use FaweCache for MCAFile

worldedit-core/src/main/java/com/boydti/fawe/FaweCache.java

@@ -5,6 +5,7 @@ import com.boydti.fawe.config.BBC;
 import com.boydti.fawe.config.Settings;
 import com.boydti.fawe.object.collection.BitArray4096;
 import com.boydti.fawe.object.collection.CleanableThreadLocal;
+import com.boydti.fawe.object.collection.VariableThreadLocal;
 import com.boydti.fawe.object.exception.FaweBlockBagException;
 import com.boydti.fawe.object.exception.FaweChunkLoadException;
 import com.boydti.fawe.object.exception.FaweException;
@@ -219,6 +220,8 @@ public enum FaweCache implements Trimable {
 
     public final CleanableThreadLocal<byte[]> BYTE_BUFFER_8192 = new CleanableThreadLocal<>(() -> new byte[8192]);
 
+    public final VariableThreadLocal BYTE_BUFFER_VAR = new VariableThreadLocal();
+
     public final CleanableThreadLocal<int[]> BLOCK_TO_PALETTE = new CleanableThreadLocal<>(() -> {
         int[] result = new int[BlockTypesCache.states.length];
         Arrays.fill(result, Integer.MAX_VALUE);

worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAChunk.java

@@ -10,7 +10,6 @@ import com.boydti.fawe.jnbt.streamer.StreamDelegate;
 import com.boydti.fawe.jnbt.streamer.ValueReader;
 import com.boydti.fawe.object.collection.BitArray4096;
 import com.boydti.fawe.object.collection.BlockVector3ChunkMap;
-import com.boydti.fawe.object.io.FastByteArrayOutputStream;
 import com.boydti.fawe.util.MathMan;
 import com.sk89q.jnbt.CompoundTag;
 import com.sk89q.jnbt.ListTag;
@@ -30,6 +29,7 @@ import com.sk89q.worldedit.world.block.BlockStateHolder;
 import com.sk89q.worldedit.world.block.BlockType;
 import com.sk89q.worldedit.world.block.BlockTypes;
 import com.sk89q.worldedit.world.block.BlockTypesCache;
+import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream;
 
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -355,7 +355,7 @@ public class MCAChunk implements IChunk {
         nbtOut.writeEndTag();
     }
 
-    public byte[] toBytes(byte[] buffer) throws IOException {
+    public FastByteArrayOutputStream toBytes(byte[] buffer) throws IOException {
         if (buffer == null) {
             buffer = new byte[8192];
         }
@@ -363,7 +363,7 @@ public class MCAChunk implements IChunk {
         try (NBTOutputStream nbtOut = new NBTOutputStream(buffered)) {
             write(nbtOut);
         }
-        return buffered.toByteArray();
+        return buffered;
     }
 
     public long getInhabitedTime() {

worldedit-core/src/main/java/com/boydti/fawe/jnbt/anvil/MCAFile.java

@@ -2,13 +2,10 @@ package com.boydti.fawe.jnbt.anvil;
 
 import com.boydti.fawe.Fawe;
 import com.boydti.fawe.FaweCache;
-import com.boydti.fawe.beta.IChunk;
 import com.boydti.fawe.beta.Trimable;
 import com.boydti.fawe.beta.implementation.IChunkExtent;
 import com.boydti.fawe.beta.implementation.processors.ExtentBatchProcessorHolder;
-import com.boydti.fawe.jnbt.streamer.StreamDelegate;
 import com.boydti.fawe.object.RunnableVal4;
-import com.boydti.fawe.object.collection.CleanableThreadLocal;
 import com.boydti.fawe.object.io.BufferedRandomAccessFile;
 import com.boydti.fawe.object.io.FastByteArrayInputStream;
 import com.boydti.fawe.util.MainUtil;
@@ -16,24 +13,28 @@ import com.boydti.fawe.util.MathMan;
 import com.sk89q.jnbt.CompoundTag;
 import com.sk89q.jnbt.NBTInputStream;
 import com.sk89q.worldedit.WorldEditException;
-import com.sk89q.worldedit.extent.Extent;
 import com.sk89q.worldedit.math.BlockVector3;
 import com.sk89q.worldedit.world.World;
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream;
 
 import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.RandomAccessFile;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.ForkJoinTask;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
+import java.util.zip.Deflater;
 import java.util.zip.Inflater;
 import java.util.zip.InflaterInputStream;
 
@@ -45,14 +46,11 @@ import java.util.zip.InflaterInputStream;
 public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, IChunkExtent {
 
     private static Field fieldBuf2;
-    private static Field fieldBuf3;
 
     static {
         try {
             fieldBuf2 = InflaterInputStream.class.getDeclaredField("buf");
             fieldBuf2.setAccessible(true);
-            fieldBuf3 = NBTInputStream.class.getDeclaredField("buf");
-            fieldBuf3.setAccessible(true);
         } catch (Throwable e) {
             e.printStackTrace();
         }
@@ -70,25 +68,7 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
     private MCAChunk[] chunks;
     private boolean[] chunkInitialized;
     private Object[] locks;
-
+    private Deflater deflater = new Deflater(1, false);
-    final ThreadLocal<byte[]> byteStore1 = new ThreadLocal<byte[]>() {
-        @Override
-        protected byte[] initialValue() {
-            return new byte[4096];
-        }
-    };
-    final ThreadLocal<byte[]> byteStore2 = new ThreadLocal<byte[]>() {
-        @Override
-        protected byte[] initialValue() {
-            return new byte[4096];
-        }
-    };
-    final ThreadLocal<byte[]> byteStore3 = new ThreadLocal<byte[]>() {
-        @Override
-        protected byte[] initialValue() {
-            return new byte[1024];
-        }
-    };
 
     public MCAFile(ForkJoinPool pool) {
         this.pool = pool;
@@ -111,9 +91,6 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                 hasChunk = true;
             }
         }
-        CleanableThreadLocal.clean(byteStore1);
-        CleanableThreadLocal.clean(byteStore2);
-        CleanableThreadLocal.clean(byteStore3);
         return !hasChunk;
     }
 
@@ -208,14 +185,6 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
         Arrays.fill(chunkInitialized, false);
     }
 
-    @Override
-    protected void finalize() throws Throwable {
-        CleanableThreadLocal.clean(byteStore1);
-        CleanableThreadLocal.clean(byteStore2);
-        CleanableThreadLocal.clean(byteStore3);
-        super.finalize();
-    }
-
     public void setDeleted(boolean deleted) {
         this.deleted = deleted;
     }
@@ -383,7 +352,7 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
         return (locations[i + 3] & 0xFF) << 12;
     }
 
-    public byte[] getChunkCompressedBytes(int offset) throws IOException {
+    public FastByteArrayInputStream getChunkCompressedBytes(int offset) throws IOException {
         if (offset == 0) {
             return null;
         }
@@ -391,51 +360,28 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
             raf.seek(offset);
             int size = raf.readInt();
             int compression = raf.read();
-            byte[] data = new byte[size];
+            byte[] data = FaweCache.IMP.BYTE_BUFFER_VAR.get(size);
-            raf.readFully(data);
+            raf.readFully(data, 0, size);
-            return data;
+            FastByteArrayInputStream result = new FastByteArrayInputStream(data, 0, size);
+            return result;
         }
     }
 
     private NBTInputStream getChunkIS(int offset) throws IOException {
         try {
-            byte[] data = getChunkCompressedBytes(offset);
+            return getChunkIS(getChunkCompressedBytes(offset));
-            FastByteArrayInputStream bais = new FastByteArrayInputStream(data);
-            InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(), 1);
-            fieldBuf2.set(iis, byteStore2.get());
-            BufferedInputStream bis = new BufferedInputStream(iis);
-            NBTInputStream nis = new NBTInputStream(bis);
-            fieldBuf3.set(nis, byteStore3.get());
-            return nis;
         } catch (IllegalAccessException unlikely) {
             unlikely.printStackTrace();
             return null;
         }
     }
 
-    public void streamChunk(int cx, int cz, StreamDelegate delegate) throws IOException {
+    private NBTInputStream getChunkIS(InputStream is) throws IllegalAccessException {
-        streamChunk(getOffset(cx, cz), delegate);
+        InflaterInputStream iis = new InflaterInputStream(is, new Inflater(), 1);
-    }
+        fieldBuf2.set(iis, FaweCache.IMP.BYTE_BUFFER_8192.get());
-
-    public void streamChunk(int offset, StreamDelegate delegate) throws IOException {
-        byte[] data = getChunkCompressedBytes(offset);
-        streamChunk(data, delegate);
-    }
-
-    public void streamChunk(byte[] data, StreamDelegate delegate) throws IOException {
-        if (data != null) {
-            try {
-                FastByteArrayInputStream bais = new FastByteArrayInputStream(data);
-                InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(), 1);
-                fieldBuf2.set(iis, byteStore2.get());
         BufferedInputStream bis = new BufferedInputStream(iis);
         NBTInputStream nis = new NBTInputStream(bis);
-                fieldBuf3.set(nis, byteStore3.get());
+        return nis;
-                nis.readNamedTagLazy(delegate);
-            } catch (IllegalAccessException unlikely) {
-                unlikely.printStackTrace();
-            }
-        }
     }
 
     /**
@@ -465,39 +411,35 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
         return list;
     }
 
-    private byte[] toBytes(MCAChunk chunk) throws Exception {
+    private FastByteArrayOutputStream toBytes(MCAChunk chunk) throws IOException {
         if (chunk.isDeleted()) {
             return null;
         }
-        byte[] uncompressed = chunk.toBytes(byteStore3.get());
+        byte[] writeBuffer = FaweCache.IMP.BYTE_BUFFER_VAR.get(4096);
-        byte[] compressed = MainUtil.compress(uncompressed, byteStore2.get(), null);
+        FastByteArrayOutputStream uncompressed = chunk.toBytes(writeBuffer);
-        return compressed;
+        if (uncompressed.array.length > writeBuffer.length) {
+            FaweCache.IMP.BYTE_BUFFER_VAR.set(uncompressed.array);
+        }
+        writeBuffer = uncompressed.array;
+        byte[] buffer = FaweCache.IMP.BYTE_BUFFER_8192.get();
+        int length = uncompressed.length;
+        uncompressed.reset();
+        // cheat, reusing the same buffer to read/write
+        int compressedLength = MainUtil.compress(uncompressed.array, length, buffer, uncompressed, deflater);
+        return uncompressed;
     }
 
-    private byte[] getChunkBytes(int cx, int cz) throws Exception {
+    private void writeSafe(RandomAccessFile raf, int offset, byte[] data, int length) throws IOException {
-        MCAChunk mca = getCachedChunk(cx, cz);
+        int len = length + 5;
-        if (mca == null) {
-            int offset = getOffset(cx, cz);
-            if (offset == 0) {
-                return null;
-            }
-            return getChunkCompressedBytes(offset);
-        }
-        return toBytes(mca);
-    }
-
-
-    private void writeSafe(RandomAccessFile raf, int offset, byte[] data) throws IOException {
-        int len = data.length + 5;
         raf.seek(offset);
         if (raf.length() - offset < len) {
             raf.setLength(((offset + len + 4095) / 4096) * 4096);
         }
         // Length of remaining data
-        raf.writeInt(data.length + 1);
+        raf.writeInt(length + 1);
         // Compression type
         raf.write(2);
-        raf.write(data);
+        raf.write(data, 0, length);
     }
 
     private void writeHeader(RandomAccessFile raf, int cx, int cz, int offsetMedium, int sizeByte, boolean writeTime) throws IOException {
@@ -561,49 +503,66 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                 file.delete();
                 return;
             }
-
             // Chunks that need to be relocated
             Int2ObjectOpenHashMap<byte[]> relocate = new Int2ObjectOpenHashMap<>();
             // The position of each chunk
             final Int2ObjectOpenHashMap<Integer> offsetMap = new Int2ObjectOpenHashMap<>(); // Offset -> <byte cx, byte cz, short size>
             // The data of each modified chunk
-            final Int2ObjectOpenHashMap<byte[]> compressedMap = new Int2ObjectOpenHashMap<>();
+            final Int2ObjectOpenHashMap<Future<byte[]>> compressedMap = new Int2ObjectOpenHashMap<>();
             // The data of each chunk that needs to be moved
-            final Int2ObjectOpenHashMap<byte[]> append = new Int2ObjectOpenHashMap<>();
+            final Int2ObjectOpenHashMap<Future<byte[]>> append = new Int2ObjectOpenHashMap<>();
-            boolean[] modified = new boolean[1];
             // Get the current time for the chunk timestamp
             long now = System.currentTimeMillis();
 
             // Load the chunks into the append or compressed map
             final ForkJoinPool finalPool = this.pool;
-            forEachCachedChunk(chunk -> {
-                if (chunk.isModified() || chunk.isDeleted()) {
-                    modified[0] = true;
-                    chunk.setLastUpdate(now);
-                    if (!chunk.isDeleted()) {
-                        MCAFile.this.pool.submit(() -> {
-                            try {
-                                byte[] compressed = toBytes(chunk);
-                                int pair = MathMan.pair((short) (chunk.getX() & 31), (short) (chunk.getZ() & 31));
-                                Int2ObjectOpenHashMap map;
-                                if (getOffset(chunk.getX(), chunk.getZ()) == 0) {
-                                    map = append;
-                                } else {
-                                    map = compressedMap;
-                                }
-                                synchronized (map) {
-                                    map.put(pair, compressed);
-                                }
-                            } catch (Throwable e) {
-                                e.printStackTrace();
-                            }
-                        });
-                    }
-                }
-            });
 
+
+            boolean modified = false;
+            for (int i = 0; i < chunks.length; i++) {
+                if (this.chunkInitialized[i]) {
+                    MCAChunk chunk = chunks[i];
+                    if (chunk != null && chunk.isModified() && !chunk.isDeleted()) {
+                        modified = true;
+                        ForkJoinTask<byte[]> future = pool.submit(() -> {
+                            FastByteArrayOutputStream compressed = toBytes(chunk);
+                            return Arrays.copyOf(compressed.array, compressed.length);
+                        });
+                    }
+                }
+            }
+//
+//            forEachCachedChunk(chunk -> {
+//                if (chunk.isModified() || chunk.isDeleted()) {
+//                    modified[0] = true;
+//                    chunk.setLastUpdate(now);
+//                    if (!chunk.isDeleted()) {
+//                        MCAFile.this.pool.submit(() -> {
+//                            try {
+//                                byte[] compressed = toBytes(chunk);
+//                                int pair = MathMan.pair((short) (chunk.getX() & 31), (short) (chunk.getZ() & 31));
+//                                Int2ObjectOpenHashMap map;
+//                                if (getOffset(chunk.getX(), chunk.getZ()) == 0) {
+//                                    map = append;
+//                                } else {
+//                                    map = compressedMap;
+//                                }
+//                                synchronized (map) {
+//                                    map.put(pair, compressed);
+//                                }
+//                            } catch (Throwable e) {
+//                                e.printStackTrace();
+//                            }
+//                        });
+//                    }
+//                }
+//            });
+
+            if (!modified) {
+                // Not modified, do nothing
+                return;
+            }
             // If any changes were detected
-            if (modified[0]) {
             file.setLastModified(now);
 
             // Load the offset data into the offset map
@@ -615,9 +574,6 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                     offsetMap.put((int) offset, (Integer) MathMan.pair(pair1, pair2));
                 }
             });
-                // Wait for previous tasks
-                pool.awaitQuiescence(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
-
 
             int start = 8192;
             int written = start;
@@ -644,7 +600,9 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                     nextOffset += size;
                     end = Math.min(start + size, end);
                     int pair = getIndex(cx, cz);
+
                     byte[] newBytes = relocate.get(pair);
+                    int newBytesLength = 0;
 
                     // newBytes is null if the chunk isn't modified or marked for moving
                     if (newBytes == null) {
@@ -657,18 +615,22 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                                 written = start + size;
                                 continue;
                             } else {
-                                    newBytes = compressedMap.get(pair);
+                                future = compressedMap.get(pair);
                             }
                         } else {
                             // The chunk needs to be moved, fetch the data if necessary
-                                newBytes = compressedMap.get(pair);
+                            future = compressedMap.get(pair);
-                                if (newBytes == null) {
+                            if (future == null) {
                                 if (cached == null || !cached.isDeleted()) {
-                                        newBytes = getChunkCompressedBytes(getOffset(cx, cz));
+                                    FastByteArrayInputStream result = getChunkCompressedBytes(getOffset(cx, cz));
                                 }
                             }
                         }
                     }
+                    if (future != null) {
+                        newBytes = future.get();
+                        newBytesLength = newBytes.length;
+                    }
 
                     if (newBytes == null) {
                         writeHeader(raf, cx, cz, 0, 0, false);
@@ -676,7 +638,7 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                     }
 
                     // The length to be written (compressed data + 5 byte chunk header)
-                        int len = newBytes.length + 5;
+                    int len = newBytesLength + 5;
                     int oldSize = (size + 4095) >> 12;
                     int newSize = (len + 4095) >> 12;
                     int nextOffset2 = end;
@@ -690,7 +652,8 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                             int nextCZ = MathMan.unpairShortY(nextCXZ);
                             MCAChunk cached = getCachedChunk(nextCX, nextCZ);
                             if (cached == null || !cached.isModified()) {
-                                    byte[] nextBytes = getChunkCompressedBytes(nextOffset2);
+                                FastByteArrayInputStream tmp = getChunkCompressedBytes(nextOffset2);
+                                byte[] nextBytes = Arrays.copyOf(tmp.array, tmp.length);
                                 relocate.put(MathMan.pair((short) (nextCX & 31), (short) (nextCZ & 31)), nextBytes);
                             }
                             int nextSize = MathMan.unpairY(nextLoc) << 12;
@@ -702,24 +665,24 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
                         }
                     }
                     // Write the chunk + chunk header
-                        writeSafe(raf, start, newBytes);
+                    writeSafe(raf, start, newBytes, newBytesLength);
                     // Write the location data (beginning of file)
                     writeHeader(raf, cx, cz, start >> 12, newSize, true);
 
-                        written = start + newBytes.length + 5;
+                    written = start + newBytesLength + 5;
                     start += newSize << 12;
                 }
 
                 // Write all the chunks which need to be appended
                 if (!append.isEmpty()) {
-                        for (Int2ObjectMap.Entry<byte[]> entry : append.int2ObjectEntrySet()) {
+                    for (Int2ObjectMap.Entry<Future<byte[]>> entry : append.int2ObjectEntrySet()) {
                         int pair = entry.getIntKey();
                         short cx = MathMan.unpairX(pair);
                         short cz = MathMan.unpairY(pair);
-                            byte[] bytes = entry.getValue();
+                        byte[] bytes = entry.getValue().get();
                         int len = bytes.length + 5;
                         int newSize = (len + 4095) >> 12;
-                            writeSafe(raf, start, bytes);
+                        writeSafe(raf, start, bytes, bytes.length);
                         writeHeader(raf, cx, cz, start >> 12, newSize, true);
                         written = start + bytes.length + 5;
                         start += newSize << 12;
@@ -739,5 +702,4 @@ public class MCAFile extends ExtentBatchProcessorHolder implements Trimable, ICh
             }
         }
     }
-    }
 }

worldedit-core/src/main/java/com/boydti/fawe/object/collection/VariableThreadLocal.java

@@ -0,0 +1,20 @@
+package com.boydti.fawe.object.collection;
+
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+public class VariableThreadLocal extends CleanableThreadLocal<byte[]> {
+    public VariableThreadLocal() {
+        super(() -> null);
+    }
+
+    public byte[] get(int size) {
+        byte[] existing = get();
+        if (existing == null || existing.length < size) {
+            int padded = ((size + 4095) / 4096) * 4096;
+            existing = new byte[padded];
+            set(existing);
+        }
+        return existing;
+    }
+}

worldedit-core/src/main/java/com/boydti/fawe/util/MainUtil.java

@@ -24,6 +24,7 @@ import com.sk89q.worldedit.extent.clipboard.io.ClipboardFormat;
 import com.sk89q.worldedit.extent.clipboard.io.ClipboardFormats;
 import com.sk89q.worldedit.history.changeset.ChangeSet;
 import com.sk89q.worldedit.util.Location;
+import it.unimi.dsi.fastutil.io.FastByteArrayOutputStream;
 import net.jpountz.lz4.LZ4BlockInputStream;
 import net.jpountz.lz4.LZ4BlockOutputStream;
 import net.jpountz.lz4.LZ4Compressor;
@@ -236,7 +237,7 @@ public class MainUtil {
         return LZ4Utils.maxCompressedLength(size);
     }
 
-    public static byte[] compress(byte[] bytes, byte[] buffer, Deflater deflate) {
+    public static int compress(byte[] bytes, int length, byte[] buffer, OutputStream out, Deflater deflate) throws IOException {
         if (buffer == null) {
             buffer = new byte[8192];
         }
@@ -245,14 +246,17 @@ public class MainUtil {
         } else {
             deflate.reset();
         }
-        deflate.setInput(bytes);
+        deflate.setInput(bytes, 0, length);
         deflate.finish();
-        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        int written = 0;
         while (!deflate.finished()) {
             int n = deflate.deflate(buffer);
-            if (n != 0) baos.write(buffer, 0, n);
+            if (n != 0) {
+                written += n;
+                out.write(buffer, 0, n);
             }
-        return baos.toByteArray();
+        }
+        return written;
     }
 
     public static byte[] decompress(byte[] bytes, byte[] buffer, Inflater inflater) throws DataFormatException {