Add 3D biomes to clipboards properly

- Also fix disk clipboard's size check, allowing for larger clipboards but with disabled biomes

worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/CPUOptimizedClipboard.java

@@ -27,7 +27,7 @@ import java.util.Map;
 public class CPUOptimizedClipboard extends LinearClipboard {
 
     private BiomeType[] biomes = null;
-    private char[] states;
+    private final char[] states;
 
     private final HashMap<IntTriple, CompoundTag> nbtMapLoc;
     private final HashMap<Integer, CompoundTag> nbtMapIndex;
@@ -52,14 +52,14 @@ public class CPUOptimizedClipboard extends LinearClipboard {
 
     @Override
     public boolean setBiome(int x, int y, int z, BiomeType biome) {
-        setBiome(getIndex(x, 0, z), biome);
+        setBiome(getBiomeIndex(x, y, z), biome);
         return true;
     }
 
     @Override
     public void setBiome(int index, BiomeType biome) {
         if (biomes == null) {
-            biomes = new BiomeType[getArea()];
+            biomes = new BiomeType[((getHeight() >> 2) + 1) * ((getLength() >> 2) + 1) * ((getWidth() >> 2) + 1)];
         }
         biomes[index] = biome;
     }
@@ -69,11 +69,12 @@ public class CPUOptimizedClipboard extends LinearClipboard {
         if (!hasBiomes()) {
             return;
         }
-        int index = 0;
         try {
-            for (int z = 0; z < getLength(); z++) {
-                for (int x = 0; x < getWidth(); x++, index++) {
-                    task.applyInt(index, biomes[index].getInternalId());
+            for (int y = 0; y < getHeight(); y ++) {
+                for (int z = 0; z < getLength(); z++) {
+                    for (int x = 0; x < getWidth(); x++) {
+                        task.applyInt(getIndex(x, y, z), biomes[getBiomeIndex(x, y, z)].getInternalId());
+                    }
                 }
             }
         } catch (IOException e) {
@@ -92,12 +93,12 @@ public class CPUOptimizedClipboard extends LinearClipboard {
 
     @Override
     public BiomeType getBiomeType(int x, int y, int z) {
-        return getBiome(getIndex(x, 0, z));
+        return getBiome(getBiomeIndex(x, y, z));
     }
 
     @Override
     public BiomeType getBiome(BlockVector3 position) {
-        return getBiome(getIndex(position.getX(), 0, position.getZ()));
+        return getBiome(getBiomeIndex(position.getX(), position.getY(), position.getZ()));
     }
 
     public void convertTilesToIndex() {
@@ -116,6 +117,10 @@ public class CPUOptimizedClipboard extends LinearClipboard {
         return nbtMapIndex.get(index);
     }
 
+    public int getBiomeIndex(int x, int y, int z) {
+        return (x >> 2) + (y >> 2) * (getWidth() >> 2) * (getLength() >> 2) + (z >> 2) * (getWidth() >> 2);
+    }
+
     public int getIndex(int x, int y, int z) {
         return x + y * getArea() + z * getWidth();
     }
@@ -176,7 +181,7 @@ public class CPUOptimizedClipboard extends LinearClipboard {
         return true;
     }
 
-    public boolean setTile(int index, CompoundTag tag) {
+    private boolean setTile(int index, CompoundTag tag) {
         final Map<String, Tag> values = new HashMap<>(tag.getValue());
         values.remove("x");
         values.remove("y");

worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/DiskOptimizedClipboard.java

@@ -55,8 +55,7 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
 
     private static final Logger log = LoggerFactory.getLogger(DiskOptimizedClipboard.class);
 
-    private static int HEADER_SIZE = 14;
-    private static final int MAX_SIZE = Short.MAX_VALUE - Short.MIN_VALUE;
+    private static final int HEADER_SIZE = 14;
 
     private final HashMap<IntTriple, CompoundTag> nbtMap;
     private final File file;
@@ -66,6 +65,7 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
 
     private FileChannel fileChannel;
     private boolean hasBiomes;
+    private boolean canHaveBiomes = true;
 
     public DiskOptimizedClipboard(Region region, UUID uuid) {
         this(region.getDimensions(), MainUtil.getFile(Fawe.get() != null ? Fawe.imp().getDirectory() : new File("."), Settings.IMP.PATHS.CLIPBOARD + File.separator + uuid + ".bd"));
@@ -77,14 +77,11 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
 
     public DiskOptimizedClipboard(BlockVector3 dimensions, File file) {
         super(dimensions);
-        if (getWidth() > MAX_SIZE) {
-            throw new IllegalArgumentException("Width of region too large");
-        }
-        if (getHeight() > MAX_SIZE) {
-            throw new IllegalArgumentException("Height of region too large");
-        }
-        if (getLength() > MAX_SIZE) {
-            throw new IllegalArgumentException("Length of region too large");
+        if (HEADER_SIZE + ((long) getVolume() << 1) >= Integer.MAX_VALUE) {
+            throw new IllegalArgumentException("Dimensions too large for this clipboard format");
+        } else if (HEADER_SIZE + ((long) getVolume() << 1) + (long) ((getHeight() >> 2) + 1) * ((getLength() >> 2) + 1) * ((getWidth() >> 2) + 1) >= Integer.MAX_VALUE) {
+            log.error("Dimensions are too large for biomes to be stored in a DiskOptimizedClipboard");
+            canHaveBiomes = false;
         }
         nbtMap = new HashMap<>();
         try {
@@ -138,7 +135,7 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
             this.braf = new RandomAccessFile(file, "rw");
             braf.setLength(file.length());
             init();
-            if (braf.length() - HEADER_SIZE == (getVolume() << 1) + getArea()) {
+            if (braf.length() - HEADER_SIZE == ((long) getVolume() << 1) + (long) ((getHeight() >> 2) + 1) * ((getLength() >> 2) + 1) * ((getWidth() >> 2) + 1)) {
                 hasBiomes = true;
             }
         } catch (IOException e) {
@@ -158,12 +155,17 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
     }
 
     private boolean initBiome() {
+        if (!canHaveBiomes) {
+            return false;
+        }
         if (!hasBiomes) {
             try {
                 hasBiomes = true;
                 close();
                 this.braf = new RandomAccessFile(file, "rw");
-                this.braf.setLength(HEADER_SIZE + (getVolume() << 1) + getArea());
+                // Since biomes represent a 4x4x4 cube, we store fewer biome bytes that volume at 1 byte per biome
+                // +1 to each too allow for cubes that lie across the region boundary
+                this.braf.setLength(HEADER_SIZE + ((long) getVolume() << 1) + (long) ((getHeight() >> 2) + 1) * ((getLength() >> 2) + 1) * ((getWidth() >> 2) + 1));
                 init();
             } catch (IOException e) {
                 e.printStackTrace();
@@ -185,14 +187,20 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
 
     @Override
     public boolean setBiome(int x, int y, int z, BiomeType biome) {
-        setBiome(getIndex(x, 0, z), biome);
+        setBiome(getBiomeIndex(x, y, z), biome);
         return true;
     }
 
     @Override
     public void setBiome(int index, BiomeType biome) {
         if (initBiome()) {
-            byteBuffer.put(HEADER_SIZE + (getVolume() << 1) + index, (byte) biome.getInternalId());
+            try {
+                byteBuffer.put(HEADER_SIZE + (getVolume() << 1) + index, (byte) biome.getInternalId());
+            } catch (IndexOutOfBoundsException e) {
+                System.out.println((long) (getHeight() >> 2) * (getLength() >> 2) * (getWidth() >> 2));
+                System.out.println(index);
+                e.printStackTrace();
+            }
         }
     }
 
@@ -210,13 +218,14 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
         if (!hasBiomes()) {
             return;
         }
-        int index = 0;
         int mbbIndex = HEADER_SIZE + (getVolume() << 1);
         try {
-            for (int z = 0; z < getLength(); z++) {
-                for (int x = 0; x < getWidth(); x++, index++, mbbIndex++) {
-                    int biome = byteBuffer.get(mbbIndex) & 0xFF;
-                    task.applyInt(index, biome);
+            for (int y = 0; y < getHeight(); y ++) {
+                for (int z = 0; z < getLength(); z++) {
+                    for (int x = 0; x < getWidth(); x++) {
+                        int biome = byteBuffer.get(mbbIndex + getBiomeIndex(x, y, z)) & 0xFF;
+                        task.applyInt(getIndex(x, y, z), biome);
+                    }
                 }
             }
         } catch (IOException e) {
@@ -227,12 +236,12 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
 
     @Override
     public BiomeType getBiomeType(int x, int y, int z) {
-        return getBiome(getIndex(x, 0, z));
+        return getBiome(getBiomeIndex(x, y, z));
     }
 
     @Override
     public BiomeType getBiome(BlockVector3 position) {
-        return getBiome(getIndex(position.getX(), 0, position.getZ()));
+        return getBiome(getBiomeIndex(position.getX(), position.getY(), position.getZ()));
     }
 
     public BlockArrayClipboard toClipboard() {
@@ -313,11 +322,6 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
         }
     }
 
-    private int ylast;
-    private int ylasti;
-    private int zlast;
-    private int zlasti;
-
     @Override
     public Collection<CompoundTag> getTileEntities() {
         return nbtMap.values();
@@ -327,6 +331,10 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
         return x + y * getArea() + z * getWidth();
     }
 
+    public int getBiomeIndex(int x, int y, int z) {
+        return (x >> 2) + (y >> 2) * (getWidth() >> 2) * (getLength() >> 2) + (z >> 2) * (getWidth() >> 2);
+    }
+
     @Override
     public BaseBlock getFullBlock(int x, int y, int z) {
         return toBaseBlock(getBlock(x, y, z), x, y, z);
@@ -459,7 +467,12 @@ public class DiskOptimizedClipboard extends LinearClipboard implements Closeable
 
     @Override
     public void removeEntity(Entity entity) {
-        this.entities.remove(entity);
+        if (!(entity instanceof BlockArrayClipboard.ClipboardEntity)) {
+            Location loc = entity.getLocation();
+            removeEntity(loc.getBlockX(), loc.getBlockY(), loc.getBlockZ(), entity.getState().getNbtData().getUUID());
+        } else {
+            this.entities.remove(entity);
+        }
     }
 
     @Override

worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/LinearClipboard.java

@@ -29,6 +29,7 @@ public abstract class LinearClipboard extends SimpleClipboard {
         super(dimensions);
     }
 
+    // We shouldn't expose methods that directly reference the index as people cannot be trusted to use it properly.
     public abstract <B extends BlockStateHolder<B>> boolean setBlock(int i, B block);
 
     public abstract BaseBlock getFullBlock(int i);

worldedit-core/src/main/java/com/boydti/fawe/object/clipboard/MemoryOptimizedClipboard.java

@@ -37,9 +37,9 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
     private static final int BLOCK_MASK = 1048575;
     private static final int BLOCK_SHIFT = 20;
 
-    private byte[][] states;
+    private final byte[][] states;
 
-    private byte[] buffer = new byte[MainUtil.getMaxCompressedLength(BLOCK_SIZE)];
+    private final byte[] buffer = new byte[MainUtil.getMaxCompressedLength(BLOCK_SIZE)];
     private byte[] biomes = null;
 
     private final HashMap<IntTriple, CompoundTag> nbtMap;
@@ -77,14 +77,14 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
 
     @Override
     public boolean setBiome(int x, int y, int z, BiomeType biome) {
-        setBiome(getIndex(x, 0, z), biome);
+        setBiome(getBiomeIndex(x, y, z), biome);
         return true;
     }
 
     @Override
     public void setBiome(int index, BiomeType biome) {
         if (biomes == null) {
-            biomes = new byte[getArea()];
+            biomes = new byte[((getHeight() >> 2) + 1) * ((getLength() >> 2) + 1) * ((getWidth() >> 2) + 1)];
         }
         biomes[index] = (byte) biome.getInternalId();
     }
@@ -95,10 +95,11 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
             return;
         }
         try {
-            int index = 0;
-            for (int z = 0; z < getLength(); z++) {
-                for (int x = 0; x < getWidth(); x++, index++) {
-                    task.applyInt(index, biomes[index] & 0xFF);
+            for (int y = 0; y < getHeight(); y ++) {
+                for (int z = 0; z < getLength(); z++) {
+                    for (int x = 0; x < getWidth(); x++) {
+                        task.applyInt(getIndex(x, y, z), biomes[getBiomeIndex(x, y, z)] & 0xFF);
+                    }
                 }
             }
         } catch (IOException e) {
@@ -117,15 +118,15 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
 
     @Override
     public BiomeType getBiomeType(int x, int y, int z) {
-        return getBiome(getIndex(x, 0, z));
+        return getBiome(getBiomeIndex(x, y, z));
     }
 
     @Override
     public BiomeType getBiome(BlockVector3 position) {
-        return getBiome(getIndex(position.getX(), 0, position.getZ()));
+        return getBiome(getBiomeIndex(position.getX(), position.getY(), position.getZ()));
     }
 
-    public int getOrdinal(int index) {
+    private int getOrdinal(int index) {
         int i = index >> BLOCK_SHIFT;
         int li = (index & BLOCK_MASK) << 1;
         if (i != lastOrdinalsI) {
@@ -155,7 +156,7 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
     private int lastIMin;
     private int lastIMax;
 
-    public int getLocalIndex(int index) {
+    private int getLocalIndex(int index) {
         if (index < lastIMin || index > lastIMax) {
             lastI = index >> BLOCK_SHIFT;
             lastIMin = lastI << BLOCK_SHIFT;
@@ -164,7 +165,7 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
         return lastI;
     }
 
-    public void setOrdinal(int index, int v) {
+    private void setOrdinal(int index, int v) {
         int i = getLocalIndex(index);
         if (i != lastOrdinalsI) {
             saveOrdinals();
@@ -197,6 +198,10 @@ public class MemoryOptimizedClipboard extends LinearClipboard {
         return x + y * getArea() + z * getWidth();
     }
 
+    public int getBiomeIndex(int x, int y, int z) {
+        return (x >> 2) + (y >> 2) * (getWidth() >> 2) * (getLength() >> 2) + (z >> 2) * (getWidth() >> 2);
+    }
+
     @Override
     public BaseBlock getFullBlock(int x, int y, int z) {
         int index = getIndex(x, y, z);

worldedit-core/src/main/java/com/sk89q/worldedit/extent/clipboard/io/FastSchematicReader.java

@@ -30,7 +30,6 @@ import com.boydti.fawe.object.io.FastByteArraysInputStream;
 import com.sk89q.jnbt.CompoundTag;
 import com.sk89q.jnbt.IntTag;
 import com.sk89q.jnbt.NBTInputStream;
-import com.sk89q.jnbt.NamedTag;
 import com.sk89q.jnbt.StringTag;
 import com.sk89q.jnbt.Tag;
 import com.sk89q.worldedit.WorldEdit;
@@ -62,7 +61,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.OptionalInt;
 import java.util.UUID;
 import java.util.function.Function;
 
@@ -318,18 +316,11 @@ public class FastSchematicReader extends NBTSchematicReader {
         }
         if (biomesOut != null && biomesOut.getSize() != 0) {
             try (FaweInputStream fis = new FaweInputStream(new LZ4BlockInputStream(new FastByteArraysInputStream(biomesOut.toByteArrays())))) {
-                if (clipboard instanceof LinearClipboard) {
-                    LinearClipboard linear = (LinearClipboard) clipboard;
-                    int volume = width * length;
-                    for (int index = 0; index < volume; index++) {
+                for (int z = 0; z < length; z++) {
+                    for (int x = 0; x < width; x++) {
                         BiomeType biome = getBiomeType(fis);
-                        linear.setBiome(index, biome);
-                    }
-                } else {
-                    for (int z = 0; z < length; z++) {
-                        for (int x = 0; x < width; x++) {
-                            BiomeType biome = getBiomeType(fis);
-                            clipboard.setBiome(x, 0, z, biome);
+                        for (int y = 0; y < height; y ++) {
+                            clipboard.setBiome(x, y, z, biome);
                         }
                     }
                 }