Initial work on new disk clipboard

worldedit-core/src/main/java/com/fastasyncworldedit/core/extent/clipboard/Clipboards.java

@@ -0,0 +1,39 @@
+package com.fastasyncworldedit.core.extent.clipboard;
+
+import com.fastasyncworldedit.core.Fawe;
+import com.fastasyncworldedit.core.IFawe;
+import com.sk89q.worldedit.extension.platform.Actor;
+import com.sk89q.worldedit.extent.clipboard.BlockArrayClipboard;
+import com.sk89q.worldedit.extent.clipboard.Clipboard;
+import com.sk89q.worldedit.math.BlockVector3;
+import com.sk89q.worldedit.regions.CuboidRegion;
+import com.sk89q.worldedit.regions.Region;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Objects;
+
+public final class Clipboards {
+
+    public static Clipboard create(Region region, BlockVector3 origin, Actor actor) {
+        if (!(region instanceof CuboidRegion)) {
+            return new BlockArrayClipboard(region, actor.getUniqueId());
+        }
+        return new DiskBasedClipboard(region.getDimensions(), region.getMinimumPoint(), origin, createActorPath(actor));
+    }
+
+    private static Path createActorPath(Actor actor) {
+        Path folder = Objects.requireNonNull(Fawe.<IFawe>platform(), "Platform not present")
+                .getDirectory().toPath()
+                .resolve("clipboards")
+                .resolve(actor.getUniqueId().toString());
+        try {
+            Files.createDirectories(folder);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        return folder;
+    }
+}

worldedit-core/src/main/java/com/fastasyncworldedit/core/extent/clipboard/DiskBasedClipboard.java

@@ -0,0 +1,282 @@
+package com.fastasyncworldedit.core.extent.clipboard;
+
+import com.fastasyncworldedit.core.util.io.MemoryFile;
+import com.sk89q.jnbt.CompoundTag;
+import com.sk89q.worldedit.WorldEditException;
+import com.sk89q.worldedit.entity.Entity;
+import com.sk89q.worldedit.extent.clipboard.Clipboard;
+import com.sk89q.worldedit.math.BlockVector3;
+import com.sk89q.worldedit.regions.CuboidRegion;
+import com.sk89q.worldedit.regions.Region;
+import com.sk89q.worldedit.util.nbt.BinaryTagIO;
+import com.sk89q.worldedit.util.nbt.CompoundBinaryTag;
+import com.sk89q.worldedit.world.biome.BiomeType;
+import com.sk89q.worldedit.world.biome.BiomeTypes;
+import com.sk89q.worldedit.world.block.BaseBlock;
+import com.sk89q.worldedit.world.block.BlockState;
+import com.sk89q.worldedit.world.block.BlockStateHolder;
+import com.sk89q.worldedit.world.block.BlockTypes;
+import com.sk89q.worldedit.world.block.BlockTypesCache;
+import it.unimi.dsi.fastutil.ints.Int2ReferenceMap;
+import it.unimi.dsi.fastutil.ints.Int2ReferenceOpenHashMap;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class DiskBasedClipboard implements Clipboard {
+
+    private static final int INVALID_INDEX = -1;
+    private static final BlockState AIR = Objects.requireNonNull(BlockTypes.AIR).getDefaultState();
+    private static final BiomeType OCEAN = BiomeTypes.OCEAN;
+
+    private final MemoryFile blockFile;
+    private MemoryFile biomeFile;
+
+    private final BlockVector3 dimensions;
+    private final BlockVector3 offset;
+    private final Path folder;
+    private final Int2ReferenceMap<CompoundBinaryTag> nbt = new Int2ReferenceOpenHashMap<>();
+    private BlockVector3 origin;
+
+    DiskBasedClipboard(
+            final BlockVector3 dimensions,
+            final BlockVector3 offset,
+            final BlockVector3 origin,
+            final Path folder
+    ) {
+        this.dimensions = dimensions;
+        this.offset = offset;
+        this.origin = origin;
+        this.folder = folder;
+        long entries = requiredEntries(dimensions);
+        try {
+            this.blockFile = MemoryFile.create(folder.resolve("blocks.dc"), entries, BlockTypesCache.states.length);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private static long requiredEntries(BlockVector3 dimensions) {
+        return (long) dimensions.getX() * dimensions.getY() * dimensions.getZ();
+    }
+
+    private static long requiredBiomeEntries(BlockVector3 dimensions) {
+        return requiredEntries(dimensions.divide(4));
+    }
+
+    @Override
+    public BaseBlock getFullBlock(final int x, final int y, final int z) {
+        final int index = toLocalIndex(x, y, z);
+        if (index != INVALID_INDEX) {
+            BlockState state = getBlock(index);
+            CompoundBinaryTag tag = this.nbt.get(index);
+            return state.toBaseBlock(tag); // passing null is fine
+        }
+        return AIR.toBaseBlock();
+    }
+
+    @Override
+    public BiomeType getBiomeType(final int x, final int y, final int z) {
+        MemoryFile memoryFile = ensureBiomeFile();
+        final int index = toLocalBiomeIndex(x, y, z);
+        if (index != INVALID_INDEX) {
+            return BiomeTypes.get(memoryFile.getValue(index));
+        }
+        return OCEAN; // as per documentation in InputExtent
+    }
+
+    @Override
+    public BlockState getBlock(final int x, final int y, final int z) {
+        final int index = toLocalIndex(x, y, z);
+        if (index != INVALID_INDEX) {
+            return getBlock(index);
+        }
+        return AIR;
+    }
+
+    @Override
+    public <B extends BlockStateHolder<B>> boolean setBlock(final int x, final int y, final int z, final B block) throws
+            WorldEditException {
+        final int index = toLocalIndex(x, y, z);
+        if (index != INVALID_INDEX) {
+            this.blockFile.setValue(index, getId(block));
+            if (block instanceof BaseBlock bb) {
+                dealWithNbt(bb, index);
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private void dealWithNbt(BaseBlock bb, int index) {
+        final CompoundBinaryTag value = bb.getNbt();
+        if (value != null) {
+            nbt.put(index, value);
+        }
+    }
+
+    @Override
+    @SuppressWarnings("deprecation")
+    public boolean setTile(final int x, final int y, final int z, final CompoundTag tile) throws WorldEditException {
+        final int index = toLocalIndex(x, y, z);
+        if (index != INVALID_INDEX) {
+            this.nbt.put(index, tile.asBinaryTag());
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public boolean fullySupports3DBiomes() {
+        return true;
+    }
+
+    @Override
+    public boolean setBiome(final int x, final int y, final int z, final BiomeType biome) {
+        MemoryFile biomeFile = ensureBiomeFile();
+        final int index = toLocalBiomeIndex(x, y, z);
+        if (index != INVALID_INDEX) {
+            biomeFile.setValue(index, biome.getInternalId());
+            return true;
+        }
+        return false;
+    }
+
+    private MemoryFile ensureBiomeFile() {
+        MemoryFile biomeFile = this.biomeFile;
+        if (biomeFile != null) {
+            return biomeFile;
+        }
+        int biomeCount = BiomeTypes.getMaxId();
+        try {
+            biomeFile = MemoryFile.create(folder.resolve("biomes.dc"), requiredBiomeEntries(this.dimensions), biomeCount);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+        this.biomeFile = biomeFile;
+        return biomeFile;
+    }
+
+    @Override
+    public Region getRegion() {
+        return new CuboidRegion(getMinimumPoint(), getMaximumPoint());
+    }
+
+    @Override
+    public BlockVector3 getDimensions() {
+        return this.dimensions;
+    }
+
+    @Override
+    public BlockVector3 getOrigin() {
+        return this.origin;
+    }
+
+    @Override
+    public void setOrigin(final BlockVector3 origin) {
+        this.origin = origin;
+    }
+
+    @Override
+    public boolean hasBiomes() {
+        return this.biomeFile != null;
+    }
+
+    @Override
+    public void removeEntity(final Entity entity) {
+
+    }
+
+    @Override
+    public void close() {
+        try {
+            this.blockFile.close();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    @Override
+    public void flush() {
+        try {
+            this.blockFile.flush();
+            writeNbt();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+    private void writeNbt() throws IOException {
+        if (this.nbt.isEmpty()) {
+            return;
+        }
+        Map<String, CompoundBinaryTag> converted = new HashMap<>();
+        for (final Int2ReferenceMap.Entry<CompoundBinaryTag> entry : this.nbt.int2ReferenceEntrySet()) {
+            converted.put(String.valueOf(entry.getIntKey()), entry.getValue());
+        }
+        CompoundBinaryTag tiles = CompoundBinaryTag.from(converted);
+        BinaryTagIO.writer().write(tiles, this.folder.resolve("entities.nbt"));
+    }
+
+    @Override
+    public BlockVector3 getMinimumPoint() {
+        return this.offset;
+    }
+
+    @Override
+    public BlockVector3 getMaximumPoint() {
+        return this.offset.add(this.dimensions.subtract(1, 1, 1));
+    }
+
+    private boolean inRegion(int x, int y, int z) {
+        return (x | y | z) >= 0 && y < this.dimensions.getY() && x < this.dimensions.getX() && z < this.dimensions.getZ();
+    }
+
+    @SuppressWarnings("deprecation")
+    private <B extends BlockStateHolder<B>> int getId(B b) {
+        return b.getOrdinal();
+    }
+
+    private int toLocalIndex(int x, int y, int z) {
+        int lx = x - this.offset.getX();
+        int ly = y - this.offset.getY();
+        int lz = z - this.offset.getZ();
+        if (inRegion(lx, ly, lz)) {
+            return toIndexUnchecked(lx, ly, lz);
+        }
+        return INVALID_INDEX;
+    }
+
+    // l = local
+    private int toIndexUnchecked(int lx, int ly, int lz) {
+        // chosen to correspond to iteration order in CuboidRegion#iterator()
+        // to minimize cache misses/page faults
+        return lx + this.dimensions.getX() * (lz + ly * this.dimensions.getZ());
+    }
+
+    private int toLocalBiomeIndex(int x, int y, int z) {
+        int lx = x - this.offset.getX();
+        int ly = y - this.offset.getY();
+        int lz = z - this.offset.getZ();
+        if (inRegion(lx, ly, lz)) {
+            return toBiomeIndexUnchecked(lx >> 2, ly >> 2, lz >> 2);
+        }
+        return INVALID_INDEX;
+    }
+
+    // b = biome, l = local
+    private int toBiomeIndexUnchecked(int blx, int bly, int blz) {
+        // chosen to correspond to iteration order in CuboidRegion#iterator()
+        // to minimize cache misses/page faults
+        return blx + (this.dimensions.getX() >> 2) * (blz + bly * (this.dimensions.getZ() >> 2));
+    }
+
+    private BlockState getBlock(final int i) {
+        return BlockTypesCache.states[this.blockFile.getValue(i)];
+    }
+
+}

worldedit-core/src/main/java/com/fastasyncworldedit/core/util/io/MemoryFile.java

@@ -0,0 +1,33 @@
+package com.fastasyncworldedit.core.util.io;
+
+import java.io.Flushable;
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+
+
+public sealed interface MemoryFile extends AutoCloseable, Flushable permits SmallMemoryFile {
+
+    /**
+     * {@return a memory-mapped file that can store up to {@code entries} integers in the range of {@code [0, valueCount)}}
+     */
+    static MemoryFile create(Path file, long entries, int valueCount) throws IOException {
+        int bitsPerEntry = MemoryFileSupport.bitsPerEntry(valueCount);
+        long bytesNeeded = MemoryFileSupport.requiredBytes(bitsPerEntry, entries);
+        if (bytesNeeded <= Integer.MAX_VALUE) {
+            return new SmallMemoryFile(FileChannel.open(file, MemoryFileSupport.OPTIONS), (int) bytesNeeded, bitsPerEntry);
+        }
+        throw new UnsupportedOperationException("too many entries: " + entries);
+    }
+
+    void setValue(int index, int value);
+
+    int getValue(int index);
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    void close() throws IOException;
+
+}

worldedit-core/src/main/java/com/fastasyncworldedit/core/util/io/MemoryFileSupport.java

@@ -0,0 +1,41 @@
+package com.fastasyncworldedit.core.util.io;
+
+import com.fastasyncworldedit.core.util.MathMan;
+import org.jetbrains.annotations.Range;
+
+import java.nio.file.OpenOption;
+import java.nio.file.StandardOpenOption;
+import java.util.EnumSet;
+import java.util.Set;
+
+final class MemoryFileSupport {
+
+    /**
+     * The amount of additional bytes required to safely call {@link java.nio.ByteBuffer#getInt(int)}
+     * and {@link java.nio.ByteBuffer#putInt(int, int)} for the last actually used byte.
+     */
+    static final int PADDING = 3;
+    static final Set<? extends OpenOption> OPTIONS = EnumSet.of(
+            StandardOpenOption.CREATE,
+            StandardOpenOption.WRITE,
+            StandardOpenOption.READ
+    );
+
+    static long requiredBytes(int bitsPerEntry, long entries) {
+        long bitsNeeded = bitsPerEntry * entries;
+        // Math.ceilDiv is Java 18+
+        return -Math.floorDiv(-bitsNeeded, 8) + MemoryFileSupport.PADDING;
+    }
+
+    static int bitsPerEntry(int valueCount) {
+        if (Integer.highestOneBit(valueCount) == Integer.lowestOneBit(valueCount)) {
+            return Integer.numberOfTrailingZeros(valueCount);
+        }
+        return MathMan.log2nlz(valueCount);
+    }
+
+    // Calculate the shift required for this bitPos
+    static @Range(from = 0, to = 7) int shift(long bitPos, long bytePos) {
+        return (int) (bitPos - (bytePos << 3));
+    }
+}

worldedit-core/src/main/java/com/fastasyncworldedit/core/util/io/SmallMemoryFile.java

@@ -0,0 +1,70 @@
+package com.fastasyncworldedit.core.util.io;
+
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+
+import static com.fastasyncworldedit.core.util.io.MemoryFileSupport.shift;
+
+final class SmallMemoryFile implements MemoryFile {
+    private final FileChannel channel;
+    private final MappedByteBuffer buffer;
+    private final int bitsPerEntry;
+    private final int entryMask;
+
+    SmallMemoryFile(FileChannel channel, int size, final int bitsPerEntry) throws IOException {
+        this.channel = channel;
+        this.buffer = this.channel.map(FileChannel.MapMode.READ_WRITE, 0, size);
+        this.buffer.order(ByteOrder.LITTLE_ENDIAN);
+        this.bitsPerEntry = bitsPerEntry;
+        this.entryMask = (1 << this.bitsPerEntry) - 1;
+    }
+
+    @Override
+    public void setValue(int index, int value) {
+        long bitPos = bitPos(index);
+        int bytePos = toBytePos(bitPos);
+        int shift = shift(bitPos, bytePos);
+        write(bytePos, shift, value);
+    }
+
+    @Override
+    public int getValue(final int index) {
+        long bitPos = bitPos(index);
+        int bytePos = toBytePos(bitPos);
+        int shift = shift(bitPos, bytePos);
+        return read(bytePos, shift);
+    }
+
+    private void write(int bytePos, int shift, int value) {
+        int mask = this.entryMask << shift;
+        int existing = this.buffer.getInt(bytePos);
+        int result = (existing & ~mask) | (value << shift);
+        this.buffer.putInt(bytePos, result);
+    }
+
+    private int read(int bytePos, int shift) {
+        return (this.buffer.getInt(bytePos) >> shift) & this.entryMask;
+    }
+
+    private static int toBytePos(long bitPos) {
+        return (int) (bitPos >> 3); // must be in int range for SmallMemoryFile
+    }
+
+    private long bitPos(int index) {
+        return (long) this.bitsPerEntry * index;
+    }
+
+    @Override
+    public void close() throws IOException {
+        flush();
+        this.channel.close();
+    }
+
+    @Override
+    public void flush() {
+        this.buffer.force();
+    }
+
+}

worldedit-core/src/main/java/com/sk89q/worldedit/command/ClipboardCommands.java

@@ -24,6 +24,7 @@ import com.fastasyncworldedit.core.FaweCache;
 import com.fastasyncworldedit.core.configuration.Caption;
 import com.fastasyncworldedit.core.configuration.Settings;
 import com.fastasyncworldedit.core.event.extent.PasteEvent;
+import com.fastasyncworldedit.core.extent.clipboard.Clipboards;
 import com.fastasyncworldedit.core.extent.clipboard.DiskOptimizedClipboard;
 import com.fastasyncworldedit.core.extent.clipboard.MultiClipboardHolder;
 import com.fastasyncworldedit.core.extent.clipboard.ReadOnlyClipboard;
@@ -154,9 +155,9 @@ public class ClipboardCommands {
         }
         session.setClipboard(null);
 
-        Clipboard clipboard = new BlockArrayClipboard(region, actor.getUniqueId());
+        final BlockVector3 origin = centerClipboard ? region.getCenter().toBlockPoint().withY(region.getMinimumY()) :
-        clipboard.setOrigin(centerClipboard ? region.getCenter().toBlockPoint().withY(region.getMinimumY()) :
+                session.getPlacementPosition(actor);
-                session.getPlacementPosition(actor));
+        Clipboard clipboard = Clipboards.create(region, origin, actor);
         ForwardExtentCopy copy = new ForwardExtentCopy(editSession, region, clipboard, region.getMinimumPoint());
         copy.setCopyingEntities(copyEntities);
         copy.setCopyingBiomes(copyBiomes);

worldedit-core/src/test/java/com/fastasyncworldedit/core/util/io/MemoryFileSupportTest.java

@@ -0,0 +1,24 @@
+package com.fastasyncworldedit.core.util.io;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class MemoryFileSupportTest {
+
+    @ParameterizedTest
+    @CsvSource(textBlock = """
+            1, 2
+            2, 3
+            2, 4
+            3, 5
+            8, 255
+            8, 256
+            9, 257
+            """)
+    void testBitsPerEntry(int expected, int entries) {
+        assertEquals(expected, MemoryFileSupport.bitsPerEntry(entries));
+    }
+
+}

worldedit-core/src/test/java/com/fastasyncworldedit/core/util/io/MemoryFileTest.java

@@ -0,0 +1,24 @@
+package com.fastasyncworldedit.core.util.io;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.random.RandomGenerator;
+
+class MemoryFileTest {
+
+    @Test
+    void writeALot(@TempDir Path dir) throws IOException {
+        Path file = dir.resolve("data.tmp");
+        int entries = Integer.MAX_VALUE - 10;
+        RandomGenerator generator = RandomGenerator.getDefault();
+        try (MemoryFile memoryFile = MemoryFile.create(file, entries, 256)) {
+            for (int i = 0; i < entries; i++) {
+                memoryFile.setValue(i, generator.nextInt(256));
+            }
+        }
+    }
+
+}