chore/feat: support non-file based inputstreams (using in memory LZ4 cache)

worldedit-bukkit/src/main/java/com/fastasyncworldedit/bukkit/util/DoNotMiniseThese.java

@@ -1,6 +1,8 @@
 package com.fastasyncworldedit.bukkit.util;
 
 import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
+import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
 import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
 import it.unimi.dsi.fastutil.longs.LongArraySet;
 import it.unimi.dsi.fastutil.longs.LongIterator;
@@ -19,5 +21,7 @@ final class DoNotMiniseThese {
     private final LongSet d = null;
     private final Int2ObjectMap<?> e = null;
     private final Object2ObjectArrayMap<?, ?> f = null;
+    private final FastBufferedInputStream g = null;
+    private final FastBufferedOutputStream h = null;
 
 }

worldedit-core/src/main/java/com/fastasyncworldedit/core/extent/clipboard/io/FastSchematicReaderV3.java

@@ -5,10 +5,12 @@ import com.fastasyncworldedit.core.extent.clipboard.SimpleClipboard;
 import com.fastasyncworldedit.core.internal.io.ResettableFileInputStream;
 import com.fastasyncworldedit.core.internal.io.VarIntStreamIterator;
 import com.fastasyncworldedit.core.math.MutableBlockVector3;
-import com.fastasyncworldedit.core.math.MutableVector3;
+import com.fastasyncworldedit.core.util.IOUtil;
+import com.fastasyncworldedit.core.util.MathMan;
 import com.sk89q.jnbt.CompoundTag;
 import com.sk89q.jnbt.NBTConstants;
 import com.sk89q.jnbt.NBTInputStream;
+import com.sk89q.jnbt.NBTOutputStream;
 import com.sk89q.worldedit.WorldEdit;
 import com.sk89q.worldedit.entity.BaseEntity;
 import com.sk89q.worldedit.extension.input.InputParseException;
@@ -19,8 +21,8 @@ import com.sk89q.worldedit.extent.clipboard.io.ClipboardReader;
 import com.sk89q.worldedit.extent.clipboard.io.sponge.VersionedDataFixer;
 import com.sk89q.worldedit.internal.util.LogManagerCompat;
 import com.sk89q.worldedit.math.BlockVector3;
-import com.sk89q.worldedit.math.Vector3;
 import com.sk89q.worldedit.util.Location;
+import com.sk89q.worldedit.util.concurrency.LazyReference;
 import com.sk89q.worldedit.util.nbt.CompoundBinaryTag;
 import com.sk89q.worldedit.world.DataFixer;
 import com.sk89q.worldedit.world.biome.BiomeType;
@@ -30,21 +32,28 @@ import com.sk89q.worldedit.world.block.BlockTypes;
 import com.sk89q.worldedit.world.block.BlockTypesCache;
 import com.sk89q.worldedit.world.entity.EntityType;
 import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
+import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
+import net.jpountz.lz4.LZ4BlockInputStream;
+import net.jpountz.lz4.LZ4BlockOutputStream;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.util.TriConsumer;
 import org.checkerframework.checker.nullness.qual.NonNull;
+import org.checkerframework.checker.nullness.qual.Nullable;
+import org.jetbrains.annotations.ApiStatus;
 
 import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
+import java.io.DataOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.HashSet;
 import java.util.Objects;
 import java.util.OptionalInt;
 import java.util.Set;
 import java.util.UUID;
-import java.util.function.BiConsumer;
 import java.util.function.BooleanSupplier;
 import java.util.function.Function;
 import java.util.zip.GZIPInputStream;
@@ -55,20 +64,22 @@ import java.util.zip.GZIPInputStream;
  * stream based approach to keep the memory overhead minimal (especially in larger schematics)
  */
 
-/**
+/**le
- * TODO: - Validate FileChannel reset performance (especially for network drive / remote backed storage)
- * TODO: ^ try to compare speed and memory / cpu usage when - instead of resetting the stream - caching the palette using LZ4 /
- * TODO    ZSTD until other data is available
  * TODO: fix tile entity locations (+ validate entity location)
  */
 @SuppressWarnings("removal") // JNBT
 public class FastSchematicReaderV3 implements ClipboardReader {
 
     private static final Logger LOGGER = LogManagerCompat.getLogger();
+    private static final byte CACHE_IDENTIFIER_END = 0x00;
+    private static final byte CACHE_IDENTIFIER_BLOCK = 0x01;
+    private static final byte CACHE_IDENTIFIER_BIOMES = 0x02;
+    private static final byte CACHE_IDENTIFIER_ENTITIES = 0x03;
+    private static final byte CACHE_IDENTIFIER_BLOCK_TILE_ENTITIES = 0x04;
 
-    private final InputStream resetableInputStream;
+    private final InputStream parentStream;
     private final MutableBlockVector3 dimensions = MutableBlockVector3.at(0, 0, 0);
-    private final Set<String> remainingTags = new HashSet<>();
+    private final Set<Byte> remainingTags;
 
     private DataInputStream dataInputStream;
     private NBTInputStream nbtInputStream;
@@ -79,18 +90,26 @@ public class FastSchematicReaderV3 implements ClipboardReader {
     private BiomeType[] biomePalette;
     private int dataVersion = -1;
 
+    // Only used if the InputStream is not file based (and therefor does not support resets based on FileChannels)
+    // and the file is unordered
+    // Data and Palette cache is separated, as the data requires a fully populated palette - and the order is not guaranteed
+    private byte[] dataCache;
+    private byte[] paletteCache;
+    private OutputStream dataCacheWriter;
+    private OutputStream paletteCacheWriter;
+
 
     public FastSchematicReaderV3(@NonNull InputStream stream) {
         Objects.requireNonNull(stream, "stream");
         if (stream instanceof FileInputStream fileInputStream) {
             stream = new ResettableFileInputStream(fileInputStream);
-        } else if (!stream.markSupported()) {
+            stream.mark(Integer.MAX_VALUE);
-            // TODO: How to handle remote schematics using URL streams? (-> SchematicCommands.java L350-352)
+            this.remainingTags = new HashSet<>();
-            // LOGGER.warn("InputStream does not support mark - will be wrapped using in memory buffer");
+        } else {
             stream = new BufferedInputStream(stream);
+            this.remainingTags = null;
         }
-        this.resetableInputStream = stream;
+        this.parentStream = stream;
-        this.resetableInputStream.mark(Integer.MAX_VALUE);
     }
 
     @Override
@@ -103,7 +122,7 @@ public class FastSchematicReaderV3 implements ClipboardReader {
         byte type;
         String tag;
         while ((type = dataInputStream.readByte()) != NBTConstants.TYPE_END) {
-            tag = readTagName();
+            tag = this.dataInputStream.readUTF();
             switch (tag) {
                 case "DataVersion" -> {
                     this.dataVersion = this.dataInputStream.readInt();
@@ -116,7 +135,14 @@ public class FastSchematicReaderV3 implements ClipboardReader {
                                     .getDataFixer()
                     );
                 }
-                case "Offset" -> readOffset();
+                case "Offset" -> {
+                    this.dataInputStream.skipNBytes(4); // Array Length field (4 byte int)
+                    this.offset = BlockVector3.at(
+                            this.dataInputStream.readInt(),
+                            this.dataInputStream.readInt(),
+                            this.dataInputStream.readInt()
+                    );
+                }
                 case "Width" -> this.dimensions.mutX(this.dataInputStream.readShort() & 0xFFFF);
                 case "Height" -> this.dimensions.mutY(this.dataInputStream.readShort() & 0xFFFF);
                 case "Length" -> this.dimensions.mutZ(this.dataInputStream.readShort() & 0xFFFF);
@@ -137,13 +163,107 @@ public class FastSchematicReaderV3 implements ClipboardReader {
             throw new IOException("Invalid schematic - missing DataVersion");
         }
 
+        if (this.supportsReset() && !remainingTags.isEmpty()) {
+            readRemainingDataReset(clipboard);
+        } else if (this.dataCacheWriter != null || this.paletteCacheWriter != null) {
+            readRemainingDataCache(clipboard);
+        }
+
+        clipboard.setOrigin(this.offset.multiply(-1));
+        if (clipboard instanceof SimpleClipboard simpleClipboard && !this.offset.equals(BlockVector3.ZERO)) {
+            clipboard = new BlockArrayClipboard(simpleClipboard, this.offset);
+        }
+        return clipboard;
+    }
+
+
+    /**
+     * Reads all locally cached data (due to reset not being available) and applies them to the clipboard.
+     * <p>
+     * Firstly, closes all cache writers (which adds the END identifier to each and fills the cache byte arrays on this instance)
+     * If required, creates all missing palettes first (as needed by all remaining data).
+     * At last writes all missing data (block states, tile entities, biomes, entities).
+     *
+     * @param clipboard The clipboard to write into.
+     * @throws IOException on I/O error.
+     */
+    private void readRemainingDataCache(Clipboard clipboard) throws IOException {
+        byte identifier;
+        if (this.paletteCacheWriter != null) {
+            this.paletteCacheWriter.close();
+        }
+        if (this.dataCacheWriter != null) {
+            this.dataCacheWriter.close();
+        }
+        if (this.paletteCache != null) {
+            try (final DataInputStream cacheStream = new DataInputStream(new FastBufferedInputStream(
+                    new LZ4BlockInputStream(new FastBufferedInputStream(new ByteArrayInputStream(this.paletteCache)))))) {
+                while ((identifier = cacheStream.readByte()) != CACHE_IDENTIFIER_END) {
+                    if (identifier == CACHE_IDENTIFIER_BLOCK) {
+                        this.readPaletteMap(cacheStream, this.provideBlockPaletteInitializer());
+                        continue;
+                    }
+                    if (identifier == CACHE_IDENTIFIER_BIOMES) {
+                        this.readPaletteMap(cacheStream, this.provideBiomePaletteInitializer());
+                        continue;
+                    }
+                    throw new IOException("invalid cache state - got identifier: 0x" + identifier);
+                }
+            }
+        }
+        try (final DataInputStream cacheStream = new DataInputStream(new FastBufferedInputStream(
+                new LZ4BlockInputStream(new FastBufferedInputStream(new ByteArrayInputStream(this.dataCache)))));
+             final NBTInputStream cacheNbtIn = new NBTInputStream(cacheStream)) {
+            while ((identifier = cacheStream.readByte()) != CACHE_IDENTIFIER_END) {
+                switch (identifier) {
+                    case CACHE_IDENTIFIER_BLOCK -> this.readPaletteData(cacheStream, this.getBlockWriter(clipboard));
+                    case CACHE_IDENTIFIER_BIOMES -> this.readPaletteData(cacheStream, this.getBiomeWriter(clipboard));
+                    case CACHE_IDENTIFIER_ENTITIES -> {
+                        cacheStream.skipNBytes(1); // list child type (TAG_Compound)
+                        this.readEntityContainers(
+                                cacheStream,
+                                cacheNbtIn,
+                                DataFixer.FixTypes.ENTITY,
+                                this.provideEntityTransformer(clipboard)
+                        );
+                    }
+                    case CACHE_IDENTIFIER_BLOCK_TILE_ENTITIES -> {
+                        cacheStream.skipNBytes(1); // list child type (TAG_Compound)
+                        this.readEntityContainers(
+                                cacheStream,
+                                cacheNbtIn,
+                                DataFixer.FixTypes.BLOCK_ENTITY,
+                                this.provideTileEntityTransformer(clipboard)
+                        );
+                    }
+                    default -> throw new IOException("invalid cache state - got identifier: 0x" + identifier);
+                }
+            }
+        }
+    }
+
+    /**
+     * Reset the main stream of this clipboard and reads all remaining data that could not be read or fixed yet.
+     * Might need two iterations if the DataVersion tag is after the Blocks tag while the Palette inside the Blocks tag is not
+     * at the first position.
+     *
+     * @param clipboard The clipboard to write into.
+     * @throws IOException on I/O error.
+     */
+    private void readRemainingDataReset(Clipboard clipboard) throws IOException {
+        byte type;
+        String tag;
         outer:
         while (!this.remainingTags.isEmpty()) {
             this.reset();
             skipHeader(this.dataInputStream);
             while ((type = dataInputStream.readByte()) != NBTConstants.TYPE_END) {
-                tag = readTagName();
+                tag = dataInputStream.readUTF();
-                if (!this.remainingTags.remove(tag)) {
+                byte b = tag.equals("Blocks") ? CACHE_IDENTIFIER_BLOCK :
+                        tag.equals("Biomes") ? CACHE_IDENTIFIER_BIOMES :
+                                tag.equals("Entities") ? CACHE_IDENTIFIER_ENTITIES :
+                                        CACHE_IDENTIFIER_END;
+                if (!this.remainingTags.remove(b)) {
                     this.nbtInputStream.readTagPayloadLazy(type, 0);
                     continue;
                 }
@@ -158,58 +278,28 @@ public class FastSchematicReaderV3 implements ClipboardReader {
                 }
             }
         }
-        clipboard.setOrigin(this.offset.multiply(-1));
-        if (clipboard instanceof SimpleClipboard simpleClipboard && !this.offset.equals(BlockVector3.ZERO)) {
-            clipboard = new BlockArrayClipboard(simpleClipboard, this.offset);
-        }
-        return clipboard;
-    }
-
-    private void readOffset() throws IOException {
-        this.dataInputStream.skipNBytes(4); // Array Length field (4 byte int)
-        this.offset = BlockVector3.at(
-                this.dataInputStream.readInt(),
-                this.dataInputStream.readInt(),
-                this.dataInputStream.readInt()
-        );
     }
 
+    /**
+     * {@inheritDoc}
+     * <p>
+     * Requires {@link #read()}, {@link #read(UUID)} or {@link #read(UUID, Function)} to be called before.
+     */
     @Override
     public OptionalInt getDataVersion() {
         return this.dataVersion > -1 ? OptionalInt.of(this.dataVersion) : OptionalInt.empty();
     }
 
     private void readBlocks(Clipboard target) throws IOException {
-        BiConsumer<Integer, Character> blockStateApplier;
-        if (target instanceof LinearClipboard linearClipboard) {
-            blockStateApplier = (dataIndex, paletteIndex) -> linearClipboard.setBlock(dataIndex, this.blockPalette[paletteIndex]);
-        } else {
-            blockStateApplier = (dataIndex, paletteIndex) -> {
-                int y = dataIndex / (dimensions.x() * dimensions.z());
-                int remainder = dataIndex - (y * dimensions.x() * dimensions.z());
-                int z = remainder / dimensions.x();
-                int x = remainder - z * dimensions.x();
-                target.setBlock(x, y, z, this.blockPalette[paletteIndex]);
-            };
-        }
         this.blockPalette = new BlockState[BlockTypesCache.states.length];
         readPalette(
                 target != null,
-                "Blocks",
+                CACHE_IDENTIFIER_BLOCK,
                 () -> this.blockPalette.length == 0,
-                (value, index) -> {
+                this.provideBlockPaletteInitializer(),
-                    value = dataFixer.fixUp(DataFixer.FixTypes.BLOCK_STATE, value);
+                this.getBlockWriter(target),
-                    try {
-                        this.blockPalette[index] = BlockState.get(value);
-                    } catch (InputParseException e) {
-                        LOGGER.warn("Invalid BlockState in palette: {}. Block will be replaced with air.", value);
-                        this.blockPalette[index] = BlockTypes.AIR.getDefaultState();
-                    }
-                },
-                blockStateApplier,
                 (type, tag) -> {
                     if (!tag.equals("BlockEntities")) {
-                        LOGGER.warn("Found additional tag in block palette: {} (0x{}). Will skip tag.", tag, type);
                         try {
                             this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_LIST, 0);
                         } catch (IOException e) {
@@ -217,15 +307,6 @@ public class FastSchematicReaderV3 implements ClipboardReader {
                         }
                         return;
                     }
-                    if (target == null || dataFixer == null) {
-                        try {
-                            this.nbtInputStream.readTagPayloadLazy(type, 0);
-                        } catch (IOException e) {
-                            LOGGER.error("Failed to skip tile entities", e);
-                        }
-                        this.remainingTags.add("Blocks");
-                        return;
-                    }
                     try {
                         this.readTileEntities(target);
                     } catch (IOException e) {
@@ -236,33 +317,13 @@ public class FastSchematicReaderV3 implements ClipboardReader {
     }
 
     private void readBiomes(Clipboard target) throws IOException {
-        BiConsumer<Integer, Character> biomeApplier;
-        if (target instanceof LinearClipboard linearClipboard) {
-            biomeApplier = (dataIndex, paletteIndex) -> linearClipboard.setBiome(dataIndex, this.biomePalette[paletteIndex]);
-        } else {
-            biomeApplier = (dataIndex, paletteIndex) -> {
-                int y = dataIndex / (dimensions.x() * dimensions.z());
-                int remainder = dataIndex - (y * dimensions.x() * dimensions.z());
-                int z = remainder / dimensions.x();
-                int x = remainder - z * dimensions.x();
-                target.setBiome(x, y, z, this.biomePalette[paletteIndex]);
-            };
-        }
         this.biomePalette = new BiomeType[BiomeType.REGISTRY.size()];
         readPalette(
                 target != null,
-                "Biomes",
+                CACHE_IDENTIFIER_BIOMES,
                 () -> this.biomePalette.length == 0,
-                (value, index) -> {
+                this.provideBiomePaletteInitializer(),
-                    value = dataFixer.fixUp(DataFixer.FixTypes.BIOME, value);
+                this.getBiomeWriter(target),
-                    BiomeType biomeType = BiomeTypes.get(value);
-                    if (biomeType == null) {
-                        biomeType = BiomeTypes.PLAINS;
-                        LOGGER.warn("Invalid biome type in palette: {}. Biome will be replaced with plains.", value);
-                    }
-                    this.biomePalette[index] = biomeType;
-                },
-                biomeApplier,
                 (type, tag) -> {
                     try {
                         this.nbtInputStream.readTagPayloadLazy(type, 0);
@@ -273,111 +334,125 @@ public class FastSchematicReaderV3 implements ClipboardReader {
         );
     }
 
-    private void readEntities(Clipboard target) throws IOException {
+    private void readEntities(@Nullable Clipboard target) throws IOException {
+        if (target == null || this.dataFixer == null) {
+            if (supportsReset()) {
+                this.remainingTags.add(CACHE_IDENTIFIER_ENTITIES);
+                this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_LIST, 0);
+                return;
+            }
+            // Easier than streaming for now
+            final NBTOutputStream cacheStream = new NBTOutputStream(this.getDataCacheWriter());
+            cacheStream.writeByte(CACHE_IDENTIFIER_ENTITIES);
+            cacheStream.writeTagPayload(this.nbtInputStream.readTagPayload(NBTConstants.TYPE_LIST, 0));
+            System.out.println("Wrote entities to cache");
+            return;
+        }
         if (this.dataInputStream.read() != NBTConstants.TYPE_COMPOUND) {
             throw new IOException("Expected a compound block for entity");
         }
-        readEntityContainers((id, pos, data) -> {
+        this.readEntityContainers(
-            final EntityType entityType = EntityType.REGISTRY.get(id);
+                this.dataInputStream, this.nbtInputStream, DataFixer.FixTypes.ENTITY, this.provideEntityTransformer(target)
-            if (entityType == null) {
+        );
-                LOGGER.warn("Unknown entity {} @ {},{},{} - skipping", id, pos.x(), pos.y(), pos.z());
-                return;
-            }
-            // Back and forth conversion, because setTile only supports JNBT CompoundTag
-            // whereas DataFixer can only handle BinaryTagCompound...
-            CompoundBinaryTag tag = this.dataFixer.fixUp(DataFixer.FixTypes.ENTITY, data.asBinaryTag());
-            if (tag == null) {
-                LOGGER.warn("Failed to fix-up entity for {} @ {},{},{} - skipping", id, pos.x(), pos.y(), pos.z());
-                return;
-            }
-            if (target.createEntity(new Location(target, pos), new BaseEntity(entityType, new CompoundTag(tag))) == null) {
-                LOGGER.warn("Failed to create entity - does the clipboard support entities?");
-            }
-        });
     }
 
     private void readTileEntities(Clipboard target) throws IOException {
         if (target == null || this.dataFixer == null) {
+            if (supportsReset()) {
+                this.remainingTags.add(CACHE_IDENTIFIER_BLOCK); // use block identifier, as this method will be called by
+                // readBlocks again
                 this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_LIST, 0);
-            this.remainingTags.add("Entities");
+                return;
+            }
+            // Easier than streaming for now
+            final NBTOutputStream cacheStream = new NBTOutputStream(this.getDataCacheWriter());
+            cacheStream.writeByte(CACHE_IDENTIFIER_BLOCK_TILE_ENTITIES);
+            cacheStream.writeTagPayload(this.nbtInputStream.readTagPayload(NBTConstants.TYPE_LIST, 0));
             return;
         }
         if (this.dataInputStream.read() != NBTConstants.TYPE_COMPOUND) {
             throw new IOException("Expected a compound block for tile entity");
         }
-        readEntityContainers((id, pos, data) -> {
+        this.readEntityContainers(
-            // Back and forth conversion, because setTile only supports JNBT CompoundTag
+                this.dataInputStream,
-            // whereas DataFixer can only handle BinaryTagCompound...
+                this.nbtInputStream,
-            CompoundBinaryTag tag = this.dataFixer.fixUp(DataFixer.FixTypes.BLOCK_ENTITY, data.asBinaryTag());
+                DataFixer.FixTypes.BLOCK_ENTITY,
-            if (tag == null) {
+                this.provideTileEntityTransformer(target)
-                LOGGER.warn("Failed to fix-up tile entity for {} @ {},{},{} - skipping",
-                        id, pos.blockX(), pos.blockY(), pos.blockZ()
         );
-                return;
-            }
-            if (!target.setTile(pos.blockX(), pos.blockY(), pos.blockZ(), new CompoundTag(tag))) {
-                LOGGER.warn("Failed to set tile entity - does the clipboard support tile entities?");
-            }
-        });
     }
 
-    private void readEntityContainers(TriConsumer<String, Vector3, CompoundTag> writer) throws IOException {
+    private void readEntityContainers(
-        MutableVector3 pos = new MutableVector3();
+            DataInputStream stream,
-        CompoundTag tag;
+            NBTInputStream nbtStream,
+            DataFixer.FixType<CompoundBinaryTag> fixType,
+            EntityTransformer transformer
+    ) throws IOException {
+        double x, y, z;
+        CompoundBinaryTag tag;
         String id;
-        int count = this.dataInputStream.readInt();
         byte type;
+        int count = stream.readInt();
         while (count-- > 0) {
-            pos.setComponents(Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE);
+            x = -1;
+            y = -1;
+            z = -1;
             tag = null;
             id = null;
-            while ((type = this.dataInputStream.readByte()) != NBTConstants.TYPE_END) {
+            while ((type = stream.readByte()) != NBTConstants.TYPE_END) {
                 switch (type) {
                     // Depending on the type of entity container (tile vs "normal") the pos consists of either doubles or ints
                     case NBTConstants.TYPE_INT_ARRAY -> {
-                        if (!readTagName().equals("Pos")) {
+                        if (!stream.readUTF().equals("Pos")) {
                             throw new IOException("Expected INT_ARRAY tag to be Pos");
                         }
-                        this.dataInputStream.skipNBytes(4); // count of following ints - for pos = 3
+                        stream.skipNBytes(4); // count of following ints - for pos = 3
-                        pos.mutX(this.dataInputStream.readInt());
+                        x = stream.readInt();
-                        pos.mutY(this.dataInputStream.readInt());
+                        y = stream.readInt();
-                        pos.mutZ(this.dataInputStream.readInt());
+                        z = stream.readInt();
                     }
                     case NBTConstants.TYPE_LIST -> {
-                        if (!readTagName().equals("Pos")) {
+                        if (!stream.readUTF().equals("Pos")) {
                             throw new IOException("Expected LIST tag to be Pos");
                         }
-                        if (this.dataInputStream.readByte() != NBTConstants.TYPE_DOUBLE) {
+                        if (stream.readByte() != NBTConstants.TYPE_DOUBLE) {
                             throw new IOException("Expected LIST Pos tag to contain DOUBLE");
                         }
-                        this.dataInputStream.skipNBytes(4); // count of following doubles - for pos = 3
+                        stream.skipNBytes(4); // count of following doubles - for pos = 3
-                        pos.mutX(this.dataInputStream.readDouble());
+                        x = stream.readDouble();
-                        pos.mutY(this.dataInputStream.readDouble());
+                        y = stream.readDouble();
-                        pos.mutZ(this.dataInputStream.readDouble());
+                        z = stream.readDouble();
                     }
                     case NBTConstants.TYPE_STRING -> {
-                        if (!readTagName().equals("Id")) {
+                        if (!stream.readUTF().equals("Id")) {
                             throw new IOException("Expected STRING tag to be Id");
                         }
-                        id = this.dataInputStream.readUTF();
+                        id = stream.readUTF();
                     }
                     case NBTConstants.TYPE_COMPOUND -> {
-                        if (!readTagName().equals("Data")) {
+                        if (!stream.readUTF().equals("Data")) {
                             throw new IOException("Expected COMPOUND tag to be Data");
                         }
-                        tag = (CompoundTag) this.nbtInputStream.readTagPayload(NBTConstants.TYPE_COMPOUND, 0);
+                        tag = ((CompoundTag) nbtStream.readTagPayload(NBTConstants.TYPE_COMPOUND, 0)).asBinaryTag();
                     }
                     default -> throw new IOException("Unexpected tag in compound: " + type);
                 }
             }
-            // Data can be actually null is not required somehow?
-            if (tag == null) {
-                continue;
-            }
             if (id == null) {
                 throw new IOException("Missing Id tag in compound");
             }
-            writer.accept(id, pos, tag);
+            if (x < 0 || y < 0 || z < 0) {
+                throw new IOException("Missing position for entity " + id);
+            }
+            if (tag == null) {
+                transformer.transform(x, y, z, id, CompoundBinaryTag.empty());
+                continue;
+            }
+            tag = this.dataFixer.fixUp(fixType, tag);
+            if (tag == null) {
+                LOGGER.warn("Failed to fix-up entity for {} @ {},{},{} - skipping", id, x, y, z);
+                continue;
+            }
+            transformer.transform(x, y, z, id, tag);
         }
     }
 
@@ -390,84 +465,264 @@ public class FastSchematicReaderV3 implements ClipboardReader {
      */
     private void readPalette(
             boolean hasClipboard,
-            String rootTag,
+            byte paletteType,
             BooleanSupplier paletteAlreadyInitialized,
-            BiConsumer<String, Character> paletteInitializer,
+            PaletteInitializer paletteInitializer,
-            BiConsumer<Integer, Character> paletteDataApplier,
+            PaletteDataApplier paletteDataApplier,
-            BiConsumer<Byte, String> additionalTag
+            AdditionalTagConsumer additionalTag
     ) throws IOException {
         boolean hasPalette = paletteAlreadyInitialized.getAsBoolean();
         byte type;
         String tag;
         while ((type = this.dataInputStream.readByte()) != NBTConstants.TYPE_END) {
-            tag = readTagName();
+            tag = this.dataInputStream.readUTF();
             if (tag.equals("Palette")) {
                 if (hasPalette) {
                     // Skip palette, as already exists
                     this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_COMPOUND, 0);
                     continue;
                 }
-                if (this.dataFixer == null) {
+                if (!this.readPaletteMap(this.dataInputStream, paletteInitializer)) {
-                    this.remainingTags.add(rootTag);
+                    if (this.supportsReset()) {
+                        // Couldn't read - skip palette for now
+                        this.remainingTags.add(paletteType);
                         this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_COMPOUND, 0);
                         continue;
                     }
-                // Read all palette entries
+                    // Reset not possible, write into cache
-                while (this.dataInputStream.readByte() != NBTConstants.TYPE_END) {
+                    final NBTOutputStream cacheWriter = new NBTOutputStream(this.getPaletteCacheWriter());
-                    String value = this.dataInputStream.readUTF();
+                    cacheWriter.write(paletteType);
-                    char index = (char) this.dataInputStream.readInt();
+                    cacheWriter.writeTagPayload(this.nbtInputStream.readTagPayload(NBTConstants.TYPE_COMPOUND, 0));
-                    paletteInitializer.accept(value, index);
+                    continue;
                 }
                 hasPalette = true;
                 continue;
             }
             if (tag.equals("Data")) {
-                // No palette or dimensions are yet available - will need to read Data next round
+                // No palette or dimensions are yet available
                 if (!hasPalette || this.dataFixer == null || !hasClipboard) {
-                    this.remainingTags.add(rootTag); // mark for read next iteration
+                    if (this.supportsReset()) {
+                        this.remainingTags.add(paletteType);
                         this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_BYTE_ARRAY, 0);
                         continue;
                     }
-                int length = this.dataInputStream.readInt();
+                    // Reset not possible, write into cache
-                // Write data into clipboard
+                    int byteLen = this.dataInputStream.readInt();
-                int i = 0;
+                    final DataOutputStream cacheWriter = new DataOutputStream(this.getDataCacheWriter());
-                if (needsVarIntReading(length)) {
+                    cacheWriter.write(paletteType);
-                    for (var iter = new VarIntStreamIterator(this.dataInputStream, length); iter.hasNext(); i++) {
+                    cacheWriter.writeInt(byteLen);
-                        paletteDataApplier.accept(i, (char) iter.nextInt());
+                    IOUtil.copy(this.dataInputStream, cacheWriter, byteLen);
-                    }
                     continue;
                 }
-                while (i < length) {
+                this.readPaletteData(this.dataInputStream, paletteDataApplier);
-                    paletteDataApplier.accept(i++, (char) this.dataInputStream.readUnsignedByte());
+                continue;
-                }
             }
             additionalTag.accept(type, tag);
         }
     }
 
-    private String readTagName() throws IOException {
+    private void readPaletteData(DataInputStream stream, PaletteDataApplier applier) throws IOException {
-        return dataInputStream.readUTF();
+        int length = stream.readInt();
+        // Write data into clipboard
+        int i = 0;
+        if (needsVarIntReading(length)) {
+            for (var iter = new VarIntStreamIterator(stream, length); iter.hasNext(); i++) {
+                applier.apply(i, (char) iter.nextInt());
+            }
+            return;
+        }
+        while (i < length) {
+            applier.apply(i++, (char) stream.readUnsignedByte());
+        }
     }
 
+    /**
+     * Reads the CompoundTag containing the palette mapping ({@code index: value}) and passes each entry to the
+     * {@link PaletteInitializer}.
+     * <p>
+     * This method expects that the identifier ({@link NBTConstants#TYPE_COMPOUND}) is already consumed from the stream.
+     *
+     * @param stream      The stream to read the data from.
+     * @param initializer The initializer called for each entry with its index and backed value.
+     * @return {@code true} if the mapping could be read, {@code false} otherwise (e.g. DataFixer is not yet available).
+     * @throws IOException on I/O error.
+     */
+    private boolean readPaletteMap(DataInputStream stream, PaletteInitializer initializer) throws IOException {
+        if (this.dataFixer == null) {
+            return false;
+        }
+        while (stream.readByte() != NBTConstants.TYPE_END) {
+            String value = stream.readUTF();
+            char index = (char) stream.readInt();
+            initializer.initialize(index, value);
+        }
+        return true;
+    }
+
+    private void indexToPosition(int index, PositionConsumer supplier) {
+        int y = index / (dimensions.x() * dimensions.z());
+        int remainder = index - (y * dimensions.x() * dimensions.z());
+        int z = remainder / dimensions.x();
+        int x = remainder - z * dimensions.x();
+        supplier.accept(x, y, z);
+    }
+
+    private PaletteDataApplier getBlockWriter(Clipboard target) {
+        if (target instanceof LinearClipboard linearClipboard) {
+            return (index, ordinal) -> linearClipboard.setBlock(index, this.blockPalette[ordinal]);
+        }
+        return (index, ordinal) -> indexToPosition(index, (x, y, z) -> target.setBlock(x, y, z, this.blockPalette[ordinal]));
+    }
+
+    private PaletteDataApplier getBiomeWriter(Clipboard target) {
+        if (target instanceof LinearClipboard linearClipboard) {
+            return (index, ordinal) -> linearClipboard.setBiome(index, this.biomePalette[ordinal]);
+        }
+        return (index, ordinal) -> indexToPosition(index, (x, y, z) -> target.setBiome(x, y, z, this.biomePalette[ordinal]));
+    }
+
+    private PaletteInitializer provideBlockPaletteInitializer() {
+        return (index, value) -> {
+            if (this.dataFixer == null) {
+                throw new IllegalStateException("Can't read block palette map if DataFixer is not yet available");
+            }
+            value = dataFixer.fixUp(DataFixer.FixTypes.BLOCK_STATE, value);
+            try {
+                this.blockPalette[index] = BlockState.get(value);
+            } catch (InputParseException e) {
+                LOGGER.warn("Invalid BlockState in palette: {}. Block will be replaced with air.", value);
+                this.blockPalette[index] = BlockTypes.AIR.getDefaultState();
+            }
+        };
+    }
+
+    private PaletteInitializer provideBiomePaletteInitializer() {
+        return (index, value) -> {
+            if (this.dataFixer == null) {
+                throw new IllegalStateException("Can't read biome palette map if DataFixer is not yet available");
+            }
+            value = dataFixer.fixUp(DataFixer.FixTypes.BIOME, value);
+            BiomeType biomeType = BiomeTypes.get(value);
+            if (biomeType == null) {
+                biomeType = BiomeTypes.PLAINS;
+                LOGGER.warn("Invalid biome type in palette: {}. Biome will be replaced with plains.", value);
+            }
+            this.biomePalette[index] = biomeType;
+        };
+    }
+
+    private EntityTransformer provideEntityTransformer(Clipboard clipboard) {
+        return (x, y, z, id, tag) -> {
+            EntityType type = EntityType.REGISTRY.get(id);
+            if (type == null) {
+                LOGGER.warn("Invalid entity id: {} - skipping", id);
+                return;
+            }
+            clipboard.createEntity(
+                    new Location(clipboard, Location.at(x, y, z).add(clipboard.getMinimumPoint().toVector3())),
+                    new BaseEntity(type, LazyReference.computed(tag))
+            );
+        };
+    }
+
+    private EntityTransformer provideTileEntityTransformer(Clipboard clipboard) {
+        return (x, y, z, id, tag) -> clipboard.setTile(
+                MathMan.roundInt(x + clipboard.getMinimumPoint().x()),
+                MathMan.roundInt(y + clipboard.getMinimumPoint().y()),
+                MathMan.roundInt(z + clipboard.getMinimumPoint().z()),
+                new CompoundTag(tag)
+        );
+    }
+
+    /**
+     * @return {@code true} if {@code Width}, {@code Length} and {@code Height} are already read from the stream
+     */
     private boolean areDimensionsAvailable() {
         return this.dimensions.x() != 0 && this.dimensions.y() != 0 && this.dimensions.z() != 0;
     }
 
+    /**
+     * Closes this reader instance and all underlying resources.
+     *
+     * @throws IOException on I/O error.
+     */
     @Override
     public void close() throws IOException {
-        resetableInputStream.close(); // closes all underlying resources implicitly
+        parentStream.close(); // closes all underlying resources implicitly
     }
 
+    /**
+     * Resets the main stream to the previously marked position ({@code 0}), if supported (see {@link #supportsReset()}).
+     * If the stream is reset, the sub streams (for DataInput and NBT) are re-created to respect the new position.
+     *
+     * @throws IOException on I/O error.
+     */
+    private void reset() throws IOException {
+        if (this.supportsReset()) {
+            this.parentStream.reset();
+            this.parentStream.mark(Integer.MAX_VALUE);
+            this.setSubStreams();
+        }
+    }
+
+    /**
+     * @return {@code true} if the stream used while instantiating the reader supports resets (without memory overhead).
+     */
+    private boolean supportsReset() {
+        return this.remainingTags != null;
+    }
+
+    /**
+     * Overwrites the DataInput- and NBT-InputStreams (e.g. when the marker of the backed stream updated).
+     *
+     * @throws IOException on I/O error.
+     */
     private void setSubStreams() throws IOException {
-        final FastBufferedInputStream buffer = new FastBufferedInputStream(new GZIPInputStream(this.resetableInputStream));
+        final FastBufferedInputStream buffer = new FastBufferedInputStream(new GZIPInputStream(this.parentStream));
         this.dataInputStream = new DataInputStream(buffer);
         this.nbtInputStream = new NBTInputStream(buffer);
     }
 
-    private void reset() throws IOException {
+    /**
-        this.resetableInputStream.reset();
+     * Creates a new cache writer for non-palette data, if none exists yet.
-        this.resetableInputStream.mark(Integer.MAX_VALUE);
+     * Returns either the already created or new one.
-        this.setSubStreams();
+     *
+     * @return the output stream for non-palette cache data.
+     */
+    private OutputStream getDataCacheWriter() {
+        if (this.dataCacheWriter == null) {
+            ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(512);
+            this.dataCacheWriter = new FastBufferedOutputStream(new LZ4BlockOutputStream(byteArrayOutputStream)) {
+                @Override
+                public void close() throws IOException {
+                    this.write(CACHE_IDENTIFIER_END);
+                    super.close();
+                    FastSchematicReaderV3.this.dataCache = byteArrayOutputStream.toByteArray();
+                }
+            };
+        }
+        return this.dataCacheWriter;
+    }
+
+    /**
+     * Creates a new cache writer for palette data, if none exists yet.
+     * Returns either the already created or new one.
+     *
+     * @return the output stream for palette cache data.
+     */
+    private OutputStream getPaletteCacheWriter() {
+        if (this.paletteCacheWriter == null) {
+            ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(256);
+            this.paletteCacheWriter = new FastBufferedOutputStream(new LZ4BlockOutputStream(byteArrayOutputStream)) {
+                @Override
+                public void close() throws IOException {
+                    this.write(CACHE_IDENTIFIER_END);
+                    super.close();
+                    FastSchematicReaderV3.this.paletteCache = byteArrayOutputStream.toByteArray();
+                }
+            };
+        }
+        return this.paletteCacheWriter;
     }
 
     private boolean needsVarIntReading(int byteArrayLength) {
@@ -485,4 +740,78 @@ public class FastSchematicReaderV3 implements ClipboardReader {
         dataInputStream.skipNBytes(1 + 2 + 9); // as above + 9 bytes = "Schematic"
     }
 
+    @ApiStatus.Internal
+    @FunctionalInterface
+    private interface PositionConsumer {
+
+        /**
+         * Called with block location coordinates.
+         *
+         * @param x the x coordinate.
+         * @param y the y coordinate.
+         * @param z the z coordinate.
+         */
+        void accept(int x, int y, int z);
+
+    }
+
+    @ApiStatus.Internal
+    @FunctionalInterface
+    private interface EntityTransformer {
+
+        /**
+         * Called for each entity from the Schematics {@code Entities} compound list.
+         *
+         * @param x   the relative x coordinate of the entity.
+         * @param y   the relative y coordinate of the entity.
+         * @param z   the relative z coordinate of the entity.
+         * @param id  the entity id as a resource location (e.g. {@code minecraft:sheep}).
+         * @param tag the - already fixed, if required - nbt data of the entity.
+         */
+        void transform(double x, double y, double z, String id, CompoundBinaryTag tag);
+
+    }
+
+    @ApiStatus.Internal
+    @FunctionalInterface
+    private interface PaletteInitializer {
+
+        /**
+         * Called for each palette entry (the mapping part, not data).
+         *
+         * @param index the index of the entry, as used in the Data byte array.
+         * @param value the value for this entry (either biome type as resource location or the block state as a string).
+         */
+        void initialize(char index, String value);
+
+    }
+
+    @ApiStatus.Internal
+    @FunctionalInterface
+    private interface PaletteDataApplier {
+
+        /**
+         * Called for each palette data entry (not the mapping part, but the var-int byte array).
+         *
+         * @param index   The index of this data entry (due to var-int behaviour not necessarily the index in the data byte array).
+         * @param ordinal The ordinal of this entry as defined in the palette mapping.
+         */
+        void apply(int index, char ordinal);
+
+    }
+
+    @ApiStatus.Internal
+    @FunctionalInterface
+    private interface AdditionalTagConsumer {
+
+        /**
+         * Called for each unknown nbt tag.
+         *
+         * @param type The type of the tag (as defined by the constants in {@link NBTConstants}).
+         * @param name The name of the tag.
+         */
+        void accept(byte type, String name);
+
+    }
+
 }