Mirror von
https://github.com/IntellectualSites/FastAsyncWorldEdit.git
synchronisiert 2024-11-19 17:30:08 +01:00
chore/feat: support non-file based inputstreams (using in memory LZ4 cache)
Dieser Commit ist enthalten in:
Pierre Maurice Schwang
Ursprung
0f3b81fcd6
Commit
fae46c6e4e
@ -1,6 +1,8 @@
|
||||
package com.fastasyncworldedit.bukkit.util;
|
||||
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
|
||||
import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
|
||||
import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
|
||||
import it.unimi.dsi.fastutil.longs.LongArraySet;
|
||||
import it.unimi.dsi.fastutil.longs.LongIterator;
|
||||
@ -19,5 +21,7 @@ final class DoNotMiniseThese {
|
||||
private final LongSet d = null;
|
||||
private final Int2ObjectMap<?> e = null;
|
||||
private final Object2ObjectArrayMap<?, ?> f = null;
|
||||
private final FastBufferedInputStream g = null;
|
||||
private final FastBufferedOutputStream h = null;
|
||||
|
||||
}
|
||||
|
||||
@ -5,10 +5,12 @@ import com.fastasyncworldedit.core.extent.clipboard.SimpleClipboard;
|
||||
import com.fastasyncworldedit.core.internal.io.ResettableFileInputStream;
|
||||
import com.fastasyncworldedit.core.internal.io.VarIntStreamIterator;
|
||||
import com.fastasyncworldedit.core.math.MutableBlockVector3;
|
||||
import com.fastasyncworldedit.core.math.MutableVector3;
|
||||
import com.fastasyncworldedit.core.util.IOUtil;
|
||||
import com.fastasyncworldedit.core.util.MathMan;
|
||||
import com.sk89q.jnbt.CompoundTag;
|
||||
import com.sk89q.jnbt.NBTConstants;
|
||||
import com.sk89q.jnbt.NBTInputStream;
|
||||
import com.sk89q.jnbt.NBTOutputStream;
|
||||
import com.sk89q.worldedit.WorldEdit;
|
||||
import com.sk89q.worldedit.entity.BaseEntity;
|
||||
import com.sk89q.worldedit.extension.input.InputParseException;
|
||||
@ -19,8 +21,8 @@ import com.sk89q.worldedit.extent.clipboard.io.ClipboardReader;
|
||||
import com.sk89q.worldedit.extent.clipboard.io.sponge.VersionedDataFixer;
|
||||
import com.sk89q.worldedit.internal.util.LogManagerCompat;
|
||||
import com.sk89q.worldedit.math.BlockVector3;
|
||||
import com.sk89q.worldedit.math.Vector3;
|
||||
import com.sk89q.worldedit.util.Location;
|
||||
import com.sk89q.worldedit.util.concurrency.LazyReference;
|
||||
import com.sk89q.worldedit.util.nbt.CompoundBinaryTag;
|
||||
import com.sk89q.worldedit.world.DataFixer;
|
||||
import com.sk89q.worldedit.world.biome.BiomeType;
|
||||
@ -30,21 +32,28 @@ import com.sk89q.worldedit.world.block.BlockTypes;
|
||||
import com.sk89q.worldedit.world.block.BlockTypesCache;
|
||||
import com.sk89q.worldedit.world.entity.EntityType;
|
||||
import it.unimi.dsi.fastutil.io.FastBufferedInputStream;
|
||||
import it.unimi.dsi.fastutil.io.FastBufferedOutputStream;
|
||||
import net.jpountz.lz4.LZ4BlockInputStream;
|
||||
import net.jpountz.lz4.LZ4BlockOutputStream;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.apache.logging.log4j.util.TriConsumer;
|
||||
import org.checkerframework.checker.nullness.qual.NonNull;
|
||||
import org.checkerframework.checker.nullness.qual.Nullable;
|
||||
import org.jetbrains.annotations.ApiStatus;
|
||||
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.DataInputStream;
|
||||
import java.io.DataOutputStream;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.util.HashSet;
|
||||
import java.util.Objects;
|
||||
import java.util.OptionalInt;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.BooleanSupplier;
|
||||
import java.util.function.Function;
|
||||
import java.util.zip.GZIPInputStream;
|
||||
@ -55,20 +64,22 @@ import java.util.zip.GZIPInputStream;
|
||||
* stream based approach to keep the memory overhead minimal (especially in larger schematics)
|
||||
*/
|
||||
|
||||
/**
|
||||
* TODO: - Validate FileChannel reset performance (especially for network drive / remote backed storage)
|
||||
* TODO: ^ try to compare speed and memory / cpu usage when - instead of resetting the stream - caching the palette using LZ4 /
|
||||
* TODO ZSTD until other data is available
|
||||
/**le
|
||||
* TODO: fix tile entity locations (+ validate entity location)
|
||||
*/
|
||||
@SuppressWarnings("removal") // JNBT
|
||||
public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
|
||||
private static final Logger LOGGER = LogManagerCompat.getLogger();
|
||||
private static final byte CACHE_IDENTIFIER_END = 0x00;
|
||||
private static final byte CACHE_IDENTIFIER_BLOCK = 0x01;
|
||||
private static final byte CACHE_IDENTIFIER_BIOMES = 0x02;
|
||||
private static final byte CACHE_IDENTIFIER_ENTITIES = 0x03;
|
||||
private static final byte CACHE_IDENTIFIER_BLOCK_TILE_ENTITIES = 0x04;
|
||||
|
||||
private final InputStream resetableInputStream;
|
||||
private final InputStream parentStream;
|
||||
private final MutableBlockVector3 dimensions = MutableBlockVector3.at(0, 0, 0);
|
||||
private final Set<String> remainingTags = new HashSet<>();
|
||||
private final Set<Byte> remainingTags;
|
||||
|
||||
private DataInputStream dataInputStream;
|
||||
private NBTInputStream nbtInputStream;
|
||||
@ -79,18 +90,26 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
private BiomeType[] biomePalette;
|
||||
private int dataVersion = -1;
|
||||
|
||||
// Only used if the InputStream is not file based (and therefor does not support resets based on FileChannels)
|
||||
// and the file is unordered
|
||||
// Data and Palette cache is separated, as the data requires a fully populated palette - and the order is not guaranteed
|
||||
private byte[] dataCache;
|
||||
private byte[] paletteCache;
|
||||
private OutputStream dataCacheWriter;
|
||||
private OutputStream paletteCacheWriter;
|
||||
|
||||
|
||||
public FastSchematicReaderV3(@NonNull InputStream stream) {
|
||||
Objects.requireNonNull(stream, "stream");
|
||||
if (stream instanceof FileInputStream fileInputStream) {
|
||||
stream = new ResettableFileInputStream(fileInputStream);
|
||||
} else if (!stream.markSupported()) {
|
||||
// TODO: How to handle remote schematics using URL streams? (-> SchematicCommands.java L350-352)
|
||||
// LOGGER.warn("InputStream does not support mark - will be wrapped using in memory buffer");
|
||||
stream.mark(Integer.MAX_VALUE);
|
||||
this.remainingTags = new HashSet<>();
|
||||
} else {
|
||||
stream = new BufferedInputStream(stream);
|
||||
this.remainingTags = null;
|
||||
}
|
||||
this.resetableInputStream = stream;
|
||||
this.resetableInputStream.mark(Integer.MAX_VALUE);
|
||||
this.parentStream = stream;
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -103,7 +122,7 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
byte type;
|
||||
String tag;
|
||||
while ((type = dataInputStream.readByte()) != NBTConstants.TYPE_END) {
|
||||
tag = readTagName();
|
||||
tag = this.dataInputStream.readUTF();
|
||||
switch (tag) {
|
||||
case "DataVersion" -> {
|
||||
this.dataVersion = this.dataInputStream.readInt();
|
||||
@ -116,7 +135,14 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
.getDataFixer()
|
||||
);
|
||||
}
|
||||
case "Offset" -> readOffset();
|
||||
case "Offset" -> {
|
||||
this.dataInputStream.skipNBytes(4); // Array Length field (4 byte int)
|
||||
this.offset = BlockVector3.at(
|
||||
this.dataInputStream.readInt(),
|
||||
this.dataInputStream.readInt(),
|
||||
this.dataInputStream.readInt()
|
||||
);
|
||||
}
|
||||
case "Width" -> this.dimensions.mutX(this.dataInputStream.readShort() & 0xFFFF);
|
||||
case "Height" -> this.dimensions.mutY(this.dataInputStream.readShort() & 0xFFFF);
|
||||
case "Length" -> this.dimensions.mutZ(this.dataInputStream.readShort() & 0xFFFF);
|
||||
@ -137,13 +163,107 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
throw new IOException("Invalid schematic - missing DataVersion");
|
||||
}
|
||||
|
||||
if (this.supportsReset() && !remainingTags.isEmpty()) {
|
||||
readRemainingDataReset(clipboard);
|
||||
} else if (this.dataCacheWriter != null || this.paletteCacheWriter != null) {
|
||||
readRemainingDataCache(clipboard);
|
||||
}
|
||||
|
||||
clipboard.setOrigin(this.offset.multiply(-1));
|
||||
if (clipboard instanceof SimpleClipboard simpleClipboard && !this.offset.equals(BlockVector3.ZERO)) {
|
||||
clipboard = new BlockArrayClipboard(simpleClipboard, this.offset);
|
||||
}
|
||||
return clipboard;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Reads all locally cached data (due to reset not being available) and applies them to the clipboard.
|
||||
* <p>
|
||||
* Firstly, closes all cache writers (which adds the END identifier to each and fills the cache byte arrays on this instance)
|
||||
* If required, creates all missing palettes first (as needed by all remaining data).
|
||||
* At last writes all missing data (block states, tile entities, biomes, entities).
|
||||
*
|
||||
* @param clipboard The clipboard to write into.
|
||||
* @throws IOException on I/O error.
|
||||
*/
|
||||
private void readRemainingDataCache(Clipboard clipboard) throws IOException {
|
||||
byte identifier;
|
||||
if (this.paletteCacheWriter != null) {
|
||||
this.paletteCacheWriter.close();
|
||||
}
|
||||
if (this.dataCacheWriter != null) {
|
||||
this.dataCacheWriter.close();
|
||||
}
|
||||
if (this.paletteCache != null) {
|
||||
try (final DataInputStream cacheStream = new DataInputStream(new FastBufferedInputStream(
|
||||
new LZ4BlockInputStream(new FastBufferedInputStream(new ByteArrayInputStream(this.paletteCache)))))) {
|
||||
while ((identifier = cacheStream.readByte()) != CACHE_IDENTIFIER_END) {
|
||||
if (identifier == CACHE_IDENTIFIER_BLOCK) {
|
||||
this.readPaletteMap(cacheStream, this.provideBlockPaletteInitializer());
|
||||
continue;
|
||||
}
|
||||
if (identifier == CACHE_IDENTIFIER_BIOMES) {
|
||||
this.readPaletteMap(cacheStream, this.provideBiomePaletteInitializer());
|
||||
continue;
|
||||
}
|
||||
throw new IOException("invalid cache state - got identifier: 0x" + identifier);
|
||||
}
|
||||
}
|
||||
}
|
||||
try (final DataInputStream cacheStream = new DataInputStream(new FastBufferedInputStream(
|
||||
new LZ4BlockInputStream(new FastBufferedInputStream(new ByteArrayInputStream(this.dataCache)))));
|
||||
final NBTInputStream cacheNbtIn = new NBTInputStream(cacheStream)) {
|
||||
while ((identifier = cacheStream.readByte()) != CACHE_IDENTIFIER_END) {
|
||||
switch (identifier) {
|
||||
case CACHE_IDENTIFIER_BLOCK -> this.readPaletteData(cacheStream, this.getBlockWriter(clipboard));
|
||||
case CACHE_IDENTIFIER_BIOMES -> this.readPaletteData(cacheStream, this.getBiomeWriter(clipboard));
|
||||
case CACHE_IDENTIFIER_ENTITIES -> {
|
||||
cacheStream.skipNBytes(1); // list child type (TAG_Compound)
|
||||
this.readEntityContainers(
|
||||
cacheStream,
|
||||
cacheNbtIn,
|
||||
DataFixer.FixTypes.ENTITY,
|
||||
this.provideEntityTransformer(clipboard)
|
||||
);
|
||||
}
|
||||
case CACHE_IDENTIFIER_BLOCK_TILE_ENTITIES -> {
|
||||
cacheStream.skipNBytes(1); // list child type (TAG_Compound)
|
||||
this.readEntityContainers(
|
||||
cacheStream,
|
||||
cacheNbtIn,
|
||||
DataFixer.FixTypes.BLOCK_ENTITY,
|
||||
this.provideTileEntityTransformer(clipboard)
|
||||
);
|
||||
}
|
||||
default -> throw new IOException("invalid cache state - got identifier: 0x" + identifier);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the main stream of this clipboard and reads all remaining data that could not be read or fixed yet.
|
||||
* Might need two iterations if the DataVersion tag is after the Blocks tag while the Palette inside the Blocks tag is not
|
||||
* at the first position.
|
||||
*
|
||||
* @param clipboard The clipboard to write into.
|
||||
* @throws IOException on I/O error.
|
||||
*/
|
||||
private void readRemainingDataReset(Clipboard clipboard) throws IOException {
|
||||
byte type;
|
||||
String tag;
|
||||
outer:
|
||||
while (!this.remainingTags.isEmpty()) {
|
||||
this.reset();
|
||||
skipHeader(this.dataInputStream);
|
||||
while ((type = dataInputStream.readByte()) != NBTConstants.TYPE_END) {
|
||||
tag = readTagName();
|
||||
if (!this.remainingTags.remove(tag)) {
|
||||
tag = dataInputStream.readUTF();
|
||||
byte b = tag.equals("Blocks") ? CACHE_IDENTIFIER_BLOCK :
|
||||
tag.equals("Biomes") ? CACHE_IDENTIFIER_BIOMES :
|
||||
tag.equals("Entities") ? CACHE_IDENTIFIER_ENTITIES :
|
||||
CACHE_IDENTIFIER_END;
|
||||
if (!this.remainingTags.remove(b)) {
|
||||
this.nbtInputStream.readTagPayloadLazy(type, 0);
|
||||
continue;
|
||||
}
|
||||
@ -158,58 +278,28 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
}
|
||||
}
|
||||
}
|
||||
clipboard.setOrigin(this.offset.multiply(-1));
|
||||
if (clipboard instanceof SimpleClipboard simpleClipboard && !this.offset.equals(BlockVector3.ZERO)) {
|
||||
clipboard = new BlockArrayClipboard(simpleClipboard, this.offset);
|
||||
}
|
||||
return clipboard;
|
||||
}
|
||||
|
||||
private void readOffset() throws IOException {
|
||||
this.dataInputStream.skipNBytes(4); // Array Length field (4 byte int)
|
||||
this.offset = BlockVector3.at(
|
||||
this.dataInputStream.readInt(),
|
||||
this.dataInputStream.readInt(),
|
||||
this.dataInputStream.readInt()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@inheritDoc}
|
||||
* <p>
|
||||
* Requires {@link #read()}, {@link #read(UUID)} or {@link #read(UUID, Function)} to be called before.
|
||||
*/
|
||||
@Override
|
||||
public OptionalInt getDataVersion() {
|
||||
return this.dataVersion > -1 ? OptionalInt.of(this.dataVersion) : OptionalInt.empty();
|
||||
}
|
||||
|
||||
private void readBlocks(Clipboard target) throws IOException {
|
||||
BiConsumer<Integer, Character> blockStateApplier;
|
||||
if (target instanceof LinearClipboard linearClipboard) {
|
||||
blockStateApplier = (dataIndex, paletteIndex) -> linearClipboard.setBlock(dataIndex, this.blockPalette[paletteIndex]);
|
||||
} else {
|
||||
blockStateApplier = (dataIndex, paletteIndex) -> {
|
||||
int y = dataIndex / (dimensions.x() * dimensions.z());
|
||||
int remainder = dataIndex - (y * dimensions.x() * dimensions.z());
|
||||
int z = remainder / dimensions.x();
|
||||
int x = remainder - z * dimensions.x();
|
||||
target.setBlock(x, y, z, this.blockPalette[paletteIndex]);
|
||||
};
|
||||
}
|
||||
this.blockPalette = new BlockState[BlockTypesCache.states.length];
|
||||
readPalette(
|
||||
target != null,
|
||||
"Blocks",
|
||||
CACHE_IDENTIFIER_BLOCK,
|
||||
() -> this.blockPalette.length == 0,
|
||||
(value, index) -> {
|
||||
value = dataFixer.fixUp(DataFixer.FixTypes.BLOCK_STATE, value);
|
||||
try {
|
||||
this.blockPalette[index] = BlockState.get(value);
|
||||
} catch (InputParseException e) {
|
||||
LOGGER.warn("Invalid BlockState in palette: {}. Block will be replaced with air.", value);
|
||||
this.blockPalette[index] = BlockTypes.AIR.getDefaultState();
|
||||
}
|
||||
},
|
||||
blockStateApplier,
|
||||
this.provideBlockPaletteInitializer(),
|
||||
this.getBlockWriter(target),
|
||||
(type, tag) -> {
|
||||
if (!tag.equals("BlockEntities")) {
|
||||
LOGGER.warn("Found additional tag in block palette: {} (0x{}). Will skip tag.", tag, type);
|
||||
try {
|
||||
this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_LIST, 0);
|
||||
} catch (IOException e) {
|
||||
@ -217,15 +307,6 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (target == null || dataFixer == null) {
|
||||
try {
|
||||
this.nbtInputStream.readTagPayloadLazy(type, 0);
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Failed to skip tile entities", e);
|
||||
}
|
||||
this.remainingTags.add("Blocks");
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.readTileEntities(target);
|
||||
} catch (IOException e) {
|
||||
@ -236,33 +317,13 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
}
|
||||
|
||||
private void readBiomes(Clipboard target) throws IOException {
|
||||
BiConsumer<Integer, Character> biomeApplier;
|
||||
if (target instanceof LinearClipboard linearClipboard) {
|
||||
biomeApplier = (dataIndex, paletteIndex) -> linearClipboard.setBiome(dataIndex, this.biomePalette[paletteIndex]);
|
||||
} else {
|
||||
biomeApplier = (dataIndex, paletteIndex) -> {
|
||||
int y = dataIndex / (dimensions.x() * dimensions.z());
|
||||
int remainder = dataIndex - (y * dimensions.x() * dimensions.z());
|
||||
int z = remainder / dimensions.x();
|
||||
int x = remainder - z * dimensions.x();
|
||||
target.setBiome(x, y, z, this.biomePalette[paletteIndex]);
|
||||
};
|
||||
}
|
||||
this.biomePalette = new BiomeType[BiomeType.REGISTRY.size()];
|
||||
readPalette(
|
||||
target != null,
|
||||
"Biomes",
|
||||
CACHE_IDENTIFIER_BIOMES,
|
||||
() -> this.biomePalette.length == 0,
|
||||
(value, index) -> {
|
||||
value = dataFixer.fixUp(DataFixer.FixTypes.BIOME, value);
|
||||
BiomeType biomeType = BiomeTypes.get(value);
|
||||
if (biomeType == null) {
|
||||
biomeType = BiomeTypes.PLAINS;
|
||||
LOGGER.warn("Invalid biome type in palette: {}. Biome will be replaced with plains.", value);
|
||||
}
|
||||
this.biomePalette[index] = biomeType;
|
||||
},
|
||||
biomeApplier,
|
||||
this.provideBiomePaletteInitializer(),
|
||||
this.getBiomeWriter(target),
|
||||
(type, tag) -> {
|
||||
try {
|
||||
this.nbtInputStream.readTagPayloadLazy(type, 0);
|
||||
@ -273,111 +334,125 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
);
|
||||
}
|
||||
|
||||
private void readEntities(Clipboard target) throws IOException {
|
||||
private void readEntities(@Nullable Clipboard target) throws IOException {
|
||||
if (target == null || this.dataFixer == null) {
|
||||
if (supportsReset()) {
|
||||
this.remainingTags.add(CACHE_IDENTIFIER_ENTITIES);
|
||||
this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_LIST, 0);
|
||||
return;
|
||||
}
|
||||
// Easier than streaming for now
|
||||
final NBTOutputStream cacheStream = new NBTOutputStream(this.getDataCacheWriter());
|
||||
cacheStream.writeByte(CACHE_IDENTIFIER_ENTITIES);
|
||||
cacheStream.writeTagPayload(this.nbtInputStream.readTagPayload(NBTConstants.TYPE_LIST, 0));
|
||||
System.out.println("Wrote entities to cache");
|
||||
return;
|
||||
}
|
||||
if (this.dataInputStream.read() != NBTConstants.TYPE_COMPOUND) {
|
||||
throw new IOException("Expected a compound block for entity");
|
||||
}
|
||||
readEntityContainers((id, pos, data) -> {
|
||||
final EntityType entityType = EntityType.REGISTRY.get(id);
|
||||
if (entityType == null) {
|
||||
LOGGER.warn("Unknown entity {} @ {},{},{} - skipping", id, pos.x(), pos.y(), pos.z());
|
||||
return;
|
||||
}
|
||||
// Back and forth conversion, because setTile only supports JNBT CompoundTag
|
||||
// whereas DataFixer can only handle BinaryTagCompound...
|
||||
CompoundBinaryTag tag = this.dataFixer.fixUp(DataFixer.FixTypes.ENTITY, data.asBinaryTag());
|
||||
if (tag == null) {
|
||||
LOGGER.warn("Failed to fix-up entity for {} @ {},{},{} - skipping", id, pos.x(), pos.y(), pos.z());
|
||||
return;
|
||||
}
|
||||
if (target.createEntity(new Location(target, pos), new BaseEntity(entityType, new CompoundTag(tag))) == null) {
|
||||
LOGGER.warn("Failed to create entity - does the clipboard support entities?");
|
||||
}
|
||||
});
|
||||
this.readEntityContainers(
|
||||
this.dataInputStream, this.nbtInputStream, DataFixer.FixTypes.ENTITY, this.provideEntityTransformer(target)
|
||||
);
|
||||
}
|
||||
|
||||
private void readTileEntities(Clipboard target) throws IOException {
|
||||
if (target == null || this.dataFixer == null) {
|
||||
if (supportsReset()) {
|
||||
this.remainingTags.add(CACHE_IDENTIFIER_BLOCK); // use block identifier, as this method will be called by
|
||||
// readBlocks again
|
||||
this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_LIST, 0);
|
||||
this.remainingTags.add("Entities");
|
||||
return;
|
||||
}
|
||||
// Easier than streaming for now
|
||||
final NBTOutputStream cacheStream = new NBTOutputStream(this.getDataCacheWriter());
|
||||
cacheStream.writeByte(CACHE_IDENTIFIER_BLOCK_TILE_ENTITIES);
|
||||
cacheStream.writeTagPayload(this.nbtInputStream.readTagPayload(NBTConstants.TYPE_LIST, 0));
|
||||
return;
|
||||
}
|
||||
if (this.dataInputStream.read() != NBTConstants.TYPE_COMPOUND) {
|
||||
throw new IOException("Expected a compound block for tile entity");
|
||||
}
|
||||
readEntityContainers((id, pos, data) -> {
|
||||
// Back and forth conversion, because setTile only supports JNBT CompoundTag
|
||||
// whereas DataFixer can only handle BinaryTagCompound...
|
||||
CompoundBinaryTag tag = this.dataFixer.fixUp(DataFixer.FixTypes.BLOCK_ENTITY, data.asBinaryTag());
|
||||
if (tag == null) {
|
||||
LOGGER.warn("Failed to fix-up tile entity for {} @ {},{},{} - skipping",
|
||||
id, pos.blockX(), pos.blockY(), pos.blockZ()
|
||||
this.readEntityContainers(
|
||||
this.dataInputStream,
|
||||
this.nbtInputStream,
|
||||
DataFixer.FixTypes.BLOCK_ENTITY,
|
||||
this.provideTileEntityTransformer(target)
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!target.setTile(pos.blockX(), pos.blockY(), pos.blockZ(), new CompoundTag(tag))) {
|
||||
LOGGER.warn("Failed to set tile entity - does the clipboard support tile entities?");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void readEntityContainers(TriConsumer<String, Vector3, CompoundTag> writer) throws IOException {
|
||||
MutableVector3 pos = new MutableVector3();
|
||||
CompoundTag tag;
|
||||
private void readEntityContainers(
|
||||
DataInputStream stream,
|
||||
NBTInputStream nbtStream,
|
||||
DataFixer.FixType<CompoundBinaryTag> fixType,
|
||||
EntityTransformer transformer
|
||||
) throws IOException {
|
||||
double x, y, z;
|
||||
CompoundBinaryTag tag;
|
||||
String id;
|
||||
int count = this.dataInputStream.readInt();
|
||||
byte type;
|
||||
int count = stream.readInt();
|
||||
while (count-- > 0) {
|
||||
pos.setComponents(Integer.MIN_VALUE, Integer.MIN_VALUE, Integer.MIN_VALUE);
|
||||
x = -1;
|
||||
y = -1;
|
||||
z = -1;
|
||||
tag = null;
|
||||
id = null;
|
||||
while ((type = this.dataInputStream.readByte()) != NBTConstants.TYPE_END) {
|
||||
while ((type = stream.readByte()) != NBTConstants.TYPE_END) {
|
||||
switch (type) {
|
||||
// Depending on the type of entity container (tile vs "normal") the pos consists of either doubles or ints
|
||||
case NBTConstants.TYPE_INT_ARRAY -> {
|
||||
if (!readTagName().equals("Pos")) {
|
||||
if (!stream.readUTF().equals("Pos")) {
|
||||
throw new IOException("Expected INT_ARRAY tag to be Pos");
|
||||
}
|
||||
this.dataInputStream.skipNBytes(4); // count of following ints - for pos = 3
|
||||
pos.mutX(this.dataInputStream.readInt());
|
||||
pos.mutY(this.dataInputStream.readInt());
|
||||
pos.mutZ(this.dataInputStream.readInt());
|
||||
stream.skipNBytes(4); // count of following ints - for pos = 3
|
||||
x = stream.readInt();
|
||||
y = stream.readInt();
|
||||
z = stream.readInt();
|
||||
}
|
||||
case NBTConstants.TYPE_LIST -> {
|
||||
if (!readTagName().equals("Pos")) {
|
||||
if (!stream.readUTF().equals("Pos")) {
|
||||
throw new IOException("Expected LIST tag to be Pos");
|
||||
}
|
||||
if (this.dataInputStream.readByte() != NBTConstants.TYPE_DOUBLE) {
|
||||
if (stream.readByte() != NBTConstants.TYPE_DOUBLE) {
|
||||
throw new IOException("Expected LIST Pos tag to contain DOUBLE");
|
||||
}
|
||||
this.dataInputStream.skipNBytes(4); // count of following doubles - for pos = 3
|
||||
pos.mutX(this.dataInputStream.readDouble());
|
||||
pos.mutY(this.dataInputStream.readDouble());
|
||||
pos.mutZ(this.dataInputStream.readDouble());
|
||||
stream.skipNBytes(4); // count of following doubles - for pos = 3
|
||||
x = stream.readDouble();
|
||||
y = stream.readDouble();
|
||||
z = stream.readDouble();
|
||||
}
|
||||
case NBTConstants.TYPE_STRING -> {
|
||||
if (!readTagName().equals("Id")) {
|
||||
if (!stream.readUTF().equals("Id")) {
|
||||
throw new IOException("Expected STRING tag to be Id");
|
||||
}
|
||||
id = this.dataInputStream.readUTF();
|
||||
id = stream.readUTF();
|
||||
}
|
||||
case NBTConstants.TYPE_COMPOUND -> {
|
||||
if (!readTagName().equals("Data")) {
|
||||
if (!stream.readUTF().equals("Data")) {
|
||||
throw new IOException("Expected COMPOUND tag to be Data");
|
||||
}
|
||||
tag = (CompoundTag) this.nbtInputStream.readTagPayload(NBTConstants.TYPE_COMPOUND, 0);
|
||||
tag = ((CompoundTag) nbtStream.readTagPayload(NBTConstants.TYPE_COMPOUND, 0)).asBinaryTag();
|
||||
}
|
||||
default -> throw new IOException("Unexpected tag in compound: " + type);
|
||||
}
|
||||
}
|
||||
// Data can be actually null is not required somehow?
|
||||
if (tag == null) {
|
||||
continue;
|
||||
}
|
||||
if (id == null) {
|
||||
throw new IOException("Missing Id tag in compound");
|
||||
}
|
||||
writer.accept(id, pos, tag);
|
||||
if (x < 0 || y < 0 || z < 0) {
|
||||
throw new IOException("Missing position for entity " + id);
|
||||
}
|
||||
if (tag == null) {
|
||||
transformer.transform(x, y, z, id, CompoundBinaryTag.empty());
|
||||
continue;
|
||||
}
|
||||
tag = this.dataFixer.fixUp(fixType, tag);
|
||||
if (tag == null) {
|
||||
LOGGER.warn("Failed to fix-up entity for {} @ {},{},{} - skipping", id, x, y, z);
|
||||
continue;
|
||||
}
|
||||
transformer.transform(x, y, z, id, tag);
|
||||
}
|
||||
}
|
||||
|
||||
@ -390,84 +465,264 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
*/
|
||||
private void readPalette(
|
||||
boolean hasClipboard,
|
||||
String rootTag,
|
||||
byte paletteType,
|
||||
BooleanSupplier paletteAlreadyInitialized,
|
||||
BiConsumer<String, Character> paletteInitializer,
|
||||
BiConsumer<Integer, Character> paletteDataApplier,
|
||||
BiConsumer<Byte, String> additionalTag
|
||||
PaletteInitializer paletteInitializer,
|
||||
PaletteDataApplier paletteDataApplier,
|
||||
AdditionalTagConsumer additionalTag
|
||||
) throws IOException {
|
||||
boolean hasPalette = paletteAlreadyInitialized.getAsBoolean();
|
||||
byte type;
|
||||
String tag;
|
||||
while ((type = this.dataInputStream.readByte()) != NBTConstants.TYPE_END) {
|
||||
tag = readTagName();
|
||||
tag = this.dataInputStream.readUTF();
|
||||
if (tag.equals("Palette")) {
|
||||
if (hasPalette) {
|
||||
// Skip palette, as already exists
|
||||
this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_COMPOUND, 0);
|
||||
continue;
|
||||
}
|
||||
if (this.dataFixer == null) {
|
||||
this.remainingTags.add(rootTag);
|
||||
if (!this.readPaletteMap(this.dataInputStream, paletteInitializer)) {
|
||||
if (this.supportsReset()) {
|
||||
// Couldn't read - skip palette for now
|
||||
this.remainingTags.add(paletteType);
|
||||
this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_COMPOUND, 0);
|
||||
continue;
|
||||
}
|
||||
// Read all palette entries
|
||||
while (this.dataInputStream.readByte() != NBTConstants.TYPE_END) {
|
||||
String value = this.dataInputStream.readUTF();
|
||||
char index = (char) this.dataInputStream.readInt();
|
||||
paletteInitializer.accept(value, index);
|
||||
// Reset not possible, write into cache
|
||||
final NBTOutputStream cacheWriter = new NBTOutputStream(this.getPaletteCacheWriter());
|
||||
cacheWriter.write(paletteType);
|
||||
cacheWriter.writeTagPayload(this.nbtInputStream.readTagPayload(NBTConstants.TYPE_COMPOUND, 0));
|
||||
continue;
|
||||
}
|
||||
hasPalette = true;
|
||||
continue;
|
||||
}
|
||||
if (tag.equals("Data")) {
|
||||
// No palette or dimensions are yet available - will need to read Data next round
|
||||
// No palette or dimensions are yet available
|
||||
if (!hasPalette || this.dataFixer == null || !hasClipboard) {
|
||||
this.remainingTags.add(rootTag); // mark for read next iteration
|
||||
if (this.supportsReset()) {
|
||||
this.remainingTags.add(paletteType);
|
||||
this.nbtInputStream.readTagPayloadLazy(NBTConstants.TYPE_BYTE_ARRAY, 0);
|
||||
continue;
|
||||
}
|
||||
int length = this.dataInputStream.readInt();
|
||||
// Write data into clipboard
|
||||
int i = 0;
|
||||
if (needsVarIntReading(length)) {
|
||||
for (var iter = new VarIntStreamIterator(this.dataInputStream, length); iter.hasNext(); i++) {
|
||||
paletteDataApplier.accept(i, (char) iter.nextInt());
|
||||
}
|
||||
// Reset not possible, write into cache
|
||||
int byteLen = this.dataInputStream.readInt();
|
||||
final DataOutputStream cacheWriter = new DataOutputStream(this.getDataCacheWriter());
|
||||
cacheWriter.write(paletteType);
|
||||
cacheWriter.writeInt(byteLen);
|
||||
IOUtil.copy(this.dataInputStream, cacheWriter, byteLen);
|
||||
continue;
|
||||
}
|
||||
while (i < length) {
|
||||
paletteDataApplier.accept(i++, (char) this.dataInputStream.readUnsignedByte());
|
||||
}
|
||||
this.readPaletteData(this.dataInputStream, paletteDataApplier);
|
||||
continue;
|
||||
}
|
||||
additionalTag.accept(type, tag);
|
||||
}
|
||||
}
|
||||
|
||||
private String readTagName() throws IOException {
|
||||
return dataInputStream.readUTF();
|
||||
private void readPaletteData(DataInputStream stream, PaletteDataApplier applier) throws IOException {
|
||||
int length = stream.readInt();
|
||||
// Write data into clipboard
|
||||
int i = 0;
|
||||
if (needsVarIntReading(length)) {
|
||||
for (var iter = new VarIntStreamIterator(stream, length); iter.hasNext(); i++) {
|
||||
applier.apply(i, (char) iter.nextInt());
|
||||
}
|
||||
return;
|
||||
}
|
||||
while (i < length) {
|
||||
applier.apply(i++, (char) stream.readUnsignedByte());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the CompoundTag containing the palette mapping ({@code index: value}) and passes each entry to the
|
||||
* {@link PaletteInitializer}.
|
||||
* <p>
|
||||
* This method expects that the identifier ({@link NBTConstants#TYPE_COMPOUND}) is already consumed from the stream.
|
||||
*
|
||||
* @param stream The stream to read the data from.
|
||||
* @param initializer The initializer called for each entry with its index and backed value.
|
||||
* @return {@code true} if the mapping could be read, {@code false} otherwise (e.g. DataFixer is not yet available).
|
||||
* @throws IOException on I/O error.
|
||||
*/
|
||||
private boolean readPaletteMap(DataInputStream stream, PaletteInitializer initializer) throws IOException {
|
||||
if (this.dataFixer == null) {
|
||||
return false;
|
||||
}
|
||||
while (stream.readByte() != NBTConstants.TYPE_END) {
|
||||
String value = stream.readUTF();
|
||||
char index = (char) stream.readInt();
|
||||
initializer.initialize(index, value);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void indexToPosition(int index, PositionConsumer supplier) {
|
||||
int y = index / (dimensions.x() * dimensions.z());
|
||||
int remainder = index - (y * dimensions.x() * dimensions.z());
|
||||
int z = remainder / dimensions.x();
|
||||
int x = remainder - z * dimensions.x();
|
||||
supplier.accept(x, y, z);
|
||||
}
|
||||
|
||||
private PaletteDataApplier getBlockWriter(Clipboard target) {
|
||||
if (target instanceof LinearClipboard linearClipboard) {
|
||||
return (index, ordinal) -> linearClipboard.setBlock(index, this.blockPalette[ordinal]);
|
||||
}
|
||||
return (index, ordinal) -> indexToPosition(index, (x, y, z) -> target.setBlock(x, y, z, this.blockPalette[ordinal]));
|
||||
}
|
||||
|
||||
private PaletteDataApplier getBiomeWriter(Clipboard target) {
|
||||
if (target instanceof LinearClipboard linearClipboard) {
|
||||
return (index, ordinal) -> linearClipboard.setBiome(index, this.biomePalette[ordinal]);
|
||||
}
|
||||
return (index, ordinal) -> indexToPosition(index, (x, y, z) -> target.setBiome(x, y, z, this.biomePalette[ordinal]));
|
||||
}
|
||||
|
||||
private PaletteInitializer provideBlockPaletteInitializer() {
|
||||
return (index, value) -> {
|
||||
if (this.dataFixer == null) {
|
||||
throw new IllegalStateException("Can't read block palette map if DataFixer is not yet available");
|
||||
}
|
||||
value = dataFixer.fixUp(DataFixer.FixTypes.BLOCK_STATE, value);
|
||||
try {
|
||||
this.blockPalette[index] = BlockState.get(value);
|
||||
} catch (InputParseException e) {
|
||||
LOGGER.warn("Invalid BlockState in palette: {}. Block will be replaced with air.", value);
|
||||
this.blockPalette[index] = BlockTypes.AIR.getDefaultState();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private PaletteInitializer provideBiomePaletteInitializer() {
|
||||
return (index, value) -> {
|
||||
if (this.dataFixer == null) {
|
||||
throw new IllegalStateException("Can't read biome palette map if DataFixer is not yet available");
|
||||
}
|
||||
value = dataFixer.fixUp(DataFixer.FixTypes.BIOME, value);
|
||||
BiomeType biomeType = BiomeTypes.get(value);
|
||||
if (biomeType == null) {
|
||||
biomeType = BiomeTypes.PLAINS;
|
||||
LOGGER.warn("Invalid biome type in palette: {}. Biome will be replaced with plains.", value);
|
||||
}
|
||||
this.biomePalette[index] = biomeType;
|
||||
};
|
||||
}
|
||||
|
||||
private EntityTransformer provideEntityTransformer(Clipboard clipboard) {
|
||||
return (x, y, z, id, tag) -> {
|
||||
EntityType type = EntityType.REGISTRY.get(id);
|
||||
if (type == null) {
|
||||
LOGGER.warn("Invalid entity id: {} - skipping", id);
|
||||
return;
|
||||
}
|
||||
clipboard.createEntity(
|
||||
new Location(clipboard, Location.at(x, y, z).add(clipboard.getMinimumPoint().toVector3())),
|
||||
new BaseEntity(type, LazyReference.computed(tag))
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
private EntityTransformer provideTileEntityTransformer(Clipboard clipboard) {
|
||||
return (x, y, z, id, tag) -> clipboard.setTile(
|
||||
MathMan.roundInt(x + clipboard.getMinimumPoint().x()),
|
||||
MathMan.roundInt(y + clipboard.getMinimumPoint().y()),
|
||||
MathMan.roundInt(z + clipboard.getMinimumPoint().z()),
|
||||
new CompoundTag(tag)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@code true} if {@code Width}, {@code Length} and {@code Height} are already read from the stream
|
||||
*/
|
||||
private boolean areDimensionsAvailable() {
|
||||
return this.dimensions.x() != 0 && this.dimensions.y() != 0 && this.dimensions.z() != 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes this reader instance and all underlying resources.
|
||||
*
|
||||
* @throws IOException on I/O error.
|
||||
*/
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
resetableInputStream.close(); // closes all underlying resources implicitly
|
||||
parentStream.close(); // closes all underlying resources implicitly
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the main stream to the previously marked position ({@code 0}), if supported (see {@link #supportsReset()}).
|
||||
* If the stream is reset, the sub streams (for DataInput and NBT) are re-created to respect the new position.
|
||||
*
|
||||
* @throws IOException on I/O error.
|
||||
*/
|
||||
private void reset() throws IOException {
|
||||
if (this.supportsReset()) {
|
||||
this.parentStream.reset();
|
||||
this.parentStream.mark(Integer.MAX_VALUE);
|
||||
this.setSubStreams();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return {@code true} if the stream used while instantiating the reader supports resets (without memory overhead).
|
||||
*/
|
||||
private boolean supportsReset() {
|
||||
return this.remainingTags != null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Overwrites the DataInput- and NBT-InputStreams (e.g. when the marker of the backed stream updated).
|
||||
*
|
||||
* @throws IOException on I/O error.
|
||||
*/
|
||||
private void setSubStreams() throws IOException {
|
||||
final FastBufferedInputStream buffer = new FastBufferedInputStream(new GZIPInputStream(this.resetableInputStream));
|
||||
final FastBufferedInputStream buffer = new FastBufferedInputStream(new GZIPInputStream(this.parentStream));
|
||||
this.dataInputStream = new DataInputStream(buffer);
|
||||
this.nbtInputStream = new NBTInputStream(buffer);
|
||||
}
|
||||
|
||||
private void reset() throws IOException {
|
||||
this.resetableInputStream.reset();
|
||||
this.resetableInputStream.mark(Integer.MAX_VALUE);
|
||||
this.setSubStreams();
|
||||
/**
|
||||
* Creates a new cache writer for non-palette data, if none exists yet.
|
||||
* Returns either the already created or new one.
|
||||
*
|
||||
* @return the output stream for non-palette cache data.
|
||||
*/
|
||||
private OutputStream getDataCacheWriter() {
|
||||
if (this.dataCacheWriter == null) {
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(512);
|
||||
this.dataCacheWriter = new FastBufferedOutputStream(new LZ4BlockOutputStream(byteArrayOutputStream)) {
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
this.write(CACHE_IDENTIFIER_END);
|
||||
super.close();
|
||||
FastSchematicReaderV3.this.dataCache = byteArrayOutputStream.toByteArray();
|
||||
}
|
||||
};
|
||||
}
|
||||
return this.dataCacheWriter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new cache writer for palette data, if none exists yet.
|
||||
* Returns either the already created or new one.
|
||||
*
|
||||
* @return the output stream for palette cache data.
|
||||
*/
|
||||
private OutputStream getPaletteCacheWriter() {
|
||||
if (this.paletteCacheWriter == null) {
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(256);
|
||||
this.paletteCacheWriter = new FastBufferedOutputStream(new LZ4BlockOutputStream(byteArrayOutputStream)) {
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
this.write(CACHE_IDENTIFIER_END);
|
||||
super.close();
|
||||
FastSchematicReaderV3.this.paletteCache = byteArrayOutputStream.toByteArray();
|
||||
}
|
||||
};
|
||||
}
|
||||
return this.paletteCacheWriter;
|
||||
}
|
||||
|
||||
private boolean needsVarIntReading(int byteArrayLength) {
|
||||
@ -485,4 +740,78 @@ public class FastSchematicReaderV3 implements ClipboardReader {
|
||||
dataInputStream.skipNBytes(1 + 2 + 9); // as above + 9 bytes = "Schematic"
|
||||
}
|
||||
|
||||
@ApiStatus.Internal
|
||||
@FunctionalInterface
|
||||
private interface PositionConsumer {
|
||||
|
||||
/**
|
||||
* Called with block location coordinates.
|
||||
*
|
||||
* @param x the x coordinate.
|
||||
* @param y the y coordinate.
|
||||
* @param z the z coordinate.
|
||||
*/
|
||||
void accept(int x, int y, int z);
|
||||
|
||||
}
|
||||
|
||||
@ApiStatus.Internal
|
||||
@FunctionalInterface
|
||||
private interface EntityTransformer {
|
||||
|
||||
/**
|
||||
* Called for each entity from the Schematics {@code Entities} compound list.
|
||||
*
|
||||
* @param x the relative x coordinate of the entity.
|
||||
* @param y the relative y coordinate of the entity.
|
||||
* @param z the relative z coordinate of the entity.
|
||||
* @param id the entity id as a resource location (e.g. {@code minecraft:sheep}).
|
||||
* @param tag the - already fixed, if required - nbt data of the entity.
|
||||
*/
|
||||
void transform(double x, double y, double z, String id, CompoundBinaryTag tag);
|
||||
|
||||
}
|
||||
|
||||
@ApiStatus.Internal
|
||||
@FunctionalInterface
|
||||
private interface PaletteInitializer {
|
||||
|
||||
/**
|
||||
* Called for each palette entry (the mapping part, not data).
|
||||
*
|
||||
* @param index the index of the entry, as used in the Data byte array.
|
||||
* @param value the value for this entry (either biome type as resource location or the block state as a string).
|
||||
*/
|
||||
void initialize(char index, String value);
|
||||
|
||||
}
|
||||
|
||||
@ApiStatus.Internal
|
||||
@FunctionalInterface
|
||||
private interface PaletteDataApplier {
|
||||
|
||||
/**
|
||||
* Called for each palette data entry (not the mapping part, but the var-int byte array).
|
||||
*
|
||||
* @param index The index of this data entry (due to var-int behaviour not necessarily the index in the data byte array).
|
||||
* @param ordinal The ordinal of this entry as defined in the palette mapping.
|
||||
*/
|
||||
void apply(int index, char ordinal);
|
||||
|
||||
}
|
||||
|
||||
@ApiStatus.Internal
|
||||
@FunctionalInterface
|
||||
private interface AdditionalTagConsumer {
|
||||
|
||||
/**
|
||||
* Called for each unknown nbt tag.
|
||||
*
|
||||
* @param type The type of the tag (as defined by the constants in {@link NBTConstants}).
|
||||
* @param name The name of the tag.
|
||||
*/
|
||||
void accept(byte type, String name);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Laden…
In neuem Issue referenzieren
Einen Benutzer sperren