Implement new chunk transformer

src/main/java/us/myles/ViaVersion/chunks/Chunk.java

@@ -0,0 +1,32 @@
+package us.myles.ViaVersion.chunks;
+
+import lombok.AccessLevel;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor(access = AccessLevel.PROTECTED)
+@Getter
+public class Chunk {
+    private final int x;
+    private final int z;
+    private final boolean groundUp;
+    private final int primaryBitmask;
+    private final ChunkSection[] sections;
+    private final byte[] biomeData;
+    private boolean unloadPacket = false;
+
+    /**
+     * Chunk unload.
+     *
+     * @param x coord
+     * @param z coord
+     */
+    protected Chunk(int x, int z) {
+        this(x, z, true, 0, new ChunkSection[16], null);
+        this.unloadPacket = true;
+    }
+
+    public boolean hasBiomeData() {
+        return biomeData != null && groundUp;
+    }
+}

src/main/java/us/myles/ViaVersion/chunks/ChunkManager.java

@@ -0,0 +1,210 @@
+package us.myles.ViaVersion.chunks;
+
+import com.google.common.collect.Sets;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.bukkit.Bukkit;
+import us.myles.ViaVersion.ConnectionInfo;
+import us.myles.ViaVersion.util.PacketUtil;
+import us.myles.ViaVersion.util.ReflectionUtil;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.ShortBuffer;
+import java.util.BitSet;
+import java.util.Set;
+import java.util.logging.Level;
+
+public class ChunkManager {
+    /**
+     * Amount of sections in a chunk.
+     */
+    private static final int SECTION_COUNT = 16;
+    /**
+     * size of each chunk section (16x16x16).
+     */
+    private static final int SECTION_SIZE = 16;
+    /**
+     * Length of biome data.
+     */
+    private static final int BIOME_DATA_LENGTH = 256;
+
+    private final ConnectionInfo info;
+    private final Set<Long> loadedChunks = Sets.newConcurrentHashSet();
+    private Method getWorldHandle;
+    private Method getChunkAt;
+    private Field getSections;
+
+    public ChunkManager(ConnectionInfo info) {
+        this.info = info;
+
+        try {
+            this.getWorldHandle = ReflectionUtil.obc("CraftWorld").getDeclaredMethod("getHandle");
+            this.getChunkAt = ReflectionUtil.nms("World").getDeclaredMethod("getChunkAt", int.class, int.class);
+            this.getSections = ReflectionUtil.nms("Chunk").getDeclaredField("sections");
+            getSections.setAccessible(true);
+        } catch(Exception e) {
+            Bukkit.getLogger().log(Level.WARNING, "Failed to initialise chunk verification", e);
+        }
+    }
+
+    /**
+     * Read chunk from 1.8 chunk data.
+     *
+     * @param input data
+     * @return Chunk
+     */
+    public Chunk readChunk(ByteBuf input) {
+        // Primary data
+        int chunkX = input.readInt();
+        int chunkZ = input.readInt();
+        long chunkHash = toLong(chunkX, chunkZ);
+        boolean groundUp = input.readByte() != 0;
+        int bitmask = input.readUnsignedShort();
+        int dataLength = PacketUtil.readVarInt(input);
+
+        // Data to be read
+        BitSet usedSections = new BitSet(16);
+        ChunkSection[] sections = new ChunkSection[16];
+        byte[] biomeData = null;
+
+        // Calculate section count from bitmask
+        for(int i = 0; i < 16; i++) {
+            if((bitmask & (1 << i)) != 0) {
+                usedSections.set(i);
+            }
+        }
+
+        // Unloading & empty chunks
+        int sectionCount = usedSections.cardinality(); // the amount of sections set
+        if(sectionCount == 0 && groundUp) {
+            if(loadedChunks.contains(chunkHash)) {
+                // This is a chunk unload packet
+                loadedChunks.remove(chunkHash);
+                return new Chunk(chunkX, chunkZ);
+            } else {
+                // Check if chunk data is invalid
+                try {
+                    Object nmsWorld = getWorldHandle.invoke(info.getPlayer().getWorld());
+                    Object nmsChunk = getChunkAt.invoke(info.getPlayer().getWorld());
+                    Object[] nmsSections = (Object[]) getSections.get(nmsChunk);
+
+                    // Check if chunk is actually empty
+                    boolean isEmpty = false;
+                    int i = 0;
+                    while(i < nmsSections.length) {
+                        if(!(isEmpty = nmsSections[i++] == null)) break;
+                    }
+                    if(isEmpty) {
+                        // not empty, LOL
+                        return null;
+                    }
+                } catch(Exception e) {
+                    Bukkit.getLogger().log(Level.WARNING, "Failed to verify chunk", e);
+                }
+            }
+        }
+
+        int startIndex = input.readerIndex();
+        loadedChunks.add(chunkHash); // mark chunk as loaded
+
+        // Read blocks
+        for(int i = 0; i < SECTION_COUNT; i++) {
+            if(!usedSections.get(i)) continue; // Section not set
+            ChunkSection section = new ChunkSection();
+            sections[i] = section;
+
+            // Read block data and convert to short buffer
+            byte[] blockData = new byte[ChunkSection.SIZE * 2];
+            input.readBytes(blockData);
+            ShortBuffer blockBuf = ByteBuffer.wrap(blockData).order(ByteOrder.LITTLE_ENDIAN).asShortBuffer();
+
+            for(int j = 0; j < ChunkSection.SIZE; j++) {
+                int mask = blockBuf.get();
+                int type = mask >> 4;
+                int data = mask & 0xF;
+                section.setBlock(j, type, data);
+            }
+        }
+
+        // Read block light
+        for(int i = 0; i < SECTION_COUNT; i++) {
+            if(!usedSections.get(i)) continue; // Section not set, has no light
+            byte[] blockLightArray = new byte[ChunkSection.LIGHT_LENGTH];
+            input.readBytes(blockLightArray);
+            sections[i].setBlockLight(blockLightArray);
+        }
+
+        // Read sky light
+        int bytesLeft = dataLength - (input.readerIndex() - startIndex);
+        if(bytesLeft >= ChunkSection.LIGHT_LENGTH) {
+            for(int i = 0; i < SECTION_COUNT; i++) {
+                if(!usedSections.get(i)) continue; // Section not set, has no light
+                byte[] skyLightArray = new byte[ChunkSection.LIGHT_LENGTH];
+                input.readBytes(skyLightArray);
+                sections[i].setSkyLight(skyLightArray);
+                bytesLeft -= ChunkSection.LIGHT_LENGTH;
+            }
+        }
+
+        // Read biome data
+        if(bytesLeft >= BIOME_DATA_LENGTH) {
+            biomeData = new byte[BIOME_DATA_LENGTH];
+            input.readBytes(biomeData);
+            bytesLeft -= BIOME_DATA_LENGTH;
+        }
+
+        // Check remaining bytes
+        if(bytesLeft > 0) {
+            Bukkit.getLogger().log(Level.WARNING, bytesLeft + " Bytes left after reading chunk! (" + groundUp + ")");
+        }
+
+        // Return chunk
+        return new Chunk(chunkX, chunkZ, groundUp, bitmask, sections, biomeData);
+    }
+
+    /**
+     * Write chunk over 1.9 protocol.
+     *
+     * @param chunk chunk
+     * @param output output
+     */
+    public void writeChunk(Chunk chunk, ByteBuf output) {
+        if(chunk.isUnloadPacket()) {
+            output.clear();
+            PacketUtil.writeVarInt(0x1D, output);
+        }
+
+        // Write primary info
+        output.writeInt(chunk.getX());
+        output.writeInt(chunk.getZ());
+        if(chunk.isUnloadPacket()) return;
+        output.writeByte(chunk.isGroundUp() ? 0x01 : 0x00);
+        PacketUtil.writeVarInt(chunk.getPrimaryBitmask(), output);
+
+        ByteBuf buf = Unpooled.buffer();
+        for(int i = 0; i < SECTION_COUNT; i++) {
+            ChunkSection section = chunk.getSections()[i];
+            if(section == null) continue; // Section not set
+            section.writeBlocks(buf);
+            section.writeBlockLight(buf);
+            if(!section.hasSkyLight()) continue; // No sky light, we're done here.
+            section.writeSkyLight(buf);
+        }
+        buf.readerIndex(0);
+        PacketUtil.writeVarInt(buf.readableBytes() + (chunk.hasBiomeData() ? 256 : 0), output);
+        output.writeBytes(buf);
+        buf.release(); // release buffer
+
+        // Write biome data
+        if(chunk.hasBiomeData()) {
+            output.writeBytes(chunk.getBiomeData());
+        }
+    }
+
+    private static long toLong(int msw, int lsw)  {
+        return ((long) msw << 32) + lsw - -2147483648L;
+    }
+}

src/main/java/us/myles/ViaVersion/chunks/ChunkSection.java

@@ -0,0 +1,143 @@
+package us.myles.ViaVersion.chunks;
+
+import com.google.common.collect.Lists;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.Unpooled;
+import org.bukkit.Material;
+import us.myles.ViaVersion.util.PacketUtil;
+
+import java.util.List;
+
+public class ChunkSection {
+    /**
+     * Size (dimensions) of blocks in a chunk section.
+     */
+    public static final int SIZE = 16 * 16 * 16; // width * depth * height
+    /**
+     * Length of the sky and block light nibble arrays.
+     */
+    public static final int LIGHT_LENGTH = 16 * 16 * 16 / 2; // size * size * size / 2 (nibble bit count)
+    /**
+     * Length of the block data array.
+     */
+//    public static final int BLOCK_LENGTH = 16 * 16 * 16 * 2; // size * size * size * 2 (char bit count)
+
+    private final List<Integer> palette = Lists.newArrayList();
+    private final int[] blocks;
+    private final NibbleArray blockLight;
+    private NibbleArray skyLight;
+
+    public ChunkSection() {
+        this.blocks = new int[SIZE];
+        this.blockLight = new NibbleArray(SIZE);
+        palette.add(0); // AIR
+    }
+
+    public void setBlock(int x, int y, int z, int type, int data) {
+        setBlock(index(x, y, z), type, data);
+    }
+
+    public void setBlock(int idx, int type, int data) {
+        int hash = type << 4 | (data & 0xF);
+        int index = palette.indexOf(hash);
+        if(index == -1) {
+            index = palette.size();
+            palette.add(hash);
+        }
+
+        blocks[idx] = index;
+    }
+
+    public void setBlockLight(byte[] data) {
+        blockLight.setHandle(data);
+    }
+
+    public void setSkyLight(byte[] data) {
+        if(data.length != LIGHT_LENGTH) throw new IllegalArgumentException("Data length != " + LIGHT_LENGTH);
+        this.skyLight = new NibbleArray(data);
+    }
+
+    private int index(int x, int y, int z) {
+        return z << 8 | y << 4 | x;
+    }
+
+    public void writeBlocks(ByteBuf output) {
+        // Write bits per block
+        int bitsPerBlock = 4;
+        while(palette.size() > 1 << bitsPerBlock) {
+            bitsPerBlock += 1;
+        }
+        long maxEntryValue = (1L << bitsPerBlock) - 1;
+        output.writeByte(bitsPerBlock);
+
+        // Write pallet (or not)
+        PacketUtil.writeVarInt(palette.size(), output);
+        for(int mappedId : palette) {
+            PacketUtil.writeVarInt(mappedId, output);
+        }
+
+        int length = (int) Math.ceil(SIZE * bitsPerBlock / 64.0);
+        PacketUtil.writeVarInt(length, output);
+        long[] data = new long[length];
+        for(int index = 0; index < blocks.length; index++) {
+            int value = blocks[index];
+            int bitIndex = index * bitsPerBlock;
+            int startIndex = bitIndex / 64;
+            int endIndex = ((index + 1) * bitsPerBlock - 1) / 64;
+            int startBitSubIndex = bitIndex % 64;
+            data[startIndex] = data[startIndex] & ~(maxEntryValue << startBitSubIndex) | ((long) value & maxEntryValue) << startBitSubIndex;
+            if(startIndex != endIndex) {
+                int endBitSubIndex = 64 - startBitSubIndex;
+                data[endIndex] = data[endIndex] >>> endBitSubIndex << endBitSubIndex | ((long) value & maxEntryValue) >> endBitSubIndex;
+            }
+        }
+        PacketUtil.writeLongs(data, output);
+    }
+
+    public void writeBlockLight(ByteBuf output) {
+        output.writeBytes(blockLight.getHandle());
+    }
+
+    public void writeSkyLight(ByteBuf output) {
+        output.writeBytes(skyLight.getHandle());
+    }
+
+    public boolean hasSkyLight() {
+        return skyLight != null;
+    }
+
+    /**
+     * Get expected size of this chunk section.
+     *
+     * @return Amount of bytes sent by this section
+     */
+    public int getExpectedSize() {
+        int bitsPerBlock = palette.size() > 255 ? 16 : 8;
+        int bytes = 1; // bits per block
+        bytes += paletteBytes(); // palette
+        bytes += countBytes(bitsPerBlock == 16 ? SIZE * 2 : SIZE); // block data length
+        bytes += (palette.size() > 255 ? 2 : 1) * SIZE; // block data
+        bytes += LIGHT_LENGTH; // block light
+        bytes += hasSkyLight() ? LIGHT_LENGTH : 0; // sky light
+        return bytes;
+    }
+
+    private int paletteBytes() {
+        // Count bytes used by pallet
+        int bytes = countBytes(palette.size());
+        for(int mappedId : palette) {
+            bytes += countBytes(mappedId);
+        }
+        return bytes;
+    }
+
+    private int countBytes(int value) {
+        // Count amount of bytes that would be sent if the value were sent as a VarInt
+        ByteBuf buf = Unpooled.buffer();
+        PacketUtil.writeVarInt(value, buf);
+        buf.readerIndex(0);
+        int bitCount = buf.readableBytes();
+        buf.release();
+        return bitCount;
+    }
+}

src/main/java/us/myles/ViaVersion/chunks/NibbleArray.java

@@ -0,0 +1,74 @@
+package us.myles.ViaVersion.chunks;
+
+import java.util.Arrays;
+
+public class NibbleArray {
+    private final byte[] handle;
+
+    public NibbleArray(int length) {
+        if(length == 0 || length % 2 != 0) {
+            throw new IllegalArgumentException("Length of nibble array must be a positive number dividable by 2!");
+        }
+
+        this.handle = new byte[length / 2];
+    }
+
+    public NibbleArray(byte[] handle) {
+        if(handle.length == 0 || handle.length % 2 != 0) {
+            throw new IllegalArgumentException("Length of nibble array must be a positive number dividable by 2!");
+        }
+
+        this.handle = handle;
+    }
+
+    public byte get(int x, int y, int z) {
+        return get(y << 8 | z << 4 | x);
+    }
+
+    public byte get(int index) {
+        byte value = handle[index / 2];
+        if(index % 2 == 0) {
+            return (byte) (value & 0xF);
+        } else {
+            return (byte) ((value >> 4) & 0xF);
+        }
+    }
+
+    public void set(int x, int y, int z, int value) {
+        set(y << 8 | z << 4 | x, value);
+    }
+
+    public void set(int index, int value) {
+        index /= 2;
+        if(index % 2 == 0) {
+            handle[index] = (byte) (handle[index] & 0xF0 | value & 0xF);
+        } else {
+            handle[index] = (byte) (handle[index] & 0xF | (value & 0xF) << 4);
+        }
+    }
+
+    public int size() {
+        return handle.length * 2;
+    }
+
+    public int actualSize() {
+        return handle.length;
+    }
+
+    public void fill(byte value) {
+        value &= 0xF; // Max nibble size (= 16)
+        Arrays.fill(handle, (byte) ((value << 4) | value));
+    }
+
+    public void setHandle(byte[] handle) {
+        if(handle.length != this.handle.length) {
+            throw new IllegalArgumentException("Length of handle must equal to size of nibble array!");
+        }
+
+        System.arraycopy(handle, 0, this.handle, 0, handle.length);
+    }
+
+    public byte[] getHandle() {
+        return handle;
+    }
+}

src/main/java/us/myles/ViaVersion/transformers/OutgoingTransformer.java

@@ -18,6 +18,8 @@ import us.myles.ViaVersion.api.ViaVersion;
 import us.myles.ViaVersion.api.boss.BossBar;
 import us.myles.ViaVersion.api.boss.BossColor;
 import us.myles.ViaVersion.api.boss.BossStyle;
+import us.myles.ViaVersion.chunks.Chunk;
+import us.myles.ViaVersion.chunks.ChunkManager;
 import us.myles.ViaVersion.metadata.MetaIndex;
 import us.myles.ViaVersion.metadata.MetadataRewriter;
 import us.myles.ViaVersion.metadata.MetadataRewriter.Entry;
@@ -40,6 +42,7 @@ public class OutgoingTransformer {
     private final ViaVersionPlugin plugin = (ViaVersionPlugin) ViaVersion.getInstance();
 
     private final ConnectionInfo info;
+    private final ChunkManager chunkManager;
     private final Map<Integer, UUID> uuidMap = new HashMap<>();
     private final Map<Integer, EntityType> clientEntityTypes = new HashMap<>();
     private final Map<Integer, Integer> vehicleMap = new HashMap<>();
@@ -51,6 +54,7 @@ public class OutgoingTransformer {
 
     public OutgoingTransformer(ConnectionInfo info) {
         this.info = info;
+        this.chunkManager = new ChunkManager(info);
     }
 
     public static String fixJson(String line) {
@@ -776,54 +780,14 @@ public class OutgoingTransformer {
             return;
         }
         if (packet == PacketType.PLAY_CHUNK_DATA) {
-            // We need to catch unloading chunk packets as defined by wiki.vg
+            // Read chunk
-            // To unload chunks, send this packet with Ground-Up Continuous=true and no 16^3 chunks (eg. Primary Bit Mask=0)
+            Chunk chunk = chunkManager.readChunk(input);
-            int chunkX = input.readInt();
+            if(chunk == null) {
-            int chunkZ = input.readInt();
-            output.writeInt(chunkX);
-            output.writeInt(chunkZ);
-
-
-            boolean groundUp = input.readBoolean();
-            output.writeBoolean(groundUp);
-
-            int bitMask = input.readUnsignedShort();
-            int size = PacketUtil.readVarInt(input);
-            byte[] data = new byte[size];
-            input.readBytes(data);
-//            if (bitMask == 0 && groundUp) {
-//                // if 256
-//                output.clear();
-//                PacketUtil.writeVarInt(PacketType.PLAY_UNLOAD_CHUNK.getNewPacketID(), output);
-//                output.writeInt(chunkX);
-//                output.writeInt(chunkZ);
-//                System.out.println("Sending unload chunk " + chunkX + " " + chunkZ + " - " + size + " bulk: " + bulk);
-//                return;
-//            }
-            boolean sk = false;
-            if (info.getLastPacket().getClass().getName().endsWith("PacketPlayOutMapChunkBulk")) {
-                try {
-                    sk = ReflectionUtil.get(info.getLastPacket(), "d", boolean.class);
-                } catch (NoSuchFieldException e) {
-                    e.printStackTrace();
-                } catch (IllegalAccessException e) {
-                    e.printStackTrace();
-                }
-            }
-            Column read = NetUtil.readOldChunkData(chunkX, chunkZ, groundUp, bitMask, data, true, sk);
-            if (read == null) {
                 throw new CancelException();
             }
-            // Write chunk section array :((
+
-            ByteBuf temp = output.alloc().buffer();
+            // Write chunk
-            try {
+            chunkManager.writeChunk(chunk, output);
-                int bitmask = NetUtil.writeNewColumn(temp, read, groundUp, sk);
-                PacketUtil.writeVarInt(bitmask, output);
-                PacketUtil.writeVarInt(temp.readableBytes(), output);
-                output.writeBytes(temp);
-            } catch (IOException e) {
-                e.printStackTrace();
-            }
             return;
         }
         output.writeBytes(input);