Paper/patches/removed/1.17/0334-Allow-Saving-of-Oversized-Chunks.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Fri, 15 Feb 2019 01:08:19 -0500
Subject: [PATCH] Allow Saving of Oversized Chunks

Note 1.17 update: With 1.17, Entities are no longer stored in chunk slices, so this needs updating!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

The Minecraft World Region File format has a hard cap of 1MB per chunk.
This is due to the fact that the header of the file format only allocates
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.

This limit can be reached fairly easily with books, resulting in the chunk being unable
to save to the world. Worse off, is that nothing printed when this occured, and silently
performed a chunk rollback on next load.

This leads to security risk with duplication and is being actively exploited.

This patch catches the too large scenario, falls back and moves any large Entity
or Tile Entity into a new compound, and this compound is saved into a different file.

On Chunk Load, we check for oversized status, and if so, we load the extra file and
merge the Entities and Tile Entities from the oversized chunk back into the level to
then be loaded as normal.

Once a chunk is returned back to normal size, the oversized flag will clear, and no
extra data file will exist.

This fix maintains compatability with all existing Anvil Region Format tools as it
does not alter the save format. They will just not know about the extra entities.

This fix also maintains compatability if someone switches server jars to one without
this fix, as the data will remain in the oversized file. Once the server returns
to a jar with this fix, the data will be restored.

diff --git a/src/main/java/net/minecraft/nbt/NbtIo.java b/src/main/java/net/minecraft/nbt/NbtIo.java
index c4fbc0bc8b80d16f09d3c6642acc7476a0817868..b3838e709c1581c25da7738c9a03a827761845b1 100644
--- a/src/main/java/net/minecraft/nbt/NbtIo.java
+++ b/src/main/java/net/minecraft/nbt/NbtIo.java
@@ -132,6 +132,7 @@ public class NbtIo {
 
     }
 
+    public static CompoundTag readNBT(DataInput datainput) throws IOException { return read(datainput); } // Paper - OBFHELPER
     public static CompoundTag read(DataInput input) throws IOException {
         return read(input, NbtAccounter.UNLIMITED);
     }
@@ -152,6 +153,7 @@ public class NbtIo {
         }
     }
 
+    public static void writeNBT(CompoundTag nbttagcompound, DataOutput dataoutput) throws IOException { write(nbttagcompound, dataoutput); } // Paper - OBFHELPER
     public static void write(CompoundTag tag, DataOutput output) throws IOException {
         writeUnnamedTag((Tag) tag, output);
     }
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java
index 1aa4d342b97f8be71c108194a6f1e0e2828aa364..424628c9588c02454558bc7e7c5bad3a3e75ec9f 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java
@@ -20,8 +20,12 @@ import java.nio.file.LinkOption;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
+import java.util.zip.InflaterInputStream; // Paper
+
 import javax.annotation.Nullable;
 import net.minecraft.Util;
+import net.minecraft.nbt.CompoundTag;
+import net.minecraft.nbt.NbtIo;
 import net.minecraft.world.level.ChunkPos;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -38,6 +42,7 @@ public class RegionFile implements AutoCloseable {
     private final IntBuffer timestamps;
     @VisibleForTesting
     protected final RegionBitmap usedSectors;
+    public final File file; // Paper
 
     public RegionFile(File file, File directory, boolean dsync) throws IOException {
         this(file.toPath(), directory.toPath(), RegionFileVersion.VERSION_DEFLATE, dsync);
@@ -45,6 +50,8 @@ public class RegionFile implements AutoCloseable {
 
     public RegionFile(Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync) throws IOException {
         this.header = ByteBuffer.allocateDirect(8192);
+        this.file = file.toFile(); // Paper
+        initOversizedState(); // Paper
         this.usedSectors = new RegionBitmap();
         this.version = outputChunkStreamVersion;
         if (!Files.isDirectory(directory, new LinkOption[0])) {
@@ -408,6 +415,74 @@ public class RegionFile implements AutoCloseable {
         void run() throws IOException;
     }
 
+    // Paper start
+    private final byte[] oversized = new byte[1024];
+    private int oversizedCount = 0;
+
+    private synchronized void initOversizedState() throws IOException {
+        File metaFile = getOversizedMetaFile();
+        if (metaFile.exists()) {
+            final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath());
+            System.arraycopy(read, 0, oversized, 0, oversized.length);
+            for (byte temp : oversized) {
+                oversizedCount += temp;
+            }
+        }
+    }
+
+    private static int getChunkIndex(int x, int z) {
+        return (x & 31) + (z & 31) * 32;
+    }
+    synchronized boolean isOversized(int x, int z) {
+        return this.oversized[getChunkIndex(x, z)] == 1;
+    }
+    synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
+        final int offset = getChunkIndex(x, z);
+        boolean previous = this.oversized[offset] == 1;
+        this.oversized[offset] = (byte) (oversized ? 1 : 0);
+        if (!previous && oversized) {
+            oversizedCount++;
+        } else if (!oversized && previous) {
+            oversizedCount--;
+        }
+        if (previous && !oversized) {
+            File oversizedFile = getOversizedFile(x, z);
+            if (oversizedFile.exists()) {
+                oversizedFile.delete();
+            }
+        }
+        if (oversizedCount > 0) {
+            if (previous != oversized) {
+                writeOversizedMeta();
+            }
+        } else if (previous) {
+            File oversizedMetaFile = getOversizedMetaFile();
+            if (oversizedMetaFile.exists()) {
+                oversizedMetaFile.delete();
+            }
+        }
+    }
+
+    private void writeOversizedMeta() throws IOException {
+        java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized);
+    }
+
+    private File getOversizedMetaFile() {
+        return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + ".oversized.nbt");
+    }
+
+    private File getOversizedFile(int x, int z) {
+        return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
+    }
+
+    synchronized CompoundTag getOversizedData(int x, int z) throws IOException {
+        File file = getOversizedFile(x, z);
+        try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) {
+            return NbtIo.readNBT((java.io.DataInput) out);
+        }
+
+    }
+    // Paper end
     class ChunkBuffer extends ByteArrayOutputStream {
 
         private final ChunkPos pos;
diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
index f33a5fc725d1d5e895f8878d82ebc4172237ad29..6d3e1bb20d1ab8ce5c9ea613322042d80550761a 100644
--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
+++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java
@@ -10,7 +10,9 @@ import java.io.File;
 import java.io.IOException;
 import javax.annotation.Nullable;
 import net.minecraft.nbt.CompoundTag;
+import net.minecraft.nbt.ListTag;
 import net.minecraft.nbt.NbtIo;
+import net.minecraft.nbt.Tag;
 import net.minecraft.server.MinecraftServer;
 import net.minecraft.util.ExceptionCollector;
 import net.minecraft.world.level.ChunkPos;
@@ -50,6 +52,74 @@ public final class RegionFileStorage implements AutoCloseable {
         }
     }
 
+    // Paper start
+    private static void printOversizedLog(String msg, File file, int x, int z) {
+        org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
+    }
+
+    private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8;
+    private static final int OVERZEALOUS_TOTAL_THRESHOLD = 1024 * 64;
+    private static final int OVERZEALOUS_THRESHOLD = 1024;
+    private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
+    private static void resetFilterThresholds() {
+        SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD));
+    }
+    static {
+        resetFilterThresholds();
+    }
+
+    static boolean isOverzealous() {
+        return SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD;
+    }
+
+
+    private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException {
+        synchronized (regionfile) {
+            try (DataInputStream datainputstream = regionfile.getReadStream(chunkCoordinate)) {
+                CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
+                CompoundTag chunk = NbtIo.readNBT((DataInput) datainputstream);
+                if (oversizedData == null) {
+                    return chunk;
+                }
+                CompoundTag oversizedLevel = oversizedData.getCompound("Level");
+                CompoundTag level = chunk.getCompound("Level");
+
+                mergeChunkList(level, oversizedLevel, "Entities");
+                mergeChunkList(level, oversizedLevel, "TileEntities");
+
+                chunk.put("Level", level);
+
+                return chunk;
+            } catch (Throwable throwable) {
+                throwable.printStackTrace();
+                throw throwable;
+            }
+        }
+    }
+
+    private static void mergeChunkList(CompoundTag level, CompoundTag oversizedLevel, String key) {
+        ListTag levelList = level.getList(key, 10);
+        ListTag oversizedList = oversizedLevel.getList(key, 10);
+
+        if (!oversizedList.isEmpty()) {
+            levelList.addAll(oversizedList);
+            level.put(key, levelList);
+        }
+    }
+
+    private static int getNBTSize(Tag nbtBase) {
+        DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream());
+        try {
+            nbtBase.write(test);
+            return test.size();
+        } catch (IOException e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+    // Paper End
+
     @Nullable
     public CompoundTag read(ChunkPos pos) throws IOException {
         // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing
@@ -59,6 +129,12 @@ public final class RegionFileStorage implements AutoCloseable {
         }
         // CraftBukkit end
         DataInputStream datainputstream = regionfile.getChunkDataInputStream(pos);
+        // Paper start
+        if (regionfile.isOversized(pos.x, pos.z)) {
+            printOversizedLog("Loading Oversized Chunk!", regionfile.file, pos.x, pos.z);
+            return readOversizedChunk(regionfile, pos);
+        }
+        // Paper end
         Throwable throwable = null;
 
         CompoundTag nbttagcompound;
@@ -99,6 +175,7 @@ public final class RegionFileStorage implements AutoCloseable {
 
         try {
             NbtIo.write(tag, (DataOutput) dataoutputstream);
+            regionfile.setOversized(pos.x, pos.z, false); // Paper - We don't do this anymore, mojang stores differently, but clear old meta flag if it exists to get rid of our own meta file once last oversized is gone
         } catch (Throwable throwable1) {
             throwable = throwable1;
             throw throwable1;