From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From: Spottedleaf Date: Thu, 20 May 2021 07:02:22 -0700 Subject: [PATCH] Fix and optimise world force upgrading The WorldUpgrader class was incorrectly modified by CB. It will store an IChunkLoader instance for all dimension types in the world, but obviously with how CB shifts around worlds only one dimension type exists per world. But this would be OK if CB did this change correctly. All IChunkLoader instances will point to the same regionfiles. And all IChunkLoader instances are going to be read from. This problem hasn't really been reported because it relies on the persistent legacy data to be converted as well to cause corruption. Why? Because the legacy data is also shared, it will result in different outputs from conversion (as once conversion for legacy persistent data takes place, it is REMOVED - so the next convert will _not_ have the data). Which means different sizes on disk. Which means different regionfile sector allocations. Which means there are 3 different possible regionfile sector allocations in memory, and none of them are going to be correct. I've fixed this by writing a world upgrader suited to CB's changes to world folder format. It was brain dead easy to add threading, so I did. diff --git a/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java new file mode 100644 index 0000000000000000000000000000000000000000..95cac7edae8ac64811fc6a2f6b97dd4a0fceb0b0 --- /dev/null +++ b/src/main/java/io/papermc/paper/world/ThreadedWorldUpgrader.java @@ -0,0 +1,209 @@ +package io.papermc.paper.world; + +import com.mojang.datafixers.DataFixer; +import com.mojang.serialization.Codec; +import net.minecraft.SharedConstants; +import net.minecraft.nbt.CompoundTag; +import net.minecraft.resources.ResourceKey; +import net.minecraft.util.worldupdate.WorldUpgrader; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.Level; +import net.minecraft.world.level.chunk.ChunkGenerator; +import net.minecraft.world.level.chunk.storage.ChunkStorage; +import net.minecraft.world.level.chunk.storage.RegionFileStorage; +import net.minecraft.world.level.dimension.DimensionType; +import net.minecraft.world.level.dimension.LevelStem; +import net.minecraft.world.level.levelgen.WorldGenSettings; +import net.minecraft.world.level.storage.DimensionDataStorage; +import net.minecraft.world.level.storage.LevelStorageSource; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.File; +import java.io.IOException; +import java.text.DecimalFormat; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; + +public class ThreadedWorldUpgrader { + + private static final Logger LOGGER = LogManager.getLogger(); + + private final ResourceKey dimensionType; + private final String worldName; + private final File worldDir; + private final ExecutorService threadPool; + private final DataFixer dataFixer; + private final Optional>> generatorKey; + private final boolean removeCaches; + + public ThreadedWorldUpgrader(final ResourceKey dimensionType, final String worldName, final File worldDir, final int threads, + final DataFixer dataFixer, final Optional>> generatorKey, final boolean removeCaches) { + this.dimensionType = dimensionType; + this.worldName = worldName; + this.worldDir = worldDir; + this.threadPool = Executors.newFixedThreadPool(Math.max(1, threads), new ThreadFactory() { + private final AtomicInteger threadCounter = new AtomicInteger(); + + @Override + public Thread newThread(final Runnable run) { + final Thread ret = new Thread(run); + + ret.setName("World upgrader thread for world " + ThreadedWorldUpgrader.this.worldName + " #" + this.threadCounter.getAndIncrement()); + ret.setUncaughtExceptionHandler((thread, throwable) -> { + LOGGER.fatal("Error upgrading world", throwable); + }); + + return ret; + } + }); + this.dataFixer = dataFixer; + this.generatorKey = generatorKey; + this.removeCaches = removeCaches; + } + + public void convert() { + final File worldFolder = LevelStorageSource.getStorageFolder(this.worldDir.toPath(), this.dimensionType).toFile(); + final DimensionDataStorage worldPersistentData = new DimensionDataStorage(new File(worldFolder, "data"), this.dataFixer); + + final File regionFolder = new File(worldFolder, "region"); + + LOGGER.info("Force upgrading " + this.worldName); + LOGGER.info("Counting regionfiles for " + this.worldName); + final File[] regionFiles = regionFolder.listFiles((final File dir, final String name) -> { + return WorldUpgrader.REGEX.matcher(name).matches(); + }); + if (regionFiles == null) { + LOGGER.info("Found no regionfiles to convert for world " + this.worldName); + return; + } + LOGGER.info("Found " + regionFiles.length + " regionfiles to convert"); + LOGGER.info("Starting conversion now for world " + this.worldName); + + final WorldInfo info = new WorldInfo(() -> worldPersistentData, + new ChunkStorage(regionFolder.toPath(), this.dataFixer, false), this.removeCaches, this.dimensionType, this.generatorKey); + + long expectedChunks = (long)regionFiles.length * (32L * 32L); + + for (final File regionFile : regionFiles) { + final ChunkPos regionPos = RegionFileStorage.getRegionFileCoordinates(regionFile.toPath()); + if (regionPos == null) { + expectedChunks -= (32L * 32L); + continue; + } + + this.threadPool.execute(new ConvertTask(info, regionPos.x >> 5, regionPos.z >> 5)); + } + this.threadPool.shutdown(); + + final DecimalFormat format = new DecimalFormat("#0.00"); + + final long start = System.nanoTime(); + + while (!this.threadPool.isTerminated()) { + final long current = info.convertedChunks.get(); + + LOGGER.info("{}% completed ({} / {} chunks)...", format.format((double)current / (double)expectedChunks * 100.0), current, expectedChunks); + + try { + Thread.sleep(1000L); + } catch (final InterruptedException ignore) {} + } + + final long end = System.nanoTime(); + + try { + info.loader.close(); + } catch (final IOException ex) { + LOGGER.fatal("Failed to close chunk loader", ex); + } + LOGGER.info("Completed conversion. Took {}s, {} out of {} chunks needed to be converted/modified ({}%)", + (int)Math.ceil((end - start) * 1.0e-9), info.modifiedChunks.get(), expectedChunks, format.format((double)info.modifiedChunks.get() / (double)expectedChunks * 100.0)); + } + + private static final class WorldInfo { + + public final Supplier persistentDataSupplier; + public final ChunkStorage loader; + public final boolean removeCaches; + public final ResourceKey worldKey; + public final Optional>> generatorKey; + public final AtomicLong convertedChunks = new AtomicLong(); + public final AtomicLong modifiedChunks = new AtomicLong(); + + private WorldInfo(final Supplier persistentDataSupplier, final ChunkStorage loader, final boolean removeCaches, + final ResourceKey worldKey, Optional>> generatorKey) { + this.persistentDataSupplier = persistentDataSupplier; + this.loader = loader; + this.removeCaches = removeCaches; + this.worldKey = worldKey; + this.generatorKey = generatorKey; + } + } + + private static final class ConvertTask implements Runnable { + + private final WorldInfo worldInfo; + private final int regionX; + private final int regionZ; + + public ConvertTask(final WorldInfo worldInfo, final int regionX, final int regionZ) { + this.worldInfo = worldInfo; + this.regionX = regionX; + this.regionZ = regionZ; + } + + @Override + public void run() { + final int regionCX = this.regionX << 5; + final int regionCZ = this.regionZ << 5; + + final Supplier persistentDataSupplier = this.worldInfo.persistentDataSupplier; + final ChunkStorage loader = this.worldInfo.loader; + final boolean removeCaches = this.worldInfo.removeCaches; + final ResourceKey worldKey = this.worldInfo.worldKey; + + for (int cz = regionCZ; cz < (regionCZ + 32); ++cz) { + for (int cx = regionCX; cx < (regionCX + 32); ++cx) { + final ChunkPos chunkPos = new ChunkPos(cx, cz); + try { + // no need to check the coordinate of the chunk, the regionfilecache does that for us + + CompoundTag chunkNBT = (loader.read(chunkPos).join()).orElse(null); + + if (chunkNBT == null) { + continue; + } + + final int versionBefore = ChunkStorage.getVersion(chunkNBT); + + chunkNBT = loader.upgradeChunkTag(worldKey, persistentDataSupplier, chunkNBT, this.worldInfo.generatorKey, chunkPos, null); + + boolean modified = versionBefore < SharedConstants.getCurrentVersion().getWorldVersion(); + + if (removeCaches) { + final CompoundTag level = chunkNBT.getCompound("Level"); + modified |= level.contains("Heightmaps"); + level.remove("Heightmaps"); + modified |= level.contains("isLightOn"); + level.remove("isLightOn"); + } + + if (modified) { + this.worldInfo.modifiedChunks.getAndIncrement(); + loader.write(chunkPos, chunkNBT); + } + } catch (final Exception ex) { + LOGGER.error("Error upgrading chunk {}", chunkPos, ex); + } finally { + this.worldInfo.convertedChunks.getAndIncrement(); + } + } + } + } + } +} diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java index 2f82002c52af7304ff6b2d6fe8f094314daf0bba..5962f7a2b185d7d54a0f9e341a4fdf6e6f1c1ec5 100644 --- a/src/main/java/net/minecraft/server/Main.java +++ b/src/main/java/net/minecraft/server/Main.java @@ -16,6 +16,7 @@ import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.function.BooleanSupplier; +import io.papermc.paper.world.ThreadedWorldUpgrader; import joptsimple.NonOptionArgumentSpec; import joptsimple.OptionParser; import joptsimple.OptionSet; @@ -314,6 +315,15 @@ public class Main { } + // Paper start - fix and optimise world upgrading + public static void convertWorldButItWorks(net.minecraft.resources.ResourceKey dimensionType, net.minecraft.world.level.storage.LevelStorageSource.LevelStorageAccess worldSession, + DataFixer dataFixer, Optional>> generatorKey, boolean removeCaches) { + int threads = Runtime.getRuntime().availableProcessors() * 3 / 8; + final ThreadedWorldUpgrader worldUpgrader = new ThreadedWorldUpgrader(dimensionType, worldSession.getLevelId(), worldSession.levelDirectory.path().toFile(), threads, dataFixer, generatorKey, removeCaches); + worldUpgrader.convert(); + } + // Paper end - fix and optimise world upgrading + public static void forceUpgrade(LevelStorageSource.LevelStorageAccess session, DataFixer dataFixer, boolean eraseCache, BooleanSupplier continueCheck, WorldGenSettings generatorOptions) { Main.LOGGER.info("Forcing world upgrade! {}", session.getLevelId()); // CraftBukkit WorldUpgrader worldupgrader = new WorldUpgrader(session, dataFixer, generatorOptions, eraseCache); diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java index 3c5b7f4b2db421d56e5832e283bd60702b2d67de..84e76fbe3eca77b112c9dff936e21cba1c83e5aa 100644 --- a/src/main/java/net/minecraft/server/MinecraftServer.java +++ b/src/main/java/net/minecraft/server/MinecraftServer.java @@ -545,11 +545,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop { - return true; - }, worlddata.worldGenSettings()); - } + // Paper - move down PrimaryLevelData iworlddataserver = worlddata; WorldGenSettings generatorsettings = worlddata.worldGenSettings(); @@ -564,6 +560,13 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop worldKey = ResourceKey.create(Registry.DIMENSION_REGISTRY, dimensionKey.location()); if (dimensionKey == LevelStem.OVERWORLD) { diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java index d9a88b29cfefcdbce7bfc477b6c1af0e51079102..c21274a72dca31c9160ecbcfa7eb42de64e91454 100644 --- a/src/main/java/net/minecraft/world/level/Level.java +++ b/src/main/java/net/minecraft/world/level/Level.java @@ -182,6 +182,15 @@ public abstract class Level implements LevelAccessor, AutoCloseable { public final Map explosionDensityCache = new HashMap<>(); // Paper - Optimize explosions public java.util.ArrayDeque redstoneUpdateInfos; // Paper - Move from Map in BlockRedstoneTorch to here + // Paper start - fix and optimise world upgrading + // copied from below + public static ResourceKey getDimensionKey(DimensionType manager) { + return ((org.bukkit.craftbukkit.CraftServer)org.bukkit.Bukkit.getServer()).getHandle().getServer().registryHolder.ownedRegistryOrThrow(net.minecraft.core.Registry.DIMENSION_TYPE_REGISTRY).getResourceKey(manager).orElseThrow(() -> { + return new IllegalStateException("Unregistered dimension type: " + manager); + }); + } + // Paper end - fix and optimise world upgrading + public CraftWorld getWorld() { return this.world; } diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java index 2a6a4a62feb1c02bef850b0cda578f6f9d46a5e3..666286f94f09299f95538ef2f19b588eb74d9dda 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java @@ -32,6 +32,28 @@ public class RegionFileStorage implements AutoCloseable { } // Paper start + public static @Nullable ChunkPos getRegionFileCoordinates(Path file) { + String fileName = file.getFileName().toString(); + if (!fileName.startsWith("r.") || !fileName.endsWith(".mca")) { + return null; + } + + String[] split = fileName.split("\\."); + + if (split.length != 4) { + return null; + } + + try { + int x = Integer.parseInt(split[1]); + int z = Integer.parseInt(split[2]); + + return new ChunkPos(x << 5, z << 5); + } catch (NumberFormatException ex) { + return null; + } + } + public synchronized RegionFile getRegionFileIfLoaded(ChunkPos chunkcoordintpair) { return this.regionCache.getAndMoveToFirst(ChunkPos.asLong(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ())); } diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java index 9447b7cf08401608b2b79e8c68f58fd977cf0d42..dd43414dde55a5b447f9f51e4e5eef12fbbcfbde 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java @@ -1204,12 +1204,7 @@ public final class CraftServer implements Server { } worlddata.checkName(name); worlddata.setModdedInfo(this.console.getServerModName(), this.console.getModdedStatus().shouldReportAsModified()); - - if (console.options.has("forceUpgrade")) { - net.minecraft.server.Main.forceUpgrade(worldSession, DataFixers.getDataFixer(), console.options.has("eraseCache"), () -> { - return true; - }, worlddata.worldGenSettings()); - } + // Paper - move down long j = BiomeManager.obfuscateSeed(creator.seed()); List list = ImmutableList.of(new PhantomSpawner(), new PatrolSpawner(), new CatSpawner(), new VillageSiege(), new WanderingTraderSpawner(worlddata)); @@ -1221,6 +1216,13 @@ public final class CraftServer implements Server { biomeProvider = generator.getDefaultBiomeProvider(worldInfo); } + // Paper start - fix and optimise world upgrading + if (console.options.has("forceUpgrade")) { + net.minecraft.server.Main.convertWorldButItWorks( + actualDimension, worldSession, DataFixers.getDataFixer(), worlddimension.generator().getTypeNameForDataFixer(), console.options.has("eraseCache") + ); + } + // Paper end - fix and optimise world upgrading ResourceKey worldKey; String levelName = this.getServer().getProperties().levelName; if (name.equals(levelName + "_nether")) {