From db3af11cea1a58ac9e8a4836d5938a65df2ee2b1 Mon Sep 17 00:00:00 2001 From: Spottedleaf Date: Thu, 9 Jan 2020 17:42:33 -0800 Subject: [PATCH] Fix race condition with regionfile being closed right after getting one (#2812) Occurs when 1 thread retrieves a regionfile, and then the regionfile is closed due to it being thrown out of cache. --- ...95-Asynchronous-chunk-IO-and-loading.patch | 168 +++++++++++++----- 1 file changed, 126 insertions(+), 42 deletions(-) diff --git a/Spigot-Server-Patches/0395-Asynchronous-chunk-IO-and-loading.patch b/Spigot-Server-Patches/0395-Asynchronous-chunk-IO-and-loading.patch index 7d84d2929f..2cd5a72c61 100644 --- a/Spigot-Server-Patches/0395-Asynchronous-chunk-IO-and-loading.patch +++ b/Spigot-Server-Patches/0395-Asynchronous-chunk-IO-and-loading.patch @@ -1,4 +1,4 @@ -From 48e633b295c6057e28af0232c11505c3e81100ff Mon Sep 17 00:00:00 2001 +From 4208dd633a9e5175b7eb7db8f090e21e35c9dd9d Mon Sep 17 00:00:00 2001 From: Spottedleaf Date: Sat, 13 Jul 2019 09:23:10 -0700 Subject: [PATCH] Asynchronous chunk IO and loading @@ -121,7 +121,7 @@ tasks required to be executed by the chunk load task (i.e lighting and some poi tasks). diff --git a/src/main/java/co/aikar/timings/WorldTimingsHandler.java b/src/main/java/co/aikar/timings/WorldTimingsHandler.java -index 3a79cde595..8de6c4816c 100644 +index 3a79cde59..8de6c4816 100644 --- a/src/main/java/co/aikar/timings/WorldTimingsHandler.java +++ b/src/main/java/co/aikar/timings/WorldTimingsHandler.java @@ -63,6 +63,17 @@ public class WorldTimingsHandler { @@ -161,7 +161,7 @@ index 3a79cde595..8de6c4816c 100644 public static Timing getTickList(WorldServer worldserver, String timingsType) { diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java -index 546a1cfe0a..1d7d1ffbf7 100644 +index 546a1cfe0..1d7d1ffbf 100644 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java @@ -1,5 +1,6 @@ @@ -237,7 +237,7 @@ index 546a1cfe0a..1d7d1ffbf7 100644 + } } diff --git a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java -index 23626bef3a..1edcecd2ee 100644 +index 23626bef3..1edcecd2e 100644 --- a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java +++ b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java @@ -9,6 +9,7 @@ import java.util.concurrent.Executors; @@ -318,7 +318,7 @@ index 23626bef3a..1edcecd2ee 100644 diff --git a/src/main/java/com/destroystokyo/paper/io/IOUtil.java b/src/main/java/com/destroystokyo/paper/io/IOUtil.java new file mode 100644 -index 0000000000..5af0ac3d9e +index 000000000..5af0ac3d9 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/IOUtil.java @@ -0,0 +1,62 @@ @@ -386,7 +386,7 @@ index 0000000000..5af0ac3d9e +} diff --git a/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java new file mode 100644 -index 0000000000..4f10a8311e +index 000000000..4f10a8311 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java @@ -0,0 +1,661 @@ @@ -1053,7 +1053,7 @@ index 0000000000..4f10a8311e +} diff --git a/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java new file mode 100644 -index 0000000000..78bd238f4c +index 000000000..78bd238f4 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java @@ -0,0 +1,276 @@ @@ -1335,7 +1335,7 @@ index 0000000000..78bd238f4c +} diff --git a/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java new file mode 100644 -index 0000000000..ee906b594b +index 000000000..ee906b594 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java @@ -0,0 +1,241 @@ @@ -1582,7 +1582,7 @@ index 0000000000..ee906b594b +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java new file mode 100644 -index 0000000000..305da47868 +index 000000000..305da4786 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java @@ -0,0 +1,149 @@ @@ -1737,7 +1737,7 @@ index 0000000000..305da47868 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java new file mode 100644 -index 0000000000..60312b85f9 +index 000000000..60312b85f --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java @@ -0,0 +1,112 @@ @@ -1855,7 +1855,7 @@ index 0000000000..60312b85f9 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java new file mode 100644 -index 0000000000..1dfa8abfd8 +index 000000000..1dfa8abfd --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java @@ -0,0 +1,40 @@ @@ -1901,7 +1901,7 @@ index 0000000000..1dfa8abfd8 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java new file mode 100644 -index 0000000000..59d73bfad7 +index 000000000..59d73bfad --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java @@ -0,0 +1,453 @@ @@ -2359,7 +2359,7 @@ index 0000000000..59d73bfad7 + +} diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java -index e9cd44fae1..1f6b1c4f16 100644 +index e9cd44fae..1f6b1c4f1 100644 --- a/src/main/java/net/minecraft/server/ChunkProviderServer.java +++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java @@ -124,11 +124,137 @@ public class ChunkProviderServer extends IChunkProvider { @@ -2529,7 +2529,7 @@ index e9cd44fae1..1f6b1c4f16 100644 } finally { playerChunkMap.callbackExecutor.run(); diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java -index a950ad801d..26f1a4b095 100644 +index a950ad801..26f1a4b09 100644 --- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java +++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java @@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet; @@ -2798,7 +2798,7 @@ index a950ad801d..26f1a4b095 100644 nbttagcompound1.set("PostProcessing", a(ichunkaccess.l())); diff --git a/src/main/java/net/minecraft/server/ChunkStatus.java b/src/main/java/net/minecraft/server/ChunkStatus.java -index 134a4f0b7d..88f1674616 100644 +index 134a4f0b7..88f167461 100644 --- a/src/main/java/net/minecraft/server/ChunkStatus.java +++ b/src/main/java/net/minecraft/server/ChunkStatus.java @@ -153,6 +153,7 @@ public class ChunkStatus { @@ -2810,7 +2810,7 @@ index 134a4f0b7d..88f1674616 100644 return ChunkStatus.r.getInt(chunkstatus.c()); } diff --git a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java -index 7210217913..f7156acb89 100644 +index 721021791..f7156acb8 100644 --- a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java +++ b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java @@ -91,7 +91,7 @@ public abstract class IAsyncTaskHandler implements Mailbox public NBTTagCompound nbttagcompound = this.read(chunkcoordintpair); -@@ -925,33 +1085,53 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { +@@ -925,33 +1085,55 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { // Paper start - chunk status cache "api" public ChunkStatus getChunkStatusOnDiskIfCached(ChunkCoordIntPair chunkPos) { - RegionFile regionFile = this.getIOWorker().getRegionFileCache().getRegionFileIfLoaded(chunkPos); ++ synchronized (this) { // Paper + RegionFile regionFile = this.getRegionFileIfLoaded(chunkPos); return regionFile == null ? null : regionFile.getStatusIfCached(chunkPos.x, chunkPos.z); ++ } // Paper } public ChunkStatus getChunkStatusOnDisk(ChunkCoordIntPair chunkPos) throws IOException { @@ -3493,7 +3495,7 @@ index 3825520fa1..d9faef8a67 100644 } public IChunkAccess getUnloadingChunk(int chunkX, int chunkZ) { -@@ -960,6 +1140,39 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { +@@ -960,6 +1142,39 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { } // Paper end @@ -3533,7 +3535,7 @@ index 3825520fa1..d9faef8a67 100644 boolean isOutsideOfRange(ChunkCoordIntPair chunkcoordintpair) { // Spigot start return isOutsideOfRange(chunkcoordintpair, false); -@@ -1304,6 +1517,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { +@@ -1304,6 +1519,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d { } @@ -3542,10 +3544,19 @@ index 3825520fa1..d9faef8a67 100644 return this.m; } diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java -index d37abf2cf3..0de24c792e 100644 +index d37abf2cf..df728e2c0 100644 --- a/src/main/java/net/minecraft/server/RegionFile.java +++ b/src/main/java/net/minecraft/server/RegionFile.java -@@ -224,7 +224,7 @@ public class RegionFile implements AutoCloseable { +@@ -36,6 +36,8 @@ public class RegionFile implements AutoCloseable { + private final RegionFileBitSet freeSectors; + public final File file; + ++ public final java.util.concurrent.locks.ReentrantLock fileLock = new java.util.concurrent.locks.ReentrantLock(true); // Paper ++ + // Paper start - Cache chunk status + private final ChunkStatus[] statuses = new ChunkStatus[32 * 32]; + +@@ -224,7 +226,7 @@ public class RegionFile implements AutoCloseable { return (i + 4096 - 1) / 4096; } @@ -3554,17 +3565,31 @@ index d37abf2cf3..0de24c792e 100644 int i = this.getOffset(chunkcoordintpair); if (i == 0) { -@@ -379,7 +379,7 @@ public class RegionFile implements AutoCloseable { - return chunkcoordintpair.j() + chunkcoordintpair.k() * 32; +@@ -380,6 +382,11 @@ public class RegionFile implements AutoCloseable { } -- public void close() throws IOException { -+ public synchronized void close() throws IOException { // Paper - Synchronized + public void close() throws IOException { ++ // Paper start - Prevent regionfiles from being closed during use ++ this.fileLock.lock(); ++ synchronized (this) { ++ try { ++ // Paper end this.closed = true; // Paper try { this.c(); +@@ -394,6 +401,10 @@ public class RegionFile implements AutoCloseable { + } + } + } ++ } finally { // Paper start - Prevent regionfiles from being closed during use ++ this.fileLock.unlock(); ++ } ++ } // Paper end + + } + diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java -index e07ae98540..d927f93211 100644 +index e07ae9854..0f201000f 100644 --- a/src/main/java/net/minecraft/server/RegionFileCache.java +++ b/src/main/java/net/minecraft/server/RegionFileCache.java @@ -9,7 +9,7 @@ import java.io.File; @@ -3576,7 +3601,7 @@ index e07ae98540..d927f93211 100644 public final Long2ObjectLinkedOpenHashMap cache = new Long2ObjectLinkedOpenHashMap(); private final File b; -@@ -20,12 +20,12 @@ public final class RegionFileCache implements AutoCloseable { +@@ -20,16 +20,27 @@ public final class RegionFileCache implements AutoCloseable { // Paper start @@ -3588,11 +3613,70 @@ index e07ae98540..d927f93211 100644 // Paper end - public RegionFile getFile(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - private > public + public synchronized RegionFile getFile(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly) throws IOException { // CraftBukkit // Paper - private > public, synchronize ++ // Paper start - add lock parameter ++ return this.getFile(chunkcoordintpair, existingOnly, false); ++ } ++ public synchronized RegionFile getFile(ChunkCoordIntPair chunkcoordintpair, boolean existingOnly, boolean lock) throws IOException { ++ // Paper end long i = ChunkCoordIntPair.pair(chunkcoordintpair.getRegionX(), chunkcoordintpair.getRegionZ()); RegionFile regionfile = (RegionFile) this.cache.getAndMoveToFirst(i); -@@ -201,7 +201,7 @@ public final class RegionFileCache implements AutoCloseable { + if (regionfile != null) { ++ // Paper start ++ if (lock) { ++ // must be in this synchronized block ++ regionfile.fileLock.lock(); ++ } ++ // Paper end + return regionfile; + } else { + if (this.cache.size() >= com.destroystokyo.paper.PaperConfig.regionFileCacheSize) { // Paper - configurable +@@ -45,6 +56,12 @@ public final class RegionFileCache implements AutoCloseable { + RegionFile regionfile1 = new RegionFile(file, this.b); + + this.cache.putAndMoveToFirst(i, regionfile1); ++ // Paper start ++ if (lock) { ++ // must be in this synchronized block ++ regionfile1.fileLock.lock(); ++ } ++ // Paper end + return regionfile1; + } + } +@@ -119,7 +136,8 @@ public final class RegionFileCache implements AutoCloseable { + + @Nullable + public NBTTagCompound read(ChunkCoordIntPair chunkcoordintpair) throws IOException { +- RegionFile regionfile = this.getFile(chunkcoordintpair, false); // CraftBukkit ++ RegionFile regionfile = this.getFile(chunkcoordintpair, false, true); // CraftBukkit // Paper ++ try { // Paper + DataInputStream datainputstream = regionfile.a(chunkcoordintpair); + // Paper start + if (regionfile.isOversized(chunkcoordintpair.x, chunkcoordintpair.z)) { +@@ -157,10 +175,14 @@ public final class RegionFileCache implements AutoCloseable { + } + + return nbttagcompound; ++ } finally { // Paper start ++ regionfile.fileLock.unlock(); ++ } // Paper end + } + + protected void write(ChunkCoordIntPair chunkcoordintpair, NBTTagCompound nbttagcompound) throws IOException { +- RegionFile regionfile = this.getFile(chunkcoordintpair, false); // CraftBukkit ++ RegionFile regionfile = this.getFile(chunkcoordintpair, false, true); // CraftBukkit // Paper ++ try { // Paper + int attempts = 0; Exception laste = null; while (attempts++ < 5) { try { // Paper + DataOutputStream dataoutputstream = regionfile.c(chunkcoordintpair); + Throwable throwable = null; +@@ -199,9 +221,12 @@ public final class RegionFileCache implements AutoCloseable { + MinecraftServer.LOGGER.error("Failed to save chunk", laste); + } // Paper end ++ } finally { // Paper start ++ regionfile.fileLock.unlock(); ++ } // Paper end } - public void close() throws IOException { @@ -3600,7 +3684,7 @@ index e07ae98540..d927f93211 100644 ObjectIterator objectiterator = this.cache.values().iterator(); while (objectiterator.hasNext()) { -@@ -213,7 +213,7 @@ public final class RegionFileCache implements AutoCloseable { +@@ -213,7 +238,7 @@ public final class RegionFileCache implements AutoCloseable { } // CraftBukkit start @@ -3610,7 +3694,7 @@ index e07ae98540..d927f93211 100644 return regionfile != null ? regionfile.chunkExists(pos) : false; diff --git a/src/main/java/net/minecraft/server/RegionFileSection.java b/src/main/java/net/minecraft/server/RegionFileSection.java -index db9f0196bd..a6d8ef5eb4 100644 +index db9f0196b..a6d8ef5eb 100644 --- a/src/main/java/net/minecraft/server/RegionFileSection.java +++ b/src/main/java/net/minecraft/server/RegionFileSection.java @@ -20,28 +20,29 @@ import javax.annotation.Nullable; @@ -3746,7 +3830,7 @@ index db9f0196bd..a6d8ef5eb4 100644 + // Paper end } diff --git a/src/main/java/net/minecraft/server/TicketType.java b/src/main/java/net/minecraft/server/TicketType.java -index 335b644351..481d954808 100644 +index 335b64435..481d95480 100644 --- a/src/main/java/net/minecraft/server/TicketType.java +++ b/src/main/java/net/minecraft/server/TicketType.java @@ -22,6 +22,7 @@ public class TicketType { @@ -3758,7 +3842,7 @@ index 335b644351..481d954808 100644 public static TicketType a(String s, Comparator comparator) { return new TicketType<>(s, comparator, 0L); diff --git a/src/main/java/net/minecraft/server/VillagePlace.java b/src/main/java/net/minecraft/server/VillagePlace.java -index c999f8c9bf..b59ef1a633 100644 +index c999f8c9b..b59ef1a63 100644 --- a/src/main/java/net/minecraft/server/VillagePlace.java +++ b/src/main/java/net/minecraft/server/VillagePlace.java @@ -24,8 +24,16 @@ public class VillagePlace extends RegionFileSection { @@ -3847,7 +3931,7 @@ index c999f8c9bf..b59ef1a633 100644 HAS_SPACE(VillagePlaceRecord::d), IS_OCCUPIED(VillagePlaceRecord::e), ANY((villageplacerecord) -> { diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java -index 90272d0c00..136c18d10b 100644 +index 90272d0c0..136c18d10 100644 --- a/src/main/java/net/minecraft/server/WorldServer.java +++ b/src/main/java/net/minecraft/server/WorldServer.java @@ -82,6 +82,79 @@ public class WorldServer extends World { @@ -3940,7 +4024,7 @@ index 90272d0c00..136c18d10b 100644 // CraftBukkit start diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java -index 2a33b90efd..0d036e5d96 100644 +index 2a33b90ef..0d036e5d9 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java @@ -555,22 +555,23 @@ public class CraftWorld implements World { @@ -4002,7 +4086,7 @@ index 2a33b90efd..0d036e5d96 100644 @Override public int getViewDistance() { diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java -index 07936eeba2..fe68df45ba 100644 +index 07936eeba..fe68df45b 100644 --- a/src/main/java/org/spigotmc/WatchdogThread.java +++ b/src/main/java/org/spigotmc/WatchdogThread.java @@ -6,6 +6,7 @@ import java.lang.management.ThreadInfo;