From 0d4cde25b635f418d3640dcdd603abf2423000f8 Mon Sep 17 00:00:00 2001 From: Aikar Date: Sun, 12 Apr 2020 15:56:03 -0400 Subject: [PATCH] Allow shutting down server during a watchdog hang gracefully If the request to shut down the server is received while we are in a watchdog hang, immediately treat it as a crash and begin the shutdown process. Shutdown process is now improved to also shutdown cleanly when not using restart scripts either. If a server is deadlocked, a server owner can send SIGUP (or any other signal the JVM understands to shut down as it currently does) and the watchdog will no longer need to wait until the full timeout, allowing you to trigger a close process and try to shut the server down gracefully, saving player and world data. Previously there was no way to trigger this outside of waiting for a full watchdog timeout, which may be set to a really long time... --- ...own-server-during-a-watchdog-hang-gr.patch | 85 +++++++++++++++++++ .../Asynchronous-chunk-IO-and-loading.patch | 62 +++++++------- 2 files changed, 116 insertions(+), 31 deletions(-) create mode 100644 Spigot-Server-Patches/Allow-shutting-down-server-during-a-watchdog-hang-gr.patch diff --git a/Spigot-Server-Patches/Allow-shutting-down-server-during-a-watchdog-hang-gr.patch b/Spigot-Server-Patches/Allow-shutting-down-server-during-a-watchdog-hang-gr.patch new file mode 100644 index 0000000000..ea6740338f --- /dev/null +++ b/Spigot-Server-Patches/Allow-shutting-down-server-during-a-watchdog-hang-gr.patch @@ -0,0 +1,85 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Aikar +Date: Sun, 12 Apr 2020 15:50:48 -0400 +Subject: [PATCH] Allow shutting down server during a watchdog hang gracefully + +If the request to shut down the server is received while we are in +a watchdog hang, immediately treat it as a crash and begin the shutdown +process. Shutdown process is now improved to also shutdown cleanly when +not using restart scripts either. + +If a server is deadlocked, a server owner can send SIGUP (or any other signal +the JVM understands to shut down as it currently does) and the watchdog +will no longer need to wait until the full timeout, allowing you to trigger +a close process and try to shut the server down gracefully, saving player and +world data. + +Previously there was no way to trigger this outside of waiting for a full watchdog +timeout, which may be set to a really long time... + +diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java +index 2686874f2..a9b533751 100644 +--- a/src/main/java/net/minecraft/server/MinecraftServer.java ++++ b/src/main/java/net/minecraft/server/MinecraftServer.java +@@ -0,0 +0,0 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant 0 && new File( split[0] ).isFile() ) +diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java +index 5bdcdcf9e..704e8426a 100644 +--- a/src/main/java/org/spigotmc/WatchdogThread.java ++++ b/src/main/java/org/spigotmc/WatchdogThread.java +@@ -0,0 +0,0 @@ public class WatchdogThread extends Thread + long currentTime = monotonicMillis(); + if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") ) + { +- boolean isLongTimeout = currentTime > lastTick + timeoutTime; ++ boolean isLongTimeout = currentTime > lastTick + timeoutTime || !MinecraftServer.getServer().isRunning(); + // Don't spam early warning dumps + if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue; + if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this... +@@ -0,0 +0,0 @@ public class WatchdogThread extends Thread + + if ( isLongTimeout ) + { +- if ( restart && !MinecraftServer.getServer().hasStopped() ) ++ if ( !MinecraftServer.getServer().hasStopped() ) + { +- RestartCommand.restart(); ++ AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us ++ AsyncCatcher.shuttingDown = true; ++ MinecraftServer.getServer().forceTicks = true; ++ if (restart) { ++ RestartCommand.addShutdownHook( SpigotConfig.restartScript ); ++ } ++ MinecraftServer.getServer().close(); + } + break; + } // Paper end +-- \ No newline at end of file diff --git a/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch b/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch index df709c3a32..290fa3fe9e 100644 --- a/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch +++ b/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch @@ -121,7 +121,7 @@ tasks required to be executed by the chunk load task (i.e lighting and some poi tasks). diff --git a/src/main/java/co/aikar/timings/WorldTimingsHandler.java b/src/main/java/co/aikar/timings/WorldTimingsHandler.java -index 27ce4a828e..30bafb214b 100644 +index 27ce4a828..30bafb214 100644 --- a/src/main/java/co/aikar/timings/WorldTimingsHandler.java +++ b/src/main/java/co/aikar/timings/WorldTimingsHandler.java @@ -0,0 +0,0 @@ public class WorldTimingsHandler { @@ -161,7 +161,7 @@ index 27ce4a828e..30bafb214b 100644 public static Timing getTickList(WorldServer worldserver, String timingsType) { diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java -index dbd1439970..6916ed30c4 100644 +index dbd143997..6916ed30c 100644 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java @@ -0,0 +0,0 @@ @@ -237,7 +237,7 @@ index dbd1439970..6916ed30c4 100644 + } } diff --git a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java -index 23626bef3a..1edcecd2ee 100644 +index 23626bef3..1edcecd2e 100644 --- a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java +++ b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java @@ -0,0 +0,0 @@ import java.util.concurrent.Executors; @@ -318,7 +318,7 @@ index 23626bef3a..1edcecd2ee 100644 diff --git a/src/main/java/com/destroystokyo/paper/io/IOUtil.java b/src/main/java/com/destroystokyo/paper/io/IOUtil.java new file mode 100644 -index 0000000000..5af0ac3d9e +index 000000000..5af0ac3d9 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/IOUtil.java @@ -0,0 +0,0 @@ @@ -386,7 +386,7 @@ index 0000000000..5af0ac3d9e +} diff --git a/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java new file mode 100644 -index 0000000000..4f10a8311e +index 000000000..4f10a8311 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java @@ -0,0 +0,0 @@ @@ -1053,7 +1053,7 @@ index 0000000000..4f10a8311e +} diff --git a/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java new file mode 100644 -index 0000000000..97f2e433c4 +index 000000000..97f2e433c --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java @@ -0,0 +0,0 @@ @@ -1336,7 +1336,7 @@ index 0000000000..97f2e433c4 +} diff --git a/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java new file mode 100644 -index 0000000000..ee906b594b +index 000000000..ee906b594 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java @@ -0,0 +0,0 @@ @@ -1583,7 +1583,7 @@ index 0000000000..ee906b594b +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java new file mode 100644 -index 0000000000..305da47868 +index 000000000..305da4786 --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java @@ -0,0 +0,0 @@ @@ -1738,7 +1738,7 @@ index 0000000000..305da47868 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java new file mode 100644 -index 0000000000..60312b85f9 +index 000000000..60312b85f --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java @@ -0,0 +0,0 @@ @@ -1856,7 +1856,7 @@ index 0000000000..60312b85f9 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java new file mode 100644 -index 0000000000..1dfa8abfd8 +index 000000000..1dfa8abfd --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java @@ -0,0 +0,0 @@ @@ -1902,7 +1902,7 @@ index 0000000000..1dfa8abfd8 +} diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java new file mode 100644 -index 0000000000..0745a2015a +index 000000000..2b20c159f --- /dev/null +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java @@ -0,0 +0,0 @@ @@ -2283,7 +2283,7 @@ index 0000000000..0745a2015a + drainChunkWaitQueue(); + + if (this.workers == null) { -+ if (Bukkit.isPrimaryThread()) { ++ if (Bukkit.isPrimaryThread() || MinecraftServer.getServer().hasStopped()) { + ((IAsyncTaskHandler)this.world.getChunkProvider().serverThreadQueue).executeAll(); + } else { + CompletableFuture wait = new CompletableFuture<>(); @@ -2399,7 +2399,7 @@ index 0000000000..0745a2015a + +} diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java -index b582171c51..03d7ce8294 100644 +index b582171c5..03d7ce829 100644 --- a/src/main/java/net/minecraft/server/ChunkProviderServer.java +++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java @@ -0,0 +0,0 @@ public class ChunkProviderServer extends IChunkProvider { @@ -2569,7 +2569,7 @@ index b582171c51..03d7ce8294 100644 } finally { playerChunkMap.callbackExecutor.run(); diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java -index a950ad801d..26f1a4b095 100644 +index a950ad801..26f1a4b09 100644 --- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java +++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java @@ -0,0 +0,0 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet; @@ -2838,7 +2838,7 @@ index a950ad801d..26f1a4b095 100644 nbttagcompound1.set("PostProcessing", a(ichunkaccess.l())); diff --git a/src/main/java/net/minecraft/server/ChunkStatus.java b/src/main/java/net/minecraft/server/ChunkStatus.java -index 134a4f0b7d..40ce30cdc2 100644 +index 134a4f0b7..40ce30cdc 100644 --- a/src/main/java/net/minecraft/server/ChunkStatus.java +++ b/src/main/java/net/minecraft/server/ChunkStatus.java @@ -0,0 +0,0 @@ public class ChunkStatus { @@ -2874,7 +2874,7 @@ index 134a4f0b7d..40ce30cdc2 100644 return this.c() >= chunkstatus.c(); } diff --git a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java -index 7e5ece9d50..cfe43e882e 100644 +index 7e5ece9d5..cfe43e882 100644 --- a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java +++ b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java @@ -0,0 +0,0 @@ public abstract class IAsyncTaskHandler implements Mailbox { @@ -3898,7 +3898,7 @@ index 1d1b267f32..4b87ca2ecb 100644 public static TicketType a(String s, Comparator comparator) { return new TicketType<>(s, comparator, 0L); diff --git a/src/main/java/net/minecraft/server/VillagePlace.java b/src/main/java/net/minecraft/server/VillagePlace.java -index c999f8c9bf..b59ef1a633 100644 +index c999f8c9b..b59ef1a63 100644 --- a/src/main/java/net/minecraft/server/VillagePlace.java +++ b/src/main/java/net/minecraft/server/VillagePlace.java @@ -0,0 +0,0 @@ public class VillagePlace extends RegionFileSection { @@ -3987,7 +3987,7 @@ index c999f8c9bf..b59ef1a633 100644 HAS_SPACE(VillagePlaceRecord::d), IS_OCCUPIED(VillagePlaceRecord::e), ANY((villageplacerecord) -> { diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java -index df7503a5ec..d4ef2403d5 100644 +index df7503a5e..d4ef2403d 100644 --- a/src/main/java/net/minecraft/server/WorldServer.java +++ b/src/main/java/net/minecraft/server/WorldServer.java @@ -0,0 +0,0 @@ public class WorldServer extends World { @@ -4080,7 +4080,7 @@ index df7503a5ec..d4ef2403d5 100644 // CraftBukkit start diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java -index a71bb86508..1d275520fb 100644 +index a71bb8650..1d275520f 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java @@ -0,0 +0,0 @@ public class CraftWorld implements World { @@ -4141,7 +4141,7 @@ index a71bb86508..1d275520fb 100644 // Spigot start @Override diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java -index 07936eeba2..5bdcdcf9e8 100644 +index 07936eeba..5bdcdcf9e 100644 --- a/src/main/java/org/spigotmc/WatchdogThread.java +++ b/src/main/java/org/spigotmc/WatchdogThread.java @@ -0,0 +0,0 @@ import java.lang.management.ThreadInfo;