Allow shutting down server during a watchdog hang gracefully

If the request to shut down the server is received while we are in
a watchdog hang, immediately treat it as a crash and begin the shutdown
process. Shutdown process is now improved to also shutdown cleanly when
not using restart scripts either.

If a server is deadlocked, a server owner can send SIGUP (or any other signal
the JVM understands to shut down as it currently does) and the watchdog
will no longer need to wait until the full timeout, allowing you to trigger
a close process and try to shut the server down gracefully, saving player and
world data.

Previously there was no way to trigger this outside of waiting for a full watchdog
timeout, which may be set to a really long time...

Spigot-Server-Patches/0391-Asynchronous-chunk-IO-and-loading.patch

@@ -1,4 +1,4 @@
-From 6e4b6c3681b093e95db973304a8cd0b04880cb76 Mon Sep 17 00:00:00 2001
+From d8323c167f2f692f00b4396f04b8ea4af7b2f3ac Mon Sep 17 00:00:00 2001
 From: Spottedleaf <Spottedleaf@users.noreply.github.com>
 Date: Sat, 13 Jul 2019 09:23:10 -0700
 Subject: [PATCH] Asynchronous chunk IO and loading
@@ -121,7 +121,7 @@ tasks required to be executed by the chunk load task (i.e lighting
 and some poi tasks).
 
 diff --git a/src/main/java/co/aikar/timings/WorldTimingsHandler.java b/src/main/java/co/aikar/timings/WorldTimingsHandler.java
-index 27ce4a828e..30bafb214b 100644
+index 27ce4a828..30bafb214 100644
 --- a/src/main/java/co/aikar/timings/WorldTimingsHandler.java
 +++ b/src/main/java/co/aikar/timings/WorldTimingsHandler.java
 @@ -61,6 +61,17 @@ public class WorldTimingsHandler {
@@ -161,7 +161,7 @@ index 27ce4a828e..30bafb214b 100644
  
      public static Timing getTickList(WorldServer worldserver, String timingsType) {
 diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
-index dbd1439970..6916ed30c4 100644
+index dbd143997..6916ed30c 100644
 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
 +++ b/src/main/java/com/destroystokyo/paper/PaperConfig.java
 @@ -1,5 +1,6 @@
@@ -237,7 +237,7 @@ index dbd1439970..6916ed30c4 100644
 +    }
  }
 diff --git a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java
-index 23626bef3a..1edcecd2ee 100644
+index 23626bef3..1edcecd2e 100644
 --- a/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java
 +++ b/src/main/java/com/destroystokyo/paper/antixray/ChunkPacketBlockControllerAntiXray.java
 @@ -9,6 +9,7 @@ import java.util.concurrent.Executors;
@@ -318,7 +318,7 @@ index 23626bef3a..1edcecd2ee 100644
  
 diff --git a/src/main/java/com/destroystokyo/paper/io/IOUtil.java b/src/main/java/com/destroystokyo/paper/io/IOUtil.java
 new file mode 100644
-index 0000000000..5af0ac3d9e
+index 000000000..5af0ac3d9
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/IOUtil.java
 @@ -0,0 +1,62 @@
@@ -386,7 +386,7 @@ index 0000000000..5af0ac3d9e
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java
 new file mode 100644
-index 0000000000..4f10a8311e
+index 000000000..4f10a8311
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/PaperFileIOThread.java
 @@ -0,0 +1,661 @@
@@ -1053,7 +1053,7 @@ index 0000000000..4f10a8311e
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java
 new file mode 100644
-index 0000000000..97f2e433c4
+index 000000000..97f2e433c
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/PrioritizedTaskQueue.java
 @@ -0,0 +1,277 @@
@@ -1336,7 +1336,7 @@ index 0000000000..97f2e433c4
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java
 new file mode 100644
-index 0000000000..ee906b594b
+index 000000000..ee906b594
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/QueueExecutorThread.java
 @@ -0,0 +1,241 @@
@@ -1583,7 +1583,7 @@ index 0000000000..ee906b594b
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java
 new file mode 100644
-index 0000000000..305da47868
+index 000000000..305da4786
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkLoadTask.java
 @@ -0,0 +1,149 @@
@@ -1738,7 +1738,7 @@ index 0000000000..305da47868
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java
 new file mode 100644
-index 0000000000..60312b85f9
+index 000000000..60312b85f
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkSaveTask.java
 @@ -0,0 +1,112 @@
@@ -1856,7 +1856,7 @@ index 0000000000..60312b85f9
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java
 new file mode 100644
-index 0000000000..1dfa8abfd8
+index 000000000..1dfa8abfd
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTask.java
 @@ -0,0 +1,40 @@
@@ -1902,7 +1902,7 @@ index 0000000000..1dfa8abfd8
 +}
 diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java
 new file mode 100644
-index 0000000000..0745a2015a
+index 000000000..2b20c159f
 --- /dev/null
 +++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java
 @@ -0,0 +1,492 @@
@@ -2283,7 +2283,7 @@ index 0000000000..0745a2015a
 +        drainChunkWaitQueue();
 +
 +        if (this.workers == null) {
-+            if (Bukkit.isPrimaryThread()) {
++            if (Bukkit.isPrimaryThread() || MinecraftServer.getServer().hasStopped()) {
 +                ((IAsyncTaskHandler<Runnable>)this.world.getChunkProvider().serverThreadQueue).executeAll();
 +            } else {
 +                CompletableFuture<Void> wait = new CompletableFuture<>();
@@ -2399,7 +2399,7 @@ index 0000000000..0745a2015a
 +
 +}
 diff --git a/src/main/java/net/minecraft/server/ChunkProviderServer.java b/src/main/java/net/minecraft/server/ChunkProviderServer.java
-index b582171c51..03d7ce8294 100644
+index b582171c5..03d7ce829 100644
 --- a/src/main/java/net/minecraft/server/ChunkProviderServer.java
 +++ b/src/main/java/net/minecraft/server/ChunkProviderServer.java
 @@ -218,11 +218,137 @@ public class ChunkProviderServer extends IChunkProvider {
@@ -2569,7 +2569,7 @@ index b582171c51..03d7ce8294 100644
          } finally {
              playerChunkMap.callbackExecutor.run();
 diff --git a/src/main/java/net/minecraft/server/ChunkRegionLoader.java b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
-index a950ad801d..26f1a4b095 100644
+index a950ad801..26f1a4b09 100644
 --- a/src/main/java/net/minecraft/server/ChunkRegionLoader.java
 +++ b/src/main/java/net/minecraft/server/ChunkRegionLoader.java
 @@ -6,6 +6,7 @@ import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
@@ -2838,7 +2838,7 @@ index a950ad801d..26f1a4b095 100644
  
          nbttagcompound1.set("PostProcessing", a(ichunkaccess.l()));
 diff --git a/src/main/java/net/minecraft/server/ChunkStatus.java b/src/main/java/net/minecraft/server/ChunkStatus.java
-index 134a4f0b7d..40ce30cdc2 100644
+index 134a4f0b7..40ce30cdc 100644
 --- a/src/main/java/net/minecraft/server/ChunkStatus.java
 +++ b/src/main/java/net/minecraft/server/ChunkStatus.java
 @@ -153,6 +153,7 @@ public class ChunkStatus {
@@ -2874,7 +2874,7 @@ index 134a4f0b7d..40ce30cdc2 100644
          return this.c() >= chunkstatus.c();
      }
 diff --git a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java
-index 7e5ece9d50..cfe43e882e 100644
+index 7e5ece9d5..cfe43e882 100644
 --- a/src/main/java/net/minecraft/server/IAsyncTaskHandler.java
 +++ b/src/main/java/net/minecraft/server/IAsyncTaskHandler.java
 @@ -91,7 +91,7 @@ public abstract class IAsyncTaskHandler<R extends Runnable> implements Mailbox<R
@@ -2887,7 +2887,7 @@ index 7e5ece9d50..cfe43e882e 100644
              ;
          }
 diff --git a/src/main/java/net/minecraft/server/IChunkLoader.java b/src/main/java/net/minecraft/server/IChunkLoader.java
-index 2f95174fcc..134c76065b 100644
+index 2f95174fc..134c76065 100644
 --- a/src/main/java/net/minecraft/server/IChunkLoader.java
 +++ b/src/main/java/net/minecraft/server/IChunkLoader.java
 @@ -3,37 +3,49 @@ package net.minecraft.server;
@@ -3012,7 +3012,7 @@ index 2f95174fcc..134c76065b 100644
 +//    Paper end
  }
 diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
-index 047a0c6591..b9d5844520 100644
+index 047a0c659..b9d584452 100644
 --- a/src/main/java/net/minecraft/server/MCUtil.java
 +++ b/src/main/java/net/minecraft/server/MCUtil.java
 @@ -598,4 +598,9 @@ public final class MCUtil {
@@ -3026,7 +3026,7 @@ index 047a0c6591..b9d5844520 100644
 +    }
  }
 diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
-index 6fa08c60bd..baeaf772be 100644
+index 6fa08c60b..baeaf772b 100644
 --- a/src/main/java/net/minecraft/server/MinecraftServer.java
 +++ b/src/main/java/net/minecraft/server/MinecraftServer.java
 @@ -789,6 +789,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
@@ -3038,7 +3038,7 @@ index 6fa08c60bd..baeaf772be 100644
  
      public String getServerIp() {
 diff --git a/src/main/java/net/minecraft/server/NextTickListEntry.java b/src/main/java/net/minecraft/server/NextTickListEntry.java
-index e9c405fb53..33cfeabdee 100644
+index e9c405fb5..33cfeabde 100644
 --- a/src/main/java/net/minecraft/server/NextTickListEntry.java
 +++ b/src/main/java/net/minecraft/server/NextTickListEntry.java
 @@ -4,7 +4,7 @@ import java.util.Comparator;
@@ -3060,7 +3060,7 @@ index e9c405fb53..33cfeabdee 100644
          this.e = t0;
          this.b = i;
 diff --git a/src/main/java/net/minecraft/server/NibbleArray.java b/src/main/java/net/minecraft/server/NibbleArray.java
-index ed8c4a87b5..996c832638 100644
+index ed8c4a87b..996c83263 100644
 --- a/src/main/java/net/minecraft/server/NibbleArray.java
 +++ b/src/main/java/net/minecraft/server/NibbleArray.java
 @@ -71,6 +71,7 @@ public class NibbleArray {
@@ -3072,7 +3072,7 @@ index ed8c4a87b5..996c832638 100644
          return this.a == null ? new NibbleArray() : new NibbleArray((byte[]) this.a.clone());
      }
 diff --git a/src/main/java/net/minecraft/server/PlayerChunk.java b/src/main/java/net/minecraft/server/PlayerChunk.java
-index 50135446f7..b38bc67758 100644
+index 50135446f..b38bc6775 100644
 --- a/src/main/java/net/minecraft/server/PlayerChunk.java
 +++ b/src/main/java/net/minecraft/server/PlayerChunk.java
 @@ -127,6 +127,18 @@ public class PlayerChunk {
@@ -3117,7 +3117,7 @@ index 50135446f7..b38bc67758 100644
                  completablefuture = (CompletableFuture) this.statusFutures.get(i);
                  if (completablefuture != null) {
 diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java
-index b4c9d544fe..7e5fa016c7 100644
+index b4c9d544f..7e5fa016c 100644
 --- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
 +++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
 @@ -63,7 +63,7 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
@@ -3600,7 +3600,7 @@ index b4c9d544fe..7e5fa016c7 100644
          return this.m;
      }
 diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
-index d37abf2cf3..df728e2c0a 100644
+index d37abf2cf..df728e2c0 100644
 --- a/src/main/java/net/minecraft/server/RegionFile.java
 +++ b/src/main/java/net/minecraft/server/RegionFile.java
 @@ -36,6 +36,8 @@ public class RegionFile implements AutoCloseable {
@@ -3645,7 +3645,7 @@ index d37abf2cf3..df728e2c0a 100644
      }
  
 diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
-index e07ae98540..0f201000f6 100644
+index e07ae9854..0f201000f 100644
 --- a/src/main/java/net/minecraft/server/RegionFileCache.java
 +++ b/src/main/java/net/minecraft/server/RegionFileCache.java
 @@ -9,7 +9,7 @@ import java.io.File;
@@ -3750,7 +3750,7 @@ index e07ae98540..0f201000f6 100644
  
          return regionfile != null ? regionfile.chunkExists(pos) : false;
 diff --git a/src/main/java/net/minecraft/server/RegionFileSection.java b/src/main/java/net/minecraft/server/RegionFileSection.java
-index db9f0196bd..a6d8ef5eb4 100644
+index db9f0196b..a6d8ef5eb 100644
 --- a/src/main/java/net/minecraft/server/RegionFileSection.java
 +++ b/src/main/java/net/minecraft/server/RegionFileSection.java
 @@ -20,28 +20,29 @@ import javax.annotation.Nullable;
@@ -3886,7 +3886,7 @@ index db9f0196bd..a6d8ef5eb4 100644
 +    // Paper end
  }
 diff --git a/src/main/java/net/minecraft/server/TicketType.java b/src/main/java/net/minecraft/server/TicketType.java
-index 1d1b267f32..4b87ca2ecb 100644
+index 1d1b267f3..4b87ca2ec 100644
 --- a/src/main/java/net/minecraft/server/TicketType.java
 +++ b/src/main/java/net/minecraft/server/TicketType.java
 @@ -22,6 +22,7 @@ public class TicketType<T> {
@@ -3898,7 +3898,7 @@ index 1d1b267f32..4b87ca2ecb 100644
      public static <T> TicketType<T> a(String s, Comparator<T> comparator) {
          return new TicketType<>(s, comparator, 0L);
 diff --git a/src/main/java/net/minecraft/server/VillagePlace.java b/src/main/java/net/minecraft/server/VillagePlace.java
-index c999f8c9bf..b59ef1a633 100644
+index c999f8c9b..b59ef1a63 100644
 --- a/src/main/java/net/minecraft/server/VillagePlace.java
 +++ b/src/main/java/net/minecraft/server/VillagePlace.java
 @@ -24,8 +24,16 @@ public class VillagePlace extends RegionFileSection<VillagePlaceSection> {
@@ -3987,7 +3987,7 @@ index c999f8c9bf..b59ef1a633 100644
  
          HAS_SPACE(VillagePlaceRecord::d), IS_OCCUPIED(VillagePlaceRecord::e), ANY((villageplacerecord) -> {
 diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java
-index df7503a5ec..d4ef2403d5 100644
+index df7503a5e..d4ef2403d 100644
 --- a/src/main/java/net/minecraft/server/WorldServer.java
 +++ b/src/main/java/net/minecraft/server/WorldServer.java
 @@ -82,6 +82,79 @@ public class WorldServer extends World {
@@ -4080,7 +4080,7 @@ index df7503a5ec..d4ef2403d5 100644
  
      // CraftBukkit start
 diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
-index a71bb86508..1d275520fb 100644
+index a71bb8650..1d275520f 100644
 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
 +++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
 @@ -554,22 +554,23 @@ public class CraftWorld implements World {
@@ -4141,7 +4141,7 @@ index a71bb86508..1d275520fb 100644
      // Spigot start
      @Override
 diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
-index 07936eeba2..5bdcdcf9e8 100644
+index 07936eeba..5bdcdcf9e 100644
 --- a/src/main/java/org/spigotmc/WatchdogThread.java
 +++ b/src/main/java/org/spigotmc/WatchdogThread.java
 @@ -6,6 +6,7 @@ import java.lang.management.ThreadInfo;

Spigot-Server-Patches/0469-Allow-shutting-down-server-during-a-watchdog-hang-gr.patch

@@ -0,0 +1,87 @@
+From e231dc58a43fc554fa22970df42b43f2e59f07ba Mon Sep 17 00:00:00 2001
+From: Aikar <aikar@aikar.co>
+Date: Sun, 12 Apr 2020 15:50:48 -0400
+Subject: [PATCH] Allow shutting down server during a watchdog hang gracefully
+
+If the request to shut down the server is received while we are in
+a watchdog hang, immediately treat it as a crash and begin the shutdown
+process. Shutdown process is now improved to also shutdown cleanly when
+not using restart scripts either.
+
+If a server is deadlocked, a server owner can send SIGUP (or any other signal
+the JVM understands to shut down as it currently does) and the watchdog
+will no longer need to wait until the full timeout, allowing you to trigger
+a close process and try to shut the server down gracefully, saving player and
+world data.
+
+Previously there was no way to trigger this outside of waiting for a full watchdog
+timeout, which may be set to a really long time...
+
+diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
+index 2686874f2..a9b533751 100644
+--- a/src/main/java/net/minecraft/server/MinecraftServer.java
++++ b/src/main/java/net/minecraft/server/MinecraftServer.java
+@@ -176,7 +176,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
+     public boolean serverAutoSave = false; // Paper
+     public File bukkitDataPackFolder;
+     public CommandDispatcher vanillaCommandDispatcher;
+-    private boolean forceTicks;
++    public boolean forceTicks; // Paper
+     // CraftBukkit end
+     // Spigot start
+     public static final int TPS = 20;
+@@ -795,6 +795,7 @@ public abstract class MinecraftServer extends IAsyncTaskHandlerReentrant<TickTas
+         }
+         // Spigot end
+         com.destroystokyo.paper.io.PaperFileIOThread.Holder.INSTANCE.close(true, true); // Paper
++        System.exit(0); // Paper
+     }
+ 
+     public String getServerIp() {
+diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
+index aefea3a9a..123de5ac9 100644
+--- a/src/main/java/org/spigotmc/RestartCommand.java
++++ b/src/main/java/org/spigotmc/RestartCommand.java
+@@ -139,7 +139,7 @@ public class RestartCommand extends Command
+     // Paper end
+ 
+     // Paper start - copied from above and modified to return if the hook registered
+-    private static boolean addShutdownHook(String restartScript)
++    public static boolean addShutdownHook(String restartScript)
+     {
+         String[] split = restartScript.split( " " );
+         if ( split.length > 0 && new File( split[0] ).isFile() )
+diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
+index 5bdcdcf9e..704e8426a 100644
+--- a/src/main/java/org/spigotmc/WatchdogThread.java
++++ b/src/main/java/org/spigotmc/WatchdogThread.java
+@@ -69,7 +69,7 @@ public class WatchdogThread extends Thread
+             long currentTime = monotonicMillis();
+             if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") )
+             {
+-                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
++                boolean isLongTimeout = currentTime > lastTick + timeoutTime || !MinecraftServer.getServer().isRunning();
+                 // Don't spam early warning dumps
+                 if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
+                 if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+@@ -135,9 +135,15 @@ public class WatchdogThread extends Thread
+ 
+                 if ( isLongTimeout )
+                 {
+-                if ( restart && !MinecraftServer.getServer().hasStopped() )
++                if ( !MinecraftServer.getServer().hasStopped() )
+                 {
+-                    RestartCommand.restart();
++                    AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
++                    AsyncCatcher.shuttingDown = true;
++                    MinecraftServer.getServer().forceTicks = true;
++                    if (restart) {
++                        RestartCommand.addShutdownHook( SpigotConfig.restartScript );
++                    }
++                    MinecraftServer.getServer().close();
+                 }
+                 break;
+                 } // Paper end
+-- 
+2.25.1
+