Merge pull request 'Implement hard shutdown after 80s' (#249) from 80shardShutdown into master

Reviewed-on: #249
Reviewed-by: YoyoNow <jwsteam@nidido.de>

SpigotCore_Main/src/de/steamwar/core/CrashDetector.java

@@ -31,13 +31,17 @@ public class CrashDetector {
 
     private static final long TIMEOUT = 20000;
     private final AtomicLong lastTick = new AtomicLong(Long.MAX_VALUE);
+    private final AtomicLong lastMessage = new AtomicLong(Long.MAX_VALUE);
     private final Thread mainThread = Thread.currentThread();
     private final Thread watchdog;
 
     private boolean run = true;
 
     public CrashDetector () {
-        Bukkit.getScheduler().runTaskTimer(Core.getInstance(), () -> lastTick.set(System.currentTimeMillis()), 0, 1);
+        Bukkit.getScheduler().runTaskTimer(Core.getInstance(), () -> {
+            lastTick.set(System.currentTimeMillis());
+            lastMessage.set(System.currentTimeMillis());
+        }, 0, 1);
         watchdog = new Thread(this::run, "SteamWar Watchdog");
         watchdog.setDaemon(true);
         watchdog.start();
@@ -56,19 +60,17 @@ public class CrashDetector {
         SWException.init();
         while (run) {
             long curTime = System.currentTimeMillis();
-            if(curTime - TIMEOUT > lastTick.get()) {
+            if(curTime - 4*TIMEOUT >= lastTick.get()) {
+                SWException.log("Server did not recover in " + (curTime - lastTick.get()) + "ms, unclean server stop", "");
+                hardStop();
+            } else if(curTime - TIMEOUT > lastMessage.get()) {
                 if(mainThread.isAlive()) {
                     SWException.log("Server hung for " + (curTime - lastTick.get()) + "ms", Arrays.stream(mainThread.getStackTrace()).map(StackTraceElement::toString).collect(Collectors.joining("\n")));
                 } else {
                     SWException.log("Server thread already dead, unclean server stop", "Core enabled: " + Core.getInstance().isEnabled());
-                    if(Core.getInstance().isEnabled()) {
+                    hardStop();
-                        Core.getInstance().onDisable();
-                    }
-                    Statement.closeAll();
-                    //System.exit(0); Does freeze, potential freezing issues: ConsoleRestoreHook, ApplicationShutdownHooks or DeleteOnExitHook
-                    Runtime.getRuntime().halt(0);
                 }
-                lastTick.set(curTime);
+                lastMessage.set(curTime);
             }
             try {
                 Thread.sleep(1000);
@@ -77,4 +79,13 @@ public class CrashDetector {
             }
         }
     }
+
+    private void hardStop() {
+        if(Core.getInstance().isEnabled()) {
+            Core.getInstance().onDisable();
+        }
+        Statement.closeAll();
+        //System.exit(0); Does freeze, potential freezing issues: ConsoleRestoreHook, ApplicationShutdownHooks or DeleteOnExitHook
+        Runtime.getRuntime().halt(0);
+    }
 }