161 Zeilen
9.9 KiB
Diff
161 Zeilen
9.9 KiB
Diff
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
|
|
From: miclebrick <miclebrick@outlook.com>
|
|
Date: Wed, 8 Aug 2018 15:30:52 -0400
|
|
Subject: [PATCH] Add Early Warning Feature to WatchDog
|
|
|
|
Detect when the server has been hung for a long duration, and start printing
|
|
thread dumps at an interval until the point of crash.
|
|
|
|
This will help diagnose what was going on in that time before the crash.
|
|
|
|
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
index 274f3a213e60c9b67dbc43c8b53efb676b8bd41e..387c423dfe69a3310f8e7aa3dbb7faafc9501e50 100644
|
|
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
|
|
@@ -1052,6 +1052,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
|
|
this.status = this.buildServerStatus();
|
|
|
|
// Spigot start
|
|
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
|
|
Arrays.fill( recentTps, 20 );
|
|
long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
|
|
lastTick = start - TICK_TIME; // Paper
|
|
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
|
|
index e6826cd0a596f063e8737dcde3c8c6c5b3f71970..1a2607d1b257cea65c82c661a6b3d46c1526a775 100644
|
|
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
|
|
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
|
|
@@ -197,6 +197,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
|
|
// Paper start
|
|
paperConfigurations.initializeGlobalConfiguration();
|
|
paperConfigurations.initializeWorldDefaultsConfiguration();
|
|
+ org.spigotmc.WatchdogThread.doStart(org.spigotmc.SpigotConfig.timeoutTime, org.spigotmc.SpigotConfig.restartOnCrash);
|
|
io.papermc.paper.command.PaperCommands.registerCommands(this);
|
|
com.destroystokyo.paper.Metrics.PaperMetrics.startMetrics();
|
|
com.destroystokyo.paper.VersionHistoryManager.INSTANCE.getClass(); // load version history now
|
|
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
index 8d1dc213ee5acaefd02c1a35140befba5e790d02..2cfcfc256acf322c5279fe967f33705fcd958623 100644
|
|
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
|
|
@@ -901,6 +901,7 @@ public final class CraftServer implements Server {
|
|
|
|
@Override
|
|
public void reload() {
|
|
+ org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
|
|
this.reloadCount++;
|
|
this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile());
|
|
this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile());
|
|
@@ -990,6 +991,7 @@ public final class CraftServer implements Server {
|
|
this.enablePlugins(PluginLoadOrder.STARTUP);
|
|
this.enablePlugins(PluginLoadOrder.POSTWORLD);
|
|
this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
|
|
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
|
|
}
|
|
|
|
@Override
|
|
diff --git a/src/main/java/org/spigotmc/SpigotConfig.java b/src/main/java/org/spigotmc/SpigotConfig.java
|
|
index ddc5f2d9aa7dda6aff132392927e3d7e3674dbff..3ac48dafe2300ff4cf4591569fec9ce4916503cd 100644
|
|
--- a/src/main/java/org/spigotmc/SpigotConfig.java
|
|
+++ b/src/main/java/org/spigotmc/SpigotConfig.java
|
|
@@ -229,7 +229,7 @@ public class SpigotConfig
|
|
SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
|
|
SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
|
|
SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
|
|
- WatchdogThread.doStart( timeoutTime, restartOnCrash );
|
|
+ // WatchdogThread.doStart( timeoutTime, restartOnCrash ); // Paper - moved to after paper config initialization
|
|
}
|
|
|
|
public static boolean bungee;
|
|
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
|
|
index a9897c494b3dc56d900356d74030359832febbaa..b47d043144c499b1499f6b4be5a16a3f75c9fcb8 100644
|
|
--- a/src/main/java/org/spigotmc/WatchdogThread.java
|
|
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
|
|
@@ -14,6 +14,10 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
|
|
private static WatchdogThread instance;
|
|
private long timeoutTime;
|
|
private boolean restart;
|
|
+ private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
|
|
+ private final long earlyWarningDelay; // Paper
|
|
+ public static volatile boolean hasStarted; // Paper
|
|
+ private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
|
|
private volatile long lastTick;
|
|
private volatile boolean stopping;
|
|
|
|
@@ -22,6 +26,8 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
|
|
super( "Paper Watchdog Thread" );
|
|
this.timeoutTime = timeoutTime;
|
|
this.restart = restart;
|
|
+ earlyWarningEvery = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningEvery, timeoutTime); // Paper
|
|
+ earlyWarningDelay = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningDelay, timeoutTime); // Paper
|
|
}
|
|
|
|
private static long monotonicMillis()
|
|
@@ -61,9 +67,18 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
|
|
while ( !this.stopping )
|
|
{
|
|
//
|
|
- if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
|
|
+ // Paper start
|
|
+ Logger log = Bukkit.getServer().getLogger();
|
|
+ long currentTime = WatchdogThread.monotonicMillis();
|
|
+ if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
|
|
{
|
|
- Logger log = Bukkit.getServer().getLogger();
|
|
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime;
|
|
+ // Don't spam early warning dumps
|
|
+ if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
|
|
+ if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
|
|
+ lastEarlyWarning = currentTime;
|
|
+ if (isLongTimeout) {
|
|
+ // Paper end
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
|
|
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
|
|
@@ -93,30 +108,46 @@ public final class WatchdogThread extends io.papermc.paper.util.TickThread // Pa
|
|
}
|
|
}
|
|
// Paper end
|
|
+ } else
|
|
+ {
|
|
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
|
|
+ log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
|
|
+ }
|
|
+ // Paper end - Different message for short timeout
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
|
|
io.papermc.paper.chunk.system.scheduling.ChunkTaskScheduler.dumpAllChunkLoadInfo(isLongTimeout); // Paper // Paper - rewrite chunk system
|
|
WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
//
|
|
+ // Paper start - Only print full dump on long timeouts
|
|
+ if ( isLongTimeout )
|
|
+ {
|
|
log.log( Level.SEVERE, "Entire Thread Dump:" );
|
|
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
|
|
for ( ThreadInfo thread : threads )
|
|
{
|
|
WatchdogThread.dumpThread( thread, log );
|
|
}
|
|
+ } else {
|
|
+ log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
|
|
+ }
|
|
+
|
|
log.log( Level.SEVERE, "------------------------------" );
|
|
|
|
+ if ( isLongTimeout )
|
|
+ {
|
|
if ( this.restart && !MinecraftServer.getServer().hasStopped() )
|
|
{
|
|
RestartCommand.restart();
|
|
}
|
|
break;
|
|
+ } // Paper end
|
|
}
|
|
|
|
try
|
|
{
|
|
- sleep( 10000 );
|
|
+ sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
|
|
} catch ( InterruptedException ex )
|
|
{
|
|
interrupt();
|