2021-06-11 14:02:28 +02:00
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Sun, 12 Apr 2020 15:50:48 -0400
Subject: [PATCH] Improved Watchdog Support
Forced Watchdog Crash support and Improve Async Shutdown
If the request to shut down the server is received while we are in
a watchdog hang, immediately treat it as a crash and begin the shutdown
process. Shutdown process is now improved to also shutdown cleanly when
not using restart scripts either.
If a server is deadlocked, a server owner can send SIGUP (or any other signal
the JVM understands to shut down as it currently does) and the watchdog
will no longer need to wait until the full timeout, allowing you to trigger
a close process and try to shut the server down gracefully, saving player and
world data.
Previously there was no way to trigger this outside of waiting for a full watchdog
timeout, which may be set to a really long time...
Additionally, fix everything to do with shutting the server down asynchronously.
Previously, nearly everything about the process was fragile and unsafe. Main might
not have actually been frozen, and might still be manipulating state.
Or, some reuest might ask main to do something in the shutdown but main is dead.
Or worse, other things might start closing down items such as the Console or Thread Pool
before we are fully shutdown.
This change tries to resolve all of these issues by moving everything into the stop
method and guaranteeing only one thread is stopping the server.
We then issue Thread Death to the main thread of another thread initiates the stop process.
We have to ensure Thread Death propagates correctly though to stop main completely.
This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.
This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
are properly accounted for and wont trip watchdog on init.
diff --git a/src/main/java/com/destroystokyo/paper/Metrics.java b/src/main/java/com/destroystokyo/paper/Metrics.java
2021-06-14 03:06:38 +02:00
index e3b74dbdf8e14219a56fab939f3174e0c2f66de6..218f5bafeed8551b55b91c7fccaf6935c8b631ca 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/com/destroystokyo/paper/Metrics.java
+++ b/src/main/java/com/destroystokyo/paper/Metrics.java
@@ -92,7 +92,12 @@ public class Metrics {
* Starts the Scheduler which submits our data every 30 minutes.
*/
private void startSubmitting() {
- final Runnable submitTask = this::submitData;
+ final Runnable submitTask = () -> {
+ if (MinecraftServer.getServer().hasStopped()) {
+ return;
+ }
+ submitData();
+ };
// Many servers tend to restart at a fixed time at xx:00 which causes an uneven distribution of requests on the
// bStats backend. To circumvent this problem, we introduce some randomness into the initial and second delay.
diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
2021-07-07 08:52:40 +02:00
index e3b605695e3b837246f72ccb364af06ea48bda45..62c3c597732e6fb30ed5367d902ea8763507a6b8 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/CrashReport.java
+++ b/src/main/java/net/minecraft/CrashReport.java
2021-07-07 08:52:40 +02:00
@@ -232,6 +232,7 @@ public class CrashReport {
2021-06-11 14:02:28 +02:00
}
public static CrashReport forThrowable(Throwable cause, String title) {
+ if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper
while (cause instanceof CompletionException && cause.getCause() != null) {
cause = cause.getCause();
}
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
2021-07-07 08:52:40 +02:00
index def1edb961ba41819ce30c427b1616578a513433..736cbb3430ba1d8c00ea43a80c978949108d8b6c 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
2021-07-07 08:52:40 +02:00
@@ -297,7 +297,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-14 03:06:38 +02:00
public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>();
2021-06-11 14:02:28 +02:00
public int autosavePeriod;
public Commands vanillaCommandDispatcher;
- private boolean forceTicks;
+ public boolean forceTicks; // Paper
// CraftBukkit end
// Spigot start
public static final int TPS = 20;
2021-07-07 08:52:40 +02:00
@@ -307,6 +307,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
public final SlackActivityAccountant slackActivityAccountant = new SlackActivityAccountant();
// Spigot end
+ public volatile Thread shutdownThread; // Paper
+ public volatile boolean abnormalExit = false; // Paper
+
public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
AtomicReference<S> atomicreference = new AtomicReference();
Thread thread = new Thread(() -> {
2021-07-07 08:52:40 +02:00
@@ -916,6 +919,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
// CraftBukkit start
private boolean hasStopped = false;
+ public volatile boolean hasFullyShutdown = false; // Paper
private final Object stopLock = new Object();
public final boolean hasStopped() {
2021-06-14 03:06:38 +02:00
synchronized (this.stopLock) {
2021-07-07 08:52:40 +02:00
@@ -930,6 +934,19 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-14 03:06:38 +02:00
if (this.hasStopped) return;
this.hasStopped = true;
2021-06-11 14:02:28 +02:00
}
+ // Paper start - kill main thread, and kill it hard
+ shutdownThread = Thread.currentThread();
+ org.spigotmc.WatchdogThread.doStop(); // Paper
+ if (!isSameThread()) {
+ MinecraftServer.LOGGER.info("Stopping main thread (Ignore any thread death message you see! - DO NOT REPORT THREAD DEATH TO PAPER)");
+ while (this.getRunningThread().isAlive()) {
+ this.getRunningThread().stop();
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException e) {}
+ }
+ }
+ // Paper end
// CraftBukkit end
MinecraftServer.LOGGER.info("Stopping server");
MinecraftTimings.stopServer(); // Paper
2021-07-07 08:52:40 +02:00
@@ -995,7 +1012,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-14 03:06:38 +02:00
this.getProfileCache().save(false); // Paper
2021-06-11 14:02:28 +02:00
}
// Spigot end
+ // Paper start - move final shutdown items here
+ LOGGER.info("Flushing Chunk IO");
com.destroystokyo.paper.io.PaperFileIOThread.Holder.INSTANCE.close(true, true); // Paper
+ LOGGER.info("Closing Thread Pool");
2021-06-14 03:06:38 +02:00
+ Util.shutdownExecutors(); // Paper
2021-06-11 14:02:28 +02:00
+ LOGGER.info("Closing Server");
+ try {
+ net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+ } catch (Exception e) {
+ }
+ this.onServerExit();
+ // Paper end
}
public String getLocalIp() {
2021-07-07 08:52:40 +02:00
@@ -1088,6 +1116,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
protected void runServer() {
try {
+ long serverStartTime = Util.getNanos(); // Paper
if (this.initServer()) {
this.nextTickTime = Util.getMillis();
this.status.setDescription(new TextComponent(this.motd));
2021-07-07 08:52:40 +02:00
@@ -1095,6 +1124,18 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
this.updateStatusIcon(this.status);
// Spigot start
+ // Paper start - move done tracking
+ LOGGER.info("Running delayed init tasks");
+ this.server.getScheduler().mainThreadHeartbeat(this.tickCount); // run all 1 tick delay tasks during init,
+ // this is going to be the first thing the tick process does anyways, so move done and run it after
+ // everything is init before watchdog tick.
+ // anything at 3+ won't be caught here but also will trip watchdog....
+ // tasks are default scheduled at -1 + delay, and first tick will tick at 1
+ String doneTime = String.format(java.util.Locale.ROOT, "%.3fs", (double) (Util.getNanos() - serverStartTime) / 1.0E9D);
+ LOGGER.info("Done ({})! For help, type \"help\"", doneTime);
+ // Paper end
+
+ org.spigotmc.WatchdogThread.tick(); // Paper
org.spigotmc.WatchdogThread.hasStarted = true; // Paper
Arrays.fill( recentTps, 20 );
long start = System.nanoTime(), curTime, tickSection = start; // Paper - Further improve server tick loop
2021-07-07 08:52:40 +02:00
@@ -1151,6 +1192,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
this.onServerCrash((CrashReport) null);
}
} catch (Throwable throwable) {
+ // Paper start
+ if (throwable instanceof ThreadDeath) {
+ MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable);
+ return;
+ }
+ // Paper end
MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable);
// Spigot Start
if ( throwable.getCause() != null )
2021-07-07 08:52:40 +02:00
@@ -1186,14 +1233,14 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
} catch (Throwable throwable1) {
MinecraftServer.LOGGER.error("Exception stopping the server", throwable1);
} finally {
- org.spigotmc.WatchdogThread.doStop(); // Spigot
+ //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
// CraftBukkit start - Restore terminal to original settings
try {
- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
} catch (Exception ignored) {
}
// CraftBukkit end
- this.onServerExit();
+ //this.exit(); // Paper - moved into stop
}
}
2021-07-07 08:52:40 +02:00
@@ -1249,6 +1296,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
@Override
public TickTask wrapRunnable(Runnable runnable) {
+ // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
+ if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
+ runnable.run();
+ runnable = () -> {};
+ }
+ // Paper end
return new TickTask(this.tickCount, runnable);
}
2021-07-07 08:52:40 +02:00
@@ -1484,6 +1537,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
try {
crashreport = CrashReport.forThrowable(throwable, "Exception ticking world");
} catch (Throwable t) {
+ if (throwable instanceof ThreadDeath) { throw (ThreadDeath)throwable; } // Paper
throw new RuntimeException("Error generating crash report", t);
}
// Spigot End
2021-07-07 08:52:40 +02:00
@@ -1961,7 +2015,8 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
2021-06-11 14:02:28 +02:00
this.packRepository.setSelected(datapacks);
2021-06-14 03:06:38 +02:00
this.worldData.setDataPackConfig(MinecraftServer.getSelectedPacks(this.packRepository));
2021-06-11 14:02:28 +02:00
datapackresources.updateGlobals();
- this.getPlayerList().saveAll();
+ if (Thread.currentThread() != this.serverThread) return; // Paper
+ //this.getPlayerList().savePlayers(); // Paper - we don't need to do this
this.getPlayerList().reloadResources();
this.functionManager.replaceLibrary(this.resources.getFunctionLibrary());
this.structureManager.onResourceManagerReload(this.resources.getResourceManager());
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
2021-07-07 08:52:40 +02:00
index b6ccc8cacb615a35a60c73f145b7bd1cf0b891ee..a335d48467d1730bfed25eb5fd9046e115f23ed0 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
2021-07-07 08:52:40 +02:00
@@ -285,7 +285,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
2021-06-11 14:02:28 +02:00
long j = Util.getNanos() - i;
String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
- DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
+ //DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s); // Paper moved to after init
if (dedicatedserverproperties.announcePlayerAchievements != null) {
((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, (MinecraftServer) this);
}
2021-07-07 08:52:40 +02:00
@@ -448,7 +448,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
2021-06-11 14:02:28 +02:00
//this.remoteStatusListener.b(); // Paper - don't wait for remote connections
}
- System.exit(0); // CraftBukkit
+ hasFullyShutdown = true; // Paper
+ System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper
}
@Override
2021-07-07 08:52:40 +02:00
@@ -781,7 +782,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
2021-06-11 14:02:28 +02:00
@Override
public void stopServer() {
super.stopServer();
- Util.shutdownExecutors();
+ //SystemUtils.h(); // Paper - moved into super
}
@Override
diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java
2021-07-07 08:52:40 +02:00
index 962028a58ee54b99be20905c1fd16cfe15c82175..6d810cdb538d078dbf7ccd2ef84a4be27eb1f3e7 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/level/ChunkMap.java
+++ b/src/main/java/net/minecraft/server/level/ChunkMap.java
2021-07-07 08:52:40 +02:00
@@ -613,6 +613,7 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider
2021-06-11 14:02:28 +02:00
MutableBoolean mutableboolean = new MutableBoolean();
do {
+ boolean isShuttingDown = level.getServer().hasStopped(); // Paper
mutableboolean.setFalse();
list.stream().map((playerchunk) -> {
CompletableFuture completablefuture;
diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java
2021-07-07 08:52:40 +02:00
index 6f3187aae34c24c6a9b22074d487006926afe524..11079851f8a375d987933b53409b789030b3220d 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/server/players/PlayerList.java
+++ b/src/main/java/net/minecraft/server/players/PlayerList.java
2021-07-07 08:52:40 +02:00
@@ -513,7 +513,7 @@ public abstract class PlayerList {
2021-06-14 03:06:38 +02:00
this.cserver.getPluginManager().callEvent(playerQuitEvent);
2021-06-11 14:02:28 +02:00
entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
- entityplayer.doTick(); // SPIGOT-924
+ if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
// CraftBukkit end
// Paper start - Remove from collideRule team if needed
diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
2021-06-16 02:50:38 +02:00
index 7bf4bf5cb2c1b54a7e2733091f48f3a824336d36..dcce05d2f4ab16424db4ab103a12188e207a457b 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
2021-06-14 03:06:38 +02:00
@@ -148,6 +148,7 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler
2021-06-11 14:02:28 +02:00
try {
task.run();
2021-06-14 03:06:38 +02:00
} catch (Exception var3) {
+ if (var3.getCause() instanceof ThreadDeath) throw var3; // Paper
LOGGER.fatal("Error executing task on {}", this.name(), var3);
2021-06-11 14:02:28 +02:00
}
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
2021-07-07 08:52:40 +02:00
index fd23c7913b7a426443515e14be0ac9814a5e934e..9cf64082555d848e4149f9a982dd770db9defa3c 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
2021-07-07 08:52:40 +02:00
@@ -848,6 +848,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable {
2021-06-11 14:02:28 +02:00
try {
tickConsumer.accept(entity);
} catch (Throwable throwable) {
+ if (throwable instanceof ThreadDeath) throw throwable; // Paper
// Paper start - Prevent tile entity and entity crashes
2021-06-21 10:09:18 +02:00
final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level.getWorld().getName(), entity.getX(), entity.getY(), entity.getZ());
MinecraftServer.LOGGER.error(msg, throwable);
2021-06-14 03:06:38 +02:00
diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
2021-06-29 02:31:35 +02:00
index fcacaa9e41c2418484d2fe9e7952be47943a2177..7d33ad1ec939de4527d0acb0915f05870c387565 100644
2021-06-14 03:06:38 +02:00
--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
2021-06-16 22:12:05 +02:00
@@ -1319,6 +1319,7 @@ public class LevelChunk implements ChunkAccess {
2021-06-14 03:06:38 +02:00
gameprofilerfiller.pop();
} catch (Throwable throwable) {
+ if (throwable instanceof ThreadDeath) throw throwable; // Paper
// Paper start - Prevent tile entity and entity crashes
2021-06-21 10:09:18 +02:00
final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ());
net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable);
2021-06-11 14:02:28 +02:00
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftServer.java b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2021-07-07 08:52:40 +02:00
index e3fc72a86aea884b9bd85803baf8cd2c0bb7eb1c..7af2baea88a29d43afffbaa11ce57d69ca606be0 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/CraftServer.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftServer.java
2021-06-29 02:31:35 +02:00
@@ -1845,7 +1845,7 @@ public final class CraftServer implements Server {
2021-06-11 14:02:28 +02:00
@Override
public boolean isPrimaryThread() {
- return Thread.currentThread().equals(console.serverThread); // Paper - Fix issues with detecting main thread properly
+ return Thread.currentThread().equals(console.serverThread) || Thread.currentThread().equals(net.minecraft.server.MinecraftServer.getServer().shutdownThread); // Paper - Fix issues with detecting main thread properly, the only time Watchdog will be used is during a crash shutdown which is a "try our best" scenario
}
// Paper start
diff --git a/src/main/java/org/bukkit/craftbukkit/Main.java b/src/main/java/org/bukkit/craftbukkit/Main.java
2021-06-17 21:12:40 +02:00
index 3f219234355edc961f07ae1a75759b6e44ddcfdf..15d6975e6470e1affad9adc73964a720a3de36e9 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/Main.java
+++ b/src/main/java/org/bukkit/craftbukkit/Main.java
@@ -12,6 +12,8 @@ import java.util.logging.Level;
import java.util.logging.Logger;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
+import net.minecraft.util.ExceptionCollector;
+import net.minecraft.world.level.lighting.LayerLightEventListener;
import net.minecrell.terminalconsole.TerminalConsoleAppender; // Paper
public class Main {
@@ -156,6 +158,36 @@ public class Main {
OptionSet options = null;
+ // Paper start - preload logger classes to avoid plugins mixing versions
+ tryPreloadClass("com.destroystokyo.paper.log.LogFullPolicy");
+ tryPreloadClass("org.apache.logging.log4j.core.Core");
+ tryPreloadClass("org.apache.logging.log4j.core.Appender");
+ tryPreloadClass("org.apache.logging.log4j.core.ContextDataInjector");
+ tryPreloadClass("org.apache.logging.log4j.core.Filter");
+ tryPreloadClass("org.apache.logging.log4j.core.ErrorHandler");
+ tryPreloadClass("org.apache.logging.log4j.core.LogEvent");
+ tryPreloadClass("org.apache.logging.log4j.core.Logger");
+ tryPreloadClass("org.apache.logging.log4j.core.LoggerContext");
+ tryPreloadClass("org.apache.logging.log4j.core.LogEventListener");
+ tryPreloadClass("org.apache.logging.log4j.core.AbstractLogEvent");
+ tryPreloadClass("org.apache.logging.log4j.message.AsynchronouslyFormattable");
+ tryPreloadClass("org.apache.logging.log4j.message.FormattedMessage");
+ tryPreloadClass("org.apache.logging.log4j.message.ParameterizedMessage");
+ tryPreloadClass("org.apache.logging.log4j.message.Message");
+ tryPreloadClass("org.apache.logging.log4j.message.MessageFactory");
+ tryPreloadClass("org.apache.logging.log4j.message.TimestampMessage");
+ tryPreloadClass("org.apache.logging.log4j.message.SimpleMessage");
+ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLogger");
+ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerContext");
+ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncQueueFullPolicy");
+ tryPreloadClass("org.apache.logging.log4j.core.async.AsyncLoggerDisruptor");
+ tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEvent");
+ tryPreloadClass("org.apache.logging.log4j.core.async.DisruptorUtil");
+ tryPreloadClass("org.apache.logging.log4j.core.async.RingBufferLogEventHandler");
+ tryPreloadClass("org.apache.logging.log4j.core.impl.ThrowableProxy");
+ tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedClassInfo");
+ tryPreloadClass("org.apache.logging.log4j.core.impl.ExtendedStackTraceElement");
+ // Paper end
try {
options = parser.parse(args);
} catch (joptsimple.OptionException ex) {
2021-06-14 03:06:38 +02:00
@@ -255,8 +287,64 @@ public class Main {
2021-06-11 14:02:28 +02:00
} catch (Throwable t) {
t.printStackTrace();
}
+ // Paper start
+ // load some required classes to avoid errors during shutdown if jar is replaced
+ // also to guarantee our version loads over plugins
+ tryPreloadClass("com.destroystokyo.paper.util.SneakyThrow");
+ tryPreloadClass("com.google.common.collect.Iterators$PeekingImpl");
+ tryPreloadClass("com.google.common.collect.MapMakerInternalMap$Values");
+ tryPreloadClass("com.google.common.collect.MapMakerInternalMap$ValueIterator");
+ tryPreloadClass("com.google.common.collect.MapMakerInternalMap$WriteThroughEntry");
+ tryPreloadClass("com.google.common.collect.Iterables");
+ for (int i = 1; i <= 15; i++) {
+ tryPreloadClass("com.google.common.collect.Iterables$" + i, false);
+ }
+ tryPreloadClass("org.apache.commons.lang3.mutable.MutableBoolean");
+ tryPreloadClass("org.apache.commons.lang3.mutable.MutableInt");
+ tryPreloadClass("org.jline.terminal.impl.MouseSupport");
+ tryPreloadClass("org.jline.terminal.impl.MouseSupport$1");
+ tryPreloadClass("org.jline.terminal.Terminal$MouseTracking");
+ tryPreloadClass("co.aikar.timings.TimingHistory");
+ tryPreloadClass("co.aikar.timings.TimingHistory$MinuteReport");
+ tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext");
+ tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$11");
+ tryPreloadClass("io.netty.channel.AbstractChannelHandlerContext$12");
+ tryPreloadClass("io.netty.channel.AbstractChannel$AbstractUnsafe$8");
+ tryPreloadClass("io.netty.util.concurrent.DefaultPromise");
+ tryPreloadClass("io.netty.util.concurrent.DefaultPromise$1");
+ tryPreloadClass("io.netty.util.internal.PromiseNotificationUtil");
+ tryPreloadClass("io.netty.util.internal.SystemPropertyUtil");
+ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler");
+ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$1");
+ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$2");
+ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$3");
+ tryPreloadClass("org.bukkit.craftbukkit.scheduler.CraftScheduler$4");
+ tryPreloadClass("org.slf4j.helpers.MessageFormatter");
+ tryPreloadClass("org.slf4j.helpers.FormattingTuple");
+ tryPreloadClass("org.slf4j.helpers.BasicMarker");
+ tryPreloadClass("org.slf4j.helpers.Util");
+ tryPreloadClass("com.destroystokyo.paper.event.player.PlayerConnectionCloseEvent");
+ tryPreloadClass("com.destroystokyo.paper.event.entity.EntityRemoveFromWorldEvent");
+ // Minecraft, seen during saving
+ tryPreloadClass(LayerLightEventListener.DummyLightLayerEventListener.class.getName());
+ tryPreloadClass(LayerLightEventListener.class.getName());
+ tryPreloadClass(ExceptionCollector.class.getName());
+ // Paper end
+ }
+ }
+
+ // Paper start
+ private static void tryPreloadClass(String className) {
+ tryPreloadClass(className, true);
+ }
+ private static void tryPreloadClass(String className, boolean printError) {
+ try {
+ Class.forName(className);
+ } catch (ClassNotFoundException e) {
+ if (printError) System.err.println("An expected class " + className + " was not found for preloading: " + e.getMessage());
}
}
+ // Paper end
private static List<String> asList(String... params) {
return Arrays.asList(params);
diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
2021-06-14 03:06:38 +02:00
index b4a19d80bbf71591f25729fd0e98590350cb31d0..d752720f2f234b9dbd2117333fee1bfad663ec02 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
@@ -12,12 +12,27 @@ public class ServerShutdownThread extends Thread {
@Override
public void run() {
try {
+ // Paper start - try to shutdown on main
+ server.safeShutdown(false, false);
+ for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
+ Thread.sleep(100);
+ }
+ if (server.hasStopped()) {
+ while (!server.hasFullyShutdown) Thread.sleep(1000);
+ return;
+ }
+ // Looks stalled, close async
org.spigotmc.AsyncCatcher.enabled = false; // Spigot
org.spigotmc.AsyncCatcher.shuttingDown = true; // Paper
+ server.forceTicks = true;
2021-06-14 03:06:38 +02:00
this.server.close();
2021-06-11 14:02:28 +02:00
+ while (!server.hasFullyShutdown) Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ // Paper end
} finally {
try {
- net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+ //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
} catch (Exception e) {
}
}
diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
2021-06-14 03:06:38 +02:00
index a142a56a920e153ed84c08cece993f10d76f7793..92d97a5810a379b427a99b4c63fb9844d823a84f 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/spigotmc/RestartCommand.java
+++ b/src/main/java/org/spigotmc/RestartCommand.java
@@ -139,7 +139,7 @@ public class RestartCommand extends Command
// Paper end
// Paper start - copied from above and modified to return if the hook registered
- private static boolean addShutdownHook(String restartScript)
+ public static boolean addShutdownHook(String restartScript)
{
String[] split = restartScript.split( " " );
if ( split.length > 0 && new File( split[0] ).isFile() )
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
2021-06-14 03:06:38 +02:00
index 1ffb208094f521883ef0e23baf5fb29380b14273..4d271cae88c16ed2419f896c728fdff612540500 100644
2021-06-11 14:02:28 +02:00
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
2021-06-14 03:06:38 +02:00
@@ -12,6 +12,7 @@ import org.bukkit.Bukkit;
2021-06-11 14:02:28 +02:00
public class WatchdogThread extends Thread
{
+ public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper
private static WatchdogThread instance;
private long timeoutTime;
private boolean restart;
2021-06-14 03:06:38 +02:00
@@ -40,6 +41,7 @@ public class WatchdogThread extends Thread
2021-06-11 14:02:28 +02:00
{
2021-06-14 03:06:38 +02:00
if ( WatchdogThread.instance == null )
2021-06-11 14:02:28 +02:00
{
+ if (timeoutTime <= 0) timeoutTime = 300; // Paper
2021-06-14 03:06:38 +02:00
WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
WatchdogThread.instance.start();
2021-06-11 14:02:28 +02:00
} else
@@ -71,12 +73,13 @@ public class WatchdogThread extends Thread
// Paper start
Logger log = Bukkit.getServer().getLogger();
2021-06-14 03:06:38 +02:00
long currentTime = WatchdogThread.monotonicMillis();
- if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
2021-06-11 14:02:28 +02:00
+ MinecraftServer server = MinecraftServer.getServer();
2021-06-14 03:06:38 +02:00
+ if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable
2021-06-11 14:02:28 +02:00
{
- boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+ boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
// Don't spam early warning dumps
if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
- if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+ if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
lastEarlyWarning = currentTime;
if (isLongTimeout) {
// Paper end
@@ -118,7 +121,7 @@ public class WatchdogThread extends Thread
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
2021-06-14 03:06:38 +02:00
com.destroystokyo.paper.io.chunk.ChunkTaskManager.dumpAllChunkLoadInfo(); // Paper
- WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
+ WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( server.serverThread.getId(), Integer.MAX_VALUE ), log );
2021-06-11 14:02:28 +02:00
log.log( Level.SEVERE, "------------------------------" );
//
// Paper start - Only print full dump on long timeouts
2021-06-14 03:06:38 +02:00
@@ -138,9 +141,25 @@ public class WatchdogThread extends Thread
2021-06-11 14:02:28 +02:00
if ( isLongTimeout )
{
2021-06-14 03:06:38 +02:00
- if ( this.restart && !MinecraftServer.getServer().hasStopped() )
2021-06-11 14:02:28 +02:00
+ if ( !server.hasStopped() )
{
- RestartCommand.restart();
+ AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
+ AsyncCatcher.shuttingDown = true;
+ server.forceTicks = true;
+ if (restart) {
+ RestartCommand.addShutdownHook( SpigotConfig.restartScript );
+ }
+ // try one last chance to safe shutdown on main incase it 'comes back'
+ server.abnormalExit = true;
+ server.safeShutdown(false, restart);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ if (!server.hasStopped()) {
+ server.close();
+ }
}
break;
} // Paper end
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
index 476f4a5cbe664ddd05474cb88553018bd334a5b8..8af159abd3d0cc94cf155fec5b384c42f69551bf 100644
--- a/src/main/resources/log4j2.xml
+++ b/src/main/resources/log4j2.xml
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="WARN" packages="com.mojang.util">
+<Configuration status="WARN" packages="com.mojang.util" shutdownHook="disable">
<Appenders>
<Queue name="ServerGuiConsole">
<PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg%n" />
