Merge pull request 'Checkpoint support' (#506) from checkpoint into master

Reviewed-on: #506
Reviewed-by: YoyoNow <jwsteam@nidido.de>

src/de/steamwar/bungeecore/EventStarter.java

@@ -54,7 +54,7 @@ public class EventStarter implements Runnable {
 
     @Override
     public void run() {
-        eventServer.entrySet().removeIf(entry -> !Subserver.getServerList().contains(entry.getValue()));
+        eventServer.entrySet().removeIf(entry -> Subserver.getSubserver(entry.getValue().getServer()) == null);
         Queue<EventFight> fights = EventFight.getFights();
 
         EventFight next;
@@ -70,10 +70,11 @@ public class EventStarter implements Runnable {
                 if(leiter != null)
                     starter.send(leiter);
 
-                Subserver subserver = starter.start();
+                starter.callback(subserver -> {
+                    eventServer.put(blue.getTeamId(), subserver);
+                    eventServer.put(red.getTeamId(), subserver);
+                }).start();
 
-                eventServer.put(blue.getTeamId(), subserver);
-                eventServer.put(red.getTeamId(), subserver);
                 Message.broadcast("EVENT_FIGHT_BROADCAST", "EVENT_FIGHT_BROADCAST_HOVER",
                         new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/event " + blue.getTeamKuerzel()), blue.getTeamColor(), blue.getTeamName(), red.getTeamColor(), red.getTeamName());
             } else {

src/de/steamwar/bungeecore/Node.java

@@ -23,6 +23,7 @@ import net.md_5.bungee.BungeeCord;
 import net.md_5.bungee.api.ProxyServer;
 
 import java.io.*;
+import java.nio.file.Files;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
@@ -30,7 +31,12 @@ import java.util.logging.Level;
 
 public abstract class Node {
 
-    private static final List<String> OPENJ9_ARGS = Arrays.asList("-Xgc:excessiveGCratio=80", "-Xsyslog:none", "-Xtrace:none", "-Xdisableexplicitgc", "-XX:+AlwaysPreTouch", "-XX:+CompactStrings", "-XX:-HeapDumpOnOutOfMemory", "-XX:+ExitOnOutOfMemoryError", "-Dlog4j.configurationFile=log4j2.xml", "-Dio.netty.allocator.numHeapArenas=1", "-Xnoclassgc", "-Xmos128M", "-Xmns48M"); // initial heap half values of memory observed by 1.19 spectate server
+    private static final List<String> OPENJ9_ARGS = Arrays.asList(
+            "-XX:+EnableCRIUSupport", "-XX:-CRIURestoreNonPortableMode",
+            "-Xgc:excessiveGCratio=80", "-Xdisableexplicitgc", "-Xnoclassgc", "-Xmos128M", "-Xmns48M", "-XX:+ExitOnOutOfMemoryError", // initial heap half values of memory observed by 1.19 spectate server
+            "-Xsyslog:none", "-Xtrace:none", "-Xverify:none",
+            "-Dlog4j.configurationFile=log4j2.xml"
+    );
     private static final Set<String> JAVA_8 = new HashSet<>();
     static {
         JAVA_8.add("paper-1.8.8.jar");
@@ -41,7 +47,6 @@ public abstract class Node {
     }
 
     private static final long MIN_FREE_MEM = 4 * 1024 * 1024L; // 4 GiB
-    private static final double MAX_LOAD = 0.8;
 
     private static final List<Node> nodes = new ArrayList<>();
 
@@ -65,9 +70,6 @@ public abstract class Node {
     protected final String hostname;
     protected volatile boolean belowLoadLimit = true;
 
-    private long previousCpuLoaded = 0;
-    private long previousCpuTotal = 0;
-
     protected Node(String hostname) {
         this.hostname = hostname;
         nodes.add(this);
@@ -101,12 +103,13 @@ public abstract class Node {
         for(String param : dParams){
             cmd.add("-D" + param);
         }
-        cmd.add("-Xmx" + xmx);
         cmd.add("-Xshareclasses:nonfatal,name=" + directory.getName());
+        cmd.add("-Xmx" + xmx);
         cmd.addAll(OPENJ9_ARGS);
         if (!JAVA_8.contains(serverJar)) {
             cmd.add("--add-opens");
             cmd.add("java.base/jdk.internal.misc=ALL-UNNAMED");
+            cmd.add("-XX:-CRIUSecProvider");
         }
         cmd.add("-jar");
         cmd.add("/binarys/" + serverJar);
@@ -120,28 +123,18 @@ public abstract class Node {
         cmd.add("nogui");
     }
 
-    protected void calcLoadLimit(BufferedReader stat, BufferedReader meminfo) throws IOException {
+    protected void calcLoadLimit(BufferedReader meminfo) throws IOException {
-        String[] cpuline = stat.readLine().split(" "); // 0-1: prefix, 2: user, 3: nice, 4: system, 5: idle, 6: iowait, 7: irq, 8: softirq, 9: steal, 10: guest, 11: guest_nice
-        long cpuLoaded = Long.parseLong(cpuline[2]) + Long.parseLong(cpuline[4]) + Long.parseLong(cpuline[6]) + Long.parseLong(cpuline[7]) + Long.parseLong(cpuline[8]) + Long.parseLong(cpuline[9]) + Long.parseLong(cpuline[10]) + Long.parseLong(cpuline[11]);
-        long cpuTotal = cpuLoaded + Long.parseLong(cpuline[3]) + Long.parseLong(cpuline[5]);
-
-        cpuLoaded -= previousCpuLoaded;
-        cpuTotal -= previousCpuTotal;
-        previousCpuLoaded += cpuLoaded;
-        previousCpuTotal += cpuTotal;
-
         String line = meminfo.readLine();
         while(!line.startsWith("MemAvailable")) {
             line = meminfo.readLine();
         }
 
         long availableMem = Long.parseLong(line.replaceAll(" +", " ").split(" ")[1]);
-        belowLoadLimit = cpuLoaded / (double)cpuTotal <= MAX_LOAD && availableMem >= MIN_FREE_MEM;
+        belowLoadLimit = availableMem >= MIN_FREE_MEM;
     }
 
     public static class LocalNode extends Node {
         private static final File MEMINFO = new File("/proc/meminfo");
-        private static final File STAT = new File("/proc/stat");
 
         public LocalNode() {
             super("sw");
@@ -158,10 +151,8 @@ public abstract class Node {
 
         @Override
         protected void calcLoadLimit() {
-            try (BufferedReader loadavg = new BufferedReader(new InputStreamReader(new FileInputStream(STAT)))) {
+            try (BufferedReader meminfo = new BufferedReader(new InputStreamReader(Files.newInputStream(MEMINFO.toPath())))) {
-                try (BufferedReader meminfo = new BufferedReader(new InputStreamReader(new FileInputStream(MEMINFO)))) {
+                calcLoadLimit(meminfo);
-                    calcLoadLimit(loadavg, meminfo);
-                }
             } catch (IOException e) {
                 BungeeCore.get().getLogger().log(Level.SEVERE, "Could not read local load", e);
                 belowLoadLimit = false;
@@ -207,13 +198,11 @@ public abstract class Node {
         @Override
         protected void calcLoadLimit() {
             try {
-                Process process = prepareExecution("cat /proc/stat /proc/meminfo").start();
+                Process process = prepareExecution("cat /proc/meminfo").start();
                 if(!process.waitFor(1, TimeUnit.SECONDS))
                     throw new IOException(hostname + " timeout");
                 try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {
-                    if(reader == null)
+                    calcLoadLimit(reader);
-                        throw new IOException("Why reader null? " + process.getInputStream());
-                    calcLoadLimit(reader, reader);
                 }
             } catch (IOException e) {
                 if(belowLoadLimit)

src/de/steamwar/bungeecore/ServerStarter.java

@@ -1,12 +1,14 @@
 package de.steamwar.bungeecore;
 
+import de.steamwar.messages.ChatSender;
 import de.steamwar.sql.EventFight;
 import de.steamwar.sql.SteamwarUser;
 import de.steamwar.sql.Team;
 import de.steamwar.sql.Tutorial;
-import de.steamwar.messages.ChatSender;
+import lombok.Getter;
 import net.md_5.bungee.api.connection.ProxiedPlayer;
 
+import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -14,6 +16,7 @@ import java.nio.file.Files;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
 import java.util.function.BooleanSupplier;
+import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -46,7 +49,9 @@ public class ServerStarter {
     private boolean allowMerge = false;
     private String fightMap = null;
     private String gameMode = null;
-    private ServerConstructor constructor = (serverName, port, builder, shutdownCallback) -> new Arenaserver(serverName, gameMode, fightMap, allowMerge, port, builder, shutdownCallback);
+    private boolean checkpoint = false;
+    private ServerConstructor constructor = (serverName, port, builder, shutdownCallback, failureCallback) -> new Arenaserver(serverName, gameMode, fightMap, allowMerge, port, builder, shutdownCallback);
+    private Consumer<Subserver> callback = subserver -> {};
 
     private final Set<ProxiedPlayer> playersToSend = new HashSet<>();
     private final Map<String, String> arguments = new HashMap<>();
@@ -117,6 +122,7 @@ public class ServerStarter {
         serverJar = version.getServerJar();
         worldDir = version.getWorldFolder(WORLDS_BASE_PATH);
         worldName = version != Version.SPIGOT_12 ? String.valueOf(SteamwarUser.get(owner).getId()) : owner.toString();
+        checkpoint = true;
 
         build(owner);
 
@@ -128,12 +134,11 @@ public class ServerStarter {
 
         // Send players to existing server
         startCondition = () -> {
-            for(Subserver subserver : Subserver.getServerList()) {
+            Bauserver subserver = Bauserver.get(owner);
-                if(subserver.getType() == Servertype.BAUSERVER && ((Bauserver)subserver).getOwner().equals(owner)) {
+            if(subserver != null) {
-                    for(ProxiedPlayer p : playersToSend)
+                for(ProxiedPlayer p : playersToSend)
-                        SubserverSystem.sendPlayer(subserver, p);
+                    SubserverSystem.sendPlayer(subserver, p);
-                    return false;
+                return false;
-                }
             }
             return true;
         };
@@ -163,19 +168,16 @@ public class ServerStarter {
             if(startingBau(owner))
                 return false;
 
-            for (Subserver subserver : Subserver.getServerList()) {
+            Bauserver subserver = Bauserver.get(owner.getUniqueId());
-                if (subserver.getType() == Servertype.BAUSERVER && ((Bauserver) subserver).getOwner().equals(owner.getUniqueId()) && subserver.hasStarted()) {
+            if(subserver != null && subserver.isStarted())
-                    subserver.stop();
+                subserver.stop();
-                    break;
-                }
-            }
 
             return !startingBau(owner);
         };
     }
 
     private void build(UUID owner) {
-        constructor = (serverName, port, builder, shutdownCallback) -> new Bauserver(serverName, owner, port, builder, shutdownCallback);
+        constructor = (serverName, port, builder, shutdownCallback, failureCallback) -> new Bauserver(serverName, owner, port, builder, shutdownCallback, failureCallback);
         serverNameProvider = port -> bauServerName(SteamwarUser.get(owner));
     }
 
@@ -185,16 +187,16 @@ public class ServerStarter {
         worldDir = version.getWorldFolder(BUILDER_BASE_PATH);
         worldName = map;
         serverNameProvider = port -> "⛏" + map;
-        constructor = (serverName, port, builder, shutdownCallback) -> new Builderserver(serverName, worldName, port, builder, shutdownCallback);
+        checkpoint = true;
+        constructor = (serverName, port, builder, shutdownCallback, failureCallback) -> new Builderserver(serverName, worldName, port, builder, shutdownCallback, failureCallback);
 
         // Send players to existing server
         startCondition = () -> {
-            for(Subserver subserver : Subserver.getServerList()) {
+            Builderserver subserver = Builderserver.get(worldName);
-                if(subserver.getType() == Servertype.BUILDER && ((Builderserver)subserver).getMap().equals(worldName)) {
+            if(subserver != null) {
-                    for(ProxiedPlayer p : playersToSend)
+                for(ProxiedPlayer p : playersToSend)
-                        SubserverSystem.sendPlayer(subserver, p);
+                    SubserverSystem.sendPlayer(subserver, p);
-                    return false;
+                return false;
-                }
             }
             return true;
         };
@@ -218,9 +220,14 @@ public class ServerStarter {
         return this;
     }
 
-    public Subserver start() {
+    public ServerStarter callback(Consumer<Subserver> callback) {
+        this.callback = callback;
+        return this;
+    }
+
+    public boolean start() {
         if(!startCondition.getAsBoolean())
-            return null;
+            return false;
 
         int port = portrange.freePort();
         String serverName = serverNameProvider.apply(port);
@@ -231,7 +238,7 @@ public class ServerStarter {
                 for (ProxiedPlayer p : playersToSend)
                     ChatSender.of(p).system("SERVER_START_OVERLOAD");
 
-                return null;
+                return false;
             }
         }
         if(worldName == null)
@@ -240,22 +247,45 @@ public class ServerStarter {
         worldSetup.run();
         arguments.put("logPath", worldName);
 
-        Subserver subserver = constructor.construct(serverName, port, node.startServer(
+        File checkpointDir = new File("/tmp/" + System.getProperty("user.name") + ".checkpoints/" + directory.getName() + "/" + worldName);
-                serverJar, directory, worldDir, worldName, port, xmx, arguments.entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue()).toArray(String[]::new)
+        if(checkpoint)
-        ), worldCleanup);
+            arguments.put("checkpoint", checkpointDir.getPath());
 
+        if(checkpoint && checkpointDir.exists()) {
+            try {
+                new DataOutputStream(Files.newOutputStream(new File(checkpointDir, "port").toPath())).writeInt(port);
+            } catch (IOException e) {
+                throw new SecurityException(e);
+            }
+
+            postStart(constructor.construct(serverName, port, node.prepareExecution(
+                    "criu", "restore", "-D", checkpointDir.getPath(), "--auto-dedup", "--shell-job", "-v"
+            ), worldCleanup, e -> regularStart(serverName, port)));
+        } else {
+            regularStart(serverName, port);
+        }
+
+        return true;
+    }
+
+    private void regularStart(String serverName, int port) {
+        postStart(constructor.construct(serverName, port, node.startServer(
+                serverJar, directory, worldDir, worldName, port, xmx, arguments.entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue()).toArray(String[]::new)
+        ), worldCleanup, null));
+    }
+
+    private void postStart(Subserver subserver) {
         for(ProxiedPlayer p : playersToSend)
             SubserverSystem.sendPlayer(subserver, p);
 
-        return subserver;
+        callback.accept(subserver);
     }
 
     private static boolean startingBau(ProxiedPlayer p) {
-        for (Subserver subserver : Subserver.getServerList()) {
+        Bauserver subserver = Bauserver.get(p.getUniqueId());
-            if (subserver.getType() == Servertype.BAUSERVER && ((Bauserver) subserver).getOwner().equals(p.getUniqueId()) && !subserver.hasStarted()) {
+        if(subserver != null && !subserver.isStarted()) {
-                Message.send("BAU_START_ALREADY", p);
+            ChatSender.of(p).system("BAU_START_ALREADY");
-                return true;
+            return true;
-            }
         }
         return false;
     }
@@ -273,7 +303,7 @@ public class ServerStarter {
     }
 
     private interface ServerConstructor {
-        Subserver construct(String serverName, int port, ProcessBuilder builder, Runnable shutdownCallback);
+        Subserver construct(String serverName, int port, ProcessBuilder builder, Runnable shutdownCallback, Consumer<Exception> failureCallback);
     }
 
     private static class Portrange {
@@ -312,6 +342,7 @@ public class ServerStarter {
         }
     }
 
+    @Getter
     public enum Version {
         SPIGOT_12("spigot-1.12.2.jar", 12),
         SPIGOT_15("spigot-1.15.2.jar", 15),
@@ -326,14 +357,6 @@ public class ServerStarter {
             this.versionSuffix = versionSuffix;
         }
 
-        public String getServerJar() {
-            return serverJar;
-        }
-
-        public int getVersionSuffix() {
-            return versionSuffix;
-        }
-
         public String getWorldFolder(String base) {
             return base + versionSuffix + "/";
         }

src/de/steamwar/bungeecore/commands/BauCommand.java

@@ -68,17 +68,17 @@ public class BauCommand extends SWCommand {
         Map<String, ServerStarter.Version> versionMap = new HashMap<>();
         versionMap.put("20", ServerStarter.Version.PAPER_20);
         versionMap.put("1.20", ServerStarter.Version.PAPER_20);
+        versionMap.put("as", ServerStarter.Version.PAPER_20);
+        versionMap.put("airship", ServerStarter.Version.PAPER_20);
+        versionMap.put("wg", ServerStarter.Version.PAPER_20);
+        versionMap.put("wargear", ServerStarter.Version.PAPER_20);
+        versionMap.put("ws", ServerStarter.Version.PAPER_20);
+        versionMap.put("warship", ServerStarter.Version.PAPER_20);
 
         versionMap.put("19", ServerStarter.Version.PAPER_19);
         versionMap.put("1.19", ServerStarter.Version.PAPER_19);
         versionMap.put("mwg", ServerStarter.Version.PAPER_19);
         versionMap.put("miniwargear", ServerStarter.Version.PAPER_19);
-        versionMap.put("wg", ServerStarter.Version.PAPER_19);
-        versionMap.put("wargear", ServerStarter.Version.PAPER_19);
-        versionMap.put("ws", ServerStarter.Version.PAPER_19);
-        versionMap.put("warship", ServerStarter.Version.PAPER_19);
-        versionMap.put("as", ServerStarter.Version.PAPER_19);
-        versionMap.put("airship", ServerStarter.Version.PAPER_19);
 
         versionMap.put("15", ServerStarter.Version.SPIGOT_15);
         versionMap.put("1.15", ServerStarter.Version.SPIGOT_15);
@@ -294,12 +294,10 @@ public class BauCommand extends SWCommand {
 
     private static void deleteWorld(ProxiedPlayer player, String world) {
         ProxyServer.getInstance().getScheduler().runAsync(BungeeCore.get(), () -> {
-            for (Subserver subserver : Subserver.getServerList()) {
+            Bauserver subserver = Bauserver.get(player.getUniqueId());
-                if (subserver.getType() == Servertype.BAUSERVER && ((Bauserver) subserver).getOwner().equals(player.getUniqueId())) {
+            if(subserver != null)
-                    subserver.stop();
+                subserver.stop();
-                    break;
+
-                }
-            }
             SubserverSystem.deleteFolder(BungeeCore.local, world);
             Message.send("BAU_DELETE_DELETED", player);
         });

src/de/steamwar/bungeecore/commands/ChallengeCommand.java

@@ -43,11 +43,9 @@ public class ChallengeCommand extends SWCommand {
                 challenges.remove(target);
                 challenges.remove(p);
 
-                Subserver arena = new ServerStarter().arena(mode, map).blueLeader(player).redLeader(target).start();
+                new ServerStarter().arena(mode, map).blueLeader(player).redLeader(target).callback(
-                if(arena != null) {
+                        arena -> Message.broadcast("CHALLENGE_BROADCAST", "CHALLENGE_BROADCAST_HOVER", new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/arena " + arena.getServer().getName()), mode.getDisplayName(), p.getName(), target.getName())
-                    Message.broadcast("CHALLENGE_BROADCAST", "CHALLENGE_BROADCAST_HOVER",
+                ).start();
-                            new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/arena " + arena.getServer().getName()), mode.getDisplayName(), p.getName(), target.getName());
-                }
             }else{
                 if(!challenges.containsKey(p)){
                     challenges.put(p, new LinkedList<>());

src/de/steamwar/bungeecore/commands/CheckCommand.java

@@ -222,7 +222,7 @@ public class CheckCommand extends SWCommand {
 
             ProxyServer.getInstance().getScheduler().runAsync(BungeeCore.get(), () -> {
                 ArenaMode mode = ArenaMode.getBySchemType(fightTypes.get(schematic.getSchemtype()));
-                if(new ServerStarter().test(mode, mode.getRandomMap(), checker).check(schematic.getId()).start() == null) {
+                if(!new ServerStarter().test(mode, mode.getRandomMap(), checker).check(schematic.getId()).start()) {
                     remove();
                     return;
                 }
@@ -318,12 +318,9 @@ public class CheckCommand extends SWCommand {
             currentCheckers.remove(checker.getUniqueId());
             currentSchems.remove(schematic.getId());
             ProxyServer.getInstance().getScheduler().runAsync(BungeeCore.get(), () -> {
-                for (Subserver subserver : Subserver.getServerList()) {
+                Bauserver subserver = Bauserver.get(checker.getUniqueId());
-                    if (subserver.getType() == Servertype.BAUSERVER && ((Bauserver) subserver).getOwner().equals(checker.getUniqueId())) {
+                if(subserver != null)
-                        subserver.stop();
+                    subserver.stop();
-                        break;
-                    }
-                }
             });
         }
 

src/de/steamwar/bungeecore/commands/FightCommand.java

@@ -159,13 +159,11 @@ public class FightCommand extends SWCommand {
 
     @Register
     public void fight(@Validator("arenaPlayer") ProxiedPlayer player, @Mapper("nonHistoricArenaMode") @OptionalValue("") @AllowNull ArenaMode arenaMode, @Mapper("arenaMap") @OptionalValue("") @AllowNull String map) {
-        createArena(player, "/fight ", true, arenaMode, map, false, (p, mode, m) -> {
+        createArena(player, "/fight ", true, arenaMode, map, false,
-            Subserver arena = new ServerStarter().arena(mode, m).blueLeader(p).start();
+                (p, mode, m) -> new ServerStarter().arena(mode, m).blueLeader(p).callback(
-            if(arena != null) {
+                        arena -> Message.broadcast("FIGHT_BROADCAST", "FIGHT_BROADCAST_HOVER", new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/arena " + arena.getServer().getName()), mode.getDisplayName(), p.getName())
-                Message.broadcast("FIGHT_BROADCAST", "FIGHT_BROADCAST_HOVER"
+                ).start()
-                        , new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/arena " + arena.getServer().getName()), mode.getDisplayName(), p.getName());
+        );
-            }
-        });
     }
 
     /**

src/de/steamwar/bungeecore/commands/HistoricCommand.java

@@ -22,7 +22,6 @@ package de.steamwar.bungeecore.commands;
 import de.steamwar.bungeecore.ArenaMode;
 import de.steamwar.bungeecore.Message;
 import de.steamwar.bungeecore.ServerStarter;
-import de.steamwar.bungeecore.Subserver;
 import de.steamwar.command.SWCommand;
 import net.md_5.bungee.api.chat.ClickEvent;
 import net.md_5.bungee.api.connection.ProxiedPlayer;
@@ -35,11 +34,9 @@ public class HistoricCommand extends SWCommand {
     @Register
     public void historic(@Validator("arenaPlayer") ProxiedPlayer player, @Mapper("historicArenaMode") @OptionalValue("") @AllowNull ArenaMode arenaMode, @Mapper("arenaMap") @OptionalValue("") @AllowNull String map) {
         FightCommand.createArena(player, "/historic ", true, arenaMode, map, true, (p, mode, m) -> {
-            Subserver arena = new ServerStarter().arena(mode, m).blueLeader(p).start();
+            new ServerStarter().arena(mode, m).blueLeader(p).callback(
-            if(arena != null) {
+                    arena -> Message.broadcast("HISTORIC_BROADCAST", "HISTORIC_BROADCAST_HOVER", new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/arena " + arena.getServer().getName()), mode.getDisplayName(), p.getName())
-                Message.broadcast("HISTORIC_BROADCAST", "HISTORIC_BROADCAST_HOVER",
+            ).start();
-                        new ClickEvent(ClickEvent.Action.RUN_COMMAND, "/arena " + arena.getServer().getName()), mode.getDisplayName(), p.getName());
-            }
         });
     }
 }