From 74697ee3121b53758852a9691a2bc1e5a342036e Mon Sep 17 00:00:00 2001 From: dordsor21 Date: Fri, 17 Sep 2021 14:38:28 +0100 Subject: [PATCH] Fix ZSTD implementation and test-loading. Don't allow disabling of the test on startup --- gradle/libs.versions.toml | 2 +- worldedit-bukkit/build.gradle.kts | 6 +- .../com/fastasyncworldedit/core/Fawe.java | 235 +++++++++--------- .../core/configuration/Settings.java | 13 +- 4 files changed, 124 insertions(+), 132 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index afaf8a785..1c918e7a1 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -36,7 +36,7 @@ truezip = "6.8.4" auto-value = "1.7.4" findbugs = "3.0.2" rhino-runtime = "1.7.13" -zstd-jni = "1.5.0-4" +zstd-jni = "1.4.8-1" # Not latest as it can be difficult to obtain latest ZSTD libs antlr4 = "4.9.1" json-simple = "1.1.1" paranamer = "2.8" diff --git a/worldedit-bukkit/build.gradle.kts b/worldedit-bukkit/build.gradle.kts index 27e5a5f15..09ab4a8e8 100644 --- a/worldedit-bukkit/build.gradle.kts +++ b/worldedit-bukkit/build.gradle.kts @@ -147,6 +147,9 @@ tasks.named("shadowJar") { // If it turns out not to be true for Spigot/Paper, our only two official platforms, this can be uncommented. // include(dependency("org.apache.logging.log4j:log4j-api")) include(dependency("org.antlr:antlr4-runtime")) + // ZSTD does not work if relocated. https://github.com/luben/zstd-jni/issues/189 Use not latest as it can be difficult + // to obtain latest ZSTD lib + include(dependency("com.github.luben:zstd-jni:1.4.8-1")) relocate("org.bstats", "com.sk89q.worldedit.bstats") { include(dependency("org.bstats:")) } @@ -162,9 +165,6 @@ tasks.named("shadowJar") { relocate("com.intellectualsites.paster", "com.fastasyncworldedit.paster") { include(dependency("com.intellectualsites.paster:Paster:1.1.1")) } - relocate("com.github.luben", "com.fastasyncworldedit.core.zstd") { - include(dependency("com.github.luben:zstd-jni:1.5.0-4")) - } relocate("net.jpountz", "com.fastasyncworldedit.core.jpountz") { include(dependency("net.jpountz:lz4-java-stream:1.0.0")) } diff --git a/worldedit-core/src/main/java/com/fastasyncworldedit/core/Fawe.java b/worldedit-core/src/main/java/com/fastasyncworldedit/core/Fawe.java index b2d7eccbc..472c45f20 100644 --- a/worldedit-core/src/main/java/com/fastasyncworldedit/core/Fawe.java +++ b/worldedit-core/src/main/java/com/fastasyncworldedit/core/Fawe.java @@ -12,9 +12,10 @@ import com.fastasyncworldedit.core.util.RandomTextureUtil; import com.fastasyncworldedit.core.util.TaskManager; import com.fastasyncworldedit.core.util.TextureUtil; import com.fastasyncworldedit.core.util.WEManager; -import com.github.luben.zstd.util.Native; +import com.github.luben.zstd.Zstd; import com.sk89q.worldedit.WorldEdit; import com.sk89q.worldedit.internal.util.LogManagerCompat; +import net.jpountz.lz4.LZ4Factory; import org.apache.logging.log4j.Logger; import javax.annotation.Nullable; @@ -81,44 +82,13 @@ public class Fawe { * The ticks-per-second timer. */ private final FaweTimer timer; - private FaweVersion version; - private TextureUtil textures; - - - private QueueHandler queueHandler; - - /** - * Get the implementation specific class. - */ - @SuppressWarnings("unchecked") - public static T imp() { - return instance != null ? (T) instance.implementation : null; - } - - /** - * Get the implementation independent class. - */ - public static Fawe get() { - return instance; - } - - /** - * This method is not for public use. If you have to ask what it does then you shouldn't be using it. - */ - public static void set(final IFawe implementation) throws InstanceAlreadyExistsException, IllegalArgumentException { - if (instance != null) { - throw new InstanceAlreadyExistsException("FAWE has already been initialized with: " + instance.implementation); - } - if (implementation == null) { - throw new IllegalArgumentException("Implementation may not be null."); - } - instance = new Fawe(implementation); - } - /** * The platform specific implementation. */ private final IFawe implementation; + private FaweVersion version; + private TextureUtil textures; + private QueueHandler queueHandler; private Thread thread; private Fawe(final IFawe implementation) { @@ -163,6 +133,94 @@ public class Fawe { TaskManager.IMP.repeat(timer, 1); } + /** + * Get the implementation specific class. + */ + @SuppressWarnings("unchecked") + public static T imp() { + return instance != null ? (T) instance.implementation : null; + } + + /** + * Get the implementation independent class. + */ + public static Fawe get() { + return instance; + } + + /** + * This method is not for public use. If you have to ask what it does then you shouldn't be using it. + */ + public static void set(final IFawe implementation) throws InstanceAlreadyExistsException, IllegalArgumentException { + if (instance != null) { + throw new InstanceAlreadyExistsException("FAWE has already been initialized with: " + instance.implementation); + } + if (implementation == null) { + throw new IllegalArgumentException("Implementation may not be null."); + } + instance = new Fawe(implementation); + } + + public static void setupInjector() { + // Check Base OS Arch for Mismatching Architectures + boolean x86OS = System.getProperty("sun.arch.data.model").contains("32"); + boolean x86JVM = System.getProperty("os.arch").contains("32"); + if (x86OS != x86JVM) { + LOGGER.info("You are running 32-bit Java on a 64-bit machine. Please upgrade to 64-bit Java."); + } + } + + public static boolean isMainThread() { + return instance == null || instance.thread == Thread.currentThread(); + } + + /** + * Non-api. Handles an input FAWE exception if not already handled, given the input boolean array. + * Looks at the {@link FaweException.Type} and decides what to do (rethrows if we want to attempt to show the error to the + * player, outputs to console where necessary). + * + * @param faweExceptionReasonsUsed boolean array that should be cached where this method is called from of length {@code + * FaweException.Type.values().length} + * @param e {@link FaweException} to handle + * @param logger {@link Logger} of the calling class + */ + public static void handleFaweException( + boolean[] faweExceptionReasonsUsed, + FaweException e, + final Logger logger + ) { + FaweException.Type type = e.getType(); + switch (type) { + case OTHER: + logger.catching(e); + throw e; + case LOW_MEMORY: + if (!faweExceptionReasonsUsed[type.ordinal()]) { + logger.warn("FaweException: " + e.getMessage()); + faweExceptionReasonsUsed[type.ordinal()] = true; + throw e; + } + case MAX_TILES: + case NO_REGION: + case MAX_CHECKS: + case MAX_CHANGES: + case MAX_ENTITIES: + case MAX_ITERATIONS: + case OUTSIDE_REGION: + if (!faweExceptionReasonsUsed[type.ordinal()]) { + faweExceptionReasonsUsed[type.ordinal()] = true; + throw e; + } else { + return; + } + default: + if (!faweExceptionReasonsUsed[type.ordinal()]) { + faweExceptionReasonsUsed[type.ordinal()] = true; + logger.warn("FaweException: " + e.getMessage()); + } + } + } + public void onDisable() { if (imp().getPreloader(false) != null) { imp().getPreloader(false).cancel(); @@ -254,45 +312,35 @@ public class Fawe { LOGGER.error("Failed to load config.", e); } Settings.IMP.QUEUE.TARGET_SIZE = Math.max(Settings.IMP.QUEUE.TARGET_SIZE, Settings.IMP.QUEUE.PARALLEL_THREADS); - } - - - public WorldEdit getWorldEdit() { - return WorldEdit.getInstance(); - } - - public static void setupInjector() { - /* - * Modify the sessions - * - EditSession supports a custom queue, and a lot of optimizations - * - LocalSession supports VirtualPlayers and undo on disk - */ - if (!Settings.IMP.EXPERIMENTAL.DISABLE_NATIVES) { - // A higher amount is currently not supported by ZSTD / ZSTD JNI + try { + byte[] in = new byte[0]; + byte[] compressed = LZ4Factory.fastestJavaInstance().fastCompressor().compress(in); + byte[] ob = new byte[100]; + assert (LZ4Factory.fastestJavaInstance().fastDecompressor().decompress(ob, compressed) == 0); + LOGGER.info("LZ4 Compression Binding loaded successfully"); + } catch (Throwable e) { + LOGGER.error("LZ4 Compression Binding Not Found.\n" + + "FAWE will still work but compression will be slower.", e); + } + try { + byte[] in = new byte[0]; + byte[] compressed = Zstd.compress(in); + byte[] ob = new byte[100]; + assert (Zstd.decompress(ob, compressed) == 0); + LOGGER.info("ZSTD Compression Binding loaded successfully"); + } catch (Throwable e) { if (Settings.IMP.CLIPBOARD.COMPRESSION_LEVEL > 6 || Settings.IMP.HISTORY.COMPRESSION_LEVEL > 6) { Settings.IMP.CLIPBOARD.COMPRESSION_LEVEL = Math.min(6, Settings.IMP.CLIPBOARD.COMPRESSION_LEVEL); Settings.IMP.HISTORY.COMPRESSION_LEVEL = Math.min(6, Settings.IMP.HISTORY.COMPRESSION_LEVEL); - } - try { - Native.load(); - } catch (Throwable e) { - LOGGER.error("ZSTD compression binding not found.\n" - + "FAWE will still work but compression won't work as well.\n", e); - } - try { - net.jpountz.util.Native.load(); - } catch (Throwable e) { - LOGGER.error("LZ4 Compression Binding Not Found.\n" - + "FAWE will still work but compression will be slower.\n", e); + LOGGER.error("ZSTD Compression Binding Not Found.\n" + + "FAWE will still work but compression won't work as well.", e); } } + Settings.IMP.save(file); + } - // Check Base OS Arch for Mismatching Architectures - boolean x86OS = System.getProperty("sun.arch.data.model").contains("32"); - boolean x86JVM = System.getProperty("os.arch").contains("32"); - if (x86OS != x86JVM) { - LOGGER.info("You are running 32-bit Java on a 64-bit machine. Please upgrade to 64-bit Java."); - } + public WorldEdit getWorldEdit() { + return WorldEdit.getInstance(); } private void setupMemoryListener() { @@ -338,10 +386,6 @@ public class Fawe { return this.thread; } - public static boolean isMainThread() { - return instance == null || instance.thread == Thread.currentThread(); - } - /** * Sets the main thread to the current thread. */ @@ -349,51 +393,4 @@ public class Fawe { return this.thread = Thread.currentThread(); } - /** - * Non-api. Handles an input FAWE exception if not already handled, given the input boolean array. - * Looks at the {@link FaweException.Type} and decides what to do (rethrows if we want to attempt to show the error to the - * player, outputs to console where necessary). - * - * @param faweExceptionReasonsUsed boolean array that should be cached where this method is called from of length {@code - * FaweException.Type.values().length} - * @param e {@link FaweException} to handle - * @param logger {@link Logger} of the calling class - */ - public static void handleFaweException( - boolean[] faweExceptionReasonsUsed, - FaweException e, - final Logger logger - ) { - FaweException.Type type = e.getType(); - switch (type) { - case OTHER: - logger.catching(e); - throw e; - case LOW_MEMORY: - if (!faweExceptionReasonsUsed[type.ordinal()]) { - logger.warn("FaweException: " + e.getMessage()); - faweExceptionReasonsUsed[type.ordinal()] = true; - throw e; - } - case MAX_TILES: - case NO_REGION: - case MAX_CHECKS: - case MAX_CHANGES: - case MAX_ENTITIES: - case MAX_ITERATIONS: - case OUTSIDE_REGION: - if (!faweExceptionReasonsUsed[type.ordinal()]) { - faweExceptionReasonsUsed[type.ordinal()] = true; - throw e; - } else { - return; - } - default: - if (!faweExceptionReasonsUsed[type.ordinal()]) { - faweExceptionReasonsUsed[type.ordinal()] = true; - logger.warn("FaweException: " + e.getMessage()); - } - } - } - } diff --git a/worldedit-core/src/main/java/com/fastasyncworldedit/core/configuration/Settings.java b/worldedit-core/src/main/java/com/fastasyncworldedit/core/configuration/Settings.java index 969fb26fa..b035877e1 100644 --- a/worldedit-core/src/main/java/com/fastasyncworldedit/core/configuration/Settings.java +++ b/worldedit-core/src/main/java/com/fastasyncworldedit/core/configuration/Settings.java @@ -224,12 +224,11 @@ public class Settings extends Config { "4 = 1 x medium, 1 x fast", "5 = 1 x medium, 2 x fast", "6 = 1 x medium, 3 x fast", - /* A higher value is currently not supported by ZSTD / ZSTD-JNI "7 = 1 x high, 1 x medium, 1 x fast", "8 = 1 x high, 1 x medium, 2 x fast", "9 = 1 x high, 1 x medium, 3 x fast (best compression)", - "NOTE: If using disk, do some compression (3+) as smaller files save faster" - */ + "NOTE: If using disk, do some compression (3+) as smaller files save faster", + " - levels over 6 require ZSTD 1.4.8+ to be installed to the system" }) public int COMPRESSION_LEVEL = 3; @Comment({ @@ -390,11 +389,6 @@ public class Settings extends Config { }) public boolean PERSISTENT_BRUSHES = true; - @Comment({ - "Disable using native libraries", - }) - public boolean DISABLE_NATIVES = false; - @Comment({ "[SAFE] Keep entities that are positioned in non-air blocks when editing an area", "Might cause client-side FPS lagg in some situations" @@ -502,7 +496,8 @@ public class Settings extends Config { " - TODO: Buffered random access with compression is not implemented on disk yet", " - 0 = No compression", " - 1 = Fast compression", - " - 2-6 = Slower compression" + " - 2-17 = Slower compression", + " - levels over 6 require ZSTD 1.4.8+ to be installed to the system" }) public int COMPRESSION_LEVEL = 1; @Comment("Number of days to keep history on disk before deleting it")