diff --git a/.gitignore b/.gitignore index 29577f5bb..1cf6c5d69 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,4 @@ plugins/ native/mbedtls native/zlib-ng native/zlib-cf +native/libdeflate diff --git a/native/compile-linux-amd64.sh b/native/compile-linux-amd64.sh deleted file mode 100755 index 135fcb539..000000000 --- a/native/compile-linux-amd64.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -if [ ! -d zlib-cf ]; then - echo "Cloning Cloudflare zlib..." - git clone -b gcc.amd64 https://github.com/cloudflare/zlib.git zlib-cf -fi - -echo "Compiling Cloudflare zlib..." -cd zlib-cf -CFLAGS="-fPIC -O3 -flto" AR=gcc-ar RANLIB=gcc-ranlib ./configure --static -make clean && make -cd .. - -# Modify as you need. -MBEDTLS_ROOT=mbedtls -CFLAGS="-O3 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack" -gcc $CFLAGS -Izlib-cf src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \ - src/main/c/jni_zlib_common.c zlib-cf/libz.a -Wl,-z,noexecstack -o src/main/resources/linux_x64/velocity-compress.so -gcc $CFLAGS -I $MBEDTLS_ROOT/include -shared $MBEDTLS_ROOT/library/aes.c $MBEDTLS_ROOT/library/aesni.c \ - $MBEDTLS_ROOT/library/platform.c $MBEDTLS_ROOT/library/platform_util.c src/main/c/jni_util.c src/main/c/jni_cipher.c \ - -o src/main/resources/linux_x64/velocity-cipher.so diff --git a/native/compile-linux.sh b/native/compile-linux.sh index 4b759df20..78072b00b 100755 --- a/native/compile-linux.sh +++ b/native/compile-linux.sh @@ -1,21 +1,20 @@ #!/bin/bash -if [ ! -d zlib-ng ]; then - echo "Cloning zlib-ng..." - git clone https://github.com/zlib-ng/zlib-ng.git +if [ ! -d libdeflate ]; then + echo "Cloning libdeflate..." + git clone https://github.com/ebiggers/libdeflate.git fi -echo "Compiling zlib-ng..." -cd zlib-ng -CFLAGS="-fPIC -O3" ./configure --zlib-compat --static -make clean && make +echo "Compiling libdeflate..." +cd libdeflate || exit +make cd .. # Modify as you need. MBEDTLS_ROOT=mbedtls -CFLAGS="-O3 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared" -gcc $CFLAGS -Izlib-ng src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \ - src/main/c/jni_zlib_common.c zlib-ng/libz.a -o src/main/resources/linux_x64/velocity-compress.so -gcc $CFLAGS -I $MBEDTLS_ROOT/include -shared $MBEDTLS_ROOT/library/aes.c $MBEDTLS_ROOT/library/aesni.c \ +CFLAGS="-O3 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack" +gcc $CFLAGS -Ilibdeflate src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \ + libdeflate/libdeflate.a -o src/main/resources/linux_x64/velocity-compress.so +gcc $CFLAGS -I $MBEDTLS_ROOT/include -shared $MBEDTLS_ROOT/library/aes.c $MBEDTLS_ROOT/library/aesni.c \ $MBEDTLS_ROOT/library/platform.c $MBEDTLS_ROOT/library/platform_util.c src/main/c/jni_util.c src/main/c/jni_cipher.c \ - -o src/main/resources/linux_x64/velocity-cipher.so \ No newline at end of file + -o src/main/resources/linux_x64/velocity-cipher.so diff --git a/native/src/main/c/jni_zlib_common.c b/native/src/main/c/jni_zlib_common.c deleted file mode 100644 index fa5a0cbab..000000000 --- a/native/src/main/c/jni_zlib_common.c +++ /dev/null @@ -1,29 +0,0 @@ -#include -#include -#include -#include -#include "jni_util.h" - -void JNICALL -check_zlib_free(JNIEnv *env, z_stream *stream, bool deflate) -{ - int ret = deflate ? deflateEnd(stream) : inflateEnd(stream); - const char *msg = stream->msg; - free((void*) stream); - - switch (ret) { - case Z_OK: - break; - case Z_STREAM_ERROR: - if (msg == NULL) { - msg = "stream state inconsistent"; - } - // fall-through - case Z_DATA_ERROR: - if (msg == NULL) { - msg = "data was discarded"; - } - throwException(env, "java/lang/IllegalArgumentException", msg); - break; - } -} \ No newline at end of file diff --git a/native/src/main/c/jni_zlib_common.h b/native/src/main/c/jni_zlib_common.h deleted file mode 100644 index ff4bc9a80..000000000 --- a/native/src/main/c/jni_zlib_common.h +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include -#include - -void JNICALL -check_zlib_free(JNIEnv *env, z_stream *stream, bool deflate); \ No newline at end of file diff --git a/native/src/main/c/jni_zlib_deflate.c b/native/src/main/c/jni_zlib_deflate.c index 6e05f7cae..7a84a5b04 100644 --- a/native/src/main/c/jni_zlib_deflate.c +++ b/native/src/main/c/jni_zlib_deflate.c @@ -2,56 +2,21 @@ #include #include #include -#include +#include #include "jni_util.h" -#include "jni_zlib_common.h" - -static jfieldID finishedID; -static jfieldID consumedID; - -JNIEXPORT void JNICALL -Java_com_velocitypowered_natives_compression_NativeZlibDeflate_initIDs(JNIEnv *env, jclass cls) -{ - finishedID = (*env)->GetFieldID(env, cls, "finished", "Z"); - consumedID = (*env)->GetFieldID(env, cls, "consumed", "I"); -} JNIEXPORT jlong JNICALL Java_com_velocitypowered_natives_compression_NativeZlibDeflate_init(JNIEnv *env, jobject obj, jint level) { - z_stream* stream = calloc(1, sizeof(z_stream)); - - if (stream == 0) { + struct libdeflate_compressor *compressor = libdeflate_alloc_compressor(level); + if (compressor == NULL) { // Out of memory! - throwException(env, "java/lang/OutOfMemoryError", "zlib allocate stream"); - return 0; - } - - int ret = deflateInit(stream, level); - if (ret == Z_OK) { - return (jlong) stream; - } else { - const char *zlib_msg = stream->msg; - free(stream); - switch (ret) { - case Z_MEM_ERROR: - throwException(env, "java/lang/OutOfMemoryError", "zlib init"); - break; - case Z_STREAM_ERROR: { - // Thanks Ken and Ritchie! - char message[32]; - snprintf(message, 32, "invalid level %d", level); - throwException(env, "java/lang/IllegalArgumentException", message); - break; - } - default: - throwException(env, "java/util/zip/DataFormatException", zlib_msg); - break; - } + throwException(env, "java/lang/OutOfMemoryError", "libdeflate allocate compressor"); return 0; } + return (jlong) compressor; } JNIEXPORT void JNICALL @@ -59,11 +24,10 @@ Java_com_velocitypowered_natives_compression_NativeZlibDeflate_free(JNIEnv *env, jobject obj, jlong ctx) { - z_stream* stream = (z_stream*) ctx; - check_zlib_free(env, stream, true); + libdeflate_free_compressor((struct libdeflate_compressor *) ctx); } -JNIEXPORT int JNICALL +JNIEXPORT jboolean JNICALL Java_com_velocitypowered_natives_compression_NativeZlibDeflate_process(JNIEnv *env, jobject obj, jlong ctx, @@ -73,38 +37,8 @@ Java_com_velocitypowered_natives_compression_NativeZlibDeflate_process(JNIEnv *e jint destinationLength, jboolean finish) { - z_stream* stream = (z_stream*) ctx; - stream->next_in = (Bytef *) sourceAddress; - stream->next_out = (Bytef *) destinationAddress; - stream->avail_in = sourceLength; - stream->avail_out = destinationLength; - - int res = deflate(stream, finish ? Z_FINISH : Z_NO_FLUSH); - switch (res) { - case Z_STREAM_END: - // The stream has ended. - (*env)->SetBooleanField(env, obj, finishedID, JNI_TRUE); - // fall-through - case Z_OK: - // Not yet completed, but progress has been made. Tell Java how many bytes we've processed. - (*env)->SetIntField(env, obj, consumedID, sourceLength - stream->avail_in); - return destinationLength - stream->avail_out; - case Z_BUF_ERROR: - // This is not fatal. Just say we need more data. Usually this applies to the next_out buffer, - // which NativeVelocityCompressor will notice and will expand the buffer. - return 0; - default: - throwException(env, "java/util/zip/DataFormatException", stream->msg); - return 0; - } -} - -JNIEXPORT void JNICALL -Java_com_velocitypowered_natives_compression_NativeZlibDeflate_reset(JNIEnv *env, - jobject obj, - jlong ctx) -{ - z_stream* stream = (z_stream*) ctx; - int ret = deflateReset(stream); - assert(ret == Z_OK); + struct libdeflate_compressor *compressor = (struct libdeflate_compressor *) ctx; + size_t produced = libdeflate_zlib_compress(compressor, (void *) sourceAddress, sourceLength, + (void *) destinationAddress, destinationLength); + return (jlong) produced; } \ No newline at end of file diff --git a/native/src/main/c/jni_zlib_inflate.c b/native/src/main/c/jni_zlib_inflate.c index 64804e160..9a81f0db8 100644 --- a/native/src/main/c/jni_zlib_inflate.c +++ b/native/src/main/c/jni_zlib_inflate.c @@ -2,50 +2,21 @@ #include #include #include -#include +#include #include "jni_util.h" -#include "jni_zlib_common.h" - -static jfieldID finishedID; -static jfieldID consumedID; - -JNIEXPORT void JNICALL -Java_com_velocitypowered_natives_compression_NativeZlibInflate_initIDs(JNIEnv *env, jclass cls) -{ - finishedID = (*env)->GetFieldID(env, cls, "finished", "Z"); - consumedID = (*env)->GetFieldID(env, cls, "consumed", "I"); -} JNIEXPORT jlong JNICALL Java_com_velocitypowered_natives_compression_NativeZlibInflate_init(JNIEnv *env, jobject obj) { - z_stream* stream = calloc(1, sizeof(z_stream)); - - if (stream == 0) { + struct libdeflate_decompressor *decompress = libdeflate_alloc_decompressor(); + if (decompress == NULL) { // Out of memory! - throwException(env, "java/lang/OutOfMemoryError", "zlib allocate stream"); + throwException(env, "java/lang/OutOfMemoryError", "libdeflate allocate decompressor"); return 0; } - int ret = inflateInit(stream); - if (ret == Z_OK) { - return (jlong) stream; - } else { - const char *zlib_msg = stream->msg; - free(stream); - switch (ret) { - case Z_MEM_ERROR: - throwException(env, "java/lang/OutOfMemoryError", "zlib init"); - return 0; - case Z_STREAM_ERROR: - throwException(env, "java/lang/IllegalArgumentException", "stream clobbered?"); - return 0; - default: - throwException(env, "java/util/zip/DataFormatException", zlib_msg); - return 0; - } - } + return (jlong) decompress; } JNIEXPORT void JNICALL @@ -53,51 +24,34 @@ Java_com_velocitypowered_natives_compression_NativeZlibInflate_free(JNIEnv *env, jobject obj, jlong ctx) { - z_stream* stream = (z_stream*) ctx; - check_zlib_free(env, stream, false); + libdeflate_free_decompressor((struct libdeflate_decompressor *) ctx); } -JNIEXPORT int JNICALL +JNIEXPORT jboolean JNICALL Java_com_velocitypowered_natives_compression_NativeZlibInflate_process(JNIEnv *env, jobject obj, jlong ctx, jlong sourceAddress, jint sourceLength, jlong destinationAddress, - jint destinationLength) + jint destinationLength, + jlong maximumSize) { - z_stream* stream = (z_stream*) ctx; - stream->next_in = (Bytef *) sourceAddress; - stream->next_out = (Bytef *) destinationAddress; - stream->avail_in = sourceLength; - stream->avail_out = destinationLength; + struct libdeflate_decompressor *decompress = (struct libdeflate_decompressor *) ctx; + enum libdeflate_result result = libdeflate_zlib_decompress(decompress, (void *) sourceAddress, + sourceLength, (void *) destinationAddress, destinationLength, NULL); - int res = inflate(stream, Z_PARTIAL_FLUSH); - switch (res) { - case Z_STREAM_END: - // The stream has ended - (*env)->SetBooleanField(env, obj, finishedID, JNI_TRUE); - // fall-through - case Z_OK: - // Not yet completed, but progress has been made. Tell Java how many bytes we've processed. - (*env)->SetIntField(env, obj, consumedID, sourceLength - stream->avail_in); - return destinationLength - stream->avail_out; - case Z_BUF_ERROR: - // This is not fatal. Just say we need more data. Usually this applies to the next_out buffer, - // which NativeVelocityCompressor will notice and will expand the buffer. - return 0; - default: - throwException(env, "java/util/zip/DataFormatException", stream->msg); - return 0; + switch (result) { + case LIBDEFLATE_SUCCESS: + // We are happy + return JNI_TRUE; + case LIBDEFLATE_BAD_DATA: + throwException(env, "java/util/zip/DataFormatException", "inflate data is bad"); + return JNI_FALSE; + case LIBDEFLATE_SHORT_OUTPUT: + case LIBDEFLATE_INSUFFICIENT_SPACE: + // These cases are the same for us. We expect the full uncompressed size to be known. + throwException(env, "java/util/zip/DataFormatException", "uncompressed size is inaccurate"); + return JNI_FALSE; } -} - -JNIEXPORT void JNICALL -Java_com_velocitypowered_natives_compression_NativeZlibInflate_reset(JNIEnv *env, - jobject obj, - jlong ctx) -{ - z_stream* stream = (z_stream*) ctx; - int ret = inflateReset(stream); - assert(ret == Z_OK); } \ No newline at end of file diff --git a/native/src/main/java/com/velocitypowered/natives/compression/Java11VelocityCompressor.java b/native/src/main/java/com/velocitypowered/natives/compression/Java11VelocityCompressor.java index 999d12caa..bc6206242 100644 --- a/native/src/main/java/com/velocitypowered/natives/compression/Java11VelocityCompressor.java +++ b/native/src/main/java/com/velocitypowered/natives/compression/Java11VelocityCompressor.java @@ -54,7 +54,8 @@ public class Java11VelocityCompressor implements VelocityCompressor { } @Override - public void inflate(ByteBuf source, ByteBuf destination, int max) throws DataFormatException { + public void inflate(ByteBuf source, ByteBuf destination, int uncompressedSize) + throws DataFormatException { ensureNotDisposed(); // We (probably) can't nicely deal with >=1 buffer nicely, so let's scream loudly. @@ -67,7 +68,7 @@ public class Java11VelocityCompressor implements VelocityCompressor { while (!inflater.finished() && inflater.getBytesRead() < source.readableBytes()) { if (!destination.isWritable()) { - ensureMaxSize(destination, max); + ensureMaxSize(destination, uncompressedSize); destination.ensureWritable(ZLIB_BUFFER_SIZE); } diff --git a/native/src/main/java/com/velocitypowered/natives/compression/JavaVelocityCompressor.java b/native/src/main/java/com/velocitypowered/natives/compression/JavaVelocityCompressor.java index 6293dd770..f2e557d1c 100644 --- a/native/src/main/java/com/velocitypowered/natives/compression/JavaVelocityCompressor.java +++ b/native/src/main/java/com/velocitypowered/natives/compression/JavaVelocityCompressor.java @@ -25,20 +25,21 @@ public class JavaVelocityCompressor implements VelocityCompressor { } @Override - public void inflate(ByteBuf source, ByteBuf destination, int max) throws DataFormatException { + public void inflate(ByteBuf source, ByteBuf destination, int uncompressedSize) + throws DataFormatException { ensureNotDisposed(); final int available = source.readableBytes(); this.setInflaterInput(source); if (destination.hasArray()) { - this.inflateDestinationIsHeap(destination, available, max); + this.inflateDestinationIsHeap(destination, available, uncompressedSize); } else { if (buf.length == 0) { buf = new byte[ZLIB_BUFFER_SIZE]; } while (!inflater.finished() && inflater.getBytesRead() < available) { - ensureMaxSize(destination, max); + ensureMaxSize(destination, uncompressedSize); int read = inflater.inflate(buf); destination.writeBytes(buf, 0, read); } diff --git a/native/src/main/java/com/velocitypowered/natives/compression/LibdeflateVelocityCompressor.java b/native/src/main/java/com/velocitypowered/natives/compression/LibdeflateVelocityCompressor.java new file mode 100644 index 000000000..73e9ff7fc --- /dev/null +++ b/native/src/main/java/com/velocitypowered/natives/compression/LibdeflateVelocityCompressor.java @@ -0,0 +1,87 @@ +package com.velocitypowered.natives.compression; + +import com.google.common.base.Preconditions; +import com.velocitypowered.natives.util.BufferPreference; +import io.netty.buffer.ByteBuf; +import java.util.zip.DataFormatException; + +public class LibdeflateVelocityCompressor implements VelocityCompressor { + + public static final VelocityCompressorFactory FACTORY = LibdeflateVelocityCompressor::new; + + private final NativeZlibInflate inflate = new NativeZlibInflate(); + private final long inflateCtx; + private final NativeZlibDeflate deflate = new NativeZlibDeflate(); + private final long deflateCtx; + private boolean disposed = false; + + private LibdeflateVelocityCompressor(int level) { + int correctedLevel = level == -1 ? 6 : level; + if (correctedLevel > 12 || correctedLevel < 1) { + throw new IllegalArgumentException("Invalid compression level " + level); + } + + this.inflateCtx = inflate.init(); + this.deflateCtx = deflate.init(level == -1 ? 6 : level); + } + + @Override + public void inflate(ByteBuf source, ByteBuf destination, int uncompressedSize) + throws DataFormatException { + ensureNotDisposed(); + source.memoryAddress(); + destination.memoryAddress(); + + // libdeflate recommends we work with a known uncompressed size - so we work strictly within + // those parameters. If the uncompressed size doesn't match the compressed size, then we will + // throw an exception from native code. + destination.ensureWritable(uncompressedSize); + + long sourceAddress = source.memoryAddress() + source.readerIndex(); + long destinationAddress = destination.memoryAddress() + destination.writerIndex(); + + inflate.process(inflateCtx, sourceAddress, source.readableBytes(), destinationAddress, + uncompressedSize); + destination.writerIndex(destination.writerIndex() + uncompressedSize); + } + + @Override + public void deflate(ByteBuf source, ByteBuf destination) throws DataFormatException { + ensureNotDisposed(); + source.memoryAddress(); + destination.memoryAddress(); + + while (true) { + long sourceAddress = source.memoryAddress() + source.readerIndex(); + long destinationAddress = destination.memoryAddress() + destination.writerIndex(); + + int produced = deflate.process(deflateCtx, sourceAddress, source.readableBytes(), + destinationAddress, destination.writableBytes()); + if (produced > 0) { + destination.writerIndex(destination.writerIndex() + produced); + return; + } + + // Insufficient room - enlarge the buffer. + destination.capacity(destination.capacity() * 2); + } + } + + private void ensureNotDisposed() { + Preconditions.checkState(!disposed, "Object already disposed"); + } + + @Override + public void dispose() { + if (!disposed) { + inflate.free(inflateCtx); + deflate.free(deflateCtx); + } + disposed = true; + } + + @Override + public BufferPreference preferredBufferType() { + return BufferPreference.DIRECT_REQUIRED; + } +} diff --git a/native/src/main/java/com/velocitypowered/natives/compression/NativeVelocityCompressor.java b/native/src/main/java/com/velocitypowered/natives/compression/NativeVelocityCompressor.java deleted file mode 100644 index b932579ae..000000000 --- a/native/src/main/java/com/velocitypowered/natives/compression/NativeVelocityCompressor.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.velocitypowered.natives.compression; - -import static com.velocitypowered.natives.compression.CompressorUtils.ZLIB_BUFFER_SIZE; -import static com.velocitypowered.natives.compression.CompressorUtils.ensureMaxSize; - -import com.google.common.base.Preconditions; -import com.velocitypowered.natives.util.BufferPreference; -import io.netty.buffer.ByteBuf; -import java.util.zip.DataFormatException; - -public class NativeVelocityCompressor implements VelocityCompressor { - - public static final VelocityCompressorFactory FACTORY = NativeVelocityCompressor::new; - - private final NativeZlibInflate inflate = new NativeZlibInflate(); - private final long inflateCtx; - private final NativeZlibDeflate deflate = new NativeZlibDeflate(); - private final long deflateCtx; - private boolean disposed = false; - - private NativeVelocityCompressor(int level) { - this.inflateCtx = inflate.init(); - this.deflateCtx = deflate.init(level); - } - - @Override - public void inflate(ByteBuf source, ByteBuf destination, int max) throws DataFormatException { - ensureNotDisposed(); - source.memoryAddress(); - destination.memoryAddress(); - - while (!inflate.finished && source.isReadable()) { - if (!destination.isWritable()) { - ensureMaxSize(destination, max); - destination.ensureWritable(ZLIB_BUFFER_SIZE); - } - int produced = inflate.process(inflateCtx, source.memoryAddress() + source.readerIndex(), - source.readableBytes(), destination.memoryAddress() + destination.writerIndex(), - destination.writableBytes()); - source.readerIndex(source.readerIndex() + inflate.consumed); - destination.writerIndex(destination.writerIndex() + produced); - } - - inflate.reset(inflateCtx); - inflate.consumed = 0; - inflate.finished = false; - } - - @Override - public void deflate(ByteBuf source, ByteBuf destination) throws DataFormatException { - ensureNotDisposed(); - source.memoryAddress(); - destination.memoryAddress(); - - while (!deflate.finished) { - if (!destination.isWritable()) { - destination.ensureWritable(ZLIB_BUFFER_SIZE); - } - int produced = deflate.process(deflateCtx, source.memoryAddress() + source.readerIndex(), - source.readableBytes(), - destination.memoryAddress() + destination.writerIndex(), destination.writableBytes(), - true); - source.readerIndex(source.readerIndex() + deflate.consumed); - destination.writerIndex(destination.writerIndex() + produced); - } - - deflate.reset(deflateCtx); - deflate.consumed = 0; - deflate.finished = false; - } - - private void ensureNotDisposed() { - Preconditions.checkState(!disposed, "Object already disposed"); - } - - @Override - public void dispose() { - if (!disposed) { - inflate.free(inflateCtx); - deflate.free(deflateCtx); - } - disposed = true; - } - - @Override - public BufferPreference preferredBufferType() { - return BufferPreference.DIRECT_REQUIRED; - } -} diff --git a/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibDeflate.java b/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibDeflate.java index 1114cab49..eb89412cb 100644 --- a/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibDeflate.java +++ b/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibDeflate.java @@ -5,21 +5,10 @@ package com.velocitypowered.natives.compression; */ class NativeZlibDeflate { - boolean finished; - int consumed; - native long init(int level); native long free(long ctx); native int process(long ctx, long sourceAddress, int sourceLength, long destinationAddress, - int destinationLength, boolean finish); - - native void reset(long ctx); - - static { - initIDs(); - } - - private static native void initIDs(); + int destinationLength); } diff --git a/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibInflate.java b/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibInflate.java index 7eabd9a05..fc6e9787f 100644 --- a/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibInflate.java +++ b/native/src/main/java/com/velocitypowered/natives/compression/NativeZlibInflate.java @@ -1,25 +1,16 @@ package com.velocitypowered.natives.compression; +import java.util.zip.DataFormatException; + /** * Represents a native interface for zlib's inflate functions. */ class NativeZlibInflate { - boolean finished; - int consumed; - native long init(); native long free(long ctx); - native int process(long ctx, long sourceAddress, int sourceLength, long destinationAddress, - int destinationLength); - - native void reset(long ctx); - - static { - initIDs(); - } - - private static native void initIDs(); + native boolean process(long ctx, long sourceAddress, int sourceLength, long destinationAddress, + int destinationLength) throws DataFormatException; } diff --git a/native/src/main/java/com/velocitypowered/natives/compression/VelocityCompressor.java b/native/src/main/java/com/velocitypowered/natives/compression/VelocityCompressor.java index 839b64569..09cad102c 100644 --- a/native/src/main/java/com/velocitypowered/natives/compression/VelocityCompressor.java +++ b/native/src/main/java/com/velocitypowered/natives/compression/VelocityCompressor.java @@ -6,10 +6,12 @@ import io.netty.buffer.ByteBuf; import java.util.zip.DataFormatException; /** - * Provides an interface to inflate and deflate {@link ByteBuf}s using zlib. + * Provides an interface to inflate and deflate {@link ByteBuf}s using zlib or a compatible + * implementation. */ public interface VelocityCompressor extends Disposable, Native { - void inflate(ByteBuf source, ByteBuf destination, int max) throws DataFormatException; + void inflate(ByteBuf source, ByteBuf destination, int uncompressedSize) + throws DataFormatException; void deflate(ByteBuf source, ByteBuf destination) throws DataFormatException; } diff --git a/native/src/main/java/com/velocitypowered/natives/util/Natives.java b/native/src/main/java/com/velocitypowered/natives/util/Natives.java index e63c87c20..ea71c75ce 100644 --- a/native/src/main/java/com/velocitypowered/natives/util/Natives.java +++ b/native/src/main/java/com/velocitypowered/natives/util/Natives.java @@ -4,7 +4,7 @@ import com.google.common.collect.ImmutableList; import com.velocitypowered.natives.NativeSetupException; import com.velocitypowered.natives.compression.Java11VelocityCompressor; import com.velocitypowered.natives.compression.JavaVelocityCompressor; -import com.velocitypowered.natives.compression.NativeVelocityCompressor; +import com.velocitypowered.natives.compression.LibdeflateVelocityCompressor; import com.velocitypowered.natives.compression.VelocityCompressorFactory; import com.velocitypowered.natives.encryption.JavaVelocityCipher; import com.velocitypowered.natives.encryption.NativeVelocityCipher; @@ -64,10 +64,11 @@ public class Natives { ImmutableList.of( new NativeCodeLoader.Variant<>(NativeConstraints.MACOS, copyAndLoadNative("/macosx/velocity-compress.dylib"), "native (macOS)", - NativeVelocityCompressor.FACTORY), + LibdeflateVelocityCompressor.FACTORY), new NativeCodeLoader.Variant<>(NativeConstraints.LINUX, - copyAndLoadNative("/linux_x64/velocity-compress.so"), "native (Linux amd64)", - NativeVelocityCompressor.FACTORY), + copyAndLoadNative("/linux_x64/velocity-compress.so"), + "libdeflate (Linux amd64)", + LibdeflateVelocityCompressor.FACTORY), new NativeCodeLoader.Variant<>(NativeConstraints.JAVA_11, () -> { }, "Java 11", () -> Java11VelocityCompressor.FACTORY), new NativeCodeLoader.Variant<>(NativeCodeLoader.ALWAYS, () -> { diff --git a/native/src/main/resources/linux_x64/velocity-compress.so b/native/src/main/resources/linux_x64/velocity-compress.so index c05574805..d72a66bfd 100755 Binary files a/native/src/main/resources/linux_x64/velocity-compress.so and b/native/src/main/resources/linux_x64/velocity-compress.so differ diff --git a/native/src/test/java/com/velocitypowered/natives/compression/VelocityCompressorTest.java b/native/src/test/java/com/velocitypowered/natives/compression/VelocityCompressorTest.java index 182423e98..98533c848 100644 --- a/native/src/test/java/com/velocitypowered/natives/compression/VelocityCompressorTest.java +++ b/native/src/test/java/com/velocitypowered/natives/compression/VelocityCompressorTest.java @@ -86,10 +86,11 @@ class VelocityCompressorTest { ByteBuf decompressed = bufSupplier.get(); source.writeBytes(TEST_DATA); + int uncompressedData = source.readableBytes(); try { compressor.deflate(source, dest); - compressor.inflate(dest, decompressed, Integer.MAX_VALUE); + compressor.inflate(dest, decompressed, uncompressedData); source.readerIndex(0); assertTrue(ByteBufUtil.equals(source, decompressed)); } finally { diff --git a/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressDecoder.java b/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressDecoder.java index cf68e134b..5a0e0462d 100644 --- a/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressDecoder.java +++ b/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressDecoder.java @@ -13,9 +13,13 @@ import java.util.List; public class MinecraftCompressDecoder extends MessageToMessageDecoder { - private static final int SOFT_MAXIMUM_UNCOMPRESSED_SIZE = 2 * 1024 * 1024; // 2MiB + private static final int VANILLA_MAXIMUM_UNCOMPRESSED_SIZE = 2 * 1024 * 1024; // 2MiB private static final int HARD_MAXIMUM_UNCOMPRESSED_SIZE = 16 * 1024 * 1024; // 16MiB + private static final int UNCOMPRESSED_CAP = + Boolean.getBoolean("velocity.increased-compression-cap") + ? HARD_MAXIMUM_UNCOMPRESSED_SIZE : VANILLA_MAXIMUM_UNCOMPRESSED_SIZE; + private final int threshold; private final VelocityCompressor compressor; @@ -28,20 +32,21 @@ public class MinecraftCompressDecoder extends MessageToMessageDecoder { protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { int claimedUncompressedSize = ProtocolUtils.readVarInt(in); if (claimedUncompressedSize == 0) { - // Strip the now-useless uncompressed size, this message is already uncompressed. + // This message is not compressed. out.add(in.retainedSlice()); return; } checkFrame(claimedUncompressedSize >= threshold, "Uncompressed size %s is less than" + " threshold %s", claimedUncompressedSize, threshold); - int allowedMax = Math.min(claimedUncompressedSize, HARD_MAXIMUM_UNCOMPRESSED_SIZE); - int initialCapacity = Math.min(claimedUncompressedSize, SOFT_MAXIMUM_UNCOMPRESSED_SIZE); + checkFrame(claimedUncompressedSize <= UNCOMPRESSED_CAP, + "Uncompressed size %s exceeds hard threshold of %s", claimedUncompressedSize, + UNCOMPRESSED_CAP); ByteBuf compatibleIn = ensureCompatible(ctx.alloc(), compressor, in); - ByteBuf uncompressed = preferredBuffer(ctx.alloc(), compressor, initialCapacity); + ByteBuf uncompressed = preferredBuffer(ctx.alloc(), compressor, claimedUncompressedSize); try { - compressor.inflate(compatibleIn, uncompressed, allowedMax); + compressor.inflate(compatibleIn, uncompressed, claimedUncompressedSize); out.add(uncompressed); } catch (Exception e) { uncompressed.release(); diff --git a/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressEncoder.java b/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressEncoder.java index 0c9105e92..c36bfd8a3 100644 --- a/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressEncoder.java +++ b/proxy/src/main/java/com/velocitypowered/proxy/protocol/netty/MinecraftCompressEncoder.java @@ -38,8 +38,11 @@ public class MinecraftCompressEncoder extends MessageToByteEncoder { @Override protected ByteBuf allocateBuffer(ChannelHandlerContext ctx, ByteBuf msg, boolean preferDirect) throws Exception { - int initialBufferSize = msg.readableBytes() <= threshold ? msg.readableBytes() + 1 : - msg.readableBytes() / 3; + // Follow the advice of https://github.com/ebiggers/libdeflate/blob/master/libdeflate.h#L103 + // here for compression. The maximum buffer size if the data compresses well (which is almost + // always the case) is one less the input buffer. + int offset = msg.readableBytes() < threshold ? 1 : -1; + int initialBufferSize = msg.readableBytes() + offset; return MoreByteBufUtils.preferredBuffer(ctx.alloc(), compressor, initialBufferSize); }