From 19dc934e391752b78f78eca60c2c1ca5f6ac647b Mon Sep 17 00:00:00 2001 From: Eugene Kliuchnikov Date: Thu, 1 Jun 2017 13:51:18 +0200 Subject: Add JNI wrappers. (#556) --- BUILD | 63 +++++- java/org/brotli/dec/BUILD | 7 +- java/org/brotli/dec/pom.xml | 8 +- java/org/brotli/integration/BUILD | 22 +- java/org/brotli/wrapper/common/BUILD | 65 ++++++ java/org/brotli/wrapper/common/BrotliCommon.java | 130 ++++++++++++ java/org/brotli/wrapper/common/CommonJNI.java | 16 ++ .../wrapper/common/SetRfcDictionaryTest.java | 102 +++++++++ .../wrapper/common/SetZeroDictionaryTest.java | 53 +++++ java/org/brotli/wrapper/common/common_jni.cc | 47 +++++ java/org/brotli/wrapper/dec/BUILD | 73 +++++++ .../brotli/wrapper/dec/BrotliDecoderChannel.java | 74 +++++++ .../wrapper/dec/BrotliDecoderChannelTest.java | 89 ++++++++ java/org/brotli/wrapper/dec/BrotliInputStream.java | 108 ++++++++++ .../brotli/wrapper/dec/BrotliInputStreamTest.java | 87 ++++++++ java/org/brotli/wrapper/dec/Decoder.java | 160 +++++++++++++++ java/org/brotli/wrapper/dec/DecoderJNI.java | 117 +++++++++++ java/org/brotli/wrapper/dec/DecoderTest.java | 82 ++++++++ java/org/brotli/wrapper/dec/decoder_jni.cc | 228 +++++++++++++++++++++ java/org/brotli/wrapper/enc/BUILD | 98 +++++++++ .../brotli/wrapper/enc/BrotliEncoderChannel.java | 82 ++++++++ .../wrapper/enc/BrotliEncoderChannelTest.java | 122 +++++++++++ .../org/brotli/wrapper/enc/BrotliOutputStream.java | 95 +++++++++ .../brotli/wrapper/enc/BrotliOutputStreamTest.java | 123 +++++++++++ java/org/brotli/wrapper/enc/Encoder.java | 200 ++++++++++++++++++ java/org/brotli/wrapper/enc/EncoderJNI.java | 111 ++++++++++ java/org/brotli/wrapper/enc/EncoderTest.java | 83 ++++++++ .../wrapper/enc/UseCustomDictionaryTest.java | 104 ++++++++++ java/org/brotli/wrapper/enc/encoder_jni.cc | 224 ++++++++++++++++++++ scripts/appveyor.yml | 2 +- 30 files changed, 2766 insertions(+), 9 deletions(-) create mode 100755 java/org/brotli/wrapper/common/BUILD create mode 100755 java/org/brotli/wrapper/common/BrotliCommon.java create mode 100755 java/org/brotli/wrapper/common/CommonJNI.java create mode 100755 java/org/brotli/wrapper/common/SetRfcDictionaryTest.java create mode 100755 java/org/brotli/wrapper/common/SetZeroDictionaryTest.java create mode 100755 java/org/brotli/wrapper/common/common_jni.cc create mode 100755 java/org/brotli/wrapper/dec/BUILD create mode 100755 java/org/brotli/wrapper/dec/BrotliDecoderChannel.java create mode 100755 java/org/brotli/wrapper/dec/BrotliDecoderChannelTest.java create mode 100755 java/org/brotli/wrapper/dec/BrotliInputStream.java create mode 100755 java/org/brotli/wrapper/dec/BrotliInputStreamTest.java create mode 100755 java/org/brotli/wrapper/dec/Decoder.java create mode 100755 java/org/brotli/wrapper/dec/DecoderJNI.java create mode 100755 java/org/brotli/wrapper/dec/DecoderTest.java create mode 100755 java/org/brotli/wrapper/dec/decoder_jni.cc create mode 100755 java/org/brotli/wrapper/enc/BUILD create mode 100755 java/org/brotli/wrapper/enc/BrotliEncoderChannel.java create mode 100755 java/org/brotli/wrapper/enc/BrotliEncoderChannelTest.java create mode 100755 java/org/brotli/wrapper/enc/BrotliOutputStream.java create mode 100755 java/org/brotli/wrapper/enc/BrotliOutputStreamTest.java create mode 100755 java/org/brotli/wrapper/enc/Encoder.java create mode 100755 java/org/brotli/wrapper/enc/EncoderJNI.java create mode 100755 java/org/brotli/wrapper/enc/EncoderTest.java create mode 100755 java/org/brotli/wrapper/enc/UseCustomDictionaryTest.java create mode 100755 java/org/brotli/wrapper/enc/encoder_jni.cc diff --git a/BUILD b/BUILD index 1355c1b..d25a5db 100644 --- a/BUILD +++ b/BUILD @@ -43,7 +43,10 @@ genrule( cc_library( name = "jni_inc", - hdrs = [":jni/jni.h", ":jni/jni_md.h"], + hdrs = [ + ":jni/jni.h", + ":jni/jni_md.h", + ], includes = ["jni"], ) @@ -141,6 +144,64 @@ cc_binary( ], ) +######################################################## +# WARNING: do not (transitively) depend on this target! +######################################################## +cc_library( + name = "jni", + srcs = [ + ":common_sources", + ":dec_sources", + ":enc_sources", + "//java/org/brotli/wrapper/common:jni_src", + "//java/org/brotli/wrapper/dec:jni_src", + "//java/org/brotli/wrapper/enc:jni_src", + ], + hdrs = [ + ":common_headers", + ":dec_headers", + ":enc_headers", + ], + deps = [ + ":brotli_inc", + ":jni_inc", + ], + alwayslink = 1, +) + +######################################################## +# WARNING: do not (transitively) depend on this target! +######################################################## +cc_library( + name = "jni_no_dictionary_data", + srcs = [ + ":common_sources", + ":dec_sources", + ":enc_sources", + "//java/org/brotli/wrapper/common:jni_src", + "//java/org/brotli/wrapper/dec:jni_src", + "//java/org/brotli/wrapper/enc:jni_src", + ], + hdrs = [ + ":common_headers", + ":dec_headers", + ":enc_headers", + ], + defines = [ + "BROTLI_EXTERNAL_DICTIONARY_DATA=", + ], + deps = [ + ":brotli_inc", + ":jni_inc", + ], + alwayslink = 1, +) + +filegroup( + name = "dictionary", + srcs = ["c/common/dictionary.bin"], +) + load("@io_bazel_rules_go//go:def.bzl", "go_prefix") go_prefix("github.com/google/brotli") diff --git a/java/org/brotli/dec/BUILD b/java/org/brotli/dec/BUILD index e7499a9..32c5897 100755 --- a/java/org/brotli/dec/BUILD +++ b/java/org/brotli/dec/BUILD @@ -7,17 +7,20 @@ licenses(["notice"]) # MIT java_library( name = "dec", - srcs = glob(["*.java"], exclude = ["*Test*.java"]), + srcs = glob( + ["*.java"], + exclude = ["*Test*.java"], + ), ) java_library( name = "test_lib", + testonly = 1, srcs = glob(["*Test*.java"]), deps = [ ":dec", "@junit_junit//jar", ], - testonly = 1, ) java_test( diff --git a/java/org/brotli/dec/pom.xml b/java/org/brotli/dec/pom.xml index ac6172f..24b7aa1 100755 --- a/java/org/brotli/dec/pom.xml +++ b/java/org/brotli/dec/pom.xml @@ -27,16 +27,16 @@ maven-compiler-plugin - **/dec/*.java + org/brotli/dec/*.java **/*Test*.java - **/dec/*Test*.java + org/brotli/dec/*Test*.java - **/dec/SetDictionaryTest.java + org/brotli/dec/SetDictionaryTest.java @@ -53,7 +53,7 @@ - **/dec/*.java + org/brotli/dec/*.java **/*Test*.java diff --git a/java/org/brotli/integration/BUILD b/java/org/brotli/integration/BUILD index 31a2be8..ac9bc2c 100755 --- a/java/org/brotli/integration/BUILD +++ b/java/org/brotli/integration/BUILD @@ -4,6 +4,10 @@ java_library( name = "bundle_helper", srcs = ["BundleHelper.java"], + visibility = [ + "//java/org/brotli/wrapper/dec:__pkg__", + "//java/org/brotli/wrapper/enc:__pkg__", + ], ) java_library( @@ -34,10 +38,26 @@ java_test( name = "bundle_checker_fuzz_test", args = [ "-s", - "java/org/brotli/integration/fuzz_data.zip" + "java/org/brotli/integration/fuzz_data.zip", ], data = ["fuzz_data.zip"], main_class = "org.brotli.integration.BundleChecker", use_testrunner = 0, runtime_deps = [":bundle_checker"], ) + +filegroup( + name = "test_data", + srcs = ["test_data.zip"], + visibility = [ + "//java/org/brotli/wrapper/dec:__pkg__", + ], +) + +filegroup( + name = "test_corpus", + srcs = ["test_corpus.zip"], + visibility = [ + "//java/org/brotli/wrapper/enc:__pkg__", + ], +) diff --git a/java/org/brotli/wrapper/common/BUILD b/java/org/brotli/wrapper/common/BUILD new file mode 100755 index 0000000..8623272 --- /dev/null +++ b/java/org/brotli/wrapper/common/BUILD @@ -0,0 +1,65 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +filegroup( + name = "jni_src", + srcs = ["common_jni.cc"], +) + +######################################### +# WARNING: do not depend on this target! +######################################### +java_library( + name = "common_no_dictionary_data", + srcs = glob( + ["*.java"], + exclude = ["*Test*.java"], + ), + deps = ["//:jni_no_dictionary_data"], +) + +######################################### +# WARNING: do not depend on this target! +######################################### +java_library( + name = "common", + srcs = glob( + ["*.java"], + exclude = ["*Test*.java"], + ), + deps = ["//:jni"], +) + +java_test( + name = "SetZeroDictionaryTest", + size = "small", + srcs = ["SetZeroDictionaryTest.java"], + data = ["//:jni_no_dictionary_data"], # Bazel JNI workaround + deps = [ + ":common_no_dictionary_data", + "//java/org/brotli/wrapper/dec", + "@junit_junit//jar", + ], +) + +filegroup( + name = "rfc_dictionary", + srcs = ["//:dictionary"], +) + +java_test( + name = "SetRfcDictionaryTest", + size = "small", + srcs = ["SetRfcDictionaryTest.java"], + data = [ + ":rfc_dictionary", + "//:jni_no_dictionary_data", # Bazel JNI workaround + ], + jvm_flags = ["-DRFC_DICTIONARY=$(location :rfc_dictionary)"], + deps = [ + ":common_no_dictionary_data", + "//java/org/brotli/wrapper/dec", + "@junit_junit//jar", + ], +) diff --git a/java/org/brotli/wrapper/common/BrotliCommon.java b/java/org/brotli/wrapper/common/BrotliCommon.java new file mode 100755 index 0000000..9419e42 --- /dev/null +++ b/java/org/brotli/wrapper/common/BrotliCommon.java @@ -0,0 +1,130 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.common; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; + +/** + * JNI wrapper for brotli common. + */ +public class BrotliCommon { + public static final int RFC_DICTIONARY_SIZE = 122784; + + /* 96cecd2ee7a666d5aa3627d74735b32a */ + private static final byte[] RFC_DICTIONARY_MD5 = { + -106, -50, -51, 46, -25, -90, 102, -43, -86, 54, 39, -41, 71, 53, -77, 42 + }; + + /* 72b41051cb61a9281ba3c4414c289da50d9a7640 */ + private static final byte[] RFC_DICTIONARY_SHA_1 = { + 114, -76, 16, 81, -53, 97, -87, 40, 27, -93, -60, 65, 76, 40, -99, -91, 13, -102, 118, 64 + }; + + /* 20e42eb1b511c21806d4d227d07e5dd06877d8ce7b3a817f378f313653f35c70 */ + private static final byte[] RFC_DICTIONARY_SHA_256 = { + 32, -28, 46, -79, -75, 17, -62, 24, 6, -44, -46, 39, -48, 126, 93, -48, + 104, 119, -40, -50, 123, 58, -127, 127, 55, -113, 49, 54, 83, -13, 92, 112 + }; + + private static boolean isDictionaryDataSet; + private static final Object mutex = new Object(); + + /** + * Checks if the given checksum matches MD5 checksum of the RFC dictionary. + */ + public static boolean checkDictionaryDataMd5(byte[] digest) { + return Arrays.equals(RFC_DICTIONARY_MD5, digest); + } + + /** + * Checks if the given checksum matches SHA-1 checksum of the RFC dictionary. + */ + public static boolean checkDictionaryDataSha1(byte[] digest) { + return Arrays.equals(RFC_DICTIONARY_SHA_1, digest); + } + + /** + * Checks if the given checksum matches SHA-256 checksum of the RFC dictionary. + */ + public static boolean checkDictionaryDataSha256(byte[] digest) { + return Arrays.equals(RFC_DICTIONARY_SHA_256, digest); + } + + /** + * Copy bytes to a new direct ByteBuffer. + * + * Direct byte buffers are used to supply native code with large data chunks. + */ + public static ByteBuffer makeNative(byte[] data) { + ByteBuffer result = ByteBuffer.allocateDirect(data.length); + result.put(data); + return result; + } + + /** + * Copies data and sets it to be brotli dictionary. + */ + public static void setDictionaryData(byte[] data) { + if (data.length != RFC_DICTIONARY_SIZE) { + throw new IllegalArgumentException("invalid dictionary size"); + } + synchronized (mutex) { + if (isDictionaryDataSet) { + return; + } + setDictionaryData(makeNative(data)); + } + } + + /** + * Reads data and sets it to be brotli dictionary. + */ + public static void setDictionaryData(InputStream src) throws IOException { + synchronized (mutex) { + if (isDictionaryDataSet) { + return; + } + ByteBuffer copy = ByteBuffer.allocateDirect(RFC_DICTIONARY_SIZE); + byte[] buffer = new byte[4096]; + int readBytes; + while ((readBytes = src.read(buffer)) != -1) { + if (copy.remaining() < readBytes) { + throw new IllegalArgumentException("invalid dictionary size"); + } + copy.put(buffer, 0, readBytes); + } + if (copy.remaining() != 0) { + throw new IllegalArgumentException("invalid dictionary size " + copy.remaining()); + } + setDictionaryData(copy); + } + } + + /** + * Sets data to be brotli dictionary. + */ + public static void setDictionaryData(ByteBuffer data) { + if (!data.isDirect()) { + throw new IllegalArgumentException("direct byte buffer is expected"); + } + if (data.capacity() != RFC_DICTIONARY_SIZE) { + throw new IllegalArgumentException("invalid dictionary size"); + } + synchronized (mutex) { + if (isDictionaryDataSet) { + return; + } + if (!CommonJNI.nativeSetDictionaryData(data)) { + throw new RuntimeException("setting dictionary failed"); + } + isDictionaryDataSet = true; + } + } +} diff --git a/java/org/brotli/wrapper/common/CommonJNI.java b/java/org/brotli/wrapper/common/CommonJNI.java new file mode 100755 index 0000000..d662546 --- /dev/null +++ b/java/org/brotli/wrapper/common/CommonJNI.java @@ -0,0 +1,16 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.common; + +import java.nio.ByteBuffer; + +/** + * JNI wrapper for brotli common. + */ +class CommonJNI { + static native boolean nativeSetDictionaryData(ByteBuffer data); +} diff --git a/java/org/brotli/wrapper/common/SetRfcDictionaryTest.java b/java/org/brotli/wrapper/common/SetRfcDictionaryTest.java new file mode 100755 index 0000000..8577800 --- /dev/null +++ b/java/org/brotli/wrapper/common/SetRfcDictionaryTest.java @@ -0,0 +1,102 @@ +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.common; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.brotli.wrapper.dec.BrotliInputStream; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link BrotliCommon}. + */ +@RunWith(JUnit4.class) +public class SetRfcDictionaryTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni_Uno_Udictionary_Udata.so").getAbsolutePath()); + } + + @Test + public void testRfcDictionaryChecksums() throws IOException, NoSuchAlgorithmException { + FileInputStream dictionary = new FileInputStream(System.getProperty("RFC_DICTIONARY")); + byte[] data = new byte[BrotliCommon.RFC_DICTIONARY_SIZE + 1]; + int offset = 0; + try { + int readBytes; + while ((readBytes = dictionary.read(data, offset, data.length - offset)) != -1) { + offset += readBytes; + if (offset > BrotliCommon.RFC_DICTIONARY_SIZE) { + break; + } + } + } finally { + dictionary.close(); + } + if (offset != BrotliCommon.RFC_DICTIONARY_SIZE) { + fail("dictionary size mismatch"); + } + + MessageDigest md5 = MessageDigest.getInstance("MD5"); + md5.update(data, 0, offset); + assertTrue(BrotliCommon.checkDictionaryDataMd5(md5.digest())); + + MessageDigest sha1 = MessageDigest.getInstance("SHA-1"); + sha1.update(data, 0, offset); + assertTrue(BrotliCommon.checkDictionaryDataSha1(sha1.digest())); + + MessageDigest sha256 = MessageDigest.getInstance("SHA-256"); + sha256.update(data, 0, offset); + assertTrue(BrotliCommon.checkDictionaryDataSha256(sha256.digest())); + } + + @Test + public void testSetRfcDictionary() throws IOException { + /* "leftdatadataleft" encoded with dictionary words. */ + byte[] data = {27, 15, 0, 0, 0, 0, -128, -29, -76, 13, 0, 0, 7, 91, 38, 49, 64, 2, 0, -32, 78, + 27, 65, -128, 32, 80, 16, 36, 8, 6}; + FileInputStream dictionary = new FileInputStream(System.getProperty("RFC_DICTIONARY")); + try { + BrotliCommon.setDictionaryData(dictionary); + } finally { + dictionary.close(); + } + + BrotliInputStream decoder = new BrotliInputStream(new ByteArrayInputStream(data)); + byte[] output = new byte[17]; + int offset = 0; + try { + int bytesRead; + while ((bytesRead = decoder.read(output, offset, 17 - offset)) != -1) { + offset += bytesRead; + } + } finally { + decoder.close(); + } + assertEquals(16, offset); + byte[] expected = { + 'l', 'e', 'f', 't', + 'd', 'a', 't', 'a', + 'd', 'a', 't', 'a', + 'l', 'e', 'f', 't', + 0 + }; + assertArrayEquals(expected, output); + } +} diff --git a/java/org/brotli/wrapper/common/SetZeroDictionaryTest.java b/java/org/brotli/wrapper/common/SetZeroDictionaryTest.java new file mode 100755 index 0000000..9046e31 --- /dev/null +++ b/java/org/brotli/wrapper/common/SetZeroDictionaryTest.java @@ -0,0 +1,53 @@ +/* Copyright 2015 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.common; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import org.brotli.wrapper.dec.BrotliInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for {@link BrotliCommon}. + */ +@RunWith(JUnit4.class) +public class SetZeroDictionaryTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni_Uno_Udictionary_Udata.so").getAbsolutePath()); + } + + @Test + public void testZeroDictionary() throws IOException { + /* "leftdatadataleft" encoded with dictionary words. */ + byte[] data = {27, 15, 0, 0, 0, 0, -128, -29, -76, 13, 0, 0, 7, 91, 38, 49, 64, 2, 0, -32, 78, + 27, 65, -128, 32, 80, 16, 36, 8, 6}; + byte[] dictionary = new byte[BrotliCommon.RFC_DICTIONARY_SIZE]; + BrotliCommon.setDictionaryData(dictionary); + + BrotliInputStream decoder = new BrotliInputStream(new ByteArrayInputStream(data)); + byte[] output = new byte[17]; + int offset = 0; + try { + int bytesRead; + while ((bytesRead = decoder.read(output, offset, 17 - offset)) != -1) { + offset += bytesRead; + } + } finally { + decoder.close(); + } + assertEquals(16, offset); + assertArrayEquals(new byte[17], output); + } +} diff --git a/java/org/brotli/wrapper/common/common_jni.cc b/java/org/brotli/wrapper/common/common_jni.cc new file mode 100755 index 0000000..2ea2ad8 --- /dev/null +++ b/java/org/brotli/wrapper/common/common_jni.cc @@ -0,0 +1,47 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +#include + +#include "../common/dictionary.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Set data to be brotli dictionary data. + * + * @param buffer direct ByteBuffer + * @returns false if dictionary data was already set; otherwise true + */ +JNIEXPORT jint JNICALL +Java_org_brotli_wrapper_common_CommonJNI_nativeSetDictionaryData( + JNIEnv* env, jobject /*jobj*/, jobject buffer) { + jobject buffer_ref = env->NewGlobalRef(buffer); + if (!buffer_ref) { + return false; + } + uint8_t* data = static_cast(env->GetDirectBufferAddress(buffer)); + if (!data) { + env->DeleteGlobalRef(buffer_ref); + return false; + } + + BrotliSetDictionaryData(data); + + const BrotliDictionary* dictionary = BrotliGetDictionary(); + if (dictionary->data != data) { + env->DeleteGlobalRef(buffer_ref); + } else { + /* Don't release reference; it is an intended memory leak. */ + } + return true; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/org/brotli/wrapper/dec/BUILD b/java/org/brotli/wrapper/dec/BUILD new file mode 100755 index 0000000..58ab3d4 --- /dev/null +++ b/java/org/brotli/wrapper/dec/BUILD @@ -0,0 +1,73 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +filegroup( + name = "jni_src", + srcs = ["decoder_jni.cc"], +) + +######################################### +# WARNING: do not depend on this target! +######################################### +java_library( + name = "dec", + srcs = glob( + ["*.java"], + exclude = ["*Test*.java"], + ), + deps = ["//:jni"], +) + +filegroup( + name = "test_bundle", + srcs = ["//java/org/brotli/integration:test_data"], +) + +java_test( + name = "BrotliDecoderChannelTest", + size = "large", + srcs = ["BrotliDecoderChannelTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + deps = [ + ":dec", + "//java/org/brotli/integration:bundle_helper", + "@junit_junit//jar", + ], +) + +java_test( + name = "BrotliInputStreamTest", + size = "large", + srcs = ["BrotliInputStreamTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + deps = [ + ":dec", + "//java/org/brotli/integration:bundle_helper", + "@junit_junit//jar", + ], +) + +java_test( + name = "DecoderTest", + size = "large", + srcs = ["DecoderTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + deps = [ + ":dec", + "//java/org/brotli/integration:bundle_helper", + "@junit_junit//jar", + ], +) diff --git a/java/org/brotli/wrapper/dec/BrotliDecoderChannel.java b/java/org/brotli/wrapper/dec/BrotliDecoderChannel.java new file mode 100755 index 0000000..e7b4bdf --- /dev/null +++ b/java/org/brotli/wrapper/dec/BrotliDecoderChannel.java @@ -0,0 +1,74 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.ReadableByteChannel; + +/** + * ReadableByteChannel that wraps native brotli decoder. + */ +public class BrotliDecoderChannel extends Decoder implements ReadableByteChannel { + /** The default internal buffer size used by the decoder. */ + private static final int DEFAULT_BUFFER_SIZE = 16384; + + private final Object mutex = new Object(); + + /** + * Creates a BrotliDecoderChannel. + * + * @param source underlying source + * @param bufferSize intermediate buffer size + * @param customDictionary initial LZ77 dictionary + */ + public BrotliDecoderChannel(ReadableByteChannel source, int bufferSize, + ByteBuffer customDictionary) throws IOException { + super(source, bufferSize, customDictionary); + } + + public BrotliDecoderChannel(ReadableByteChannel source, int bufferSize) throws IOException { + super(source, bufferSize, null); + } + + public BrotliDecoderChannel(ReadableByteChannel source) throws IOException { + this(source, DEFAULT_BUFFER_SIZE); + } + + @Override + public boolean isOpen() { + synchronized (mutex) { + return !closed; + } + } + + @Override + public void close() throws IOException { + synchronized (mutex) { + super.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + synchronized (mutex) { + if (closed) { + throw new ClosedChannelException(); + } + int result = 0; + while (dst.hasRemaining()) { + int outputSize = decode(); + if (outputSize == -1) { + return result == 0 ? -1 : result; + } + result += consume(dst); + } + return result; + } + } +} diff --git a/java/org/brotli/wrapper/dec/BrotliDecoderChannelTest.java b/java/org/brotli/wrapper/dec/BrotliDecoderChannelTest.java new file mode 100755 index 0000000..b6fc036 --- /dev/null +++ b/java/org/brotli/wrapper/dec/BrotliDecoderChannelTest.java @@ -0,0 +1,89 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import static org.junit.Assert.assertEquals; + +import org.brotli.integration.BundleHelper; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for {@link org.brotli.wrapper.dec.BrotliDecoderChannel}. */ +@RunWith(AllTests.class) +public class BrotliDecoderChannelTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new ChannelTestCase(entry)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class ChannelTestCase extends TestCase { + final String entryName; + ChannelTestCase(String entryName) { + super("BrotliDecoderChannelTest." + entryName); + this.entryName = entryName; + } + + @Override + protected void runTest() throws Throwable { + BrotliDecoderChannelTest.run(entryName); + } + } + + private static void run(String entryName) throws Throwable { + InputStream bundle = getBundle(); + byte[] compressed; + try { + compressed = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + if (compressed == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + ReadableByteChannel src = Channels.newChannel(new ByteArrayInputStream(compressed)); + ReadableByteChannel decoder = new BrotliDecoderChannel(src); + long crc; + try { + crc = BundleHelper.fingerprintStream(Channels.newInputStream(decoder)); + } finally { + decoder.close(); + } + assertEquals(BundleHelper.getExpectedFingerprint(entryName), crc); + } +} diff --git a/java/org/brotli/wrapper/dec/BrotliInputStream.java b/java/org/brotli/wrapper/dec/BrotliInputStream.java new file mode 100755 index 0000000..63da868 --- /dev/null +++ b/java/org/brotli/wrapper/dec/BrotliInputStream.java @@ -0,0 +1,108 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; + +/** + * InputStream that wraps native brotli decoder. + */ +public class BrotliInputStream extends InputStream { + /** The default internal buffer size used by the decoder. */ + private static final int DEFAULT_BUFFER_SIZE = 16384; + + private final Decoder decoder; + + /** + * Creates a BrotliInputStream. + * + * @param source underlying source + * @param bufferSize intermediate buffer size + * @param customDictionary initial LZ77 dictionary + */ + public BrotliInputStream(InputStream source, int bufferSize, ByteBuffer customDictionary) + throws IOException { + this.decoder = new Decoder(Channels.newChannel(source), bufferSize, customDictionary); + } + + public BrotliInputStream(InputStream source, int bufferSize) throws IOException { + this.decoder = new Decoder(Channels.newChannel(source), bufferSize, null); + } + + public BrotliInputStream(InputStream source) throws IOException { + this(source, DEFAULT_BUFFER_SIZE); + } + + @Override + public void close() throws IOException { + decoder.close(); + } + + @Override + public int available() { + return (decoder.buffer != null) ? decoder.buffer.remaining() : 0; + } + + @Override + public int read() throws IOException { + if (decoder.closed) { + throw new IOException("read after close"); + } + if (decoder.decode() == -1) { + return -1; + } + return decoder.buffer.get() & 0xFF; + } + + @Override + public int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (decoder.closed) { + throw new IOException("read after close"); + } + if (decoder.decode() == -1) { + return -1; + } + int result = 0; + while (len > 0) { + int limit = Math.min(len, decoder.buffer.remaining()); + decoder.buffer.get(b, off, limit); + off += limit; + len -= limit; + result += limit; + if (decoder.decode() == -1) { + break; + } + } + return result; + } + + @Override + public long skip(long n) throws IOException { + if (decoder.closed) { + throw new IOException("read after close"); + } + long result = 0; + while (n > 0) { + if (decoder.decode() == -1) { + break; + } + int limit = (int) Math.min(n, (long) decoder.buffer.remaining()); + decoder.discard(limit); + result += limit; + n -= limit; + } + return result; + } +} diff --git a/java/org/brotli/wrapper/dec/BrotliInputStreamTest.java b/java/org/brotli/wrapper/dec/BrotliInputStreamTest.java new file mode 100755 index 0000000..aec26a0 --- /dev/null +++ b/java/org/brotli/wrapper/dec/BrotliInputStreamTest.java @@ -0,0 +1,87 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import static org.junit.Assert.assertEquals; + +import org.brotli.integration.BundleHelper; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for {@link org.brotli.wrapper.dec.BrotliInputStream}. */ +@RunWith(AllTests.class) +public class BrotliInputStreamTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new StreamTestCase(entry)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class StreamTestCase extends TestCase { + final String entryName; + StreamTestCase(String entryName) { + super("BrotliInputStreamTest." + entryName); + this.entryName = entryName; + } + + @Override + protected void runTest() throws Throwable { + BrotliInputStreamTest.run(entryName); + } + } + + private static void run(String entryName) throws Throwable { + InputStream bundle = getBundle(); + byte[] compressed; + try { + compressed = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + if (compressed == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + InputStream src = new ByteArrayInputStream(compressed); + InputStream decoder = new BrotliInputStream(src); + long crc; + try { + crc = BundleHelper.fingerprintStream(decoder); + } finally { + decoder.close(); + } + assertEquals(BundleHelper.getExpectedFingerprint(entryName), crc); + } +} diff --git a/java/org/brotli/wrapper/dec/Decoder.java b/java/org/brotli/wrapper/dec/Decoder.java new file mode 100755 index 0000000..366045f --- /dev/null +++ b/java/org/brotli/wrapper/dec/Decoder.java @@ -0,0 +1,160 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.util.ArrayList; + +/** + * Base class for InputStream / Channel implementations. + */ +class Decoder { + private final ReadableByteChannel source; + private final DecoderJNI.Wrapper decoder; + ByteBuffer buffer; + boolean closed; + + /** + * Creates a Decoder wrapper. + * + * @param source underlying source + * @param inputBufferSize read buffer size + */ + public Decoder(ReadableByteChannel source, int inputBufferSize, ByteBuffer customDictionary) + throws IOException { + if (inputBufferSize <= 0) { + throw new IllegalArgumentException("buffer size must be positive"); + } + if (source == null) { + throw new NullPointerException("source can not be null"); + } + this.source = source; + this.decoder = new DecoderJNI.Wrapper(inputBufferSize, customDictionary); + } + + private void fail(String message) throws IOException { + try { + close(); + } catch (IOException ex) { + /* Ignore */ + } + throw new IOException(message); + } + + /** + * Continue decoding. + * + * @return -1 if stream is finished, or number of bytes available in read buffer (> 0) + */ + int decode() throws IOException { + while (true) { + if (buffer != null) { + if (!buffer.hasRemaining()) { + buffer = null; + } else { + return buffer.remaining(); + } + } + + switch (decoder.getStatus()) { + case DONE: + return -1; + + case OK: + decoder.push(0); + break; + + case NEEDS_MORE_INPUT: + ByteBuffer inputBuffer = decoder.getInputBuffer(); + inputBuffer.clear(); + int bytesRead = source.read(inputBuffer); + if (bytesRead == -1) { + fail("unexpected end of input"); + } + decoder.push(bytesRead); + break; + + case NEEDS_MORE_OUTPUT: + buffer = decoder.pull(); + break; + + default: + fail("corrupted input"); + } + } + } + + void discard(int length) { + buffer.position(buffer.position() + length); + if (!buffer.hasRemaining()) { + buffer = null; + } + } + + int consume(ByteBuffer dst) { + ByteBuffer slice = buffer.slice(); + int limit = Math.min(slice.remaining(), dst.remaining()); + slice.limit(limit); + dst.put(slice); + discard(limit); + return limit; + } + + void close() throws IOException { + if (closed) { + return; + } + closed = true; + decoder.destroy(); + source.close(); + } + + /** + * Decodes the given data buffer. + */ + public static byte[] decompress(byte[] data) throws IOException { + DecoderJNI.Wrapper decoder = new DecoderJNI.Wrapper(data.length, null); + ArrayList output = new ArrayList(); + int totalOutputSize = 0; + try { + decoder.getInputBuffer().put(data); + decoder.push(data.length); + while (decoder.getStatus() != DecoderJNI.Status.DONE) { + switch (decoder.getStatus()) { + case OK: + decoder.push(0); + break; + + case NEEDS_MORE_OUTPUT: + ByteBuffer buffer = decoder.pull(); + byte[] chunk = new byte[buffer.remaining()]; + buffer.get(chunk); + output.add(chunk); + totalOutputSize += chunk.length; + break; + + default: + throw new IOException("corrupted input"); + } + } + } finally { + decoder.destroy(); + } + if (output.size() == 1) { + return output.get(0); + } + byte[] result = new byte[totalOutputSize]; + int offset = 0; + for (byte[] chunk : output) { + System.arraycopy(chunk, 0, result, offset, chunk.length); + offset += chunk.length; + } + return result; + } +} diff --git a/java/org/brotli/wrapper/dec/DecoderJNI.java b/java/org/brotli/wrapper/dec/DecoderJNI.java new file mode 100755 index 0000000..ffd3ce9 --- /dev/null +++ b/java/org/brotli/wrapper/dec/DecoderJNI.java @@ -0,0 +1,117 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * JNI wrapper for brotli decoder. + */ +class DecoderJNI { + private static native ByteBuffer nativeCreate(long[] context, ByteBuffer customDictionary); + private static native void nativePush(long[] context, int length); + private static native ByteBuffer nativePull(long[] context); + private static native void nativeDestroy(long[] context); + + enum Status { + ERROR, + DONE, + NEEDS_MORE_INPUT, + NEEDS_MORE_OUTPUT, + OK + }; + + static class Wrapper { + private final long[] context = new long[2]; + private final ByteBuffer inputBuffer; + private Status lastStatus = Status.NEEDS_MORE_INPUT; + + Wrapper(int inputBufferSize, ByteBuffer customDictionary) throws IOException { + if (customDictionary != null && !customDictionary.isDirect()) { + throw new IllegalArgumentException("LZ77 dictionary must be direct ByteBuffer"); + } + this.context[1] = inputBufferSize; + this.inputBuffer = nativeCreate(this.context, customDictionary); + if (this.context[0] == 0) { + throw new IOException("failed to initialize native brotli decoder"); + } + } + + void push(int length) { + if (length < 0) { + throw new IllegalArgumentException("negative block length"); + } + if (context[0] == 0) { + throw new IllegalStateException("brotli decoder is already destroyed"); + } + if (lastStatus != Status.NEEDS_MORE_INPUT && lastStatus != Status.OK) { + throw new IllegalStateException("pushing input to decoder in " + lastStatus + " state"); + } + if (lastStatus == Status.OK && length != 0) { + throw new IllegalStateException("pushing input to decoder in OK state"); + } + nativePush(context, length); + parseStatus(); + } + + private void parseStatus() { + long status = context[1]; + if (status == 1) { + lastStatus = Status.DONE; + } else if (status == 2) { + lastStatus = Status.NEEDS_MORE_INPUT; + } else if (status == 3) { + lastStatus = Status.NEEDS_MORE_OUTPUT; + } else if (status == 4) { + lastStatus = Status.OK; + } else { + lastStatus = Status.ERROR; + } + } + + Status getStatus() { + return lastStatus; + } + + ByteBuffer getInputBuffer() { + return inputBuffer; + } + + ByteBuffer pull() { + if (context[0] == 0) { + throw new IllegalStateException("brotli decoder is already destroyed"); + } + if (lastStatus != Status.NEEDS_MORE_OUTPUT) { + throw new IllegalStateException("pulling output from decoder in " + lastStatus + " state"); + } + ByteBuffer result = nativePull(context); + parseStatus(); + return result; + } + + /** + * Releases native resources. + */ + void destroy() { + if (context[0] == 0) { + throw new IllegalStateException("brotli decoder is already destroyed"); + } + nativeDestroy(context); + context[0] = 0; + } + + @Override + protected void finalize() throws Throwable { + if (context[0] != 0) { + /* TODO: log resource leak? */ + destroy(); + } + super.finalize(); + } + } +} diff --git a/java/org/brotli/wrapper/dec/DecoderTest.java b/java/org/brotli/wrapper/dec/DecoderTest.java new file mode 100755 index 0000000..0a8970f --- /dev/null +++ b/java/org/brotli/wrapper/dec/DecoderTest.java @@ -0,0 +1,82 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.dec; + +import static org.junit.Assert.assertEquals; + +import org.brotli.integration.BundleHelper; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for {@link org.brotli.wrapper.dec.Decoder}. */ +@RunWith(AllTests.class) +public class DecoderTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new DecoderTestCase(entry)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class DecoderTestCase extends TestCase { + final String entryName; + DecoderTestCase(String entryName) { + super("DecoderTest." + entryName); + this.entryName = entryName; + } + + @Override + protected void runTest() throws Throwable { + DecoderTest.run(entryName); + } + } + + private static void run(String entryName) throws Throwable { + InputStream bundle = getBundle(); + byte[] compressed; + try { + compressed = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + if (compressed == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + byte[] decompressed = Decoder.decompress(compressed); + + long crc = BundleHelper.fingerprintStream(new ByteArrayInputStream(decompressed)); + assertEquals(BundleHelper.getExpectedFingerprint(entryName), crc); + } +} diff --git a/java/org/brotli/wrapper/dec/decoder_jni.cc b/java/org/brotli/wrapper/dec/decoder_jni.cc new file mode 100755 index 0000000..b06cb5d --- /dev/null +++ b/java/org/brotli/wrapper/dec/decoder_jni.cc @@ -0,0 +1,228 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +#include + +#include + +#include + +namespace { +/* A structure used to persist the decoder's state in between calls. */ +typedef struct DecoderHandle { + BrotliDecoderState* state; + + jobject custom_dictionary_ref; + + uint8_t* input_start; + size_t input_offset; + size_t input_length; +} DecoderHandle; + +/* Obtain handle from opaque pointer. */ +DecoderHandle* getHandle(void* opaque) { + return static_cast(opaque); +} + +} /* namespace */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Creates a new Decoder. + * + * Cookie to address created decoder is stored in out_cookie. In case of failure + * cookie is 0. + * + * @param ctx {out_cookie, in_directBufferSize} tuple + * @returns direct ByteBuffer if directBufferSize is not 0; otherwise null + */ +JNIEXPORT jobject JNICALL +Java_org_brotli_wrapper_dec_DecoderJNI_nativeCreate( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject custom_dictionary) { + bool ok = true; + DecoderHandle* handle = nullptr; + jlong context[2]; + env->GetLongArrayRegion(ctx, 0, 2, context); + size_t input_size = context[1]; + context[0] = 0; + handle = new (std::nothrow) DecoderHandle(); + ok = !!handle; + + if (ok) { + handle->custom_dictionary_ref = nullptr; + handle->input_offset = 0; + handle->input_length = 0; + handle->input_start = nullptr; + + if (input_size == 0) { + ok = false; + } else { + handle->input_start = new (std::nothrow) uint8_t[input_size]; + ok = !!handle->input_start; + } + } + + if (ok) { + handle->state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + ok = !!handle->state; + } + + if (ok && !!custom_dictionary) { + handle->custom_dictionary_ref = env->NewGlobalRef(custom_dictionary); + if (!!handle->custom_dictionary_ref) { + uint8_t* custom_dictionary_address = static_cast( + env->GetDirectBufferAddress(handle->custom_dictionary_ref)); + if (!!custom_dictionary_address) { + jlong capacity = + env->GetDirectBufferCapacity(handle->custom_dictionary_ref); + ok = (capacity > 0) && (capacity < (1 << 24)); + if (ok) { + size_t custom_dictionary_size = static_cast(capacity); + BrotliDecoderSetCustomDictionary( + handle->state, custom_dictionary_size, custom_dictionary_address); + } + } else { + ok = false; + } + } else { + ok = false; + } + } + + if (ok) { + /* TODO: future versions (e.g. when 128-bit architecture comes) + might require thread-safe cookie<->handle mapping. */ + context[0] = reinterpret_cast(handle); + } else if (!!handle) { + if (!!handle->custom_dictionary_ref) { + env->DeleteGlobalRef(handle->custom_dictionary_ref); + } + if (!!handle->input_start) delete[] handle->input_start; + delete handle; + } + + env->SetLongArrayRegion(ctx, 0, 2, context); + + if (!ok) { + return nullptr; + } + + return env->NewDirectByteBuffer(handle->input_start, input_size); +} + +/** + * Push data to decoder. + * + * status codes: + * - 0 error happened + * - 1 stream is finished, no more input / output expected + * - 2 needs more input to process further + * - 3 needs more output to process further + * - 4 ok, can proceed further without additional input + * + * @param ctx {in_cookie, out_status} tuple + * @param input_length number of bytes provided in input or direct input; + * 0 to process further previous input + */ +JNIEXPORT void JNICALL +Java_org_brotli_wrapper_dec_DecoderJNI_nativePush( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jint input_length) { + jlong context[2]; + env->GetLongArrayRegion(ctx, 0, 2, context); + DecoderHandle* handle = getHandle(reinterpret_cast(context[0])); + context[1] = 0; /* ERROR */ + env->SetLongArrayRegion(ctx, 0, 2, context); + + if (input_length != 0) { + /* Still have unconsumed data. Workflow is broken. */ + if (handle->input_offset < handle->input_length) { + return; + } + handle->input_offset = 0; + handle->input_length = input_length; + } + + /* Actual decompression. */ + const uint8_t* in = handle->input_start + handle->input_offset; + size_t in_size = handle->input_length - handle->input_offset; + size_t out_size = 0; + BrotliDecoderResult status = BrotliDecoderDecompressStream( + handle->state, &in_size, &in, &out_size, nullptr, nullptr); + handle->input_offset = handle->input_length - in_size; + switch (status) { + case BROTLI_DECODER_RESULT_SUCCESS: + /* Bytes after stream end are not allowed. */ + context[1] = (handle->input_offset == handle->input_length) ? 1 : 0; + break; + + case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + context[1] = 2; + break; + + case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + context[1] = 3; + break; + + default: + context[1] = 0; + break; + } + env->SetLongArrayRegion(ctx, 0, 2, context); +} + +/** + * Pull decompressed data from decoder. + * + * @param ctx {in_cookie, out_status} tuple + * @returns direct ByteBuffer; all the produced data MUST be consumed before + * any further invocation; null in case of error + */ +JNIEXPORT jobject JNICALL +Java_org_brotli_wrapper_dec_DecoderJNI_nativePull( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx) { + jlong context[2]; + env->GetLongArrayRegion(ctx, 0, 2, context); + DecoderHandle* handle = getHandle(reinterpret_cast(context[0])); + size_t data_length = 0; + const uint8_t* data = BrotliDecoderTakeOutput(handle->state, &data_length); + if (BrotliDecoderHasMoreOutput(handle->state)) { + context[1] = 3; + } else if (BrotliDecoderIsFinished(handle->state)) { + /* Bytes after stream end are not allowed. */ + context[1] = (handle->input_offset == handle->input_length) ? 1 : 0; + } else { + /* Can proceed, or more data is required? */ + context[1] = (handle->input_offset == handle->input_length) ? 2 : 4; + } + env->SetLongArrayRegion(ctx, 0, 2, context); + return env->NewDirectByteBuffer(const_cast(data), data_length); +} + +/** + * Releases all used resources. + * + * @param ctx {in_cookie} tuple + */ +JNIEXPORT void JNICALL +Java_org_brotli_wrapper_dec_DecoderJNI_nativeDestroy( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx) { + jlong context[2]; + env->GetLongArrayRegion(ctx, 0, 2, context); + DecoderHandle* handle = getHandle(reinterpret_cast(context[0])); + BrotliDecoderDestroyInstance(handle->state); + if (!!handle->custom_dictionary_ref) { + env->DeleteGlobalRef(handle->custom_dictionary_ref); + } + delete[] handle->input_start; + delete handle; +} + +#ifdef __cplusplus +} +#endif diff --git a/java/org/brotli/wrapper/enc/BUILD b/java/org/brotli/wrapper/enc/BUILD new file mode 100755 index 0000000..2290ab4 --- /dev/null +++ b/java/org/brotli/wrapper/enc/BUILD @@ -0,0 +1,98 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +filegroup( + name = "jni_src", + srcs = ["encoder_jni.cc"], +) + +######################################### +# WARNING: do not depend on this target! +######################################### +java_library( + name = "enc", + srcs = glob( + ["*.java"], + exclude = ["*Test*.java"], + ), + deps = ["//:jni"], +) + +filegroup( + name = "test_bundle", + srcs = ["//java/org/brotli/integration:test_corpus"], +) + +java_test( + name = "BrotliEncoderChannelTest", + size = "large", + srcs = ["BrotliEncoderChannelTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + shard_count = 15, + deps = [ + ":enc", + "//java/org/brotli/integration:bundle_helper", + "//java/org/brotli/wrapper/dec", + "@junit_junit//jar", + ], +) + +java_test( + name = "BrotliOutputStreamTest", + size = "large", + srcs = ["BrotliOutputStreamTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + shard_count = 15, + deps = [ + ":enc", + "//java/org/brotli/integration:bundle_helper", + "//java/org/brotli/wrapper/dec", + "@junit_junit//jar", + ], +) + +java_test( + name = "EncoderTest", + size = "large", + srcs = ["EncoderTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + shard_count = 15, + deps = [ + ":enc", + "//java/org/brotli/integration:bundle_helper", + "//java/org/brotli/wrapper/dec", + "@junit_junit//jar", + ], +) + +java_test( + name = "UseCustomDictionaryTest", + size = "large", + srcs = ["UseCustomDictionaryTest.java"], + data = [ + ":test_bundle", + "//:jni", # Bazel JNI workaround + ], + jvm_flags = ["-DTEST_BUNDLE=$(location :test_bundle)"], + shard_count = 15, + deps = [ + ":enc", + "//java/org/brotli/integration:bundle_helper", + "//java/org/brotli/wrapper/common", + "//java/org/brotli/wrapper/dec", + "@junit_junit//jar", + ], +) diff --git a/java/org/brotli/wrapper/enc/BrotliEncoderChannel.java b/java/org/brotli/wrapper/enc/BrotliEncoderChannel.java new file mode 100755 index 0000000..95a8b20 --- /dev/null +++ b/java/org/brotli/wrapper/enc/BrotliEncoderChannel.java @@ -0,0 +1,82 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.enc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.WritableByteChannel; + +/** + * WritableByteChannel that wraps native brotli encoder. + */ +public class BrotliEncoderChannel extends Encoder implements WritableByteChannel { + /** The default internal buffer size used by the decoder. */ + private static final int DEFAULT_BUFFER_SIZE = 16384; + + private final Object mutex = new Object(); + + /** + * Creates a BrotliEncoderChannel. + * + * @param destination underlying destination + * @param params encoding settings + * @param bufferSize intermediate buffer size + * @param customDictionary initial LZ77 dictionary + */ + public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params, + int bufferSize, ByteBuffer customDictionary) throws IOException { + super(destination, params, bufferSize, customDictionary); + } + + public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params, + int bufferSize) throws IOException { + super(destination, params, bufferSize, null); + } + + public BrotliEncoderChannel(WritableByteChannel destination, Encoder.Parameters params) + throws IOException { + this(destination, params, DEFAULT_BUFFER_SIZE); + } + + public BrotliEncoderChannel(WritableByteChannel destination) throws IOException { + this(destination, new Encoder.Parameters()); + } + + @Override + public boolean isOpen() { + synchronized (mutex) { + return !closed; + } + } + + @Override + public void close() throws IOException { + synchronized (mutex) { + super.close(); + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + synchronized (mutex) { + if (closed) { + throw new ClosedChannelException(); + } + int result = 0; + while (src.hasRemaining() && encode(EncoderJNI.Operation.PROCESS)) { + int limit = Math.min(src.remaining(), inputBuffer.remaining()); + ByteBuffer slice = src.slice(); + slice.limit(limit); + inputBuffer.put(slice); + result += limit; + src.position(src.position() + limit); + } + return result; + } + } +} diff --git a/java/org/brotli/wrapper/enc/BrotliEncoderChannelTest.java b/java/org/brotli/wrapper/enc/BrotliEncoderChannelTest.java new file mode 100755 index 0000000..1ab7599 --- /dev/null +++ b/java/org/brotli/wrapper/enc/BrotliEncoderChannelTest.java @@ -0,0 +1,122 @@ +package org.brotli.wrapper.enc; + +import static org.junit.Assert.assertEquals; + +import org.brotli.integration.BundleHelper; +import org.brotli.wrapper.dec.BrotliInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for {@link org.brotli.wrapper.enc.BrotliEncoderChannel}. */ +@RunWith(AllTests.class) +public class BrotliEncoderChannelTest { + + private enum TestMode { + WRITE_ALL, + WRITE_CHUNKS + } + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + private static final int CHUNK_SIZE = 256; + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new ChannleTestCase(entry, TestMode.WRITE_ALL)); + suite.addTest(new ChannleTestCase(entry, TestMode.WRITE_CHUNKS)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class ChannleTestCase extends TestCase { + final String entryName; + final TestMode mode; + ChannleTestCase(String entryName, TestMode mode) { + super("BrotliEncoderChannelTest." + entryName + "." + mode.name()); + this.entryName = entryName; + this.mode = mode; + } + + @Override + protected void runTest() throws Throwable { + BrotliEncoderChannelTest.run(entryName, mode); + } + } + + private static void run(String entryName, TestMode mode) throws Throwable { + InputStream bundle = getBundle(); + byte[] original; + try { + original = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + if (original == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + if ((mode == TestMode.WRITE_CHUNKS) && (original.length <= CHUNK_SIZE)) { + return; + } + + ByteArrayOutputStream dst = new ByteArrayOutputStream(); + WritableByteChannel encoder = new BrotliEncoderChannel(Channels.newChannel(dst)); + ByteBuffer src = ByteBuffer.wrap(original); + try { + switch (mode) { + case WRITE_ALL: + encoder.write(src); + break; + + case WRITE_CHUNKS: + while (src.hasRemaining()) { + int limit = Math.min(CHUNK_SIZE, src.remaining()); + ByteBuffer slice = src.slice(); + slice.limit(limit); + src.position(src.position() + limit); + encoder.write(slice); + } + break; + } + } finally { + encoder.close(); + } + + InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(dst.toByteArray())); + try { + long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original)); + long crc = BundleHelper.fingerprintStream(decoder); + assertEquals(originalCrc, crc); + } finally { + decoder.close(); + } + } +} diff --git a/java/org/brotli/wrapper/enc/BrotliOutputStream.java b/java/org/brotli/wrapper/enc/BrotliOutputStream.java new file mode 100755 index 0000000..1cee434 --- /dev/null +++ b/java/org/brotli/wrapper/enc/BrotliOutputStream.java @@ -0,0 +1,95 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.enc; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; + +/** + * Output stream that wraps native brotli encoder. + */ +public class BrotliOutputStream extends OutputStream { + /** The default internal buffer size used by the encoder. */ + private static final int DEFAULT_BUFFER_SIZE = 16384; + + private final Encoder encoder; + + /** + * Creates a BrotliOutputStream. + * + * @param destination underlying destination + * @param params encoding settings + * @param bufferSize intermediate buffer size + * @param customDictionary initial LZ77 dictionary + */ + public BrotliOutputStream(OutputStream destination, Encoder.Parameters params, int bufferSize, + ByteBuffer customDictionary) throws IOException { + this.encoder = new Encoder( + Channels.newChannel(destination), params, bufferSize, customDictionary); + } + + public BrotliOutputStream(OutputStream destination, Encoder.Parameters params, int bufferSize) + throws IOException { + this.encoder = new Encoder(Channels.newChannel(destination), params, bufferSize, null); + } + + public BrotliOutputStream(OutputStream destination, Encoder.Parameters params) + throws IOException { + this(destination, params, DEFAULT_BUFFER_SIZE); + } + + public BrotliOutputStream(OutputStream destination) throws IOException { + this(destination, new Encoder.Parameters()); + } + + @Override + public void close() throws IOException { + encoder.close(); + } + + @Override + public void flush() throws IOException { + if (encoder.closed) { + throw new IOException("write after close"); + } + encoder.flush(); + } + + @Override + public void write(int b) throws IOException { + if (encoder.closed) { + throw new IOException("write after close"); + } + while (!encoder.encode(EncoderJNI.Operation.PROCESS)) { + // Busy-wait loop. + } + encoder.inputBuffer.put((byte) b); + } + + @Override + public void write(byte[] b) throws IOException { + this.write(b, 0, b.length); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (encoder.closed) { + throw new IOException("write after close"); + } + while (len > 0) { + if (!encoder.encode(EncoderJNI.Operation.PROCESS)) { + continue; + } + int limit = Math.min(len, encoder.inputBuffer.remaining()); + encoder.inputBuffer.put(b, off, limit); + off += limit; + len -= limit; + } + } +} diff --git a/java/org/brotli/wrapper/enc/BrotliOutputStreamTest.java b/java/org/brotli/wrapper/enc/BrotliOutputStreamTest.java new file mode 100755 index 0000000..a436e81 --- /dev/null +++ b/java/org/brotli/wrapper/enc/BrotliOutputStreamTest.java @@ -0,0 +1,123 @@ +package org.brotli.wrapper.enc; + +import static org.junit.Assert.assertEquals; + +import org.brotli.integration.BundleHelper; +import org.brotli.wrapper.dec.BrotliInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for {@link org.brotli.wrapper.enc.BrotliOutputStream}. */ +@RunWith(AllTests.class) +public class BrotliOutputStreamTest { + + private enum TestMode { + WRITE_ALL, + WRITE_CHUNKS, + WRITE_BYTE + } + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + private static final int CHUNK_SIZE = 256; + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new StreamTestCase(entry, TestMode.WRITE_ALL)); + suite.addTest(new StreamTestCase(entry, TestMode.WRITE_CHUNKS)); + suite.addTest(new StreamTestCase(entry, TestMode.WRITE_BYTE)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class StreamTestCase extends TestCase { + final String entryName; + final TestMode mode; + StreamTestCase(String entryName, TestMode mode) { + super("BrotliOutputStreamTest." + entryName + "." + mode.name()); + this.entryName = entryName; + this.mode = mode; + } + + @Override + protected void runTest() throws Throwable { + BrotliOutputStreamTest.run(entryName, mode); + } + } + + private static void run(String entryName, TestMode mode) throws Throwable { + InputStream bundle = getBundle(); + byte[] original; + try { + original = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + if (original == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + if ((mode == TestMode.WRITE_CHUNKS) && (original.length <= CHUNK_SIZE)) { + return; + } + + ByteArrayOutputStream dst = new ByteArrayOutputStream(); + OutputStream encoder = new BrotliOutputStream(dst); + try { + switch (mode) { + case WRITE_ALL: + encoder.write(original); + break; + + case WRITE_CHUNKS: + for (int offset = 0; offset < original.length; offset += CHUNK_SIZE) { + encoder.write(original, offset, Math.min(CHUNK_SIZE, original.length - offset)); + } + break; + + case WRITE_BYTE: + for (byte singleByte : original) { + encoder.write(singleByte); + } + break; + } + } finally { + encoder.close(); + } + + InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(dst.toByteArray())); + try { + long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original)); + long crc = BundleHelper.fingerprintStream(decoder); + assertEquals(originalCrc, crc); + } finally { + decoder.close(); + } + } +} diff --git a/java/org/brotli/wrapper/enc/Encoder.java b/java/org/brotli/wrapper/enc/Encoder.java new file mode 100755 index 0000000..55cc369 --- /dev/null +++ b/java/org/brotli/wrapper/enc/Encoder.java @@ -0,0 +1,200 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.enc; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; + +/** + * Base class for OutputStream / Channel implementations. + */ +public class Encoder { + private final WritableByteChannel destination; + private final EncoderJNI.Wrapper encoder; + final ByteBuffer inputBuffer; + ByteBuffer buffer; + boolean closed; + + /** + * Brotli encoder settings. + */ + public static final class Parameters { + private int quality = -1; + private int lgwin = -1; + + public Parameters() { } + + private Parameters(Parameters other) { + this.quality = other.quality; + this.lgwin = other.lgwin; + } + + /** + * @param quality compression quality, or -1 for default + */ + public Parameters setQuality(int quality) { + if (quality < -1 || quality > 11) { + throw new IllegalArgumentException("quality should be in range [0, 11], or -1"); + } + this.quality = quality; + return this; + } + + /** + * @param lgwin log2(LZ window size), or -1 for default + */ + public Parameters setWindow(int lgwin) { + if ((lgwin != -1) && ((lgwin < 10) || (lgwin > 24))) { + throw new IllegalArgumentException("lgwin should be in range [10, 24], or -1"); + } + this.lgwin = lgwin; + return this; + } + } + + /** + * Creates a Encoder wrapper. + * + * @param destination underlying destination + * @param params encoding parameters + * @param inputBufferSize read buffer size + */ + Encoder(WritableByteChannel destination, Parameters params, int inputBufferSize, + ByteBuffer customDictionary) throws IOException { + if (inputBufferSize <= 0) { + throw new IllegalArgumentException("buffer size must be positive"); + } + if (destination == null) { + throw new NullPointerException("destination can not be null"); + } + this.destination = destination; + this.encoder = new EncoderJNI.Wrapper( + inputBufferSize, params.quality, params.lgwin, customDictionary); + this.inputBuffer = this.encoder.getInputBuffer(); + } + + private void fail(String message) throws IOException { + try { + close(); + } catch (IOException ex) { + /* Ignore */ + } + throw new IOException(message); + } + + /** + * @param force repeat pushing until all output is consumed + * @return true if all encoder output is consumed + */ + boolean pushOutput(boolean force) throws IOException { + while (buffer != null) { + if (buffer.hasRemaining()) { + destination.write(buffer); + } + if (!buffer.hasRemaining()) { + buffer = null; + } else if (!force) { + return false; + } + } + return true; + } + + /** + * @return true if there is space in inputBuffer. + */ + boolean encode(EncoderJNI.Operation op) throws IOException { + boolean force = (op != EncoderJNI.Operation.PROCESS); + if (force) { + inputBuffer.limit(inputBuffer.position()); + } else if (inputBuffer.hasRemaining()) { + return true; + } + boolean hasInput = true; + while (true) { + if (!encoder.isSuccess()) { + fail("encoding failed"); + } else if (!pushOutput(force)) { + return false; + } else if (encoder.hasMoreOutput()) { + buffer = encoder.pull(); + } else if (encoder.hasRemainingInput()) { + encoder.push(op, 0); + } else if (hasInput) { + encoder.push(op, inputBuffer.limit()); + hasInput = false; + } else { + inputBuffer.clear(); + return true; + } + } + } + + void flush() throws IOException { + encode(EncoderJNI.Operation.FLUSH); + } + + void close() throws IOException { + if (closed) { + return; + } + closed = true; + try { + encode(EncoderJNI.Operation.FINISH); + } finally { + encoder.destroy(); + destination.close(); + } + } + + /** + * Encodes the given data buffer. + */ + public static byte[] compress(byte[] data, Parameters params) throws IOException { + EncoderJNI.Wrapper encoder = new EncoderJNI.Wrapper( + data.length, params.quality, params.lgwin, null); + ArrayList output = new ArrayList(); + int totalOutputSize = 0; + try { + encoder.getInputBuffer().put(data); + encoder.push(EncoderJNI.Operation.FINISH, data.length); + while (true) { + if (!encoder.isSuccess()) { + throw new IOException("encoding failed"); + } else if (encoder.hasMoreOutput()) { + ByteBuffer buffer = encoder.pull(); + byte[] chunk = new byte[buffer.remaining()]; + buffer.get(chunk); + output.add(chunk); + totalOutputSize += chunk.length; + } else if (encoder.hasRemainingInput()) { + encoder.push(EncoderJNI.Operation.FINISH, 0); + } else { + break; + } + } + } finally { + encoder.destroy(); + } + if (output.size() == 1) { + return output.get(0); + } + byte[] result = new byte[totalOutputSize]; + int offset = 0; + for (byte[] chunk : output) { + System.arraycopy(chunk, 0, result, offset, chunk.length); + offset += chunk.length; + } + return result; + } + + public static byte[] compress(byte[] data) throws IOException { + return compress(data, new Parameters()); + } +} diff --git a/java/org/brotli/wrapper/enc/EncoderJNI.java b/java/org/brotli/wrapper/enc/EncoderJNI.java new file mode 100755 index 0000000..dd7cff3 --- /dev/null +++ b/java/org/brotli/wrapper/enc/EncoderJNI.java @@ -0,0 +1,111 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.wrapper.enc; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * JNI wrapper for brotli encoder. + */ +class EncoderJNI { + private static native ByteBuffer nativeCreate(long[] context, ByteBuffer customDictionary); + private static native void nativePush(long[] context, int length); + private static native ByteBuffer nativePull(long[] context); + private static native void nativeDestroy(long[] context); + + enum Operation { + PROCESS, + FLUSH, + FINISH + } + + static class Wrapper { + protected final long[] context = new long[4]; + private final ByteBuffer inputBuffer; + + Wrapper(int inputBufferSize, int quality, int lgwin, ByteBuffer customDictionary) + throws IOException { + if (customDictionary != null && !customDictionary.isDirect()) { + throw new IllegalArgumentException("LZ77 dictionary must be direct ByteBuffer"); + } + this.context[1] = inputBufferSize; + this.context[2] = quality; + this.context[3] = lgwin; + this.inputBuffer = nativeCreate(this.context, customDictionary); + if (this.context[0] == 0) { + throw new IOException("failed to initialize native brotli encoder"); + } + this.context[1] = 1; + this.context[2] = 0; + this.context[3] = 0; + } + + void push(Operation op, int length) { + if (length < 0) { + throw new IllegalArgumentException("negative block length"); + } + if (context[0] == 0) { + throw new IllegalStateException("brotli encoder is already destroyed"); + } + if (!isSuccess() || hasMoreOutput()) { + throw new IllegalStateException("pushing input to encoder in unexpected state"); + } + if (hasRemainingInput() && length != 0) { + throw new IllegalStateException("pushing input to encoder over previous input"); + } + context[1] = op.ordinal(); + nativePush(context, length); + } + + boolean isSuccess() { + return context[1] != 0; + } + + boolean hasMoreOutput() { + return context[2] != 0; + } + + boolean hasRemainingInput() { + return context[3] != 0; + } + + ByteBuffer getInputBuffer() { + return inputBuffer; + } + + ByteBuffer pull() { + if (context[0] == 0) { + throw new IllegalStateException("brotli encoder is already destroyed"); + } + if (!isSuccess() || !hasMoreOutput()) { + throw new IllegalStateException("pulling while data is not ready"); + } + return nativePull(context); + } + + /** + * Releases native resources. + */ + void destroy() { + if (context[0] == 0) { + throw new IllegalStateException("brotli encoder is already destroyed"); + } + nativeDestroy(context); + context[0] = 0; + } + + @Override + protected void finalize() throws Throwable { + if (context[0] != 0) { + /* TODO: log resource leak? */ + destroy(); + } + super.finalize(); + } + } +} diff --git a/java/org/brotli/wrapper/enc/EncoderTest.java b/java/org/brotli/wrapper/enc/EncoderTest.java new file mode 100755 index 0000000..8328d45 --- /dev/null +++ b/java/org/brotli/wrapper/enc/EncoderTest.java @@ -0,0 +1,83 @@ +package org.brotli.wrapper.enc; + +import static org.junit.Assert.assertEquals; + +import org.brotli.integration.BundleHelper; +import org.brotli.wrapper.dec.BrotliInputStream; +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for {@link org.brotli.wrapper.enc.Encoder}. */ +@RunWith(AllTests.class) +public class EncoderTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new EncoderTestCase(entry)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class EncoderTestCase extends TestCase { + final String entryName; + EncoderTestCase(String entryName) { + super("EncoderTest." + entryName); + this.entryName = entryName; + } + + @Override + protected void runTest() throws Throwable { + EncoderTest.run(entryName); + } + } + + private static void run(String entryName) throws Throwable { + InputStream bundle = getBundle(); + byte[] original; + try { + original = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + if (original == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + byte[] compressed = Encoder.compress(original, new Encoder.Parameters().setQuality(6)); + + InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(compressed)); + try { + long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original)); + long crc = BundleHelper.fingerprintStream(decoder); + assertEquals(originalCrc, crc); + } finally { + decoder.close(); + } + } +} diff --git a/java/org/brotli/wrapper/enc/UseCustomDictionaryTest.java b/java/org/brotli/wrapper/enc/UseCustomDictionaryTest.java new file mode 100755 index 0000000..d46f997 --- /dev/null +++ b/java/org/brotli/wrapper/enc/UseCustomDictionaryTest.java @@ -0,0 +1,104 @@ +package org.brotli.wrapper.enc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import org.brotli.integration.BundleHelper; +import org.brotli.wrapper.common.BrotliCommon; +import org.brotli.wrapper.dec.BrotliInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.List; +import junit.framework.TestCase; +import junit.framework.TestSuite; +import org.junit.runner.RunWith; +import org.junit.runners.AllTests; + +/** Tests for compression / decompression aided with LZ77 dictionary. */ +@RunWith(AllTests.class) +public class UseCustomDictionaryTest { + + // TODO: remove when Bazel get JNI support. + static { + System.load(new java.io.File(new java.io.File(System.getProperty("java.library.path")), + "liblibjni.so").getAbsolutePath()); + } + + static InputStream getBundle() throws IOException { + return new FileInputStream(System.getProperty("TEST_BUNDLE")); + } + + /** Creates a test suite. */ + public static TestSuite suite() throws IOException { + TestSuite suite = new TestSuite(); + InputStream bundle = getBundle(); + try { + List entries = BundleHelper.listEntries(bundle); + for (String entry : entries) { + suite.addTest(new UseCustomDictionaryTestCase(entry)); + } + } finally { + bundle.close(); + } + return suite; + } + + /** Test case with a unique name. */ + static class UseCustomDictionaryTestCase extends TestCase { + final String entryName; + UseCustomDictionaryTestCase(String entryName) { + super("UseCustomDictionaryTest." + entryName); + this.entryName = entryName; + } + + @Override + protected void runTest() throws Throwable { + UseCustomDictionaryTest.run(entryName); + } + } + + private static void run(String entryName) throws Throwable { + InputStream bundle = getBundle(); + byte[] original; + try { + original = BundleHelper.readEntry(bundle, entryName); + } finally { + bundle.close(); + } + + if (original == null) { + throw new RuntimeException("Can't read bundle entry: " + entryName); + } + + ByteBuffer dictionary = BrotliCommon.makeNative(original); + + ByteArrayOutputStream dst = new ByteArrayOutputStream(); + OutputStream encoder = new BrotliOutputStream(dst, + new Encoder.Parameters().setQuality(11).setWindow(23), 1 << 23, dictionary); + try { + encoder.write(original); + } finally { + encoder.close(); + } + + byte[] compressed = dst.toByteArray(); + + // Just copy self from LZ77 dictionary -> ultimate compression ratio. + assertTrue(compressed.length < 80 + original.length / 65536); + + InputStream decoder = new BrotliInputStream(new ByteArrayInputStream(compressed), + 1 << 23, dictionary); + try { + long originalCrc = BundleHelper.fingerprintStream(new ByteArrayInputStream(original)); + long crc = BundleHelper.fingerprintStream(decoder); + assertEquals(originalCrc, crc); + } finally { + decoder.close(); + } + } +} diff --git a/java/org/brotli/wrapper/enc/encoder_jni.cc b/java/org/brotli/wrapper/enc/encoder_jni.cc new file mode 100755 index 0000000..adc7c12 --- /dev/null +++ b/java/org/brotli/wrapper/enc/encoder_jni.cc @@ -0,0 +1,224 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +#include + +#include + +#include + +namespace { +/* A structure used to persist the encoder's state in between calls. */ +typedef struct EncoderHandle { + BrotliEncoderState* state; + + jobject custom_dictionary_ref; + + uint8_t* input_start; + size_t input_offset; + size_t input_last; +} EncoderHandle; + +/* Obtain handle from opaque pointer. */ +EncoderHandle* getHandle(void* opaque) { + return static_cast(opaque); +} + +} /* namespace */ + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Creates a new Encoder. + * + * Cookie to address created encoder is stored in out_cookie. In case of failure + * cookie is 0. + * + * @param ctx {out_cookie, in_directBufferSize, in_quality, in_lgwin} tuple + * @returns direct ByteBuffer if directBufferSize is not 0; otherwise null + */ +JNIEXPORT jobject JNICALL +Java_org_brotli_wrapper_enc_EncoderJNI_nativeCreate( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jobject custom_dictionary) { + bool ok = true; + EncoderHandle* handle = nullptr; + jlong context[4]; + env->GetLongArrayRegion(ctx, 0, 4, context); + size_t input_size = context[1]; + context[0] = 0; + handle = new (std::nothrow) EncoderHandle(); + ok = !!handle; + + if (ok) { + handle->custom_dictionary_ref = nullptr; + handle->input_offset = 0; + handle->input_last = 0; + handle->input_start = nullptr; + + if (input_size == 0) { + ok = false; + } else { + handle->input_start = new (std::nothrow) uint8_t[input_size]; + ok = !!handle->input_start; + } + } + + if (ok) { + handle->state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + ok = !!handle->state; + } + + if (ok) { + int quality = context[2]; + if (quality >= 0) { + BrotliEncoderSetParameter(handle->state, BROTLI_PARAM_QUALITY, quality); + } + int lgwin = context[3]; + if (lgwin >= 0) { + BrotliEncoderSetParameter(handle->state, BROTLI_PARAM_LGWIN, lgwin); + } + } + + if (ok && !!custom_dictionary) { + handle->custom_dictionary_ref = env->NewGlobalRef(custom_dictionary); + if (!!handle->custom_dictionary_ref) { + uint8_t* custom_dictionary_address = static_cast( + env->GetDirectBufferAddress(handle->custom_dictionary_ref)); + if (!!custom_dictionary_address) { + jlong capacity = + env->GetDirectBufferCapacity(handle->custom_dictionary_ref); + ok = (capacity > 0) && (capacity < (1 << 24)); + if (ok) { + size_t custom_dictionary_size = static_cast(capacity); + BrotliEncoderSetCustomDictionary( + handle->state, custom_dictionary_size, custom_dictionary_address); + } + } else { + ok = false; + } + } else { + ok = false; + } + } + + if (ok) { + /* TODO: future versions (e.g. when 128-bit architecture comes) + might require thread-safe cookie<->handle mapping. */ + context[0] = reinterpret_cast(handle); + } else if (!!handle) { + if (!!handle->custom_dictionary_ref) { + env->DeleteGlobalRef(handle->custom_dictionary_ref); + } + if (!!handle->input_start) delete[] handle->input_start; + delete handle; + } + + env->SetLongArrayRegion(ctx, 0, 1, context); + + if (!ok) { + return nullptr; + } + + return env->NewDirectByteBuffer(handle->input_start, input_size); +} + +/** + * Push data to encoder. + * + * @param ctx {in_cookie, in_operation_out_success, out_has_more_output, + * out_has_remaining_input} tuple + * @param input_length number of bytes provided in input or direct input; + * 0 to process further previous input + */ +JNIEXPORT void JNICALL +Java_org_brotli_wrapper_enc_EncoderJNI_nativePush( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx, jint input_length) { + jlong context[4]; + env->GetLongArrayRegion(ctx, 0, 4, context); + EncoderHandle* handle = getHandle(reinterpret_cast(context[0])); + int operation = context[1]; + context[1] = 0; /* ERROR */ + env->SetLongArrayRegion(ctx, 0, 4, context); + + BrotliEncoderOperation op; + switch (operation) { + case 0: op = BROTLI_OPERATION_PROCESS; break; + case 1: op = BROTLI_OPERATION_FLUSH; break; + case 2: op = BROTLI_OPERATION_FINISH; break; + default: return; /* ERROR */ + } + + if (input_length != 0) { + /* Still have unconsumed data. Workflow is broken. */ + if (handle->input_offset < handle->input_last) { + return; + } + handle->input_offset = 0; + handle->input_last = input_length; + } + + /* Actual compression. */ + const uint8_t* in = handle->input_start + handle->input_offset; + size_t in_size = handle->input_last - handle->input_offset; + size_t out_size = 0; + BROTLI_BOOL status = BrotliEncoderCompressStream( + handle->state, op, &in_size, &in, &out_size, nullptr, nullptr); + handle->input_offset = handle->input_last - in_size; + if (!!status) { + context[1] = 1; + context[2] = BrotliEncoderHasMoreOutput(handle->state) ? 1 : 0; + context[3] = (handle->input_offset != handle->input_last) ? 1 : 0; + } + env->SetLongArrayRegion(ctx, 0, 4, context); +} + +/** + * Pull decompressed data from encoder. + * + * @param ctx {in_cookie, out_success, out_has_more_output, + * out_has_remaining_input} tuple + * @returns direct ByteBuffer; all the produced data MUST be consumed before + * any further invocation; null in case of error + */ +JNIEXPORT jobject JNICALL +Java_org_brotli_wrapper_enc_EncoderJNI_nativePull( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx) { + jlong context[4]; + env->GetLongArrayRegion(ctx, 0, 4, context); + EncoderHandle* handle = getHandle(reinterpret_cast(context[0])); + size_t data_length = 0; + const uint8_t* data = BrotliEncoderTakeOutput(handle->state, &data_length); + context[1] = 1; + context[2] = BrotliEncoderHasMoreOutput(handle->state) ? 1 : 0; + context[3] = (handle->input_offset != handle->input_last) ? 1 : 0; + env->SetLongArrayRegion(ctx, 0, 4, context); + return env->NewDirectByteBuffer(const_cast(data), data_length); +} + +/** + * Releases all used resources. + * + * @param ctx {in_cookie} tuple + */ +JNIEXPORT void JNICALL +Java_org_brotli_wrapper_enc_EncoderJNI_nativeDestroy( + JNIEnv* env, jobject /*jobj*/, jlongArray ctx) { + jlong context[2]; + env->GetLongArrayRegion(ctx, 0, 2, context); + EncoderHandle* handle = getHandle(reinterpret_cast(context[0])); + BrotliEncoderDestroyInstance(handle->state); + if (!!handle->custom_dictionary_ref) { + env->DeleteGlobalRef(handle->custom_dictionary_ref); + } + delete[] handle->input_start; + delete handle; +} + +#ifdef __cplusplus +} +#endif diff --git a/scripts/appveyor.yml b/scripts/appveyor.yml index 617f324..b5b1294 100644 --- a/scripts/appveyor.yml +++ b/scripts/appveyor.yml @@ -62,7 +62,7 @@ install: pip install --disable-pip-version-check --user --upgrade pip # install/upgrade setuptools and wheel to build packages - pip install --upgrade setuptools wheel + pip install --upgrade setuptools six wheel } before_build: -- cgit v1.1