diff options
author | Eugene Kliuchnikov <eustas@google.com> | 2019-04-12 13:57:42 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-04-12 13:57:42 +0200 |
commit | 4b2b2d4f83ffeaac7708e44409fe34896a01a278 (patch) | |
tree | 04dff6010a8fad91ba93eff45a8730ed5fc40f14 /java/org | |
parent | 9cd01c0437e8b6010434d3491a348a5645de624b (diff) | |
download | brotli-4b2b2d4f83ffeaac7708e44409fe34896a01a278.zip brotli-4b2b2d4f83ffeaac7708e44409fe34896a01a278.tar.gz brotli-4b2b2d4f83ffeaac7708e44409fe34896a01a278.tar.bz2 |
Update (#749)
Update:
* Bazel: fix MSVC configuration
* C: common: extended documentation and helpers around distance codes
* C: common: enable BROTLI_DCHECK in "debug" builds
* C: common: fix implicit trailing zero in `kPrefixSuffix`
* C: dec: fix possible bit reader discharge for "large-window" mode
* C: dec: simplify distance decoding via lookup table
* C: dec: reuse decoder state members memory via union with lookup table
* C: dec: add decoder state diagram
* C: enc: clarify access to static dictionary
* C: enc: improve static dictionary hash
* C: enc: add "stream offset" parameter for parallel encoding
* C: enc: reorganize hasher; now Q2-Q3 require exactly 256KiB
to avoid global TCMalloc lock
* C: enc: fix rare access to uninitialized data in ring-buffer
* C: enc: reorganize logging / checks in `write_bits.h`
* Java: dec: add "large-window" support
* Java: dec: improve speed
* Java: dec: debug and 32-bit mode are now activated via system properties
* Java: dec: demystify some state variables (use better names)
* Dictionary generator: add single input mode
* Java: dec: modernize tests
* Bazel: js: pick working commit for closure rules
Diffstat (limited to 'java/org')
-rw-r--r-- | java/org/brotli/dec/BUILD | 51 | ||||
-rw-r--r-- | java/org/brotli/dec/BitReader.java | 31 | ||||
-rw-r--r-- | java/org/brotli/dec/BitReaderTest.java | 19 | ||||
-rw-r--r-- | java/org/brotli/dec/BrotliInputStream.java | 18 | ||||
-rw-r--r-- | java/org/brotli/dec/Decode.java | 606 | ||||
-rwxr-xr-x | java/org/brotli/dec/EagerStreamTest.java | 2 | ||||
-rw-r--r-- | java/org/brotli/dec/Huffman.java | 17 | ||||
-rw-r--r-- | java/org/brotli/dec/State.java | 14 | ||||
-rw-r--r-- | java/org/brotli/dec/SynthTest.java | 156 | ||||
-rw-r--r-- | java/org/brotli/dec/Utils.java | 11 | ||||
-rw-r--r-- | java/org/brotli/dec/build_defs.bzl | 34 | ||||
-rw-r--r-- | java/org/brotli/dec/pom.xml | 10 | ||||
-rw-r--r-- | java/org/brotli/wrapper/common/BUILD | 4 | ||||
-rw-r--r-- | java/org/brotli/wrapper/dec/BUILD | 6 | ||||
-rw-r--r-- | java/org/brotli/wrapper/dec/BrotliInputStream.java | 4 | ||||
-rw-r--r-- | java/org/brotli/wrapper/dec/Decoder.java | 4 | ||||
-rw-r--r-- | java/org/brotli/wrapper/dec/DecoderJNI.java | 3 | ||||
-rwxr-xr-x | java/org/brotli/wrapper/dec/EagerStreamTest.java | 2 | ||||
-rw-r--r-- | java/org/brotli/wrapper/enc/BUILD | 6 | ||||
-rw-r--r-- | java/org/brotli/wrapper/enc/EncoderJNI.java | 6 |
20 files changed, 741 insertions, 263 deletions
diff --git a/java/org/brotli/dec/BUILD b/java/org/brotli/dec/BUILD index e6d3a4d..8c61fe7 100644 --- a/java/org/brotli/dec/BUILD +++ b/java/org/brotli/dec/BUILD @@ -5,6 +5,11 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # MIT +TEST_DEPS = [ + ":dec", + "@junit_junit//jar", +] + java_library( name = "dec", srcs = glob( @@ -14,48 +19,40 @@ java_library( proguard_specs = ["proguard.cfg"], ) -java_library( - name = "test_lib", - testonly = 1, - srcs = glob(["*Test*.java"]), - deps = [ - ":dec", - "@junit_junit//jar", - ], -) +load(":build_defs.bzl", "brotli_java_test") -java_test( +brotli_java_test( name = "BitReaderTest", - test_class = "org.brotli.dec.BitReaderTest", - runtime_deps = [":test_lib"], + srcs = ["BitReaderTest.java"], + deps = TEST_DEPS, ) -java_test( +brotli_java_test( name = "DecodeTest", - test_class = "org.brotli.dec.DecodeTest", - runtime_deps = [":test_lib"], + srcs = ["DecodeTest.java"], + deps = TEST_DEPS, ) -java_test( +brotli_java_test( name = "DictionaryTest", - test_class = "org.brotli.dec.DictionaryTest", - runtime_deps = [":test_lib"], + srcs = ["DictionaryTest.java"], + deps = TEST_DEPS, ) -java_test( +brotli_java_test( name = "EagerStreamTest", - test_class = "org.brotli.dec.EagerStreamTest", - runtime_deps = [":test_lib"], + srcs = ["EagerStreamTest.java"], + deps = TEST_DEPS, ) -java_test( +brotli_java_test( name = "SynthTest", - test_class = "org.brotli.dec.SynthTest", - runtime_deps = [":test_lib"], + srcs = ["SynthTest.java"], + deps = TEST_DEPS, ) -java_test( +brotli_java_test( name = "TransformTest", - test_class = "org.brotli.dec.TransformTest", - runtime_deps = [":test_lib"], + srcs = ["TransformTest.java"], + deps = TEST_DEPS, ) diff --git a/java/org/brotli/dec/BitReader.java b/java/org/brotli/dec/BitReader.java index 5d54e01..6dfeedc 100644 --- a/java/org/brotli/dec/BitReader.java +++ b/java/org/brotli/dec/BitReader.java @@ -12,9 +12,14 @@ package org.brotli.dec; final class BitReader { // Possible values: {5, 6}. 5 corresponds to 32-bit build, 6 to 64-bit. This value is used for - // conditional compilation -> produced artifacts might be binary INCOMPATIBLE (JLS 13.2). - private static final int LOG_BITNESS = 6; - private static final int BITNESS = 1 << LOG_BITNESS; + // JIT conditional compilation. + private static final int LOG_BITNESS = Utils.getLogBintness(); + + // Not only Java compiler prunes "if (const false)" code, but JVM as well. + // Code under "if (DEBUG != 0)" have zero performance impact (outside unit tests). + private static final int DEBUG = Utils.isDebugMode(); + + static final int BITNESS = 1 << LOG_BITNESS; private static final int BYTENESS = BITNESS / 8; private static final int CAPACITY = 4096; @@ -89,7 +94,16 @@ final class BitReader { } } + static void assertAccumulatorHealthy(State s) { + if (s.bitOffset > BITNESS) { + throw new IllegalStateException("Accumulator underloaded: " + s.bitOffset); + } + } + static void fillBitWindow(State s) { + if (DEBUG != 0) { + assertAccumulatorHealthy(s); + } if (s.bitOffset >= HALF_BITNESS) { // Same as doFillBitWindow. JVM fails to inline it. if (BITNESS == 64) { @@ -103,7 +117,10 @@ final class BitReader { } } - private static void doFillBitWindow(State s) { + static void doFillBitWindow(State s) { + if (DEBUG != 0) { + assertAccumulatorHealthy(s); + } if (BITNESS == 64) { s.accumulator64 = ((long) s.intBuffer[s.halfOffset++] << HALF_BITNESS) | (s.accumulator64 >>> HALF_BITNESS); @@ -122,6 +139,12 @@ final class BitReader { } } + /** + * Fetches bits from accumulator. + * + * WARNING: accumulator MUST contain at least the specified amount of bits, + * otherwise BitReader will become broken. + */ static int readFewBits(State s, int n) { int val = peekBits(s) & ((1 << n) - 1); s.bitOffset += n; diff --git a/java/org/brotli/dec/BitReaderTest.java b/java/org/brotli/dec/BitReaderTest.java index fa57640..da59ebc 100644 --- a/java/org/brotli/dec/BitReaderTest.java +++ b/java/org/brotli/dec/BitReaderTest.java @@ -32,4 +32,23 @@ public class BitReaderTest { } fail("BrotliRuntimeException should have been thrown by BitReader.checkHealth"); } + + @Test + public void testAccumulatorUnderflowDetected() { + State reader = new State(); + Decode.initState(reader, new ByteArrayInputStream(new byte[8])); + // 65 bits is enough for both 32 and 64 bit systems. + BitReader.readBits(reader, 13); + BitReader.readBits(reader, 13); + BitReader.readBits(reader, 13); + BitReader.readBits(reader, 13); + BitReader.readBits(reader, 13); + try { + BitReader.fillBitWindow(reader); + } catch (IllegalStateException ex) { + // This exception is expected. + return; + } + fail("IllegalStateException should have been thrown by 'broken' BitReader"); + } } diff --git a/java/org/brotli/dec/BrotliInputStream.java b/java/org/brotli/dec/BrotliInputStream.java index 5cc2e28..b99e40a 100644 --- a/java/org/brotli/dec/BrotliInputStream.java +++ b/java/org/brotli/dec/BrotliInputStream.java @@ -84,18 +84,12 @@ public class BrotliInputStream extends InputStream { } } - public void setEager(boolean eager) { - boolean isEager = (state.isEager != 0); - if (eager == isEager) { - /* Shortcut for no-op change. */ - return; - } - if (eager) { - Decode.setEager(state); - } else { - /* Once decoder is "eager", there is no way back. */ - throw new IllegalStateException("Brotli decoder has been already switched to eager mode"); - } + public void enableEagerOutput() { + Decode.enableEagerOutput(state); + } + + public void enableLargeWindow() { + Decode.enableLargeWindow(state); } /** diff --git a/java/org/brotli/dec/Decode.java b/java/org/brotli/dec/Decode.java index 60bf9c6..560c635 100644 --- a/java/org/brotli/dec/Decode.java +++ b/java/org/brotli/dec/Decode.java @@ -14,6 +14,11 @@ import java.io.InputStream; */ final class Decode { + static final int MIN_LARGE_WINDOW_BITS = 10; + /* Maximum was chosen to be 30 to allow efficient decoder implementation. + * Format allows bigger window, but Java does not support 2G+ arrays. */ + static final int MAX_LARGE_WINDOW_BITS = 30; + //---------------------------------------------------------------------------- // RunningState //---------------------------------------------------------------------------- @@ -35,7 +40,7 @@ final class Decode { private static final int DEFAULT_CODE_LENGTH = 8; private static final int CODE_LENGTH_REPEAT_CODE = 16; private static final int NUM_LITERAL_CODES = 256; - private static final int NUM_INSERT_AND_COPY_CODES = 704; + private static final int NUM_COMMAND_CODES = 704; private static final int NUM_BLOCK_LENGTH_CODES = 26; private static final int LITERAL_CONTEXT_BITS = 6; private static final int DISTANCE_CONTEXT_BITS = 2; @@ -44,10 +49,19 @@ final class Decode { private static final int HUFFMAN_TABLE_MASK = 0xFF; /** - * Maximum possible Huffman table size for an alphabet size of 704, max code length 15 and root - * table bits 8. + * Maximum possible Huffman table size for an alphabet size of (index * 32), + * max code length 15 and root table bits 8. + * The biggest alphabet is "command" - 704 symbols. Though "distance" alphabet could theoretically + * outreach that limit (for 62 extra bit distances), practically it is limited by + * MAX_ALLOWED_DISTANCE and never gets bigger than 544 symbols. */ - static final int HUFFMAN_TABLE_SIZE = 1080; + static final int[] MAX_HUFFMAN_TABLE_SIZE = { + 256, 402, 436, 468, 500, 534, 566, 598, 630, 662, 694, 726, 758, 790, 822, + 854, 886, 920, 952, 984, 1016, 1048, 1080 + }; + + private static final int HUFFMAN_TABLE_SIZE_26 = 396; + private static final int HUFFMAN_TABLE_SIZE_258 = 632; private static final int CODE_LENGTH_CODES = 18; private static final int[] CODE_LENGTH_CODE_ORDER = { @@ -56,7 +70,7 @@ final class Decode { private static final int NUM_DISTANCE_SHORT_CODES = 16; private static final int[] DISTANCE_SHORT_CODE_INDEX_OFFSET = { - 3, 2, 1, 0, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2 + 0, 3, 2, 1, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3 }; private static final int[] DISTANCE_SHORT_CODE_VALUE_OFFSET = { @@ -86,6 +100,17 @@ final class Decode { static final int MAX_TRANSFORMED_WORD_LENGTH = 5 + MAX_WORD_LENGTH + 8; + private static final int MAX_DISTANCE_BITS = 24; + private static final int MAX_LARGE_WINDOW_DISTANCE_BITS = 62; + + /** + * Safe distance limit. + * + * Limit ((1 << 31) - 4) allows safe distance calculation without overflows, + * given the distance alphabet size is limited to corresponding size. + */ + private static final int MAX_ALLOWED_DISTANCE = 0x7FFFFFFC; + //---------------------------------------------------------------------------- // Prefix code LUT. //---------------------------------------------------------------------------- @@ -98,33 +123,103 @@ final class Decode { 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 8, 9, 10, 11, 12, 13, 24 }; - static final int[] INSERT_LENGTH_OFFSET = { - 0, 1, 2, 3, 4, 5, 6, 8, 10, 14, 18, 26, 34, 50, 66, 98, 130, 194, 322, 578, 1090, 2114, 6210, - 22594 + static final short[] INSERT_LENGTH_N_BITS = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, 0x03, 0x03, + 0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x18 }; - static final int[] INSERT_LENGTH_N_BITS = { - 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 12, 14, 24 + static final short[] COPY_LENGTH_N_BITS = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x02, 0x02, + 0x03, 0x03, 0x04, 0x04, 0x05, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x18 }; - static final int[] COPY_LENGTH_OFFSET = { - 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 18, 22, 30, 38, 54, 70, 102, 134, 198, 326, 582, 1094, - 2118 - }; + // Each command is represented with 4x16-bit values: + // * [insertLenExtraBits, copyLenExtraBits] + // * insertLenOffset + // * copyLenOffset + // * distanceContext + static final short[] CMD_LOOKUP = new short[NUM_COMMAND_CODES * 4]; - static final int[] COPY_LENGTH_N_BITS = { - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 7, 8, 9, 10, 24 - }; + static { + unpackCommandLookupTable(CMD_LOOKUP); + } - static final int[] INSERT_RANGE_LUT = { - 0, 0, 8, 8, 0, 16, 8, 16, 16 - }; + private static int log2floor(int i) { + int result = -1; + int step = 16; + while (step > 0) { + if ((i >>> step) != 0) { + result += step; + i = i >>> step; + } + step = step >> 1; + } + return result + i; + } - static final int[] COPY_RANGE_LUT = { - 0, 8, 0, 8, 16, 0, 16, 8, 16 - }; + private static int calculateDistanceAlphabetSize(int npostfix, int ndirect, int maxndistbits) { + return NUM_DISTANCE_SHORT_CODES + ndirect + 2 * (maxndistbits << npostfix); + } + + // TODO: add a correctness test for this function when + // large-window and dictionary are implemented. + private static int calculateDistanceAlphabetLimit(int maxDistance, int npostfix, int ndirect) { + if (maxDistance < ndirect + (2 << npostfix)) { + throw new IllegalArgumentException("maxDistance is too small"); + } + int offset = ((maxDistance - ndirect) >> npostfix) + 4; + int ndistbits = log2floor(offset) - 1; + int group = ((ndistbits - 1) << 1) | ((offset >> ndistbits) & 1); + return ((group - 1) << npostfix) + (1 << npostfix) + ndirect + NUM_DISTANCE_SHORT_CODES; + } + + private static void unpackCommandLookupTable(short[] cmdLookup) { + short[] insertLengthOffsets = new short[24]; + short[] copyLengthOffsets = new short[24]; + copyLengthOffsets[0] = 2; + for (int i = 0; i < 23; ++i) { + insertLengthOffsets[i + 1] = + (short) (insertLengthOffsets[i] + (1 << INSERT_LENGTH_N_BITS[i])); + copyLengthOffsets[i + 1] = + (short) (copyLengthOffsets[i] + (1 << COPY_LENGTH_N_BITS[i])); + } + for (int cmdCode = 0; cmdCode < NUM_COMMAND_CODES; ++cmdCode) { + int rangeIdx = cmdCode >>> 6; + /* -4 turns any regular distance code to negative. */ + int distanceContextOffset = -4; + if (rangeIdx >= 2) { + rangeIdx -= 2; + distanceContextOffset = 0; + } + int insertCode = (((0x29850 >>> (rangeIdx * 2)) & 0x3) << 3) | ((cmdCode >>> 3) & 7); + int copyCode = (((0x26244 >>> (rangeIdx * 2)) & 0x3) << 3) | (cmdCode & 7); + short copyLengthOffset = copyLengthOffsets[copyCode]; + int distanceContext = + distanceContextOffset + (copyLengthOffset > 4 ? 3 : copyLengthOffset - 2); + int index = cmdCode * 4; + cmdLookup[index + 0] = + (short) (INSERT_LENGTH_N_BITS[insertCode] | (COPY_LENGTH_N_BITS[copyCode] << 8)); + cmdLookup[index + 1] = insertLengthOffsets[insertCode]; + cmdLookup[index + 2] = copyLengthOffsets[copyCode]; + cmdLookup[index + 3] = (short) distanceContext; + } + } + + /** + * Reads brotli stream header and parses "window bits". + * + * @param s initialized state, before any read is performed. + * @return -1 if header is invalid + */ private static int decodeWindowBits(State s) { + /* Change the meaning of flag. Before that step it means "decoder must be capable of reading + * "large-window" brotli stream. After this step it means that "large-window" feature + * is actually detected. Despite the window size could be same as before (lgwin = 10..24), + * encoded distances are allowed to be much greater, thus bigger dictinary could be used. */ + int largeWindowEnabled = s.isLargeWindow; + s.isLargeWindow = 0; + BitReader.fillBitWindow(s); if (BitReader.readFewBits(s, 1) == 0) { return 16; @@ -135,7 +230,25 @@ final class Decode { } n = BitReader.readFewBits(s, 3); if (n != 0) { - return 8 + n; + if (n == 1) { + if (largeWindowEnabled == 0) { + /* Reserved value in regular brotli stream. */ + return -1; + } + s.isLargeWindow = 1; + /* Check "reserved" bit for future (post-large-window) extensions. */ + if (BitReader.readFewBits(s, 1) == 1) { + return -1; + } + n = BitReader.readFewBits(s, 6); + if (n < MIN_LARGE_WINDOW_BITS || n > MAX_LARGE_WINDOW_BITS) { + /* Encoded window bits value is too small or too big. */ + return -1; + } + return n; + } else { + return 8 + n; + } } return 17; } @@ -147,13 +260,20 @@ final class Decode { * * @param s initialized state, before any read is performed. */ - static void setEager(State s) { + static void enableEagerOutput(State s) { if (s.runningState != INITIALIZED) { throw new IllegalStateException("State MUST be freshly initialized"); } s.isEager = 1; } + static void enableLargeWindow(State s) { + if (s.runningState != INITIALIZED) { + throw new IllegalStateException("State MUST be freshly initialized"); + } + s.isLargeWindow = 1; + } + /** * Associate input with decoder state. * @@ -164,7 +284,13 @@ final class Decode { if (s.runningState != UNINITIALIZED) { throw new IllegalStateException("State MUST be uninitialized"); } - s.blockTrees = new int[6 * HUFFMAN_TABLE_SIZE]; + /* 6 trees + 1 extra "offset" slot to simplify table decoding logic. */ + s.blockTrees = new int[7 + 3 * (HUFFMAN_TABLE_SIZE_258 + HUFFMAN_TABLE_SIZE_26)]; + s.blockTrees[0] = 7; + s.distRbIdx = 3; + int maxDistanceAlphabetLimit = calculateDistanceAlphabetLimit(MAX_ALLOWED_DISTANCE, 3, 15 << 3); + s.distExtraBits = new byte[maxDistanceAlphabetLimit]; + s.distOffset = new int[maxDistanceAlphabetLimit]; s.input = input; BitReader.initBitReader(s); s.runningState = INITIALIZED; @@ -246,11 +372,12 @@ final class Decode { /** * Decodes the next Huffman code from bit-stream. */ - private static int readSymbol(int[] table, int offset, State s) { + private static int readSymbol(int[] tableGroup, int tableIdx, State s) { + int offset = tableGroup[tableIdx]; int val = BitReader.peekBits(s); offset += val & HUFFMAN_TABLE_MASK; - int bits = table[offset] >> 16; - int sym = table[offset] & 0xFFFF; + int bits = tableGroup[offset] >> 16; + int sym = tableGroup[offset] & 0xFFFF; if (bits <= HUFFMAN_TABLE_BITS) { s.bitOffset += bits; return sym; @@ -258,27 +385,18 @@ final class Decode { offset += sym; int mask = (1 << bits) - 1; offset += (val & mask) >>> HUFFMAN_TABLE_BITS; - s.bitOffset += ((table[offset] >> 16) + HUFFMAN_TABLE_BITS); - return table[offset] & 0xFFFF; + s.bitOffset += ((tableGroup[offset] >> 16) + HUFFMAN_TABLE_BITS); + return tableGroup[offset] & 0xFFFF; } - private static int readBlockLength(int[] table, int offset, State s) { + private static int readBlockLength(int[] tableGroup, int tableIdx, State s) { BitReader.fillBitWindow(s); - int code = readSymbol(table, offset, s); + int code = readSymbol(tableGroup, tableIdx, s); int n = BLOCK_LENGTH_N_BITS[code]; BitReader.fillBitWindow(s); return BLOCK_LENGTH_OFFSET[code] + BitReader.readBits(s, n); } - private static int translateShortCodes(int code, int[] ringBuffer, int index) { - if (code < NUM_DISTANCE_SHORT_CODES) { - index += DISTANCE_SHORT_CODE_INDEX_OFFSET[code]; - index &= 3; - return ringBuffer[index] + DISTANCE_SHORT_CODE_VALUE_OFFSET[code]; - } - return code - NUM_DISTANCE_SHORT_CODES + 1; - } - private static void moveToFront(int[] v, int index) { int value = v[index]; for (; index > 0; index--) { @@ -308,9 +426,9 @@ final class Decode { int repeat = 0; int repeatCodeLen = 0; int space = 32768; - int[] table = new int[32]; - - Huffman.buildHuffmanTable(table, 0, 5, codeLengthCodeLengths, CODE_LENGTH_CODES); + int[] table = new int[32 + 1]; /* Speculative single entry table group. */ + int tableIdx = table.length - 1; + Huffman.buildHuffmanTable(table, tableIdx, 5, codeLengthCodeLengths, CODE_LENGTH_CODES); while (symbol < numSymbols && space > 0) { BitReader.readMoreInput(s); @@ -361,85 +479,128 @@ final class Decode { Utils.fillIntsWithZeroes(codeLengths, symbol, numSymbols); } - static int checkDupes(int[] symbols, int length) { + private static void checkDupes(int[] symbols, int length) { for (int i = 0; i < length - 1; ++i) { for (int j = i + 1; j < length; ++j) { if (symbols[i] == symbols[j]) { - return 0; + throw new BrotliRuntimeException("Duplicate simple Huffman code symbol"); // COV_NF_LINE } } } - return 1; } - // TODO: Use specialized versions for smaller tables. - static void readHuffmanCode(int alphabetSize, int[] table, int offset, State s) { - int ok = 1; - int simpleCodeOrSkip; - BitReader.readMoreInput(s); - // TODO: Avoid allocation. - int[] codeLengths = new int[alphabetSize]; - BitReader.fillBitWindow(s); - simpleCodeOrSkip = BitReader.readFewBits(s, 2); - if (simpleCodeOrSkip == 1) { // Read symbols, codes & code lengths directly. - int maxBitsCounter = alphabetSize - 1; - int maxBits = 0; - int[] symbols = new int[4]; - int numSymbols = BitReader.readFewBits(s, 2) + 1; - while (maxBitsCounter != 0) { - maxBitsCounter >>= 1; - maxBits++; - } - // TODO: uncomment when codeLengths is reused. - // Utils.fillWithZeroes(codeLengths, 0, alphabetSize); - for (int i = 0; i < numSymbols; i++) { - BitReader.fillBitWindow(s); - symbols[i] = BitReader.readFewBits(s, maxBits) % alphabetSize; - codeLengths[symbols[i]] = 2; - } - codeLengths[symbols[0]] = 1; - switch (numSymbols) { - case 2: - codeLengths[symbols[1]] = 1; - break; - case 4: - if (BitReader.readFewBits(s, 1) == 1) { - codeLengths[symbols[2]] = 3; - codeLengths[symbols[3]] = 3; - } else { - codeLengths[symbols[0]] = 2; - } - break; - default: - break; - } - ok = checkDupes(symbols, numSymbols); - } else { // Decode Huffman-coded code lengths. - int[] codeLengthCodeLengths = new int[CODE_LENGTH_CODES]; - int space = 32; - int numCodes = 0; - for (int i = simpleCodeOrSkip; i < CODE_LENGTH_CODES && space > 0; i++) { - int codeLenIdx = CODE_LENGTH_CODE_ORDER[i]; - BitReader.fillBitWindow(s); - int p = BitReader.peekBits(s) & 15; - // TODO: Demultiplex FIXED_TABLE. - s.bitOffset += FIXED_TABLE[p] >> 16; - int v = FIXED_TABLE[p] & 0xFFFF; - codeLengthCodeLengths[codeLenIdx] = v; - if (v != 0) { - space -= (32 >> v); - numCodes++; - } + /** + * Reads up to 4 symbols directly and applies predefined histograms. + */ + private static int readSimpleHuffmanCode(int alphabetSizeMax, int alphabetSizeLimit, + int[] tableGroup, int tableIdx, State s) { + // TODO: Avoid allocation? + int[] codeLengths = new int[alphabetSizeLimit]; + int[] symbols = new int[4]; + + int maxBits = 1 + log2floor(alphabetSizeMax - 1); + + int numSymbols = BitReader.readFewBits(s, 2) + 1; + for (int i = 0; i < numSymbols; i++) { + BitReader.fillBitWindow(s); + int symbol = BitReader.readFewBits(s, maxBits); + if (symbol >= alphabetSizeLimit) { + throw new BrotliRuntimeException("Can't readHuffmanCode"); // COV_NF_LINE } - if (space != 0 && numCodes != 1) { - ok = 0; + symbols[i] = symbol; + } + checkDupes(symbols, numSymbols); + + int histogramId = numSymbols; + if (numSymbols == 4) { + histogramId += BitReader.readFewBits(s, 1); + } + + switch (histogramId) { + case 1: + codeLengths[symbols[0]] = 1; + break; + + case 2: + codeLengths[symbols[0]] = 1; + codeLengths[symbols[1]] = 1; + break; + + case 3: + codeLengths[symbols[0]] = 1; + codeLengths[symbols[1]] = 2; + codeLengths[symbols[2]] = 2; + break; + + case 4: // uniform 4-symbol histogram + codeLengths[symbols[0]] = 2; + codeLengths[symbols[1]] = 2; + codeLengths[symbols[2]] = 2; + codeLengths[symbols[3]] = 2; + break; + + case 5: // prioritized 4-symbol histogram + codeLengths[symbols[0]] = 1; + codeLengths[symbols[1]] = 2; + codeLengths[symbols[2]] = 3; + codeLengths[symbols[3]] = 3; + break; + + default: + break; + } + + // TODO: Use specialized version? + return Huffman.buildHuffmanTable( + tableGroup, tableIdx, HUFFMAN_TABLE_BITS, codeLengths, alphabetSizeLimit); + } + + // Decode Huffman-coded code lengths. + private static int readComplexHuffmanCode(int alphabetSizeLimit, int skip, + int[] tableGroup, int tableIdx, State s) { + // TODO: Avoid allocation? + int[] codeLengths = new int[alphabetSizeLimit]; + int[] codeLengthCodeLengths = new int[CODE_LENGTH_CODES]; + int space = 32; + int numCodes = 0; + for (int i = skip; i < CODE_LENGTH_CODES && space > 0; i++) { + int codeLenIdx = CODE_LENGTH_CODE_ORDER[i]; + BitReader.fillBitWindow(s); + int p = BitReader.peekBits(s) & 15; + // TODO: Demultiplex FIXED_TABLE. + s.bitOffset += FIXED_TABLE[p] >> 16; + int v = FIXED_TABLE[p] & 0xFFFF; + codeLengthCodeLengths[codeLenIdx] = v; + if (v != 0) { + space -= (32 >> v); + numCodes++; } - readHuffmanCodeLengths(codeLengthCodeLengths, alphabetSize, codeLengths, s); } - if (ok == 0) { - throw new BrotliRuntimeException("Can't readHuffmanCode"); // COV_NF_LINE + if (space != 0 && numCodes != 1) { + throw new BrotliRuntimeException("Corrupted Huffman code histogram"); // COV_NF_LINE + } + + readHuffmanCodeLengths(codeLengthCodeLengths, alphabetSizeLimit, codeLengths, s); + + return Huffman.buildHuffmanTable( + tableGroup, tableIdx, HUFFMAN_TABLE_BITS, codeLengths, alphabetSizeLimit); + } + + /** + * Decodes Huffman table from bit-stream. + * + * @return number of slots used by resulting Huffman table + */ + private static int readHuffmanCode(int alphabetSizeMax, int alphabetSizeLimit, + int[] tableGroup, int tableIdx, State s) { + BitReader.readMoreInput(s); + BitReader.fillBitWindow(s); + int simpleCodeOrSkip = BitReader.readFewBits(s, 2); + if (simpleCodeOrSkip == 1) { + return readSimpleHuffmanCode(alphabetSizeMax, alphabetSizeLimit, tableGroup, tableIdx, s); + } else { + return readComplexHuffmanCode(alphabetSizeLimit, simpleCodeOrSkip, tableGroup, tableIdx, s); } - Huffman.buildHuffmanTable(table, offset, HUFFMAN_TABLE_BITS, codeLengths, alphabetSize); } private static int decodeContextMap(int contextMapSize, byte[] contextMap, State s) { @@ -457,12 +618,16 @@ final class Decode { if (useRleForZeros != 0) { maxRunLengthPrefix = BitReader.readFewBits(s, 4) + 1; } - int[] table = new int[HUFFMAN_TABLE_SIZE]; - readHuffmanCode(numTrees + maxRunLengthPrefix, table, 0, s); + int alphabetSize = numTrees + maxRunLengthPrefix; + int tableSize = MAX_HUFFMAN_TABLE_SIZE[(alphabetSize + 31) >> 5]; + /* Speculative single entry table group. */ + int[] table = new int[tableSize + 1]; + int tableIdx = table.length - 1; + readHuffmanCode(alphabetSize, alphabetSize, table, tableIdx, s); for (int i = 0; i < contextMapSize; ) { BitReader.readMoreInput(s); BitReader.fillBitWindow(s); - int code = readSymbol(table, 0, s); + int code = readSymbol(table, tableIdx, s); if (code == 0) { contextMap[i] = 0; i++; @@ -493,8 +658,8 @@ final class Decode { final int[] ringBuffers = s.rings; final int offset = 4 + treeType * 2; BitReader.fillBitWindow(s); - int blockType = readSymbol(s.blockTrees, treeType * HUFFMAN_TABLE_SIZE, s); - int result = readBlockLength(s.blockTrees, (treeType + 3) * HUFFMAN_TABLE_SIZE, s); + int blockType = readSymbol(s.blockTrees, 2 * treeType, s); + int result = readBlockLength(s.blockTrees, 2 * treeType + 1, s); if (blockType == 1) { blockType = ringBuffers[offset + 1] + 1; @@ -515,8 +680,7 @@ final class Decode { s.literalBlockLength = decodeBlockTypeAndLength(s, 0, s.numLiteralBlockTypes); int literalBlockType = s.rings[5]; s.contextMapSlice = literalBlockType << LITERAL_CONTEXT_BITS; - s.literalTreeIndex = s.contextMap[s.contextMapSlice] & 0xFF; - s.literalTree = s.hGroup0[s.literalTreeIndex]; + s.literalTreeIdx = s.contextMap[s.contextMapSlice] & 0xFF; int contextMode = s.contextModes[literalBlockType]; s.contextLookupOffset1 = contextMode << 9; s.contextLookupOffset2 = s.contextLookupOffset1 + 256; @@ -524,7 +688,7 @@ final class Decode { private static void decodeCommandBlockSwitch(State s) { s.commandBlockLength = decodeBlockTypeAndLength(s, 1, s.numCommandBlockTypes); - s.treeCommandOffset = s.hGroup1[s.rings[7]]; + s.commandTreeIdx = s.rings[7]; } private static void decodeDistanceBlockSwitch(State s) { @@ -563,9 +727,9 @@ final class Decode { return; } // TODO: Reset? Do we need this? - s.hGroup0 = new int[0]; - s.hGroup1 = new int[0]; - s.hGroup2 = new int[0]; + s.literalTreeGroup = new int[0]; + s.commandTreeGroup = new int[0]; + s.distanceTreeGroup = new int[0]; BitReader.readMoreInput(s); decodeMetaBlockLength(s); @@ -592,12 +756,57 @@ final class Decode { } private static int readMetablockPartition(State s, int treeType, int numBlockTypes) { + int offset = s.blockTrees[2 * treeType]; if (numBlockTypes <= 1) { + s.blockTrees[2 * treeType + 1] = offset; + s.blockTrees[2 * treeType + 2] = offset; return 1 << 28; } - readHuffmanCode(numBlockTypes + 2, s.blockTrees, treeType * HUFFMAN_TABLE_SIZE, s); - readHuffmanCode(NUM_BLOCK_LENGTH_CODES, s.blockTrees, (treeType + 3) * HUFFMAN_TABLE_SIZE, s); - return readBlockLength(s.blockTrees, (treeType + 3) * HUFFMAN_TABLE_SIZE, s); + + int blockTypeAlphabetSize = numBlockTypes + 2; + offset += readHuffmanCode( + blockTypeAlphabetSize, blockTypeAlphabetSize, s.blockTrees, 2 * treeType, s); + s.blockTrees[2 * treeType + 1] = offset; + + int blockLengthAlphabetSize = NUM_BLOCK_LENGTH_CODES; + offset += readHuffmanCode( + blockLengthAlphabetSize, blockLengthAlphabetSize, s.blockTrees, 2 * treeType + 1, s); + s.blockTrees[2 * treeType + 2] = offset; + + return readBlockLength(s.blockTrees, 2 * treeType + 1, s); + } + + private static void calculateDistanceLut(State s, int alphabetSizeLimit) { + byte[] distExtraBits = s.distExtraBits; + int[] distOffset = s.distOffset; + int npostfix = s.distancePostfixBits; + int ndirect = s.numDirectDistanceCodes; + int postfix = 1 << npostfix; + int bits = 1; + int half = 0; + + /* Skip short codes. */ + int i = NUM_DISTANCE_SHORT_CODES; + + /* Fill direct codes. */ + for (int j = 0; j < ndirect; ++j) { + distExtraBits[i] = 0; + distOffset[i] = j + 1; + ++i; + } + + /* Fill regular distance codes. */ + while (i < alphabetSizeLimit) { + int base = ndirect + ((((2 + half) << bits) - 4) << npostfix) + 1; + /* Always fill the complete group. */ + for (int j = 0; j < postfix; ++j) { + distExtraBits[i] = (byte) bits; + distOffset[i] = base + j; + ++i; + } + bits = bits + half; + half = half ^ 1; + } } private static void readMetablockHuffmanCodesAndContextMaps(State s) { @@ -611,10 +820,8 @@ final class Decode { BitReader.readMoreInput(s); BitReader.fillBitWindow(s); s.distancePostfixBits = BitReader.readFewBits(s, 2); - s.numDirectDistanceCodes = - NUM_DISTANCE_SHORT_CODES + (BitReader.readFewBits(s, 4) << s.distancePostfixBits); + s.numDirectDistanceCodes = BitReader.readFewBits(s, 4) << s.distancePostfixBits; s.distancePostfixMask = (1 << s.distancePostfixBits) - 1; - int numDistanceCodes = s.numDirectDistanceCodes + (48 << s.distancePostfixBits); // TODO: Reuse? s.contextModes = new byte[s.numLiteralBlockTypes]; for (int i = 0; i < s.numLiteralBlockTypes;) { @@ -622,7 +829,7 @@ final class Decode { int limit = Math.min(i + 96, s.numLiteralBlockTypes); for (; i < limit; ++i) { BitReader.fillBitWindow(s); - s.contextModes[i] = (byte) (BitReader.readFewBits(s, 2)); + s.contextModes[i] = (byte) BitReader.readFewBits(s, 2); } BitReader.readMoreInput(s); } @@ -644,18 +851,29 @@ final class Decode { int numDistTrees = decodeContextMap(s.numDistanceBlockTypes << DISTANCE_CONTEXT_BITS, s.distContextMap, s); - s.hGroup0 = decodeHuffmanTreeGroup(NUM_LITERAL_CODES, numLiteralTrees, s); - s.hGroup1 = - decodeHuffmanTreeGroup(NUM_INSERT_AND_COPY_CODES, s.numCommandBlockTypes, s); - s.hGroup2 = decodeHuffmanTreeGroup(numDistanceCodes, numDistTrees, s); + s.literalTreeGroup = decodeHuffmanTreeGroup(NUM_LITERAL_CODES, NUM_LITERAL_CODES, + numLiteralTrees, s); + s.commandTreeGroup = decodeHuffmanTreeGroup(NUM_COMMAND_CODES, NUM_COMMAND_CODES, + s.numCommandBlockTypes, s); + int distanceAlphabetSizeMax = calculateDistanceAlphabetSize( + s.distancePostfixBits, s.numDirectDistanceCodes, MAX_DISTANCE_BITS); + int distanceAlphabetSizeLimit = distanceAlphabetSizeMax; + if (s.isLargeWindow == 1) { + distanceAlphabetSizeMax = calculateDistanceAlphabetSize( + s.distancePostfixBits, s.numDirectDistanceCodes, MAX_LARGE_WINDOW_DISTANCE_BITS); + distanceAlphabetSizeLimit = calculateDistanceAlphabetLimit( + MAX_ALLOWED_DISTANCE, s.distancePostfixBits, s.numDirectDistanceCodes); + } + s.distanceTreeGroup = decodeHuffmanTreeGroup(distanceAlphabetSizeMax, distanceAlphabetSizeLimit, + numDistTrees, s); + calculateDistanceLut(s, distanceAlphabetSizeLimit); s.contextMapSlice = 0; s.distContextMapSlice = 0; - s.contextLookupOffset1 = (int) (s.contextModes[0]) << 9; + s.contextLookupOffset1 = s.contextModes[0] * 512; s.contextLookupOffset2 = s.contextLookupOffset1 + 256; - s.literalTreeIndex = 0; - s.literalTree = s.hGroup0[0]; - s.treeCommandOffset = s.hGroup1[0]; + s.literalTreeIdx = 0; + s.commandTreeIdx = 0; s.rings[4] = 1; s.rings[5] = 0; @@ -706,13 +924,14 @@ final class Decode { } } - private static int[] decodeHuffmanTreeGroup(int alphabetSize, int n, State s) { - int[] group = new int[n + (n * HUFFMAN_TABLE_SIZE)]; + private static int[] decodeHuffmanTreeGroup(int alphabetSizeMax, int alphabetSizeLimit, + int n, State s) { + int maxTableSize = MAX_HUFFMAN_TABLE_SIZE[(alphabetSizeLimit + 31) >> 5]; + int[] group = new int[n + n * maxTableSize]; int next = n; - for (int i = 0; i < n; i++) { + for (int i = 0; i < n; ++i) { group[i] = next; - Decode.readHuffmanCode(alphabetSize, group, next, s); - next += HUFFMAN_TABLE_SIZE; + next += readHuffmanCode(alphabetSizeMax, alphabetSizeLimit, group, i, s); } return group; } @@ -738,7 +957,7 @@ final class Decode { } if (s.runningState == INITIALIZED) { int windowBits = decodeWindowBits(s); - if (windowBits == 9) { /* Reserved case for future expansion. */ + if (windowBits == -1) { /* Reserved case for future expansion. */ throw new BrotliRuntimeException("Invalid 'windowBits' code"); } s.maxRingBufferSize = 1 << windowBits; @@ -780,23 +999,21 @@ final class Decode { } s.commandBlockLength--; BitReader.fillBitWindow(s); - int cmdCode = readSymbol(s.hGroup1, s.treeCommandOffset, s); - int rangeIdx = cmdCode >>> 6; - s.distanceCode = 0; - if (rangeIdx >= 2) { - rangeIdx -= 2; - s.distanceCode = -1; - } - int insertCode = INSERT_RANGE_LUT[rangeIdx] + ((cmdCode >>> 3) & 7); + int cmdCode = readSymbol(s.commandTreeGroup, s.commandTreeIdx, s) << 2; + short insertAndCopyExtraBits = CMD_LOOKUP[cmdCode]; + int insertLengthOffset = CMD_LOOKUP[cmdCode + 1]; + int copyLengthOffset = CMD_LOOKUP[cmdCode + 2]; + s.distanceCode = CMD_LOOKUP[cmdCode + 3]; BitReader.fillBitWindow(s); - int insertBits = INSERT_LENGTH_N_BITS[insertCode]; - int insertExtra = BitReader.readBits(s, insertBits); - s.insertLength = INSERT_LENGTH_OFFSET[insertCode] + insertExtra; - int copyCode = COPY_RANGE_LUT[rangeIdx] + (cmdCode & 7); + { + int extraBits = insertAndCopyExtraBits & 0xFF; + s.insertLength = insertLengthOffset + BitReader.readBits(s, extraBits); + } BitReader.fillBitWindow(s); - int copyBits = COPY_LENGTH_N_BITS[copyCode]; - int copyExtra = BitReader.readBits(s, copyBits); - s.copyLength = COPY_LENGTH_OFFSET[copyCode] + copyExtra; + { + int extraBits = insertAndCopyExtraBits >> 8; + s.copyLength = copyLengthOffset + BitReader.readBits(s, extraBits); + } s.j = 0; s.runningState = INSERT_LOOP; @@ -811,8 +1028,7 @@ final class Decode { } s.literalBlockLength--; BitReader.fillBitWindow(s); - ringBuffer[s.pos] = - (byte) readSymbol(s.hGroup0, s.literalTree, s); + ringBuffer[s.pos] = (byte) readSymbol(s.literalTreeGroup, s.literalTreeIdx, s); s.pos++; s.j++; if (s.pos >= fence) { @@ -829,14 +1045,13 @@ final class Decode { if (s.literalBlockLength == 0) { decodeLiteralBlockSwitch(s); } - int literalTreeIndex = s.contextMap[s.contextMapSlice - + (Context.LOOKUP[s.contextLookupOffset1 + prevByte1] - | Context.LOOKUP[s.contextLookupOffset2 + prevByte2])] & 0xFF; + int literalContext = Context.LOOKUP[s.contextLookupOffset1 + prevByte1] + | Context.LOOKUP[s.contextLookupOffset2 + prevByte2]; + int literalTreeIdx = s.contextMap[s.contextMapSlice + literalContext] & 0xFF; s.literalBlockLength--; prevByte2 = prevByte1; BitReader.fillBitWindow(s); - prevByte1 = readSymbol( - s.hGroup0, s.hGroup0[literalTreeIndex], s); + prevByte1 = readSymbol(s.literalTreeGroup, literalTreeIdx, s); ringBuffer[s.pos] = (byte) prevByte1; s.pos++; s.j++; @@ -855,36 +1070,38 @@ final class Decode { s.runningState = MAIN_LOOP; continue; } - if (s.distanceCode < 0) { + int distanceCode = s.distanceCode; + if (distanceCode < 0) { + // distanceCode in untouched; assigning it 0 won't affect distance ring buffer rolling. + s.distance = s.rings[s.distRbIdx]; + } else { BitReader.readMoreInput(s); if (s.distanceBlockLength == 0) { decodeDistanceBlockSwitch(s); } s.distanceBlockLength--; BitReader.fillBitWindow(s); - s.distanceCode = readSymbol(s.hGroup2, s.hGroup2[ - s.distContextMap[s.distContextMapSlice - + (s.copyLength > 4 ? 3 : s.copyLength - 2)] & 0xFF], s); - if (s.distanceCode >= s.numDirectDistanceCodes) { - s.distanceCode -= s.numDirectDistanceCodes; - int postfix = s.distanceCode & s.distancePostfixMask; - s.distanceCode >>>= s.distancePostfixBits; - int n = (s.distanceCode >>> 1) + 1; - int offset = ((2 + (s.distanceCode & 1)) << n) - 4; - BitReader.fillBitWindow(s); - int distanceExtra = BitReader.readBits(s, n); - s.distanceCode = s.numDirectDistanceCodes + postfix - + ((offset + distanceExtra) << s.distancePostfixBits); + int distTreeIdx = s.distContextMap[s.distContextMapSlice + distanceCode] & 0xFF; + distanceCode = readSymbol(s.distanceTreeGroup, distTreeIdx, s); + if (distanceCode < NUM_DISTANCE_SHORT_CODES) { + int index = (s.distRbIdx + DISTANCE_SHORT_CODE_INDEX_OFFSET[distanceCode]) & 0x3; + s.distance = s.rings[index] + DISTANCE_SHORT_CODE_VALUE_OFFSET[distanceCode]; + if (s.distance < 0) { + throw new BrotliRuntimeException("Negative distance"); // COV_NF_LINE + } + } else { + int extraBits = s.distExtraBits[distanceCode]; + int bits; + if (s.bitOffset + extraBits <= BitReader.BITNESS) { + bits = BitReader.readFewBits(s, extraBits); + } else { + BitReader.fillBitWindow(s); + bits = BitReader.readBits(s, extraBits); + } + s.distance = s.distOffset[distanceCode] + (bits << s.distancePostfixBits); } } - // Convert the distance code to the actual distance by possibly looking up past distances - // from the ringBuffer. - s.distance = translateShortCodes(s.distanceCode, s.rings, s.distRbIdx); - if (s.distance < 0) { - throw new BrotliRuntimeException("Negative distance"); // COV_NF_LINE - } - if (s.maxDistance != s.maxBackwardDistance && s.pos < s.maxBackwardDistance) { s.maxDistance = s.pos; @@ -897,9 +1114,9 @@ final class Decode { continue; } - if (s.distanceCode > 0) { - s.rings[s.distRbIdx & 3] = s.distance; - s.distRbIdx++; + if (distanceCode > 0) { + s.distRbIdx = (s.distRbIdx + 1) & 0x3; + s.rings[s.distRbIdx] = s.distance; } if (s.copyLength > s.metaBlockLength) { @@ -916,7 +1133,10 @@ final class Decode { int dstEnd = dst + copyLength; if ((srcEnd < ringBufferMask) && (dstEnd < ringBufferMask)) { if (copyLength < 12 || (srcEnd > dst && dstEnd > src)) { - for (int k = 0; k < copyLength; ++k) { + for (int k = 0; k < copyLength; k += 4) { + ringBuffer[dst++] = ringBuffer[src++]; + ringBuffer[dst++] = ringBuffer[src++]; + ringBuffer[dst++] = ringBuffer[src++]; ringBuffer[dst++] = ringBuffer[src++]; } } else { @@ -945,6 +1165,10 @@ final class Decode { continue; case TRANSFORM: + // This check is done here to unburden the hot loop. + if (s.distance > MAX_ALLOWED_DISTANCE) { + throw new BrotliRuntimeException("Invalid backward reference"); // COV_NF_LINE + } if (s.copyLength >= MIN_WORD_LENGTH && s.copyLength <= MAX_WORD_LENGTH) { int offset = DICTIONARY_OFFSETS_BY_LENGTH[s.copyLength]; diff --git a/java/org/brotli/dec/EagerStreamTest.java b/java/org/brotli/dec/EagerStreamTest.java index 069ae34..8e079c0 100755 --- a/java/org/brotli/dec/EagerStreamTest.java +++ b/java/org/brotli/dec/EagerStreamTest.java @@ -375,7 +375,7 @@ public class EagerStreamTest { ps = new ProxyStream(new ByteArrayInputStream(DATA)); reader = new BrotliInputStream(ps, 1); - reader.setEager(true); + reader.enableEagerOutput(); reader.read(buffer); reader.close(); int eagerReadBytes = ps.readBytes; diff --git a/java/org/brotli/dec/Huffman.java b/java/org/brotli/dec/Huffman.java index 868701a..38f7f29 100644 --- a/java/org/brotli/dec/Huffman.java +++ b/java/org/brotli/dec/Huffman.java @@ -58,9 +58,12 @@ final class Huffman { /** * Builds Huffman lookup table assuming code lengths are in symbol order. + * + * @return number of slots used by resulting Huffman table */ - static void buildHuffmanTable(int[] rootTable, int tableOffset, int rootBits, int[] codeLengths, + static int buildHuffmanTable(int[] tableGroup, int tableIdx, int rootBits, int[] codeLengths, int codeLengthsSize) { + int tableOffset = tableGroup[tableIdx]; int key; // Reversed prefix code. int[] sorted = new int[codeLengthsSize]; // Symbols sorted by code length. // TODO: fill with zeroes? @@ -93,9 +96,9 @@ final class Huffman { // Special case code with only one value. if (offset[MAX_LENGTH] == 1) { for (key = 0; key < totalSize; key++) { - rootTable[tableOffset + key] = sorted[0]; + tableGroup[tableOffset + key] = sorted[0]; } - return; + return totalSize; } // Fill in root table. @@ -103,7 +106,8 @@ final class Huffman { symbol = 0; for (int len = 1, step = 2; len <= rootBits; len++, step <<= 1) { for (; count[len] > 0; count[len]--) { - replicateValue(rootTable, tableOffset + key, step, tableSize, len << 16 | sorted[symbol++]); + replicateValue(tableGroup, tableOffset + key, step, tableSize, + len << 16 | sorted[symbol++]); key = getNextKey(key, len); } } @@ -120,13 +124,14 @@ final class Huffman { tableSize = 1 << tableBits; totalSize += tableSize; low = key & mask; - rootTable[tableOffset + low] = + tableGroup[tableOffset + low] = (tableBits + rootBits) << 16 | (currentOffset - tableOffset - low); } - replicateValue(rootTable, currentOffset + (key >> rootBits), step, tableSize, + replicateValue(tableGroup, currentOffset + (key >> rootBits), step, tableSize, (len - rootBits) << 16 | sorted[symbol++]); key = getNextKey(key, len); } } + return totalSize; } } diff --git a/java/org/brotli/dec/State.java b/java/org/brotli/dec/State.java index 2dc46d5..54a5a2d 100644 --- a/java/org/brotli/dec/State.java +++ b/java/org/brotli/dec/State.java @@ -13,6 +13,7 @@ final class State { byte[] contextModes; byte[] contextMap; byte[] distContextMap; + byte[] distExtraBits; byte[] output; byte[] byteBuffer; // BitReader @@ -21,9 +22,10 @@ final class State { int[] intBuffer; // BitReader int[] rings; int[] blockTrees; - int[] hGroup0; - int[] hGroup1; - int[] hGroup2; + int[] literalTreeGroup; + int[] commandTreeGroup; + int[] distanceTreeGroup; + int[] distOffset; long accumulator64; // BitReader: pre-fetched bits. @@ -48,15 +50,14 @@ final class State { int maxDistance; int distRbIdx; int trivialLiteralContext; - int literalTreeIndex; - int literalTree; + int literalTreeIdx; + int commandTreeIdx; int j; int insertLength; int contextMapSlice; int distContextMapSlice; int contextLookupOffset1; int contextLookupOffset2; - int treeCommandOffset; int distanceCode; int numDirectDistanceCodes; int distancePostfixMask; @@ -73,6 +74,7 @@ final class State { int ringBufferBytesWritten; int ringBufferBytesReady; int isEager; + int isLargeWindow; InputStream input; // BitReader diff --git a/java/org/brotli/dec/SynthTest.java b/java/org/brotli/dec/SynthTest.java index de91c37..e269e6b 100644 --- a/java/org/brotli/dec/SynthTest.java +++ b/java/org/brotli/dec/SynthTest.java @@ -12,6 +12,7 @@ import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Arrays; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -624,7 +625,7 @@ public class SynthTest { * // one ins/copy and dist block type * vlq_blocktypes: 1 * vlq_blocktypes: 1 - * ndirect: 0 0 + * ndirect: 0, 0 * // two MSB6 literal context modes * bits: "00", "00" * // two literal prefix codes @@ -680,7 +681,7 @@ public class SynthTest { * // one ins/copy and dist block type * vlq_blocktypes: 1 * vlq_blocktypes: 1 - * ndirect: 0 0 + * ndirect: 0, 0 * // two MSB6 literal context modes * bits: "00", "00" * // two literal prefix codes @@ -984,6 +985,46 @@ public class SynthTest { } @Test + public void testDistanceLut() { + byte[] compressed = { + (byte) 0x8b, (byte) 0x02, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x80, + (byte) 0xe3, (byte) 0xb4, (byte) 0x0d, (byte) 0x00, (byte) 0x00, (byte) 0x07, (byte) 0x5b, + (byte) 0x26, (byte) 0x31, (byte) 0x40, (byte) 0x02, (byte) 0x00, (byte) 0xe0, (byte) 0x4e, + (byte) 0x1b, (byte) 0x99, (byte) 0x86, (byte) 0x46, (byte) 0xc6, (byte) 0x22, (byte) 0x14, + (byte) 0x00, (byte) 0x00, (byte) 0x03, (byte) 0x00, (byte) 0x00, (byte) 0x1c, (byte) 0xa7, + (byte) 0x6d, (byte) 0x00, (byte) 0x00, (byte) 0x38, (byte) 0xd8, (byte) 0x32, (byte) 0x89, + (byte) 0x01, (byte) 0x12, (byte) 0x21, (byte) 0x91, (byte) 0x69, (byte) 0x62, (byte) 0x6a, + (byte) 0x36 + }; + checkSynth( + /* + * main_header + * metablock_header_easy: 6, 0 // implicit ndirect: 0, 0 + * command_easy: 3, "abc", 3 // Insert "abc", copy "abc" + * metablock_header_begin: 0, 0, 6, 0 + * vlq_blocktypes: 1 // num litetal block types + * vlq_blocktypes: 1 // num command block types + * vlq_blocktypes: 1 // num distance block types + * ndirect: 3, 0 + * bits: "00" // literal context modes + * vlq_blocktypes: 1 // num literal Huffman trees + * // command has no context -> num trees == num block types + * vlq_blocktypes: 1 // num distance Huffman trees + * huffman_fixed: 256 + * huffman_fixed: 704 + * huffman_simple: 0,1,67, 18 + * command_inscopy_easy: 3, 3 // Insert 3, copy 3 + * command_literals_easy: "def" + * // 0-bit Huffman code : dcode = 18 -> third direct distance + * metablock_lastempty // make sure that no extra distance bits are read + */ + compressed, + true, + "abcabcdefdef" + ); + } + + @Test public void testEmpty() { byte[] compressed = { (byte) 0x3b @@ -1264,7 +1305,7 @@ public class SynthTest { * // one ins/copy and dist block type * vlq_blocktypes: 1 * vlq_blocktypes: 1 - * ndirect: 0 0 + * ndirect: 0, 0 * // two MSB6 literal context modes * bits: "00", "00" * // two literal prefix codes @@ -2707,6 +2748,87 @@ public class SynthTest { */ @Test + public void testStressReadDistanceExtraBits() { + byte[] compressed = { + (byte) 0x4f, (byte) 0xfe, (byte) 0xff, (byte) 0x3f, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x80, (byte) 0xe3, (byte) 0xb4, (byte) 0x0d, (byte) 0x00, (byte) 0x00, (byte) 0x07, + (byte) 0x5b, (byte) 0x26, (byte) 0x31, (byte) 0x40, (byte) 0x02, (byte) 0x00, (byte) 0xe0, + (byte) 0x4e, (byte) 0x9b, (byte) 0xf6, (byte) 0x69, (byte) 0xef, (byte) 0xff, (byte) 0x0c, + (byte) 0x8d, (byte) 0x8c, (byte) 0x05, (byte) 0x10, (byte) 0x00, (byte) 0x00, (byte) 0x00, + (byte) 0x00, (byte) 0x00, (byte) 0x38, (byte) 0x4e, (byte) 0xdb, (byte) 0x00, (byte) 0x00, + (byte) 0x70, (byte) 0xb0, (byte) 0x65, (byte) 0x12, (byte) 0x03, (byte) 0x24, (byte) 0xa8, + (byte) 0xaa, (byte) 0xef, (byte) 0xab, (byte) 0xaa, (byte) 0x7f, (byte) 0x24, (byte) 0x16, + (byte) 0x35, (byte) 0x8f, (byte) 0xac, (byte) 0x9e, (byte) 0x3d, (byte) 0xf7, (byte) 0xf3, + (byte) 0xe3, (byte) 0x0a, (byte) 0xfc, (byte) 0xff, (byte) 0x03, (byte) 0x00, (byte) 0x00, + (byte) 0x78, (byte) 0x01, (byte) 0x08, (byte) 0x30, (byte) 0x31, (byte) 0x32, (byte) 0x33, + (byte) 0x34, (byte) 0x35, (byte) 0x36, (byte) 0x37, (byte) 0x38, (byte) 0x39, (byte) 0x41, + (byte) 0x42, (byte) 0x43, (byte) 0x44, (byte) 0x45, (byte) 0x46, (byte) 0x30, (byte) 0x31, + (byte) 0x32, (byte) 0x33, (byte) 0x34, (byte) 0x35, (byte) 0x36, (byte) 0x37, (byte) 0x38, + (byte) 0x39, (byte) 0x41, (byte) 0x42, (byte) 0x43, (byte) 0x44, (byte) 0x45, (byte) 0x46, + (byte) 0x30, (byte) 0x31, (byte) 0x32, (byte) 0x33, (byte) 0x34, (byte) 0x35, (byte) 0x36, + (byte) 0x37, (byte) 0x38, (byte) 0x39, (byte) 0x41, (byte) 0x42, (byte) 0x43, (byte) 0x44, + (byte) 0x45, (byte) 0x46, (byte) 0x03 + }; + /* This line is added manually. */ + char[] stub = new char[8388602]; Arrays.fill(stub, 'c'); String hex = "0123456789ABCDEF"; + checkSynth( + /* + * main_header: 24 + * metablock_header_easy: 8388605, 0 // 2^23 - 3 = shortest 22-bit distance + * command_easy: 8388602, "abc", 1 + * metablock_header_begin: 0, 0, 3, 0 + * vlq_blocktypes: 1 // num litetal block types + * vlq_blocktypes: 1 // num command block types + * vlq_blocktypes: 1 // num distance block types + * ndirect: 0, 0 + * bits: "00" // literal context modes + * vlq_blocktypes: 1 // num literal Huffman trees + * // command has no context -> num trees == num block types + * vlq_blocktypes: 1 // num distance Huffman trees + * huffman_fixed: 256 + * huffman_fixed: 704 + * // Begin of distance Huffman tree. First 15 codes have lengths 1 to 15. + * // Symbol that corresponds to first half of 22-bit distance range is also + * // 15. All other symbols are 0. + * hskip: 0 + * clcl_ordered: 4,4,4,4, 4,4,4,4, 4,4,4,4, 4,4, 5,5,5,5 + * set_prefix_cl_rle: "0000", "0001", "0010", "0011", \ + * "0100", "0101", "0110", "0111", \ + * "1000", "1001", "1010", "1011", \ + * "1100", "1101", \ + * "11100", "11101", "11110", "11111" + * cl_rle: 1 + * cl_rle: 2 + * cl_rle: 3 + * cl_rle: 4 + * cl_rle: 5 + * cl_rle: 6 + * cl_rle: 7 + * cl_rle: 8 + * cl_rle: 9 + * cl_rle: 10 + * cl_rle: 11 + * cl_rle: 12 + * cl_rle: 13 + * cl_rle: 14 + * cl_rle: 15 + * cl_rle_rep_0: 43 + * cl_rle: 15 // literal number 97, that is, the letter 'a' + * // end of literal Huffman tree + * command_inscopy_easy: 0, 3 // Insert 0, copy 3 + * // 15 bits of distance code plus 22 extra bits + * command_dist_bits: "111111111111111", "0000000000000000000000" + * metablock_uncompressed: "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF" + * metablock_lastempty + */ + compressed, + true, + /* This line is modified manually. */ + "abc" + new String(stub) + "abc" + hex + hex + hex + ); + } + + @Test public void testTooManySymbolsRepeated() { byte[] compressed = { (byte) 0x1b, (byte) 0x03, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x01, (byte) 0x80, @@ -2785,6 +2907,34 @@ public class SynthTest { ); } + @Test + public void testZeroCostLiterals() { + byte[] compressed = { + (byte) 0x9b, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0x00, (byte) 0x20, (byte) 0x54, + (byte) 0x00, (byte) 0x00, (byte) 0x38, (byte) 0xd8, (byte) 0x32, (byte) 0x89, (byte) 0x01, + (byte) 0x12, (byte) 0x00, (byte) 0x00, (byte) 0x77, (byte) 0xda, (byte) 0xcc, (byte) 0xe1, + (byte) 0x7b, (byte) 0xfa, (byte) 0x0f + }; + /* This lines is added manually. */ + char[] expected = new char[16777216]; Arrays.fill(expected, '*'); + checkSynth( + /* + * main_header + * metablock_header_begin: 1, 0, 16777216, 0 + * metablock_header_trivial_context + * huffman_simple: 0,1,256, 42 // Single symbol alphabet + * huffman_fixed: 704 + * huffman_fixed: 64 + * command_inscopy_easy: 16777216, 0 + * // 16777216 times 0 bits + */ + compressed, + true, + /* This line is modified manually. */ + new String(expected) + ); + } + /* GENERATED CODE END */ } diff --git a/java/org/brotli/dec/Utils.java b/java/org/brotli/dec/Utils.java index 1583c75..2d04aec 100644 --- a/java/org/brotli/dec/Utils.java +++ b/java/org/brotli/dec/Utils.java @@ -88,4 +88,15 @@ final class Utils { static void flipBuffer(Buffer buffer) { buffer.flip(); } + + static int isDebugMode() { + boolean assertsEnabled = Boolean.parseBoolean(System.getProperty("BROTLI_ENABLE_ASSERTS")); + return assertsEnabled ? 1 : 0; + } + + // See BitReader.LOG_BITNESS + static int getLogBintness() { + boolean isLongExpensive = Boolean.parseBoolean(System.getProperty("BROTLI_32_BIT_CPU")); + return isLongExpensive ? 5 : 6; + } } diff --git a/java/org/brotli/dec/build_defs.bzl b/java/org/brotli/dec/build_defs.bzl new file mode 100644 index 0000000..d0a015c --- /dev/null +++ b/java/org/brotli/dec/build_defs.bzl @@ -0,0 +1,34 @@ +"""Utilities for Java brotli tests.""" + +_TEST_JVM_FLAGS = [ + "-DBROTLI_ENABLE_ASSERTS=true", +] + +def brotli_java_test(name, main_class = None, jvm_flags = None, **kwargs): + """test duplication rule that creates 32/64-bit test pair.""" + + if jvm_flags == None: + jvm_flags = [] + jvm_flags = jvm_flags + _TEST_JVM_FLAGS + + test_package = native.package_name().replace("/", ".") + if main_class == None: + test_class = test_package + "." + name + else: + test_class = None + + native.java_test( + name = name + "_32", + main_class = main_class, + test_class = test_class, + jvm_flags = jvm_flags + ["-DBROTLI_32_BIT_CPU=true"], + **kwargs + ) + + native.java_test( + name = name + "_64", + main_class = main_class, + test_class = test_class, + jvm_flags = jvm_flags + ["-DBROTLI_32_BIT_CPU=false"], + **kwargs + ) diff --git a/java/org/brotli/dec/pom.xml b/java/org/brotli/dec/pom.xml index 24b7aa1..bf609dc 100644 --- a/java/org/brotli/dec/pom.xml +++ b/java/org/brotli/dec/pom.xml @@ -42,6 +42,16 @@ </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>3.0.0-M3</version> + <configuration> + <systemPropertyVariables> + <BROTLI_ENABLE_ASSERTS>true</BROTLI_ENABLE_ASSERTS> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-source-plugin</artifactId> <version>2.4</version> <executions> diff --git a/java/org/brotli/wrapper/common/BUILD b/java/org/brotli/wrapper/common/BUILD index 48b02f3..4f7ed84 100644 --- a/java/org/brotli/wrapper/common/BUILD +++ b/java/org/brotli/wrapper/common/BUILD @@ -35,7 +35,6 @@ java_library( java_test( name = "SetZeroDictionaryTest", - test_class = "org.brotli.wrapper.common.SetZeroDictionaryTest", size = "small", data = [ ":brotli_jni_no_dictionary_data", # Bazel JNI workaround @@ -43,12 +42,12 @@ java_test( jvm_flags = [ "-DBROTLI_JNI_LIBRARY=$(location :brotli_jni_no_dictionary_data)", ], + test_class = "org.brotli.wrapper.common.SetZeroDictionaryTest", runtime_deps = [":test_lib"], ) java_test( name = "SetRfcDictionaryTest", - test_class = "org.brotli.wrapper.common.SetRfcDictionaryTest", size = "small", data = [ ":brotli_jni_no_dictionary_data", # Bazel JNI workaround @@ -56,5 +55,6 @@ java_test( jvm_flags = [ "-DBROTLI_JNI_LIBRARY=$(location :brotli_jni_no_dictionary_data)", ], + test_class = "org.brotli.wrapper.common.SetRfcDictionaryTest", runtime_deps = [":test_lib"], ) diff --git a/java/org/brotli/wrapper/dec/BUILD b/java/org/brotli/wrapper/dec/BUILD index fcf0dbf..754541a 100644 --- a/java/org/brotli/wrapper/dec/BUILD +++ b/java/org/brotli/wrapper/dec/BUILD @@ -39,7 +39,6 @@ filegroup( java_test( name = "BrotliDecoderChannelTest", - test_class = "org.brotli.wrapper.dec.BrotliDecoderChannelTest", size = "large", data = [ ":brotli_jni", # Bazel JNI workaround @@ -49,12 +48,12 @@ java_test( "-DBROTLI_JNI_LIBRARY=$(location :brotli_jni)", "-DTEST_BUNDLE=$(location :test_bundle)", ], + test_class = "org.brotli.wrapper.dec.BrotliDecoderChannelTest", runtime_deps = [":test_lib"], ) java_test( name = "BrotliInputStreamTest", - test_class = "org.brotli.wrapper.dec.BrotliInputStreamTest", size = "large", data = [ ":brotli_jni", # Bazel JNI workaround @@ -64,12 +63,12 @@ java_test( "-DBROTLI_JNI_LIBRARY=$(location :brotli_jni)", "-DTEST_BUNDLE=$(location :test_bundle)", ], + test_class = "org.brotli.wrapper.dec.BrotliInputStreamTest", runtime_deps = [":test_lib"], ) java_test( name = "DecoderTest", - test_class = "org.brotli.wrapper.dec.DecoderTest", size = "large", data = [ ":brotli_jni", # Bazel JNI workaround @@ -79,5 +78,6 @@ java_test( "-DBROTLI_JNI_LIBRARY=$(location :brotli_jni)", "-DTEST_BUNDLE=$(location :test_bundle)", ], + test_class = "org.brotli.wrapper.dec.DecoderTest", runtime_deps = [":test_lib"], ) diff --git a/java/org/brotli/wrapper/dec/BrotliInputStream.java b/java/org/brotli/wrapper/dec/BrotliInputStream.java index 26f7a82..6e2e6e5 100644 --- a/java/org/brotli/wrapper/dec/BrotliInputStream.java +++ b/java/org/brotli/wrapper/dec/BrotliInputStream.java @@ -34,8 +34,8 @@ public class BrotliInputStream extends InputStream { this(source, DEFAULT_BUFFER_SIZE); } - public void setEager(boolean eager) { - decoder.setEager(eager); + public void enableEagerOutput() { + decoder.enableEagerOutput(); } @Override diff --git a/java/org/brotli/wrapper/dec/Decoder.java b/java/org/brotli/wrapper/dec/Decoder.java index ae4d817..26183ab 100644 --- a/java/org/brotli/wrapper/dec/Decoder.java +++ b/java/org/brotli/wrapper/dec/Decoder.java @@ -50,8 +50,8 @@ public class Decoder { throw new IOException(message); } - public void setEager(boolean eager) { - this.eager = eager; + public void enableEagerOutput() { + this.eager = true; } /** diff --git a/java/org/brotli/wrapper/dec/DecoderJNI.java b/java/org/brotli/wrapper/dec/DecoderJNI.java index 320705c..2319b1e 100644 --- a/java/org/brotli/wrapper/dec/DecoderJNI.java +++ b/java/org/brotli/wrapper/dec/DecoderJNI.java @@ -30,6 +30,7 @@ public class DecoderJNI { private final long[] context = new long[3]; private final ByteBuffer inputBuffer; private Status lastStatus = Status.NEEDS_MORE_INPUT; + private boolean fresh = true; public Wrapper(int inputBufferSize) throws IOException { this.context[1] = inputBufferSize; @@ -52,6 +53,7 @@ public class DecoderJNI { if (lastStatus == Status.OK && length != 0) { throw new IllegalStateException("pushing input to decoder in OK state"); } + fresh = false; nativePush(context, length); parseStatus(); } @@ -90,6 +92,7 @@ public class DecoderJNI { if (lastStatus != Status.NEEDS_MORE_OUTPUT && !hasOutput()) { throw new IllegalStateException("pulling output from decoder in " + lastStatus + " state"); } + fresh = false; ByteBuffer result = nativePull(context); parseStatus(); return result; diff --git a/java/org/brotli/wrapper/dec/EagerStreamTest.java b/java/org/brotli/wrapper/dec/EagerStreamTest.java index 9166092..919f6e3 100755 --- a/java/org/brotli/wrapper/dec/EagerStreamTest.java +++ b/java/org/brotli/wrapper/dec/EagerStreamTest.java @@ -56,7 +56,7 @@ public class EagerStreamTest extends BrotliJniTestBase { } }; BrotliInputStream reader = new BrotliInputStream(source); - reader.setEager(true); + reader.enableEagerOutput(); int count = 0; while (true) { log.append("^").append(count); diff --git a/java/org/brotli/wrapper/enc/BUILD b/java/org/brotli/wrapper/enc/BUILD index 42ad23e..b3d10b9 100644 --- a/java/org/brotli/wrapper/enc/BUILD +++ b/java/org/brotli/wrapper/enc/BUILD @@ -40,7 +40,6 @@ filegroup( java_test( name = "BrotliEncoderChannelTest", - test_class = "org.brotli.wrapper.enc.BrotliEncoderChannelTest", size = "large", data = [ ":brotli_jni", # Bazel JNI workaround @@ -51,12 +50,12 @@ java_test( "-DTEST_BUNDLE=$(location :test_bundle)", ], shard_count = 15, + test_class = "org.brotli.wrapper.enc.BrotliEncoderChannelTest", runtime_deps = [":test_lib"], ) java_test( name = "BrotliOutputStreamTest", - test_class = "org.brotli.wrapper.enc.BrotliOutputStreamTest", size = "large", data = [ ":brotli_jni", # Bazel JNI workaround @@ -67,12 +66,12 @@ java_test( "-DTEST_BUNDLE=$(location :test_bundle)", ], shard_count = 15, + test_class = "org.brotli.wrapper.enc.BrotliOutputStreamTest", runtime_deps = [":test_lib"], ) java_test( name = "EncoderTest", - test_class = "org.brotli.wrapper.enc.EncoderTest", size = "large", data = [ ":brotli_jni", # Bazel JNI workaround @@ -83,5 +82,6 @@ java_test( "-DTEST_BUNDLE=$(location :test_bundle)", ], shard_count = 15, + test_class = "org.brotli.wrapper.enc.EncoderTest", runtime_deps = [":test_lib"], ) diff --git a/java/org/brotli/wrapper/enc/EncoderJNI.java b/java/org/brotli/wrapper/enc/EncoderJNI.java index 6627f5a..5013629 100644 --- a/java/org/brotli/wrapper/enc/EncoderJNI.java +++ b/java/org/brotli/wrapper/enc/EncoderJNI.java @@ -27,9 +27,13 @@ class EncoderJNI { static class Wrapper { protected final long[] context = new long[5]; private final ByteBuffer inputBuffer; + private boolean fresh = true; Wrapper(int inputBufferSize, int quality, int lgwin) throws IOException { + if (inputBufferSize <= 0) { + throw new IOException("buffer size must be positive"); + } this.context[1] = inputBufferSize; this.context[2] = quality; this.context[3] = lgwin; @@ -56,6 +60,7 @@ class EncoderJNI { throw new IllegalStateException("pushing input to encoder over previous input"); } context[1] = op.ordinal(); + fresh = false; nativePush(context, length); } @@ -86,6 +91,7 @@ class EncoderJNI { if (!isSuccess() || !hasMoreOutput()) { throw new IllegalStateException("pulling while data is not ready"); } + fresh = false; return nativePull(context); } |