From 66e798d46ae95af5fcd569b5b2ec71001688ced4 Mon Sep 17 00:00:00 2001 From: Eugene Kliuchnikov Date: Mon, 10 Apr 2017 15:39:00 +0200 Subject: Update API to v1.0.0 (#537) Make Java decoder fully transpilable to C#. --- common/version.h | 2 +- enc/encode.c | 17 -------------- include/brotli/encode.h | 19 --------------- java/org/brotli/dec/BitReader.java | 3 ++- java/org/brotli/dec/BrotliInputStream.java | 2 ++ java/org/brotli/dec/Decode.java | 3 ++- java/org/brotli/dec/DecodeTest.java | 13 ++++++----- java/org/brotli/dec/DictionaryTest.java | 22 +++++++++++------- java/org/brotli/dec/SynthTest.java | 33 ++------------------------ java/org/brotli/dec/TransformTest.java | 37 ++++++++++++++++++------------ 10 files changed, 52 insertions(+), 99 deletions(-) diff --git a/common/version.h b/common/version.h index a201860..10fe01f 100755 --- a/common/version.h +++ b/common/version.h @@ -14,6 +14,6 @@ BrotliEncoderVersion methods. */ /* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */ -#define BROTLI_VERSION 0x0006000 +#define BROTLI_VERSION 0x1000000 #endif /* BROTLI_COMMON_VERSION_H_ */ diff --git a/enc/encode.c b/enc/encode.c index 372af65..d111573 100644 --- a/enc/encode.c +++ b/enc/encode.c @@ -1792,23 +1792,6 @@ uint32_t BrotliEncoderVersion(void) { return BROTLI_VERSION; } - -/* DEPRECATED >>> */ -size_t BrotliEncoderInputBlockSize(BrotliEncoderState* s) { - return InputBlockSize(s); -} -void BrotliEncoderCopyInputToRingBuffer(BrotliEncoderState* s, - const size_t input_size, - const uint8_t* input_buffer) { - CopyInputToRingBuffer(s, input_size, input_buffer); -} -BROTLI_BOOL BrotliEncoderWriteData( - BrotliEncoderState* s, const BROTLI_BOOL is_last, - const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output) { - return EncodeData(s, is_last, force_flush, out_size, output); -} -/* <<< DEPRECATED */ - #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif diff --git a/include/brotli/encode.h b/include/brotli/encode.h index 72e8324..98a5ff8 100755 --- a/include/brotli/encode.h +++ b/include/brotli/encode.h @@ -36,11 +36,6 @@ extern "C" { /** Maximal value for ::BROTLI_PARAM_QUALITY parameter. */ #define BROTLI_MAX_QUALITY 11 -BROTLI_DEPRECATED static const int kBrotliMinWindowBits = - BROTLI_MIN_WINDOW_BITS; -BROTLI_DEPRECATED static const int kBrotliMaxWindowBits = - BROTLI_MAX_WINDOW_BITS; - /** Options for ::BROTLI_PARAM_MODE parameter. */ typedef enum BrotliEncoderMode { /** @@ -228,20 +223,6 @@ BROTLI_ENC_API BrotliEncoderState* BrotliEncoderCreateInstance( */ BROTLI_ENC_API void BrotliEncoderDestroyInstance(BrotliEncoderState* state); -/* Calculates maximum input size that can be processed at once. */ -BROTLI_DEPRECATED BROTLI_ENC_API size_t BrotliEncoderInputBlockSize( - BrotliEncoderState* state); - -/* Copies the given input data to the internal ring buffer. */ -BROTLI_DEPRECATED BROTLI_ENC_API void BrotliEncoderCopyInputToRingBuffer( - BrotliEncoderState* state, const size_t input_size, - const uint8_t* input_buffer); - -/* Processes the accumulated input. */ -BROTLI_DEPRECATED BROTLI_ENC_API BROTLI_BOOL BrotliEncoderWriteData( - BrotliEncoderState* state, const BROTLI_BOOL is_last, - const BROTLI_BOOL force_flush, size_t* out_size, uint8_t** output); - /** * Prepends imaginary LZ77 dictionary. * diff --git a/java/org/brotli/dec/BitReader.java b/java/org/brotli/dec/BitReader.java index 28f6cf2..a3bb53d 100755 --- a/java/org/brotli/dec/BitReader.java +++ b/java/org/brotli/dec/BitReader.java @@ -79,7 +79,8 @@ final class BitReader { try { while (bytesRead < BYTE_READ_SIZE) { int len = br.input.read(br.byteBuffer, bytesRead, BYTE_READ_SIZE - bytesRead); - if (len == -1) { + // EOF is -1 in Java, but 0 in C#. + if (len <= 0) { br.endOfStreamReached = true; br.tailBytes = bytesRead; bytesRead += 3; diff --git a/java/org/brotli/dec/BrotliInputStream.java b/java/org/brotli/dec/BrotliInputStream.java index a6ce5e9..a886b5d 100755 --- a/java/org/brotli/dec/BrotliInputStream.java +++ b/java/org/brotli/dec/BrotliInputStream.java @@ -166,5 +166,7 @@ public class BrotliInputStream extends InputStream { } catch (BrotliRuntimeException ex) { throw new IOException("Brotli stream decoding failed", ex); } + + // <{[INJECTED CODE]}> } } diff --git a/java/org/brotli/dec/Decode.java b/java/org/brotli/dec/Decode.java index f1e9da6..7c515dd 100755 --- a/java/org/brotli/dec/Decode.java +++ b/java/org/brotli/dec/Decode.java @@ -126,7 +126,8 @@ final class Decode { return sym; } offset += sym; - offset += (val & ((1L << bits) - 1)) >>> HUFFMAN_TABLE_BITS; + int mask = (1 << bits) - 1; + offset += (val & mask) >>> HUFFMAN_TABLE_BITS; br.bitOffset += ((table[offset] >> 16) + HUFFMAN_TABLE_BITS); return table[offset] & 0xFFFF; } diff --git a/java/org/brotli/dec/DecodeTest.java b/java/org/brotli/dec/DecodeTest.java index 18546c1..690ab9f 100755 --- a/java/org/brotli/dec/DecodeTest.java +++ b/java/org/brotli/dec/DecodeTest.java @@ -11,7 +11,6 @@ import static org.junit.Assert.assertArrayEquals; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -26,18 +25,20 @@ public class DecodeTest { byte[] buffer = new byte[65536]; ByteArrayInputStream input = new ByteArrayInputStream(data); ByteArrayOutputStream output = new ByteArrayOutputStream(); - InputStream brotliInput = new BrotliInputStream(input); + BrotliInputStream brotliInput = new BrotliInputStream(input); if (byByte) { + byte[] oneByte = new byte[1]; while (true) { int next = brotliInput.read(); if (next == -1) { break; } - output.write(next); + oneByte[0] = (byte) next; + output.write(oneByte, 0, 1); } } else { while (true) { - int len = brotliInput.read(buffer); + int len = brotliInput.read(buffer, 0, buffer.length); if (len <= 0) { break; } @@ -52,10 +53,10 @@ public class DecodeTest { byte[] buffer = new byte[65536]; ByteArrayInputStream input = new ByteArrayInputStream(data); ByteArrayOutputStream output = new ByteArrayOutputStream(); - InputStream brotliInput = new BrotliInputStream( + BrotliInputStream brotliInput = new BrotliInputStream( input, BrotliInputStream.DEFAULT_INTERNAL_BUFFER_SIZE, dictionary); while (true) { - int len = brotliInput.read(buffer); + int len = brotliInput.read(buffer, 0, buffer.length); if (len <= 0) { break; } diff --git a/java/org/brotli/dec/DictionaryTest.java b/java/org/brotli/dec/DictionaryTest.java index eb0b46e..cf5c1cd 100755 --- a/java/org/brotli/dec/DictionaryTest.java +++ b/java/org/brotli/dec/DictionaryTest.java @@ -8,8 +8,6 @@ package org.brotli.dec; import static org.junit.Assert.assertEquals; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -20,12 +18,20 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class DictionaryTest { + private static long crc64(byte[] data) { + long crc = -1; + for (int i = 0; i < data.length; ++i) { + long c = (crc ^ (long) (data[i] & 0xFF)) & 0xFF; + for (int k = 0; k < 8; k++) { + c = (c >>> 1) ^ (-(c & 1L) & -3932672073523589310L); + } + crc = c ^ (crc >>> 8); + } + return ~crc; + } + @Test - public void testGetData() throws NoSuchAlgorithmException { - MessageDigest md = MessageDigest.getInstance("SHA-256"); - md.update(Dictionary.getData()); - byte[] digest = md.digest(); - String sha256 = String.format("%064x", new java.math.BigInteger(1, digest)); - assertEquals("20e42eb1b511c21806d4d227d07e5dd06877d8ce7b3a817f378f313653f35c70", sha256); + public void testGetData() { + assertEquals(37084801881332636L, crc64(Dictionary.getData())); } } diff --git a/java/org/brotli/dec/SynthTest.java b/java/org/brotli/dec/SynthTest.java index e4b065d..939e323 100755 --- a/java/org/brotli/dec/SynthTest.java +++ b/java/org/brotli/dec/SynthTest.java @@ -12,8 +12,6 @@ import static org.junit.Assert.fail; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -28,9 +26,9 @@ public class SynthTest { byte[] buffer = new byte[65536]; ByteArrayInputStream input = new ByteArrayInputStream(data); ByteArrayOutputStream output = new ByteArrayOutputStream(); - InputStream brotliInput = new BrotliInputStream(input); + BrotliInputStream brotliInput = new BrotliInputStream(input); while (true) { - int len = brotliInput.read(buffer); + int len = brotliInput.read(buffer, 0, buffer.length); if (len <= 0) { break; } @@ -2024,33 +2022,6 @@ public class SynthTest { false, ""); } - @Ignore("Java implementation forbids extra bytes after the stream end.") - @Test - public void testSimplePrefixPlusExtraData() { - byte[] compressed = { - (byte) 0x1b, (byte) 0x03, (byte) 0x00, (byte) 0x00, (byte) 0xa0, (byte) 0xc3, (byte) 0xc4, - (byte) 0xc6, (byte) 0xc8, (byte) 0x02, (byte) 0x00, (byte) 0x70, (byte) 0xb0, (byte) 0x65, - (byte) 0x12, (byte) 0x03, (byte) 0x24, (byte) 0x00, (byte) 0x00, (byte) 0xee, (byte) 0xb4, - (byte) 0x51, (byte) 0xa0, (byte) 0x1d, (byte) 0x55, (byte) 0xaa - }; - checkSynth( -/* - main_header - metablock_header_begin: 1, 0, 4, 0 - metablock_header_trivial_context - huffman_simple: 1,4,256, 97,98,99,100 // ascii codes for a, b, c, d - huffman_fixed: 704 - huffman_fixed: 64 - command_inscopy_easy: 4, 0 - command_literal_bits: 0, 10, 110, 111 // a, b, c, d - byte_boundary - bits: "01010101", "10101010" - */ - compressed, - true, "" - + "abcd"); - } - @Test public void testTooManySymbolsRepeated() { byte[] compressed = { diff --git a/java/org/brotli/dec/TransformTest.java b/java/org/brotli/dec/TransformTest.java index f83d4b4..2449815 100755 --- a/java/org/brotli/dec/TransformTest.java +++ b/java/org/brotli/dec/TransformTest.java @@ -9,9 +9,6 @@ package org.brotli.dec; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -22,29 +19,44 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) public class TransformTest { + private static long crc64(byte[] data) { + long crc = -1; + for (int i = 0; i < data.length; ++i) { + long c = (crc ^ (long) (data[i] & 0xFF)) & 0xFF; + for (int k = 0; k < 8; k++) { + c = (c >>> 1) ^ (-(c & 1L) & -3932672073523589310L); + } + crc = c ^ (crc >>> 8); + } + return ~crc; + } + @Test public void testTrimAll() { byte[] output = new byte[2]; - byte[] input = "word".getBytes(StandardCharsets.UTF_8); + byte[] input = {119, 111, 114, 100}; // "word" Transform transform = new Transform("[", WordTransformType.OMIT_FIRST_5, "]"); Transform.transformDictionaryWord(output, 0, input, 0, input.length, transform); - assertArrayEquals(output, "[]".getBytes(StandardCharsets.UTF_8)); + byte[] expectedOutput = {91, 93}; // "[]" + assertArrayEquals(expectedOutput, output); } @Test public void testCapitalize() { byte[] output = new byte[8]; - byte[] input = "qæप".getBytes(StandardCharsets.UTF_8); + byte[] input = {113, -61, -90, -32, -92, -86}; // "qæप" Transform transform = new Transform("[", WordTransformType.UPPERCASE_ALL, "]"); Transform.transformDictionaryWord(output, 0, input, 0, input.length, transform); - assertArrayEquals(output, "[QÆय]".getBytes(StandardCharsets.UTF_8)); + byte[] expectedOutput = {91, 81, -61, -122, -32, -92, -81, 93}; // "[QÆय]" + assertArrayEquals(expectedOutput, output); } @Test - public void testAllTransforms() throws NoSuchAlgorithmException { + public void testAllTransforms() { /* This string allows to apply all transforms: head and tail cutting, capitalization and turning to upper case; all results will be mutually different. */ - byte[] testWord = Transform.readUniBytes("o123456789abcdef"); + // "o123456789abcdef" + byte[] testWord = {111, 49, 50, 51, 52, 53, 54, 55, 56, 57, 97, 98, 99, 100, 101, 102}; byte[] output = new byte[2259]; int offset = 0; for (int i = 0; i < Transform.TRANSFORMS.length; ++i) { @@ -53,11 +65,6 @@ public class TransformTest { output[offset++] = -1; } assertEquals(output.length, offset); - - MessageDigest md = MessageDigest.getInstance("SHA-256"); - md.update(output); - byte[] digest = md.digest(); - String sha256 = String.format("%064x", new java.math.BigInteger(1, digest)); - assertEquals("60f1c7e45d788e24938c5a3919aaf41a7d8ad474d0ced6b9e4c0079f4d1da8c4", sha256); + assertEquals(8929191060211225186L, crc64(output)); } } -- cgit v1.1