Update

Documentation:
  - add note that brotli is a "stream" format, not an archive-like
  - regenerate .1 with Pandoc
Build:
  - drop legacy "BROTLI_BUILD_PORTABLE" option
  - drop "BROTLI_SANITIZED" definition
Code:
  - c: comb includes
  - c/enc: extract encoder state into separate header
  - c/enc: drop designated q10 codepath
  - c/enc: dealing better with flushing of empty stream
  - fix MSVC compilation
API:
  - py: use library version instead of one in version.h
  - c: add plugable API to report consumed input / produced output
  - c/java: support "lean" prepared dictionaries (without copy of source)

7 files changed, 36 insertions, 16 deletions

diff --git a/java/org/brotli/dec/BrotliInputStream.java b/java/org/brotli/dec/BrotliInputStream.java
index 5eca238..7bbe2f6 100644
--- a/java/org/brotli/dec/BrotliInputStream.java
+++ b/java/org/brotli/dec/BrotliInputStream.java
@@ -19,6 +19,14 @@ public class BrotliInputStream extends InputStream {
   public static final int DEFAULT_INTERNAL_BUFFER_SIZE = 256;
 
   /**
+   * Value expected by InputStream contract when stream is over.
+   *
+   * In Java it is -1.
+   * In C# it is 0 (should be patched during transpilation).
+   */
+  private static final int END_OF_STREAM_MARKER = -1;
+
+  /**
    * Internal buffer used for efficient byte-by-byte reading.
    */
   private byte[] buffer;
@@ -112,7 +120,8 @@ public class BrotliInputStream extends InputStream {
     if (bufferOffset >= remainingBufferBytes) {
       remainingBufferBytes = read(buffer, 0, buffer.length);
       bufferOffset = 0;
-      if (remainingBufferBytes == -1) {
+      if (remainingBufferBytes == END_OF_STREAM_MARKER) {
+        // Both Java and C# return the same value for EOF on single-byte read.
         return -1;
       }
     }
@@ -151,10 +160,9 @@ public class BrotliInputStream extends InputStream {
       state.outputLength = destLen;
       state.outputUsed = 0;
       Decode.decompress(state);
-      if (state.outputUsed == 0) {
-        return -1;
-      }
-      return state.outputUsed + copyLen;
+      copyLen += state.outputUsed;
+      copyLen = (copyLen > 0) ? copyLen : END_OF_STREAM_MARKER;
+      return copyLen;
     } catch (BrotliRuntimeException ex) {
       throw new IOException("Brotli stream decoding failed", ex);
     }
diff --git a/java/org/brotli/dec/Decode.java b/java/org/brotli/dec/Decode.java
index b139ef7..c386995 100644
--- a/java/org/brotli/dec/Decode.java
+++ b/java/org/brotli/dec/Decode.java
@@ -919,6 +919,7 @@ final class Decode {
   private static int writeRingBuffer(State s) {
     int toWrite = Math.min(s.outputLength - s.outputUsed,
         s.ringBufferBytesReady - s.ringBufferBytesWritten);
+    // TODO(eustas): DCHECK(toWrite >= 0)
     if (toWrite != 0) {
       System.arraycopy(s.ringBuffer, s.ringBufferBytesWritten, s.output,
           s.outputOffset + s.outputUsed, toWrite);
diff --git a/java/org/brotli/dec/build_defs.bzl b/java/org/brotli/dec/build_defs.bzl
index fd23a0d..d4f280b 100644
--- a/java/org/brotli/dec/build_defs.bzl
+++ b/java/org/brotli/dec/build_defs.bzl
@@ -5,13 +5,20 @@ _TEST_JVM_FLAGS = [
 ]
 
 def brotli_java_test(name, main_class = None, jvm_flags = None, **kwargs):
-    """test duplication rule that creates 32/64-bit test pair."""
+    """test duplication rule that creates 32/64-bit test pair.
+
+    Args:
+       name: target name prefix
+       main_class: override for test_class
+       jvm_flags: base Java VM options
+       **kwargs: pass-through
+    """
 
     if jvm_flags == None:
         jvm_flags = []
     jvm_flags = jvm_flags + _TEST_JVM_FLAGS
 
-    test_package = native.package_name().replace("/", ".").replace("javatests.", "")
+    test_package = native.package_name().replace("/", ".").replace("third_party.brotli.java.", "")
 
     if main_class == None:
         test_class = test_package + "." + name
@@ -23,6 +30,7 @@ def brotli_java_test(name, main_class = None, jvm_flags = None, **kwargs):
         main_class = main_class,
         test_class = test_class,
         jvm_flags = jvm_flags + ["-DBROTLI_32_BIT_CPU=true"],
+        visibility = ["//visibility:private"],
         **kwargs
     )
 
@@ -31,5 +39,6 @@ def brotli_java_test(name, main_class = None, jvm_flags = None, **kwargs):
         main_class = main_class,
         test_class = test_class,
         jvm_flags = jvm_flags + ["-DBROTLI_32_BIT_CPU=false"],
+        visibility = ["//visibility:private"],
         **kwargs
     )
diff --git a/java/org/brotli/wrapper/dec/decoder_jni.cc b/java/org/brotli/wrapper/dec/decoder_jni.cc
index 3328a1a..42e6bae 100644
--- a/java/org/brotli/wrapper/dec/decoder_jni.cc
+++ b/java/org/brotli/wrapper/dec/decoder_jni.cc
@@ -4,12 +4,12 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
-#include "decoder_jni.h"
-
-#include <new>
+#include "decoder_jni.h"  // NOLINT: build/include
 
 #include <brotli/decode.h>
 
+#include <new>
+
 namespace {
 /* A structure used to persist the decoder's state in between calls. */
 typedef struct DecoderHandle {
diff --git a/java/org/brotli/wrapper/dec/decoder_jni_onload.cc b/java/org/brotli/wrapper/dec/decoder_jni_onload.cc
index 2f93de0..b69f954 100644
--- a/java/org/brotli/wrapper/dec/decoder_jni_onload.cc
+++ b/java/org/brotli/wrapper/dec/decoder_jni_onload.cc
@@ -6,7 +6,7 @@
 
 #include <jni.h>
 
-#include "decoder_jni.h"
+#include "decoder_jni.h"   // NOLINT: build/include
 
 #ifdef __cplusplus
 extern "C" {
@@ -36,7 +36,7 @@ JNIEXPORT jint JNI_OnLoad(JavaVM* vm, void* reserved) {
   }
 
   jclass clazz =
-      env->FindClass("com/google/compression/brotli/wrapper/dec/DecoderJNI");
+      env->FindClass("org/brotli/wrapper/dec/DecoderJNI");
   if (clazz == nullptr) {
     return -1;
   }
diff --git a/java/org/brotli/wrapper/enc/EncoderJNI.java b/java/org/brotli/wrapper/enc/EncoderJNI.java
index 3e77207..b8e32d2 100644
--- a/java/org/brotli/wrapper/enc/EncoderJNI.java
+++ b/java/org/brotli/wrapper/enc/EncoderJNI.java
@@ -30,8 +30,10 @@ class EncoderJNI {
 
   private static class PreparedDictionaryImpl implements PreparedDictionary {
     private ByteBuffer data;
+    /** Reference to (non-copied) LZ data. */
+    private ByteBuffer rawData;
 
-    private PreparedDictionaryImpl(ByteBuffer data) {
+    private PreparedDictionaryImpl(ByteBuffer data, ByteBuffer rawData) {
       this.data = data;
     }
 
@@ -45,6 +47,7 @@ class EncoderJNI {
       try {
         ByteBuffer data = this.data;
         this.data = null;
+        this.rawData = null;
         nativeDestroyDictionary(data);
       } finally {
         super.finalize();
@@ -66,7 +69,7 @@ class EncoderJNI {
     if (dictionaryData == null) {
       throw new IllegalStateException("OOM");
     }
-    return new PreparedDictionaryImpl(dictionaryData);
+    return new PreparedDictionaryImpl(dictionaryData, dictionary);
   }
 
   static class Wrapper {
diff --git a/java/org/brotli/wrapper/enc/encoder_jni.cc b/java/org/brotli/wrapper/enc/encoder_jni.cc
index adcc7bf..796908b 100644
--- a/java/org/brotli/wrapper/enc/encoder_jni.cc
+++ b/java/org/brotli/wrapper/enc/encoder_jni.cc
@@ -4,12 +4,11 @@
    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
 */
 
+#include <brotli/encode.h>
 #include <jni.h>
 
 #include <new>
 
-#include <brotli/encode.h>
-
 namespace {
 /* A structure used to persist the encoder's state in between calls. */
 typedef struct EncoderHandle {