New feature: "Large Window Brotli" (#640)

* New feature: "Large Window Brotli"

By setting special encoder/decoder flag it is now possible to extend
LZ-window up to 30 bits; though produced stream will not be RFC7932
compliant.

Added new dictionary generator - "DSH". It combines speed of "Sieve"
and quality of "DM". Plus utilities to prepare train corpora
(remove unique strings).

Improved compression ratio: now two sub-blocks could be stitched:
the last copy command could be extended to span the next sub-block.

Fixed compression ineffectiveness caused by floating numbers rounding and
wrong cost heuristic.

Other C changes:
 - combined / moved `context.h` to `common`
 - moved transforms to `common`
 - unified some aspects of code formatting
 - added an abstraction for encoder (static) dictionary
 - moved default allocator/deallocator functions to `common`

brotli CLI:
 - window size is auto-adjusted if not specified explicitly

Java:
 - added "eager" decoding both to JNI wrapper and pure decoder
 - huge speed-up of `DictionaryData` initialization

* Add dictionaryless compressed dictionary

* Fix `sources.lst`

* Fix `sources.lst` and add a note that `libtool` is also required.

* Update setup.py

* Fix `EagerStreamTest`

* Fix BUILD file

* Add missing `libdivsufsort` dependency

* Fix "unused parameter" warning.

2 files changed, 20 insertions, 6 deletions

diff --git a/c/include/brotli/decode.h b/c/include/brotli/decode.h
index 1acf605..61a4326 100644
--- a/c/include/brotli/decode.h
+++ b/c/include/brotli/decode.h
@@ -34,11 +34,11 @@ typedef struct BrotliDecoderStateStruct BrotliDecoderState;
 typedef enum {
   /** Decoding error, e.g. corrupted input or memory allocation problem. */
   BROTLI_DECODER_RESULT_ERROR = 0,
-  /** Decoding successfully completed */
+  /** Decoding successfully completed. */
   BROTLI_DECODER_RESULT_SUCCESS = 1,
-  /** Partially done; should be called again with more input */
+  /** Partially done; should be called again with more input. */
   BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT = 2,
-  /** Partially done; should be called again with more output */
+  /** Partially done; should be called again with more output. */
   BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT = 3
 } BrotliDecoderResult;
 
@@ -83,8 +83,9 @@ typedef enum {
   BROTLI_ERROR_CODE(_ERROR_FORMAT_, WINDOW_BITS, -13) SEPARATOR            \
   BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR              \
   BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR              \
+  BROTLI_ERROR_CODE(_ERROR_FORMAT_, DISTANCE, -16) SEPARATOR               \
                                                                            \
-  /* -16..-17 codes are reserved */                                        \
+  /* -17 code is reserved */                                               \
                                                                            \
   BROTLI_ERROR_CODE(_ERROR_, COMPOUND_DICTIONARY, -18) SEPARATOR           \
   BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR            \
@@ -135,7 +136,11 @@ typedef enum BrotliDecoderParameter {
    * Ring buffer is allocated according to window size, despite the real size of
    * the content.
    */
-  BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0
+  BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0,
+  /**
+   * Flag that determines if "Large Window Brotli" is used.
+   */
+  BROTLI_DECODER_PARAM_LARGE_WINDOW = 1
 } BrotliDecoderParameter;
 
 /**
diff --git a/c/include/brotli/encode.h b/c/include/brotli/encode.h
index 1fa85cc..0b5c8c7 100644
--- a/c/include/brotli/encode.h
+++ b/c/include/brotli/encode.h
@@ -27,6 +27,11 @@ extern "C" {
  * @note equal to @c BROTLI_MAX_DISTANCE_BITS constant.
  */
 #define BROTLI_MAX_WINDOW_BITS 24
+/**
+ * Maximal value for ::BROTLI_PARAM_LGWIN parameter
+ * in "Large Window Brotli" (32-bit).
+ */
+#define BROTLI_LARGE_MAX_WINDOW_BITS 30
 /** Minimal value for ::BROTLI_PARAM_LGBLOCK parameter. */
 #define BROTLI_MIN_INPUT_BLOCK_BITS 16
 /** Maximal value for ::BROTLI_PARAM_LGBLOCK parameter. */
@@ -176,7 +181,11 @@ typedef enum BrotliEncoderParameter {
    *
    * The default value is 0, which means that the total input size is unknown.
    */
-  BROTLI_PARAM_SIZE_HINT = 5
+  BROTLI_PARAM_SIZE_HINT = 5,
+  /**
+   * Flag that determines if "Large Window Brotli" is used.
+   */
+  BROTLI_PARAM_LARGE_WINDOW = 6
 } BrotliEncoderParameter;
 
 /**