Update (#560)

Update:
 * add decoder API to avoid ringbuffer reallocation
 * fix MSVC warnings
 * remove dead code

9 files changed, 103 insertions, 27 deletions

diff --git a/c/dec/decode.c b/c/dec/decode.c
index d3951f8..53a9b55 100644
--- a/c/dec/decode.c
+++ b/c/dec/decode.c
@@ -39,6 +39,11 @@ extern "C" {
 #define HUFFMAN_TABLE_BITS 8U
 #define HUFFMAN_TABLE_MASK 0xff
 
+/* We need the slack region for the following reasons:
+    - doing up to two 16-byte copies for fast backward copying
+    - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
+static const uint32_t kRingBufferWriteAheadSlack = 42;
+
 static const uint8_t kCodeLengthCodeOrder[BROTLI_CODE_LENGTH_CODES] = {
   1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15,
 };
@@ -52,6 +57,17 @@ static const uint8_t kCodeLengthPrefixValue[16] = {
   0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5,
 };
 
+BROTLI_BOOL BrotliDecoderSetParameter(
+    BrotliDecoderState* state, BrotliDecoderParameter p, uint32_t value) {
+  switch (p) {
+    case BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION:
+      state->canny_ringbuffer_allocation = !!value ? 0 : 1;
+      return BROTLI_TRUE;
+
+    default: return BROTLI_FALSE;
+  }
+}
+
 BrotliDecoderState* BrotliDecoderCreateInstance(
     brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) {
   BrotliDecoderState* state = 0;
@@ -1247,17 +1263,13 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) {
 */
 static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer(
     BrotliDecoderState* s) {
-  /* We need the slack region for the following reasons:
-      - doing up to two 16-byte copies for fast backward copying
-      - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */
-  static const int kRingBufferWriteAheadSlack = 42;
   uint8_t* old_ringbuffer = s->ringbuffer;
   if (s->ringbuffer_size == s->new_ringbuffer_size) {
     return BROTLI_TRUE;
   }
 
-  s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size +
-      kRingBufferWriteAheadSlack));
+  s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size) +
+      kRingBufferWriteAheadSlack);
   if (s->ringbuffer == 0) {
     /* Restore previous value. */
     s->ringbuffer = old_ringbuffer;
@@ -1369,8 +1381,13 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize(
   output_size += s->meta_block_remaining_len;
   min_size = min_size < output_size ? output_size : min_size;
 
-  while ((new_ringbuffer_size >> 1) >= min_size) {
-    new_ringbuffer_size >>= 1;
+  if (!!s->canny_ringbuffer_allocation) {
+    /* Reduce ring buffer size to save memory when server is unscrupulous.
+       In worst case memory usage might be 1.5x bigger for a short period of
+       ring buffer reallocation.*/
+    while ((new_ringbuffer_size >> 1) >= min_size) {
+      new_ringbuffer_size >>= 1;
+    }
   }
 
   s->new_ringbuffer_size = new_ringbuffer_size;
diff --git a/c/dec/state.c b/c/dec/state.c
index b7431f2..0db73fb 100644
--- a/c/dec/state.c
+++ b/c/dec/state.c
@@ -87,7 +87,11 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
   s->custom_dict_size = 0;
 
   s->is_last_metablock = 0;
+  s->is_uncompressed = 0;
+  s->is_metadata = 0;
   s->should_wrap_ringbuffer = 0;
+  s->canny_ringbuffer_allocation = 1;
+
   s->window_bits = 0;
   s->max_distance = 0;
   s->dist_rb[0] = 16;
diff --git a/c/dec/state.h b/c/dec/state.h
index 5946124..00c373b 100644
--- a/c/dec/state.h
+++ b/c/dec/state.h
@@ -172,7 +172,7 @@ struct BrotliDecoderStateStruct {
   uint32_t space;
 
   HuffmanCode table[32];
-  /* List of of symbol chains. */
+  /* List of heads of symbol chains. */
   uint16_t* symbol_lists;
   /* Storage from symbol_lists. */
   uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 +
@@ -215,6 +215,7 @@ struct BrotliDecoderStateStruct {
   unsigned int is_uncompressed : 1;
   unsigned int is_metadata : 1;
   unsigned int should_wrap_ringbuffer : 1;
+  unsigned int canny_ringbuffer_allocation : 1;
   unsigned int size_nibbles : 8;
   uint32_t window_bits;
 
diff --git a/c/enc/brotli_bit_stream.c b/c/enc/brotli_bit_stream.c
index 4874695..2907510 100644
--- a/c/enc/brotli_bit_stream.c
+++ b/c/enc/brotli_bit_stream.c
@@ -1325,17 +1325,6 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block,
   }
 }
 
-void BrotliStoreSyncMetaBlock(size_t* BROTLI_RESTRICT storage_ix,
-                              uint8_t* BROTLI_RESTRICT storage) {
-  /* Empty metadata meta-block bit pattern:
-       1 bit:  is_last (0)
-       2 bits: num nibbles (3)
-       1 bit:  reserved (0)
-       2 bits: metadata length bytes (0) */
-  BrotliWriteBits(6, 6, storage_ix, storage);
-  JumpToByteBoundary(storage_ix, storage);
-}
-
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif
diff --git a/c/enc/brotli_bit_stream.h b/c/enc/brotli_bit_stream.h
index a98f98f..2c8bfed 100644
--- a/c/enc/brotli_bit_stream.h
+++ b/c/enc/brotli_bit_stream.h
@@ -96,10 +96,6 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock(
     BROTLI_BOOL is_final_block, const uint8_t* input, size_t position,
     size_t mask, size_t len, size_t* storage_ix, uint8_t* storage);
 
-/* Stores an empty metadata meta-block and syncs to a byte boundary. */
-BROTLI_INTERNAL void BrotliStoreSyncMetaBlock(size_t* storage_ix,
-                                              uint8_t* storage);
-
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif
diff --git a/c/enc/cluster_inc.h b/c/enc/cluster_inc.h
index cf279bd..22ecb3c 100644
--- a/c/enc/cluster_inc.h
+++ b/c/enc/cluster_inc.h
@@ -17,6 +17,8 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)(
     size_t* num_pairs) CODE({
   BROTLI_BOOL is_good_pair = BROTLI_FALSE;
   HistogramPair p;
+  p.idx1 = p.idx2 = 0;
+  p.cost_diff = p.cost_combo = 0;
   if (idx1 == idx2) {
     return;
   }
diff --git a/c/include/brotli/decode.h b/c/include/brotli/decode.h
index 93cbe38..b18e9e4 100755
--- a/c/include/brotli/decode.h
+++ b/c/include/brotli/decode.h
@@ -126,6 +126,29 @@ typedef enum {
  */
 #define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE
 
+/** Options to be used with ::BrotliDecoderSetParameter. */
+typedef enum BrotliDecoderParameter {
+  /**
+   * Disable "canny" ring buffer allocation strategy.
+   *
+   * Ring buffer is allocated according to window size, despite the real size of
+   * the content.
+   */
+  BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0
+} BrotliDecoderParameter;
+
+/**
+ * Sets the specified parameter to the given decoder instance.
+ *
+ * @param state decoder instance
+ * @param param parameter to set
+ * @param value new parameter value
+ * @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid
+ * @returns ::BROTLI_TRUE if value is accepted
+ */
+BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter(
+    BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value);
+
 /**
  * Creates an instance of ::BrotliDecoderState and initializes it.
  *
diff --git a/c/tools/brotli.c b/c/tools/brotli.c
index ff0cabf..3cf6297 100755
--- a/c/tools/brotli.c
+++ b/c/tools/brotli.c
@@ -697,8 +697,8 @@ static BROTLI_BOOL CloseFiles(Context* context, BROTLI_BOOL success) {
 static const size_t kFileBufferSize = 1 << 16;
 
 static BROTLI_BOOL DecompressFile(Context* context, BrotliDecoderState* s) {
-  size_t available_in;
-  const uint8_t* next_in;
+  size_t available_in = 0;
+  const uint8_t* next_in = NULL;
   size_t available_out = kFileBufferSize;
   uint8_t* next_out = context->output;
   BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
diff --git a/docs/decode.h.3 b/docs/decode.h.3
index f4b95fe..447c67b 100755
--- a/docs/decode.h.3
+++ b/docs/decode.h.3
@@ -1,4 +1,4 @@
-.TH "decode.h" 3 "Sun May 21 2017" "Brotli" \" -*- nroff -*-
+.TH "decode.h" 3 "Tue Jun 13 2017" "Brotli" \" -*- nroff -*-
 .ad l
 .nh
 .SH NAME
@@ -23,6 +23,10 @@ decode.h \- API for Brotli decompression\&.
 
 .in +1c
 .ti -1c
+.RI "typedef enum \fBBrotliDecoderParameter\fP \fBBrotliDecoderParameter\fP"
+.br
+.RI "\fIOptions to be used with \fBBrotliDecoderSetParameter\fP\&. \fP"
+.ti -1c
 .RI "typedef struct BrotliDecoderStateStruct \fBBrotliDecoderState\fP"
 .br
 .RI "\fIOpaque structure that holds decoder state\&. \fP"
@@ -72,6 +76,10 @@ decode.h \- API for Brotli decompression\&.
 .br
 .RI "\fIPrepends LZ77 dictionary\&. \fP"
 .ti -1c
+.RI "\fBBROTLI_BOOL\fP \fBBrotliDecoderSetParameter\fP (\fBBrotliDecoderState\fP *state, \fBBrotliDecoderParameter\fP param, uint32_t value)"
+.br
+.RI "\fISets the specified parameter to the given decoder instance\&. \fP"
+.ti -1c
 .RI "const uint8_t * \fBBrotliDecoderTakeOutput\fP (\fBBrotliDecoderState\fP *state, size_t *size)"
 .br
 .RI "\fIAcquires pointer to internal output buffer\&. \fP"
@@ -115,6 +123,10 @@ BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_)
 The value of the last error code, negative integer\&. All other error code values are in the range from \fBBROTLI_LAST_ERROR_CODE\fP to \fC-1\fP\&. There are also 4 other possible non-error codes \fC0\fP \&.\&. \fC3\fP in \fBBrotliDecoderErrorCode\fP enumeration\&. 
 .SH "Typedef Documentation"
 .PP 
+.SS "typedef enum \fBBrotliDecoderParameter\fP  \fBBrotliDecoderParameter\fP"
+
+.PP
+Options to be used with \fBBrotliDecoderSetParameter\fP\&. 
 .SS "typedef struct BrotliDecoderStateStruct \fBBrotliDecoderState\fP"
 
 .PP
@@ -125,6 +137,16 @@ Opaque structure that holds decoder state\&. Allocated and initialized with \fBB
 
 .PP
 Error code for detailed logging / production debugging\&. See \fBBrotliDecoderGetErrorCode\fP and \fBBROTLI_LAST_ERROR_CODE\fP\&. 
+.SS "enum \fBBrotliDecoderParameter\fP"
+
+.PP
+Options to be used with \fBBrotliDecoderSetParameter\fP\&. 
+.PP
+\fBEnumerator\fP
+.in +1c
+.TP
+\fB\fIBROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION \fP\fP
+Disable 'canny' ring buffer allocation strategy\&. Ring buffer is allocated according to window size, despite the real size of the content\&. 
 .SS "enum \fBBrotliDecoderResult\fP"
 
 .PP
@@ -360,6 +382,28 @@ Clean up and free state with \fBBrotliDecoderDestroyInstance\fP
 .RE
 .PP
 
+.SS "\fBBROTLI_BOOL\fP BrotliDecoderSetParameter (\fBBrotliDecoderState\fP * state, \fBBrotliDecoderParameter\fP param, uint32_t value)"
+
+.PP
+Sets the specified parameter to the given decoder instance\&. 
+.PP
+\fBParameters:\fP
+.RS 4
+\fIstate\fP decoder instance 
+.br
+\fIparam\fP parameter to set 
+.br
+\fIvalue\fP new parameter value 
+.RE
+.PP
+\fBReturns:\fP
+.RS 4
+\fBBROTLI_FALSE\fP if parameter is unrecognized, or value is invalid 
+.PP
+\fBBROTLI_TRUE\fP if value is accepted 
+.RE
+.PP
+
 .SS "const uint8_t* BrotliDecoderTakeOutput (\fBBrotliDecoderState\fP * state, size_t * size)"
 
 .PP