From 05d5f3d77a3445b464c230b382855979e1b89fef Mon Sep 17 00:00:00 2001 From: Eugene Kliuchnikov Date: Tue, 13 Jun 2017 12:52:56 +0200 Subject: Update (#560) Update: * add decoder API to avoid ringbuffer reallocation * fix MSVC warnings * remove dead code --- c/dec/decode.c | 33 +++++++++++++++++++++++++-------- c/dec/state.c | 4 ++++ c/dec/state.h | 3 ++- c/enc/brotli_bit_stream.c | 11 ----------- c/enc/brotli_bit_stream.h | 4 ---- c/enc/cluster_inc.h | 2 ++ c/include/brotli/decode.h | 23 +++++++++++++++++++++++ c/tools/brotli.c | 4 ++-- docs/decode.h.3 | 46 +++++++++++++++++++++++++++++++++++++++++++++- 9 files changed, 103 insertions(+), 27 deletions(-) diff --git a/c/dec/decode.c b/c/dec/decode.c index d3951f8..53a9b55 100644 --- a/c/dec/decode.c +++ b/c/dec/decode.c @@ -39,6 +39,11 @@ extern "C" { #define HUFFMAN_TABLE_BITS 8U #define HUFFMAN_TABLE_MASK 0xff +/* We need the slack region for the following reasons: + - doing up to two 16-byte copies for fast backward copying + - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */ +static const uint32_t kRingBufferWriteAheadSlack = 42; + static const uint8_t kCodeLengthCodeOrder[BROTLI_CODE_LENGTH_CODES] = { 1, 2, 3, 4, 0, 5, 17, 6, 16, 7, 8, 9, 10, 11, 12, 13, 14, 15, }; @@ -52,6 +57,17 @@ static const uint8_t kCodeLengthPrefixValue[16] = { 0, 4, 3, 2, 0, 4, 3, 1, 0, 4, 3, 2, 0, 4, 3, 5, }; +BROTLI_BOOL BrotliDecoderSetParameter( + BrotliDecoderState* state, BrotliDecoderParameter p, uint32_t value) { + switch (p) { + case BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION: + state->canny_ringbuffer_allocation = !!value ? 0 : 1; + return BROTLI_TRUE; + + default: return BROTLI_FALSE; + } +} + BrotliDecoderState* BrotliDecoderCreateInstance( brotli_alloc_func alloc_func, brotli_free_func free_func, void* opaque) { BrotliDecoderState* state = 0; @@ -1247,17 +1263,13 @@ static void BROTLI_NOINLINE WrapRingBuffer(BrotliDecoderState* s) { */ static BROTLI_BOOL BROTLI_NOINLINE BrotliEnsureRingBuffer( BrotliDecoderState* s) { - /* We need the slack region for the following reasons: - - doing up to two 16-byte copies for fast backward copying - - inserting transformed dictionary word (5 prefix + 24 base + 8 suffix) */ - static const int kRingBufferWriteAheadSlack = 42; uint8_t* old_ringbuffer = s->ringbuffer; if (s->ringbuffer_size == s->new_ringbuffer_size) { return BROTLI_TRUE; } - s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size + - kRingBufferWriteAheadSlack)); + s->ringbuffer = (uint8_t*)BROTLI_ALLOC(s, (size_t)(s->new_ringbuffer_size) + + kRingBufferWriteAheadSlack); if (s->ringbuffer == 0) { /* Restore previous value. */ s->ringbuffer = old_ringbuffer; @@ -1369,8 +1381,13 @@ static void BROTLI_NOINLINE BrotliCalculateRingBufferSize( output_size += s->meta_block_remaining_len; min_size = min_size < output_size ? output_size : min_size; - while ((new_ringbuffer_size >> 1) >= min_size) { - new_ringbuffer_size >>= 1; + if (!!s->canny_ringbuffer_allocation) { + /* Reduce ring buffer size to save memory when server is unscrupulous. + In worst case memory usage might be 1.5x bigger for a short period of + ring buffer reallocation.*/ + while ((new_ringbuffer_size >> 1) >= min_size) { + new_ringbuffer_size >>= 1; + } } s->new_ringbuffer_size = new_ringbuffer_size; diff --git a/c/dec/state.c b/c/dec/state.c index b7431f2..0db73fb 100644 --- a/c/dec/state.c +++ b/c/dec/state.c @@ -87,7 +87,11 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s, s->custom_dict_size = 0; s->is_last_metablock = 0; + s->is_uncompressed = 0; + s->is_metadata = 0; s->should_wrap_ringbuffer = 0; + s->canny_ringbuffer_allocation = 1; + s->window_bits = 0; s->max_distance = 0; s->dist_rb[0] = 16; diff --git a/c/dec/state.h b/c/dec/state.h index 5946124..00c373b 100644 --- a/c/dec/state.h +++ b/c/dec/state.h @@ -172,7 +172,7 @@ struct BrotliDecoderStateStruct { uint32_t space; HuffmanCode table[32]; - /* List of of symbol chains. */ + /* List of heads of symbol chains. */ uint16_t* symbol_lists; /* Storage from symbol_lists. */ uint16_t symbols_lists_array[BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1 + @@ -215,6 +215,7 @@ struct BrotliDecoderStateStruct { unsigned int is_uncompressed : 1; unsigned int is_metadata : 1; unsigned int should_wrap_ringbuffer : 1; + unsigned int canny_ringbuffer_allocation : 1; unsigned int size_nibbles : 8; uint32_t window_bits; diff --git a/c/enc/brotli_bit_stream.c b/c/enc/brotli_bit_stream.c index 4874695..2907510 100644 --- a/c/enc/brotli_bit_stream.c +++ b/c/enc/brotli_bit_stream.c @@ -1325,17 +1325,6 @@ void BrotliStoreUncompressedMetaBlock(BROTLI_BOOL is_final_block, } } -void BrotliStoreSyncMetaBlock(size_t* BROTLI_RESTRICT storage_ix, - uint8_t* BROTLI_RESTRICT storage) { - /* Empty metadata meta-block bit pattern: - 1 bit: is_last (0) - 2 bits: num nibbles (3) - 1 bit: reserved (0) - 2 bits: metadata length bytes (0) */ - BrotliWriteBits(6, 6, storage_ix, storage); - JumpToByteBoundary(storage_ix, storage); -} - #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif diff --git a/c/enc/brotli_bit_stream.h b/c/enc/brotli_bit_stream.h index a98f98f..2c8bfed 100644 --- a/c/enc/brotli_bit_stream.h +++ b/c/enc/brotli_bit_stream.h @@ -96,10 +96,6 @@ BROTLI_INTERNAL void BrotliStoreUncompressedMetaBlock( BROTLI_BOOL is_final_block, const uint8_t* input, size_t position, size_t mask, size_t len, size_t* storage_ix, uint8_t* storage); -/* Stores an empty metadata meta-block and syncs to a byte boundary. */ -BROTLI_INTERNAL void BrotliStoreSyncMetaBlock(size_t* storage_ix, - uint8_t* storage); - #if defined(__cplusplus) || defined(c_plusplus) } /* extern "C" */ #endif diff --git a/c/enc/cluster_inc.h b/c/enc/cluster_inc.h index cf279bd..22ecb3c 100644 --- a/c/enc/cluster_inc.h +++ b/c/enc/cluster_inc.h @@ -17,6 +17,8 @@ BROTLI_INTERNAL void FN(BrotliCompareAndPushToQueue)( size_t* num_pairs) CODE({ BROTLI_BOOL is_good_pair = BROTLI_FALSE; HistogramPair p; + p.idx1 = p.idx2 = 0; + p.cost_diff = p.cost_combo = 0; if (idx1 == idx2) { return; } diff --git a/c/include/brotli/decode.h b/c/include/brotli/decode.h index 93cbe38..b18e9e4 100755 --- a/c/include/brotli/decode.h +++ b/c/include/brotli/decode.h @@ -126,6 +126,29 @@ typedef enum { */ #define BROTLI_LAST_ERROR_CODE BROTLI_DECODER_ERROR_UNREACHABLE +/** Options to be used with ::BrotliDecoderSetParameter. */ +typedef enum BrotliDecoderParameter { + /** + * Disable "canny" ring buffer allocation strategy. + * + * Ring buffer is allocated according to window size, despite the real size of + * the content. + */ + BROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION = 0 +} BrotliDecoderParameter; + +/** + * Sets the specified parameter to the given decoder instance. + * + * @param state decoder instance + * @param param parameter to set + * @param value new parameter value + * @returns ::BROTLI_FALSE if parameter is unrecognized, or value is invalid + * @returns ::BROTLI_TRUE if value is accepted + */ +BROTLI_DEC_API BROTLI_BOOL BrotliDecoderSetParameter( + BrotliDecoderState* state, BrotliDecoderParameter param, uint32_t value); + /** * Creates an instance of ::BrotliDecoderState and initializes it. * diff --git a/c/tools/brotli.c b/c/tools/brotli.c index ff0cabf..3cf6297 100755 --- a/c/tools/brotli.c +++ b/c/tools/brotli.c @@ -697,8 +697,8 @@ static BROTLI_BOOL CloseFiles(Context* context, BROTLI_BOOL success) { static const size_t kFileBufferSize = 1 << 16; static BROTLI_BOOL DecompressFile(Context* context, BrotliDecoderState* s) { - size_t available_in; - const uint8_t* next_in; + size_t available_in = 0; + const uint8_t* next_in = NULL; size_t available_out = kFileBufferSize; uint8_t* next_out = context->output; BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; diff --git a/docs/decode.h.3 b/docs/decode.h.3 index f4b95fe..447c67b 100755 --- a/docs/decode.h.3 +++ b/docs/decode.h.3 @@ -1,4 +1,4 @@ -.TH "decode.h" 3 "Sun May 21 2017" "Brotli" \" -*- nroff -*- +.TH "decode.h" 3 "Tue Jun 13 2017" "Brotli" \" -*- nroff -*- .ad l .nh .SH NAME @@ -23,6 +23,10 @@ decode.h \- API for Brotli decompression\&. .in +1c .ti -1c +.RI "typedef enum \fBBrotliDecoderParameter\fP \fBBrotliDecoderParameter\fP" +.br +.RI "\fIOptions to be used with \fBBrotliDecoderSetParameter\fP\&. \fP" +.ti -1c .RI "typedef struct BrotliDecoderStateStruct \fBBrotliDecoderState\fP" .br .RI "\fIOpaque structure that holds decoder state\&. \fP" @@ -72,6 +76,10 @@ decode.h \- API for Brotli decompression\&. .br .RI "\fIPrepends LZ77 dictionary\&. \fP" .ti -1c +.RI "\fBBROTLI_BOOL\fP \fBBrotliDecoderSetParameter\fP (\fBBrotliDecoderState\fP *state, \fBBrotliDecoderParameter\fP param, uint32_t value)" +.br +.RI "\fISets the specified parameter to the given decoder instance\&. \fP" +.ti -1c .RI "const uint8_t * \fBBrotliDecoderTakeOutput\fP (\fBBrotliDecoderState\fP *state, size_t *size)" .br .RI "\fIAcquires pointer to internal output buffer\&. \fP" @@ -115,6 +123,10 @@ BROTLI_DECODER_ERROR_CODES_LIST(CASE_, NEWLINE_) The value of the last error code, negative integer\&. All other error code values are in the range from \fBBROTLI_LAST_ERROR_CODE\fP to \fC-1\fP\&. There are also 4 other possible non-error codes \fC0\fP \&.\&. \fC3\fP in \fBBrotliDecoderErrorCode\fP enumeration\&. .SH "Typedef Documentation" .PP +.SS "typedef enum \fBBrotliDecoderParameter\fP \fBBrotliDecoderParameter\fP" + +.PP +Options to be used with \fBBrotliDecoderSetParameter\fP\&. .SS "typedef struct BrotliDecoderStateStruct \fBBrotliDecoderState\fP" .PP @@ -125,6 +137,16 @@ Opaque structure that holds decoder state\&. Allocated and initialized with \fBB .PP Error code for detailed logging / production debugging\&. See \fBBrotliDecoderGetErrorCode\fP and \fBBROTLI_LAST_ERROR_CODE\fP\&. +.SS "enum \fBBrotliDecoderParameter\fP" + +.PP +Options to be used with \fBBrotliDecoderSetParameter\fP\&. +.PP +\fBEnumerator\fP +.in +1c +.TP +\fB\fIBROTLI_DECODER_PARAM_DISABLE_RING_BUFFER_REALLOCATION \fP\fP +Disable 'canny' ring buffer allocation strategy\&. Ring buffer is allocated according to window size, despite the real size of the content\&. .SS "enum \fBBrotliDecoderResult\fP" .PP @@ -360,6 +382,28 @@ Clean up and free state with \fBBrotliDecoderDestroyInstance\fP .RE .PP +.SS "\fBBROTLI_BOOL\fP BrotliDecoderSetParameter (\fBBrotliDecoderState\fP * state, \fBBrotliDecoderParameter\fP param, uint32_t value)" + +.PP +Sets the specified parameter to the given decoder instance\&. +.PP +\fBParameters:\fP +.RS 4 +\fIstate\fP decoder instance +.br +\fIparam\fP parameter to set +.br +\fIvalue\fP new parameter value +.RE +.PP +\fBReturns:\fP +.RS 4 +\fBBROTLI_FALSE\fP if parameter is unrecognized, or value is invalid +.PP +\fBBROTLI_TRUE\fP if value is accepted +.RE +.PP + .SS "const uint8_t* BrotliDecoderTakeOutput (\fBBrotliDecoderState\fP * state, size_t * size)" .PP -- cgit v1.1