diff options
author | Eugene Kliuchnikov <eustas@google.com> | 2016-12-12 10:27:13 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2016-12-12 10:27:13 +0100 |
commit | 0ee416139f47e656f936e724dee1a332aa0e393d (patch) | |
tree | 86dfa0069d206eb882c39091df46f2b743c0643a | |
parent | 4a60128c1339f74c13f0c8ae27a5dfe426b14208 (diff) | |
download | brotli-0ee416139f47e656f936e724dee1a332aa0e393d.zip brotli-0ee416139f47e656f936e724dee1a332aa0e393d.tar.gz brotli-0ee416139f47e656f936e724dee1a332aa0e393d.tar.bz2 |
Update python brotli wrapper (#479)
* Update python brotli wrapper
* release GIL on CPU intensive blocks, fixes #476
* use BrotliDecoderTakeOutput (less memory, less memcpy)
* Python: Convert bro.py tests to unittest style (#478)
* Create unittest-style tests for `bro.py` decompression and compression
* Delete old tests for `bro.py`
* Update test method generation to properly create a Cartesian product
of iterables using `itertools.product`
* Update python brotli wrapper
* release GIL on CPU intensive blocks, fixes #476
* use BrotliDecoderTakeOutput (less memory, less memcpy)
-rw-r--r-- | python/_brotli.cc | 31 |
1 files changed, 20 insertions, 11 deletions
diff --git a/python/_brotli.cc b/python/_brotli.cc index c084e8b..669f9e4 100644 --- a/python/_brotli.cc +++ b/python/_brotli.cc @@ -90,6 +90,7 @@ static int lgblock_convertor(PyObject *o, int *lgblock) { static BROTLI_BOOL compress_stream(BrotliEncoderState* enc, BrotliEncoderOperation op, std::vector<uint8_t>* output, uint8_t* input, size_t input_length) { BROTLI_BOOL ok = BROTLI_TRUE; + Py_BEGIN_ALLOW_THREADS size_t available_in = input_length; const uint8_t* next_in = input; @@ -116,6 +117,7 @@ static BROTLI_BOOL compress_stream(BrotliEncoderState* enc, BrotliEncoderOperati break; } + Py_END_ALLOW_THREADS return ok; } @@ -201,8 +203,12 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj BrotliEncoderSetParameter(self->enc, BROTLI_PARAM_LGBLOCK, (uint32_t)lgblock); if (custom_dictionary_length != 0) { + /* Unlike decoder, encoder processes dictionary immediately, that is why + it makes sense to release python GIL. */ + Py_BEGIN_ALLOW_THREADS BrotliEncoderSetCustomDictionary(self->enc, custom_dictionary_length, custom_dictionary); + Py_END_ALLOW_THREADS } return 0; @@ -431,8 +437,10 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key return NULL; std::vector<uint8_t> output; - const size_t kBufferSize = 65536; - uint8_t* buffer = new uint8_t[kBufferSize]; + + /* >>> Pure C block; release python GIL. */ + Py_BEGIN_ALLOW_THREADS + BrotliDecoderState* state = BrotliDecoderCreateInstance(0, 0, 0); if (custom_dictionary_length != 0) { BrotliDecoderSetCustomDictionary(state, custom_dictionary_length, custom_dictionary); @@ -440,24 +448,25 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { - size_t available_out = kBufferSize; - uint8_t* next_out = buffer; + size_t available_out = 0; result = BrotliDecoderDecompressStream(state, &length, &input, - &available_out, &next_out, 0); - size_t used_out = kBufferSize - available_out; - if (used_out != 0) - output.insert(output.end(), buffer, buffer + used_out); + &available_out, 0, 0); + const uint8_t* next_out = BrotliDecoderTakeOutput(state, &available_out); + if (available_out != 0) + output.insert(output.end(), next_out, next_out + available_out); } ok = result == BROTLI_DECODER_RESULT_SUCCESS; + BrotliDecoderDestroyInstance(state); + + Py_END_ALLOW_THREADS + /* <<< Pure C block end. Python GIL reacquired. */ + if (ok) { ret = PyBytes_FromStringAndSize((char*)(output.size() ? &output[0] : NULL), output.size()); } else { PyErr_SetString(BrotliError, "BrotliDecompress failed"); } - BrotliDecoderDestroyInstance(state); - delete[] buffer; - return ret; } |