diff options
Diffstat (limited to 'python')
-rw-r--r-- | python/_brotli.cc | 63 | ||||
-rwxr-xr-x | python/bro.py | 20 | ||||
-rw-r--r-- | python/brotli.py | 7 | ||||
-rw-r--r-- | python/tests/bro_test.py | 16 | ||||
-rw-r--r-- | python/tests/compress_test.py | 13 |
5 files changed, 18 insertions, 101 deletions
diff --git a/python/_brotli.cc b/python/_brotli.cc index 60567f5..d17b90d 100644 --- a/python/_brotli.cc +++ b/python/_brotli.cc @@ -125,7 +125,7 @@ PyDoc_STRVAR(brotli_Compressor_doc, "An object to compress a byte string.\n" "\n" "Signature:\n" -" Compressor(mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, dictionary='')\n" +" Compressor(mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0)\n" "\n" "Args:\n" " mode (int, optional): The compression mode can be MODE_GENERIC (default),\n" @@ -138,8 +138,6 @@ PyDoc_STRVAR(brotli_Compressor_doc, " lgblock (int, optional): Base 2 logarithm of the maximum input block size.\n" " Range is 16 to 24. If set to 0, the value will be set based on the\n" " quality. Defaults to 0.\n" -" dictionary (bytes, optional): Custom dictionary. Only last sliding window\n" -" size bytes will be used.\n" "\n" "Raises:\n" " brotli.error: If arguments are invalid.\n"); @@ -174,20 +172,16 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj int quality = -1; int lgwin = -1; int lgblock = -1; - uint8_t* custom_dictionary = NULL; - size_t custom_dictionary_length = 0; int ok; - static const char *kwlist[] = { - "mode", "quality", "lgwin", "lgblock", "dictionary", NULL}; + static const char *kwlist[] = {"mode", "quality", "lgwin", "lgblock", NULL}; - ok = PyArg_ParseTupleAndKeywords(args, keywds, "|O&O&O&O&s#:Compressor", + ok = PyArg_ParseTupleAndKeywords(args, keywds, "|O&O&O&O&:Compressor", const_cast<char **>(kwlist), &mode_convertor, &mode, &quality_convertor, &quality, &lgwin_convertor, &lgwin, - &lgblock_convertor, &lgblock, - &custom_dictionary, &custom_dictionary_length); + &lgblock_convertor, &lgblock); if (!ok) return -1; if (!self->enc) @@ -202,15 +196,6 @@ static int brotli_Compressor_init(brotli_Compressor *self, PyObject *args, PyObj if (lgblock != -1) BrotliEncoderSetParameter(self->enc, BROTLI_PARAM_LGBLOCK, (uint32_t)lgblock); - if (custom_dictionary_length != 0) { - /* Unlike decoder, encoder processes dictionary immediately, that is why - it makes sense to release python GIL. */ - Py_BEGIN_ALLOW_THREADS - BrotliEncoderSetCustomDictionary(self->enc, custom_dictionary_length, - custom_dictionary); - Py_END_ALLOW_THREADS - } - return 0; } @@ -432,11 +417,7 @@ PyDoc_STRVAR(brotli_Decompressor_doc, "An object to decompress a byte string.\n" "\n" "Signature:\n" -" Decompressor(dictionary='')\n" -"\n" -"Args:\n" -" dictionary (bytes, optional): Custom dictionary. Only last sliding window\n" -" size bytes will be used.\n" +" Decompressor()\n" "\n" "Raises:\n" " brotli.error: If arguments are invalid.\n"); @@ -467,26 +448,17 @@ static PyObject* brotli_Decompressor_new(PyTypeObject *type, PyObject *args, PyO } static int brotli_Decompressor_init(brotli_Decompressor *self, PyObject *args, PyObject *keywds) { - uint8_t* custom_dictionary = NULL; - size_t custom_dictionary_length = 0; int ok; - static const char *kwlist[] = { - "dictionary", NULL}; + static const char *kwlist[] = {NULL}; - ok = PyArg_ParseTupleAndKeywords(args, keywds, "|s#:Decompressor", - const_cast<char **>(kwlist), - &custom_dictionary, &custom_dictionary_length); + ok = PyArg_ParseTupleAndKeywords(args, keywds, "|:Decompressor", + const_cast<char **>(kwlist)); if (!ok) return -1; if (!self->dec) return -1; - if (custom_dictionary_length != 0) { - BrotliDecoderSetCustomDictionary(self->dec, custom_dictionary_length, - custom_dictionary); - } - return 0; } @@ -644,8 +616,6 @@ PyDoc_STRVAR(brotli_decompress__doc__, "\n" "Args:\n" " string (bytes): The compressed input data.\n" -" dictionary (bytes, optional): Custom dictionary. MUST be the same data\n" -" as passed to compress method.\n" "\n" "Returns:\n" " The decompressed byte string.\n" @@ -655,19 +625,15 @@ PyDoc_STRVAR(brotli_decompress__doc__, static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *keywds) { PyObject *ret = NULL; - const uint8_t *input, *custom_dictionary; - size_t length, custom_dictionary_length; + const uint8_t *input; + size_t length; int ok; - static const char *kwlist[] = {"string", "dictionary", NULL}; + static const char *kwlist[] = {"string", NULL}; - custom_dictionary = NULL; - custom_dictionary_length = 0; - - ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|s#:decompress", + ok = PyArg_ParseTupleAndKeywords(args, keywds, "s#|:decompress", const_cast<char **>(kwlist), - &input, &length, - &custom_dictionary, &custom_dictionary_length); + &input, &length); if (!ok) return NULL; @@ -677,9 +643,6 @@ static PyObject* brotli_decompress(PyObject *self, PyObject *args, PyObject *key Py_BEGIN_ALLOW_THREADS BrotliDecoderState* state = BrotliDecoderCreateInstance(0, 0, 0); - if (custom_dictionary_length != 0) { - BrotliDecoderSetCustomDictionary(state, custom_dictionary_length, custom_dictionary); - } BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT; while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { diff --git a/python/bro.py b/python/bro.py index 2a84e42..7a094b4 100755 --- a/python/bro.py +++ b/python/bro.py @@ -114,13 +114,6 @@ def main(args=None): help='Base 2 logarithm of the maximum input block size. ' 'Range is 16 to 24. If set to 0, the value will be set based ' 'on the quality. Defaults to 0.') - params.add_argument( - '--custom-dictionary', - metavar='FILE', - type=str, - dest='dictfile', - help='Custom dictionary file.', - default=None) # set default values using global DEFAULT_PARAMS dictionary parser.set_defaults(**DEFAULT_PARAMS) @@ -145,25 +138,16 @@ def main(args=None): else: outfile = get_binary_stdio('stdout') - if options.dictfile: - if not os.path.isfile(options.dictfile): - parser.error('file "%s" not found' % options.dictfile) - with open(options.dictfile, 'rb') as dictfile: - custom_dictionary = dictfile.read() - else: - custom_dictionary = '' - try: if options.decompress: - data = brotli.decompress(data, dictionary=custom_dictionary) + data = brotli.decompress(data) else: data = brotli.compress( data, mode=options.mode, quality=options.quality, lgwin=options.lgwin, - lgblock=options.lgblock, - dictionary=custom_dictionary) + lgblock=options.lgblock) except brotli.error as e: parser.exit(1, 'bro: error: %s: %s' % (e, options.infile or 'sys.stdin')) diff --git a/python/brotli.py b/python/brotli.py index 9c9bb4f..d66966b 100644 --- a/python/brotli.py +++ b/python/brotli.py @@ -23,8 +23,7 @@ Compressor = _brotli.Compressor Decompressor = _brotli.Decompressor # Compress a byte string. -def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, - dictionary=''): +def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0): """Compress a byte string. Args: @@ -39,8 +38,6 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, lgblock (int, optional): Base 2 logarithm of the maximum input block size. Range is 16 to 24. If set to 0, the value will be set based on the quality. Defaults to 0. - dictionary (bytes, optional): Custom dictionary. Only last sliding window - size bytes will be used. Returns: The compressed byte string. @@ -49,7 +46,7 @@ def compress(string, mode=MODE_GENERIC, quality=11, lgwin=22, lgblock=0, brotli.error: If arguments are invalid, or compressor fails. """ compressor = Compressor(mode=mode, quality=quality, lgwin=lgwin, - lgblock=lgblock, dictionary=dictionary) + lgblock=lgblock) return compressor.process(string) + compressor.finish() # Decompress a compressed byte string. diff --git a/python/tests/bro_test.py b/python/tests/bro_test.py index ef8c45c..c0d55dc 100644 --- a/python/tests/bro_test.py +++ b/python/tests/bro_test.py @@ -62,8 +62,6 @@ class TestBroCompress(_test_utils.TestCase): temp_compressed = _test_utils.get_temp_compressed_name(test_data) original = test_data args = [PYTHON, BRO, '-f', '-d'] - if 'dictionary' in kwargs: - args.extend(['--custom-dictionary', str(kwargs['dictionary'])]) args.extend(['-i', temp_compressed, '-o', temp_uncompressed]) subprocess.check_call(args, env=TEST_ENV) self.assertFilesMatch(temp_uncompressed, original) @@ -75,8 +73,6 @@ class TestBroCompress(_test_utils.TestCase): args.extend(['-q', str(kwargs['quality'])]) if 'lgwin' in kwargs: args.extend(['--lgwin', str(kwargs['lgwin'])]) - if 'dictionary' in kwargs: - args.extend(['--custom-dictionary', str(kwargs['dictionary'])]) args.extend(['-i', test_data, '-o', temp_compressed]) subprocess.check_call(args, env=TEST_ENV) @@ -87,8 +83,6 @@ class TestBroCompress(_test_utils.TestCase): args.extend(['-q', str(kwargs['quality'])]) if 'lgwin' in kwargs: args.extend(['--lgwin', str(kwargs['lgwin'])]) - if 'dictionary' in kwargs: - args.extend(['--custom-dictionary', str(kwargs['dictionary'])]) with open(temp_compressed, 'wb') as out_file: with open(test_data, 'rb') as in_file: subprocess.check_call( @@ -102,16 +96,6 @@ class TestBroCompress(_test_utils.TestCase): self._compress_pipe(test_data, **kwargs) self._check_decompression(test_data) - def _test_compress_file_custom_dictionary(self, test_data, **kwargs): - kwargs['dictionary'] = test_data - self._compress_file(test_data, **kwargs) - self._check_decompression(test_data, **kwargs) - - def _test_compress_pipe_custom_dictionary(self, test_data, **kwargs): - kwargs['dictionary'] = test_data - self._compress_pipe(test_data, **kwargs) - self._check_decompression(test_data, **kwargs) - _test_utils.generate_test_methods( TestBroCompress, variants=TestBroCompress.VARIANTS) diff --git a/python/tests/compress_test.py b/python/tests/compress_test.py index 79e7097..46ff68f 100644 --- a/python/tests/compress_test.py +++ b/python/tests/compress_test.py @@ -14,11 +14,7 @@ class TestCompress(_test_utils.TestCase): VARIANTS = {'quality': (1, 6, 9, 11), 'lgwin': (10, 15, 20, 24)} def _check_decompression(self, test_data, **kwargs): - # Only dictionary is supported as a kwarg to brotli.decompress. - if 'dictionary' in kwargs: - kwargs = {'dictionary': kwargs['dictionary']} - else: - kwargs = {} + kwargs = {} # Write decompression to temp file and verify it matches the original. temp_uncompressed = _test_utils.get_temp_uncompressed_name(test_data) temp_compressed = _test_utils.get_temp_compressed_name(test_data) @@ -38,13 +34,6 @@ class TestCompress(_test_utils.TestCase): self._compress(test_data, **kwargs) self._check_decompression(test_data, **kwargs) - def _test_compress_custom_dictionary(self, test_data, **kwargs): - with open(test_data, 'rb') as in_file: - dictionary = in_file.read() - kwargs['dictionary'] = dictionary - self._compress(test_data, **kwargs) - self._check_decompression(test_data, **kwargs) - _test_utils.generate_test_methods(TestCompress, variants=TestCompress.VARIANTS) |