From a657d9969dfb9655f4b483a658ad5da9dda3e52e Mon Sep 17 00:00:00 2001 From: Eugene Kliuchnikov Date: Wed, 22 Mar 2017 12:41:19 +0100 Subject: Add go wrapper, streamline java decoder: (#524) * add (c)brotli golang wrapper * remove (language-specific) enums in java decoder --- BUILD | 4 + WORKSPACE | 9 + go/cbrotli/BUILD | 23 +++ go/cbrotli/cbrotli.go | 254 +++++++++++++++++++++++++++ go/cbrotli/cbrotli_test.go | 271 +++++++++++++++++++++++++++++ go/cbrotli/internal/BUILD | 19 ++ go/cbrotli/internal/decoder.go | 110 ++++++++++++ go/cbrotli/internal/encoder.go | 135 ++++++++++++++ java/org/brotli/dec/BUILD | 6 + java/org/brotli/dec/EnumTest.java | 53 ++++++ java/org/brotli/dec/RunningState.java | 28 +-- java/org/brotli/dec/State.java | 4 +- java/org/brotli/dec/Transform.java | 10 +- java/org/brotli/dec/WordTransformType.java | 57 +++--- 14 files changed, 932 insertions(+), 51 deletions(-) create mode 100755 go/cbrotli/BUILD create mode 100755 go/cbrotli/cbrotli.go create mode 100755 go/cbrotli/cbrotli_test.go create mode 100755 go/cbrotli/internal/BUILD create mode 100755 go/cbrotli/internal/decoder.go create mode 100755 go/cbrotli/internal/encoder.go create mode 100755 java/org/brotli/dec/EnumTest.java diff --git a/BUILD b/BUILD index 1dea6d8..bc86186 100644 --- a/BUILD +++ b/BUILD @@ -100,3 +100,7 @@ cc_binary( ":brotlienc", ], ) + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix") + +go_prefix("github.com/google/brotli") diff --git a/WORKSPACE b/WORKSPACE index 9771d45..4c3c9b7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -7,3 +7,12 @@ maven_jar( name = "junit_junit", artifact = "junit:junit:4.12", ) + +git_repository( + name = "io_bazel_rules_go", + remote = "https://github.com/bazelbuild/rules_go.git", + tag = "0.4.1", +) +load("@io_bazel_rules_go//go:def.bzl", "go_repositories") + +go_repositories() diff --git a/go/cbrotli/BUILD b/go/cbrotli/BUILD new file mode 100755 index 0000000..4bea8a0 --- /dev/null +++ b/go/cbrotli/BUILD @@ -0,0 +1,23 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +load("@io_bazel_rules_go//go:def.bzl", "go_prefix", "go_library", "go_test") + +go_prefix("github.com/google/brotli") + +go_library( + name = "cbrotli", + srcs = ["cbrotli.go"], + deps = [ + "//go/cbrotli/internal:decoder", + "//go/cbrotli/internal:encoder", + ], +) + +go_test( + name = "cbrotli_test", + size = "small", + srcs = ["cbrotli_test.go"], + library = ":cbrotli", +) diff --git a/go/cbrotli/cbrotli.go b/go/cbrotli/cbrotli.go new file mode 100755 index 0000000..a7009d0 --- /dev/null +++ b/go/cbrotli/cbrotli.go @@ -0,0 +1,254 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Distributed under MIT license. +// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + +// Package cbrotli compresses and decompresses data with C-Brotli library. +package cbrotli + +import ( + "bytes" + "io" + + "github.com/google/brotli/go/cbrotli/internal/decoder" + "github.com/google/brotli/go/cbrotli/internal/encoder" +) + +// An internalError reports an error in the (c-)brotli code itself. +type internalError string + +func (e internalError) Error() string { + return "cbrotli: internal error: " + string(e) +} + +//------------------------------------------------------------------------------ +// Encoder +//------------------------------------------------------------------------------ + +// WriterOptions configures Writer. +type WriterOptions struct { + // Quality controls the compression-speed vs compression-density trade-offs. + // The higher the quality, the slower the compression. Range is 0 to 11. + Quality int + // LGWin is the base 2 logarithm of the sliding window size. + // Range is 10 to 24. 0 indicates automatic configuration based on Quality. + LGWin int +} + +// Writer implements io.WriteCloser, an io.Writer decorator that produces +// Brotli-encoded data. +type Writer struct { + dst io.Writer + encoder encoder.Encoder + closed bool +} + +// NewWriter initializes new Writer instance. +// Close MUST be called to free resources. +func NewWriter(dst io.Writer, options WriterOptions) *Writer { + return &Writer{ + dst: dst, + encoder: encoder.New(options.Quality, options.LGWin), + } +} + +// Close implements io.Closer. Close MUST be invoked to free native resources. +// Also Close implicitly flushes remaining data to the decorated writer. +func (z *Writer) Close() error { + if z.closed { + return nil + } + defer z.encoder.Close() + _, err := z.writeChunk(nil, encoder.Finish) + z.closed = true + return err +} + +func (z *Writer) writeChunk(p []byte, op encoder.Operation) (int, error) { + if z.closed { + return 0, internalError("write after close") + } + var totalBytesConsumed int + var err error + for { + bytesConsumed, output, status := z.encoder.CompressStream(p, op) + if status == encoder.Error { + err = internalError("encoder failure") + break + } + p = p[bytesConsumed:] + totalBytesConsumed += bytesConsumed + _, err = z.dst.Write(output) + if err != nil { + break + } + if len(p) == 0 && status == encoder.Done { + break + } + } + return totalBytesConsumed, err +} + +// Write implements io.Writer. +func (z *Writer) Write(p []byte) (int, error) { + return z.writeChunk(p, encoder.Process) +} + +// Flush outputs encoded data for all input provided to Write. The resulting +// output can be decoded to match all input before Flush, but the stream is +// not yet complete until after Close. +// Flush has a negative impact on compression. +func (z *Writer) Flush() error { + _, err := z.writeChunk(nil, encoder.Finish) + return err +} + +// Encode returns content encoded with Brotli. +func Encode(content []byte, options WriterOptions) ([]byte, error) { + var buf bytes.Buffer + writer := NewWriter(&buf, options) + defer writer.Close() + _, err := writer.Write(content) + if err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +//------------------------------------------------------------------------------ +// Decoder +//------------------------------------------------------------------------------ + +// Reader implements io.ReadCloser, an io.Reader decorator that decodes +// Brotli-encoded data. +type Reader struct { + src io.Reader + decoder decoder.Decoder + buf []byte // intermediate read buffer pointed to by next + eof bool // true if all compressed stream is decoded + next []byte // buffered data to be passed to decoder + output []byte // data produced by decoder, but not yet consumed + srcErr error // last source reader error + err error // reader state; nil if it is OK to read further + closed bool // true is stream is already closed +} + +// NewReader initializes new Reader instance. +// Close MUST be called to free resources. +func NewReader(src io.Reader) *Reader { + return &Reader{ + src: src, + decoder: decoder.New(), + buf: make([]byte, 32*1024), + eof: false, + } +} + +// Close implements io.Closer. Close MUST be invoked to free native resources. +func (z *Reader) Close() error { + if z.closed { + return nil + } + z.decoder.Close() + z.err = internalError("read after close") + z.closed = true + return nil +} + +func isEOF(src io.Reader) bool { + n, err := src.Read(make([]byte, 1)) + return n == 0 && err == io.EOF +} + +// Read implements io.Reader. +func (z *Reader) Read(p []byte) (int, error) { + // Any error state is unrecoverable. + if z.err != nil { + return 0, z.err + } + // See io.Reader documentation. + if len(p) == 0 { + return 0, nil + } + + var totalOutBytes int + + // There is no practical limit for amount of bytes being consumed by decoder + // before producing any output. Continue feeding decoder until some data is + // produced + for { + // Push already produced output first. + if outBytes := len(z.output); outBytes != 0 { + outBytes = copy(p, z.output) + p = p[outBytes:] + z.output = z.output[outBytes:] + totalOutBytes += outBytes + // Output buffer is full. + if len(p) == 0 { + break + } + continue + } + // No more produced output left. + // If no more output is expected, then we are finished. + if z.eof { + z.err = io.EOF + break + } + // Replenish buffer (might cause blocking read), only if necessary. + if len(z.next) == 0 && totalOutBytes == 0 && z.srcErr != io.EOF { + var n int + n, z.srcErr = z.src.Read(z.buf) + z.next = z.buf[:n] + if z.srcErr != nil && z.srcErr != io.EOF { + z.err = z.srcErr + break + } + } + // Do decoding. + consumed, output, status := z.decoder.DecompressStream(z.next) + z.output = output + z.next = z.next[consumed:] + if status == decoder.Error { + // When error happens, the remaining output does not matter. + z.err = internalError("decoder failure") + break + } else if status == decoder.Done { + // Decoder stream is closed; no further input is expected. + if len(z.next) != 0 || (z.srcErr != io.EOF && !isEOF(z.src)) { + z.err = internalError("excessive input") + break + } + // No more output is expected; keep pushing output. + z.eof = true + continue + } else { + // If can not move any further... + if consumed == 0 && len(z.output) == 0 { + // Unexpected end of input. + if z.srcErr == io.EOF || totalOutBytes == 0 { + z.err = io.ErrUnexpectedEOF + } + // Postpone blocking reads for the next invocation. + break + } + // Continue pushing output. + } + } + return totalOutBytes, z.err +} + +// Decode decodes Brotli encoded data. +func Decode(encodedData []byte) ([]byte, error) { + var buf bytes.Buffer + reader := NewReader(bytes.NewReader(encodedData)) + defer reader.Close() + _, err := io.Copy(&buf, reader) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/go/cbrotli/cbrotli_test.go b/go/cbrotli/cbrotli_test.go new file mode 100755 index 0000000..4fe0960 --- /dev/null +++ b/go/cbrotli/cbrotli_test.go @@ -0,0 +1,271 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Distributed under MIT license. +// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + +package cbrotli + +import ( + "bytes" + "fmt" + "io" + "math" + "math/rand" + "testing" +) + +func checkCompressedData(compressedData, wantOriginalData []byte) error { + uncompressed, err := Decode(compressedData) + if err != nil { + return fmt.Errorf("brotli decompress failed: %v", err) + } + if !bytes.Equal(uncompressed, wantOriginalData) { + if len(wantOriginalData) != len(uncompressed) { + return fmt.Errorf(""+ + "Data doesn't uncompress to the original value.\n"+ + "Length of original: %v\n"+ + "Length of uncompressed: %v", + len(wantOriginalData), len(uncompressed)) + } + for i := range wantOriginalData { + if wantOriginalData[i] != uncompressed[i] { + return fmt.Errorf(""+ + "Data doesn't uncompress to the original value.\n"+ + "Original at %v is %v\n"+ + "Uncompressed at %v is %v", + i, wantOriginalData[i], i, uncompressed[i]) + } + } + } + return nil +} + +func TestEncoderNoWrite(t *testing.T) { + out := bytes.Buffer{} + e := NewWriter(&out, WriterOptions{Quality: 5}) + if err := e.Close(); err != nil { + t.Errorf("Close()=%v, want nil", err) + } + // Check Write after close. + if _, err := e.Write([]byte("hi")); err == nil { + t.Errorf("No error after Close() + Write()") + } +} + +func TestEncoderEmptyWrite(t *testing.T) { + out := bytes.Buffer{} + e := NewWriter(&out, WriterOptions{Quality: 5}) + n, err := e.Write([]byte("")) + if n != 0 || err != nil { + t.Errorf("Write()=%v,%v, want 0, nil", n, err) + } + if err := e.Close(); err != nil { + t.Errorf("Close()=%v, want nil", err) + } +} + +func TestWriter(t *testing.T) { + // Test basic encoder usage. + input := []byte("

Hello world

") + out := bytes.Buffer{} + e := NewWriter(&out, WriterOptions{Quality: 1}) + in := bytes.NewReader([]byte(input)) + n, err := io.Copy(e, in) + if err != nil { + t.Errorf("Copy Error: %v", err) + } + if int(n) != len(input) { + t.Errorf("Copy() n=%v, want %v", n, len(input)) + } + if err := e.Close(); err != nil { + t.Errorf("Close Error after copied %d bytes: %v", n, err) + } + if err := checkCompressedData(out.Bytes(), input); err != nil { + t.Error(err) + } +} + +func TestEncoderStreams(t *testing.T) { + // Test that output is streamed. + // Adjust window size to ensure the encoder outputs at least enough bytes + // to fill the window. + const lgWin = 16 + windowSize := int(math.Pow(2, lgWin)) + input := make([]byte, 2*windowSize) + rand.Read(input) + out := bytes.Buffer{} + e := NewWriter(&out, WriterOptions{Quality: 11, LGWin: lgWin}) + halfInput := input[:len(input)/2] + in := bytes.NewReader(halfInput) + + n, err := io.Copy(e, in) + if err != nil { + t.Errorf("Copy Error: %v", err) + } + + // We've fed more data than the sliding window size. Check that some + // compressed data has been output. + if out.Len() == 0 { + t.Errorf("Output length is after %d bytes written", n) + } + if err := e.Close(); err != nil { + t.Errorf("Close Error after copied %d bytes: %v", n, err) + } + if err := checkCompressedData(out.Bytes(), halfInput); err != nil { + t.Error(err) + } +} + +func TestEncoderLargeInput(t *testing.T) { + input := make([]byte, 1000000) + rand.Read(input) + out := bytes.Buffer{} + e := NewWriter(&out, WriterOptions{Quality: 5}) + in := bytes.NewReader(input) + + n, err := io.Copy(e, in) + if err != nil { + t.Errorf("Copy Error: %v", err) + } + if int(n) != len(input) { + t.Errorf("Copy() n=%v, want %v", n, len(input)) + } + if err := e.Close(); err != nil { + t.Errorf("Close Error after copied %d bytes: %v", n, err) + } + if err := checkCompressedData(out.Bytes(), input); err != nil { + t.Error(err) + } +} + +func TestEncoderFlush(t *testing.T) { + input := make([]byte, 1000) + rand.Read(input) + out := bytes.Buffer{} + e := NewWriter(&out, WriterOptions{Quality: 5}) + in := bytes.NewReader(input) + _, err := io.Copy(e, in) + if err != nil { + t.Fatalf("Copy Error: %v", err) + } + if err := e.Flush(); err != nil { + t.Fatalf("Flush(): %v", err) + } + decompressed := make([]byte, 1000) + reader := NewReader(bytes.NewReader(out.Bytes())) + n, err := reader.Read(decompressed) + if n != len(decompressed) || err != nil { + t.Errorf("Expected <%v, nil>, but <%v, %v>", len(decompressed), n, err) + } + reader.Close() + if !bytes.Equal(decompressed, input) { + t.Errorf(""+ + "Decompress after flush: %v\n"+ + "%q\n"+ + "want:\n%q", + err, decompressed, input) + } + if err := e.Close(); err != nil { + t.Errorf("Close(): %v", err) + } +} + +func TestReader(t *testing.T) { + content := bytes.Repeat([]byte("hello world!"), 10000) + encoded, _ := Encode(content, WriterOptions{Quality: 5}) + r := NewReader(bytes.NewReader(encoded)) + var decodedOutput bytes.Buffer + n, err := io.Copy(&decodedOutput, r) + if err != nil { + t.Fatalf("Copy(): n=%v, err=%v", n, err) + } + if err := r.Close(); err != nil { + t.Errorf("Close(): %v", err) + } + if got := decodedOutput.Bytes(); !bytes.Equal(got, content) { + t.Errorf(""+ + "Reader output:\n"+ + "%q\n"+ + "want:\n"+ + "%q", + got, content) + } +} + +func TestDecode(t *testing.T) { + content := bytes.Repeat([]byte("hello world!"), 10000) + encoded, _ := Encode(content, WriterOptions{Quality: 5}) + decoded, err := Decode(encoded) + if err != nil { + t.Errorf("Decode: %v", err) + } + if !bytes.Equal(decoded, content) { + t.Errorf(""+ + "Decode content:\n"+ + "%q\n"+ + "want:\n"+ + "%q", + decoded, content) + } +} + +func TestDecodeFuzz(t *testing.T) { + // Test that the decoder terminates with corrupted input. + content := bytes.Repeat([]byte("hello world!"), 100) + src := rand.NewSource(0) + encoded, _ := Encode(content, WriterOptions{Quality: 5}) + for i := 0; i < 100; i++ { + enc := append([]byte{}, encoded...) + for j := 0; j < 5; j++ { + enc[int(src.Int63())%len(enc)] = byte(src.Int63() % 256) + } + Decode(enc) + } +} + +func TestDecodeTrailingData(t *testing.T) { + content := bytes.Repeat([]byte("hello world!"), 100) + encoded, _ := Encode(content, WriterOptions{Quality: 5}) + _, err := Decode(append(encoded, 0)) + if err == nil { + t.Errorf("Expected 'excessive input' error") + } +} + +func TestEncodeDecode(t *testing.T) { + for _, test := range []struct { + data []byte + repeats int + }{ + {nil, 0}, + {[]byte("A"), 1}, + {[]byte("

Hello world

"), 10}, + {[]byte("

Hello world

"), 1000}, + } { + t.Logf("case %q x %d", test.data, test.repeats) + input := bytes.Repeat(test.data, test.repeats) + encoded, err := Encode(input, WriterOptions{Quality: 5}) + if err != nil { + t.Errorf("Encode: %v", err) + } + // Inputs are compressible, but may be too small to compress. + if maxSize := len(input)/2 + 20; len(encoded) >= maxSize { + t.Errorf(""+ + "Encode returned %d bytes, want <%d\n"+ + "Encoded=%q", + len(encoded), maxSize, encoded) + } + decoded, err := Decode(encoded) + if err != nil { + t.Errorf("Decode: %v", err) + } + if !bytes.Equal(decoded, input) { + t.Errorf(""+ + "Decode content:\n"+ + "%q\n"+ + "want:\n"+ + "%q", + decoded, input) + } + } +} diff --git a/go/cbrotli/internal/BUILD b/go/cbrotli/internal/BUILD new file mode 100755 index 0000000..a3e3b87 --- /dev/null +++ b/go/cbrotli/internal/BUILD @@ -0,0 +1,19 @@ +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +load("@io_bazel_rules_go//go:def.bzl", "cgo_library") + +cgo_library( + name = "decoder", + srcs = ["decoder.go"], + visibility = ["//go/cbrotli:__subpackages__"], + cdeps = ["//:brotlidec"], +) + +cgo_library( + name = "encoder", + srcs = ["encoder.go"], + visibility = ["//go/cbrotli:__subpackages__"], + cdeps = ["//:brotlienc"], +) diff --git a/go/cbrotli/internal/decoder.go b/go/cbrotli/internal/decoder.go new file mode 100755 index 0000000..74d4c21 --- /dev/null +++ b/go/cbrotli/internal/decoder.go @@ -0,0 +1,110 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Distributed under MIT license. +// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + +// Package decoder wraps the brotli decoder C API used by package brotli. +package decoder + +/* +#include + +// Wrap BrotliDecoderDecompressStream so that it doesn't take variable (in-out) +// pointers. Instead of updated pointer, deltas are saved in auxiliary struct. + +struct DecompressStreamResult { + size_t bytes_consumed; + const uint8_t* output_data; + size_t output_data_size; + BrotliDecoderResult status; +}; + +struct DecompressStreamResult DecompressStream(BrotliDecoderState* s, + const uint8_t* encoded_data, size_t encoded_data_size) { + struct DecompressStreamResult result; + size_t available_in = encoded_data_size; + const uint8_t* next_in = encoded_data; + size_t available_out = 0; + result.status = BrotliDecoderDecompressStream(s, + &available_in, &next_in, &available_out, 0, 0); + result.bytes_consumed = encoded_data_size - available_in; + result.output_data = 0; + result.output_data_size = 0; + if (result.status != BROTLI_DECODER_RESULT_ERROR) { + result.output_data = BrotliDecoderTakeOutput(s, &result.output_data_size); + if (BrotliDecoderIsFinished(s)) { + result.status = BROTLI_DECODER_RESULT_SUCCESS; + } + } + return result; +} + +*/ +import "C" +import ( + "unsafe" +) + +// Status represents internal state after DecompressStream invokation +type Status int + +const ( + // Error happened + Error Status = iota + // Done means that no more output will be produced + Done + // Ok means that more output might be produced with no additional input + Ok +) + +// Decoder is the Brotli c-decoder handle. +type Decoder struct { + state *C.BrotliDecoderState +} + +// New returns a new Brotli c-decoder handle. +// Close MUST be called to free resources. +func New() Decoder { + return Decoder{state: C.BrotliDecoderCreateInstance(nil, nil, nil)} +} + +// Close frees resources used by decoder. +func (z *Decoder) Close() { + C.BrotliDecoderDestroyInstance(z.state) + z.state = nil +} + +func goStatus(cStatus C.BrotliDecoderResult) (status Status) { + switch cStatus { + case C.BROTLI_DECODER_RESULT_SUCCESS: + return Done + case C.BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT: + return Ok + case C.BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT: + return Ok + } + return Error +} + +// cBytes casts a Go []byte into a C uint8_t*. We pass &buf[0] directly to C, +// which is legal because C doesn't save the pointer longer than the call and +// the byte array itself doesn't contain any pointers. +func cBytes(buf []byte) (*C.uint8_t, C.size_t) { + if len(buf) == 0 { + return (*C.uint8_t)(nil), 0 + } + return (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)) +} + +// DecompressStream reads Brotli-encoded bytes from in, and returns produced +// bytes. Output contents should not be modified. Liveness of output is +// hard-limited by Decoder liveness; slice becomes invalid when any Decoder +// method is invoked. +func (z *Decoder) DecompressStream(in []byte) ( + bytesConsumed int, output []byte, status Status) { + cin, cinSize := cBytes(in) + result := C.DecompressStream(z.state, cin, cinSize) + output = C.GoBytes( + unsafe.Pointer(result.output_data), C.int(result.output_data_size)) + return int(result.bytes_consumed), output, goStatus(result.status) +} diff --git a/go/cbrotli/internal/encoder.go b/go/cbrotli/internal/encoder.go new file mode 100755 index 0000000..b8dd21c --- /dev/null +++ b/go/cbrotli/internal/encoder.go @@ -0,0 +1,135 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Distributed under MIT license. +// See file LICENSE for detail or copy at https://opensource.org/licenses/MIT + +// Package encoder wraps the brotli encoder C API used by package brotli. +package encoder + +/* +#include + +// Wrap BrotliEncoderCompressStream so that it doesn't take variable (in-out) +// pointers. Instead of updated pointer, deltas are saved in auxiliary struct. +struct CompressStreamResult { + size_t bytes_consumed; + const uint8_t* output_data; + size_t output_data_size; + int success; + int has_more; +}; + +struct CompressStreamResult CompressStream( + BrotliEncoderState* s, BrotliEncoderOperation op, + const uint8_t* data, size_t data_size) { + struct CompressStreamResult result; + size_t available_in = data_size; + const uint8_t* next_in = data; + size_t available_out = 0; + result.success = BrotliEncoderCompressStream(s, op, + &available_in, &next_in, &available_out, 0, 0) ? 1 : 0; + result.bytes_consumed = data_size - available_in; + result.output_data = 0; + result.output_data_size = 0; + if (result.success) { + result.output_data = BrotliEncoderTakeOutput(s, &result.output_data_size); + } + result.has_more = BrotliEncoderHasMoreOutput(s) ? 1 : 0; + return result; +} +*/ +import "C" +import ( + "unsafe" +) + +// Operation represents type of request to CompressStream +type Operation int + +const ( + // Process input + Process Operation = iota + // Flush input processed so far + Flush + // Finish stream + Finish +) + +// Status represents internal state after CompressStream invocation +type Status int + +const ( + // Error happened + Error Status = iota + // Done means that no more output will be produced + Done + // Ok means that more output might be produced with no additional input + Ok +) + +// Encoder is the Brotli c-encoder handle. +type Encoder struct { + state *C.BrotliEncoderState +} + +// New returns a new Brotli c-encoder handle. +// quality and lgWin are described in third_party/Brotli/enc/encode.h. +// Close MUST be called to free resources. +func New(quality, lgWin int) Encoder { + state := C.BrotliEncoderCreateInstance(nil, nil, nil) + // TODO(b/18187008): Check if LGBLOCK or MODE are useful to Flywheel. + C.BrotliEncoderSetParameter( + state, C.BROTLI_PARAM_QUALITY, (C.uint32_t)(quality)) + C.BrotliEncoderSetParameter( + state, C.BROTLI_PARAM_LGWIN, (C.uint32_t)(lgWin)) + return Encoder{state} +} + +// Close frees resources used by encoder. +func (z *Encoder) Close() { + C.BrotliEncoderDestroyInstance(z.state) + z.state = nil +} + +// cBytes casts a Go []byte into a C uint8_t*. We pass &buf[0] directly to C, +// which is legal because C doesn't save the pointer longer than the call and +// the byte array itself doesn't contain any pointers. +func cBytes(buf []byte) (*C.uint8_t, C.size_t) { + if len(buf) == 0 { + return (*C.uint8_t)(nil), 0 + } + return (*C.uint8_t)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)) +} + +func cOperation(op Operation) (cOp C.BrotliEncoderOperation) { + switch op { + case Flush: + return C.BROTLI_OPERATION_FLUSH + case Finish: + return C.BROTLI_OPERATION_FINISH + } + return C.BROTLI_OPERATION_PROCESS +} + +// CompressStream processes data and produces Brotli-encoded bytes. Encoder may +// consume considerable amount of input before the first output bytes come out. +// Flush and Finish operations force Encoder to produce output that corresponds +// to input consumed so far. Output contents should not be modified. Liveness of +// output is hard-limited by Encoder liveness; slice becomes invalid when any +// Encoder method is invoked. +func (z *Encoder) CompressStream(in []byte, op Operation) ( + bytesConsumed int, output []byte, status Status) { + cin, cinSize := cBytes(in) + result := C.CompressStream(z.state, cOperation(op), cin, cinSize) + output = C.GoBytes( + unsafe.Pointer(result.output_data), C.int(result.output_data_size)) + var outcome Status + if result.success == 0 { + outcome = Error + } else if result.has_more != 0 { + outcome = Ok + } else { + outcome = Done + } + return int(result.bytes_consumed), output, outcome +} diff --git a/java/org/brotli/dec/BUILD b/java/org/brotli/dec/BUILD index 8b1c9f3..6119041 100755 --- a/java/org/brotli/dec/BUILD +++ b/java/org/brotli/dec/BUILD @@ -39,6 +39,12 @@ java_test( ) java_test( + name = "EnumTest", + test_class = "org.brotli.dec.EnumTest", + runtime_deps = [":test_lib"], +) + +java_test( name = "SynthTest", test_class = "org.brotli.dec.SynthTest", runtime_deps = [":test_lib"], diff --git a/java/org/brotli/dec/EnumTest.java b/java/org/brotli/dec/EnumTest.java new file mode 100755 index 0000000..8e7b593 --- /dev/null +++ b/java/org/brotli/dec/EnumTest.java @@ -0,0 +1,53 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +package org.brotli.dec; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.TreeSet; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Tests for Enum-like classes. + */ +@RunWith(JUnit4.class) +public class EnumTest { + + private void checkEnumClass(Class clazz) { + TreeSet values = new TreeSet(); + for (Field f : clazz.getDeclaredFields()) { + assertEquals("int", f.getType().getName()); + assertEquals(Modifier.FINAL | Modifier.STATIC, f.getModifiers()); + Integer value = null; + try { + value = f.getInt(null); + } catch (IllegalAccessException ex) { + fail("Inaccessible field"); + } + assertFalse(values.contains(value)); + values.add(value); + } + assertEquals(0, values.first().intValue()); + assertEquals(values.size(), values.last() + 1); + } + + @Test + public void testRunningState() { + checkEnumClass(RunningState.class); + } + + @Test + public void testWordTransformType() { + checkEnumClass(WordTransformType.class); + } +} diff --git a/java/org/brotli/dec/RunningState.java b/java/org/brotli/dec/RunningState.java index 4fe22be..692267f 100755 --- a/java/org/brotli/dec/RunningState.java +++ b/java/org/brotli/dec/RunningState.java @@ -9,18 +9,18 @@ package org.brotli.dec; /** * Enumeration of decoding state-machine. */ -enum RunningState { - UNINITIALIZED, - BLOCK_START, - COMPRESSED_BLOCK_START, - MAIN_LOOP, - READ_METADATA, - COPY_UNCOMPRESSED, - INSERT_LOOP, - COPY_LOOP, - COPY_WRAP_BUFFER, - TRANSFORM, - FINISHED, - CLOSED, - WRITE +final class RunningState { + static final int UNINITIALIZED = 0; + static final int BLOCK_START = 1; + static final int COMPRESSED_BLOCK_START = 2; + static final int MAIN_LOOP = 3; + static final int READ_METADATA = 4; + static final int COPY_UNCOMPRESSED = 5; + static final int INSERT_LOOP = 6; + static final int COPY_LOOP = 7; + static final int COPY_WRAP_BUFFER = 8; + static final int TRANSFORM = 9; + static final int FINISHED = 10; + static final int CLOSED = 11; + static final int WRITE = 12; } diff --git a/java/org/brotli/dec/State.java b/java/org/brotli/dec/State.java index 3da3358..c4646a8 100755 --- a/java/org/brotli/dec/State.java +++ b/java/org/brotli/dec/State.java @@ -14,8 +14,8 @@ import java.io.IOException; import java.io.InputStream; final class State { - RunningState runningState = UNINITIALIZED; - RunningState nextRunningState; + int runningState = UNINITIALIZED; + int nextRunningState; final BitReader br = new BitReader(); byte[] ringBuffer; final int[] blockTypeTrees = new int[3 * Huffman.HUFFMAN_MAX_TABLE_SIZE]; diff --git a/java/org/brotli/dec/Transform.java b/java/org/brotli/dec/Transform.java index ecfef45..59a8450 100755 --- a/java/org/brotli/dec/Transform.java +++ b/java/org/brotli/dec/Transform.java @@ -33,10 +33,10 @@ import static org.brotli.dec.WordTransformType.UPPERCASE_FIRST; final class Transform { private final byte[] prefix; - private final WordTransformType type; + private final int type; private final byte[] suffix; - Transform(String prefix, WordTransformType type, String suffix) { + Transform(String prefix, int type, String suffix) { this.prefix = readUniBytes(prefix); this.type = type; this.suffix = readUniBytes(suffix); @@ -188,14 +188,14 @@ final class Transform { } // Copy trimmed word. - WordTransformType op = transform.type; - tmp = op.omitFirst; + int op = transform.type; + tmp = WordTransformType.getOmitFirst(op); if (tmp > len) { tmp = len; } wordOffset += tmp; len -= tmp; - len -= op.omitLast; + len -= WordTransformType.getOmitLast(op); i = len; while (i > 0) { dst[offset++] = word[wordOffset++]; diff --git a/java/org/brotli/dec/WordTransformType.java b/java/org/brotli/dec/WordTransformType.java index 357b978..ed67b51 100755 --- a/java/org/brotli/dec/WordTransformType.java +++ b/java/org/brotli/dec/WordTransformType.java @@ -12,37 +12,34 @@ package org.brotli.dec; *

There are two simple types of transforms: omit X first/last symbols, two character-case * transforms and the identity transform. */ -enum WordTransformType { - IDENTITY(0, 0), - OMIT_LAST_1(0, 1), - OMIT_LAST_2(0, 2), - OMIT_LAST_3(0, 3), - OMIT_LAST_4(0, 4), - OMIT_LAST_5(0, 5), - OMIT_LAST_6(0, 6), - OMIT_LAST_7(0, 7), - OMIT_LAST_8(0, 8), - OMIT_LAST_9(0, 9), - UPPERCASE_FIRST(0, 0), - UPPERCASE_ALL(0, 0), - OMIT_FIRST_1(1, 0), - OMIT_FIRST_2(2, 0), - OMIT_FIRST_3(3, 0), - OMIT_FIRST_4(4, 0), - OMIT_FIRST_5(5, 0), - OMIT_FIRST_6(6, 0), - OMIT_FIRST_7(7, 0), - /* - * brotli specification doesn't use OMIT_FIRST_8(8, 0) transform. - * Probably, it would be used in future format extensions. - */ - OMIT_FIRST_9(9, 0); +final class WordTransformType { + static final int IDENTITY = 0; + static final int OMIT_LAST_1 = 1; + static final int OMIT_LAST_2 = 2; + static final int OMIT_LAST_3 = 3; + static final int OMIT_LAST_4 = 4; + static final int OMIT_LAST_5 = 5; + static final int OMIT_LAST_6 = 6; + static final int OMIT_LAST_7 = 7; + static final int OMIT_LAST_8 = 8; + static final int OMIT_LAST_9 = 9; + static final int UPPERCASE_FIRST = 10; + static final int UPPERCASE_ALL = 11; + static final int OMIT_FIRST_1 = 12; + static final int OMIT_FIRST_2 = 13; + static final int OMIT_FIRST_3 = 14; + static final int OMIT_FIRST_4 = 15; + static final int OMIT_FIRST_5 = 16; + static final int OMIT_FIRST_6 = 17; + static final int OMIT_FIRST_7 = 18; + static final int OMIT_FIRST_8 = 19; + static final int OMIT_FIRST_9 = 20; - final int omitFirst; - final int omitLast; + static int getOmitFirst(int type) { + return type >= OMIT_FIRST_1 ? (type - OMIT_FIRST_1 + 1) : 0; + } - WordTransformType(int omitFirst, int omitLast) { - this.omitFirst = omitFirst; - this.omitLast = omitLast; + static int getOmitLast(int type) { + return type <= OMIT_LAST_9 ? (type - OMIT_LAST_1 + 1) : 0; } } -- cgit v1.1