Verbose CLI + start pulling "Shared-Brotli" (#722)

* Verbose CLI + start pulling "Shared-Brotli"

 * vesbose CLI output; fix #666
 * pull `SHIFT` transforms; currently this is semantically dead code;
   later it will be used by "Shared-Brotli"

9 files changed, 127 insertions, 24 deletions

diff --git a/c/common/transform.c b/c/common/transform.c
index 426e635..c182053 100755
--- a/c/common/transform.c
+++ b/c/common/transform.c
@@ -166,6 +166,7 @@ static BrotliTransforms kBrotliTransforms = {
   kPrefixSuffixMap,
   sizeof(kTransformsData) / (3 * sizeof(kTransformsData[0])),
   kTransformsData,
+  NULL,  /* no extra parameters */
   {0, 12, 27, 23, 42, 63, 56, 48, 59, 64}
 };
 
@@ -190,6 +191,48 @@ static int ToUpperCase(uint8_t* p) {
   return 3;
 }
 
+static int Shift(uint8_t* word, int word_len, uint16_t parameter) {
+  /* Limited sign extension: scalar < (1 << 24). */
+  uint32_t scalar =
+      (parameter & 0x7FFFu) + (0x1000000u - (parameter & 0x8000u));
+  if (word[0] < 0x80) {
+    /* 1-byte rune / 0sssssss / 7 bit scalar (ASCII). */
+    scalar += (uint32_t)word[0];
+    word[0] = (uint8_t)(scalar & 0x7Fu);
+    return 1;
+  } else if (word[0] < 0xC0) {
+    /* Continuation / 10AAAAAA. */
+    return 1;
+  } else if (word[0] < 0xE0) {
+    /* 2-byte rune / 110sssss AAssssss / 11 bit scalar. */
+    if (word_len < 2) return 1;
+    scalar += (uint32_t)((word[1] & 0x3Fu) | ((word[0] & 0x1Fu) << 6u));
+    word[0] = (uint8_t)(0xC0 | ((scalar >> 6u) & 0x1F));
+    word[1] = (uint8_t)((word[1] & 0xC0) | (scalar & 0x3F));
+    return 2;
+  } else if (word[0] < 0xF0) {
+    /* 3-byte rune / 1110ssss AAssssss BBssssss / 16 bit scalar. */
+    if (word_len < 3) return word_len;
+    scalar += (uint32_t)((word[2] & 0x3Fu) | ((word[1] & 0x3Fu) << 6u) |
+        ((word[0] & 0x0Fu) << 12u));
+    word[0] = (uint8_t)(0xE0 | ((scalar >> 12u) & 0x0F));
+    word[1] = (uint8_t)((word[1] & 0xC0) | ((scalar >> 6u) & 0x3F));
+    word[2] = (uint8_t)((word[2] & 0xC0) | (scalar & 0x3F));
+    return 3;
+  } else if (word[0] < 0xF8) {
+    /* 4-byte rune / 11110sss AAssssss BBssssss CCssssss / 21 bit scalar. */
+    if (word_len < 4) return word_len;
+    scalar += (uint32_t)((word[3] & 0x3Fu) | ((word[2] & 0x3Fu) << 6u) |
+        ((word[1] & 0x3Fu) << 12u) | ((word[0] & 0x07u) << 18u));
+    word[0] = (uint8_t)(0xF0 | ((scalar >> 18u) & 0x07));
+    word[1] = (uint8_t)((word[1] & 0xC0) | ((scalar >> 12u) & 0x3F));
+    word[2] = (uint8_t)((word[2] & 0xC0) | ((scalar >> 6u) & 0x3F));
+    word[3] = (uint8_t)((word[3] & 0xC0) | (scalar & 0x3F));
+    return 4;
+  }
+  return 1;
+}
+
 int BrotliTransformDictionaryWord(uint8_t* dst, const uint8_t* word, int len,
     const BrotliTransforms* transforms, int transform_idx) {
   int idx = 0;
@@ -221,6 +264,19 @@ int BrotliTransformDictionaryWord(uint8_t* dst, const uint8_t* word, int len,
         uppercase += step;
         len -= step;
       }
+    } else if (t == BROTLI_TRANSFORM_SHIFT_FIRST) {
+      uint16_t param = (uint16_t)(transforms->params[transform_idx * 2]
+          + (transforms->params[transform_idx * 2 + 1] << 8u));
+      Shift(&dst[idx - len], len, param);
+    } else if (t == BROTLI_TRANSFORM_SHIFT_ALL) {
+      uint16_t param = (uint16_t)(transforms->params[transform_idx * 2]
+          + (transforms->params[transform_idx * 2 + 1] << 8u));
+      uint8_t* shift = &dst[idx - len];
+      while (len > 0) {
+        int step = Shift(shift, len, param);
+        shift += step;
+        len -= step;
+      }
     }
   }
   {
diff --git a/c/common/transform.h b/c/common/transform.h
index 456c12d..b6f86cc 100755
--- a/c/common/transform.h
+++ b/c/common/transform.h
@@ -37,6 +37,8 @@ enum BrotliWordTransformType {
   BROTLI_TRANSFORM_OMIT_FIRST_7 = 18,
   BROTLI_TRANSFORM_OMIT_FIRST_8 = 19,
   BROTLI_TRANSFORM_OMIT_FIRST_9 = 20,
+  BROTLI_TRANSFORM_SHIFT_FIRST = 21,
+  BROTLI_TRANSFORM_SHIFT_ALL = 22,
   BROTLI_NUM_TRANSFORM_TYPES  /* Counts transforms, not a transform itself. */
 };
 
@@ -50,6 +52,9 @@ typedef struct BrotliTransforms {
   uint32_t num_transforms;
   /* Each entry is a [prefix_id, transform, suffix_id] triplet. */
   const uint8_t* transforms;
+  /* Shift for BROTLI_TRANSFORM_SHIFT_FIRST and BROTLI_TRANSFORM_SHIFT_ALL,
+     must be NULL if and only if no such transforms are present. */
+  const uint8_t* params;
   /* Indices of transforms like ["", BROTLI_TRANSFORM_OMIT_LAST_#, ""].
      0-th element corresponds to ["", BROTLI_TRANSFORM_IDENTITY, ""].
      -1, if cut-off transform does not exist. */
diff --git a/c/enc/backward_references.h b/c/enc/backward_references.h
index 3a41466..f82a80d 100644
--- a/c/enc/backward_references.h
+++ b/c/enc/backward_references.h
@@ -25,9 +25,9 @@ extern "C" {
    initially the total amount of commands output by previous
    CreateBackwardReferences calls, and must be incremented by the amount written
    by this call. */
-BROTLI_INTERNAL void BrotliCreateBackwardReferences(
-    size_t num_bytes, size_t position, const uint8_t* ringbuffer,
-    size_t ringbuffer_mask, const BrotliEncoderParams* params,
+BROTLI_INTERNAL void BrotliCreateBackwardReferences(size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params,
     HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals);
 
diff --git a/c/enc/backward_references_hq.c b/c/enc/backward_references_hq.c
index 96b0e70..5737f75 100644
--- a/c/enc/backward_references_hq.c
+++ b/c/enc/backward_references_hq.c
@@ -419,8 +419,8 @@ static size_t UpdateNodes(
   size_t k;
   size_t gap = 0;
 
-  EvaluateNode(block_start, pos, max_backward_limit, gap, starting_dist_cache,
-      model, queue, nodes);
+  EvaluateNode(block_start, pos, max_backward_limit, gap,
+      starting_dist_cache, model, queue, nodes);
 
   {
     const PosData* posdata = StartPosQueueAt(queue, 0);
@@ -587,9 +587,10 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
     {
       size_t distance = ZopfliNodeCopyDistance(next);
       size_t len_code = ZopfliNodeLengthCode(next);
-      size_t max_distance =
-          BROTLI_MIN(size_t, block_start + pos, max_backward_limit);
-      BROTLI_BOOL is_dictionary = TO_BROTLI_BOOL(distance > max_distance + gap);
+      size_t max_distance = BROTLI_MIN(size_t,
+          block_start + pos, max_backward_limit);
+      BROTLI_BOOL is_dictionary =
+          TO_BROTLI_BOOL(distance > max_distance + gap);
       size_t dist_code = ZopfliNodeDistanceCode(next);
       InitCommand(&commands[i], &params->dist, insert_length,
           copy_length, (int)len_code - (int)copy_length, dist_code);
diff --git a/c/enc/backward_references_hq.h b/c/enc/backward_references_hq.h
index 1e4275d..fb1ff3f 100644
--- a/c/enc/backward_references_hq.h
+++ b/c/enc/backward_references_hq.h
@@ -23,14 +23,16 @@ extern "C" {
 #endif
 
 BROTLI_INTERNAL void BrotliCreateZopfliBackwardReferences(MemoryManager* m,
-    size_t num_bytes, size_t position, const uint8_t* ringbuffer,
-    size_t ringbuffer_mask, const BrotliEncoderParams* params,
+    size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params,
     HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals);
 
 BROTLI_INTERNAL void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
-    size_t num_bytes, size_t position, const uint8_t* ringbuffer,
-    size_t ringbuffer_mask, const BrotliEncoderParams* params,
+    size_t num_bytes,
+    size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
+    const BrotliEncoderParams* params,
     HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
     Command* commands, size_t* num_commands, size_t* num_literals);
 
diff --git a/c/enc/backward_references_inc.h b/c/enc/backward_references_inc.h
index c18cdb0..e29daf3 100644
--- a/c/enc/backward_references_inc.h
+++ b/c/enc/backward_references_inc.h
@@ -60,7 +60,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
         FN(FindLongestMatch)(hasher,
             &params->dictionary,
             ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
-            max_distance, gap, params->dist.max_distance, &sr2);
+            max_distance, gap, params->dist.max_distance,
+            &sr2);
         if (sr2.score >= sr.score + cost_diff_lazy) {
           /* Ok, let's just write one byte for now and start a match from the
              next byte. */
@@ -76,7 +77,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
       }
       apply_random_heuristics =
           position + 2 * sr.len + random_heuristics_window_size;
-      max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
+      max_distance = BROTLI_MIN(size_t,
+          position, max_backward_limit);
       {
         /* The first 16 codes are special short-codes,
            and the minimum offset is 1. */
diff --git a/c/enc/encode.c b/c/enc/encode.c
index 141e70a..c82f2d3 100644
--- a/c/enc/encode.c
+++ b/c/enc/encode.c
@@ -114,8 +114,6 @@ typedef struct BrotliEncoderStateStruct {
   BROTLI_BOOL is_initialized_;
 } BrotliEncoderStateStruct;
 
-static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s);
-
 static size_t InputBlockSize(BrotliEncoderState* s) {
   return (size_t)1 << s->params.lgblock;
 }
diff --git a/c/enc/encoder_dict.c b/c/enc/encoder_dict.c
index 8b2f6ad..e58ca67 100755
--- a/c/enc/encoder_dict.c
+++ b/c/enc/encoder_dict.c
@@ -24,7 +24,6 @@ void BrotliInitEncoderDictionary(BrotliEncoderDictionary* dict) {
 
   dict->cutoffTransformsCount = kCutoffTransformsCount;
   dict->cutoffTransforms = kCutoffTransforms;
-
 }
 
 #if defined(__cplusplus) || defined(c_plusplus)
diff --git a/c/tools/brotli.c b/c/tools/brotli.c
index ce05b64..838539a 100644
--- a/c/tools/brotli.c
+++ b/c/tools/brotli.c
@@ -86,10 +86,10 @@ typedef struct {
   /* Parameters */
   int quality;
   int lgwin;
+  int verbosity;
   BROTLI_BOOL force_overwrite;
   BROTLI_BOOL junk_source;
   BROTLI_BOOL copy_stat;
-  BROTLI_BOOL verbose;
   BROTLI_BOOL write_to_stdout;
   BROTLI_BOOL test_integrity;
   BROTLI_BOOL decompress;
@@ -121,6 +121,12 @@ typedef struct {
   const uint8_t* next_in;
   size_t available_out;
   uint8_t* next_out;
+
+  /* Reporting */
+  /* size_t would be large enough,
+     until 4GiB+ files are compressed / decompressed on 32-bit CPUs. */
+  size_t total_in;
+  size_t total_out;
 } Context;
 
 /* Parse up to 5 decimal digits. */
@@ -279,11 +285,11 @@ static Command ParseParams(Context* params) {
           command = COMMAND_TEST_INTEGRITY;
           continue;
         } else if (c == 'v') {
-          if (params->verbose) {
+          if (params->verbosity > 0) {
             fprintf(stderr, "argument --verbose / -v already set\n");
             return COMMAND_INVALID;
           }
-          params->verbose = BROTLI_TRUE;
+          params->verbosity = 1;
           continue;
         } else if (c == 'V') {
           /* Don't parse further. */
@@ -415,11 +421,11 @@ static Command ParseParams(Context* params) {
         command_set = BROTLI_TRUE;
         command = COMMAND_TEST_INTEGRITY;
       } else if (strcmp("verbose", arg) == 0) {
-        if (params->verbose) {
+        if (params->verbosity > 0) {
           fprintf(stderr, "argument --verbose / -v already set\n");
           return COMMAND_INVALID;
         }
-        params->verbose = BROTLI_TRUE;
+        params->verbosity = 1;
       } else if (strcmp("version", arg) == 0) {
         /* Don't parse further. */
         return COMMAND_VERSION;
@@ -787,6 +793,8 @@ static void InitializeBuffers(Context* context) {
   context->next_in = NULL;
   context->available_out = kFileBufferSize;
   context->next_out = context->output;
+  context->total_in = 0;
+  context->total_out = 0;
 }
 
 static BROTLI_BOOL HasMoreInput(Context* context) {
@@ -796,6 +804,7 @@ static BROTLI_BOOL HasMoreInput(Context* context) {
 static BROTLI_BOOL ProvideInput(Context* context) {
   context->available_in =
       fread(context->input, 1, kFileBufferSize, context->fin);
+  context->total_in += context->available_in;
   context->next_in = context->input;
   if (ferror(context->fin)) {
     fprintf(stderr, "failed to read input [%s]: %s\n",
@@ -808,6 +817,7 @@ static BROTLI_BOOL ProvideInput(Context* context) {
 /* Internal: should be used only in Provide-/Flush-Output. */
 static BROTLI_BOOL WriteOutput(Context* context) {
   size_t out_size = (size_t)(context->next_out - context->output);
+  context->total_out += out_size;
   if (out_size == 0) return BROTLI_TRUE;
   if (context->test_integrity) return BROTLI_TRUE;
 
@@ -833,6 +843,25 @@ static BROTLI_BOOL FlushOutput(Context* context) {
   return BROTLI_TRUE;
 }
 
+static void PrintBytes(size_t value) {
+  if (value < 1024) {
+    fprintf(stderr, "%d B", (int)value);
+  } else if (value < 1048576) {
+    fprintf(stderr, "%0.3f KiB", (double)value / 1024.0);
+  } else if (value < 1073741824) {
+    fprintf(stderr, "%0.3f MiB", (double)value / 1048576.0);
+  } else {
+    fprintf(stderr, "%0.3f GiB", (double)value / 1073741824.0);
+  }
+}
+
+static void PrintFileProcessingProgress(Context* context) {
+  fprintf(stderr, "[%s]: ", PrintablePath(context->current_input_path));
+  PrintBytes(context->total_in);
+  fprintf(stderr, " -> ");
+  PrintBytes(context->total_out);
+}
+
 static BROTLI_BOOL DecompressFile(Context* context, BrotliDecoderState* s) {
   BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;
   InitializeBuffers(context);
@@ -853,6 +882,11 @@ static BROTLI_BOOL DecompressFile(Context* context, BrotliDecoderState* s) {
                 PrintablePath(context->current_input_path));
         return BROTLI_FALSE;
       }
+      if (context->verbosity > 0) {
+        fprintf(stderr, "Decompressed ");
+        PrintFileProcessingProgress(context);
+        fprintf(stderr, "\n");
+      }
       return BROTLI_TRUE;
     } else {
       fprintf(stderr, "corrupt input [%s]\n",
@@ -915,7 +949,13 @@ static BROTLI_BOOL CompressFile(Context* context, BrotliEncoderState* s) {
     }
 
     if (BrotliEncoderIsFinished(s)) {
-      return FlushOutput(context);
+      if (!FlushOutput(context)) return BROTLI_FALSE;
+      if (context->verbosity > 0) {
+        fprintf(stderr, "Compressed ");
+        PrintFileProcessingProgress(context);
+        fprintf(stderr, "\n");
+      }
+      return BROTLI_TRUE;
     }
   }
 }
@@ -979,11 +1019,11 @@ int main(int argc, char** argv) {
 
   context.quality = 11;
   context.lgwin = -1;
+  context.verbosity = 0;
   context.force_overwrite = BROTLI_FALSE;
   context.junk_source = BROTLI_FALSE;
   context.copy_stat = BROTLI_TRUE;
   context.test_integrity = BROTLI_FALSE;
-  context.verbose = BROTLI_FALSE;
   context.write_to_stdout = BROTLI_FALSE;
   context.decompress = BROTLI_FALSE;
   context.large_window = BROTLI_FALSE;