new, more ergonomic, exported C functions, plus a transparent fallbac…

…k from multithreaded to single threaded if num_threads=1

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brotli"
-version = "3.0.3"
+version = "3.1.0"
 authors = ["Daniel Reiter Horn <danielrh@dropbox.com>", "The Brotli Authors"]
 description = "A brotli compressor and decompressor that with an interface avoiding the rust stdlib. This makes it suitable for embedded devices and kernels. It is designed with a pluggable allocator so that the standard lib's allocator may be employed. The default build also includes a stdlib allocator and stream interface. Disable this with --features=no-stdlib. All included code is safe."
 license = "BSD-3-Clause/MIT"

c/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "brotli-ffi"
-version = "1.0.1"
+version = "1.1.0"
 authors = ["Daniel Reiter Horn <danielrh@dropbox.com>", "The Brotli Authors"]
 description = "A brotli compressor and decompressor that with an interface exactly matching https://github.com/google/brotli. All included code is safe except the thin exported C-compatible functions."
 license = "BSD-3-Clause/MIT"
@@ -19,7 +19,7 @@ crate-type=["cdylib", "staticlib", "rlib"]
 lto=true
 
 [dependencies]
-"brotli" = {version="~3.0", default-features=false}
+"brotli" = {version="~3.1", default-features=false}
 
 [features]
 validation=["brotli/validation"]

c/brotli/broccoli.h

@@ -32,7 +32,17 @@ BroccoliResult BroccoliConcatStream(
     size_t *available_out,
     uint8_t **output_buf_ptr);
 
+BroccoliResult BroccoliConcatStreaming(
+    BroccoliState *state,
+    size_t *available_in,
+    const uint8_t *input_buf_ptr,
+    size_t *available_out,
+    uint8_t *output_buf_ptr);
+
 BroccoliResult BroccoliConcatFinish(BroccoliState * state,
                               size_t *available_out,
                               uint8_t**output_buf);
+BroccoliResult BroccoliConcatFinished(BroccoliState * state,
+                              size_t *available_out,
+                              uint8_t*output_buf);
 #endif

c/brotli/decode.h

@@ -118,6 +118,21 @@ typedef enum {
 #undef BROTLI_ERROR_CODE_ENUM_ITEM_
 #undef BROTLI_COMMA_
 
+typedef struct HuffmanCodeStruct {
+    uint16_t value;
+    uint8_t bits;
+} HuffmanCode;
+
+
+
+typedef struct BrotliDecoderReturnInfoStruct {
+    size_t decoded_size;
+    char error[256];
+    BrotliDecoderResult result;
+    BrotliDecoderErrorCode code;
+} BrotliDecoderReturnInfo;
+
+
 /**
  * The value of the last error code, negative integer.
  *
@@ -205,6 +220,25 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
     size_t* decoded_size,
     uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(*decoded_size)]);
 
+BROTLI_DEC_API BrotliDecoderReturnInfo BrotliDecoderDecompressWithReturnInfo(
+    size_t encoded_size,
+    const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
+    size_t decoded_size,
+    uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(decoded_size)]);
+
+BROTLI_DEC_API BrotliDecoderReturnInfo BrotliDecoderDecompressPrealloc(
+    size_t encoded_size,
+    const uint8_t encoded_buffer[BROTLI_ARRAY_PARAM(encoded_size)],
+    size_t decoded_size,
+    uint8_t decoded_buffer[BROTLI_ARRAY_PARAM(decoded_size)],
+    size_t scratch_u8_size,
+    uint8_t scratch_u8_buffer[BROTLI_ARRAY_PARAM(scratch_u8_size)],
+    size_t scratch_u32_size,
+    uint32_t scratch_u32_buffer[BROTLI_ARRAY_PARAM(scratch_u32_size)],
+    size_t scratch_hc_size,
+    HuffmanCode scratch_hc_buffer[BROTLI_ARRAY_PARAM(scratch_hc_size)]
+    );
+
 /**
  * Decompresses the input stream to the output stream.
  *
@@ -247,6 +281,10 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStream(
   BrotliDecoderState* state, size_t* available_in, const uint8_t** next_in,
   size_t* available_out, uint8_t** next_out, size_t* total_out);
 
+BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompressStreaming(
+  BrotliDecoderState* state, size_t* available_in, const uint8_t* next_in,
+  size_t* available_out, uint8_t* next_out);
+
 /**
  * Checks if decoder has more output.
  *

c/brotli/encode.h

@@ -384,6 +384,10 @@ BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStream(
     const uint8_t** next_in, size_t* available_out, uint8_t** next_out,
     size_t* total_out);
 
+BROTLI_ENC_API BROTLI_BOOL BrotliEncoderCompressStreaming(
+    BrotliEncoderState* state, BrotliEncoderOperation op, size_t* available_in,
+    const uint8_t* next_in, size_t* available_out, uint8_t* next_out);
+
 /**
  * Checks if encoder instance reached the final state.
  *

c/go/interface_test.go

@@ -80,6 +80,39 @@ func TestCompressRoundtrip(*testing.T) {
 	}
 }
 
+func TestCompressRoundtripMulti(*testing.T) {
+	tmp := testData()
+	data := tmp[:len(tmp)-17]
+	outBuffer := bytes.NewBuffer(nil)
+	var options = brotli.CompressionOptions{
+		NumThreads: 16,
+		Quality:    9,
+		Catable:    true,
+		Appendable: true,
+		Magic:      true,
+	}
+	writer := brotli.NewMultiCompressionWriter(
+		brotli.NewDecompressionWriter(
+			outBuffer,
+		),
+		options,
+	)
+	_, err := writer.Write(data[:])
+	if err != nil {
+		panic(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		panic(err)
+	}
+	if len(outBuffer.Bytes()) == 0 {
+		panic("Zero output buffer")
+	}
+	if !bytes.Equal(outBuffer.Bytes(), data[:]) {
+		panic(fmt.Sprintf("Bytes not equal %d, %d", len(outBuffer.Bytes()), len(data)))
+	}
+}
+
 func TestRejectCorruptBuffers(*testing.T) {
 	tmp := testData()
 	data := tmp[:len(tmp)-17]

c/py/brotli.py

@@ -1,7 +1,7 @@
 import ctypes
 import sys
 
-from ctypes import c_uint, pointer, POINTER, c_size_t, c_void_p, c_uint32, c_ubyte, c_char_p, byref
+from ctypes import c_uint, c_int, pointer, POINTER, c_size_t, c_void_p, c_uint32, c_ubyte, c_char_p, byref
 class BrotliEncoderWorkPool(ctypes.Structure):
     pass
 BrotliEncoderWorkPool= ctypes.POINTER(BrotliEncoderWorkPool)
@@ -15,6 +15,7 @@ class BrotliEncoderWorkPool(ctypes.Structure):
             brotli_library=ctypes.CDLL("target/release/libbrotli_ffi.dylib")
         except OSError:
             brotli_library=ctypes.CDLL("target/release/libbrotli_ffi.so")
+
 _BrotliEncoderCreateWorkPool = brotli_library.BrotliEncoderCreateWorkPool
 _BrotliEncoderCreateWorkPool.restype = POINTER(BrotliEncoderWorkPool)
 _BrotliEncoderCompressWorkPool = brotli_library.BrotliEncoderCompressWorkPool
@@ -23,6 +24,14 @@ class BrotliDecoderState(ctypes.Structure):
     pass
 class BrotliDecompressorException(Exception):
     pass
+class BrotliDecoderReturnInfo(ctypes.Structure):
+    _fields_ = [('decoded_size', c_size_t),
+                ('error_string', c_ubyte * 256),
+                ('result', c_int),
+                ('error_code', c_int),
+    ]
+BrotliDecoderDecompressWithReturnInfo = brotli_library.BrotliDecoderDecompressWithReturnInfo
+BrotliDecoderDecompressWithReturnInfo.restype = BrotliDecoderReturnInfo
 _BrotliDecoderCreateInstance = brotli_library.BrotliDecoderCreateInstance
 _BrotliDecoderCreateInstance.restype = POINTER(BrotliDecoderState)
 _BrotliDecoderDestroyInstance = brotli_library.BrotliDecoderDestroyInstance
@@ -127,20 +136,20 @@ def BrotliDecode(any_input, expected_size=4096 * 1024, max_expected_size = 256 *
         decoded_size = c_size_t(expected_size)
         decoded = (c_ubyte * decoded_size.value)()
 
-        res = brotli_library.BrotliDecoderDecompress(len(input),
-                                                     input,
-                                                     byref(decoded_size),
-                                                     byref(decoded))
-        if res == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
+        res = BrotliDecoderDecompressWithReturnInfo(len(input),
+                                                    input,
+                                                    decoded_size,
+                                                    byref(decoded))
+        if res.result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
             raise BrotliDecompressorException("EarlyEOF")
-        elif res == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
-            expected_size *= 2
-        elif res == BROTLI_DECODER_RESULT_SUCCESS:
-            return bytearray(decoded[:decoded_size.value])
-        else:
+        elif res.result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
             expected_size *= 2
             if expected_size > max_expected_size:
                 raise BrotliDecompressorException("Brotli file > " + str(max_expected_size) + " or corrupt brotli file")
+        elif res.result == BROTLI_DECODER_RESULT_SUCCESS:
+            return bytearray(decoded[:res.decoded_size])
+        else:
+            raise BrotliDecompressorException(''.join(chr(x) for x in res.error_string))
 
 
 def BrotliCompress(

c/py/brotli_test.py

@@ -45,6 +45,17 @@ def test_tiny_alloc(self):
         rt = BrotliDecode(output, 2)
         assert rt == self.test_data
         assert len(output) < 1024 * 1024
+    def test_single_thread(self):
+        output = BrotliCompress(self.test_data,
+                                {
+                                    BROTLI_PARAM_QUALITY:5,
+                                    BROTLI_PARAM_CATABLE:1,
+                                    BROTLI_PARAM_MAGIC_NUMBER:1,
+                                },
+                                1)
+        rt = BrotliDecode(output, 2)
+        assert rt == self.test_data
+        assert len(output) < 1024 * 1024
     def test_memory_view(self):
         output = BrotliCompress(memoryview(self.test_data),
                                 {
@@ -82,7 +93,7 @@ def test_rnd(self):
         rt = BrotliDecode(output)
         assert rt == random_data
         assert len(output) > 130000
-    def test_1(self):
+    def test_truncation(self):
         output = BrotliCompress(self.test_data[:65536],
                                 {
                                     BROTLI_PARAM_QUALITY:6,
@@ -100,6 +111,24 @@ def test_1(self):
             pass
         else:
             assert False, "Should have errored"
+    def test_corruption(self):
+        output = BrotliCompress(self.test_data[:65536],
+                                {
+                                    BROTLI_PARAM_QUALITY:6,
+                                    BROTLI_PARAM_CATABLE:1,
+                                    BROTLI_PARAM_MAGIC_NUMBER:1,
+                                },
+                                8)
+        corrupt = output[:len(output)/2] + output[len(output)/2 + 1:]
+        rt = BrotliDecode(output)
+        assert rt == self.test_data[:65536]
+        assert len(output) < 1024 * 1024
+        try:
+            BrotliDecode(corrupt)
+        except BrotliDecompressorException:
+            pass
+        else:
+            assert False, "Should have errored"
 if __name__ == '__main__':
     unittest.main()
 

src/enc/encode.rs

@@ -3530,7 +3530,7 @@ pub fn BrotliEncoderCompressStream<Alloc: BrotliAlloc,
   1i32
 }
 
-pub fn BrotliEncoderIsFinished<Alloc:BrotliAlloc>(s: &mut BrotliEncoderStateStruct<Alloc>) -> i32 {
+pub fn BrotliEncoderIsFinished<Alloc:BrotliAlloc>(s: &BrotliEncoderStateStruct<Alloc>) -> i32 {
   if !!((*s).stream_state_ as (i32) == BrotliEncoderStreamState::BROTLI_STREAM_FINISHED as (i32) &&
         (BrotliEncoderHasMoreOutput(s) == 0)) {
     1i32

src/ffi/broccoli.rs

@@ -92,6 +92,22 @@ pub unsafe extern fn BroccoliConcatStream(
   ret
 }
 
+#[no_mangle]
+pub unsafe extern fn BroccoliConcatStreaming(
+  state: *mut BroccoliState,
+  available_in: *mut usize,
+  mut input_buf: *const u8,
+  available_out: *mut usize,
+  mut output_buf: *mut u8) -> BroccoliResult {
+  BroccoliConcatStream(
+    state,
+    available_in,
+    &mut input_buf,
+    available_out,
+    &mut output_buf)
+
+}
+
 #[no_mangle]
 pub unsafe extern fn BroccoliConcatFinish(
   state: *mut BroccoliState,
@@ -106,3 +122,15 @@ pub unsafe extern fn BroccoliConcatFinish(
   *state = BroccoliState::from(bro_catli);
   ret
 }
+
+// exactly the same as BrotliConcatFinish but without the indirect
+#[no_mangle]
+pub unsafe extern fn BroccoliConcatFinished(
+  state: *mut BroccoliState,
+  available_out: *mut usize,
+  mut output_buf: *mut u8) -> BroCatliResult {
+  BroccoliConcatFinish(
+    state,
+    available_out,
+    &mut output_buf)
+}

src/ffi/compressor.rs

@@ -237,6 +237,26 @@ pub unsafe extern fn BrotliEncoderCompress(
     },
   }
 }
+
+#[no_mangle]
+pub unsafe extern fn BrotliEncoderCompressStreaming(
+  state_ptr: *mut BrotliEncoderState,
+  op: BrotliEncoderOperation,
+  available_in: *mut usize,
+  mut input_buf: *const u8,
+  available_out: *mut usize,
+  mut output_buf: *mut u8,
+) -> i32 {
+  BrotliEncoderCompressStream(state_ptr,
+                              op,
+                              available_in,
+                              &mut input_buf,
+                              available_out,
+                              &mut output_buf,
+                              core::ptr::null_mut())
+
+}
+
 #[no_mangle]
 pub unsafe extern fn BrotliEncoderCompressStream(
   state_ptr: *mut BrotliEncoderState,