Merge pull request #114 from Cyan4973/dev

Dev

NEWS

@@ -1,3 +1,8 @@
+v0.4.7
+Improved : small compression speed improvement in HC mode
+Changed : `zstd_decompress.c` has ZSTD_LEGACY_SUPPORT to 0 by default
+fix : bt search bug
+
 v0.4.6
 fix : fast compression mode on Windows
 New : cmake configuration file, thanks to Artyom Dymchenko

contrib/cmake/lib/CMakeLists.txt

@@ -66,7 +66,8 @@ SET(Sources
 
 SET(Headers
         ${LIBRARY_DIR}/bitstream.h
-        ${LIBRARY_DIR}/error.h
+        ${LIBRARY_DIR}/error_private.h
+        ${LIBRARY_DIR}/error_public.h
         ${LIBRARY_DIR}/fse.h
         ${LIBRARY_DIR}/fse_static.h
         ${LIBRARY_DIR}/huff0.h

lib/bitstream.h

@@ -50,8 +50,8 @@ extern "C" {
 /******************************************
 *  Includes
 ******************************************/
-#include "mem.h"     /* unaligned access routines */
-#include "error.h"   /* error codes and messages */
+#include "mem.h"            /* unaligned access routines */
+#include "error_private.h"  /* error codes and messages */
 
 
 /********************************************

lib/error.h → lib/error_private.h

@@ -1,6 +1,6 @@
 /* ******************************************************************
    Error codes and messages
-   Copyright (C) 2013-2015, Yann Collet
+   Copyright (C) 2013-2016, Yann Collet
 
    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
 
@@ -28,9 +28,10 @@
    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
    You can contact the author at :
-   - Source repository : https://github.com/Cyan4973/FiniteStateEntropy
-   - Public forum : https://groups.google.com/forum/#!forum/lz4c
+   - Source repository : https://github.com/Cyan4973/zstd
 ****************************************************************** */
+/* Note : this module is expected to remain private, do not expose it */
+
 #ifndef ERROR_H_MODULE
 #define ERROR_H_MODULE
 
@@ -39,56 +40,65 @@ extern "C" {
 #endif
 
 
-/******************************************
+/* *****************************************
 *  Includes
 ******************************************/
-#include <stddef.h>    /* size_t, ptrdiff_t */
+#include <stddef.h>        /* size_t, ptrdiff_t */
+#include "error_public.h"  /* enum list */
 
 
-/******************************************
+/* *****************************************
 *  Compiler-specific
 ******************************************/
-#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#if defined(__GNUC__)
+#  define ERR_STATIC static __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
 #  define ERR_STATIC static inline
 #elif defined(_MSC_VER)
 #  define ERR_STATIC static __inline
-#elif defined(__GNUC__)
-#  define ERR_STATIC static __attribute__((unused))
 #else
 #  define ERR_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
 #endif
 
 
-/******************************************
-*  Error Management
+/* *****************************************
+*  Error Codes
 ******************************************/
 #define PREFIX(name) ZSTD_error_##name
 
+#ifdef ERROR
+#  undef ERROR   /* reported already defined on VS 2015 by Rich Geldreich */
+#endif
 #define ERROR(name) (size_t)-PREFIX(name)
 
-#define ERROR_LIST(ITEM) \
-        ITEM(PREFIX(No_Error)) ITEM(PREFIX(GENERIC)) \
-        ITEM(PREFIX(prefix_unknown)) ITEM(PREFIX(frameParameter_unsupported)) ITEM(PREFIX(frameParameter_unsupportedBy32bitsImplementation)) \
-        ITEM(PREFIX(init_missing)) ITEM(PREFIX(memory_allocation)) ITEM(PREFIX(stage_wrong)) \
-        ITEM(PREFIX(dstSize_tooSmall)) ITEM(PREFIX(srcSize_wrong)) \
-        ITEM(PREFIX(corruption_detected)) \
-        ITEM(PREFIX(tableLog_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooLarge)) ITEM(PREFIX(maxSymbolValue_tooSmall)) \
-        ITEM(PREFIX(maxCode))
-
-#define ERROR_GENERATE_ENUM(ENUM) ENUM,
-typedef enum { ERROR_LIST(ERROR_GENERATE_ENUM) } ERR_codes;  /* enum is exposed, to detect & handle specific errors; compare function result to -enum value */
+ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
 
-#define ERROR_CONVERTTOSTRING(STRING) #STRING,
-#define ERROR_GENERATE_STRING(EXPR) ERROR_CONVERTTOSTRING(EXPR)
-static const char* ERR_strings[] = { ERROR_LIST(ERROR_GENERATE_STRING) };
 
-ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
+/* *****************************************
+*  Error Strings
+******************************************/
 
 ERR_STATIC const char* ERR_getErrorName(size_t code)
 {
     static const char* codeError = "Unspecified error code";
-    if (ERR_isError(code)) return ERR_strings[-(int)(code)];
-    return codeError;
+    switch( (size_t)(0-code) )
+    {
+    case ZSTD_error_No_Error: return "No error detected";
+    case ZSTD_error_GENERIC:  return "Error (generic)";
+    case ZSTD_error_prefix_unknown: return "Unknown frame descriptor";
+    case ZSTD_error_frameParameter_unsupported: return "Unsupported frame parameter";
+    case ZSTD_error_frameParameter_unsupportedBy32bitsImplementation: return "Frame parameter unsupported in 32-bits mode";
+    case ZSTD_error_init_missing: return "Context should be init first";
+    case ZSTD_error_memory_allocation: return "Allocation error : not enough memory";
+    case ZSTD_error_dstSize_tooSmall: return "Destination buffer is too small";
+    case ZSTD_error_srcSize_wrong: return "Src size incorrect";
+    case ZSTD_error_corruption_detected: return "Corrupted block detected";
+    case ZSTD_error_tableLog_tooLarge: return "tableLog requires too much memory";
+    case ZSTD_error_maxSymbolValue_tooLarge: return "Unsupported max possible Symbol Value : too large";
+    case ZSTD_error_maxSymbolValue_tooSmall: return "Specified maxSymbolValue is too small";
+    case ZSTD_error_maxCode:
+    default: return codeError;
+    }
 }
 
 

lib/error_public.h

@@ -0,0 +1,70 @@
+/* ******************************************************************
+   Error codes list
+   Copyright (C) 2016, Yann Collet
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - Source repository : https://github.com/Cyan4973/zstd
+****************************************************************** */
+#ifndef ERROR_PUBLIC_H_MODULE
+#define ERROR_PUBLIC_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/* ****************************************
+*  error list
+******************************************/
+enum {
+  ZSTD_error_No_Error,
+  ZSTD_error_GENERIC,
+  ZSTD_error_prefix_unknown,
+  ZSTD_error_frameParameter_unsupported,
+  ZSTD_error_frameParameter_unsupportedBy32bitsImplementation,
+  ZSTD_error_init_missing,
+  ZSTD_error_memory_allocation,
+  ZSTD_error_stage_wrong,
+  ZSTD_error_dstSize_tooSmall,
+  ZSTD_error_srcSize_wrong,
+  ZSTD_error_corruption_detected,
+  ZSTD_error_tableLog_tooLarge,
+  ZSTD_error_maxSymbolValue_tooLarge,
+  ZSTD_error_maxSymbolValue_tooSmall,
+  ZSTD_error_maxCode
+};
+
+/* note : functions provide error codes in reverse negative order,
+          so compare with (size_t)(0-enum) */
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ERROR_PUBLIC_H_MODULE */

lib/legacy/zstd_legacy.h

@@ -40,8 +40,8 @@ extern "C" {
 /* *************************************
 *  Includes
 ***************************************/
-#include "mem.h"        /* MEM_STATIC */
-#include "error.h"      /* ERROR */
+#include "mem.h"            /* MEM_STATIC */
+#include "error_private.h"  /* ERROR */
 #include "zstd_v01.h"
 #include "zstd_v02.h"
 #include "zstd_v03.h"
@@ -51,7 +51,7 @@ MEM_STATIC unsigned ZSTD_isLegacy (U32 magicNumberLE)
 	switch(magicNumberLE)
 	{
 		case ZSTDv01_magicNumberLE :
-		case ZSTDv02_magicNumber : 
+		case ZSTDv02_magicNumber :
 		case ZSTDv03_magicNumber : return 1;
 		default : return 0;
 	}

lib/legacy/zstd_v01.c

@@ -1630,6 +1630,7 @@ static size_t ZSTD_decodeLiteralsBlock(void* ctx,
             ip += litcSize;
             break;
         }
+    case bt_end:
     default:
         return (size_t)-ZSTD_ERROR_GENERIC;
     }

lib/zstd.h

@@ -44,7 +44,7 @@ extern "C" {
 
 
 /* ***************************************************************
-*  Tuning parameters
+*  Export parameters
 *****************************************************************/
 /*!
 *  ZSTD_DLL_EXPORT :
@@ -62,7 +62,7 @@ extern "C" {
 ***************************************/
 #define ZSTD_VERSION_MAJOR    0    /* for breaking interface changes  */
 #define ZSTD_VERSION_MINOR    4    /* for new (non-breaking) interface capabilities */
-#define ZSTD_VERSION_RELEASE  6    /* for tweaks, bug-fixes, or development */
+#define ZSTD_VERSION_RELEASE  7    /* for tweaks, bug-fixes, or development */
 #define ZSTD_VERSION_NUMBER  (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
 ZSTDLIB_API unsigned ZSTD_versionNumber (void);
 

lib/zstd_buffered.c

@@ -39,7 +39,7 @@
 *  Includes
 ***************************************/
 #include <stdlib.h>
-#include "error.h"
+#include "error_private.h"
 #include "zstd_static.h"
 #include "zstd_buffered_static.h"
 
@@ -243,6 +243,8 @@ static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
                 zbc->stage = ZBUFFcs_load;
                 break;
             }
+        default:
+            return ERROR(GENERIC);   /* impossible */
         }
     }
 
@@ -534,6 +536,7 @@ size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbc, void* dst, size_t* maxDstSizePt
                 notDone = 0;
                 break;
             }
+        default: return ERROR(GENERIC);   /* impossible */
         }
     }
 

lib/zstd_compress.c

@@ -39,7 +39,6 @@
 #  define FORCE_INLINE static __forceinline
 #  include <intrin.h>                    /* For Visual 2005 */
 #  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
-#  pragma warning(disable : 4324)        /* disable: C4324: padded structure */
 #else
 #  define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
 #  ifdef __GNUC__
@@ -1043,18 +1042,43 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
     const U32 current = (U32)(ip-base);
     const U32 btLow = btMask >= current ? 0 : current - btMask;
     U32* smallerPtr = bt + 2*(current&btMask);
-    U32* largerPtr  = bt + 2*(current&btMask) + 1;
+    U32* largerPtr  = smallerPtr + 1;
     U32 dummy32;   /* to be nullified at the end */
     const U32 windowLow = zc->lowLimit;
     U32 matchEndIdx = current+8;
+    U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0);
+    U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1);
+    predictedSmall += (predictedSmall>0);
+    predictedLarge += (predictedLarge>0);
 
     hashTable[h] = current;   /* Update Hash Table */
 
     while (nbCompares-- && (matchIndex > windowLow))
     {
         U32* nextPtr = bt + 2*(matchIndex & btMask);
+        const U32* predictPtr = bt + 2*((matchIndex-1) & btMask);   /* written this way, as bt is a roll buffer */
         size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger);   /* guaranteed minimum nb of common bytes */
 
+        if (matchIndex == predictedSmall)
+        {   /* no need to check length, result known */
+            *smallerPtr = matchIndex;
+            if (matchIndex <= btLow) { smallerPtr=&dummy32; break; }   /* beyond tree size, stop the search */
+            smallerPtr = nextPtr+1;               /* new "smaller" => larger of match */
+            matchIndex = nextPtr[1];              /* new matchIndex larger than previous (closer to current) */
+            predictedSmall = predictPtr[1] + (predictPtr[1]>0);
+            continue;
+        }
+
+        if (matchIndex == predictedLarge)
+        {
+            *largerPtr = matchIndex;
+            if (matchIndex <= btLow) { largerPtr=&dummy32; break; }   /* beyond tree size, stop the search */
+            largerPtr = nextPtr;
+            matchIndex = nextPtr[0];
+            predictedLarge = predictPtr[0] + (predictPtr[0]>0);
+            continue;
+        }
+
         if ((!extDict) || (matchIndex+matchLength >= dictLimit))
         {
             match = base + matchIndex;

lib/zstd_decompress.c

@@ -44,10 +44,10 @@
 
 /*!
 *  LEGACY_SUPPORT :
-*  ZSTD_decompress() can decode older formats (starting from zstd 0.1+)
+*  ZSTD_decompress() can decode older formats (v0.1+) if set to 1
 */
 #ifndef ZSTD_LEGACY_SUPPORT
-#  define ZSTD_LEGACY_SUPPORT 1
+#  define ZSTD_LEGACY_SUPPORT 0
 #endif
 
 
@@ -799,7 +799,7 @@ size_t ZSTD_decompress(void* dst, size_t maxDstSize, const void* src, size_t src
 #else
     ZSTD_DCtx dctx;
     return ZSTD_decompressDCtx(&dctx, dst, maxDstSize, src, srcSize);
-#endif // defined
+#endif
 }
 
 

lib/zstd_internal.h

@@ -41,7 +41,7 @@ extern "C" {
 *  Includes
 ***************************************/
 #include "mem.h"
-#include "error.h"
+#include "error_private.h"
 
 
 /* *************************************

lib/zstd_static.h

@@ -284,8 +284,8 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
     /*    W,  C,  H,  S,  L, strat */
     { 0, 17, 12, 12,  1,  4, ZSTD_fast    },  /* level  0 - never used */
     { 0, 17, 12, 13,  1,  6, ZSTD_fast    },  /* level  1 */
-    { 0, 17, 15, 16,  1,  5, ZSTD_fast    },  /* level  2 */
-    { 0, 17, 16, 17,  1,  5, ZSTD_fast    },  /* level  3 */
+    { 0, 17, 14, 16,  1,  5, ZSTD_fast    },  /* level  2 */
+    { 0, 17, 15, 17,  1,  5, ZSTD_fast    },  /* level  3 */
     { 0, 17, 13, 15,  2,  4, ZSTD_greedy  },  /* level  4 */
     { 0, 17, 15, 17,  3,  4, ZSTD_greedy  },  /* level  5 */
     { 0, 17, 14, 17,  3,  4, ZSTD_lazy    },  /* level  6 */
@@ -308,7 +308,7 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
     /*     W,  C,  H,  S,  L, strat */
     {  0,  0,  0,  0,  0,  0, ZSTD_fast    },  /* level  0 - never used */
     {  0, 14, 14, 14,  1,  4, ZSTD_fast    },  /* level  1 */
-    {  0, 14, 14, 16,  1,  4, ZSTD_fast    },  /* level  1 */
+    {  0, 14, 14, 16,  1,  4, ZSTD_fast    },  /* level  2 */
     {  0, 14, 14, 14,  5,  4, ZSTD_greedy  },  /* level  3 */
     {  0, 14, 14, 14,  8,  4, ZSTD_greedy  },  /* level  4 */
     {  0, 14, 11, 14,  6,  4, ZSTD_lazy    },  /* level  5 */
@@ -334,7 +334,7 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = {
 /* *************************************
 *  Error management
 ***************************************/
-#include "error.h"
+#include "error_public.h"
 
 
 #if defined (__cplusplus)