Merge branch 'dev' into clean

.github/workflows/dev-long-tests.yml

@@ -50,6 +50,13 @@ jobs:
     - name: thread sanitizer zstreamtest
       run: CC=clang ZSTREAM_TESTTIME=-T3mn make tsan-test-zstream
 
+  ubsan-zstreamtest:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: undefined behavior sanitizer zstreamtest
+      run: CC=clang make uasan-test-zstream
+
   # lasts ~15mn
   tsan-fuzztest:
     runs-on: ubuntu-latest
@@ -69,6 +76,13 @@ jobs:
         make gcc8install
         CC=gcc-8 make -j uasan-test-zstd </dev/null V=1
 
+  clang-asan-ubsan-testzstd:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + ASan + UBSan + Test Zstd
+      run: CC=clang make -j uasan-test-zstd </dev/null V=1
+
   gcc-asan-ubsan-testzstd-32bit:
     runs-on: ubuntu-latest
     steps:
@@ -93,6 +107,13 @@ jobs:
         make gcc8install
         CC=gcc-8 FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
 
+  clang-asan-ubsan-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + ASan + UBSan + Fuzz Test
+      run: CC=clang FUZZER_FLAGS="--long-tests" make clean uasan-fuzztest
+
   gcc-asan-ubsan-fuzz32:
     runs-on: ubuntu-latest
     steps:
@@ -103,6 +124,16 @@ jobs:
         make libc6install
         CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
 
+  clang-asan-ubsan-fuzz32:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + ASan + UBSan + Fuzz Test 32bit
+      run: |
+        sudo apt-get -qqq update
+        make libc6install
+        CC=clang CFLAGS="-O3 -m32" FUZZER_FLAGS="--long-tests" make uasan-fuzztest
+
   asan-ubsan-regression:
     runs-on: ubuntu-latest
     steps:
@@ -110,6 +141,13 @@ jobs:
     - name: ASan + UBSan + Regression Test
       run: make -j uasanregressiontest
 
+  clang-ubsan-regression:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: clang + ASan + UBSan + Regression Test
+      run: CC=clang make -j uasanregressiontest
+
   msan-regression:
     runs-on: ubuntu-latest
     steps:

contrib/match_finders/zstd_edist.c

@@ -26,7 +26,7 @@
 *  Constants
 ***************************************/
 
-/* Just a sential for the entires of the diagonal matrix */
+/* Just a sential for the entries of the diagonal matrix */
 #define ZSTD_EDIST_DIAG_MAX (S32)(1 << 30)
 
 /* How large should a snake be to be considered a 'big' snake. 
@@ -57,10 +57,10 @@ typedef struct {
     const BYTE* src;
     size_t dictSize;
     size_t srcSize;
-    S32* forwardDiag;            /* Entires of the forward diagonal stored here */ 
-    S32* backwardDiag;           /* Entires of the backward diagonal stored here.
+    S32* forwardDiag;            /* Entries of the forward diagonal stored here */
+    S32* backwardDiag;           /* Entries of the backward diagonal stored here.
                                   *   Note: this buffer and the 'forwardDiag' buffer 
-                                  *   are contiguous. See the ZSTD_eDist_genSequences */ 
+                                  *   are contiguous. See the ZSTD_eDist_genSequences */
     ZSTD_eDist_match* matches;   /* Accumulate matches of length 1 in this buffer. 
                                   *   In a subsequence post-processing step, we combine 
                                   *   contiguous matches. */
@@ -460,7 +460,7 @@ static size_t ZSTD_eDist_convertMatchesToSequences(ZSTD_Sequence* sequences,
 }
 
 /*-*************************************
-*  Interal utils
+*  Internal utils
 ***************************************/
 
 static size_t ZSTD_eDist_hamingDist(const BYTE* const a,

contrib/seqBench/Makefile

@@ -0,0 +1,58 @@
+# ################################################################
+# Copyright (c) 2018-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+PROGDIR = ../../programs
+LIBDIR  = ../../lib
+
+LIBZSTD = $(LIBDIR)/libzstd.a
+
+CPPFLAGS+= -I$(LIBDIR) -I$(LIBDIR)/common -I$(LIBDIR)/dictBuilder -I$(PROGDIR)
+
+CFLAGS  ?= -O3 -g
+CFLAGS  += -std=gnu99
+DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
+            -Wstrict-aliasing=1 -Wswitch-enum \
+            -Wstrict-prototypes -Wundef -Wpointer-arith \
+            -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \
+            -Wredundant-decls
+CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
+
+
+default: seqBench
+
+all : seqBench
+
+seqBench: util.o timefn.o benchfn.o datagen.o xxhash.o seqBench.c $(LIBZSTD)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+.PHONY: $(LIBZSTD)
+$(LIBZSTD):
+	$(MAKE) -C $(LIBDIR) libzstd.a CFLAGS="$(CFLAGS)"
+
+benchfn.o: $(PROGDIR)/benchfn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+timefn.o: $(PROGDIR)/timefn.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+datagen.o: $(PROGDIR)/datagen.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+util.o: $(PROGDIR)/util.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+xxhash.o : $(LIBDIR)/common/xxhash.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ -c
+
+
+clean:
+	$(RM) *.o
+	$(MAKE) -C $(LIBDIR) clean > /dev/null
+	$(RM) seqBench

contrib/seqBench/seqBench.c

@@ -0,0 +1,53 @@
+#define ZSTD_STATIC_LINKING_ONLY
+#include <zstd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+int main(int argc, char *argv[]) {
+    ZSTD_CCtx* zc = ZSTD_createCCtx();
+
+    if (argc != 2) {
+        printf("Usage: seqBench <file>\n"); // TODO provide the block delim option here
+        return 1;
+    }
+
+    FILE *f = fopen(argv[1], "rb");
+    fseek(f, 0, SEEK_END);
+    long inBufSize = ftell(f);
+    fseek(f, 0, SEEK_SET);
+
+    char *inBuf = malloc(inBufSize + 1);
+    fread(inBuf, inBufSize, 1, f);
+    fclose(f);
+
+    size_t seqsSize = ZSTD_sequenceBound(inBufSize);
+    ZSTD_Sequence *seqs = (ZSTD_Sequence*)malloc(seqsSize * sizeof(ZSTD_Sequence));
+    char *outBuf = malloc(ZSTD_compressBound(inBufSize));
+
+    ZSTD_generateSequences(zc, seqs, seqsSize, inBuf, inBufSize);
+    ZSTD_CCtx_setParameter(zc, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
+    size_t outBufSize = ZSTD_compressSequences(zc, outBuf, inBufSize, seqs, seqsSize, inBuf, inBufSize);
+    if (ZSTD_isError(outBufSize)) {
+        printf("ERROR: %lu\n", outBufSize);
+        return 1;
+    }
+
+    char *validationBuf = malloc(inBufSize);
+    ZSTD_decompress(validationBuf, inBufSize, outBuf, outBufSize);
+
+    if (memcmp(inBuf, validationBuf, inBufSize) == 0) {
+        printf("Compression and decompression were successful!\n");
+    } else {
+        printf("ERROR: input and validation buffers don't match!\n");
+        for (int i = 0; i < inBufSize; i++) {
+            if (inBuf[i] != validationBuf[i]) {
+                printf("First bad index: %d\n", i);
+                break;
+            }
+        }
+    }
+
+    return 0;
+}

lib/compress/zstd_compress.c

@@ -3003,6 +3003,10 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc)
     zc->seqCollector.seqIndex += seqStoreSeqSize;
 }
 
+size_t ZSTD_sequenceBound(size_t srcSize) {
+    return (srcSize / ZSTD_MINMATCH_MIN) + 1;
+}
+
 size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
                               size_t outSeqsSize, const void* src, size_t srcSize)
 {

lib/decompress/zstd_decompress.c

@@ -2059,7 +2059,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                     if (ZSTD_isError(decompressedSize)) return decompressedSize;
                     DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()")
                     ip = istart + cSize;
-                    op += decompressedSize;
+                    op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */
                     zds->expected = 0;
                     zds->streamStage = zdss_init;
                     someMoreWork = 0;
@@ -2177,14 +2177,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
                 break;
             }
         case zdss_flush:
-            {   size_t const toFlushSize = zds->outEnd - zds->outStart;
+            if (op != NULL) {
+                size_t const toFlushSize = zds->outEnd - zds->outStart;
                 size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize);
+
                 op += flushedSize;
+
                 zds->outStart += flushedSize;
                 if (flushedSize == toFlushSize) {  /* flush completed */
                     zds->streamStage = zdss_read;
                     if ( (zds->outBuffSize < zds->fParams.frameContentSize)
-                      && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
+                    && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) {
                         DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)",
                                 (int)(zds->outBuffSize - zds->outStart),
                                 (U32)zds->fParams.blockSizeMax);

lib/zstd.h

@@ -836,7 +836,7 @@ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
 /*! ZSTD_decompressStream() :
  * Streaming decompression function.
  * Call repetitively to consume full input updating it as necessary.
- * Funtion will update both input and output `pos` fields exposing current state via these fields:
+ * Function will update both input and output `pos` fields exposing current state via these fields:
  * - `input.pos < input.size`, some input remaining and caller should provide remaining input
  *   on the next call.
  * - `output.pos < output.size`, decoder finished and flushed all remaining buffers.
@@ -844,7 +844,7 @@ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
  *   call ZSTD_decompressStream() again to flush remaining data to output.
  * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX.
  *
- * @return : 0 when a frame is completly decoded and fully flushed,
+ * @return : 0 when a frame is completely decoded and fully flushed,
  *           or an error code, which can be tested using ZSTD_isError(),
  *           or any other value > 0, which means there is some decoding or flushing to do to complete current frame.
  */
@@ -1396,6 +1396,15 @@ typedef enum {
   ZSTD_sf_explicitBlockDelimiters = 1    /* Representation of ZSTD_Sequence contains explicit block delimiters */
 } ZSTD_sequenceFormat_e;
 
+/*! ZSTD_sequenceBound() :
+ * `srcSize` : size of the input buffer
+ *  @return : upper-bound for the number of sequences that can be generated
+ *            from a buffer of srcSize bytes
+ *
+ *  note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence).
+ */
+ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize);
+
 /*! ZSTD_generateSequences() :
  * Generate sequences using ZSTD_compress2(), given a source buffer.
  *

tests/.gitignore

@@ -12,6 +12,7 @@ zstreamtest
 zstreamtest32
 zstreamtest_asan
 zstreamtest_tsan
+zstreamtest_ubsan
 zstreamtest-dll
 datagen
 paramgrill

tests/Makefile

@@ -190,6 +190,11 @@ zstreamtest_tsan : CFLAGS += -fsanitize=thread
 zstreamtest_tsan : $(ZSTREAMFILES)
 	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
 
+CLEAN += zstreamtest_ubsan
+zstreamtest_ubsan : CFLAGS += -fsanitize=undefined
+zstreamtest_ubsan : $(ZSTREAMFILES)
+	$(LINK.c) $(MULTITHREAD) $^ -o $@$(EXT)
+
 # note : broken : requires symbols unavailable from dynamic library
 zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c  # xxh symbols not exposed from dll
 zstreamtest-dll : $(ZSTREAM_LOCAL_FILES)

tests/zstreamtest.c

@@ -522,7 +522,7 @@ static int basicUnitTests(U32 seed, double compressibility)
     }
     DISPLAYLEVEL(3, "OK \n");
 
-    DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++);
+    DISPLAYLEVEL(3, "test%3i : NULL output and NULL input : ", testNb++);
     inBuff.src = NULL;
     inBuff.size = 0;
     inBuff.pos = 0;
@@ -548,6 +548,36 @@ static int basicUnitTests(U32 seed, double compressibility)
     {   size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff);
         if (ret != 0) goto _output_error;
     }
+    DISPLAYLEVEL(3, "OK\n");
+
+    DISPLAYLEVEL(3, "test%3i : NULL output buffer with non-NULL input : ", testNb++);
+    {
+        const char* test = "aa";
+        inBuff.src = test;
+        inBuff.size = 2;
+        inBuff.pos = 0;
+        outBuff.dst = NULL;
+        outBuff.size = 0;
+        outBuff.pos = 0;
+        CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) );
+        CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed");
+        CHECK_Z( ZSTD_endStream(zc, &outBuff) );
+        outBuff.dst = (char*)(compressedBuffer);
+        outBuff.size = compressedBufferSize;
+        outBuff.pos = 0;
+        {   size_t const r = ZSTD_endStream(zc, &outBuff);
+            CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r);
+        }
+        inBuff.src = outBuff.dst;
+        inBuff.size = outBuff.pos;
+        inBuff.pos = 0;
+        outBuff.dst = NULL;
+        outBuff.size = 0;
+        outBuff.pos = 0;
+        CHECK_Z( ZSTD_initDStream(zd) );
+        CHECK_Z(ZSTD_decompressStream(zd, &outBuff, &inBuff));
+    }
+
     DISPLAYLEVEL(3, "OK\n");
     /* _srcSize compression test */
     DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH);