mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-25 10:57:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/facebook/dev' into dev11

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-12-13 12:19:19 +01:00
commit 334b8962b0
21 changed files with 202 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
NEWS
View File

@ -1,16 +1,19 @@
v1.1.2
Improved : faster decompression speed at ultra compression settings and in 32-bits mode
API : streaming : decompression : changed : implicit reset on starting new frames without init
API : experimental : added : dictID retrieval functions
API : zbuff : changed : prototypes now generate deprecation warnings
lib : improved : faster decompression speed at ultra compression settings and 32-bits mode
lib : changed : only public ZSTD_ symbols are now exposed
lib : changed : reduced usage of stack memory
lib : fixed : several corner case bugs, by Nick Terrell
cli : new : gzstd, experimental version able to decode .gz files, by Przemyslaw Skibinski
cli : new : preserve file attributes
cli : new : added zstdless and zstdgrep tools
cli : fixed : status displays total amount decoded, even for file consisting of multiple frames (like pzstd)
cli : fixed : zstdcat
API : changed : zbuff prototypes now generate deprecation warnings
API : changed : streaming decompression implicit reset on starting new frame
API : added experimental : dictID retrieval functions
zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
Changed : zbuff source files moved to lib/deprecated
Changed : reduced stack memory use
install : better compatibility with FreeBSD, by Dimitry Andric
source tree : changed : zbuff source files moved to lib/deprecated
v1.1.1
New : command -M#, --memory=, --memlimit=, --memlimit-decompress= to limit allowed memory consumption

10
contrib/pzstd/Options.cpp
View File

@ -22,19 +22,13 @@
defined(__CYGWIN__)
#include <io.h> /* _isatty */
#define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
#if defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || \
defined(_POSIX_SOURCE) || \
(defined(__APPLE__) && \
defined( \
__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ \
*/
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
#include <unistd.h> /* isatty */
#define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else
#define IS_CONSOLE(stdStream) 0
#endif
#endif
namespace pzstd {

21
examples/simple_compression.c
View File

@ -8,9 +8,9 @@
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror
#include <string.h> // strerror
#include <stdlib.h> // malloc, free, exit
#include <stdio.h> // fprintf, perror, fopen, etc.
#include <string.h> // strlen, strcat, memset, strerror
#include <errno.h> // errno
#include <sys/stat.h> // stat
#include <zstd.h> // presumes zstd library is installed
@ -45,13 +45,18 @@ static void* malloc_orDie(size_t size)
static void* loadFile_orDie(const char* fileName, size_t* size)
{
off_t const buffSize = fsize_orDie(fileName);
off_t const fileSize = fsize_orDie(fileName);
size_t const buffSize = (size_t)fileSize;
if ((off_t)buffSize < fileSize) { /* narrowcast overflow */
fprintf(stderr, "%s : filesize too large \n", fileName);
exit(4);
}
FILE* const inFile = fopen_orDie(fileName, "rb");
void* const buffer = malloc_orDie(buffSize);
size_t const readSize = fread(buffer, 1, buffSize, inFile);
if (readSize != (size_t)buffSize) {
fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
exit(4);
exit(5);
}
fclose(inFile); /* can't fail, read only */
*size = buffSize;
@ -65,11 +70,11 @@ static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSi
size_t const wSize = fwrite(buff, 1, buffSize, oFile);
if (wSize != (size_t)buffSize) {
fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
exit(5);
exit(6);
}
if (fclose(oFile)) {
perror(fileName);
exit(6);
exit(7);
}
}
@ -84,7 +89,7 @@ static void compress_orDie(const char* fname, const char* oname)
size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
if (ZSTD_isError(cSize)) {
fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
exit(7);
exit(8);
}
saveFile_orDie(oname, cBuff, cSize);

10
examples/streaming_compression.c
View File

@ -7,11 +7,9 @@
*/
#include <stdlib.h> // malloc, exit
#include <stdio.h> // fprintf, perror, feof
#include <string.h> // strerror
#include <errno.h> // errno
#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being
#include <stdlib.h> // malloc, free, exit
#include <stdio.h> // fprintf, perror, feof, fopen, etc.
#include <string.h> // strlen, memset, strcat
#include <zstd.h> // presumes zstd library is installed
@ -82,7 +80,7 @@ static void compressFile_orDie(const char* fname, const char* outName, int cLeve
ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */
if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
fwrite_orDie(buffOut, output.pos, fout);
}
}

16
lib/Makefile
View File

@ -47,9 +47,9 @@ ifeq ($(shell uname), Darwin)
SHARED_EXT = dylib
SHARED_EXT_MAJOR = $(LIBVER_MAJOR).$(SHARED_EXT)
SHARED_EXT_VER = $(LIBVER).$(SHARED_EXT)
SONAME_FLAGS = -install_name $(PREFIX)/lib/$@.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
SONAME_FLAGS = -install_name $(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR) -compatibility_version $(LIBVER_MAJOR) -current_version $(LIBVER)
else
SONAME_FLAGS = -Wl,-soname=$@.$(SHARED_EXT).$(LIBVER_MAJOR)
SONAME_FLAGS = -Wl,-soname=libzstd.$(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT = so
SHARED_EXT_MAJOR = $(SHARED_EXT).$(LIBVER_MAJOR)
SHARED_EXT_VER = $(SHARED_EXT).$(LIBVER)
@ -107,15 +107,15 @@ libzstd.pc: libzstd.pc.in
install: libzstd.a libzstd libzstd.pc
@install -d -m 755 $(DESTDIR)$(LIBDIR)/pkgconfig/ $(DESTDIR)$(INCLUDEDIR)/
@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
@install -m 755 libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
@cp -a libzstd.$(SHARED_EXT_MAJOR) $(DESTDIR)$(LIBDIR)
@cp -a libzstd.$(SHARED_EXT) $(DESTDIR)$(LIBDIR)
@cp -a libzstd.pc $(DESTDIR)$(LIBDIR)/pkgconfig/
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # prototypes generate deprecation warnings
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
@install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR) # prototypes generate deprecation warnings
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
@echo zstd static and shared library installed
uninstall:

2
lib/common/zstd_internal.h
View File

@ -147,7 +147,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTD_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;

29
lib/compress/zstd_compress.c
View File

@ -33,6 +33,7 @@ typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZS
/*-*************************************
* Helper functions
***************************************/
#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; }
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
@ -148,6 +149,14 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
}
/** ZSTD_cycleLog() :
* condition for correct operation : hashLog > 1 */
static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat)
{
U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2);
return hashLog - btScale;
}
/** ZSTD_adjustCParams() :
optimize `cPar` for a given input (`srcSize` and `dictSize`).
mostly downsizing to reduce memory consumption and initialization.
@ -166,9 +175,9 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2);
U32 const maxChainLog = cPar.windowLog+btPlus;
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
{ U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy);
if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog);
}
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
@ -2274,16 +2283,16 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
if (remaining < blockSize) blockSize = remaining;
/* preemptive overflow correction */
if (cctx->lowLimit > (1<<30)) {
U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2);
U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */
U32 const correction = cctx->lowLimit - newLowLimit;
if (cctx->lowLimit > (2U<<30)) {
U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1;
U32 const current = (U32)(ip - cctx->base);
U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog);
U32 const correction = current - newCurrent;
ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30);
ZSTD_reduceIndex(cctx, correction);
cctx->base += correction;
cctx->dictBase += correction;
cctx->lowLimit = newLowLimit;
cctx->lowLimit -= correction;
cctx->dictLimit -= correction;
if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0;
else cctx->nextToUpdate -= correction;

12
lib/decompress/zstd_decompress.c
View File

@ -963,13 +963,13 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
/* match within prefix */
if (sequence.offset < 8) {
@ -997,7 +997,7 @@ size_t ZSTD_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}
@ -1183,13 +1183,13 @@ size_t ZSTD_execSequenceLong(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
if (op > oend_w || sequence.matchLength < MINMATCH) {
U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */
#endif
/* match within prefix */
@ -1218,7 +1218,7 @@ size_t ZSTD_execSequenceLong(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

6
lib/legacy/zstd_v01.c
View File

@ -1354,7 +1354,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -1803,7 +1803,7 @@ static size_t ZSTD_execSequence(BYTE* op,
} else { ZSTD_copy8(op, match); }
op += 8; match += 8;
if (endMatch > oend-12)
if (endMatch > oend-(16-MINMATCH))
{
if (op < oend-8)
{
@ -1814,7 +1814,7 @@ static size_t ZSTD_execSequence(BYTE* op,
while (op<endMatch) *op++ = *match++;
}
else
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
/* restore, in case of overlap */
if (overlapRisk) memcpy(endMatch, saved, qutt);

6
lib/legacy/zstd_v02.c
View File

@ -2808,7 +2808,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3206,7 +3206,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12)
if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@ -3218,7 +3218,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
}

6
lib/legacy/zstd_v03.c
View File

@ -2449,7 +2449,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -2847,7 +2847,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12)
if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@ -2859,7 +2859,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
}

8
lib/legacy/zstd_v04.c
View File

@ -487,7 +487,7 @@ static void ZSTD_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { ZSTD_copy8(d,s); d+=8; s+=8; }
/*! ZSTD_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
static void ZSTD_wildcopy(void* dst, const void* src, size_t length)
static void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3107,7 +3107,7 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
if (op > oend_8 || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3134,7 +3134,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12)
if (oMatchEnd > oend-(16-MINMATCH))
{
if (op < oend_8)
{
@ -3146,7 +3146,7 @@ static size_t ZSTD_execSequence(BYTE* op,
}
else
{
ZSTD_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

8
lib/legacy/zstd_v05.c
View File

@ -509,7 +509,7 @@ static void ZSTDv05_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv05_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTDv05_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3325,7 +3325,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
if (op > oend_8 || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3348,7 +3348,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
}
op += 8; match += 8;
if (oMatchEnd > oend-12) {
if (oMatchEnd > oend-(16-MINMATCH)) {
if (op < oend_8) {
ZSTDv05_wildcopy(op, match, oend_8 - op);
match += oend_8 - op;
@ -3357,7 +3357,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
while (op < oMatchEnd)
*op++ = *match++;
} else {
ZSTDv05_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTDv05_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

6
lib/legacy/zstd_v06.c
View File

@ -537,7 +537,7 @@ static void ZSTDv06_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv06_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTDv06_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3470,7 +3470,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
if (op > oend_8 || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3503,7 +3503,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTDv06_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTDv06_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

6
lib/legacy/zstd_v07.c
View File

@ -2845,7 +2845,7 @@ static void ZSTDv07_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
/*! ZSTDv07_wildcopy() :
* custom version of memcpy(), can copy up to 7 bytes too many (8 bytes if length==0) */
#define WILDCOPY_OVERLENGTH 8
MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, size_t length)
MEM_STATIC void ZSTDv07_wildcopy(void* dst, const void* src, ptrdiff_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
@ -3693,7 +3693,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
if (op > oend_w || sequence.matchLength < MINMATCH) {
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
@ -3726,7 +3726,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
}
while (op < oMatchEnd) *op++ = *match++;
} else {
ZSTDv07_wildcopy(op, match, sequence.matchLength-8); /* works even if matchLength < 8 */
ZSTDv07_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */
}
return sequenceLength;
}

22
programs/util.h
View File

@ -25,7 +25,7 @@ extern "C" {
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#if _MSC_VER <= 1800 /* (1800 = Visual Studio 2013) */
#define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
#endif
#endif
@ -49,11 +49,11 @@ extern "C" {
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#if defined(_MSC_VER)
#include <sys/utime.h> /* utime */
#include <io.h> /* _chmod */
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
#else
#include <unistd.h> /* chown, stat */
#include <utime.h> /* utime */
# include <unistd.h> /* chown, stat */
# include <utime.h> /* utime */
#endif
#include <time.h> /* time */
#include <errno.h>
@ -98,7 +98,7 @@ extern "C" {
# define SET_HIGH_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L)
# if defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || (defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 199309L))
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
@ -408,16 +408,15 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
{
size_t pos;
unsigned i, nbFiles;
char *bufend, *buf;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
buf = (char*)malloc(LIST_SIZE_INCREASE);
if (!buf) return NULL;
bufend = buf + LIST_SIZE_INCREASE;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t len = strlen(inputNames[i]);
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
@ -439,8 +438,7 @@ UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned i
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++)
{
for (i=0, pos=0; i<nbFiles; i++) {
fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1;
}

3
programs/zstdcli.c
View File

@ -43,7 +43,8 @@
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
#elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#else

1
tests/.gitignore vendored
View File

@ -11,6 +11,7 @@ datagen
paramgrill
paramgrill32
roundTripCrash
longmatch
# Tmp test directory
zstdtest

10
tests/Makefile
View File

@ -124,6 +124,9 @@ datagen : $(PRGDIR)/datagen.c datagencli.c
roundTripCrash : $(ZSTD_FILES) roundTripCrash.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
longmatch : $(ZSTD_FILES) longmatch.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
namespaceTest:
if $(CC) namespaceTest.c ../lib/common/xxhash.c -o $@ ; then echo compilation should fail; exit 1 ; fi
$(RM) $@
@ -140,7 +143,7 @@ clean:
fullbench-lib$(EXT) fullbench-dll$(EXT) \
fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \
zstreamtest$(EXT) zstreamtest32$(EXT) \
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT)
datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT)
@echo Cleaning completed
@ -180,7 +183,7 @@ zstd-playTests: datagen
file $(ZSTD)
ZSTD="$(QEMU_SYS) $(ZSTD)" ./playTests.sh $(ZSTDRTTEST)
test: test-zstd test-fullbench test-fuzzer test-zstream
test: test-zstd test-fullbench test-fuzzer test-zstream test-longmatch
test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32
@ -237,4 +240,7 @@ test-zstream: zstreamtest
test-zstream32: zstreamtest32
$(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME)
test-longmatch: longmatch
$(QEMU_SYS) ./longmatch
endif

90
tests/longmatch.c Normal file
View File

@ -0,0 +1,90 @@
#include <stdio.h>
#include <stddef.h>
#include <stdlib.h>
#include <stdint.h>
#include "mem.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
int compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) {
ZSTD_inBuffer in = { data, size, 0 };
while (in.pos < in.size) {
ZSTD_outBuffer tmp = out;
const size_t rc = ZSTD_compressStream(ctx, &tmp, &in);
if (ZSTD_isError(rc)) {
return 1;
}
}
{
ZSTD_outBuffer tmp = out;
const size_t rc = ZSTD_flushStream(ctx, &tmp);
if (rc != 0) { return 1; }
}
return 0;
}
int main(int argc, const char** argv) {
ZSTD_CStream *ctx;
ZSTD_parameters params;
size_t rc;
unsigned windowLog;
(void)argc;
(void)argv;
/* Create stream */
ctx = ZSTD_createCStream();
if (!ctx) { return 1; }
/* Set parameters */
memset(&params, 0, sizeof(params));
params.cParams.windowLog = 18;
params.cParams.chainLog = 13;
params.cParams.hashLog = 14;
params.cParams.searchLog = 1;
params.cParams.searchLength = 7;
params.cParams.targetLength = 16;
params.cParams.strategy = ZSTD_fast;
windowLog = params.cParams.windowLog;
/* Initialize stream */
rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0);
if (ZSTD_isError(rc)) { return 2; }
{
U64 compressed = 0;
const U64 toCompress = ((U64)1) << 33;
const size_t size = 1 << windowLog;
size_t pos = 0;
char *srcBuffer = (char*) malloc(1 << windowLog);
char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog));
ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 };
const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
const size_t randomData = (1 << windowLog) - 2*sizeof(match);
size_t i;
printf("\n === Long Match Test === \n");
printf("Creating random data to produce long matches \n");
for (i = 0; i < sizeof(match); ++i) {
srcBuffer[i] = match[i];
}
for (i = 0; i < randomData; ++i) {
srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF);
}
for (i = 0; i < sizeof(match); ++i) {
srcBuffer[sizeof(match) + randomData + i] = match[i];
}
printf("Compressing, trying to generate a segfault \n");
if (compress(ctx, out, srcBuffer, size)) {
return 1;
}
compressed += size;
while (compressed < toCompress) {
const size_t block = rand() % (size - pos + 1);
if (pos == size) { pos = 0; }
if (compress(ctx, out, srcBuffer + pos, block)) {
return 1;
}
pos += block;
compressed += block;
}
printf("Compression completed successfully (no error triggered)\n");
free(srcBuffer);
free(dstBuffer);
}
return 0;
}

10
tests/playTests.sh
View File

@ -31,8 +31,10 @@ case "$OS" in
;;
esac
case "$OSTYPE" in
darwin*) MD5SUM="md5 -r" ;;
UNAME=$(uname)
case "$UNAME" in
Darwin) MD5SUM="md5 -r" ;;
FreeBSD) MD5SUM="gmd5sum" ;;
*) MD5SUM="md5sum" ;;
esac
@ -228,8 +230,8 @@ cp ../programs/*.h dirTestDict
$MD5SUM dirTestDict/* > tmph1
$ZSTD -f --rm dirTestDict/* -D tmpDictC
$ZSTD -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default
case "$OSTYPE" in
darwin*) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5
case "$UNAME" in
Darwin) $ECHO "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5
*) $MD5SUM -c tmph1 ;;
esac
rm -rf dirTestDict