diff --git a/NEWS b/NEWS
index 580889542..36aa45e91 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,9 @@ cli : fixed : status displays total amount decoded, even for file consisting of
cli : fixed : zstdcat
API : changed : zbuff prototypes now generate deprecation warnings
API : changed : streaming decompression implicit reset on starting new frame
+API : added experimental : dictID retrieval functions
+zlib_wrapper : added support for gz* functions, by Przemyslaw Skibinski
+Changed : zbuff source files moved to lib/deprecated
Changed : reduced stack memory use
v1.1.1
diff --git a/build/VS2005/fullbench/fullbench.vcproj b/build/VS2005/fullbench/fullbench.vcproj
index c28d1f696..c67490c6f 100644
--- a/build/VS2005/fullbench/fullbench.vcproj
+++ b/build/VS2005/fullbench/fullbench.vcproj
@@ -363,14 +363,6 @@
RelativePath="..\..\..\lib\decompress\huf_decompress.c"
>
-
-
-
-
@@ -425,10 +417,6 @@
RelativePath="..\..\..\lib\common\mem.h"
>
-
-
diff --git a/build/VS2005/fuzzer/fuzzer.vcproj b/build/VS2005/fuzzer/fuzzer.vcproj
index dd7450faa..b1ac81365 100644
--- a/build/VS2005/fuzzer/fuzzer.vcproj
+++ b/build/VS2005/fuzzer/fuzzer.vcproj
@@ -429,10 +429,6 @@
RelativePath="..\..\..\lib\common\xxhash.h"
>
-
-
diff --git a/build/VS2005/zstd/zstd.vcproj b/build/VS2005/zstd/zstd.vcproj
index 223285ef1..9f49e3cb6 100644
--- a/build/VS2005/zstd/zstd.vcproj
+++ b/build/VS2005/zstd/zstd.vcproj
@@ -379,14 +379,6 @@
RelativePath="..\..\..\lib\common\xxhash.c"
>
-
-
-
-
@@ -481,14 +473,6 @@
RelativePath="..\..\..\lib\common\xxhash.h"
>
-
-
-
-
diff --git a/build/VS2005/zstdlib/zstdlib.vcproj b/build/VS2005/zstdlib/zstdlib.vcproj
index 7a0a76e67..1b78986b2 100644
--- a/build/VS2005/zstdlib/zstdlib.vcproj
+++ b/build/VS2005/zstdlib/zstdlib.vcproj
@@ -360,11 +360,11 @@
>
-
-
diff --git a/build/cmake/lib/CMakeLists.txt b/build/cmake/lib/CMakeLists.txt
index f970fe7e9..d22ddeada 100644
--- a/build/cmake/lib/CMakeLists.txt
+++ b/build/cmake/lib/CMakeLists.txt
@@ -64,25 +64,25 @@ SET(Sources
${LIBRARY_DIR}/common/fse_decompress.c
${LIBRARY_DIR}/compress/fse_compress.c
${LIBRARY_DIR}/compress/huf_compress.c
- ${LIBRARY_DIR}/compress/zbuff_compress.c
${LIBRARY_DIR}/compress/zstd_compress.c
${LIBRARY_DIR}/decompress/huf_decompress.c
- ${LIBRARY_DIR}/decompress/zbuff_decompress.c
${LIBRARY_DIR}/decompress/zstd_decompress.c
${LIBRARY_DIR}/dictBuilder/divsufsort.c
- ${LIBRARY_DIR}/dictBuilder/zdict.c)
+ ${LIBRARY_DIR}/dictBuilder/zdict.c
+ ${LIBRARY_DIR}/deprecated/zbuff_compress.c
+ ${LIBRARY_DIR}/deprecated/zbuff_decompress.c)
SET(Headers
+ ${LIBRARY_DIR}/zstd.h
${LIBRARY_DIR}/common/bitstream.h
${LIBRARY_DIR}/common/error_private.h
${LIBRARY_DIR}/common/zstd_errors.h
${LIBRARY_DIR}/common/fse.h
${LIBRARY_DIR}/common/huf.h
${LIBRARY_DIR}/common/mem.h
- ${LIBRARY_DIR}/common/zbuff.h
${LIBRARY_DIR}/common/zstd_internal.h
- ${LIBRARY_DIR}/zstd.h
- ${LIBRARY_DIR}/dictBuilder/zdict.h)
+ ${LIBRARY_DIR}/dictBuilder/zdict.h
+ ${LIBRARY_DIR}/deprecated/zbuff.h)
IF (ZSTD_LEGACY_SUPPORT)
SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
@@ -165,7 +165,7 @@ IF (UNIX)
SET(INSTALL_INCLUDE_DIR ${PREFIX}/include)
# install target
- INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/common/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
+ INSTALL(FILES ${LIBRARY_DIR}/zstd.h ${LIBRARY_DIR}/deprecated/zbuff.h ${LIBRARY_DIR}/dictBuilder/zdict.h DESTINATION ${INSTALL_INCLUDE_DIR})
INSTALL(TARGETS libzstd_static DESTINATION ${INSTALL_LIBRARY_DIR})
INSTALL(TARGETS libzstd_shared LIBRARY DESTINATION ${INSTALL_LIBRARY_DIR})
diff --git a/build/cmake/tests/CMakeLists.txt b/build/cmake/tests/CMakeLists.txt
index f5ece894e..7f9c38e1a 100644
--- a/build/cmake/tests/CMakeLists.txt
+++ b/build/cmake/tests/CMakeLists.txt
@@ -50,9 +50,6 @@ ADD_EXECUTABLE(fuzzer ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/fuzzer.c)
TARGET_LINK_LIBRARIES(fuzzer libzstd_static)
IF (UNIX)
- ADD_EXECUTABLE(zbufftest ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/zbufftest.c)
- TARGET_LINK_LIBRARIES(zbufftest libzstd_static)
-
ADD_EXECUTABLE(paramgrill ${PROGRAMS_DIR}/datagen.c ${TESTS_DIR}/paramgrill.c)
TARGET_LINK_LIBRARIES(paramgrill libzstd_static m) #m is math library
diff --git a/lib/Makefile b/lib/Makefile
index 8f316aa68..bf1088d68 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -23,24 +23,22 @@ PREFIX ?= /usr/local
LIBDIR ?= $(PREFIX)/lib
INCLUDEDIR=$(PREFIX)/include
-CPPFLAGS= -I. -I./common -DXXH_NAMESPACE=XXH_
-CFLAGS ?= -O3
-CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
- -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
- -Wpointer-arith
-FLAGS = $(CPPFLAGS) $(CFLAGS) $(MOREFLAGS)
+CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
+CFLAGS ?= -O3
+CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wstrict-aliasing=1 \
+ -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \
+ -Wpointer-arith
+CFLAGS += $(MOREFLAGS)
+FLAGS = $(CPPFLAGS) $(CFLAGS)
-ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c)
-ZSTD_EXCLUDE := compress/zbuff_compress.c decompress/zbuff_decompress.c
-ZSTD_FILES := $(filter-out $(ZSTD_EXCLUDE), $(ZSTD_FILES))
-
+ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
ifeq ($(ZSTD_LEGACY_SUPPORT), 0)
CPPFLAGS += -DZSTD_LEGACY_SUPPORT=0
else
-ZSTD_FILES+= legacy/*.c
CPPFLAGS += -I./legacy -DZSTD_LEGACY_SUPPORT=1
+ZSTD_FILES+= $(wildcard legacy/*.c)
endif
# OS X linker doesn't support -soname, and use different extension
@@ -90,8 +88,8 @@ libzstd : $(LIBZSTD)
lib: libzstd.a libzstd
clean:
- @$(RM) -f core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
- @$(RM) -f decompress/*.o
+ @$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc dll/libzstd.dll dll/libzstd.lib
+ @$(RM) decompress/*.o
@echo Cleaning library completed
#------------------------------------------------------------------------
@@ -116,7 +114,7 @@ install: libzstd.a libzstd libzstd.pc
@install -m 644 libzstd.a $(DESTDIR)$(LIBDIR)/libzstd.a
@install -m 644 zstd.h $(DESTDIR)$(INCLUDEDIR)/zstd.h
@install -m 644 common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
- @install -m 644 common/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h
+ @install -m 644 deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)/zbuff.h # prototypes generate deprecation warnings
@install -m 644 dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)/zdict.h
@echo zstd static and shared library installed
diff --git a/lib/README.md b/lib/README.md
index d33ad52c2..3357e3d87 100644
--- a/lib/README.md
+++ b/lib/README.md
@@ -56,15 +56,15 @@ file it should be linked with `dll\libzstd.dll`. For example:
```
gcc $(CFLAGS) -Iinclude/ test-dll.c -o test-dll dll\libzstd.dll
```
-The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
+The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
#### Obsolete streaming API
Streaming is now provided within `zstd.h`.
-Older streaming API is still available within `common/zbuff.h`.
-It is now deprecated, and will be removed in a future version.
-Consider migrating towards newer streaming API in `zstd.h`.
+Older streaming API is still available within `deprecated/zbuff.h`.
+It will be removed in a future version.
+Consider migrating code towards newer streaming API in `zstd.h`.
#### Miscellaneous
diff --git a/lib/common/zstd_common.c b/lib/common/zstd_common.c
index 54bc91c89..f30128c79 100644
--- a/lib/common/zstd_common.c
+++ b/lib/common/zstd_common.c
@@ -16,7 +16,6 @@
#include "error_private.h"
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */
-#include "zbuff.h" /* declaration of ZBUFF_isError, ZBUFF_getErrorName */
/*-****************************************
@@ -44,16 +43,11 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
* provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
-
-/* **************************************************************
-* ZBUFF Error Management
-****************************************************************/
+/* --- ZBUFF Error Management (deprecated) --- */
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
-
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
-
/*=**************************************************************
* Custom allocator
****************************************************************/
diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c
index 7c4f54bd3..70dd4ccaa 100644
--- a/lib/decompress/zstd_decompress.c
+++ b/lib/decompress/zstd_decompress.c
@@ -1778,6 +1778,45 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
}
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
+{
+ if (dictSize < 8) return 0;
+ if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0;
+ return MEM_readLE32((const char*)dict + 4);
+}
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
+{
+ if (ddict==NULL) return 0;
+ return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize);
+}
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize)
+{
+ ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 };
+ size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize);
+ if (ZSTD_isError(hError)) return 0;
+ return zfp.dictID;
+}
+
/*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary
diff --git a/lib/common/zbuff.h b/lib/deprecated/zbuff.h
similarity index 99%
rename from lib/common/zbuff.h
rename to lib/deprecated/zbuff.h
index e8af504de..8e55c39a5 100644
--- a/lib/common/zbuff.h
+++ b/lib/deprecated/zbuff.h
@@ -202,6 +202,7 @@ ZBUFF_DEPRECATED("use ZSTD_initDStream_usingDict") size_t ZBUFF_compressInit_adv
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
+
#endif /* ZBUFF_STATIC_LINKING_ONLY */
diff --git a/lib/compress/zbuff_compress.c b/lib/deprecated/zbuff_compress.c
similarity index 100%
rename from lib/compress/zbuff_compress.c
rename to lib/deprecated/zbuff_compress.c
diff --git a/lib/decompress/zbuff_decompress.c b/lib/deprecated/zbuff_decompress.c
similarity index 100%
rename from lib/decompress/zbuff_decompress.c
rename to lib/deprecated/zbuff_decompress.c
diff --git a/lib/zstd.h b/lib/zstd.h
index b7a8bb989..a10a9eaab 100644
--- a/lib/zstd.h
+++ b/lib/zstd.h
@@ -53,8 +53,6 @@ extern "C" {
*********************************************************************************************************/
/*------ Version ------*/
-ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */
-
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 2
@@ -65,6 +63,7 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
+ZSTDLIB_API unsigned ZSTD_versionNumber (void);
/***************************************
@@ -462,6 +461,30 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
+/*! ZSTD_getDictID_fromDict() :
+ * Provides the dictID stored within dictionary.
+ * if @return == 0, the dictionary is not conformant with Zstandard specification.
+ * It can still be loaded, but as a content-only dictionary. */
+unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
+
+/*! ZSTD_getDictID_fromDDict() :
+ * Provides the dictID of the dictionary loaded into `ddict`.
+ * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
+ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
+unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
+
+/*! ZSTD_getDictID_fromFrame() :
+ * Provides the dictID required to decompressed the frame stored within `src`.
+ * If @return == 0, the dictID could not be decoded.
+ * This could for one of the following reasons :
+ * - The frame does not require a dictionary to be decoded (most common case).
+ * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information.
+ * Note : this use case also happens when using a non-conformant dictionary.
+ * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
+ * - This is not a Zstandard frame.
+ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
+unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
+
/********************************************************************
* Advanced streaming functions
diff --git a/programs/fileio.c b/programs/fileio.c
index 8c54e3c5e..a493e97c5 100644
--- a/programs/fileio.c
+++ b/programs/fileio.c
@@ -35,6 +35,9 @@
#include "zstd.h"
#ifdef ZSTD_GZDECOMPRESS
#include "zlib.h"
+#if !defined(z_const)
+ #define z_const
+#endif
#endif
@@ -465,6 +468,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
****************************************************************************/
typedef struct {
void* srcBuffer;
+ size_t srcBufferLoaded;
size_t srcBufferSize;
void* dstBuffer;
size_t dstBufferSize;
@@ -582,63 +586,18 @@ static void FIO_fwriteSparseEnd(FILE* file, unsigned storedSkips)
} }
}
-/** FIO_decompressFrame() :
- @return : size of decoded frame
-*/
-unsigned long long FIO_decompressFrame(dRess_t ress,
- FILE* finput, size_t alreadyLoaded,
- U64 alreadyDecoded)
-{
- U64 frameSize = 0;
- size_t readSize;
- U32 storedSkips = 0;
-
- ZSTD_resetDStream(ress.dctx);
-
- /* Header loading (optional, saves one loop) */
- { size_t const toLoad = 9 - alreadyLoaded; /* assumption : 9 >= alreadyLoaded */
- size_t const loadedSize = fread(((char*)ress.srcBuffer) + alreadyLoaded, 1, toLoad, finput);
- readSize = alreadyLoaded + loadedSize;
- }
-
- /* Main decompression Loop */
- while (1) {
- ZSTD_inBuffer inBuff = { ress.srcBuffer, readSize, 0 };
- ZSTD_outBuffer outBuff= { ress.dstBuffer, ress.dstBufferSize, 0 };
- size_t const readSizeHint = ZSTD_decompressStream(ress.dctx, &outBuff, &inBuff );
- if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
-
- /* Write block */
- storedSkips = FIO_fwriteSparse(ress.dstFile, ress.dstBuffer, outBuff.pos, storedSkips);
- frameSize += outBuff.pos;
- DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
-
- if (readSizeHint == 0) break; /* end of frame */
- if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
-
- /* Fill input buffer */
- { size_t const toRead = MIN(readSizeHint, ress.srcBufferSize); /* support large skippable frames */
- readSize = fread(ress.srcBuffer, 1, toRead, finput);
- if (readSize < toRead) EXM_THROW(39, "Read error : premature end");
- } }
-
- FIO_fwriteSparseEnd(ress.dstFile, storedSkips);
-
- return frameSize;
-}
-
/** FIO_passThrough() : just copy input into output, for compatibility with gzip -df mode
@return : 0 (no error) */
-static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize)
+static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_t bufferSize, size_t alreadyLoaded)
{
size_t const blockSize = MIN(64 KB, bufferSize);
size_t readFromInput = 1;
unsigned storedSkips = 0;
- /* assumption : first 4 bytes already loaded (magic number detection), and stored within buffer */
- { size_t const sizeCheck = fwrite(buffer, 1, 4, foutput);
- if (sizeCheck != 4) EXM_THROW(50, "Pass-through write error"); }
+ /* assumption : ress->srcBufferLoaded bytes already loaded and stored within buffer */
+ { size_t const sizeCheck = fwrite(buffer, 1, alreadyLoaded, foutput);
+ if (sizeCheck != alreadyLoaded) EXM_THROW(50, "Pass-through write error"); }
while (readFromInput) {
readFromInput = fread(buffer, 1, blockSize, finput);
@@ -649,8 +608,60 @@ static unsigned FIO_passThrough(FILE* foutput, FILE* finput, void* buffer, size_
return 0;
}
+
+/** FIO_decompressFrame() :
+ @return : size of decoded frame
+*/
+unsigned long long FIO_decompressFrame(dRess_t* ress,
+ FILE* finput,
+ U64 alreadyDecoded)
+{
+ U64 frameSize = 0;
+ U32 storedSkips = 0;
+
+ ZSTD_resetDStream(ress->dctx);
+
+ /* Header loading (optional, saves one loop) */
+ { size_t const toRead = 9;
+ if (ress->srcBufferLoaded < toRead)
+ ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
+ }
+
+ /* Main decompression Loop */
+ while (1) {
+ ZSTD_inBuffer inBuff = { ress->srcBuffer, ress->srcBufferLoaded, 0 };
+ ZSTD_outBuffer outBuff= { ress->dstBuffer, ress->dstBufferSize, 0 };
+ size_t const readSizeHint = ZSTD_decompressStream(ress->dctx, &outBuff, &inBuff);
+ if (ZSTD_isError(readSizeHint)) EXM_THROW(36, "Decoding error : %s", ZSTD_getErrorName(readSizeHint));
+
+ /* Write block */
+ storedSkips = FIO_fwriteSparse(ress->dstFile, ress->dstBuffer, outBuff.pos, storedSkips);
+ frameSize += outBuff.pos;
+ DISPLAYUPDATE(2, "\rDecoded : %u MB... ", (U32)((alreadyDecoded+frameSize)>>20) );
+
+ if (inBuff.pos > 0) {
+ memmove(ress->srcBuffer, (char*)ress->srcBuffer + inBuff.pos, inBuff.size - inBuff.pos);
+ ress->srcBufferLoaded -= inBuff.pos;
+ }
+
+ if (readSizeHint == 0) break; /* end of frame */
+ if (inBuff.size != inBuff.pos) EXM_THROW(37, "Decoding error : should consume entire input");
+
+ /* Fill input buffer */
+ { size_t const toRead = MIN(readSizeHint, ress->srcBufferSize); /* support large skippable frames */
+ if (ress->srcBufferLoaded < toRead)
+ ress->srcBufferLoaded += fread(((char*)ress->srcBuffer) + ress->srcBufferLoaded, 1, toRead - ress->srcBufferLoaded, finput);
+ if (ress->srcBufferLoaded < toRead) EXM_THROW(39, "Read error : premature end");
+ } }
+
+ FIO_fwriteSparseEnd(ress->dstFile, storedSkips);
+
+ return frameSize;
+}
+
+
#ifdef ZSTD_GZDECOMPRESS
-static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, const char* srcFileName, size_t alreadyLoaded)
+static unsigned long long FIO_decompressGzFrame(dRess_t* ress, FILE* srcFile, const char* srcFileName)
{
unsigned long long outFileSize = 0;
z_stream strm;
@@ -662,32 +673,34 @@ static unsigned long long FIO_decompressGzFrame(dRess_t ress, FILE* srcFile, con
strm.avail_in = Z_NULL;
if (inflateInit2(&strm, 15 /* maxWindowLogSize */ + 16 /* gzip only */) != Z_OK) return 0; /* see http://www.zlib.net/manual.html */
- strm.next_out = ress.dstBuffer;
- strm.avail_out = ress.dstBufferSize;
+ strm.next_out = ress->dstBuffer;
+ strm.avail_out = ress->dstBufferSize;
+ strm.avail_in = ress->srcBufferLoaded;
+ strm.next_in = (z_const unsigned char*)ress->srcBuffer;
for ( ; ; ) {
- if (alreadyLoaded) {
- strm.avail_in = alreadyLoaded;
- strm.next_in = (z_const unsigned char*)ress.srcBuffer;
- alreadyLoaded = 0;
- } else {
- if (fread(ress.srcBuffer, 1, 1/*ress.srcBufferSize*/, srcFile) == 0) break;
- strm.next_in = (z_const unsigned char*)ress.srcBuffer;
- strm.avail_in = 1;
+ int ret;
+ if (strm.avail_in == 0) {
+ ress->srcBufferLoaded = fread(ress->srcBuffer, 1, ress->srcBufferSize, srcFile);
+ if (ress->srcBufferLoaded == 0) break;
+ strm.next_in = (z_const unsigned char*)ress->srcBuffer;
+ strm.avail_in = ress->srcBufferLoaded;
}
- { int const ret = inflate(&strm, Z_NO_FLUSH);
- if (ret == Z_STREAM_END) break;
- if (ret != Z_OK) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
- }
-
- { size_t const decompBytes = ress.dstBufferSize - strm.avail_out;
+ ret = inflate(&strm, Z_NO_FLUSH);
+ if (ret != Z_OK && ret != Z_STREAM_END) { DISPLAY("zstd: %s: inflate error %d \n", srcFileName, ret); return 0; }
+ { size_t const decompBytes = ress->dstBufferSize - strm.avail_out;
if (decompBytes) {
- if (fwrite(ress.dstBuffer, 1, decompBytes, ress.dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
+ if (fwrite(ress->dstBuffer, 1, decompBytes, ress->dstFile) != decompBytes) EXM_THROW(31, "Write error : cannot write to output file");
outFileSize += decompBytes;
- strm.next_out = ress.dstBuffer;
- strm.avail_out = ress.dstBufferSize;
- } } }
+ strm.next_out = ress->dstBuffer;
+ strm.avail_out = ress->dstBufferSize;
+ }
+ }
+ if (ret == Z_STREAM_END) break;
+ }
+ if (strm.avail_in > 0) memmove(ress->srcBuffer, strm.next_in, strm.avail_in);
+ ress->srcBufferLoaded = strm.avail_in;
inflateEnd(&strm);
return outFileSize;
}
@@ -717,19 +730,19 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
for ( ; ; ) {
/* check magic number -> version */
size_t const toRead = 4;
- size_t const sizeCheck = fread(ress.srcBuffer, (size_t)1, toRead, srcFile);
const BYTE* buf = (const BYTE*)ress.srcBuffer;
- if (sizeCheck==0) {
+ if (ress.srcBufferLoaded < toRead)
+ ress.srcBufferLoaded += fread((char*)ress.srcBuffer + ress.srcBufferLoaded, (size_t)1, toRead - ress.srcBufferLoaded, srcFile);
+ if (ress.srcBufferLoaded==0) {
if (readSomething==0) { DISPLAY("zstd: %s: unexpected end of file \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
break; /* no more input */
}
readSomething = 1; /* there is at least >= 4 bytes in srcFile */
- if (sizeCheck != toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
+ if (ress.srcBufferLoaded < toRead) { DISPLAY("zstd: %s: unknown header \n", srcFileName); fclose(srcFile); return 1; } /* srcFileName is empty */
if (buf[0] == 31 && buf[1] == 139) { /* gz header */
#ifdef ZSTD_GZDECOMPRESS
- unsigned long long const result = FIO_decompressGzFrame(ress, srcFile, srcFileName, toRead);
+ unsigned long long const result = FIO_decompressGzFrame(&ress, srcFile, srcFileName);
if (result == 0) return 1;
- printf("gzip=%d\n", (int)result);
filesize += result;
#else
DISPLAYLEVEL(1, "zstd: %s: gzip file cannot be uncompressed (zstd compiled without ZSTD_GZDECOMPRESS) -- ignored \n", srcFileName);
@@ -738,8 +751,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
} else {
if (!ZSTD_isFrame(ress.srcBuffer, toRead)) {
if ((g_overwrite) && !strcmp (dstFileName, stdoutmark)) { /* pass-through mode */
- unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize);
- printf("pass-through=%d\n", (int)result);
+ unsigned const result = FIO_passThrough(ress.dstFile, srcFile, ress.srcBuffer, ress.srcBufferSize, ress.srcBufferLoaded);
if (fclose(srcFile)) EXM_THROW(32, "zstd: %s close error", srcFileName); /* error should never happen */
return result;
} else {
@@ -747,8 +759,7 @@ static int FIO_decompressSrcFile(dRess_t ress, const char* dstFileName, const ch
fclose(srcFile);
return 1;
} }
- filesize += FIO_decompressFrame(ress, srcFile, toRead, filesize);
- printf("zstd filesize=%d\n", (int)filesize);
+ filesize += FIO_decompressFrame(&ress, srcFile, filesize);
}
}
diff --git a/tests/Makefile b/tests/Makefile
index ca899d2f8..fbee21448 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -37,10 +37,10 @@ FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS)
ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c
-ZSTDCOMP_FILES := $(ZSTDDIR)/compress/zstd_compress.c $(ZSTDDIR)/compress/fse_compress.c $(ZSTDDIR)/compress/huf_compress.c
-ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/zstd_decompress.c $(ZSTDDIR)/decompress/huf_decompress.c
+ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c
+ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c
ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES)
-ZBUFF_FILES := $(ZSTDDIR)/compress/zbuff_compress.c $(ZSTDDIR)/decompress/zbuff_decompress.c
+ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c
ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c
@@ -53,7 +53,7 @@ EXT =
endif
VOID = /dev/null
-ZBUFFTEST = -T2mn
+ZSTREAM_TESTTIME = -T2mn
FUZZERTEST= -T5mn
ZSTDRTTEST= --test-large-data
@@ -61,9 +61,9 @@ ZSTDRTTEST= --test-large-data
default: fullbench
-all: fullbench fuzzer zstreamtest paramgrill datagen
+all: fullbench fuzzer zstreamtest paramgrill datagen zbufftest
-all32: fullbench32 fuzzer32 zstreamtest32
+all32: fullbench32 fuzzer32 zstreamtest32 zbufftest32
@@ -93,19 +93,21 @@ fullbench-dll: $(PRGDIR)/datagen.c fullbench.c
$(MAKE) -C $(ZSTDDIR) libzstd
$(CC) $(FLAGS) $^ -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll
-fuzzer : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
fuzzer : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
-fuzzer32 : CPPFLAGS += -I$(ZSTDDIR)/dictBuilder
fuzzer32 : $(ZSTD_FILES) $(ZDICT_FILES) $(PRGDIR)/datagen.c fuzzer.c
$(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
-zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
- $(CC) $(FLAGS) $^ -o $@$(EXT)
+zbufftest : CPPFLAGS += -I$(ZSTDDIR)/deprecated
+zbufftest : CFLAGS += -Wno-deprecated-declarations
+zbufftest : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT) # flag required to silence deprecation warnings
-zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
- $(CC) -m32 $(FLAGS) $^ -o $@$(EXT)
+zbufftest32 : CPPFLAGS += -I$(ZSTDDIR)/deprecated
+zbufftest32 : CFLAGS += -Wno-deprecated-declarations -m32
+zbufftest32 : $(ZSTD_FILES) $(ZBUFF_FILES) $(PRGDIR)/datagen.c zbufftest.c
+ $(CC) $(FLAGS) $^ -o $@$(EXT)
zstreamtest : $(ZSTD_FILES) $(PRGDIR)/datagen.c zstreamtest.c
$(CC) $(FLAGS) $^ -o $@$(EXT)
@@ -194,9 +196,20 @@ test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd
test-zstd-nolegacy: zstd-nolegacy zstd-playTests
test-gzstd: gzstd
+ $(PRGDIR)/zstd README.md test-zstd-speed.py
gzip README.md test-zstd-speed.py
+ cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst
+ cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz
$(PRGDIR)/zstd -d README.md.gz -o README2.md
$(PRGDIR)/zstd -d README.md.gz test-zstd-speed.py.gz
+ $(PRGDIR)/zstd -d zstd_gz.zst gz_zstd.gz
+ diff -q zstd_gz gz_zstd
+ echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst
+ echo Hello World GZIP | gzip -c - >hello.gz
+ echo Hello World TEXT >hello.txt
+ cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz
+ $(PRGDIR)/zstd -dcf hello.*
+ $(PRGDIR)/zstd -dcf - /* clock_t */
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
#include "zstd.h" /* ZSTD_VERSION_STRING */
-#include "zstd_errors.h" /* ZSTD_getErrorCode */
+#include "zstd_errors.h" /* ZSTD_getErrorCode */
#include "zdict.h" /* ZDICT_trainFromBuffer */
#include "datagen.h" /* RDG_genBuffer */
#include "mem.h"
@@ -246,6 +246,7 @@ static int basicUnitTests(U32 seed, double compressibility)
size_t const sampleUnitSize = 8 KB;
U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize);
size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t));
+ U32 dictID;
if (dictBuffer==NULL || samplesSizes==NULL) {
free(dictBuffer);
@@ -261,10 +262,9 @@ static int basicUnitTests(U32 seed, double compressibility)
DISPLAYLEVEL(4, "OK, created dictionary of size %u \n", (U32)dictSize);
DISPLAYLEVEL(4, "test%3i : check dictID : ", testNb++);
- { U32 const dictID = ZDICT_getDictID(dictBuffer, dictSize);
- if (dictID==0) goto _output_error;
- DISPLAYLEVEL(4, "OK : %u \n", dictID);
- }
+ dictID = ZDICT_getDictID(dictBuffer, dictSize);
+ if (dictID==0) goto _output_error;
+ DISPLAYLEVEL(4, "OK : %u \n", dictID);
DISPLAYLEVEL(4, "test%3i : compress with dictionary : ", testNb++);
cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, ZSTD_compressBound(CNBuffSize),
@@ -273,6 +273,18 @@ static int basicUnitTests(U32 seed, double compressibility)
if (ZSTD_isError(cSize)) goto _output_error;
DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/CNBuffSize*100);
+ DISPLAYLEVEL(4, "test%3i : retrieve dictID from dictionary : ", testNb++);
+ { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize);
+ if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
+ DISPLAYLEVEL(4, "test%3i : retrieve dictID from frame : ", testNb++);
+ { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize);
+ if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */
+ }
+ DISPLAYLEVEL(4, "OK \n");
+
DISPLAYLEVEL(4, "test%3i : frame built with dictionary should be decompressible : ", testNb++);
CHECKPLUS(r, ZSTD_decompress_usingDict(dctx,
decodedBuffer, CNBuffSize,
diff --git a/zlibWrapper/.gitignore b/zlibWrapper/.gitignore
index c3376bad8..8ce15613c 100644
--- a/zlibWrapper/.gitignore
+++ b/zlibWrapper/.gitignore
@@ -2,11 +2,14 @@
_*
example.*
example_zstd.*
+example_gz.*
fitblk.*
fitblk_zstd.*
zwrapbench.*
foo.gz
+minigzip
+minigzip_zstd
example
example_zstd
fitblk
diff --git a/zlibWrapper/Makefile b/zlibWrapper/Makefile
index 49d44e35a..a0e641d94 100644
--- a/zlibWrapper/Makefile
+++ b/zlibWrapper/Makefile
@@ -13,12 +13,12 @@ ZLIB_PATH ?= .
ZSTDLIBDIR = ../lib
ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a
ZLIBWRAPPER_PATH = .
-GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
+GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o
EXAMPLE_PATH = examples
PROGRAMS_PATH = ../programs
TEST_FILE = ../doc/zstd_compression_format.md
-CPPFLAGS = -DXXH_NAMESPACE=XXH_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
+CPPFLAGS = -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH)
CFLAGS ?= $(MOREFLAGS) -O3 -std=gnu99
CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum -Wdeclaration-after-statement -Wstrict-prototypes -Wundef -Wstrict-aliasing=1