From 4bf317dd0099bd11690ebee4d72f7ac6919e5964 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sun, 28 Aug 2016 07:43:34 -0700 Subject: [PATCH] first version supporting legacy streams (transparent decoding) --- lib/common/error_private.h | 1 + lib/common/error_public.h | 3 +- lib/decompress/zstd_decompress.c | 50 ++++++++- lib/legacy/zstd_legacy.h | 186 +++++++++++++++++++++++++++---- lib/zstd.h | 6 +- 5 files changed, 215 insertions(+), 31 deletions(-) diff --git a/lib/common/error_private.h b/lib/common/error_private.h index 34c0c4c04..977fe3d45 100644 --- a/lib/common/error_private.h +++ b/lib/common/error_private.h @@ -93,6 +93,7 @@ ERR_STATIC const char* ERR_getErrorString(ERR_enum code) case PREFIX(no_error): return "No error detected"; case PREFIX(GENERIC): return "Error (generic)"; case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(parameter_unknown): return "Unknown parameter type"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; diff --git a/lib/common/error_public.h b/lib/common/error_public.h index 10f020cfe..5a6d7107c 100644 --- a/lib/common/error_public.h +++ b/lib/common/error_public.h @@ -41,13 +41,14 @@ extern "C" { #include /* size_t */ -/* **************************************** +/*-**************************************** * error codes list ******************************************/ typedef enum { ZSTD_error_no_error, ZSTD_error_GENERIC, ZSTD_error_prefix_unknown, + ZSTD_error_version_unsupported, ZSTD_error_parameter_unknown, ZSTD_error_frameParameter_unsupported, ZSTD_error_frameParameter_unsupportedBy32bits, diff --git a/lib/decompress/zstd_decompress.c b/lib/decompress/zstd_decompress.c index b96f487ab..5b95de96f 100644 --- a/lib/decompress/zstd_decompress.c +++ b/lib/decompress/zstd_decompress.c @@ -1308,8 +1308,8 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); #endif return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); + dst, dstCapacity, + src, srcSize); } @@ -1337,6 +1337,11 @@ struct ZSTD_DStream_s { BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; size_t lhSize; ZSTD_customMem customMem; + void* dictContent; + size_t dictSize; + const void* dictSource; + void* legacyContext; + U32 legacyVersion; }; /* typedef'd to ZSTD_DStream within "zstd.h" */ @@ -1372,6 +1377,13 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) ZSTD_freeDCtx(zds->zd); if (zds->inBuff) zds->customMem.customFree(zds->customMem.opaque, zds->inBuff); if (zds->outBuff) zds->customMem.customFree(zds->customMem.opaque, zds->outBuff); + if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (zds->legacyContext) { + ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->legacyVersion); + zds->legacyContext = NULL; + } +#endif zds->customMem.customFree(zds->customMem.opaque, zds); return 0; } @@ -1386,7 +1398,18 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - return ZSTD_decompressBegin_usingDict(zds->zd, dict, dictSize); + if ((dict != zds->dictSource) || (dictSize != zds->dictSize)) { /* new dictionary */ + if (dictSize > zds->dictSize) { + if (zds->dictContent) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); + zds->dictContent = zds->customMem.customAlloc(zds->customMem.opaque, dictSize); + if (zds->dictContent == NULL) return ERROR(memory_allocation); + } + memcpy(zds->dictContent, dict, dictSize); + zds->dictSize = dictSize; + } + if (zds->legacyContext) zds->customMem.customFree(zds->customMem.opaque, zds->dictContent); /* legacy restarts from scratch on each call, to detect restart */ + zds->legacyVersion = 0; + return 0; } size_t ZSTD_initDStream(ZSTD_DStream* zds) @@ -1432,6 +1455,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB char* op = ostart; U32 someMoreWork = 1; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) + return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input); +#endif + while (someMoreWork) { switch(zds->stage) { @@ -1439,8 +1467,19 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB return ERROR(init_missing); case zdss_loadHeader : - { size_t const hSize = ZSTD_getFrameParams(&(zds->fParams), zds->headerBuffer, zds->lhSize); - if (ZSTD_isError(hSize)) return hSize; + { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart) + if (legacyVersion) { + zds->legacyVersion = legacyVersion; + return ZSTD_decompressLegacyStream(&zds->legacyContext, zds->legacyVersion, output, input); + } else { + return hSize; /* error */ + } } +#else + return hSize; +#endif if (hSize != 0) { /* need more input */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ @@ -1454,6 +1493,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } /* Consume header */ + ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize); { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ diff --git a/lib/legacy/zstd_legacy.h b/lib/legacy/zstd_legacy.h index 6c2a1019b..a5a767e49 100644 --- a/lib/legacy/zstd_legacy.h +++ b/lib/legacy/zstd_legacy.h @@ -42,6 +42,7 @@ extern "C" { ***************************************/ #include "mem.h" /* MEM_STATIC */ #include "error_private.h" /* ERROR */ +#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer */ #include "zstd_v01.h" #include "zstd_v02.h" #include "zstd_v03.h" @@ -76,32 +77,30 @@ MEM_STATIC unsigned ZSTD_isLegacy(const void* src, size_t srcSize) MEM_STATIC unsigned long long ZSTD_getDecompressedSize_legacy(const void* src, size_t srcSize) { - if (srcSize < 4) return 0; - - { U32 const version = ZSTD_isLegacy(src, srcSize); - if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ - if (version==5) { - ZSTDv05_parameters fParams; - size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.srcSize; - } - if (version==6) { - ZSTDv06_frameParams fParams; - size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } - if (version==7) { - ZSTDv07_frameParams fParams; - size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); - if (frResult != 0) return 0; - return fParams.frameContentSize; - } - return 0; /* should not be possible */ + U32 const version = ZSTD_isLegacy(src, srcSize); + if (version < 5) return 0; /* no decompressed size in frame header, or not a legacy format */ + if (version==5) { + ZSTDv05_parameters fParams; + size_t const frResult = ZSTDv05_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.srcSize; } + if (version==6) { + ZSTDv06_frameParams fParams; + size_t const frResult = ZSTDv06_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } + if (version==7) { + ZSTDv07_frameParams fParams; + size_t const frResult = ZSTDv07_getFrameParams(&fParams, src, srcSize); + if (frResult != 0) return 0; + return fParams.frameContentSize; + } + return 0; /* should not be possible */ } + MEM_STATIC size_t ZSTD_decompressLegacy( void* dst, size_t dstCapacity, const void* src, size_t compressedSize, @@ -148,6 +147,147 @@ MEM_STATIC size_t ZSTD_decompressLegacy( } +MEM_STATIC void* ZSTD_createLegacyStreamContext(U32 version) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return NULL; + case 4 : return ZBUFFv04_createDCtx(); + case 5 : return ZBUFFv05_createDCtx(); + case 6 : return ZBUFFv06_createDCtx(); + case 7 : return ZBUFFv07_createDCtx(); + } +} + +MEM_STATIC size_t ZSTD_freeLegacyStreamContext(void* legacyContext, U32 version) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return ERROR(version_unsupported); + case 4 : return ZBUFFv04_freeDCtx((ZBUFFv04_DCtx*)legacyContext); + case 5 : return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx*)legacyContext); + case 6 : return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx*)legacyContext); + case 7 : return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx*)legacyContext); + } +} + + +MEM_STATIC void ZSTD_initLegacyStream(void* legacyContext, U32 version, + const void* dict, size_t dictSize) +{ + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return; + case 4 : + { + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) legacyContext; + ZBUFFv04_decompressInit(dctx); + ZBUFFv04_decompressWithDictionary(dctx, dict, dictSize); + return; + } + case 5 : + { + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) legacyContext; + ZBUFFv05_decompressInitDictionary(dctx, dict, dictSize); + return; + } + case 6 : + { + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) legacyContext; + ZBUFFv06_decompressInitDictionary(dctx, dict, dictSize); + return; + } + case 7 : + { + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) legacyContext; + ZBUFFv07_decompressInitDictionary(dctx, dict, dictSize); + return; + } + } +} + + + +MEM_STATIC size_t ZSTD_decompressLegacyStream(void** legacyContext, U32 version, + ZSTD_outBuffer* output, ZSTD_inBuffer* input, + const void* dict, size_t dictSize) +{ + if (*legacyContext == NULL) { + *legacyContext = ZSTD_createLegacyStreamContext(version); + if (*legacyContext==NULL) return ERROR(memory_allocation); + ZSTD_initLegacyStream(*legacyContext, version, dict, dictSize); + } + + switch(version) + { + default : + case 1 : + case 2 : + case 3 : + return ERROR(version_unsupported); + case 4 : + { + ZBUFFv04_DCtx* dctx = (ZBUFFv04_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv04_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + case 5 : + { + ZBUFFv05_DCtx* dctx = (ZBUFFv05_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv05_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + case 6 : + { + ZBUFFv06_DCtx* dctx = (ZBUFFv06_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv06_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + case 7 : + { + ZBUFFv07_DCtx* dctx = (ZBUFFv07_DCtx*) (*legacyContext); + const void* src = (const char*)input->src + input->pos; + size_t readSize = input->size - input->pos; + void* dst = (char*)output->dst + output->pos; + size_t decodedSize = output->size - output->pos; + size_t const hintSize = ZBUFFv07_decompressContinue(dctx, dst, &decodedSize, src, &readSize); + output->pos += decodedSize; + input->pos += readSize; + return hintSize; + } + } +} + #if defined (__cplusplus) } diff --git a/lib/zstd.h b/lib/zstd.h index 706bc09b5..1f3dddb2e 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -208,7 +208,7 @@ typedef struct ZSTD_outBuffer_s { } ZSTD_outBuffer; -/*====== compression ======*/ +/*====== streaming compression ======*/ /*-*********************************************************************** * Streaming compression - howto @@ -225,7 +225,9 @@ typedef struct ZSTD_outBuffer_s { * The function will automatically update both `pos`. * Note that it may not consume the entire input, in which case `pos < size`, * and it's up to the caller to present again remaining data. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency) +* @return : a size hint, preferred nb of bytes to use as input for next function call +* (it's just a hint, to help latency a little, any other value will work fine) +* (note : the size hint is guaranteed to be <= ZSTD_CStreamInSize() ) * or an error code, which can be tested using ZSTD_isError(). * * At any moment, it's possible to flush whatever data remains within buffer, using ZSTD_flushStream().