diff --git a/lib/common/zstd.h b/lib/common/zstd.h index b52b1de64..47bae1157 100644 --- a/lib/common/zstd.h +++ b/lib/common/zstd.h @@ -441,7 +441,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds User will have to take in charge required information to regenerate data, such as compressed and content sizes. A few rules to respect : - - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) + - Uncompressed block size must be <= MIN (128 KB, 1 << windowLog) + If you need to compress more, cut data into multiple blocks + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. - Compressing and decompressing require a context structure diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index 98239b5d1..52dc72dd7 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -274,9 +274,9 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) note : 'params' is expected to be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params, U64 frameContentSize, - U32 reset, U32 fullBlockSize) + U32 reset) { /* note : params considered validated here */ - const size_t blockSize = fullBlockSize ? ZSTD_BLOCKSIZE_MAX : MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); + const size_t blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params.cParams.windowLog); const U32 divider = (params.cParams.searchLength==3) ? 3 : 4; const size_t maxNbSeq = blockSize / divider; const size_t tokenSpace = blockSize + 11*maxNbSeq; @@ -358,7 +358,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) if (srcCCtx->stage!=1) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0, 1); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ /* copy tables */ @@ -2560,7 +2560,8 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* zc, size_t ZSTD_compressBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - if (srcSize > ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << zc->params.cParams.windowLog); + if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); ZSTD_LOG_BLOCK("%p: ZSTD_compressBlock searchLength=%d\n", zc->base, zc->params.cParams.searchLength); return ZSTD_compressContinue_internal(zc, dst, dstCapacity, src, srcSize, 0); } @@ -2695,7 +2696,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1, (pledgedSrcSize==0) ); + size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); if (ZSTD_isError(resetError)) return resetError; return ZSTD_compress_insertDictionary(zc, dict, dictSize); diff --git a/lib/dictBuilder/zdict.c b/lib/dictBuilder/zdict.c index 27e95d76a..0378a313a 100644 --- a/lib/dictBuilder/zdict.c +++ b/lib/dictBuilder/zdict.c @@ -581,13 +581,14 @@ typedef struct #define MAXREPOFFSET 1024 -static void ZDICT_countEStats(EStats_ress_t esr, +static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, const void* src, size_t srcSize) { + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_MAX, 1 << params.cParams.windowLog); size_t cSize; - if (srcSize > ZSTD_BLOCKSIZE_MAX) srcSize = ZSTD_BLOCKSIZE_MAX; /* protection vs large samples */ + if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } } @@ -721,7 +722,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, /* collect stats on all files */ for (u=0; u