mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-24 09:56:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Adjust advanced parameters to source size

Browse Source 
In the new advanced API, adjust the parameters even if they are explicitly
set. This mainly applies to the `windowLog`, and accordingly the `hashLog`
and `chainLog`, when the source size is known.
This commit is contained in:
Nick Terrell 2018-06-14 16:24:18 -07:00
parent f70f829ff5
commit 3841dbac84
4 changed files with 27 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
lib/compress/zstd_compress.c
View File

@ -150,21 +150,6 @@ size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
/* private API call, for dictBuilder only */
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) { return &(ctx->seqStore); }
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize)
{
ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize);
if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog;
if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog;
if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog;
if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog;
if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength;
if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength;
if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy;
return cParams;
}
static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams(
ZSTD_compressionParameters cParams)
{
@ -761,6 +746,22 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar,
return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize);
}
ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams(
const ZSTD_CCtx_params* CCtxParams, U64 srcSizeHint, size_t dictSize)
{
ZSTD_compressionParameters cParams = ZSTD_getCParams(CCtxParams->compressionLevel, srcSizeHint, dictSize);
if (CCtxParams->ldmParams.enableLdm) cParams.windowLog = ZSTD_LDM_DEFAULT_WINDOW_LOG;
if (CCtxParams->cParams.windowLog) cParams.windowLog = CCtxParams->cParams.windowLog;
if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog;
if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog;
if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog;
if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength;
if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength;
if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy;
assert(!ZSTD_checkCParams(cParams));
return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize);
}
static size_t
ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams,
const U32 forCCtx)
@ -1078,7 +1079,6 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
if (params.ldmParams.enableLdm) {
/* Adjust long distance matching parameters */
params.ldmParams.windowLog = params.cParams.windowLog;
ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
assert(params.ldmParams.hashEveryLog < 32);
@ -3638,6 +3638,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
params.cParams = ZSTD_getCParamsFromCCtxParams(
&cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
#ifdef ZSTD_MULTITHREAD
if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) {
params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */

10
lib/compress/zstd_ldm.c
View File

@ -9,6 +9,7 @@
#include "zstd_ldm.h"
#include "debug.h"
#include "zstd_fast.h" /* ZSTD_fillHashTable() */
#include "zstd_double_fast.h" /* ZSTD_fillDoubleHashTable() */
@ -20,7 +21,7 @@
void ZSTD_ldm_adjustParameters(ldmParams_t* params,
ZSTD_compressionParameters const* cParams)
{
U32 const windowLog = cParams->windowLog;
params->windowLog = cParams->windowLog;
ZSTD_STATIC_ASSERT(LDM_BUCKET_SIZE_LOG <= ZSTD_LDM_BUCKETSIZELOG_MAX);
DEBUGLOG(4, "ZSTD_ldm_adjustParameters");
if (!params->bucketSizeLog) params->bucketSizeLog = LDM_BUCKET_SIZE_LOG;
@ -33,12 +34,13 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
params->minMatchLength = minMatch;
}
if (params->hashLog == 0) {
params->hashLog = MAX(ZSTD_HASHLOG_MIN, windowLog - LDM_HASH_RLOG);
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
}
if (params->hashEveryLog == 0) {
params->hashEveryLog =
windowLog < params->hashLog ? 0 : windowLog - params->hashLog;
params->hashEveryLog = params->windowLog < params->hashLog
? 0
: params->windowLog - params->hashLog;
}
params->bucketSizeLog = MIN(params->bucketSizeLog, params->hashLog);
}

1
lib/compress/zstdmt_compress.c
View File

@ -429,7 +429,6 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
/* Adjust parameters */
if (params.ldmParams.enableLdm) {
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
params.ldmParams.windowLog = params.cParams.windowLog;
ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
assert(params.ldmParams.hashEveryLog < 32);

7
programs/fileio.c
View File

@ -759,8 +759,9 @@ FIO_compressZstdFrame(const cRess_t* ressPtr,
DISPLAYLEVEL(6, "compression using zstd format \n");
/* init */
if (fileSize != UTIL_FILESIZE_UNKNOWN)
ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize);
if (fileSize != UTIL_FILESIZE_UNKNOWN) {
CHECK(ZSTD_CCtx_setPledgedSrcSize(ress.cctx, fileSize));
}
(void)compressionLevel; (void)srcFileName;
/* Main compression loop */
@ -1755,7 +1756,7 @@ int FIO_decompressMultipleFilenames(const char** srcNamesTable, unsigned nbFiles
&& strcmp(suffixPtr, LZMA_EXTENSION)
&& strcmp(suffixPtr, LZ4_EXTENSION)) ) {
const char* suffixlist = ZSTD_EXTENSION
#ifdef ZSTD_GZCOMPRESS
#ifdef ZSTD_GZCOMPRESS
"/" GZ_EXTENSION
#endif
#ifdef ZSTD_LZMACOMPRESS