mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-24 20:16:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

added function to control monotonic memory budget increase of ZSTD_defaultCParameters[0]

Browse Source 
It's a runtime test, based on assert(),
played once, on first ZSTD_getCParams() usage,
when ZSTD_DEBUG is enabled.
This commit is contained in:
Yann Collet 2017-06-28 15:34:56 -07:00
parent adbe74a8ac
commit 813535105b
3 changed files with 58 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/zstd_manual.html
View File

@ -412,9 +412,9 @@ size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDCtxSize(void);
</b><p> These functions make it possible to estimate memory usage
of a future {D,C}Ctx, before its creation.
The objective is to guide decision before allocation.
ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
ZSTD_estimateCCtxSize() will provide a budget large for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
Note : CCtx estimation is only correct for single-threaded compression
</p></pre><BR>
@ -423,8 +423,9 @@ size_t ZSTD_estimateDCtxSize(void);
size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams);
size_t ZSTD_estimateDStreamSize(size_t windowSize);
size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize);
</b><p> ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
</b><p> ZSTD_estimateCStreamSize() will provide a budget large for any compression level up to selected one.
It will also consider src size to be arbitrarily "large", which is worst case.
If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
Note : CStream estimation is only correct for single-threaded compression.
ZSTD_DStream memory budget depends on window Size.

46
lib/compress/zstd_compress.c
View File

@ -4098,6 +4098,43 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
},
};
/* This function just controls
* the monotonic memory budget increase of ZSTD_defaultCParameters[0].
* Run only once, on first ZSTD_getCParams() usage, when ZSTD_DEBUG is enabled
*/
MEM_STATIC void ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget(void)
{
# define ZSTD_TABLECOST(h,c) ((1<<(h)) + (1<<(c)))
# define ZDCP_FIELD(l,field) (ZSTD_defaultCParameters[0][l].field)
# define ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(l) { \
assert(ZDCP_FIELD(l,windowLog) <= ZDCP_FIELD(l+1,windowLog) ); \
assert(ZSTD_TABLECOST(ZDCP_FIELD(l,hashLog), ZDCP_FIELD(l,chainLog)) <= ZSTD_TABLECOST(ZDCP_FIELD(l+1,hashLog), ZDCP_FIELD(l+1,chainLog)) ); \
}
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(1);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(2);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(3);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(4);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(5);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(6);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(7);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(8);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(9);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(10);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(11);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(12);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(13);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(14);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(15);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(16);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(17);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(18);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(19);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(20);
ZSTD_CHECK_MONOTONIC_INCREASE_LEVEL(21);
assert(ZSTD_maxCLevel()==22);
}
/*! ZSTD_getCParams() :
* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`.
* Size values are optional, provide 0 if not known or unused */
@ -4108,6 +4145,15 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l
U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */
if (compressionLevel <= 0) compressionLevel = ZSTD_CLEVEL_DEFAULT; /* 0 == default; no negative compressionLevel yet */
if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL;
#if defined(ZSTD_DEBUG) && (ZSTD_DEBUG>=1)
static int g_monotonicTest = 1;
if (g_monotonicTest) {
ZSTD_check_compressionLevel_monotonicIncrease_memoryBudget();
g_monotonicTest=0;
}
#endif
{ ZSTD_compressionParameters const cp = ZSTD_defaultCParameters[tableID][compressionLevel];
return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); }
}

11
lib/zstd.h
View File

@ -496,9 +496,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/*! ZSTD_estimate*() :
* These functions make it possible to estimate memory usage
* of a future {D,C}Ctx, before its creation.
* The objective is to guide decision before allocation.
* ZSTD_estimateCCtxSize() will consider src size to be arbitrarily "large".
* If srcSize is known to be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* Note : CCtx estimation is only correct for single-threaded compression */
ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel);
@ -506,8 +506,9 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cPa
ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
/*! ZSTD_estimate?StreamSize() :
* ZSTD_estimateCStreamSize() will consider src size to be arbitrarily "large".
* If srcSize is known to be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize() will provide a budget large enough for any compression level up to selected one.
* It will also consider src size to be arbitrarily "large", which is worst case.
* If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation.
* ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel.
* Note : CStream estimation is only correct for single-threaded compression.
* ZSTD_DStream memory budget depends on window Size.