From 15fdeb9e41306e8bb8d6a909a7be643768d75e6b Mon Sep 17 00:00:00 2001 From: Stella Lau Date: Thu, 24 Aug 2017 16:28:49 -0700 Subject: [PATCH] Enforce nbThreads<=1 for estimateCCtxSize --- lib/compress/zstd_compress.c | 63 ++++++++++++++++++++---------------- lib/zstd.h | 7 ++-- tests/Makefile | 1 + 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index f3f602230..4513d9831 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -678,29 +678,34 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u size_t ZSTD_estimateCCtxSize_advanced_opaque(const ZSTD_CCtx_params* params) { - ZSTD_compressionParameters const cParams = params->cParams; - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); - U32 const divider = (cParams.searchLength==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = blockSize + 11*maxNbSeq; - size_t const chainSize = - (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); - size_t const hSize = ((size_t)1) << cParams.hashLog; - U32 const hashLog3 = (cParams.searchLength>3) ? - 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + /* Estimate CCtx size is supported for single-threaded compression only. */ + if (params->nbThreads > 1) { + return 0; + } + { ZSTD_compressionParameters const cParams = params->cParams; + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + U32 const divider = (cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + size_t const chainSize = + (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); + size_t const hSize = ((size_t)1) << cParams.hashLog; + U32 const hashLog3 = (cParams.searchLength>3) ? + 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const entropySpace = sizeof(ZSTD_entropyCTables_t); + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - size_t const optBudget = - ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<cParams.windowLog); - size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; - size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; - size_t const streamingSize = inBuffSize + outBuffSize; + if (params->nbThreads > 1) { + return 0; + } + { size_t const CCtxSize = ZSTD_estimateCCtxSize_advanced_opaque(params); + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << params->cParams.windowLog); + size_t const inBuffSize = ((size_t)1 << params->cParams.windowLog) + blockSize; + size_t const outBuffSize = ZSTD_compressBound(blockSize) + 1; + size_t const streamingSize = inBuffSize + outBuffSize; - return CCtxSize + streamingSize; + return CCtxSize + streamingSize; + } } size_t ZSTD_estimateCStreamSize_advanced(ZSTD_compressionParameters cParams) diff --git a/lib/zstd.h b/lib/zstd.h index 4cff6974b..03994f9a8 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -498,7 +498,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCCtxSize_advanced() can provide a tighter estimation. * ZSTD_estimateCCtxSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_advanced_opaque() can be used in tandem with ZSTD_CCtxParam_setParameter(). + * ZSTD_estimateCCtxSize_advanced_opaque() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return 0 if ZSTD_p_nbThreads is set to a value > 1. * Note : CCtx estimation is only correct for single-threaded compression */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_advanced(ZSTD_compressionParameters cParams); @@ -510,7 +510,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_advanced() can provide a tighter estimation. * ZSTD_estimateCStreamSize_advanced() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_advanced_opaque() can be used in tandem with ZSTD_CCtxParam_setParameter(). + * ZSTD_estimateCStreamSize_advanced_opaque() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return 0 if ZSTD_p_nbThreads is set to a value > 1. * Note : CStream estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -1113,7 +1113,8 @@ size_t ZSTD_compress_generic_simpleArgs ( * - ZSTD_compress_generic() : Do compression using the CCtx. * - ZSTD_freeCCtxParams() : Free the memory. * - * This can be used with ZSTD_estimateCCtxSize_opaque() for static allocation. */ + * This can be used with ZSTD_estimateCCtxSize_opaque() for static allocation + * for single-threaded compression. */ ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); diff --git a/tests/Makefile b/tests/Makefile index 3734f7737..006059b72 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -323,6 +323,7 @@ test-zstream: zstreamtest $(QEMU_SYS) ./zstreamtest $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) $(QEMU_SYS) ./zstreamtest --mt $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) $(QEMU_SYS) ./zstreamtest --newapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest --opaqueapi $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) test-zstream32: zstreamtest32 $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS)