diff --git a/lib/common/zstd_internal.h b/lib/common/zstd_internal.h index a68c7cbce..53631b936 100644 --- a/lib/common/zstd_internal.h +++ b/lib/common/zstd_internal.h @@ -304,7 +304,6 @@ struct ZSTD_CCtx_params_s { unsigned overlapSizeLog; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ - const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index d6f50e60c..d1d321011 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -215,6 +215,7 @@ size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned } } + #define ZSTD_CLEVEL_CUSTOM 999 static void ZSTD_cLevelToCParams(ZSTD_CCtx* cctx) { @@ -3151,10 +3152,8 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (frame && (cctx->stage==ZSTDcs_init)) { - fhSize = ZSTD_writeFrameHeader( - dst, dstCapacity, - cctx->appliedParams, - cctx->pledgedSrcSizePlusOne-1, cctx->dictID); + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity,cctx->appliedParams, + cctx->pledgedSrcSizePlusOne-1, cctx->dictID); if (ZSTD_isError(fhSize)) return fhSize; dstCapacity -= fhSize; dst = (char*)dst + fhSize; @@ -3444,8 +3443,7 @@ size_t ZSTD_compressBegin_advanced_internal( * @return : 0, or an error code */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_parameters params, - unsigned long long pledgedSrcSize) + ZSTD_parameters params, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); @@ -3571,11 +3569,8 @@ size_t ZSTD_compress_advanced_internal( return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } -size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, - size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize, - int compressionLevel) +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, + const void* dict, size_t dictSize, int compressionLevel) { ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0); params.fParams.contentSizeFlag = 1; @@ -3606,8 +3601,7 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, unsigned byReference) { DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict)); - DEBUGLOG(5, "CCtx estimate : %u", - (U32)ZSTD_estimateCCtxSize_advanced(cParams)); + DEBUGLOG(5, "CCtx estimate : %u", (U32)ZSTD_estimateCCtxSize_advanced(cParams)); return sizeof(ZSTD_CDict) + ZSTD_estimateCCtxSize_advanced(cParams) + (byReference ? 0 : dictSize); } @@ -3645,7 +3639,7 @@ static size_t ZSTD_initCDict_internal( } cdict->dictContentSize = dictSize; - { ZSTD_frameParameters const fParams = { 0 /*contentSizeFlag */, + { ZSTD_frameParameters const fParams = { 0 /* contentSizeFlag */, 0 /* checksumFlag */, 0 /* noDictIDFlag */ }; /* dummy */ /* TODO: correct? */ ZSTD_CCtx_params cctxParams = cdict->refContext->requestedParams; @@ -3666,7 +3660,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference, ZSTD_dictMode_e dictMode, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { - DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", dictMode); + DEBUGLOG(5, "ZSTD_createCDict_advanced, mode %u", (U32)dictMode); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); @@ -3677,8 +3671,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, return NULL; } cdict->refContext = cctx; - if (ZSTD_isError( ZSTD_initCDict_internal( - cdict, + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dictBuffer, dictSize, byReference, dictMode, cParams) )) { @@ -3735,8 +3728,7 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, ZSTD_compressionParameters cParams) { size_t const cctxSize = ZSTD_estimateCCtxSize_advanced(cParams); - size_t const neededSize = sizeof(ZSTD_CDict) - + (byReference ? 0 : dictSize) + size_t const neededSize = sizeof(ZSTD_CDict) + (byReference ? 0 : dictSize) + cctxSize; ZSTD_CDict* const cdict = (ZSTD_CDict*) workspace; void* ptr; @@ -3755,9 +3747,10 @@ ZSTD_CDict* ZSTD_initStaticCDict(void* workspace, size_t workspaceSize, } cdict->refContext = ZSTD_initStaticCCtx(ptr, cctxSize); - if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, - 1 /* byReference */, - dictMode, cParams) )) + if (ZSTD_isError( ZSTD_initCDict_internal(cdict, + dict, dictSize, + 1 /* byReference */, dictMode, + cParams) )) return NULL; return cdict; @@ -3897,12 +3890,10 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) * Note : not static, but hidden (not exposed). Used by zstdmt_compress.c * Assumption 1 : params are valid * Assumption 2 : either dict, or cdict, is defined, not both */ -size_t ZSTD_initCStream_internal( - ZSTD_CStream* zcs, - const void* dict, size_t dictSize, - const ZSTD_CDict* cdict, - ZSTD_CCtx_params params, - unsigned long long pledgedSrcSize) +size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + const ZSTD_CDict* cdict, + ZSTD_CCtx_params params, unsigned long long pledgedSrcSize) { assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); assert(!((dict) && (cdict))); /* either dict or cdict, not both */ @@ -3931,10 +3922,10 @@ size_t ZSTD_initCStream_internal( zcs->cdictLocal = NULL; zcs->cdict = cdict; } + zcs->requestedParams = params; - return ZSTD_resetCStream_internal( - zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize); + return ZSTD_resetCStream_internal(zcs, NULL, 0, zcs->cdict, params, pledgedSrcSize); } /* ZSTD_initCStream_usingCDict_advanced() : @@ -4167,6 +4158,7 @@ size_t ZSTDMT_initCStream_internal(ZSTDMT_CCtx* zcs, const void* dict, size_t dictSize, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, unsigned long long pledgedSrcSize); + size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, @@ -4182,7 +4174,6 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, const void* const prefix = cctx->prefix; size_t const prefixSize = cctx->prefixSize; ZSTD_CCtx_params params = cctx->requestedParams; - if (params.compressionLevel != ZSTD_CLEVEL_CUSTOM) params.cParams = ZSTD_getCParams(params.compressionLevel, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index 4b8360933..a52cb8aa0 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -62,7 +62,7 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void) DEBUGLOG(MUTEX_WAIT_TIME_DLEVEL, "Thread took %llu microseconds to acquire mutex %s \n", \ elapsedTime, #mutex); \ } } \ - } else { pthread_mutex_lock(mutex); } \ + } else pthread_mutex_lock(mutex); \ } #else @@ -743,7 +743,7 @@ size_t ZSTDMT_initCStream_internal( ZSTD_freeCDict(zcs->cdictLocal); /* TODO: cctxParam version? Is this correct? */ zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, - 0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */ + 0 /* byRef */, ZSTD_dm_auto, /* note : a loadPrefix becomes an internal CDict */ requestedParams.cParams, zcs->cMem); zcs->cdict = zcs->cdictLocal; if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); diff --git a/lib/zstd.h b/lib/zstd.h index 3e7574c78..e207e46bf 100644 --- a/lib/zstd.h +++ b/lib/zstd.h @@ -987,7 +987,7 @@ typedef enum { /* advanced parameters - may not remain available after API update */ ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, - * even when referencing into Dictionary content (default:0) */ + * even when referencing into Dictionary content (default:0) */ } ZSTD_cParameter; @@ -1101,6 +1101,7 @@ size_t ZSTD_compress_generic_simpleArgs ( const void* src, size_t srcSize, size_t* srcPos, ZSTD_EndDirective endOp); + /** ZSTD_CCtx_params : * * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure diff --git a/programs/fileio.c b/programs/fileio.c index 669052d8b..1dd8008e8 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -214,6 +214,7 @@ void FIO_setOverlapLog(unsigned overlapLog){ g_overlapLog = overlapLog; } + /*-************************************* * Functions ***************************************/ diff --git a/tests/zstreamtest.c b/tests/zstreamtest.c index 8f974b461..98e121949 100644 --- a/tests/zstreamtest.c +++ b/tests/zstreamtest.c @@ -1204,298 +1204,6 @@ _output_error: goto _cleanup; } - -/* Tests for ZSTD_compress_generic() API */ -#if 0 -static int fuzzerTests_newAPI(U32 seed, U32 nbTests, unsigned startTest, double compressibility, int bigTests) -{ - U32 const maxSrcLog = bigTests ? 24 : 22; - static const U32 maxSampleLog = 19; - size_t const srcBufferSize = (size_t)1<= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } - else { DISPLAYUPDATE(2, "\r%6u ", testNb); } - FUZ_rand(&coreSeed); - lseed = coreSeed ^ prime32; - - /* states full reset (deliberately not synchronized) */ - /* some issues can only happen when reusing states */ - if ((FUZ_rand(&lseed) & 0xFF) == 131) { - DISPLAYLEVEL(5, "Creating new context \n"); - ZSTD_freeCCtx(zc); - zc = ZSTD_createCCtx(); - CHECK(zc==NULL, "ZSTD_createCCtx allocation error"); - resetAllowed=0; - } - if ((FUZ_rand(&lseed) & 0xFF) == 132) { - ZSTD_freeDStream(zd); - zd = ZSTD_createDStream(); - CHECK(zd==NULL, "ZSTD_createDStream allocation error"); - ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ - } - - /* srcBuffer selection [0-4] */ - { U32 buffNb = FUZ_rand(&lseed) & 0x7F; - if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ - else { - buffNb >>= 3; - if (buffNb & 7) { - const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ - buffNb = tnb[buffNb >> 3]; - } else { - const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ - buffNb = tnb[buffNb >> 3]; - } } - srcBuffer = cNoiseBuffer[buffNb]; - } - - /* compression init */ - CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/ - if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ - && oldTestLog /* at least one test happened */ && resetAllowed) { - maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); - if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; - { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_compressionLevel, compressionLevel) ); - } - } else { - U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; - U32 const cLevelCandidate = (FUZ_rand(&lseed) % - (ZSTD_maxCLevel() - - (MAX(testLog, dictLog) / 3))) + - 1; - U32 const cLevel = MIN(cLevelCandidate, cLevelMax); - maxTestSize = FUZ_rLogLength(&lseed, testLog); - oldTestLog = testLog; - /* random dictionary selection */ - dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; - { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); - dict = srcBuffer + dictStart; - if (!dictSize) dict=NULL; - } - { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; - ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); - - /* mess with compression parameters */ - cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; - cParams.searchLength += (FUZ_rand(&lseed) & 3) - 1; - cParams.targetLength = (U32)(cParams.targetLength * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); - cParams = ZSTD_adjustCParams(cParams, 0, 0); - - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_hashLog, cParams.hashLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_chainLog, cParams.chainLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_searchLog, cParams.searchLog) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_minMatch, cParams.searchLength) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_targetLength, cParams.targetLength) ); - - /* unconditionally set, to be sync with decoder */ - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_refDictContent, FUZ_rand(&lseed) & 1) ); - if (FUZ_rand(&lseed) & 1) { - CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); - if (dict && dictSize) { - /* test that compression parameters are rejected (correctly) after loading a non-NULL dictionary */ - size_t const setError = ZSTD_CCtx_setParameter(zc, ZSTD_p_windowLog, cParams.windowLog-1) ; - CHECK(!ZSTD_isError(setError), "ZSTD_CCtx_setParameter should have failed"); - } } else { - CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); - } - - /* mess with frame parameters */ - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_checksumFlag, FUZ_rand(&lseed) & 1) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_dictIDFlag, FUZ_rand(&lseed) & 1) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_contentSizeFlag, FUZ_rand(&lseed) & 1) ); - if (FUZ_rand(&lseed) & 1) CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); - DISPLAYLEVEL(5, "pledgedSrcSize : %u \n", (U32)pledgedSrcSize); - - /* multi-threading parameters */ - { U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; - U32 const nbThreads = MIN(nbThreadsCandidate, nbThreadsMax); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_nbThreads, nbThreads) ); - if (nbThreads > 1) { - U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_overlapSizeLog, FUZ_rand(&lseed) % 10) ); - CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_p_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog)) ); - } } } } - - /* multi-segments compression test */ - XXH64_reset(&xxhState, 0); - { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; - for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) { - /* compress random chunks into randomly sized dst buffers */ - size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); - size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); - size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); - ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush; - ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; - outBuff.size = outBuff.pos + dstBuffSize; - - CHECK_Z( ZSTD_compress_generic(zc, &outBuff, &inBuff, flush) ); - DISPLAYLEVEL(5, "compress consumed %u bytes (total : %u) \n", - (U32)inBuff.pos, (U32)(totalTestSize + inBuff.pos)); - - XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); - memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); - totalTestSize += inBuff.pos; - } - - /* final frame epilogue */ - { size_t remainingToFlush = (size_t)(-1); - while (remainingToFlush) { - ZSTD_inBuffer inBuff = { NULL, 0, 0 }; - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); - size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); - outBuff.size = outBuff.pos + adjustedDstSize; - DISPLAYLEVEL(5, "End-flush into dst buffer of size %u \n", (U32)adjustedDstSize); - remainingToFlush = ZSTD_compress_generic(zc, &outBuff, &inBuff, ZSTD_e_end); - CHECK(ZSTD_isError(remainingToFlush), - "ZSTD_compress_generic w/ ZSTD_e_end error : %s", - ZSTD_getErrorName(remainingToFlush) ); - } } - crcOrig = XXH64_digest(&xxhState); - cSize = outBuff.pos; - DISPLAYLEVEL(5, "Frame completed : %u bytes \n", (U32)cSize); - } - - /* multi - fragments decompression test */ - if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { - DISPLAYLEVEL(5, "resetting DCtx (dict:%08X) \n", (U32)(size_t)dict); - CHECK_Z( ZSTD_resetDStream(zd) ); - } else { - DISPLAYLEVEL(5, "using dict of size %u \n", (U32)dictSize); - CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); - } - { size_t decompressionResult = 1; - ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; - ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; - for (totalGenSize = 0 ; decompressionResult ; ) { - size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); - size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); - inBuff.size = inBuff.pos + readCSrcSize; - outBuff.size = inBuff.pos + dstBuffSize; - DISPLAYLEVEL(5, "ZSTD_decompressStream input %u bytes (pos:%u/%u)\n", - (U32)readCSrcSize, (U32)inBuff.pos, (U32)cSize); - decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); - CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); - DISPLAYLEVEL(5, "inBuff.pos = %u \n", (U32)readCSrcSize); - } - CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (U32)outBuff.pos, (U32)totalTestSize); - CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (U32)inBuff.pos, (U32)cSize); - { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); - if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); - CHECK (crcDest!=crcOrig, "decompressed data corrupted"); - } } - - /*===== noisy/erroneous src decompression test =====*/ - - /* add some noise */ - { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; - U32 nn; for (nn=0; nn