From 70e8c389f1099d4601c6b5d3c14abf79b76aa732 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Wed, 10 Feb 2016 13:37:52 +0100 Subject: [PATCH] Refactoring for portability --- lib/zstd_compress.c | 81 +++---- lib/zstd_opt.c | 511 +++++++++++++++++------------------------- lib/zstd_static.h | 9 +- programs/paramgrill.c | 240 ++++++++------------ 4 files changed, 345 insertions(+), 496 deletions(-) diff --git a/lib/zstd_compress.c b/lib/zstd_compress.c index 47f655946..7f8385b37 100644 --- a/lib/zstd_compress.c +++ b/lib/zstd_compress.c @@ -88,31 +88,33 @@ typedef struct { BYTE* matchLength; BYTE* dumpsStart; BYTE* dumps; + /* opt */ U32* matchLengthFreq; U32* litLengthFreq; U32* litFreq; U32* offCodeFreq; - U32 matchLengthSum; - U32 litLengthSum; - U32 litSum; - U32 offCodeSum; + U32 matchLengthSum; + U32 litLengthSum; + U32 litSum; + U32 offCodeSum; } seqStore_t; static void ZSTD_resetFreqs(seqStore_t* ssPtr) { - ssPtr->matchLengthSum = (1<litLengthSum = (1<matchLengthSum = 512; // (1<litLengthSum = 256; // (1<litSum = (1<offCodeSum = (1<litFreq[i] = 1; - for (int i=0; i<=MaxLL; i++) - ssPtr->litLengthFreq[i] = 1; - for (int i=0; i<=MaxML; i++) - ssPtr->matchLengthFreq[i] = 1; - for (int i=0; i<=MaxOff; i++) - ssPtr->offCodeFreq[i] = 1; + for (u=0; u<=MaxLit; u++) + ssPtr->litFreq[u] = 1; + for (u=0; u<=MaxLL; u++) + ssPtr->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + ssPtr->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + ssPtr->offCodeFreq[u] = 1; } static void ZSTD_resetSeqStore(seqStore_t* ssPtr) @@ -173,7 +175,9 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) static unsigned ZSTD_highbit(U32 val); -/** ZSTD_validateParams +#define CLAMP(val,min,max) { if (valmax) val=max; } + +/** ZSTD_validateParams() correct params value to remain within authorized range optimize for srcSize if srcSize > 0 */ void ZSTD_validateParams(ZSTD_parameters* params) @@ -182,25 +186,21 @@ void ZSTD_validateParams(ZSTD_parameters* params) /* validate params */ if (MEM_32bits()) if (params->windowLog > 25) params->windowLog = 25; /* 32 bits mode cannot flush > 24 bits */ - if (params->windowLog > ZSTD_WINDOWLOG_MAX) params->windowLog = ZSTD_WINDOWLOG_MAX; - if (params->windowLog < ZSTD_WINDOWLOG_MIN) params->windowLog = ZSTD_WINDOWLOG_MIN; + CLAMP(params->windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMP(params->contentLog, ZSTD_CONTENTLOG_MIN, ZSTD_CONTENTLOG_MAX); + CLAMP(params->hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMP(params->searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + CLAMP(params->searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); + CLAMP(params->sufficientLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + if ((U32)params->strategy>(U32)ZSTD_opt_bt) params->strategy = ZSTD_opt_bt; /* correct params, to use less memory */ if ((params->srcSize > 0) && (params->srcSize < (1<srcSize)-1) + 1; if (params->windowLog > srcLog) params->windowLog = srcLog; } - if (params->windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) params->windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (params->contentLog > params->windowLog+btPlus) params->contentLog = params->windowLog+btPlus; /* <= ZSTD_CONTENTLOG_MAX */ - if (params->contentLog < ZSTD_CONTENTLOG_MIN) params->contentLog = ZSTD_CONTENTLOG_MIN; - if (params->hashLog > ZSTD_HASHLOG_MAX) params->hashLog = ZSTD_HASHLOG_MAX; - if (params->hashLog < ZSTD_HASHLOG_MIN) params->hashLog = ZSTD_HASHLOG_MIN; - if (params->searchLog > ZSTD_SEARCHLOG_MAX) params->searchLog = ZSTD_SEARCHLOG_MAX; - if (params->searchLog < ZSTD_SEARCHLOG_MIN) params->searchLog = ZSTD_SEARCHLOG_MIN; - if (params->searchLength> ZSTD_SEARCHLENGTH_MAX) params->searchLength = ZSTD_SEARCHLENGTH_MAX; - if (params->searchLength< ZSTD_SEARCHLENGTH_MIN) params->searchLength = ZSTD_SEARCHLENGTH_MIN; - if ((U32)params->strategy>(U32)ZSTD_opt_bt) params->strategy = ZSTD_opt_bt; } @@ -234,18 +234,19 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, zc->lowLimit = 0; zc->params = params; zc->blockSize = blockSize; - zc->seqStore.offsetStart = (U32*) (zc->seqStore.buffer); + + zc->seqStore.litFreq = (U32*) (zc->seqStore.buffer); + zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<seqStore.offsetStart = zc->seqStore.offCodeFreq + (1<seqStore.offCodeStart = (BYTE*) (zc->seqStore.offsetStart + (blockSize>>2)); zc->seqStore.litStart = zc->seqStore.offCodeStart + (blockSize>>2); zc->seqStore.litLengthStart = zc->seqStore.litStart + blockSize; zc->seqStore.matchLengthStart = zc->seqStore.litLengthStart + (blockSize>>2); zc->seqStore.dumpsStart = zc->seqStore.matchLengthStart + (blockSize>>2); - BYTE* dumpsEnd= zc->seqStore.dumpsStart + (blockSize>>2); - zc->seqStore.litFreq = (U32*)(dumpsEnd); - zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (1<seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (1<seqStore.XXX = zc->seqStore.offCodeFreq + (1<seqStore.XXX = zc->seqStore.dumpsStart + (blockSize>>4); zc->hbSize = 0; zc->stage = 0; @@ -562,7 +563,6 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, const size_t maxCSize = srcSize - minGain; BYTE* seqHead; - /* Compress literals */ { size_t cSize; @@ -1194,8 +1194,8 @@ void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, /* ************************************* * Binary Tree search ***************************************/ -/** ZSTD_insertBt1 : add one or multiple positions to tree -* @ip : assumed <= iend-8 +/** ZSTD_insertBt1() : add one or multiple positions to tree +* ip : assumed <= iend-8 * @return : nb of positions added */ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, U32 extDict) @@ -1230,9 +1230,9 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co while (nbCompares-- && (matchIndex > windowLow)) { U32* nextPtr = bt + 2*(matchIndex & btMask); - const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ if (matchIndex == predictedSmall) { /* no need to check length, result known */ *smallerPtr = matchIndex; @@ -1242,7 +1242,6 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co predictedSmall = predictPtr[1] + (predictPtr[1]>0); continue; } - if (matchIndex == predictedLarge) { *largerPtr = matchIndex; if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ @@ -2314,8 +2313,9 @@ size_t ZSTD_compress(void* dst, size_t maxDstSize, const void* src, size_t srcSi } -/*- Pre-defined compression levels -*/ +/*-===== Pre-defined compression levels =====-*/ +#define ZSTD_MAX_CLEVEL 25 unsigned ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { @@ -2345,7 +2345,8 @@ static const ZSTD_parameters ZSTD_defaultParameters[4][ZSTD_MAX_CLEVEL+1] = { { 0, 0, 22, 20, 22, 4, 4, ZSTD_lazy2 }, /* level 21 = 11 + L=4 */ // 41902762 lazy1=42087013 norep1=42911693 { 0, 0, 23, 21, 22, 5, 4, ZSTD_btlazy2 }, /* level 22 = 16 + L=4 */ // 41233150 btlazy1=41560211 norep1=42322286 { 0, 32, 23, 21, 22, 5, 4, ZSTD_opt }, /* level 23 */ - { 0, 32, 23, 21, 22, 5, 4, ZSTD_opt_bt }, /* level 24 */ + { 0, 32, 23, 21, 22, 5, 4, ZSTD_opt_bt }, /* level 24 = 16 + btopt */ + { 0, 64, 26, 27, 25, 10, 4, ZSTD_opt_bt }, /* level 25 = 20 + btopt */ }, { /* for srcSize <= 256 KB */ /* SL, W, C, H, S, L, strat */ diff --git a/lib/zstd_opt.c b/lib/zstd_opt.c index 8a9b1e2b1..43ae8b570 100644 --- a/lib/zstd_opt.c +++ b/lib/zstd_opt.c @@ -16,7 +16,7 @@ typedef struct U32 litlen; U32 rep; U32 rep2; -} ZSTD_optimal_t; +} ZSTD_optimal_t; #define ZSTD_OPT_DEBUG 0 // 1 = tableID=0; 5 = check encoded sequences @@ -24,11 +24,11 @@ typedef struct #if 1 #define ZSTD_LOG_PARSER(fmt, args...) ;// printf(fmt, ##args) #define ZSTD_LOG_PRICE(fmt, args...) ;//printf(fmt, ##args) - #define ZSTD_LOG_ENCODE(fmt, args...) ;//printf(fmt, ##args) + #define ZSTD_LOG_ENCODE(fmt, args...) ;//printf(fmt, ##args) #else #define ZSTD_LOG_PARSER(fmt, args...) printf(fmt, ##args) #define ZSTD_LOG_PRICE(fmt, args...) printf(fmt, ##args) - #define ZSTD_LOG_ENCODE(fmt, args...) printf(fmt, ##args) + #define ZSTD_LOG_ENCODE(fmt, args...) printf(fmt, ##args) #endif #define ZSTD_LOG_TRY_PRICE(fmt, args...) ;//printf(fmt, ##args) @@ -37,46 +37,23 @@ typedef struct #define ZSTD_FREQ_THRESHOLD (256) -// log2_32 is from http://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers -const U32 tab32[32] = { - 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31}; - -U32 log2_32 (U32 value) -{ - value |= value >> 1; - value |= value >> 2; - value |= value >> 4; - value |= value >> 8; - value |= value >> 16; - return tab32[(U32)(value*0x07C4ACDD) >> 27]; -} - FORCE_INLINE U32 ZSTD_getLiteralPriceReal(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals) { U32 price = 0; if (litLength > 0) { + U32 freq, u; /* literals */ - for (U32 i=0; i < litLength; i++) - price += log2_32(seqStorePtr->litSum) - log2_32(seqStorePtr->litFreq[literals[i]]); + for (u=0; u < litLength; u++) + price += ZSTD_highbit(seqStorePtr->litSum) - ZSTD_highbit(seqStorePtr->litFreq[literals[u]]); /* literal Length */ - U32 freq; - if (litLength >= MaxLL) { - freq = seqStorePtr->litLengthFreq[MaxLL]; - if (litLength<255 + MaxLL) { - price += 8; - } else { - price += 8; - if (litLength < (1<<15)) price += 16; else price += 24; - } } - else - freq = seqStorePtr->litLengthFreq[litLength]; - price += log2_32(seqStorePtr->litLengthSum) - log2_32(freq); + price += ((litLength >= MaxLL)*8) + ((litLength >= 255+MaxLL)*16) + ((litLength>=(1<<15))*8); + if (litLength >= MaxLL) litLength = MaxLL; + freq = seqStorePtr->litLengthFreq[litLength]; + price += ZSTD_highbit(seqStorePtr->litLengthSum) - ZSTD_highbit(freq); + /* note : seems that using matchLengthSum (litSum > ZSTD_FREQ_THRESHOLD) + if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD) return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals); - + /* backup eval */ return 1 + (litLength<<3); } @@ -96,25 +73,18 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* seqStorePtr, U32 litLength, co FORCE_INLINE U32 ZSTD_getMatchPriceReal(seqStore_t* seqStorePtr, U32 offset, U32 matchLength) { - /* match offset */ - BYTE offCode = (BYTE)ZSTD_highbit(offset) + 1; - if (offset==0) - offCode = 0; - U32 price = log2_32(seqStorePtr->offCodeSum) - log2_32(seqStorePtr->offCodeFreq[offCode]); + U32 freq; + + /* offset */ + BYTE offCode = offset ? (BYTE)ZSTD_highbit(offset) + 1 : 0; + U32 price = ZSTD_highbit(seqStorePtr->offCodeSum) - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]); price += offCode; /* match Length */ - U32 freq; - if (matchLength >= MaxML) { - freq = seqStorePtr->matchLengthFreq[MaxML]; - if (matchLength < 255+MaxML) { - price += 8; - } else { - price += 8; - if (matchLength < (1<<15)) price += 16; else price += 24; - } } - else freq = seqStorePtr->matchLengthFreq[matchLength]; - price += log2_32(seqStorePtr->matchLengthSum) - log2_32(freq); + price += ((matchLength >= MaxML)*8) + ((matchLength >= 255+MaxML)*16) + ((matchLength>=(1<<15))*8); + if (matchLength >= MaxML) matchLength = MaxML; + freq = seqStorePtr->matchLengthFreq[matchLength]; + price += ZSTD_highbit(seqStorePtr->matchLengthSum) - ZSTD_highbit(freq); return price; } @@ -124,7 +94,7 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT { if (seqStorePtr->litSum > ZSTD_FREQ_THRESHOLD) return ZSTD_getLiteralPriceReal(seqStorePtr, litLength, literals) + ZSTD_getMatchPriceReal(seqStorePtr, offset, matchLength); - + return (litLength<<3) + ZSTD_highbit((U32)matchLength+1) + Offbits + ZSTD_highbit((U32)offset+1); } @@ -132,6 +102,7 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) { + U32 u; #if 0 static const BYTE* g_start = NULL; if (g_start==NULL) g_start = literals; @@ -141,14 +112,14 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B #endif /* literals */ seqStorePtr->litSum += litLength; - for (U32 i=0; i < litLength; i++) - seqStorePtr->litFreq[literals[i]]++; - + for (u=0; u < litLength; u++) + seqStorePtr->litFreq[literals[u]]++; + /* literal Length */ seqStorePtr->litLengthSum++; if (litLength >= MaxLL) seqStorePtr->litLengthFreq[MaxLL]++; - else + else seqStorePtr->litLengthFreq[litLength]++; /* match offset */ @@ -161,7 +132,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B seqStorePtr->matchLengthSum++; if (matchLength >= MaxML) seqStorePtr->matchLengthFreq[MaxML]++; - else + else seqStorePtr->matchLengthFreq[matchLength]++; } @@ -211,8 +182,8 @@ U32 ZSTD_insertBtAndGetAllMatches ( U32 matchEndIdx = current+8; U32 dummy32; /* to be nullified at the end */ U32 mnum = 0; - - bestLength = 0; + + bestLength = MINMATCH-1; hashTable[h] = current; /* Update Hash Table */ while (nbCompares-- && (matchIndex > windowLow)) { @@ -232,18 +203,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( } if (matchLength > bestLength) { - if (matchLength > matchEndIdx - matchIndex) - matchEndIdx = matchIndex + (U32)matchLength; - { - if (matchLength >= MINMATCH) { - bestLength = matchLength; - matches[mnum].off = current - matchIndex; - matches[mnum].len = (U32)matchLength; - matches[mnum].back = 0; - mnum++; - } - if (matchLength > ZSTD_OPT_NUM) break; - } + if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = current - matchIndex; + matches[mnum].len = (U32)matchLength; + matches[mnum].back = 0; + mnum++; + if (matchLength > ZSTD_OPT_NUM) break; if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ } @@ -262,8 +228,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ largerPtr = nextPtr; matchIndex = nextPtr[0]; - } - } + } } *smallerPtr = *largerPtr = 0; @@ -290,8 +255,7 @@ FORCE_INLINE U32 ZSTD_BtGetAllMatches_selectMLS ( const BYTE* ip, const BYTE* const iLowLimit, const BYTE* const iHighLimit, const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, U32 minml) { - if (iLowLimit) {}; // skip warnings - + (void)iLowLimit; /* unused */ switch(matchLengthSearch) { default : @@ -303,7 +267,7 @@ FORCE_INLINE U32 ZSTD_BtGetAllMatches_selectMLS ( /** Tree updater, providing best match */ FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ -size_t ZSTD_BtGetAllMatches_extDict ( +U32 ZSTD_BtGetAllMatches_extDict ( ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iLimit, const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, U32 minml) @@ -385,14 +349,14 @@ U32 ZSTD_HcGetAllMatches_generic ( } /* save best solution */ - if (currentMl > minml) { - minml = currentMl; + if (currentMl > minml) { + minml = currentMl; matches[mnum].off = current - matchIndex; matches[mnum].len = (U32)currentMl; matches[mnum].back = back; mnum++; if (currentMl > ZSTD_OPT_NUM) break; - if (ip+currentMl == iHighLimit) break; /* best possible, and avoid read overflow*/ + if (ip+currentMl == iHighLimit) break; /* best possible, and avoid read overflow*/ } if (matchIndex <= minChain) break; @@ -458,11 +422,11 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_optimal_t opt[ZSTD_OPT_NUM+4]; ZSTD_match_t matches[ZSTD_OPT_NUM+1]; - const uint8_t *inr; + const BYTE* inr; U32 skip_num, cur, cur2, match_num, last_pos, litlen, price; const U32 sufficient_len = ctx->params.sufficientLength; - const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt); + const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt); /* init */ @@ -472,6 +436,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, /* Match Loop */ while (ip < ilimit) { + U32 u; U32 mlen=0; U32 best_mlen=0; U32 best_off=0; @@ -485,7 +450,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, if (MEM_read32(ip+1) == MEM_read32(ip+1 - rep_1)) { /* repcode : we take it */ mlen = (U32)ZSTD_count(ip+1+MINMATCH, ip+1+MINMATCH-rep_1, iend) + MINMATCH; - + ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen); if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; @@ -505,7 +470,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, best_mlen = (last_pos) ? last_pos : MINMATCH; - + if (faster_get_matches && last_pos) match_num = 0; else @@ -518,8 +483,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, opt[0].rep2 = rep_2; opt[0].mlen = 1; - if (match_num && matches[match_num-1].len > sufficient_len) - { + if (match_num && matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; @@ -528,50 +492,45 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } // set prices using matches at position = 0 - for (U32 i = 0; i < match_num; i++) - { - mlen = (i>0) ? matches[i-1].len+1 : best_mlen; - best_mlen = (matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM; + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM; ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; while (mlen <= best_mlen) { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH); + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; } } - if (last_pos < MINMATCH) { + if (last_pos < MINMATCH) { // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - ip++; continue; + ip++; continue; } // check further positions for (skip_num = 0, cur = 1; cur <= last_pos; cur++) - { + { + size_t cur_rep; inr = ip + cur; - if (opt[cur-1].mlen == 1) - { + if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; - if (cur > litlen) - { + if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); ZSTD_LOG_TRY_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-base), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } - else + } else price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } - else - { + } else { litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); ZSTD_LOG_TRY_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-base), price, cur, litlen, (int)ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1)); } - + ZSTD_LOG_TRY_PRICE("%d: TRY4 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur, opt[cur].price); if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) @@ -583,53 +542,40 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - - if (opt[cur-mlen].off) - { + + if (opt[cur-mlen].off) { opt[cur].rep2 = opt[cur-mlen].rep; opt[cur].rep = opt[cur-mlen].off; ZSTD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } - else - { - if (cur!=mlen && opt[cur-mlen].litlen == 0) - { + } else { + if (cur!=mlen && opt[cur-mlen].litlen == 0) { opt[cur].rep2 = opt[cur-mlen].rep; opt[cur].rep = opt[cur-mlen].rep2; ZSTD_LOG_PARSER("%d: COPYREP2 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } - else - { + } else { opt[cur].rep2 = opt[cur-mlen].rep2; opt[cur].rep = opt[cur-mlen].rep; ZSTD_LOG_PARSER("%d: COPYREP3 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } - } + } } - ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); + ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); - - - size_t cur_rep; best_mlen = 0; if (!opt[cur].off && opt[cur].mlen != 1) { cur_rep = opt[cur].rep2; - ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); - } - else { + ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); + } else { cur_rep = opt[cur].rep; - ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); + ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); } - if (MEM_read32(inr) == MEM_read32(inr - cur_rep)) // check rep - { - mlen = (U32)ZSTD_count(inr+MINMATCH, inr+MINMATCH - cur_rep, iend) + MINMATCH; + if (MEM_read32(inr) == MEM_read32(inr - cur_rep)) { // check rep + mlen = (U32)ZSTD_count(inr+MINMATCH, inr+MINMATCH - cur_rep, iend) + MINMATCH; ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-base), mlen, 0, opt[cur].rep, cur, opt[cur].off); - if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) - { + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { best_mlen = mlen; best_off = 0; ZSTD_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-base), sufficient_len, best_mlen, best_off, last_pos); @@ -637,20 +583,15 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, goto _storeSequence; } - if (opt[cur].mlen == 1) - { + if (opt[cur].mlen == 1) { litlen = opt[cur].litlen; - if (cur > litlen) - { + if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, 0, mlen - MINMATCH); ZSTD_LOG_TRY_PRICE("%d: TRY5 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-base), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } - else + } else price = ZSTD_getPrice(seqStorePtr, litlen, anchor, 0, mlen - MINMATCH); - } - else - { + } else { litlen = 0; price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - MINMATCH); ZSTD_LOG_TRY_PRICE("%d: TRY7 price=%d cur=%d litlen=0 getprice=%d\n", (int)(inr-base), price, cur, (int)ZSTD_getPrice(seqStorePtr, 0, NULL, 0, mlen - MINMATCH)); @@ -662,31 +603,23 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price); - do - { + do { if (cur + mlen > last_pos || price <= opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH SET_PRICE(cur + mlen, mlen, 0, litlen, price); mlen--; - } - while (mlen >= MINMATCH); + } while (mlen >= MINMATCH); } - - if (faster_get_matches && skip_num > 0) - { - skip_num--; - continue; - } + if (faster_get_matches && skip_num > 0) { skip_num--; continue; } - best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; + best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; - match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen); + match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); - if (match_num > 0 && matches[match_num-1].len > sufficient_len) - { + if (match_num > 0 && matches[match_num-1].len > sufficient_len) { cur -= matches[match_num-1].back; best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; @@ -695,77 +628,69 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, } - // set prices using matches at position = cur - for (U32 i = 0; i < match_num; i++) - { - mlen = (i>0) ? matches[i-1].len+1 : best_mlen; - cur2 = cur - matches[i].back; - best_mlen = (cur2 + matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM - cur2; + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + cur2 = cur - matches[u].back; + best_mlen = (cur2 + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur2; - ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); - if (mlen < matches[i].back + 1) - mlen = matches[i].back + 1; + ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[u].len, matches[i].off, best_mlen, last_pos); + if (mlen < matches[u].back + 1) + mlen = matches[u].back + 1; - while (mlen <= best_mlen) - { - if (opt[cur2].mlen == 1) - { + while (mlen <= best_mlen) { + if (opt[cur2].mlen == 1) { litlen = opt[cur2].litlen; if (cur2 > litlen) - price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[i].off, mlen - MINMATCH); + price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[u].off, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH); - } - else - { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + } else { litlen = 0; - price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[i].off, mlen - MINMATCH); + price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); } - ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[u].back, mlen, best_mlen, matches[u].off, price, litlen, cur - litlen, opt[cur - litlen].price); ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price); if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price)) - SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + SET_PRICE(cur2 + mlen, mlen, matches[u].off, litlen, price); mlen++; - } - } + } } } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; - // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur); + // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur); /* store sequence */ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set - for (U32 i = 1; i <= last_pos; i++) - ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep); + for (u = 1; u <= last_pos; u++) + ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); + ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep); opt[0].mlen = 1; U32 offset; - while (1) - { + while (1) { mlen = opt[cur].mlen; ZSTD_LOG_PARSER("%d: cur=%d mlen=%d\n", (int)(ip-base), cur, mlen); offset = opt[cur].off; - opt[cur].mlen = best_mlen; + opt[cur].mlen = best_mlen; opt[cur].off = best_off; best_mlen = mlen; - best_off = offset; + best_off = offset; if (mlen > cur) break; cur -= mlen; } - - for (U32 i = 0; i <= last_pos;) - { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2); - i += opt[i].mlen; + + for (u = 0; u <= last_pos;) { + ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); + u += opt[u].mlen; } cur = 0; @@ -789,7 +714,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } else { - if (litLength == 0) + if (litLength == 0) { best_off = rep_2; rep_2 = rep_1; @@ -798,7 +723,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); - + #if ZSTD_OPT_DEBUG >= 5 int ml2; if (offset) @@ -839,7 +764,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set anchor += best_mlen+MINMATCH; ip = anchor; continue; // faster when present ... (?) - } + } } /* Last Literals */ @@ -880,11 +805,11 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ZSTD_optimal_t opt[ZSTD_OPT_NUM+4]; ZSTD_match_t matches[ZSTD_OPT_NUM+1]; - const uint8_t *inr; + const BYTE* inr; U32 skip_num, cur, cur2, match_num, last_pos, litlen, price; const U32 sufficient_len = ctx->params.sufficientLength; - const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt); + const U32 faster_get_matches = (ctx->params.strategy == ZSTD_opt); /* init */ @@ -894,6 +819,8 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* Match Loop */ while (ip < ilimit) { + U32 u; + U32 offset; U32 mlen=0; U32 best_mlen=0; U32 best_off=0; @@ -914,7 +841,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; mlen = (U32)ZSTD_count_2segments(ip+1+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; - + ZSTD_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-base), (int)rep_1, (int)mlen); if (depth==0 || mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { ip+=1; best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; @@ -922,19 +849,16 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } litlen = opt[0].litlen + 1; - do - { + do { price = ZSTD_getPrice(seqStorePtr, litlen, anchor, 0, mlen - MINMATCH); if (mlen + 1 > last_pos || price < opt[mlen + 1].price) SET_PRICE(mlen + 1, mlen, 0, litlen, price); mlen--; - } - while (mlen >= MINMATCH); - } - } + } while (mlen >= MINMATCH); + } } best_mlen = (last_pos) ? last_pos : MINMATCH; - + if (faster_get_matches && last_pos) match_num = 0; else @@ -947,8 +871,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, opt[0].rep2 = rep_2; opt[0].mlen = 1; - if (match_num && matches[match_num-1].len > sufficient_len) - { + if (match_num && matches[match_num-1].len > sufficient_len) { best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; cur = 0; @@ -957,50 +880,43 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, } // set prices using matches at position = 0 - for (U32 i = 0; i < match_num; i++) - { - mlen = (i>0) ? matches[i-1].len+1 : best_mlen; - best_mlen = (matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM; - ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[i].len, matches[i].off, (int)best_mlen, (int)last_pos); + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = (matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM; + ZSTD_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-base), matches[u].len, matches[u].off, (int)best_mlen, (int)last_pos); litlen = opt[0].litlen; - while (mlen <= best_mlen) - { - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH); + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); if (mlen > last_pos || price < opt[mlen].price) - SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); mlen++; - } - } + } } - if (last_pos < MINMATCH) { - // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ - ip++; continue; + if (last_pos < MINMATCH) { + // ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + ip++; continue; } // check further positions for (skip_num = 0, cur = 1; cur <= last_pos; cur++) - { + { + size_t cur_rep; inr = ip + cur; - if (opt[cur-1].mlen == 1) - { + if (opt[cur-1].mlen == 1) { litlen = opt[cur-1].litlen + 1; - if (cur > litlen) - { + if (cur > litlen) { price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); ZSTD_LOG_TRY_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-base), cur - litlen, opt[cur - litlen].price, price, cur, litlen); - } - else + } else price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); - } - else - { + } else { litlen = 1; - price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); ZSTD_LOG_TRY_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-base), price, cur, litlen, (int)ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1)); } - + ZSTD_LOG_TRY_PRICE("%d: TRY4 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur, opt[cur].price); if (cur > last_pos || price <= opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) @@ -1012,43 +928,35 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, continue; mlen = opt[cur].mlen; - - if (opt[cur-mlen].off) - { + + if (opt[cur-mlen].off) { opt[cur].rep2 = opt[cur-mlen].rep; opt[cur].rep = opt[cur-mlen].off; ZSTD_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } - else - { - if (cur!=mlen && opt[cur-mlen].litlen == 0) - { + } else { + if (cur!=mlen && opt[cur-mlen].litlen == 0) { opt[cur].rep2 = opt[cur-mlen].rep; opt[cur].rep = opt[cur-mlen].rep2; ZSTD_LOG_PARSER("%d: COPYREP2 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } - else - { + } else { opt[cur].rep2 = opt[cur-mlen].rep2; opt[cur].rep = opt[cur-mlen].rep; ZSTD_LOG_PARSER("%d: COPYREP3 cur=%d mlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, mlen, opt[cur].rep, opt[cur].rep2); - } - } + } } - ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); + ZSTD_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(inr-base), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); - size_t cur_rep; best_mlen = 0; if (!opt[cur].off && opt[cur].mlen != 1) { cur_rep = opt[cur].rep2; - ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); + ZSTD_LOG_PARSER("%d: try REP2 rep2=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); } else { cur_rep = opt[cur].rep; - ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); + ZSTD_LOG_PARSER("%d: try REP1 rep=%d mlen=%d\n", (int)(inr-base), cur_rep, mlen); } const U32 repIndex = (U32)(current+cur - cur_rep); @@ -1107,19 +1015,18 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, if (faster_get_matches && skip_num > 0) { - skip_num--; + skip_num--; continue; } - best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; + best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; - match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen); + match_num = getAllMatches(ctx, inr, ip, iend, maxSearches, mls, matches, best_mlen); ZSTD_LOG_PARSER("%d: ZSTD_GetAllMatches match_num=%d\n", (int)(inr-base), match_num); - if (match_num > 0 && matches[match_num-1].len > sufficient_len) - { + if (match_num > 0 && matches[match_num-1].len > sufficient_len) { cur -= matches[match_num-1].back; best_mlen = matches[match_num-1].len; best_off = matches[match_num-1].off; @@ -1129,109 +1036,94 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, // set prices using matches at position = cur - for (U32 i = 0; i < match_num; i++) - { - mlen = (i>0) ? matches[i-1].len+1 : best_mlen; - cur2 = cur - matches[i].back; - best_mlen = (cur2 + matches[i].len < ZSTD_OPT_NUM) ? matches[i].len : ZSTD_OPT_NUM - cur2; + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + cur2 = cur - matches[u].back; + best_mlen = (cur2 + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur2; - ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); - if (mlen < matches[i].back + 1) - mlen = matches[i].back + 1; + ZSTD_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-base), cur, cur2, matches[u].len, matches[u].off, best_mlen, last_pos); + if (mlen < matches[u].back + 1) + mlen = matches[u].back + 1; - while (mlen <= best_mlen) - { - if (opt[cur2].mlen == 1) - { + while (mlen <= best_mlen) { + if (opt[cur2].mlen == 1) { litlen = opt[cur2].litlen; if (cur2 > litlen) - price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[i].off, mlen - MINMATCH); + price = opt[cur2 - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur2-litlen, matches[u].off, mlen - MINMATCH); else - price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[i].off, mlen - MINMATCH); - } - else - { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off, mlen - MINMATCH); + } else { litlen = 0; - price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[i].off, mlen - MINMATCH); + price = opt[cur2].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off, mlen - MINMATCH); } - ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + ZSTD_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-base), matches[u].back, mlen, best_mlen, matches[u].off, price, litlen, cur - litlen, opt[cur - litlen].price); ZSTD_LOG_TRY_PRICE("%d: TRY8 price=%d opt[%d].price=%d\n", (int)(inr-base), price, cur2 + mlen, opt[cur2 + mlen].price); if (cur2 + mlen > last_pos || (price < opt[cur2 + mlen].price)) - SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + SET_PRICE(cur2 + mlen, mlen, matches[u].off, litlen, price); mlen++; - } - } + } } } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) best_mlen = opt[last_pos].mlen; best_off = opt[last_pos].off; cur = last_pos - best_mlen; - // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur); + // printf("%d: start=%d best_mlen=%d best_off=%d cur=%d\n", (int)(ip - base), (int)(start - ip), (int)best_mlen, (int)best_off, cur); /* store sequence */ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set - for (U32 i = 1; i <= last_pos; i++) - ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2); - ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep); + for (u = 1; u <= last_pos; u++) + ZSTD_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u]litlen, opt[u].rep, opt[u].rep2); + ZSTD_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-base+cur), (int)cur, (int)last_pos, (int)best_mlen, (int)best_off, opt[cur].rep); opt[0].mlen = 1; - U32 offset; - while (1) - { + while (1) { mlen = opt[cur].mlen; ZSTD_LOG_PARSER("%d: cur=%d mlen=%d\n", (int)(ip-base), cur, mlen); offset = opt[cur].off; - opt[cur].mlen = best_mlen; + opt[cur].mlen = best_mlen; opt[cur].off = best_off; best_mlen = mlen; - best_off = offset; + best_off = offset; if (mlen > cur) break; cur -= mlen; } - - for (U32 i = 0; i <= last_pos;) - { - ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep, opt[i].rep2); - i += opt[i].mlen; + + for (u = 0; u <= last_pos;) { + ZSTD_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+u), u, last_pos, opt[u].price, opt[u].off, opt[u].mlen, opt[u].litlen, opt[u].rep, opt[u].rep2); + u += opt[u].mlen; } cur = 0; - while (cur < last_pos) - { + while (cur < last_pos) { + U32 litLength; ZSTD_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d rep2=%d\n", (int)(ip-base+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep, opt[cur].rep2); mlen = opt[cur].mlen; if (mlen == 1) { ip++; cur++; continue; } offset = opt[cur].off; cur += mlen; - - U32 litLength = (U32)(ip - anchor); + litLength = (U32)(ip - anchor); ZSTD_LOG_ENCODE("%d/%d: ENCODE1 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); - if (offset) - { + if (offset) { rep_2 = rep_1; rep_1 = offset; - } - else - { - if (litLength == 0) - { + } else { + if (litLength == 0) { best_off = rep_2; rep_2 = rep_1; rep_1 = best_off; - } - } + } } ZSTD_LOG_ENCODE("%d/%d: ENCODE2 literals=%d mlen=%d off=%d rep1=%d rep2=%d\n", (int)(ip-base), (int)(iend-base), (int)(litLength), (int)mlen, (int)(offset), (int)rep_1, (int)rep_2); - + #if ZSTD_OPT_DEBUG >= 5 int ml2; if (offset) @@ -1258,14 +1150,14 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set } - // check immediate repcode + /* check immediate repcode */ while (anchor <= ilimit) { const U32 repIndex = (U32)((anchor-base) - rep_2); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; const BYTE* const repMatch = repBase + repIndex; - if ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ - if (MEM_read32(anchor) == MEM_read32(repMatch)) { - /* repcode detected we should take it */ + if ( ((U32)((dictLimit-1) - repIndex) >= 3) /* intentional overflow */ + && (MEM_read32(anchor) == MEM_read32(repMatch)) ) { + /* repcode detected, let's take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; mlen = (U32)ZSTD_count_2segments(anchor+MINMATCH, repMatch+MINMATCH, iend, repEnd, prefixStart) + MINMATCH; offset = rep_2; rep_2 = rep_1; rep_1 = offset; /* swap offset history */ @@ -1277,8 +1169,7 @@ _storeSequence: // cur, last_pos, best_mlen, best_off have to be set continue; /* faster when present ... (?) */ } break; - } - } + } } /* Last Literals */ { diff --git a/lib/zstd_static.h b/lib/zstd_static.h index da2335bd3..8b7d63b23 100644 --- a/lib/zstd_static.h +++ b/lib/zstd_static.h @@ -54,6 +54,8 @@ extern "C" { #define ZSTD_WINDOWLOG_MAX 26 #define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 11 +#define ZSTD_TARGETLENGTH_MIN 4 +#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CONTENTLOG_MIN 4 #define ZSTD_HASHLOG_MAX 28 @@ -63,16 +65,16 @@ extern "C" { #define ZSTD_SEARCHLENGTH_MAX 7 #define ZSTD_SEARCHLENGTH_MIN 4 -/** from faster to stronger */ +/* from faster to stronger */ typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_opt, ZSTD_opt_bt } ZSTD_strategy; typedef struct { U64 srcSize; /* optional : tells how much bytes are present in the frame. Use 0 if not known. */ - U32 sufficientLength; /* size of matches which is acceptable (used only by the optimal parser): larger == more compression, slower */ + U32 sufficientLength; /* size of matches which is acceptable (used only by the optimal parser): larger == more compression, slower */ U32 windowLog; /* largest match distance : larger == more compression, more memory needed during decompression */ U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /* dispatch table : larger == more memory, faster */ + U32 hashLog; /* dispatch table : larger == faster, more memory */ U32 searchLog; /* nb of searches : larger == more compression, slower */ U32 searchLength; /* size of matches : larger == faster decompression, sometimes less compression */ ZSTD_strategy strategy; @@ -82,7 +84,6 @@ typedef struct /* ************************************* * Advanced functions ***************************************/ -#define ZSTD_MAX_CLEVEL 24 ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getParams() : diff --git a/programs/paramgrill.c b/programs/paramgrill.c index a34da88ce..fd75aa327 100644 --- a/programs/paramgrill.c +++ b/programs/paramgrill.c @@ -1,6 +1,6 @@ /* - paramgrill.c - parameter tester for zstd_hc - Copyright (C) Yann Collet 2015 + paramgrill.c - parameter tester for zstd + Copyright (C) Yann Collet 2015-2016 GPL v2 License @@ -19,11 +19,10 @@ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. You can contact the author at : - - zstd source repository : https://github.com/Cyan4973/zstd - - ztsd public forum : https://groups.google.com/forum/#!forum/lz4c + - zstd homepage : http://www.zstd.net/ */ -/************************************** +/*-************************************ * Compiler Options **************************************/ /* Disable some Visual warning messages */ @@ -48,8 +47,8 @@ #endif -/************************************** -* Includes +/*-************************************ +* Dependencies **************************************/ #include /* malloc */ #include /* fprintf, fopen, ftello64 */ @@ -71,7 +70,7 @@ #include "xxhash.h" -/************************************** +/*-************************************ * Compiler Options **************************************/ /* S_ISREG & gettimeofday() are not supported by MSVC */ @@ -80,7 +79,7 @@ #endif -/************************************** +/*-************************************ * Constants **************************************/ #define PROGRAM_DESCRIPTION "ZSTD_HC parameters tester" @@ -98,6 +97,8 @@ #define NBLOOPS 2 #define TIMELOOP 2000 +#define NB_LEVELS_TRACKED 30 + static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); #define DEFAULT_CHUNKSIZE (4<<20) @@ -110,13 +111,13 @@ static const int g_maxVariationTime = 60000; /* 60 sec */ static const int g_maxNbVariations = 64; -/************************************** +/*-************************************ * Macros **************************************/ #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -/************************************** +/*-************************************ * Benchmark Parameters **************************************/ static U32 g_nbIterations = NBLOOPS; @@ -126,7 +127,7 @@ static U32 g_rand = 1; static U32 g_singleRun = 0; static U32 g_target = 0; static U32 g_noSeed = 0; -static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, ZSTD_greedy }; +static ZSTD_parameters g_params = { 0, 0, 0, 0, 0, 0, 0, ZSTD_greedy }; void BMK_SetNbIterations(int nbLoops) { @@ -135,7 +136,7 @@ void BMK_SetNbIterations(int nbLoops) } -/********************************************************* +/*-******************************************************* * Private functions *********************************************************/ @@ -187,8 +188,7 @@ static size_t BMK_findMaxMem(U64 requiredMem) if (requiredMem > maxMemory) requiredMem = maxMemory; requiredMem += 2*step; - while (!testmem) - { + while (!testmem) { requiredMem -= step; testmem = (BYTE*) malloc ((size_t)requiredMem); } @@ -226,7 +226,7 @@ U32 FUZ_rand(U32* src) } -/********************************************************* +/*-******************************************************* * Bench functions *********************************************************/ typedef struct { @@ -265,14 +265,14 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, U32 Hlog = params.hashLog; U32 Slog = params.searchLog; U32 Slength = params.searchLength; + U32 Tlength = params.sufficientLength; ZSTD_strategy strat = params.strategy; char name[30] = { 0 }; U64 crcOrig; /* Memory allocation & restrictions */ - snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%1u", Wlog, Clog, Hlog, Slog, Slength, strat); - if (!compressedBuffer || !resultBuffer || !blockTable) - { + snprintf(name, 30, "Sw%02uc%02uh%02us%02ul%1ut%03uS%1u", Wlog, Clog, Hlog, Slog, Slength, Tlength, strat); + if (!compressedBuffer || !resultBuffer || !blockTable) { DISPLAY("\nError: not enough memory!\n"); free(compressedBuffer); free(resultBuffer); @@ -290,8 +290,7 @@ static size_t BMK_benchParam(BMK_result_t* resultPtr, const char* srcPtr = (const char*)srcBuffer; char* cPtr = (char*)compressedBuffer; char* resPtr = (char*)resultBuffer; - for (i=0; i O_ratio) DISPLAY ("Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", @@ -518,8 +504,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel); continue; } - if (W_CMemUsed_note < O_CMemUsed_note) - { + if (W_CMemUsed_note < O_CMemUsed_note) { /* uses too much memory for compression for too little benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", @@ -527,8 +512,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, O_ratio, (double)(O_CMemUsed) / 1024 / 1024, cLevel); continue; } - if (W_CSpeed_note < O_CSpeed_note ) - { + if (W_CSpeed_note < O_CSpeed_note ) { /* too large compression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", @@ -536,8 +520,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, O_ratio, (double)(winners[cLevel].result.cSpeed) / 1000., cLevel); continue; } - if (W_DSpeed_note < O_DSpeed_note ) - { + if (W_DSpeed_note < O_DSpeed_note ) { /* too large decompression speed difference for the compression benefit */ if (W_ratio > O_ratio) DISPLAY ("Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", @@ -554,9 +537,7 @@ static int BMK_seed(winnerInfo_t* winners, const ZSTD_parameters params, BMK_printWinner(stdout, cLevel, testResult, params, srcSize); better = 1; - } - - } + } } return better; } @@ -567,10 +548,9 @@ static ZSTD_parameters* sanitizeParams(ZSTD_parameters params) { g_params = params; if (params.strategy == ZSTD_fast) - { - g_params.contentLog = 0; - g_params.searchLog = 0; - } + g_params.contentLog = 0, g_params.searchLog = 0; + if ((params.strategy != ZSTD_opt) && (params.strategy != ZSTD_opt_bt)) + g_params.sufficientLength = 0; return &g_params; } @@ -578,8 +558,7 @@ static ZSTD_parameters* sanitizeParams(ZSTD_parameters params) static void paramVariation(ZSTD_parameters* p) { U32 nbChanges = (FUZ_rand(&g_rand) & 3) + 1; - for (; nbChanges; nbChanges--) - { + for (; nbChanges; nbChanges--) { const U32 changeID = FUZ_rand(&g_rand) % 12; switch(changeID) { @@ -607,6 +586,10 @@ static void paramVariation(ZSTD_parameters* p) p->strategy = (ZSTD_strategy)(((U32)p->strategy)+1); break; case 11: p->strategy = (ZSTD_strategy)(((U32)p->strategy)-1); break; + case 12: + p->sufficientLength *= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; + case 13: + p->sufficientLength /= 1 + ((double)(FUZ_rand(&g_rand)&255)) / 256.; break; } } ZSTD_validateParams(p); @@ -632,8 +615,7 @@ static void playAround(FILE* f, winnerInfo_t* winners, int nbVariations = 0; const int startTime = BMK_GetMilliStart(); - while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) - { + while (BMK_GetMilliSpan(startTime) < g_maxVariationTime) { ZSTD_parameters p = params; if (nbVariations++ > g_maxNbVariations) break; @@ -658,15 +640,15 @@ static void playAround(FILE* f, winnerInfo_t* winners, static void potentialRandomParams(ZSTD_parameters* p, U32 inverseChance) { U32 chance = (FUZ_rand(&g_rand) % (inverseChance+1)); - if (!chance) - { + if (!chance) { /* totally random entry */ p->contentLog = FUZ_rand(&g_rand) % (ZSTD_CONTENTLOG_MAX+1 - ZSTD_CONTENTLOG_MIN) + ZSTD_CONTENTLOG_MIN; p->hashLog = FUZ_rand(&g_rand) % (ZSTD_HASHLOG_MAX+1 - ZSTD_HASHLOG_MIN) + ZSTD_HASHLOG_MIN; p->searchLog = FUZ_rand(&g_rand) % (ZSTD_SEARCHLOG_MAX+1 - ZSTD_SEARCHLOG_MIN) + ZSTD_SEARCHLOG_MIN; p->windowLog = FUZ_rand(&g_rand) % (ZSTD_WINDOWLOG_MAX+1 - ZSTD_WINDOWLOG_MIN) + ZSTD_WINDOWLOG_MIN; p->searchLength=FUZ_rand(&g_rand) % (ZSTD_SEARCHLENGTH_MAX+1 - ZSTD_SEARCHLENGTH_MIN) + ZSTD_SEARCHLENGTH_MIN; - p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_btlazy2+1)); + p->sufficientLength=FUZ_rand(&g_rand) % (ZSTD_TARGETLENGTH_MAX+1 - ZSTD_TARGETLENGTH_MIN) + ZSTD_TARGETLENGTH_MIN; + p->strategy = (ZSTD_strategy) (FUZ_rand(&g_rand) % (ZSTD_opt_bt+1)); ZSTD_validateParams(p); } } @@ -676,9 +658,8 @@ static void BMK_selectRandomStart( const void* srcBuffer, size_t srcSize, ZSTD_CCtx* ctx) { - U32 id = (FUZ_rand(&g_rand) % (ZSTD_MAX_CLEVEL+1)); - if ((id==0) || (winners[id].params.windowLog==0)) - { + U32 id = (FUZ_rand(&g_rand) % (ZSTD_maxCLevel()+1)); + if ((id==0) || (winners[id].params.windowLog==0)) { /* totally random entry */ ZSTD_parameters p; potentialRandomParams(&p, 1); @@ -695,14 +676,14 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) { ZSTD_CCtx* ctx = ZSTD_createCCtx(); ZSTD_parameters params; - winnerInfo_t winners[ZSTD_MAX_CLEVEL+1]; + winnerInfo_t winners[NB_LEVELS_TRACKED]; int i; + unsigned u; const char* rfName = "grillResults.txt"; FILE* f; const size_t blockSize = g_blockSize ? g_blockSize : srcSize; - if (g_singleRun) - { + if (g_singleRun) { BMK_result_t testResult; g_params.srcSize = blockSize; ZSTD_validateParams(&g_params); @@ -718,8 +699,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) if (g_target) g_cSpeedTarget[1] = g_target * 1000; - else - { + else { /* baseline config for level 1 */ BMK_result_t testResult; params = ZSTD_getParams(1, blockSize); @@ -728,14 +708,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) } /* establish speed objectives (relative to level 1) */ - for (i=2; i<=ZSTD_MAX_CLEVEL; i++) - g_cSpeedTarget[i] = (g_cSpeedTarget[i-1] * 25) >> 5; + for (u=2; u<=ZSTD_maxCLevel(); u++) + g_cSpeedTarget[u] = (g_cSpeedTarget[u-1] * 25) >> 5; /* populate initial solution */ { - const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL; - for (i=1; i<=maxSeeds; i++) - { + const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + for (i=1; i<=maxSeeds; i++) { params = ZSTD_getParams(i, blockSize); ZSTD_validateParams(¶ms); BMK_seed(winners, params, srcBuffer, srcSize, ctx); @@ -746,8 +725,7 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) /* start tests */ { const int milliStart = BMK_GetMilliStart(); - do - { + do { BMK_selectRandomStart(f, winners, srcBuffer, srcSize, ctx); } while (BMK_GetMilliSpan(milliStart) < g_grillDuration); } @@ -764,17 +742,13 @@ static void BMK_benchMem(void* srcBuffer, size_t srcSize) static int benchSample(void) { - char* origBuff; + void* origBuff; size_t benchedSize = sampleSize; const char* name = "Sample 10MiB"; /* Allocation */ - origBuff = (char*) malloc((size_t)benchedSize); - if(!origBuff) - { - DISPLAY("\nError: not enough memory!\n"); - return 12; - } + origBuff = malloc(benchedSize); + if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } /* Fill buffer */ RDG_genBuffer(origBuff, benchedSize, g_compressibility, 0.0, 0); @@ -794,8 +768,7 @@ int benchFiles(char** fileNamesTable, int nbFiles) int fileIdx=0; /* Loop for each file */ - while (fileIdx inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } /* Alloc */ origBuff = (char*) malloc((size_t)benchedSize); - if(!origBuff) - { + if(!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; @@ -835,8 +804,7 @@ int benchFiles(char** fileNamesTable, int nbFiles) readSize = fread(origBuff, 1, benchedSize, inFile); fclose(inFile); - if(readSize != benchedSize) - { + if(readSize != benchedSize) { DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); free(origBuff); return 13; @@ -862,8 +830,7 @@ int optimizeForSize(char* inFileName) /* Check file existence */ inFile = fopen( inFileName, "rb" ); - if (inFile==NULL) - { + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } @@ -873,14 +840,11 @@ int optimizeForSize(char* inFileName) benchedSize = (size_t) BMK_findMaxMem(inFileSize*3) / 3; if ((U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; if (benchedSize < inFileSize) - { DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20)); - } /* Alloc */ origBuff = (char*) malloc((size_t)benchedSize); - if(!origBuff) - { + if(!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; @@ -891,8 +855,7 @@ int optimizeForSize(char* inFileName) readSize = fread(origBuff, 1, benchedSize, inFile); fclose(inFile); - if(readSize != benchedSize) - { + if(readSize != benchedSize) { DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); free(origBuff); return 13; @@ -916,9 +879,8 @@ int optimizeForSize(char* inFileName) /* find best solution from default params */ { - const int maxSeeds = g_noSeed ? 1 : ZSTD_MAX_CLEVEL; - for (i=1; i<=maxSeeds; i++) - { + const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + for (i=1; i<=maxSeeds; i++) { params = ZSTD_getParams(i, blockSize); BMK_benchParam(&candidate, origBuff, benchedSize, ctx, params); if ( (candidate.cSize < winner.result.cSize) @@ -927,16 +889,14 @@ int optimizeForSize(char* inFileName) winner.params = params; winner.result = candidate; BMK_printWinner(stdout, i, winner.result, winner.params, benchedSize); - } - } + } } } BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); /* start tests */ { const int milliStart = BMK_GetMilliStart(); - do - { + do { params = winner.params; paramVariation(¶ms); potentialRandomParams(¶ms, 16); @@ -950,13 +910,11 @@ int optimizeForSize(char* inFileName) /* improvement found => new winner */ if ( (candidate.cSize < winner.result.cSize) - ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) ) - { + ||((candidate.cSize == winner.result.cSize) && (candidate.cSpeed > winner.result.cSpeed)) ) { winner.params = params; winner.result = candidate; BMK_printWinner(stdout, 99, winner.result, winner.params, benchedSize); } - } while (BMK_GetMilliSpan(milliStart) < g_grillDuration); } @@ -1008,6 +966,12 @@ int main(int argc, char** argv) U32 optimizer = 0; U32 main_pause = 0; + /* checks */ + if (NB_LEVELS_TRACKED <= ZSTD_maxCLevel()) { + DISPLAY("Error : NB_LEVELS_TRACKED <= ZSTD_maxCLevel() \n"); + exit(1); + } + /* Welcome message */ DISPLAY(WELCOME_MESSAGE); @@ -1022,12 +986,10 @@ int main(int argc, char** argv) if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } /* Decode command (note : aggregated commands are allowed) */ - if (argument[0]=='-') - { + if (argument[0]=='-') { argument++; - while (argument[0]!=0) - { + while (argument[0]!=0) { switch(argument[0]) { @@ -1050,8 +1012,7 @@ int main(int argc, char** argv) argument++; { U32 proba32 = 0; - while ((argument[0]>= '0') && (argument[0]<= '9')) - { + while ((argument[0]>= '0') && (argument[0]<= '9')) { proba32 *= 10; proba32 += argument[0] - '0'; argument++; @@ -1070,8 +1031,7 @@ int main(int argc, char** argv) g_singleRun = 1; argument++; g_params = ZSTD_getParams(2, g_blockSize); - for ( ; ; ) - { + for ( ; ; ) { switch(*argument) { case 'w': @@ -1107,8 +1067,7 @@ int main(int argc, char** argv) case 't': /* strategy */ g_params.strategy = (ZSTD_strategy)0; argument++; - while ((*argument>= '0') && (*argument<='9')) - { + while ((*argument>= '0') && (*argument<='9')) { g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy *10); g_params.strategy = (ZSTD_strategy)((U32)g_params.strategy + *argument++ - '0'); } @@ -1132,8 +1091,7 @@ int main(int argc, char** argv) case 'T': argument++; g_target = 0; - while ((*argument >= '0') && (*argument <= '9')) - { + while ((*argument >= '0') && (*argument <= '9')) { g_target *= 10; g_target += *argument - '0'; argument++; @@ -1167,8 +1125,7 @@ int main(int argc, char** argv) if (filenamesStart==0) result = benchSample(); - else - { + else { if (optimizer) result = optimizeForSize(input_filename); else @@ -1179,4 +1136,3 @@ int main(int argc, char** argv) return result; } -