mirrors/lz4
0
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2024-12-01 13:53:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

LZ4HC_getSearchNum

Browse Source
This commit is contained in:
Przemyslaw Skibinski 2016-12-28 15:18:19 +01:00
parent 98f9d6c726
commit 1c80b9af4e
3 changed files with 26 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/lz4hc.c
View File

@ -486,6 +486,14 @@ _Search3:
return (int) (((char*)op)-dest); return (int) (((char*)op)-dest);
} }
static int LZ4HC_getSearchNum(int compressionLevel)
{
switch (compressionLevel) {
default: return 0; /* unused */
case 11: return 128;
case 12: return 1<<10;
}
}
static int LZ4HC_compress_generic ( static int LZ4HC_compress_generic (
LZ4HC_CCtx_internal* const ctx, LZ4HC_CCtx_internal* const ctx,
@ -497,13 +505,14 @@ static int LZ4HC_compress_generic (
limitedOutput_directive limit limitedOutput_directive limit
) )
{ {
// printf("LZ4HC_compress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel);
if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT; if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
if (compressionLevel > 9) { if (compressionLevel > 9) {
switch (compressionLevel) { switch (compressionLevel) {
case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit); case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
case 11: ctx->searchNum = 128; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0); case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
default: default:
case 12: ctx->searchNum = 1<<10; return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1); case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
} }
} }
return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit); return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
@ -554,11 +563,13 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.base = NULL;
LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
} }
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{ {
LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse; LZ4HC_CCtx_internal* ctxPtr = &LZ4_streamHCPtr->internal_donotuse;
// printf("LZ4_loadDictHC dictSize=%d\n", (int)dictSize);
if (dictSize > 64 KB) { if (dictSize > 64 KB) {
dictionary += dictSize - 64 KB; dictionary += dictSize - 64 KB;
dictSize = 64 KB; dictSize = 64 KB;
@ -577,6 +588,7 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{ {
// printf("LZ4HC_setExternalDict\n");
if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN) if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS); LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
else else

18
lib/lz4opt.h
View File

@ -37,6 +37,7 @@
#define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) #define LZ4_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ4_LOG_TREE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ4_OPT_NUM (1<<12) #define LZ4_OPT_NUM (1<<12)
@ -106,6 +107,8 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
size_t matchLength = 0; size_t matchLength = 0;
U32* HashPos; U32* HashPos;
LZ4_LOG_TREE("LZ4HC_BinTree_InsertAndGetAllMatches nbAttempts=%d\n", nbAttempts);
if (ip + MINMATCH > iHighLimit) return 1; if (ip + MINMATCH > iHighLimit) return 1;
/* HC4 match finder */ /* HC4 match finder */
@ -117,6 +120,7 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
ptr1 = &DELTANEXTMAXD(current*2); ptr1 = &DELTANEXTMAXD(current*2);
delta0 = delta1 = (U16)(current - matchIndex); delta0 = delta1 = (U16)(current - matchIndex);
LZ4_LOG_TREE("matchIndex[%u] current[%u] lowLimit[%u]\n", matchIndex, current, lowLimit);
while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) { while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--; nbAttempts--;
if (matchIndex >= dictLimit) { if (matchIndex >= dictLimit) {
@ -141,29 +145,30 @@ FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
matches[mnum].len = (int)matchLength; matches[mnum].len = (int)matchLength;
mnum++; mnum++;
} }
if (best_mlen > LZ4_OPT_NUM) break; if (best_mlen > LZ4_OPT_NUM) { LZ4_LOG_TREE("best_mlen > LZ4_OPT_NUM\n"); break; }
} }
if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */ if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ { LZ4_LOG_TREE("ip+matchLength > iHighLimit\n"); break; } /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
if (*(ip+matchLength) < *(match+matchLength)) { if (*(ip+matchLength) < *(match+matchLength)) {
*ptr0 = delta0; *ptr0 = delta0;
ptr0 = &DELTANEXTMAXD(matchIndex*2); ptr0 = &DELTANEXTMAXD(matchIndex*2);
if (*ptr0 == (U16)-1) break; if (*ptr0 == (U16)-1) { LZ4_LOG_TREE("*ptr0 == (U16)-1"); break; }
delta0 = *ptr0; delta0 = *ptr0;
delta1 += delta0; delta1 += delta0;
matchIndex -= delta0; matchIndex -= delta0;
} else { } else {
*ptr1 = delta1; *ptr1 = delta1;
ptr1 = &DELTANEXTMAXD(matchIndex*2+1); ptr1 = &DELTANEXTMAXD(matchIndex*2+1);
if (*ptr1 == (U16)-1) break; if (*ptr1 == (U16)-1) { LZ4_LOG_TREE("*ptr1 == (U16)-1\n"); break; }
delta1 = *ptr1; delta1 = *ptr1;
delta0 += delta1; delta0 += delta1;
matchIndex -= delta1; matchIndex -= delta1;
} }
} }
LZ4_LOG_TREE("nbAttempts=%d mnum=%d\n", nbAttempts, mnum);
*ptr0 = (U16)-1; *ptr0 = (U16)-1;
*ptr1 = (U16)-1; *ptr1 = (U16)-1;
if (matchNum) *matchNum = mnum; if (matchNum) *matchNum = mnum;
@ -178,7 +183,7 @@ FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* cons
const BYTE* const base = ctx->base; const BYTE* const base = ctx->base;
const U32 target = (U32)(ip - base); const U32 target = (U32)(ip - base);
U32 idx = ctx->nextToUpdateBT; U32 idx = ctx->nextToUpdateBT;
LZ4_LOG_TREE("LZ4HC_updateBinTree %d->%d nbAttempts=%d\n", idx, target, ctx->searchNum);
while(idx < target) while(idx < target)
idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL); idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL);
} }
@ -206,7 +211,7 @@ FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
opt[pos].off = (int)offset; \ opt[pos].off = (int)offset; \
opt[pos].litlen = (int)litlen; \ opt[pos].litlen = (int)litlen; \
opt[pos].price = (int)price; \ opt[pos].price = (int)price; \
LZ4_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ LZ4_LOG_PARSER("%u: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-(const BYTE*)source), (int)(pos), (int)last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \
} }
@ -330,7 +335,6 @@ static int LZ4HC_compress_optimal (
price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen); price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
} }
LZ4_LOG_PARSER("%d: Found2 mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-(const BYTE*)source), (int)mlen, (int)best_mlen, matches[i].off, (int)price, (int)litlen, (int)(cur - litlen), opt[cur - litlen].price);
if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) { if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
} }

2
tests/fuzzer.c
View File

@ -601,7 +601,7 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);
ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1); ret = LZ4_compress_HC_continue(&LZ4dictHC, block, compressedBuffer, blockSize, blockContinueCompressedSize-1);
FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer"); FUZ_CHECKTEST(ret>0, "LZ4_compressHC_limitedOutput_continue using ExtDict should fail : one missing byte for output buffer (%i != %i)", ret, blockContinueCompressedSize);
FUZ_DISPLAYTEST; FUZ_DISPLAYTEST;
LZ4_loadDictHC(&LZ4dictHC, dict, dictSize); LZ4_loadDictHC(&LZ4dictHC, dict, dictSize);