mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-25 00:56:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

removed ZSTD_highbit() from zstd_internal.h, as it is only used by zstd_compress.c

Browse Source
This commit is contained in:
Yann Collet 2016-06-04 20:17:38 +02:00
parent 16f729957f
commit 49bb0041af
5 changed files with 59 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/common/zstd.h
View File

@ -162,7 +162,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
/*--- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB526 /* v0.6 */
#define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27))
@ -350,7 +350,7 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds
ZSTD_decompressContinue() needs previous data blocks during decompression, up to (1 << windowlog).
They should preferably be located contiguously, prior to current block. Alternatively, a round buffer is also possible.
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity)
@result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity).
It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some header.
A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero.

26
lib/common/zstd_internal.h
View File

@ -85,7 +85,7 @@
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 1, 2, 8 };
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
#define ZSTD_BLOCKHEADERSIZE 3 /* because C standard does not allow a static const value to be defined using another static const value .... :( */
#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */
static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE;
typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t;
@ -162,28 +162,6 @@ MEM_STATIC void ZSTD_wildcopy(void* dst, const void* src, size_t length)
while (op < oend);
}
MEM_STATIC unsigned ZSTD_highbit(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-*******************************************
* Private interfaces
@ -211,7 +189,7 @@ typedef struct {
MEM_STATIC void ZSTD_statsInit(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsResetFreqs(ZSTD_stats_t* stats) { (void)stats; }
MEM_STATIC void ZSTD_statsUpdatePrices(ZSTD_stats_t* stats, size_t litLength, const BYTE* literals, size_t offset, size_t matchLength) { (void)stats; (void)litLength; (void)literals; (void)offset; (void)matchLength; }
#endif // #if ZSTD_OPT_DEBUG == 3
#endif /* #if ZSTD_OPT_DEBUG == 3 */
typedef struct {
void* buffer;

57
lib/compress/zstd_compress.c
View File

@ -71,6 +71,27 @@ static const U32 g_searchStrength = 8; /* control skip over incompressible dat
***************************************/
size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; }
static U32 ZSTD_highbit32(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-*************************************
* Sequence storage
@ -176,8 +197,6 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
}
static unsigned ZSTD_highbit(U32 val);
/** ZSTD_checkCParams_advanced() :
temporary work-around, while the compressor compatibility remains limited regarding windowLog < 18 */
size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSize)
@ -205,7 +224,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = ZSTD_highbit((U32)(rSize)-1) + 1;
U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1;
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
@ -649,7 +668,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
size_t u;
for (u=0; u<nbSeq; u++) {
U32 const ll = llTable[u];
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit(ll) + LL_deltaCode : LL_Code[ll];
llCodeTable[u] = (ll>63) ? (BYTE)ZSTD_highbit32(ll) + LL_deltaCode : LL_Code[ll];
}
if (seqStorePtr->longLengthID==1)
llCodeTable[seqStorePtr->longLengthPos] = MaxLL;
@ -659,7 +678,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
{ const U32* const offsetTable = seqStorePtr->offsetStart;
BYTE* const ofCodeTable = seqStorePtr->offCodeStart;
size_t u;
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit(offsetTable[u]);
for (u=0; u<nbSeq; u++) ofCodeTable[u] = (BYTE)ZSTD_highbit32(offsetTable[u]);
}
/* ML codes */
@ -677,7 +696,7 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq)
size_t u;
for (u=0; u<nbSeq; u++) {
U32 const ml = mlTable[u];
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit(ml) + ML_deltaCode : ML_Code[ml];
mlCodeTable[u] = (ml>127) ? (BYTE)ZSTD_highbit32(ml) + ML_deltaCode : ML_Code[ml];
}
if (seqStorePtr->longLengthID==2)
mlCodeTable[seqStorePtr->longLengthPos] = MaxML;
@ -1425,7 +1444,7 @@ static size_t ZSTD_insertBtAndFindBestMatch (
if (matchLength > bestLength) {
if (matchLength > matchEndIdx - matchIndex)
matchEndIdx = matchIndex + (U32)matchLength;
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit(current-matchIndex+1) - ZSTD_highbit((U32)offsetPtr[0]+1)) )
if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) )
bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex;
if (ip+matchLength == iend) /* equal : no way to know if inf or sup */
break; /* drop, to guarantee consistency (miss a little bit of compression) */
@ -1711,14 +1730,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
int const gain2 = (int)(mlRep * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
if ((mlRep >= EQUAL_READ32) && (gain2 > gain1))
matchLength = mlRep, offset = 0, start = ip;
}
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */
@ -1730,14 +1749,14 @@ void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx,
if ((offset) && (MEM_read32(ip) == MEM_read32(ip - rep[0]))) {
size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-rep[0], iend) + EQUAL_READ32;
int const gain2 = (int)(ml2 * 4);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1))
matchLength = ml2, offset = 0, start = ip;
}
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue;
@ -1883,7 +1902,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
int const gain2 = (int)(repLength * 3);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit((U32)offset+1) + 1);
int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip;
} }
@ -1891,8 +1910,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* search match, depth 1 */
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 4);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue; /* search a better one */
@ -1913,7 +1932,7 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend;
size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32;
int gain2 = (int)(repLength * 4);
int gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 1);
int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1);
if ((repLength >= EQUAL_READ32) && (gain2 > gain1))
matchLength = repLength, offset = 0, start = ip;
} }
@ -1921,8 +1940,8 @@ void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx,
/* search match, depth 2 */
{ size_t offset2=99999999;
size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls);
int const gain2 = (int)(ml2*4 - ZSTD_highbit((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit((U32)offset+1) + 7);
int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */
int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7);
if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) {
matchLength = ml2, offset = offset2, start = ip;
continue;

32
lib/compress/zstd_opt.h
View File

@ -41,10 +41,10 @@
***************************************/
FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr)
{
ssPtr->log2matchLengthSum = ZSTD_highbit(ssPtr->matchLengthSum+1);
ssPtr->log2litLengthSum = ZSTD_highbit(ssPtr->litLengthSum+1);
ssPtr->log2litSum = ZSTD_highbit(ssPtr->litSum+1);
ssPtr->log2offCodeSum = ZSTD_highbit(ssPtr->offCodeSum+1);
ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1);
ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1);
ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1);
ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1);
ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum));
}
@ -106,7 +106,7 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
U32 price, u;
if (litLength == 0)
return ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[0]+1);
return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1);
/* literals */
if (ssPtr->cachedLiterals == literals) {
@ -114,13 +114,13 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength;
price = ssPtr->cachedPrice + additional * ssPtr->log2litSum;
for (u=0; u < additional; u++)
price -= ZSTD_highbit(ssPtr->litFreq[literals2[u]]+1);
price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1);
ssPtr->cachedPrice = price;
ssPtr->cachedLitLength = litLength;
} else {
price = litLength * ssPtr->log2litSum;
for (u=0; u < litLength; u++)
price -= ZSTD_highbit(ssPtr->litFreq[literals[u]]+1);
price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1);
if (litLength >= 12) {
ssPtr->cachedLiterals = literals;
@ -139,8 +139,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit(ssPtr->litLengthFreq[llCode]+1);
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1);
}
return price;
@ -150,8 +150,8 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength)
{
/* offset */
BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit(seqStorePtr->offCodeFreq[offCode]+1);
BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
/* match Length */
{ static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
@ -163,8 +163,8 @@ FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYT
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit(seqStorePtr->matchLengthFreq[mlCode]+1);
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1);
}
return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor;
@ -190,13 +190,13 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
24, 24, 24, 24, 24, 24, 24, 24,
24, 24, 24, 24, 24, 24, 24, 24 };
const BYTE LL_deltaCode = 19;
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit(litLength) + LL_deltaCode : LL_Code[litLength];
const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength];
seqStorePtr->litLengthFreq[llCode]++;
seqStorePtr->litLengthSum++;
}
/* match offset */
{ BYTE offCode = (BYTE)ZSTD_highbit(offset+1);
{ BYTE offCode = (BYTE)ZSTD_highbit32(offset+1);
seqStorePtr->offCodeSum++;
seqStorePtr->offCodeFreq[offCode]++;
}
@ -211,7 +211,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42,
42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 };
const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit(matchLength) + ML_deltaCode : ML_Code[matchLength];
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
seqStorePtr->matchLengthFreq[mlCode]++;
seqStorePtr->matchLengthSum++;
}

2
lib/dictBuilder/zdict.c
View File

@ -60,7 +60,7 @@
#include "error_private.h"
#include "fse.h"
#include "huf_static.h"
#include "zstd_internal.h"
#include "zstd_internal.h" /* includes zstd.h */
#include "xxhash.h"
#include "divsufsort.h"
#define ZDICT_STATIC_LINKING_ONLY