mirror of
https://github.com/facebook/zstd.git
synced 2024-12-01 02:56:45 +08:00
Updated fullbench tests
This commit is contained in:
Yann Collet
parent
18a68714d0
commit
759433d679
14
lib/zstd.c
14
lib/zstd.c
@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
|
||||
//U32 limit = 4 * BLOCKSIZE;
|
||||
//const U32 updateRate = 2 * BLOCKSIZE;
|
||||
|
||||
// Init
|
||||
/* Init */
|
||||
if (ctx->base==NULL) ctx->base = src, ctx->current=0;
|
||||
if (src != ctx->base + ctx->current) // not contiguous
|
||||
if (src != ctx->base + ctx->current) /* not contiguous */
|
||||
{
|
||||
ZSTD_resetCCtx(ctx);
|
||||
ctx->base = src;
|
||||
@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
|
||||
}
|
||||
*/
|
||||
|
||||
// compress
|
||||
/* compress */
|
||||
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
|
||||
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
|
||||
if (cSize == 0)
|
||||
{
|
||||
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
|
||||
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
}
|
||||
else
|
||||
@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
|
||||
BYTE* const ostart = dst;
|
||||
BYTE* op = ostart;
|
||||
|
||||
// Header
|
||||
/* Header */
|
||||
{
|
||||
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
|
||||
if(ZSTD_isError(headerSize)) return headerSize;
|
||||
@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
|
||||
maxDstSize -= headerSize;
|
||||
}
|
||||
|
||||
// Compression
|
||||
/* Compression */
|
||||
{
|
||||
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
|
||||
if (ZSTD_isError(cSize)) return cSize;
|
||||
@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
|
||||
maxDstSize -= cSize;
|
||||
}
|
||||
|
||||
// Close frame
|
||||
/* Close frame */
|
||||
{
|
||||
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
|
||||
if(ZSTD_isError(endSize)) return endSize;
|
||||
|
||||
@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
|
||||
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
|
||||
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)
|
||||
|
||||
test-fullbench: fullbench
|
||||
test-fullbench: fullbench datagen
|
||||
./fullbench -i1
|
||||
./datagen -P0 -g516K > tmp
|
||||
./fullbench -i1 tmp
|
||||
@rm tmp
|
||||
|
||||
test-fullbench32: fullbench32
|
||||
test-fullbench32: fullbench32 datagen
|
||||
./fullbench32 -i1
|
||||
./datagen -P0 -g516K > tmp
|
||||
./fullbench32 -i1 tmp
|
||||
@rm tmp
|
||||
|
||||
test-fuzzer: fuzzer
|
||||
./fuzzer
|
||||
|
||||
@ -56,6 +56,24 @@
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* OS-specific Includes
|
||||
**************************************/
|
||||
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
|
||||
# include <fcntl.h> /* _O_BINARY */
|
||||
# include <io.h> /* _setmode, _isatty */
|
||||
# ifdef __MINGW32__
|
||||
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
|
||||
# endif
|
||||
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
|
||||
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
|
||||
#else
|
||||
# include <unistd.h> /* isatty */
|
||||
# define SET_BINARY_MODE(file)
|
||||
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
|
||||
#endif
|
||||
|
||||
|
||||
/**************************************
|
||||
* Constants
|
||||
**************************************/
|
||||
@ -70,7 +88,6 @@
|
||||
#define CDG_SIZE_DEFAULT (64 KB)
|
||||
#define CDG_SEED_DEFAULT 0
|
||||
#define CDG_COMPRESSIBILITY_DEFAULT 50
|
||||
#define CDG_LITDENSITY_DEFAULT 12
|
||||
#define PRIME1 2654435761U
|
||||
#define PRIME2 2246822519U
|
||||
|
||||
@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
|
||||
{
|
||||
U32 rand32 = *src;
|
||||
rand32 *= PRIME1;
|
||||
rand32 += PRIME2;
|
||||
rand32 ^= PRIME2;
|
||||
rand32 = CDG_rotl32(rand32, 13);
|
||||
*src = rand32;
|
||||
return rand32;
|
||||
@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
|
||||
|
||||
#define LTSIZE 8192
|
||||
#define LTMASK (LTSIZE-1)
|
||||
static const char firstChar = '(';
|
||||
static const char lastChar = '}';
|
||||
static void* CDG_createLiteralDistrib(double ld)
|
||||
{
|
||||
char* lt = malloc(LTSIZE);
|
||||
BYTE* lt = malloc(LTSIZE);
|
||||
U32 i = 0;
|
||||
char character = '0';
|
||||
BYTE character = '0';
|
||||
BYTE firstChar = '(';
|
||||
BYTE lastChar = '}';
|
||||
|
||||
if (ld==0.0)
|
||||
{
|
||||
character = 0;
|
||||
firstChar = 0;
|
||||
lastChar =255;
|
||||
}
|
||||
while (i<LTSIZE)
|
||||
{
|
||||
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
|
||||
@ -146,8 +169,10 @@ static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba
|
||||
U32 pos=1;
|
||||
U32 genBlockSize = 128 KB;
|
||||
void* ldctx = CDG_createLiteralDistrib(litProba);
|
||||
FILE* fout = stdout;
|
||||
|
||||
/* Build initial prefix */
|
||||
/* init */
|
||||
SET_BINARY_MODE(stdout);
|
||||
fullbuff[0] = CDG_genChar(seed, ldctx);
|
||||
while (pos<32 KB)
|
||||
{
|
||||
@ -206,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba
|
||||
}
|
||||
}
|
||||
|
||||
/* output datagen */
|
||||
pos=0;
|
||||
for (;pos+512<=genBlockSize;pos+=512)
|
||||
printf("%512.512s", buff+pos);
|
||||
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
|
||||
/* output generated data */
|
||||
fwrite(buff, 1, genBlockSize, fout);
|
||||
/* Regenerate prefix */
|
||||
memcpy(fullbuff, buff + 96 KB, 32 KB);
|
||||
}
|
||||
@ -238,8 +260,8 @@ static int CDG_usage(char* programName)
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
int argNb;
|
||||
U32 proba = CDG_COMPRESSIBILITY_DEFAULT;
|
||||
U32 litProba = CDG_LITDENSITY_DEFAULT;
|
||||
double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
|
||||
double litProba = proba / 3.6;
|
||||
U64 size = CDG_SIZE_DEFAULT;
|
||||
U32 seed = CDG_SEED_DEFAULT;
|
||||
char* programName;
|
||||
@ -290,25 +312,28 @@ int main(int argc, char** argv)
|
||||
break;
|
||||
case 'P':
|
||||
argument++;
|
||||
proba=0;
|
||||
proba=0.0;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
proba *= 10;
|
||||
proba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (proba>100) proba=100;
|
||||
if (proba>100.) proba=100.;
|
||||
proba /= 100.;
|
||||
litProba = proba / 4.;
|
||||
break;
|
||||
case 'L':
|
||||
argument++;
|
||||
litProba=0;
|
||||
litProba=0.;
|
||||
while ((*argument>='0') && (*argument<='9'))
|
||||
{
|
||||
litProba *= 10;
|
||||
litProba += *argument - '0';
|
||||
argument++;
|
||||
}
|
||||
if (litProba>100) litProba=100;
|
||||
if (litProba>100.) litProba=100.;
|
||||
litProba /= 100.;
|
||||
break;
|
||||
case 'v':
|
||||
displayLevel = 4;
|
||||
@ -324,9 +349,9 @@ int main(int argc, char** argv)
|
||||
|
||||
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
|
||||
DISPLAYLEVEL(3, "Seed = %u \n", seed);
|
||||
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
|
||||
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
|
||||
|
||||
CDG_generate(size, &seed, ((double)proba) / 100, ((double)litProba) / 100);
|
||||
CDG_generate(size, &seed, proba, litProba);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
{
|
||||
blockProperties_t bp;
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
}
|
||||
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
|
||||
memcpy(buff2, dstBuff+7, g_cSize);
|
||||
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
|
||||
@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
|
||||
ip += 4; // Jump magic Number
|
||||
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
|
||||
if (bp.blockType != bt_compressed)
|
||||
{
|
||||
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
|
||||
free(dstBuff);
|
||||
free(buff2);
|
||||
return 0;
|
||||
}
|
||||
iend = ip + 3 + blockSize; // Get end of first block
|
||||
ip += 3; // jump first block header
|
||||
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
|
||||
@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
default : ;
|
||||
}
|
||||
|
||||
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
|
||||
|
||||
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
|
||||
{
|
||||
double averageTime;
|
||||
@ -457,7 +474,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
|
||||
U32 nbRounds=0;
|
||||
|
||||
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
|
||||
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
|
||||
|
||||
milliTime = BMK_GetMilliStart();
|
||||
while(BMK_GetMilliStart() == milliTime);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user