mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-28 02:46:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1 from Cyan4973/dev

Browse Source 
Dev
This commit is contained in:
Yann Collet 2015-01-24 13:47:43 +01:00
commit adcffd085c
4 changed files with 98 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/zstd.c
View File

@ -1004,9 +1004,9 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
//U32 limit = 4 * BLOCKSIZE;
//const U32 updateRate = 2 * BLOCKSIZE;
// Init
/* Init */
if (ctx->base==NULL) ctx->base = src, ctx->current=0;
if (src != ctx->base + ctx->current) // not contiguous
if (src != ctx->base + ctx->current) /* not contiguous */
{
ZSTD_resetCCtx(ctx);
ctx->base = src;
@ -1035,12 +1035,12 @@ size_t ZSTD_compressContinue(ZSTD_cctx_t cctx, void* dst, size_t maxDstSize, con
}
*/
// compress
/* compress */
if (maxDstSize < ZSTD_blockHeaderSize) return (size_t)-ZSTD_ERROR_maxDstSize_tooSmall;
cSize = ZSTD_compressBlock(ctx, op+ZSTD_blockHeaderSize, maxDstSize-ZSTD_blockHeaderSize, ip, blockSize);
if (cSize == 0)
{
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize);
cSize = ZSTD_noCompressBlock(op, maxDstSize, ip, blockSize); /* block is not compressible */
if (ZSTD_isError(cSize)) return cSize;
}
else
@ -1084,7 +1084,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
BYTE* const ostart = dst;
BYTE* op = ostart;
// Header
/* Header */
{
size_t headerSize = ZSTD_compressBegin(ctx, dst, maxDstSize);
if(ZSTD_isError(headerSize)) return headerSize;
@ -1092,7 +1092,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
maxDstSize -= headerSize;
}
// Compression
/* Compression */
{
size_t cSize = ZSTD_compressContinue(ctx, op, maxDstSize, src, srcSize);
if (ZSTD_isError(cSize)) return cSize;
@ -1100,7 +1100,7 @@ static size_t ZSTD_compressCCtx(void* ctx, void* dst, size_t maxDstSize, const v
maxDstSize -= cSize;
}
// Close frame
/* Close frame */
{
size_t endSize = ZSTD_compressEnd(ctx, op, maxDstSize);
if(ZSTD_isError(endSize)) return endSize;

10
programs/Makefile
View File

@ -127,11 +127,17 @@ test-zstd32: zstd32 datagen
./datagen -g256MB | ./zstd32 -v | ./zstd32 -d > $(VOID)
./datagen -g6GB | ./zstd32 -vq | ./zstd32 -d > $(VOID)
test-fullbench: fullbench
test-fullbench: fullbench datagen
./fullbench -i1
./datagen -P0 -g516K > tmp
./fullbench -i1 tmp
@rm tmp
test-fullbench32: fullbench32
test-fullbench32: fullbench32 datagen
./fullbench32 -i1
./datagen -P0 -g516K > tmp
./fullbench32 -i1 tmp
@rm tmp
test-fuzzer: fuzzer
./fuzzer

87
programs/datagen.c
View File

@ -56,6 +56,24 @@
#endif
/**************************************
* OS-specific Includes
**************************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _isatty */
# ifdef __MINGW32__
int _fileno(FILE *stream); /* MINGW somehow forgets to include this windows declaration into <stdio.h> */
# endif
# define SET_BINARY_MODE(file) _setmode(_fileno(file), _O_BINARY)
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#else
# include <unistd.h> /* isatty */
# define SET_BINARY_MODE(file)
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#endif
/**************************************
* Constants
**************************************/
@ -85,7 +103,6 @@
* Local Parameters
**************************************/
static unsigned no_prompt = 0;
static char* programName;
static unsigned displayLevel = 2;
@ -97,7 +114,7 @@ static unsigned int CDG_rand(U32* src)
{
U32 rand32 = *src;
rand32 *= PRIME1;
rand32 += PRIME2;
rand32 ^= PRIME2;
rand32 = CDG_rotl32(rand32, 13);
*src = rand32;
return rand32;
@ -106,14 +123,20 @@ static unsigned int CDG_rand(U32* src)
#define LTSIZE 8192
#define LTMASK (LTSIZE-1)
static const char firstChar = '(';
static const char lastChar = '}';
static void* CDG_createLiteralDistrib(double ld)
{
char* lt = malloc(LTSIZE);
BYTE* lt = malloc(LTSIZE);
U32 i = 0;
char character = '0';
BYTE character = '0';
BYTE firstChar = '(';
BYTE lastChar = '}';
if (ld==0.0)
{
character = 0;
firstChar = 0;
lastChar =255;
}
while (i<LTSIZE)
{
U32 weight = (U32)((double)(LTSIZE - i) * ld) + 1;
@ -137,7 +160,7 @@ static char CDG_genChar(U32* seed, const void* ltctx)
#define CDG_RAND15BITS ((CDG_rand(seed) >> 3) & 32767)
#define CDG_RANDLENGTH ( ((CDG_rand(seed) >> 7) & 7) ? (CDG_rand(seed) & 15) : (CDG_rand(seed) & 511) + 15)
#define CDG_DICTSIZE (32 KB)
static void CDG_generate(U64 size, U32* seed, double matchProba)
static void CDG_generate(U64 size, U32* seed, double matchProba, double litProba)
{
BYTE fullbuff[CDG_DICTSIZE + 128 KB + 1];
BYTE* buff = fullbuff + CDG_DICTSIZE;
@ -145,10 +168,11 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
U32 P32 = (U32)(32768 * matchProba);
U32 pos=1;
U32 genBlockSize = 128 KB;
double literalDistrib = 0.13;
void* ldctx = CDG_createLiteralDistrib(literalDistrib);
void* ldctx = CDG_createLiteralDistrib(litProba);
FILE* fout = stdout;
/* Build initial prefix */
/* init */
SET_BINARY_MODE(stdout);
fullbuff[0] = CDG_genChar(seed, ldctx);
while (pos<32 KB)
{
@ -207,11 +231,8 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
}
}
/* output datagen */
pos=0;
for (;pos+512<=genBlockSize;pos+=512)
printf("%512.512s", buff+pos);
for (;pos<genBlockSize;pos++) printf("%c", buff[pos]);
/* output generated data */
fwrite(buff, 1, genBlockSize, fout);
/* Regenerate prefix */
memcpy(fullbuff, buff + 96 KB, 32 KB);
}
@ -221,7 +242,7 @@ static void CDG_generate(U64 size, U32* seed, double matchProba)
/*********************************************************
* Command line
*********************************************************/
static int CDG_usage(void)
static int CDG_usage(char* programName)
{
DISPLAY( "Compressible data generator\n");
DISPLAY( "Usage :\n");
@ -239,9 +260,11 @@ static int CDG_usage(void)
int main(int argc, char** argv)
{
int argNb;
int proba = CDG_COMPRESSIBILITY_DEFAULT;
double proba = (double)CDG_COMPRESSIBILITY_DEFAULT / 100;
double litProba = proba / 3.6;
U64 size = CDG_SIZE_DEFAULT;
U32 seed = CDG_SEED_DEFAULT;
char* programName;
/* Check command line */
programName = argv[0];
@ -262,7 +285,7 @@ int main(int argc, char** argv)
switch(*argument)
{
case 'h':
return CDG_usage();
return CDG_usage(programName);
case 'g':
argument++;
size=0;
@ -287,23 +310,37 @@ int main(int argc, char** argv)
argument++;
}
break;
case 'p':
case 'P':
argument++;
proba=0;
proba=0.0;
while ((*argument>='0') && (*argument<='9'))
{
proba *= 10;
proba += *argument - '0';
argument++;
}
if (proba<0) proba=0;
if (proba>100) proba=100;
if (proba>100.) proba=100.;
proba /= 100.;
litProba = proba / 4.;
break;
case 'L':
argument++;
litProba=0.;
while ((*argument>='0') && (*argument<='9'))
{
litProba *= 10;
litProba += *argument - '0';
argument++;
}
if (litProba>100.) litProba=100.;
litProba /= 100.;
break;
case 'v':
displayLevel = 4;
argument++;
break;
default: ;
default:
return CDG_usage(programName);
}
}
@ -312,9 +349,9 @@ int main(int argc, char** argv)
DISPLAYLEVEL(4, "Data Generator %s \n", ZSTD_VERSION);
DISPLAYLEVEL(3, "Seed = %u \n", seed);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", proba);
if (proba!=CDG_COMPRESSIBILITY_DEFAULT) DISPLAYLEVEL(3, "Compressibility : %i%%\n", (U32)(proba*100));
CDG_generate(size, &seed, ((double)proba) / 100);
CDG_generate(size, &seed, proba, litProba);
return 0;
}

26
programs/fullbench.c
View File

@ -383,7 +383,7 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
}
/* Allocation */
dstBuffSize = srcSize + 512;
dstBuffSize = ZSTD_compressBound(srcSize);
dstBuff = malloc(dstBuffSize);
buff2 = malloc(dstBuffSize);
if ((!dstBuff) || (!buff2))
@ -403,6 +403,14 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
{
blockProperties_t bp;
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ZSTD_getcBlockSize(dstBuff+4, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
g_cSize = ZSTD_getcBlockSize(dstBuff+7, dstBuffSize, &bp) + 3;
memcpy(buff2, dstBuff+7, g_cSize);
//srcSize = benchFunction(dstBuff, dstBuffSize, buff2, src, srcSize); // real speed
@ -418,6 +426,13 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
ZSTD_compress(dstBuff, dstBuffSize, src, srcSize);
ip += 4; // Jump magic Number
blockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); // Get first block compressed size
if (bp.blockType != bt_compressed)
{
DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n");
free(dstBuff);
free(buff2);
return 0;
}
iend = ip + 3 + blockSize; // Get end of first block
ip += 3; // jump first block header
ip += ZSTD_getcBlockSize(ip, iend - ip, &bp) + 3; // jump literal sub block and its header
@ -450,6 +465,8 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
default : ;
}
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
for (loopNb = 1; loopNb <= nbIterations; loopNb++)
{
double averageTime;
@ -457,7 +474,6 @@ size_t benchMem(void* src, size_t srcSize, U32 benchNb)
U32 nbRounds=0;
DISPLAY("%2i- %-30.30s : \r", loopNb, benchName);
{ size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } /* warming up memory */
milliTime = BMK_GetMilliStart();
while(BMK_GetMilliStart() == milliTime);
@ -545,7 +561,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", inFileName, (int)(benchedSize>>20));
}
// Alloc
/* Alloc */
origBuff = (char*) malloc((size_t)benchedSize);
if(!origBuff)
{
@ -554,7 +570,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 12;
}
// Fill input buffer
/* Fill input buffer */
DISPLAY("Loading %s... \r", inFileName);
readSize = fread(origBuff, 1, benchedSize, inFile);
fclose(inFile);
@ -566,7 +582,7 @@ int benchFiles(char** fileNamesTable, int nbFiles, U32 benchNb)
return 13;
}
// bench
/* bench */
DISPLAY("\r%79s\r", "");
DISPLAY(" %s : \n", inFileName);
if (benchNb)