mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-27 23:46:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

bench: added cli command -S to benchmark multiple files separately

Browse Source 
Currently, all files are joined by default,
they are compressed separately but benchmarked together,
providing a single final result.

Benchmarking files separately make it possible to accurately measure difference for each file.
This is expected to be useful while tuning optimal parser.
This commit is contained in:
Yann Collet 2017-11-17 00:22:55 -08:00
parent 8accfa7fcc
commit d898fb7ba6
4 changed files with 43 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
programs/bench.c
View File

@ -135,6 +135,11 @@ void BMK_setRealTime(unsigned priority) {
g_realTime = (priority>0);
}
static U32 g_separateFiles = 0;
void BMK_setSeparateFiles(unsigned separate) {
g_separateFiles = (separate>0);
}
static U32 g_ldmFlag = 0;
void BMK_setLdmFlag(unsigned ldmFlag) {
g_ldmFlag = ldmFlag;
@ -482,11 +487,11 @@ static size_t BMK_findMaxMem(U64 requiredMem)
return (size_t)(requiredMem);
}
static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
static void BMK_benchCLevel(const void* srcBuffer, size_t benchedSize,
const char* displayName, int cLevel, int cLevelLast,
const size_t* fileSizes, unsigned nbFiles,
const void* dictBuffer, size_t dictBufferSize,
ZSTD_compressionParameters *compressionParams)
const ZSTD_compressionParameters* const compressionParams)
{
int l;
@ -518,7 +523,7 @@ static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize,
At most, fills `buffer` entirely */
static void BMK_loadFiles(void* buffer, size_t bufferSize,
size_t* fileSizes,
const char** fileNamesTable, unsigned nbFiles)
const char* const * const fileNamesTable, unsigned nbFiles)
{
size_t pos = 0, totalSize = 0;
unsigned n;
@ -550,16 +555,17 @@ static void BMK_loadFiles(void* buffer, size_t bufferSize,
if (totalSize == 0) EXM_THROW(12, "no data to bench");
}
static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, const char* dictFileName, int cLevel,
int cLevelLast, ZSTD_compressionParameters *compressionParams)
static void BMK_benchFileTable(const char* const * const fileNamesTable, unsigned const nbFiles,
const char* const dictFileName,
int const cLevel, int const cLevelLast,
const ZSTD_compressionParameters* const compressionParams)
{
void* srcBuffer;
size_t benchedSize;
void* dictBuffer = NULL;
size_t dictBufferSize = 0;
size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
size_t* const fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t));
U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles);
char mfName[20] = {0};
if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes");
@ -588,13 +594,26 @@ static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, co
BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles);
/* Bench */
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams);
}
if (g_separateFiles) {
const BYTE* srcPtr = (const BYTE*)srcBuffer;
U32 fileNb;
for (fileNb=0; fileNb<nbFiles; fileNb++) {
size_t const fileSize = fileSizes[fileNb];
BMK_benchCLevel(srcPtr, fileSize,
fileNamesTable[fileNb], cLevel, cLevelLast,
fileSizes+fileNb, 1,
dictBuffer, dictBufferSize, compressionParams);
srcPtr += fileSize;
}
} else {
char mfName[20] = {0};
snprintf (mfName, sizeof(mfName), " %u files", nbFiles);
{ const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0];
BMK_benchCLevel(srcBuffer, benchedSize,
displayName, cLevel, cLevelLast,
fileSizes, nbFiles,
dictBuffer, dictBufferSize, compressionParams);
} }
/* clean up */
free(srcBuffer);

1
programs/bench.h
View File

@ -25,6 +25,7 @@ void BMK_setBlockSize(size_t blockSize);
void BMK_setNbThreads(unsigned nbThreads);
void BMK_setRealTime(unsigned priority);
void BMK_setNotificationLevel(unsigned level);
void BMK_setSeparateFiles(unsigned separate);
void BMK_setAdditionalParam(int additionalParam);
void BMK_setDecodeOnlyMode(unsigned decodeFlag);
void BMK_setLdmFlag(unsigned ldmFlag);

2
programs/util.h
View File

@ -336,7 +336,7 @@ UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
}
UTIL_STATIC U64 UTIL_getTotalFileSize(const char** fileNamesTable, unsigned nbFiles)
UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
int error = 0;

8
programs/zstdcli.c
View File

@ -377,6 +377,7 @@ int main(int argCount, const char* argv[])
lastCommand = 0,
nbThreads = 1,
setRealTimePrio = 0,
separateFiles = 0,
ldmFlag = 0;
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
size_t blockSize = 0;
@ -633,6 +634,12 @@ int main(int argCount, const char* argv[])
blockSize = readU32FromChar(&argument);
break;
/* benchmark files separately (hidden option) */
case 'S':
argument++;
separateFiles = 1;
break;
#endif /* ZSTD_NOBENCH */
/* nb of threads (hidden option) */
@ -751,6 +758,7 @@ int main(int argCount, const char* argv[])
if (operation==zom_bench) {
#ifndef ZSTD_NOBENCH
BMK_setNotificationLevel(g_displayLevel);
BMK_setSeparateFiles(separateFiles);
BMK_setBlockSize(blockSize);
BMK_setNbThreads(nbThreads);
BMK_setRealTime(setRealTimePrio);