mirrors/zstd
0
0
Fork 0
mirror of https://github.com/facebook/zstd.git synced 2024-11-28 23:36:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix : srcSize written in frame header when multiple files compressed

Browse Source 
This information used to be disabled when nbFiles>1.
It was badly initialized later in the code, resulting in an error.
This commit is contained in:
Yann Collet 2017-09-29 18:05:18 -07:00
parent 7c9669f272
commit db1668a43b
2 changed files with 15 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/compress/zstd_compress.c
View File

@ -822,11 +822,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc,
ZSTD_compResetPolicy_e const crp,
ZSTD_buffered_policy_e const zbuff)
{
DEBUGLOG(4, "ZSTD_resetCCtx_internal");
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
DEBUGLOG(4, "pledgedSrcSize: %u", (U32)pledgedSrcSize);
if (crp == ZSTDcrp_continue) {
if (ZSTD_equivalentParams(params, zc->appliedParams)) {
DEBUGLOG(5, "ZSTD_equivalentParams()==1");
DEBUGLOG(4, "ZSTD_equivalentParams()==1");
assert(!(params.ldmParams.enableLdm &&
params.ldmParams.hashEveryLog == ZSTD_LDM_HASHEVERYLOG_NOTSET));
zc->entropy->hufCTable_repeatMode = HUF_repeat_none;
@ -2011,8 +2013,6 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
ZSTD_buffered_policy_e zbuff)
{
DEBUGLOG(4, "ZSTD_compressBegin_internal");
DEBUGLOG(4, "dict ? %s", dict ? "dict" : (cdict ? "cdict" : "none"));
DEBUGLOG(4, "dictMode : %u", (U32)dictMode);
/* params are supposed to be fully validated at this point */
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@ -2485,7 +2485,7 @@ size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
ZSTD_CCtx_params params = zcs->requestedParams;
params.fParams.contentSizeFlag = (pledgedSrcSize > 0);
params.cParams = ZSTD_getCParamsFromCCtxParams(params, pledgedSrcSize, 0);
DEBUGLOG(5, "ZSTD_resetCStream");
DEBUGLOG(4, "ZSTD_resetCStream");
return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dm_auto, zcs->cdict, params, pledgedSrcSize);
}
@ -2497,6 +2497,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const ZSTD_CDict* cdict,
ZSTD_CCtx_params params, unsigned long long pledgedSrcSize)
{
DEBUGLOG(4, "ZSTD_initCStream_internal");
assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams)));
assert(!((dict) && (cdict))); /* either dict or cdict, not both */
@ -2768,6 +2769,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/);
memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */
assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */
DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage");
#ifdef ZSTD_MULTITHREAD
if (params.nbThreads > 1) {

15
programs/fileio.c
View File

@ -757,6 +757,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
U64 readsize = 0;
U64 compressedfilesize = 0;
U64 const fileSize = UTIL_getFileSize(srcFileName);
DISPLAYLEVEL(5, "%s: %u bytes \n", srcFileName, (U32)fileSize);
switch (g_compressionType) {
case FIO_zstdCompression:
@ -796,7 +797,7 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* init */
#ifdef ZSTD_NEWAPI
/* nothing, reset is implied */
CHECK( ZSTD_resetCStream(ress.cctx, fileSize) ); /* to pass fileSize */
#elif defined(ZSTD_MULTITHREAD)
CHECK( ZSTDMT_resetCStream(ress.cctx, fileSize) );
#else
@ -849,10 +850,10 @@ static int FIO_compressFilename_internal(cRess_t ress,
/* End of Frame */
{ size_t result = 1;
while (result!=0) { /* note : is there any possibility of endless loop ? */
while (result != 0) {
ZSTD_outBuffer outBuff = { ress.dstBuffer, ress.dstBufferSize, 0 };
#ifdef ZSTD_NEWAPI
ZSTD_inBuffer inBuff = { NULL, 0, 0};
ZSTD_inBuffer inBuff = { NULL, 0, 0 };
result = ZSTD_compress_generic(ress.cctx,
&outBuff, &inBuff, ZSTD_e_end);
#elif defined(ZSTD_MULTITHREAD)
@ -863,8 +864,10 @@ static int FIO_compressFilename_internal(cRess_t ress,
if (ZSTD_isError(result))
EXM_THROW(26, "Compression error during frame end : %s",
ZSTD_getErrorName(result));
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos) EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName); }
{ size_t const sizeCheck = fwrite(ress.dstBuffer, 1, outBuff.pos, dstFile);
if (sizeCheck!=outBuff.pos)
EXM_THROW(27, "Write error : cannot write frame end into %s", dstFileName);
}
compressedfilesize += outBuff.pos;
}
}
@ -974,7 +977,7 @@ int FIO_compressMultipleFilenames(const char** inFileNamesTable, unsigned nbFile
char* dstFileName = (char*)malloc(FNSPACE);
size_t const suffixSize = suffix ? strlen(suffix) : 0;
U64 const srcSize = (nbFiles != 1) ? 0 : UTIL_getFileSize(inFileNamesTable[0]) ;
int const isRegularFile = (nbFiles > 1) ? 0 : UTIL_isRegularFile(inFileNamesTable[0]); /* won't write frame content size when nbFiles > 1 */
int const isRegularFile = (nbFiles > 1) ? 1 : UTIL_isRegularFile(inFileNamesTable[0]); /* if nbFiles > 1, it's not stdin */
cRess_t ress = FIO_createCResources(dictFileName, compressionLevel, srcSize, isRegularFile, comprParams);
/* init */