mirrors/lz4
0
0
Fork 0
mirror of https://github.com/lz4/lz4.git synced 2024-11-26 19:34:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix 1308

Browse Source 
PR #1308 introduces a new symbol, `LZ4_compress_fast_extState_destSize()`,
which, upon inspection, doesn't seem to work as advertised.
In particular, it fails if `dstCapacity` is not large enough,
as opposed to adjusting input size to fit within `dstCapacity`.

Even if it had worked, it would have resulted in problems
for the existing `LZ4_compress_fast_extState()`,
that would have stopped failing when `dstCapacity` is not large enough,
resulting in silent input truncation.

Changed the approach.
It appears the wanted functionality already exists within `lz4` source code,
and is called `LZ4_compress_destSize_extState()`.
It wasn't exposed dues to a (non-verified) problem regarding lz4 state on exit:
it's claimed that since the `_destSize` doesn't completely consume the input,
the state may end up in an "unfinished" status,
making it dangerous to re-employ in later invocations.

Fixed that by enforcing a state initialization at the end of operation.

Also : added some tests,
ensure that `LZ4_compress_fast_extState()` fails as expected when `dstCapacity` is not large enough,
and ensure that `LZ4_compress_destSize_extState()` does succeed in the same circumstance,
by consume less input.

Finally, expose `LZ4_compress_destSize_extState()` as an advanced experimental symbol.
This commit is contained in:
Yann Collet 2023-12-27 20:38:30 -08:00
parent c8673f6d5b
commit 88e477db6b
3 changed files with 44 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
lib/lz4.c
View File

@ -1377,29 +1377,24 @@ LZ4_FORCE_INLINE int LZ4_compress_generic(
int LZ4_compress_fast_extState(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration)
{
return LZ4_compress_fast_extState_destSize(state, source, dest, &inputSize, maxOutputSize, acceleration);
}
int LZ4_compress_fast_extState_destSize(void* state, const char* src, char* dst, int *srcSizePtr, int dstCapacity, int acceleration)
{
LZ4_stream_t_internal* const ctx = & LZ4_initStream(state, sizeof(LZ4_stream_t)) -> internal_donotuse;
assert(ctx != NULL);
if (acceleration < 1) acceleration = LZ4_ACCELERATION_DEFAULT;
if (acceleration > LZ4_ACCELERATION_MAX) acceleration = LZ4_ACCELERATION_MAX;
if (dstCapacity >= LZ4_compressBound(*srcSizePtr)) {
if (*srcSizePtr < LZ4_64Klimit) {
return LZ4_compress_generic(ctx, src, dst, *srcSizePtr, srcSizePtr, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
if (maxOutputSize >= LZ4_compressBound(inputSize)) {
if (inputSize < LZ4_64Klimit) {
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, byU16, noDict, noDictIssue, acceleration);
} else {
const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, src, dst, *srcSizePtr, srcSizePtr, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, 0, notLimited, tableType, noDict, noDictIssue, acceleration);
}
} else {
if (*srcSizePtr < LZ4_64Klimit) {
return LZ4_compress_generic(ctx, src, dst, *srcSizePtr, srcSizePtr, dstCapacity, limitedOutput, byU16, noDict, noDictIssue, acceleration);
if (inputSize < LZ4_64Klimit) {
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration);
} else {
const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, src, dst, *srcSizePtr, srcSizePtr, dstCapacity, limitedOutput, tableType, noDict, noDictIssue, acceleration);
const tableType_t tableType = ((sizeof(void*)==4) && ((uptrval)source > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(ctx, source, dest, inputSize, NULL, maxOutputSize, limitedOutput, tableType, noDict, noDictIssue, acceleration);
}
}
}
@ -1480,22 +1475,30 @@ int LZ4_compress_default(const char* src, char* dst, int srcSize, int dstCapacit
/* Note!: This function leaves the stream in an unclean/broken state!
* It is not safe to subsequently use the same state with a _fastReset() or
* _continue() call without resetting it. */
static int LZ4_compress_destSize_extState (LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize)
static int LZ4_compress_destSize_extState_internal(LZ4_stream_t* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration)
{
void* const s = LZ4_initStream(state, sizeof (*state));
assert(s != NULL); (void)s;
if (targetDstSize >= LZ4_compressBound(*srcSizePtr)) { /* compression success is guaranteed */
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1);
return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, acceleration);
} else {
if (*srcSizePtr < LZ4_64Klimit) {
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, 1);
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, byU16, noDict, noDictIssue, acceleration);
} else {
tableType_t const addrMode = ((sizeof(void*)==4) && ((uptrval)src > LZ4_DISTANCE_MAX)) ? byPtr : byU32;
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, 1);
return LZ4_compress_generic(&state->internal_donotuse, src, dst, *srcSizePtr, srcSizePtr, targetDstSize, fillOutput, addrMode, noDict, noDictIssue, acceleration);
} }
}
int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration)
{
int const r = LZ4_compress_destSize_extState_internal((LZ4_stream_t*)state, src, dst, srcSizePtr, targetDstSize, acceleration);
/* clean the state on exit */
LZ4_initStream(state, sizeof (LZ4_stream_t));
return r;
}
int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize)
{
@ -1507,7 +1510,7 @@ int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targe
LZ4_stream_t* const ctx = &ctxBody;
#endif
int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize);
int result = LZ4_compress_destSize_extState_internal(ctx, src, dst, srcSizePtr, targetDstSize, 1);
#if (LZ4_HEAPMODE)
FREEMEM(ctx);

16
lib/lz4.h
View File

@ -243,14 +243,6 @@ LZ4LIB_API int LZ4_compress_fast (const char* src, char* dst, int srcSize, int d
LZ4LIB_API int LZ4_sizeofState(void);
LZ4LIB_API int LZ4_compress_fast_extState (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_compress_fast_extState_destSize() :
* Same as LZ4_compress_fast(), but compresses as much data as possible from 'src' buffer into
* already allocated buffer 'dst', of size >= 'dstCapacity'. This function either compresses the
* entire 'src' content into 'dst' if it's large enough, or fills 'dst' buffer completely with as
* much data as possible from 'src'.
*/
LZ4LIB_API int LZ4_compress_fast_extState_destSize(void* state, const char* src, char* dst, int *srcSizePtr, int dstCapacity, int acceleration);
/*! LZ4_compress_destSize() :
* Reverse the logic : compresses as much data as possible from 'src' buffer
* into already allocated buffer 'dst', of size >= 'targetDestSize'.
@ -274,7 +266,7 @@ LZ4LIB_API int LZ4_compress_fast_extState_destSize(void* state, const char* src,
* a dstCapacity which is > decompressedSize, by at least 1 byte.
* See https://github.com/lz4/lz4/issues/859 for details
*/
LZ4LIB_API int LZ4_compress_destSize (const char* src, char* dst, int* srcSizePtr, int targetDstSize);
LZ4LIB_API int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize);
/*! LZ4_decompress_safe_partial() :
@ -571,6 +563,12 @@ LZ4_decompress_safe_partial_usingDict(const char* src, char* dst,
*/
LZ4LIB_STATIC_API int LZ4_compress_fast_extState_fastReset (void* state, const char* src, char* dst, int srcSize, int dstCapacity, int acceleration);
/*! LZ4_compress_destSize_extState() :
* Same as LZ4_compress_destSize(), but using an externally allocated state.
* Also: exposes @acceleration
*/
int LZ4_compress_destSize_extState(void* state, const char* src, char* dst, int* srcSizePtr, int targetDstSize, int acceleration);
/*! LZ4_attach_dictionary() :
* This is an experimental API that allows
* efficient use of a static dictionary many times.

17
tests/fuzzer.c
View File

@ -478,10 +478,23 @@ static int FUZ_test(U32 seed, U32 nbCycles, const U32 startCycle, const double c
/* Test compression using external state */
FUZ_DISPLAYTEST("test LZ4_compress_fast_extState()");
{ int const r = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState() failed"); }
FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState() failed");
FUZ_DISPLAYTEST("test LZ4_compress_fast_extState() with a too small destination buffer (must fail)");
{ int const r2 = LZ4_compress_fast_extState(stateLZ4, block, compressedBuffer, blockSize, r-1, 8);
FUZ_CHECKTEST(r2!=0, "LZ4_compress_fast_extState() should have failed");
}
FUZ_DISPLAYTEST("test LZ4_compress_destSize_extState() with a too small destination buffer (must succeed, by compressing less than full input)");
{ int inputSize = blockSize;
int const r3 = LZ4_compress_destSize_extState(stateLZ4, block, compressedBuffer, &inputSize, r-1, 8);
FUZ_CHECKTEST(r3==0, "LZ4_compress_destSize_extState() failed");
FUZ_CHECKTEST(inputSize>=blockSize, "LZ4_compress_destSize_extState() should consume less than full input");
}
}
/* Test compression using fast reset external state*/
FUZ_DISPLAYTEST();
FUZ_DISPLAYTEST("test LZ4_compress_fast_extState_fastReset()");
{ int const r = LZ4_compress_fast_extState_fastReset(stateLZ4, block, compressedBuffer, blockSize, (int)compressedBufferSize, 8);
FUZ_CHECKTEST(r==0, "LZ4_compress_fast_extState_fastReset() failed"); }