removed FlexArray pattern from CCtxPool

within ZSTDMT_.
This pattern is flagged by less forgiving variants of ubsan
notably used during compilation of the Linux Kernel.

There are 2 other places in the code where this pattern is used.
This fixes just one of them.
This commit is contained in:
Yann Collet 2023-10-07 20:19:45 -07:00
parent c692b8d12d
commit e8ff7d18eb
3 changed files with 27 additions and 17 deletions

View File

@ -178,6 +178,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
{ {
DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx);
if (cctx==NULL) return 0; /* support free on NULL */ if (cctx==NULL) return 0; /* support free on NULL */
RETURN_ERROR_IF(cctx->staticSize, memory_allocation, RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
"not compatible with static CCtx"); "not compatible with static CCtx");

View File

@ -350,15 +350,16 @@ typedef struct {
int totalCCtx; int totalCCtx;
int availCCtx; int availCCtx;
ZSTD_customMem cMem; ZSTD_customMem cMem;
ZSTD_CCtx* cctx[1]; /* variable size */ ZSTD_CCtx** cctxs;
} ZSTDMT_CCtxPool; } ZSTDMT_CCtxPool;
/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ /* note : all CCtx borrowed from the pool must be reverted back to the pool _before_ freeing the pool */
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
{ {
int cid; int cid;
for (cid=0; cid<pool->totalCCtx; cid++) for (cid=0; cid<pool->totalCCtx; cid++)
ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ ZSTD_freeCCtx(pool->cctxs[cid]); /* note : compatible with free on NULL */
ZSTD_customFree(pool->cctxs, pool->cMem);
ZSTD_pthread_mutex_destroy(&pool->poolMutex); ZSTD_pthread_mutex_destroy(&pool->poolMutex);
ZSTD_customFree(pool, pool->cMem); ZSTD_customFree(pool, pool->cMem);
} }
@ -373,14 +374,19 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
assert(nbWorkers > 0); assert(nbWorkers > 0);
if (!cctxPool) return NULL; if (!cctxPool) return NULL;
if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
ZSTDMT_freeCCtxPool(cctxPool);
return NULL;
}
cctxPool->totalCCtx = nbWorkers;
cctxPool->cctxs = (ZSTD_CCtx**)ZSTD_customCalloc(nbWorkers * sizeof(ZSTD_CCtx*), cMem);
if (!cctxPool->cctxs) {
ZSTD_customFree(cctxPool, cMem); ZSTD_customFree(cctxPool, cMem);
return NULL; return NULL;
} }
cctxPool->cMem = cMem; cctxPool->cMem = cMem;
cctxPool->totalCCtx = nbWorkers; cctxPool->cctxs[0] = ZSTD_createCCtx_advanced(cMem);
if (!cctxPool->cctxs[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */
cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem);
if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers);
return cctxPool; return cctxPool;
} }
@ -404,14 +410,15 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
{ unsigned const nbWorkers = cctxPool->totalCCtx; { unsigned const nbWorkers = cctxPool->totalCCtx;
size_t const poolSize = sizeof(*cctxPool) size_t const poolSize = sizeof(*cctxPool)
+ (nbWorkers-1) * sizeof(ZSTD_CCtx*); + (nbWorkers-1) * sizeof(ZSTD_CCtx*);
unsigned u; size_t const arraySize = cctxPool->totalCCtx * sizeof(ZSTD_CCtx*);
size_t totalCCtxSize = 0; size_t totalCCtxSize = 0;
unsigned u;
for (u=0; u<nbWorkers; u++) { for (u=0; u<nbWorkers; u++) {
totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]); totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctxs[u]);
} }
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
assert(nbWorkers > 0); assert(nbWorkers > 0);
return poolSize + totalCCtxSize; return poolSize + arraySize + totalCCtxSize;
} }
} }
@ -421,7 +428,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
if (cctxPool->availCCtx) { if (cctxPool->availCCtx) {
cctxPool->availCCtx--; cctxPool->availCCtx--;
{ ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; { ZSTD_CCtx* const cctx = cctxPool->cctxs[cctxPool->availCCtx];
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
return cctx; return cctx;
} } } }
@ -435,7 +442,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
if (cctx==NULL) return; /* compatibility with release on NULL */ if (cctx==NULL) return; /* compatibility with release on NULL */
ZSTD_pthread_mutex_lock(&pool->poolMutex); ZSTD_pthread_mutex_lock(&pool->poolMutex);
if (pool->availCCtx < pool->totalCCtx) if (pool->availCCtx < pool->totalCCtx)
pool->cctx[pool->availCCtx++] = cctx; pool->cctxs[pool->availCCtx++] = cctx;
else { else {
/* pool overflow : should not happen, since totalCCtx==nbWorkers */ /* pool overflow : should not happen, since totalCCtx==nbWorkers */
DEBUGLOG(4, "CCtx pool overflow : free cctx"); DEBUGLOG(4, "CCtx pool overflow : free cctx");

View File

@ -1119,6 +1119,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
size_t const srcSize1 = kWindowSize / 2; size_t const srcSize1 = kWindowSize / 2;
size_t const srcSize2 = kWindowSize * 10; size_t const srcSize2 = kWindowSize * 10;
CHECK(cctx!=NULL);
CHECK(dctx!=NULL);
CHECK(dict!=NULL);
if (CNBuffSize < dictSize) goto _output_error; if (CNBuffSize < dictSize) goto _output_error;
RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
@ -1140,6 +1143,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
CHECK_Z(cSize); CHECK_Z(cSize);
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
/* Streaming decompression to catch out of bounds offsets. */ /* Streaming decompression to catch out of bounds offsets. */
{ {
@ -1153,24 +1157,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
/* Round trip once with a dictionary. */ /* Round trip once with a dictionary. */
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
{ { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
cSize = out.pos; cSize = out.pos;
} }
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
{
ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
cSize = out.pos; cSize = out.pos;
} }
/* Streaming decompression to catch out of bounds offsets. */ /* Streaming decompression to catch out of bounds offsets. */
{ { ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
CHECK_Z(dSize); CHECK_Z(dSize);