mirror of
https://github.com/facebook/zstd.git
synced 2024-12-03 05:36:44 +08:00
removed FlexArray pattern from CCtxPool
within ZSTDMT_. This pattern is flagged by less forgiving variants of ubsan notably used during compilation of the Linux Kernel. There are 2 other places in the code where this pattern is used. This fixes just one of them.
This commit is contained in:
parent
c692b8d12d
commit
e8ff7d18eb
@ -178,6 +178,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx)
|
|||||||
|
|
||||||
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
|
||||||
{
|
{
|
||||||
|
DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx);
|
||||||
if (cctx==NULL) return 0; /* support free on NULL */
|
if (cctx==NULL) return 0; /* support free on NULL */
|
||||||
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
|
RETURN_ERROR_IF(cctx->staticSize, memory_allocation,
|
||||||
"not compatible with static CCtx");
|
"not compatible with static CCtx");
|
||||||
|
@ -350,15 +350,16 @@ typedef struct {
|
|||||||
int totalCCtx;
|
int totalCCtx;
|
||||||
int availCCtx;
|
int availCCtx;
|
||||||
ZSTD_customMem cMem;
|
ZSTD_customMem cMem;
|
||||||
ZSTD_CCtx* cctx[1]; /* variable size */
|
ZSTD_CCtx** cctxs;
|
||||||
} ZSTDMT_CCtxPool;
|
} ZSTDMT_CCtxPool;
|
||||||
|
|
||||||
/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
|
/* note : all CCtx borrowed from the pool must be reverted back to the pool _before_ freeing the pool */
|
||||||
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
|
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
|
||||||
{
|
{
|
||||||
int cid;
|
int cid;
|
||||||
for (cid=0; cid<pool->totalCCtx; cid++)
|
for (cid=0; cid<pool->totalCCtx; cid++)
|
||||||
ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */
|
ZSTD_freeCCtx(pool->cctxs[cid]); /* note : compatible with free on NULL */
|
||||||
|
ZSTD_customFree(pool->cctxs, pool->cMem);
|
||||||
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
|
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
|
||||||
ZSTD_customFree(pool, pool->cMem);
|
ZSTD_customFree(pool, pool->cMem);
|
||||||
}
|
}
|
||||||
@ -373,14 +374,19 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
|
|||||||
assert(nbWorkers > 0);
|
assert(nbWorkers > 0);
|
||||||
if (!cctxPool) return NULL;
|
if (!cctxPool) return NULL;
|
||||||
if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
|
if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) {
|
||||||
|
ZSTDMT_freeCCtxPool(cctxPool);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
cctxPool->totalCCtx = nbWorkers;
|
||||||
|
cctxPool->cctxs = (ZSTD_CCtx**)ZSTD_customCalloc(nbWorkers * sizeof(ZSTD_CCtx*), cMem);
|
||||||
|
if (!cctxPool->cctxs) {
|
||||||
ZSTD_customFree(cctxPool, cMem);
|
ZSTD_customFree(cctxPool, cMem);
|
||||||
return NULL;
|
return NULL;
|
||||||
}
|
}
|
||||||
cctxPool->cMem = cMem;
|
cctxPool->cMem = cMem;
|
||||||
cctxPool->totalCCtx = nbWorkers;
|
cctxPool->cctxs[0] = ZSTD_createCCtx_advanced(cMem);
|
||||||
|
if (!cctxPool->cctxs[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
|
||||||
cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */
|
cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */
|
||||||
cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem);
|
|
||||||
if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; }
|
|
||||||
DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers);
|
DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers);
|
||||||
return cctxPool;
|
return cctxPool;
|
||||||
}
|
}
|
||||||
@ -404,14 +410,15 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool)
|
|||||||
{ unsigned const nbWorkers = cctxPool->totalCCtx;
|
{ unsigned const nbWorkers = cctxPool->totalCCtx;
|
||||||
size_t const poolSize = sizeof(*cctxPool)
|
size_t const poolSize = sizeof(*cctxPool)
|
||||||
+ (nbWorkers-1) * sizeof(ZSTD_CCtx*);
|
+ (nbWorkers-1) * sizeof(ZSTD_CCtx*);
|
||||||
unsigned u;
|
size_t const arraySize = cctxPool->totalCCtx * sizeof(ZSTD_CCtx*);
|
||||||
size_t totalCCtxSize = 0;
|
size_t totalCCtxSize = 0;
|
||||||
|
unsigned u;
|
||||||
for (u=0; u<nbWorkers; u++) {
|
for (u=0; u<nbWorkers; u++) {
|
||||||
totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctx[u]);
|
totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctxs[u]);
|
||||||
}
|
}
|
||||||
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||||
assert(nbWorkers > 0);
|
assert(nbWorkers > 0);
|
||||||
return poolSize + totalCCtxSize;
|
return poolSize + arraySize + totalCCtxSize;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -421,7 +428,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool)
|
|||||||
ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
|
ZSTD_pthread_mutex_lock(&cctxPool->poolMutex);
|
||||||
if (cctxPool->availCCtx) {
|
if (cctxPool->availCCtx) {
|
||||||
cctxPool->availCCtx--;
|
cctxPool->availCCtx--;
|
||||||
{ ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx];
|
{ ZSTD_CCtx* const cctx = cctxPool->cctxs[cctxPool->availCCtx];
|
||||||
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex);
|
||||||
return cctx;
|
return cctx;
|
||||||
} }
|
} }
|
||||||
@ -435,7 +442,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx)
|
|||||||
if (cctx==NULL) return; /* compatibility with release on NULL */
|
if (cctx==NULL) return; /* compatibility with release on NULL */
|
||||||
ZSTD_pthread_mutex_lock(&pool->poolMutex);
|
ZSTD_pthread_mutex_lock(&pool->poolMutex);
|
||||||
if (pool->availCCtx < pool->totalCCtx)
|
if (pool->availCCtx < pool->totalCCtx)
|
||||||
pool->cctx[pool->availCCtx++] = cctx;
|
pool->cctxs[pool->availCCtx++] = cctx;
|
||||||
else {
|
else {
|
||||||
/* pool overflow : should not happen, since totalCCtx==nbWorkers */
|
/* pool overflow : should not happen, since totalCCtx==nbWorkers */
|
||||||
DEBUGLOG(4, "CCtx pool overflow : free cctx");
|
DEBUGLOG(4, "CCtx pool overflow : free cctx");
|
||||||
|
@ -1119,6 +1119,9 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||||||
size_t const srcSize1 = kWindowSize / 2;
|
size_t const srcSize1 = kWindowSize / 2;
|
||||||
size_t const srcSize2 = kWindowSize * 10;
|
size_t const srcSize2 = kWindowSize * 10;
|
||||||
|
|
||||||
|
CHECK(cctx!=NULL);
|
||||||
|
CHECK(dctx!=NULL);
|
||||||
|
CHECK(dict!=NULL);
|
||||||
if (CNBuffSize < dictSize) goto _output_error;
|
if (CNBuffSize < dictSize) goto _output_error;
|
||||||
|
|
||||||
RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
|
RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed);
|
||||||
@ -1140,6 +1143,7 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||||||
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
|
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1);
|
||||||
CHECK_Z(cSize);
|
CHECK_Z(cSize);
|
||||||
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
|
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
|
||||||
|
|
||||||
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
|
cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2);
|
||||||
/* Streaming decompression to catch out of bounds offsets. */
|
/* Streaming decompression to catch out of bounds offsets. */
|
||||||
{
|
{
|
||||||
@ -1153,24 +1157,22 @@ static int basicUnitTests(U32 const seed, double compressibility)
|
|||||||
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
|
CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2));
|
||||||
/* Round trip once with a dictionary. */
|
/* Round trip once with a dictionary. */
|
||||||
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
|
CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize));
|
||||||
{
|
{ ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
|
||||||
ZSTD_inBuffer in = {CNBuffer, srcSize1, 0};
|
|
||||||
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
|
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
|
||||||
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
|
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
|
||||||
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
|
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
|
||||||
cSize = out.pos;
|
cSize = out.pos;
|
||||||
}
|
}
|
||||||
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
|
CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize));
|
||||||
{
|
|
||||||
ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
|
{ ZSTD_inBuffer in = {CNBuffer, srcSize2, 0};
|
||||||
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
|
ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0};
|
||||||
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
|
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush));
|
||||||
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
|
CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));
|
||||||
cSize = out.pos;
|
cSize = out.pos;
|
||||||
}
|
}
|
||||||
/* Streaming decompression to catch out of bounds offsets. */
|
/* Streaming decompression to catch out of bounds offsets. */
|
||||||
{
|
{ ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
||||||
ZSTD_inBuffer in = {compressedBuffer, cSize, 0};
|
|
||||||
ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
|
ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0};
|
||||||
size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
|
size_t const dSize = ZSTD_decompressStream(dctx, &out, &in);
|
||||||
CHECK_Z(dSize);
|
CHECK_Z(dSize);
|
||||||
|
Loading…
Reference in New Issue
Block a user