From e8ff7d18ebdb7af55ad73f92c5192e74bdc85ca2 Mon Sep 17 00:00:00 2001 From: Yann Collet Date: Sat, 7 Oct 2023 20:19:45 -0700 Subject: [PATCH] removed FlexArray pattern from CCtxPool within ZSTDMT_. This pattern is flagged by less forgiving variants of ubsan notably used during compilation of the Linux Kernel. There are 2 other places in the code where this pattern is used. This fixes just one of them. --- lib/compress/zstd_compress.c | 1 + lib/compress/zstdmt_compress.c | 29 ++++++++++++++++++----------- tests/fuzzer.c | 14 ++++++++------ 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c index a51c0079258..b79266fdb7c 100644 --- a/lib/compress/zstd_compress.c +++ b/lib/compress/zstd_compress.c @@ -178,6 +178,7 @@ static void ZSTD_freeCCtxContent(ZSTD_CCtx* cctx) size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) { + DEBUGLOG(3, "ZSTD_freeCCtx (address: %p)", (void*)cctx); if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c index add99d769be..236daab2627 100644 --- a/lib/compress/zstdmt_compress.c +++ b/lib/compress/zstdmt_compress.c @@ -350,15 +350,16 @@ typedef struct { int totalCCtx; int availCCtx; ZSTD_customMem cMem; - ZSTD_CCtx* cctx[1]; /* variable size */ + ZSTD_CCtx** cctxs; } ZSTDMT_CCtxPool; -/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ +/* note : all CCtx borrowed from the pool must be reverted back to the pool _before_ freeing the pool */ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) { int cid; for (cid=0; cidtotalCCtx; cid++) - ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ + ZSTD_freeCCtx(pool->cctxs[cid]); /* note : compatible with free on NULL */ + ZSTD_customFree(pool->cctxs, pool->cMem); ZSTD_pthread_mutex_destroy(&pool->poolMutex); ZSTD_customFree(pool, pool->cMem); } @@ -373,14 +374,19 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, assert(nbWorkers > 0); if (!cctxPool) return NULL; if (ZSTD_pthread_mutex_init(&cctxPool->poolMutex, NULL)) { + ZSTDMT_freeCCtxPool(cctxPool); + return NULL; + } + cctxPool->totalCCtx = nbWorkers; + cctxPool->cctxs = (ZSTD_CCtx**)ZSTD_customCalloc(nbWorkers * sizeof(ZSTD_CCtx*), cMem); + if (!cctxPool->cctxs) { ZSTD_customFree(cctxPool, cMem); return NULL; } cctxPool->cMem = cMem; - cctxPool->totalCCtx = nbWorkers; + cctxPool->cctxs[0] = ZSTD_createCCtx_advanced(cMem); + if (!cctxPool->cctxs[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } cctxPool->availCCtx = 1; /* at least one cctx for single-thread mode */ - cctxPool->cctx[0] = ZSTD_createCCtx_advanced(cMem); - if (!cctxPool->cctx[0]) { ZSTDMT_freeCCtxPool(cctxPool); return NULL; } DEBUGLOG(3, "cctxPool created, with %u workers", nbWorkers); return cctxPool; } @@ -404,14 +410,15 @@ static size_t ZSTDMT_sizeof_CCtxPool(ZSTDMT_CCtxPool* cctxPool) { unsigned const nbWorkers = cctxPool->totalCCtx; size_t const poolSize = sizeof(*cctxPool) + (nbWorkers-1) * sizeof(ZSTD_CCtx*); - unsigned u; + size_t const arraySize = cctxPool->totalCCtx * sizeof(ZSTD_CCtx*); size_t totalCCtxSize = 0; + unsigned u; for (u=0; ucctx[u]); + totalCCtxSize += ZSTD_sizeof_CCtx(cctxPool->cctxs[u]); } ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); assert(nbWorkers > 0); - return poolSize + totalCCtxSize; + return poolSize + arraySize + totalCCtxSize; } } @@ -421,7 +428,7 @@ static ZSTD_CCtx* ZSTDMT_getCCtx(ZSTDMT_CCtxPool* cctxPool) ZSTD_pthread_mutex_lock(&cctxPool->poolMutex); if (cctxPool->availCCtx) { cctxPool->availCCtx--; - { ZSTD_CCtx* const cctx = cctxPool->cctx[cctxPool->availCCtx]; + { ZSTD_CCtx* const cctx = cctxPool->cctxs[cctxPool->availCCtx]; ZSTD_pthread_mutex_unlock(&cctxPool->poolMutex); return cctx; } } @@ -435,7 +442,7 @@ static void ZSTDMT_releaseCCtx(ZSTDMT_CCtxPool* pool, ZSTD_CCtx* cctx) if (cctx==NULL) return; /* compatibility with release on NULL */ ZSTD_pthread_mutex_lock(&pool->poolMutex); if (pool->availCCtx < pool->totalCCtx) - pool->cctx[pool->availCCtx++] = cctx; + pool->cctxs[pool->availCCtx++] = cctx; else { /* pool overflow : should not happen, since totalCCtx==nbWorkers */ DEBUGLOG(4, "CCtx pool overflow : free cctx"); diff --git a/tests/fuzzer.c b/tests/fuzzer.c index c920fbb130c..d70a669047b 100644 --- a/tests/fuzzer.c +++ b/tests/fuzzer.c @@ -1119,6 +1119,9 @@ static int basicUnitTests(U32 const seed, double compressibility) size_t const srcSize1 = kWindowSize / 2; size_t const srcSize2 = kWindowSize * 10; + CHECK(cctx!=NULL); + CHECK(dctx!=NULL); + CHECK(dict!=NULL); if (CNBuffSize < dictSize) goto _output_error; RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); @@ -1140,6 +1143,7 @@ static int basicUnitTests(U32 const seed, double compressibility) cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); CHECK_Z(cSize); CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); /* Streaming decompression to catch out of bounds offsets. */ { @@ -1153,24 +1157,22 @@ static int basicUnitTests(U32 const seed, double compressibility) CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); /* Round trip once with a dictionary. */ CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); - { - ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; + { ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); cSize = out.pos; } CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); - { - ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; + + { ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); cSize = out.pos; } /* Streaming decompression to catch out of bounds offsets. */ - { - ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + { ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); CHECK_Z(dSize);