From 17beeb5d1a978cb7775d37ee5f2b184368262ef5 Mon Sep 17 00:00:00 2001 From: daniellerozenblit <48103643+daniellerozenblit@users.noreply.github.com> Date: Thu, 12 Dec 2024 13:09:29 -0500 Subject: [PATCH] Change CLI to employ multithreading by default (#4211) * Change CLI to employ multithreading by default * Document changes to benchmarking, print number of threads for display level >= 4, and add lower bound of 1 for the default number of threads --- programs/fileio_common.h | 11 ++++---- programs/zstd.1.md | 5 +++- programs/zstdcli.c | 56 +++++++++++++++++++++------------------- 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/programs/fileio_common.h b/programs/fileio_common.h index 55491b8e328..7a014ee4bb1 100644 --- a/programs/fileio_common.h +++ b/programs/fileio_common.h @@ -28,11 +28,14 @@ extern "C" { #define GB *(1U<<30) #undef MAX #define MAX(a,b) ((a)>(b) ? (a) : (b)) +#undef MIN /* in case it would be already defined */ +#define MIN(a,b) ((a) < (b) ? (a) : (b)) extern FIO_display_prefs_t g_display_prefs; -#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) -#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__) +#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__) +#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__) +#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__) #define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } } extern UTIL_time_t g_displayClock; @@ -56,10 +59,6 @@ extern UTIL_time_t g_displayClock; #define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }} #define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } } -#undef MIN /* in case it would be already defined */ -#define MIN(a,b) ((a) < (b) ? (a) : (b)) - - #define EXM_THROW(error, ...) \ { \ DISPLAYLEVEL(1, "zstd: "); \ diff --git a/programs/zstd.1.md b/programs/zstd.1.md index fabf8d6fcee..b4e848640fd 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -343,7 +343,7 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a `ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression. If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message. -`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200. +`ZSTD_NBTHREADS` has a default value of `max(1, min(4, nbCores/4))`, and is capped at ZSTDMT_NBWORKERS_MAX==200. `zstd` must be compiled with multithread support for this variable to have any effect. They can both be overridden by corresponding command line arguments: @@ -664,9 +664,12 @@ BENCHMARK The `zstd` CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer's performance. `zstd -b [FILE(s)]` will benchmark `zstd` for both compression and decompression using default compression level. Note that results are very dependent on the content being compressed. + It's possible to pass multiple files to the benchmark, and even a directory with `-r DIRECTORY`. When no `FILE` is provided, the benchmark will use a procedurally generated `lorem ipsum` text. +Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default in order to match the behavior of the normal CLI I/O. + * `-b#`: benchmark file(s) using compression level # * `-e#`: diff --git a/programs/zstdcli.c b/programs/zstdcli.c index 90e090b5883..024552d52b8 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -8,22 +8,6 @@ * You may select, at your option, one of the above-listed licenses. */ - -/*-************************************ -* Tuning parameters -**************************************/ -#ifndef ZSTDCLI_CLEVEL_DEFAULT -# define ZSTDCLI_CLEVEL_DEFAULT 3 -#endif - -#ifndef ZSTDCLI_CLEVEL_MAX -# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */ -#endif - -#ifndef ZSTDCLI_NBTHREADS_DEFAULT -# define ZSTDCLI_NBTHREADS_DEFAULT 1 -#endif - /*-************************************ * Dependencies **************************************/ @@ -46,6 +30,23 @@ #endif #include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */ #include "fileio_asyncio.h" +#include "fileio_common.h" + +/*-************************************ +* Tuning parameters +**************************************/ +#ifndef ZSTDCLI_CLEVEL_DEFAULT +# define ZSTDCLI_CLEVEL_DEFAULT 3 +#endif + +#ifndef ZSTDCLI_CLEVEL_MAX +# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */ +#endif + +#ifndef ZSTDCLI_NBTHREADS_DEFAULT +#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4)) +#endif + /*-************************************ @@ -99,9 +100,7 @@ typedef enum { cover, fastCover, legacy } dictType; /*-************************************ * Display Macros **************************************/ -#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__) -#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__) -#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__) +#undef DISPLAYLEVEL #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } } static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */ @@ -760,7 +759,7 @@ static int init_cLevel(void) { } #ifdef ZSTD_MULTITHREAD -static unsigned init_nbThreads(void) { +static unsigned default_nbThreads(void) { const char* const env = getenv(ENV_NBTHREADS); if (env != NULL) { const char* ptr = env; @@ -855,7 +854,7 @@ int main(int argCount, const char* argv[]) ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto; ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto; FIO_compressionType_t cType = FIO_zstdCompression; - unsigned nbWorkers = 0; + int nbWorkers = -1; /* -1 means unset */ double compressibility = -1.0; /* lorem ipsum generator */ unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */ size_t blockSize = 0; @@ -896,7 +895,6 @@ int main(int argCount, const char* argv[]) #endif ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto; - /* init */ checkLibVersion(); (void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */ @@ -904,9 +902,6 @@ int main(int argCount, const char* argv[]) assert(argCount >= 1); if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); } programName = lastNameFromPath(programName); -#ifdef ZSTD_MULTITHREAD - nbWorkers = init_nbThreads(); -#endif /* preset behaviors */ if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0; @@ -1298,7 +1293,7 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(3, WELCOME_MESSAGE); #ifdef ZSTD_MULTITHREAD - if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) { + if ((operation==zom_decompress) && (nbWorkers > 1)) { DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n"); } if ((nbWorkers==0) && (!singleThread)) { @@ -1311,6 +1306,15 @@ int main(int argCount, const char* argv[]) DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers); } } + /* Resolve to default if nbWorkers is still unset */ + if (nbWorkers == -1) { + if (operation == zom_decompress) { + nbWorkers = 1; + } else { + nbWorkers = default_nbThreads(); + } + } + DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers); #else (void)singleThread; (void)nbWorkers; (void)defaultLogicalCores; #endif