diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 6cf42b27718022..c7df36dcce6912 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -628,6 +628,11 @@ Miscellaneous options .. versionadded:: 3.13 + * :samp:`-X gc_preset={preset}` Tune the cyclic garbage collector + according to the specified preset. See :envvar:`PYTHON_GC_PRESET`. + + .. versionadded:: 3.14 + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -976,6 +981,34 @@ conflict. .. versionadded:: 3.4 +.. envvar:: PYTHON_GC_PRESET + + Tune the cyclic garbage collector (GC) according to the specificed + high-level preset objective. Possible preset values are: + + * ``min-memory``: prioritize freeing resources quickly in exchange for + higher GC cost and lower overall throughput. + + * ``min-overhead``: prioritize throughput (lowest runtime cost) in exchange + for higher peak memory usage and potentially delayed freeing. File + descriptiors and sockets, for example, should be cleaned by context + handlers rather than relying on the GC if this preset is used. + + * ``min-latency``: prioritize keeping GC pauses low, in exchange for higher + GC cost. This preset is not yet implemented and is equivalent to the + ``balanced`` preset at this time. + + * ``balanced``: a combination of the above three presets, with tuning that + is intended to work well for most programs. + + The default preset in version 3.14 is ``min-memory``. In future Python + versions, the default may be changed to ``balanced``. If the stategy is set + to something not recognized as a valid preset, the default preset will be + used and an error will not be raised. + + .. versionadded:: 3.14 + + .. envvar:: PYTHONMALLOC Set the Python memory allocators and/or install debug hooks. diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index c2cb4e3cdd92fb..22b432a8a56d94 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -142,6 +142,7 @@ typedef struct PyConfig { unsigned long hash_seed; int faulthandler; int tracemalloc; + wchar_t *gc_preset; int perf_profiling; int import_time; int code_debug_ranges; diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index cf96f661e6cd7e..3323d21fc480a5 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -362,6 +362,7 @@ struct _gc_thread_state { extern void _PyGC_InitState(struct _gc_runtime_state *); +extern PyStatus _PyGC_InitConfig(PyInterpreterState *interp); extern Py_ssize_t _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason); extern void _PyGC_CollectNoFail(PyThreadState *tstate); diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-29-14-40-24.gh-issue-124771.EznvS8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-29-14-40-24.gh-issue-124771.EznvS8.rst new file mode 100644 index 00000000000000..ac3bd54f8554c2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-29-14-40-24.gh-issue-124771.EznvS8.rst @@ -0,0 +1,4 @@ +Add ``PYTHON_GC_PRESET`` and the corresponding ``-X`` option +``gc_preset``. This can be used indicate to the cyclic garbage collector +what kind of performance trade-offs are preferred when tuning the GC +parameters. diff --git a/Python/gc.c b/Python/gc.c index 028657eb8999c1..ab82afca902cb5 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -166,7 +166,6 @@ _PyGC_InitState(GCState *gcstate) #undef INIT_HEAD } - PyStatus _PyGC_Init(PyInterpreterState *interp) { @@ -186,6 +185,40 @@ _PyGC_Init(PyInterpreterState *interp) return _PyStatus_OK(); } +static void +gc_set_preset(PyInterpreterState *interp, const PyConfig *config) +{ + if (config->gc_preset == NULL) { + return; + } + if (wcscmp(config->gc_preset, L"min-memory") == 0) { + // This is currently the default. In upcoming versions it + // might be more aggressive than the default, which would become + // "balanced". + interp->gc.young.threshold = 700; + return; + } + if (wcscmp(config->gc_preset, L"min-overhead") == 0) { + interp->gc.young.threshold = 20000; + return; + } + if (wcscmp(config->gc_preset, L"min-latency") == 0 || + wcscmp(config->gc_preset, L"balanced") == 0) { + // these are the same for now. If we get an incremental GC merged, + // then the "min-latency" setting could tune to have lower GC pauses + // than the default and balanced strategies. + interp->gc.young.threshold = 7000; + return; + } +} + +PyStatus +_PyGC_InitConfig(PyInterpreterState *interp) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + gc_set_preset(interp, config); + return _PyStatus_OK(); +} /* _gc_prev values diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index a5bc9b9b5782b2..f8165a8ba4e073 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -838,7 +838,6 @@ _PyGC_InitState(GCState *gcstate) gcstate->young.threshold = 2000; } - PyStatus _PyGC_Init(PyInterpreterState *interp) { @@ -857,6 +856,41 @@ _PyGC_Init(PyInterpreterState *interp) return _PyStatus_OK(); } +static void +gc_set_preset(PyInterpreterState *interp, const PyConfig *config) +{ + if (config->gc_preset == NULL) { + return; + } + if (wcscmp(config->gc_preset, L"min-memory") == 0) { + // This is currently the default. In upcoming versions it + // might be more aggressive than the default, which would become + // "balanced". + interp->gc.young.threshold = 700; + return; + } + if (wcscmp(config->gc_preset, L"min-overhead") == 0) { + interp->gc.young.threshold = 20000; + return; + } + if (wcscmp(config->gc_preset, L"min-latency") == 0 || + wcscmp(config->gc_preset, L"balanced") == 0) { + // these are the same for now. If we get an incremental GC merged, + // then the "min-latency" setting could tune to have lower GC pauses + // than the default and balanced strategies. + interp->gc.young.threshold = 7000; + return; + } +} + +PyStatus +_PyGC_InitConfig(PyInterpreterState *interp) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + gc_set_preset(interp, config); + return _PyStatus_OK(); +} + static void debug_cycle(const char *msg, PyObject *op) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 58ac5e7d7eaeff..adcb1965e97c8f 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -165,6 +165,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS), SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS), SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), + SPEC(gc_preset, WSTR_OPT, READ_ONLY, NO_SYS), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site @@ -318,6 +319,7 @@ The following implementation-specific options are available:\n\ the interactive interpreter; only works on debug builds\n\ -X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ +-X gc_preset=STRAT: set the GC tuning preset; also PYTHON_GC_PRESET\n\ -X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ -X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ also PYTHONWARNDEFAULTENCODING\ @@ -342,6 +344,7 @@ static const char usage_envvars[] = " on Python memory allocators. Use PYTHONMALLOC=debug to\n" " install debug hooks.\n" "PYTHONMALLOCSTATS: print memory allocator statistics\n" +"PYTHON_GC_PRESET: set garbage collector (GC) tuning preset.\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" " coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" " display of locale coercion and locale compatibility warnings\n" @@ -913,6 +916,7 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->base_exec_prefix); CLEAR(config->platlibdir); CLEAR(config->sys_path_0); + CLEAR(config->gc_preset); CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); @@ -1941,6 +1945,24 @@ config_init_tracemalloc(PyConfig *config) return _PyStatus_OK(); } +static PyStatus +config_init_gc_preset(PyConfig *config) +{ + if (config->gc_preset != NULL) { + return _PyStatus_OK(); + } + PyStatus status = CONFIG_GET_ENV_DUP(config, &config->gc_preset, + L"PYTHON_GC_PRESET", "PYTHON_GC_PRESET"); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + const wchar_t *value = config_get_xoption_value(config, L"gc_preset"); + if (value) { + config->gc_preset = _PyMem_RawWcsdup(value); + } + return _PyStatus_OK(); +} + static PyStatus config_init_int_max_str_digits(PyConfig *config) { @@ -2410,6 +2432,11 @@ config_read(PyConfig *config, int compute_path_config) } #endif + status = config_init_gc_preset(config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = config_read_complex_options(config); if (_PyStatus_EXCEPTION(status)) { return status; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8aebbe5c405ffe..5948b35997c444 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -902,6 +902,11 @@ pycore_interp_init(PyThreadState *tstate) const PyConfig *config = _PyInterpreterState_GetConfig(interp); + status = _PyGC_InitConfig(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _PyImport_InitCore(tstate, sysmod, config->_install_importlib); if (_PyStatus_EXCEPTION(status)) { goto done;