add param to control maxBytes for single dbt.log file (#8200) (#8241)

* add param to control maxBytes for single dbt.log file

* nits

* nits

* Update core/dbt/cli/params.py

Co-authored-by: Peter Webb <peter.webb@dbtlabs.com>

---------

Co-authored-by: Peter Webb <peter.webb@dbtlabs.com>
(cherry picked from commit f392add)

Co-authored-by: Chenyu Li <chenyu.li@dbtlabs.com>

.changes/unreleased/Under the Hood-20230724-150654.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: A way to control maxBytes for a single dbt.log file
+time: 2023-07-24T15:06:54.263822-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "8199"

core/dbt/cli/main.py

@@ -132,6 +132,7 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult:
 @p.enable_legacy_logger
 @p.fail_fast
 @p.log_cache_events
+@p.log_file_max_bytes
 @p.log_format
 @p.log_format_file
 @p.log_level

core/dbt/cli/params.py

@@ -171,6 +171,15 @@
     default=True,
 )
 
+log_file_max_bytes = click.option(
+    "--log-file-max-bytes",
+    envvar="DBT_LOG_FILE_MAX_BYTES",
+    help="Configure the max file size in bytes for a single dbt.log file, before rolling over. 0 means no limit.",
+    default=10 * 1024 * 1024,  # 10mb
+    type=click.INT,
+    hidden=True,
+)
+
 log_path = click.option(
     "--log-path",
     envvar="DBT_LOG_PATH",

core/dbt/events/eventmgr.py

@@ -80,6 +80,7 @@ class LoggerConfig:
     use_colors: bool = False
     output_stream: Optional[TextIO] = None
     output_file_name: Optional[str] = None
+    output_file_max_bytes: Optional[int] = 10 * 1024 * 1024  # 10 mb
     logger: Optional[Any] = None
 
 
@@ -100,7 +101,7 @@ def __init__(self, event_manager: "EventManager", config: LoggerConfig) -> None:
             file_handler = RotatingFileHandler(
                 filename=str(config.output_file_name),
                 encoding="utf8",
-                maxBytes=10 * 1024 * 1024,  # 10 mb
+                maxBytes=config.output_file_max_bytes,  # type: ignore
                 backupCount=5,
             )
             self._python_logger = self._get_python_log_for_handler(file_handler)

core/dbt/events/functions.py

@@ -64,7 +64,11 @@ def setup_event_logger(flags, callbacks: List[Callable[[EventMsg], None]] = [])
             log_level_file = EventLevel.DEBUG if flags.DEBUG else EventLevel(flags.LOG_LEVEL_FILE)
             EVENT_MANAGER.add_logger(
                 _get_logfile_config(
-                    log_file, flags.USE_COLORS_FILE, log_file_format, log_level_file
+                    log_file,
+                    flags.USE_COLORS_FILE,
+                    log_file_format,
+                    log_level_file,
+                    flags.LOG_FILE_MAX_BYTES,
                 )
             )
 
@@ -122,7 +126,11 @@ def _stdout_filter(
 
 
 def _get_logfile_config(
-    log_path: str, use_colors: bool, line_format: LineFormat, level: EventLevel
+    log_path: str,
+    use_colors: bool,
+    line_format: LineFormat,
+    level: EventLevel,
+    log_file_max_bytes: int,
 ) -> LoggerConfig:
     return LoggerConfig(
         name="file_log",
@@ -132,6 +140,7 @@ def _get_logfile_config(
         scrubber=env_scrubber,
         filter=partial(_logfile_filter, bool(get_flags().LOG_CACHE_EVENTS), line_format),
         output_file_name=log_path,
+        output_file_max_bytes=log_file_max_bytes,
     )
 
 

core/dbt/tests/fixtures/project.py

@@ -483,6 +483,7 @@ def project(
         DEBUG=False,
         LOG_CACHE_EVENTS=False,
         QUIET=False,
+        LOG_FILE_MAX_BYTES=1000000,
     )
     setup_event_logger(log_flags)
     orig_cwd = os.getcwd()

test/unit/test_graph.py

@@ -18,6 +18,7 @@
 from dbt.contracts.files import SourceFile, FileHash, FilePath
 from dbt.contracts.graph.manifest import MacroManifest, ManifestStateCheck
 from dbt.graph import NodeSelector, parse_difference
+from dbt.events.functions import setup_event_logger
 
 try:
     from queue import Empty
@@ -140,6 +141,7 @@ def get_config(self, extra_cfg=None):
 
         config = config_from_parts_or_dicts(project=cfg, profile=self.profile)
         dbt.flags.set_from_args(Namespace(), config)
+        setup_event_logger(dbt.flags.get_flags())
         object.__setattr__(dbt.flags.get_flags(), "PARTIAL_PARSE", False)
         return config
 

tests/unit/test_cli_flags.py

@@ -54,6 +54,11 @@ def test_log_path_default(self, run_context):
         assert hasattr(flags, "LOG_PATH")
         assert getattr(flags, "LOG_PATH") == Path("logs")
 
+    def test_log_file_max_size_default(self, run_context):
+        flags = Flags(run_context)
+        assert hasattr(flags, "LOG_FILE_MAX_BYTES")
+        assert getattr(flags, "LOG_FILE_MAX_BYTES") == 10 * 1024 * 1024
+
     @pytest.mark.parametrize(
         "set_stats_param,do_not_track,expected_anonymous_usage_stats",
         [

tests/unit/test_functions.py

@@ -2,7 +2,7 @@
 import pytest
 
 import dbt.flags as flags
-from dbt.events.functions import msg_to_dict, warn_or_error
+from dbt.events.functions import msg_to_dict, warn_or_error, setup_event_logger
 from dbt.events.types import InfoLevel, NoNodesForSelectionCriteria
 from dbt.exceptions import EventCompilationError
 
@@ -59,3 +59,13 @@ def __init__(self):
         assert (
             False
         ), f"We expect `msg_to_dict` to gracefully handle exceptions, but it raised {exc}"
+
+
+def test_setup_event_logger_specify_max_bytes(mocker):
+    patched_file_handler = mocker.patch("dbt.events.eventmgr.RotatingFileHandler")
+    args = Namespace(log_file_max_bytes=1234567)
+    flags.set_from_args(args, {})
+    setup_event_logger(flags.get_flags())
+    patched_file_handler.assert_called_once_with(
+        filename="logs/dbt.log", encoding="utf8", maxBytes=1234567, backupCount=5
+    )