From fb305a146b1601ede8e1fd46c87b5c8d64db98c2 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 19 Sep 2023 14:05:21 +0100 Subject: [PATCH 1/4] Fix #6497: Support global flags passed in after subcommands --- .../unreleased/Fixes-20230919-140514.yaml | 6 + core/dbt/cli/main.py | 132 +++++++++--------- tests/unit/test_cli_flags.py | 19 ++- 3 files changed, 86 insertions(+), 71 deletions(-) create mode 100644 .changes/unreleased/Fixes-20230919-140514.yaml diff --git a/.changes/unreleased/Fixes-20230919-140514.yaml b/.changes/unreleased/Fixes-20230919-140514.yaml new file mode 100644 index 00000000000..7990e1fabf8 --- /dev/null +++ b/.changes/unreleased/Fixes-20230919-140514.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Support global flags passed in after subcommands +time: 2023-09-19T14:05:14.600303+01:00 +custom: + Author: aranke + Issue: "6497" diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 9b6e8459cb0..cc3a53d6e14 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -1,3 +1,4 @@ +import functools from copy import copy from dataclasses import dataclass from typing import Callable, List, Optional, Union @@ -118,6 +119,48 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: ) +# approach from https://github.com/pallets/click/issues/108#issuecomment-280489786 +def global_flags(func): + @click.pass_context + @p.cache_selected_only + @p.debug + @p.deprecated_print + @p.enable_legacy_logger + @p.fail_fast + @p.log_cache_events + @p.log_file_max_bytes + @p.log_format + @p.log_format_file + @p.log_level + @p.log_level_file + @p.log_path + @p.macro_debugging + @p.partial_parse + @p.partial_parse_file_path + @p.partial_parse_file_diff + @p.populate_cache + @p.print + @p.printer_width + @p.quiet + @p.record_timing_info + @p.send_anonymous_usage_stats + @p.single_threaded + @p.static_parser + @p.use_colors + @p.use_colors_file + @p.use_experimental_parser + @p.version + @p.version_check + @p.warn_error + @p.warn_error_options + @p.write_json + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + + return wrapper + + # dbt @click.group( context_settings={"help_option_names": ["-h", "--help"]}, @@ -125,39 +168,7 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: no_args_is_help=True, epilog="Specify one of these sub-commands and you can find more help from there.", ) -@click.pass_context -@p.cache_selected_only -@p.debug -@p.deprecated_print -@p.enable_legacy_logger -@p.fail_fast -@p.log_cache_events -@p.log_file_max_bytes -@p.log_format -@p.log_format_file -@p.log_level -@p.log_level_file -@p.log_path -@p.macro_debugging -@p.partial_parse -@p.partial_parse_file_path -@p.partial_parse_file_diff -@p.populate_cache -@p.print -@p.printer_width -@p.quiet -@p.record_timing_info -@p.send_anonymous_usage_stats -@p.single_threaded -@p.static_parser -@p.use_colors -@p.use_colors_file -@p.use_experimental_parser -@p.version -@p.version_check -@p.warn_error -@p.warn_error_options -@p.write_json +@global_flags def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com @@ -166,11 +177,9 @@ def cli(ctx, **kwargs): # dbt build @cli.command("build") -@click.pass_context @p.defer @p.deprecated_defer @p.exclude -@p.fail_fast @p.favor_state @p.deprecated_favor_state @p.full_refresh @@ -190,7 +199,6 @@ def cli(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -212,7 +220,7 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") -@click.pass_context +@global_flags @p.profile @p.profiles_dir @p.project_dir @@ -234,14 +242,14 @@ def clean(ctx, **kwargs): # dbt docs @cli.group() -@click.pass_context +@global_flags def docs(ctx, **kwargs): """Generate or serve the documentation website for your project""" # dbt docs generate @docs.command("generate") -@click.pass_context +@global_flags @p.compile_docs @p.defer @p.deprecated_defer @@ -261,7 +269,6 @@ def docs(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -283,7 +290,7 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") -@click.pass_context +@global_flags @p.browser @p.port @p.profile @@ -311,7 +318,7 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") -@click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -335,7 +342,6 @@ def docs_serve(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -358,7 +364,7 @@ def compile(ctx, **kwargs): # dbt show @cli.command("show") -@click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -382,7 +388,6 @@ def compile(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -405,7 +410,7 @@ def show(ctx, **kwargs): # dbt debug @cli.command("debug") -@click.pass_context +@global_flags @p.debug_connection @p.config_dir @p.profile @@ -413,7 +418,6 @@ def show(ctx, **kwargs): @p.project_dir @p.target @p.vars -@p.version_check @requires.postflight @requires.preflight def debug(ctx, **kwargs): @@ -431,7 +435,7 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") -@click.pass_context +@global_flags @p.profile @p.profiles_dir_exists_false @p.project_dir @@ -451,7 +455,7 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") -@click.pass_context +@global_flags # for backwards compatibility, accept 'project_name' as an optional positional argument @click.argument("project_name", required=False) @p.profile @@ -473,7 +477,7 @@ def init(ctx, **kwargs): # dbt list @cli.command("list") -@click.pass_context +@global_flags @p.exclude @p.indirect_selection @p.models @@ -518,7 +522,7 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") -@click.pass_context +@global_flags @p.profile @p.profiles_dir @p.project_dir @@ -526,7 +530,6 @@ def list(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -542,13 +545,12 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") -@click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.favor_state @p.deprecated_favor_state @p.exclude -@p.fail_fast @p.full_refresh @p.profile @p.profiles_dir @@ -562,7 +564,6 @@ def parse(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -584,7 +585,7 @@ def run(ctx, **kwargs): # dbt retry @cli.command("retry") -@click.pass_context +@global_flags @p.project_dir @p.profiles_dir @p.vars @@ -592,7 +593,6 @@ def run(ctx, **kwargs): @p.target @p.state @p.threads -@p.fail_fast @requires.postflight @requires.preflight @requires.profile @@ -614,7 +614,7 @@ def retry(ctx, **kwargs): # dbt clone @cli.command("clone") -@click.pass_context +@global_flags @p.defer_state @p.exclude @p.full_refresh @@ -629,7 +629,6 @@ def retry(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.preflight @requires.profile @requires.project @@ -651,7 +650,7 @@ def clone(ctx, **kwargs): # dbt run operation @cli.command("run-operation") -@click.pass_context +@global_flags @click.argument("macro") @p.args @p.profile @@ -682,7 +681,7 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") -@click.pass_context +@global_flags @p.exclude @p.full_refresh @p.profile @@ -698,7 +697,6 @@ def run_operation(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile @@ -719,7 +717,7 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") -@click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -758,14 +756,14 @@ def snapshot(ctx, **kwargs): # dbt source @cli.group() -@click.pass_context +@global_flags def source(ctx, **kwargs): """Manage your project's sources""" # dbt source freshness @source.command("freshness") -@click.pass_context +@global_flags @p.exclude @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? @p.profile @@ -807,11 +805,10 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") -@click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude -@p.fail_fast @p.favor_state @p.deprecated_favor_state @p.indirect_selection @@ -828,7 +825,6 @@ def freshness(ctx, **kwargs): @p.target_path @p.threads @p.vars -@p.version_check @requires.postflight @requires.preflight @requires.profile diff --git a/tests/unit/test_cli_flags.py b/tests/unit/test_cli_flags.py index d5d78a229ae..e4d347ad4d0 100644 --- a/tests/unit/test_cli_flags.py +++ b/tests/unit/test_cli_flags.py @@ -1,10 +1,10 @@ -import pytest - -import click from multiprocessing import get_context from pathlib import Path from typing import List, Optional +import click +import pytest + from dbt.cli.exceptions import DbtUsageException from dbt.cli.flags import Flags from dbt.cli.main import cli @@ -356,6 +356,19 @@ def test_duplicate_flags_raises_error(self): with pytest.raises(DbtUsageException): Flags(context) + def test_global_flag_at_child_context(self): + parent_context_a = self.make_dbt_context("parent_context_a", ["--no-use-colors"]) + child_context_a = self.make_dbt_context("child_context_a", ["run"], parent_context_a) + flags_a = Flags(child_context_a) + + parent_context_b = self.make_dbt_context("parent_context_b", ["run"]) + child_context_b = self.make_dbt_context( + "child_context_b", ["--no-use-colors"], parent_context_b + ) + flags_b = Flags(child_context_b) + + assert flags_a.USE_COLORS == flags_b.USE_COLORS + def _create_flags_from_dict(self, cmd, d): write_file("", "profiles.yml") result = Flags.from_dict(cmd, d) From 800c6b1f35962e9b97ab8889683bdaa67d52da27 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 19 Sep 2023 14:57:43 +0100 Subject: [PATCH 2/4] Don't put click.pass_context in global_flags --- core/dbt/cli/main.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index cc3a53d6e14..987b84a5c4c 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -121,7 +121,6 @@ def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: # approach from https://github.com/pallets/click/issues/108#issuecomment-280489786 def global_flags(func): - @click.pass_context @p.cache_selected_only @p.debug @p.deprecated_print @@ -169,6 +168,7 @@ def wrapper(*args, **kwargs): epilog="Specify one of these sub-commands and you can find more help from there.", ) @global_flags +@click.pass_context def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com @@ -177,6 +177,7 @@ def cli(ctx, **kwargs): # dbt build @cli.command("build") +@click.pass_context @p.defer @p.deprecated_defer @p.exclude @@ -220,6 +221,7 @@ def build(ctx, **kwargs): # dbt clean @cli.command("clean") +@click.pass_context @global_flags @p.profile @p.profiles_dir @@ -242,6 +244,7 @@ def clean(ctx, **kwargs): # dbt docs @cli.group() +@click.pass_context @global_flags def docs(ctx, **kwargs): """Generate or serve the documentation website for your project""" @@ -249,6 +252,7 @@ def docs(ctx, **kwargs): # dbt docs generate @docs.command("generate") +@click.pass_context @global_flags @p.compile_docs @p.defer @@ -291,6 +295,7 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") @global_flags +@click.pass_context @p.browser @p.port @p.profile @@ -319,6 +324,7 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") @global_flags +@click.pass_context @p.defer @p.deprecated_defer @p.exclude @@ -365,6 +371,7 @@ def compile(ctx, **kwargs): # dbt show @cli.command("show") @global_flags +@click.pass_context @p.defer @p.deprecated_defer @p.exclude @@ -411,6 +418,7 @@ def show(ctx, **kwargs): # dbt debug @cli.command("debug") @global_flags +@click.pass_context @p.debug_connection @p.config_dir @p.profile @@ -436,6 +444,7 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") @global_flags +@click.pass_context @p.profile @p.profiles_dir_exists_false @p.project_dir @@ -456,6 +465,7 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") @global_flags +@click.pass_context # for backwards compatibility, accept 'project_name' as an optional positional argument @click.argument("project_name", required=False) @p.profile @@ -478,6 +488,7 @@ def init(ctx, **kwargs): # dbt list @cli.command("list") @global_flags +@click.pass_context @p.exclude @p.indirect_selection @p.models @@ -523,6 +534,7 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") @global_flags +@click.pass_context @p.profile @p.profiles_dir @p.project_dir @@ -546,6 +558,7 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") @global_flags +@click.pass_context @p.defer @p.deprecated_defer @p.favor_state @@ -586,6 +599,7 @@ def run(ctx, **kwargs): # dbt retry @cli.command("retry") @global_flags +@click.pass_context @p.project_dir @p.profiles_dir @p.vars @@ -615,6 +629,7 @@ def retry(ctx, **kwargs): # dbt clone @cli.command("clone") @global_flags +@click.pass_context @p.defer_state @p.exclude @p.full_refresh @@ -651,6 +666,7 @@ def clone(ctx, **kwargs): # dbt run operation @cli.command("run-operation") @global_flags +@click.pass_context @click.argument("macro") @p.args @p.profile @@ -682,6 +698,7 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") @global_flags +@click.pass_context @p.exclude @p.full_refresh @p.profile @@ -718,6 +735,7 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") @global_flags +@click.pass_context @p.defer @p.deprecated_defer @p.exclude @@ -757,6 +775,7 @@ def snapshot(ctx, **kwargs): # dbt source @cli.group() @global_flags +@click.pass_context def source(ctx, **kwargs): """Manage your project's sources""" @@ -764,6 +783,7 @@ def source(ctx, **kwargs): # dbt source freshness @source.command("freshness") @global_flags +@click.pass_context @p.exclude @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? @p.profile @@ -806,6 +826,7 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") @global_flags +@click.pass_context @p.defer @p.deprecated_defer @p.exclude From 9f014bce7821aa06dd84c6142afbdddf3580a658 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 19 Sep 2023 14:59:43 +0100 Subject: [PATCH 3/4] missed a spot --- core/dbt/cli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index 987b84a5c4c..e79f769e9e7 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -177,6 +177,7 @@ def cli(ctx, **kwargs): # dbt build @cli.command("build") +@global_flags @click.pass_context @p.defer @p.deprecated_defer From efdedc3029a0caf246836ab4486537a06864bc95 Mon Sep 17 00:00:00 2001 From: Kshitij Aranke Date: Tue, 19 Sep 2023 15:35:16 +0100 Subject: [PATCH 4/4] move a few flags out of global --- core/dbt/cli/main.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/core/dbt/cli/main.py b/core/dbt/cli/main.py index e79f769e9e7..ab501e015f4 100644 --- a/core/dbt/cli/main.py +++ b/core/dbt/cli/main.py @@ -128,7 +128,6 @@ def global_flags(func): @p.fail_fast @p.log_cache_events @p.log_file_max_bytes - @p.log_format @p.log_format_file @p.log_level @p.log_level_file @@ -150,8 +149,6 @@ def global_flags(func): @p.use_experimental_parser @p.version @p.version_check - @p.warn_error - @p.warn_error_options @p.write_json @functools.wraps(func) def wrapper(*args, **kwargs): @@ -167,8 +164,11 @@ def wrapper(*args, **kwargs): no_args_is_help=True, epilog="Specify one of these sub-commands and you can find more help from there.", ) -@global_flags @click.pass_context +@global_flags +@p.warn_error +@p.warn_error_options +@p.log_format def cli(ctx, **kwargs): """An ELT tool for managing your SQL transformations and data models. For more documentation on these commands, visit: docs.getdbt.com @@ -177,8 +177,8 @@ def cli(ctx, **kwargs): # dbt build @cli.command("build") -@global_flags @click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -295,8 +295,8 @@ def docs_generate(ctx, **kwargs): # dbt docs serve @docs.command("serve") -@global_flags @click.pass_context +@global_flags @p.browser @p.port @p.profile @@ -324,8 +324,8 @@ def docs_serve(ctx, **kwargs): # dbt compile @cli.command("compile") -@global_flags @click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -371,8 +371,8 @@ def compile(ctx, **kwargs): # dbt show @cli.command("show") -@global_flags @click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -418,8 +418,8 @@ def show(ctx, **kwargs): # dbt debug @cli.command("debug") -@global_flags @click.pass_context +@global_flags @p.debug_connection @p.config_dir @p.profile @@ -444,8 +444,8 @@ def debug(ctx, **kwargs): # dbt deps @cli.command("deps") -@global_flags @click.pass_context +@global_flags @p.profile @p.profiles_dir_exists_false @p.project_dir @@ -465,8 +465,8 @@ def deps(ctx, **kwargs): # dbt init @cli.command("init") -@global_flags @click.pass_context +@global_flags # for backwards compatibility, accept 'project_name' as an optional positional argument @click.argument("project_name", required=False) @p.profile @@ -488,8 +488,8 @@ def init(ctx, **kwargs): # dbt list @cli.command("list") -@global_flags @click.pass_context +@global_flags @p.exclude @p.indirect_selection @p.models @@ -534,8 +534,8 @@ def list(ctx, **kwargs): # dbt parse @cli.command("parse") -@global_flags @click.pass_context +@global_flags @p.profile @p.profiles_dir @p.project_dir @@ -558,8 +558,8 @@ def parse(ctx, **kwargs): # dbt run @cli.command("run") -@global_flags @click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.favor_state @@ -599,8 +599,8 @@ def run(ctx, **kwargs): # dbt retry @cli.command("retry") -@global_flags @click.pass_context +@global_flags @p.project_dir @p.profiles_dir @p.vars @@ -629,8 +629,8 @@ def retry(ctx, **kwargs): # dbt clone @cli.command("clone") -@global_flags @click.pass_context +@global_flags @p.defer_state @p.exclude @p.full_refresh @@ -666,8 +666,8 @@ def clone(ctx, **kwargs): # dbt run operation @cli.command("run-operation") -@global_flags @click.pass_context +@global_flags @click.argument("macro") @p.args @p.profile @@ -698,8 +698,8 @@ def run_operation(ctx, **kwargs): # dbt seed @cli.command("seed") -@global_flags @click.pass_context +@global_flags @p.exclude @p.full_refresh @p.profile @@ -735,8 +735,8 @@ def seed(ctx, **kwargs): # dbt snapshot @cli.command("snapshot") -@global_flags @click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude @@ -775,16 +775,16 @@ def snapshot(ctx, **kwargs): # dbt source @cli.group() -@global_flags @click.pass_context +@global_flags def source(ctx, **kwargs): """Manage your project's sources""" # dbt source freshness @source.command("freshness") -@global_flags @click.pass_context +@global_flags @p.exclude @p.output_path # TODO: Is this ok to re-use? We have three different output params, how much can we consolidate? @p.profile @@ -826,8 +826,8 @@ def freshness(ctx, **kwargs): # dbt test @cli.command("test") -@global_flags @click.pass_context +@global_flags @p.defer @p.deprecated_defer @p.exclude