Add a --fast-deps option to comfy install/reinstall that enables ne…

…w faster/unified/uv-based dependency install (#141) * added uv script * --fast-deps option now works with installing comfyui core and comfyui-manager * `comfy node install --fast-deps` is now supported * `--fast-deps` installs of core/nodes now auto-detect gpu if `torch` is already installed * added section on developing comfy-cli and ComfyUI-Manager together * clean up specification of --fast-deps option * fix python 3.9 incompatible type hint syntax * picked lint * post rebase cleanup * make `DependencyCompiler` more flexible/testable * fix routine for finding custom node dirs * remove awkward `fastInstallComfyDeps` shim function * mark `staticmethod`s with UpperCamelCase names * cleanup `DependencyCompiler.InstallBuildDeps` * started adding tests for code in `uv.py` module * test_uv now runs, dies on mock extension vs extension dependency conflict * `uv` subprocess calls now print cmd/output on error - output is currently unsatisfactory in the case of a dependency conflict - doesn't print the names of conflicting top-level packages. The uv maintainers have expressed interest in fixing this: astral-sh/uv#1854 * add parsing of `uv compile` conflict error * add `resolve_strategy="ask"` option to `DependencyCompiler.Compile` - will manually prompt user for resolution in case of extension-vs-extension dependency conflict, using `ui.prompt_select` * basic implementation of ext-vs-ext conflict resolution via manual user input now functional - needs better output * improved feedback/UX of manual dependency conflict resolution * fixup user input for conflict resolution; fixup test-uv paths * `test_compile` in `test_uv` now works/runs correctly via pytest - still needs an actual assert at the end * added `assert` to `test_compile`. Might still be a bit fragile * made `test_compile` more robust * small type hint fix * mark old py dep install funcs with `pip_` prefix * in `uv.py`, make func/methods `snake_case`, static methods `Snake_Case` * ci test fix * disable nuisance lint rule (W0707)
Comfy-Org · Aug 16, 2024 · 71cd6ef · 71cd6ef
1 parent fe102a8
commit 71cd6ef
Show file tree

Hide file tree

Showing 14 changed files with 563 additions and 22 deletions.
diff --git a/.gitignore b/.gitignore
@@ -10,6 +10,8 @@ __pycache__/
 .vscode/settings.json
 .idea/
 .vscode/
+*.code-workspace
+.history
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -45,6 +47,9 @@ share/python-wheels/
 *.manifest
 *.spec
 
+# temporary files created by tests
+tests/temp/
+
 venv/
 
 bisect_state.json
diff --git a/.pylintrc b/.pylintrc
@@ -16,6 +16,8 @@ disable=
     W0212, # protected-access
     C0301, # line-too-long
     C0103, # invalid-name
+    W1510, # subprocess-run-check
+    W0707, # raise-missing-from
 
     # TODO
     W3101, # missing timeout on request

diff --git a/DEV_README.md b/DEV_README.md
@@ -83,6 +83,26 @@ def remove(name: str):
 - Use `rich` for all console output
   - For progress reporting, use either [`rich.progress`](https://rich.readthedocs.io/en/stable/progress.html)
 
+## Develop comfy-cli and ComfyUI-Manager (cm-cli) together
+### Make changes to both
+1. Fork your own branches of `comfy-cli` and `ComfyUI-Manager`, make changes
+2. Be sure to commit any changes to `ComfyUI-Manager` to a new branch, and push to remote
+
+### Try out changes to both
+1. clone the changed branch of `comfy-cli`, then live install `comfy-cli`:
+  - `pip install -e comfy-cli`
+2. Go to a test dir and run:
+  - `comfy --here install --manager-url=<path-or-url-to-fork-of-ComfyUI-Manager>`
+3. Run:
+  - `cd ComfyUI/custom_nodes/ComfyUI-Manager/ && git checkout <changed-branch> && cd -`
+4. Further changes can be pulled into these copies of the `comfy-cli` and `ComfyUI-Manager` repos
+
+### Debug both simultaneously
+1. Follow instructions above to get working install with changes
+2. Add breakpoints directly to code: `import ipdb; ipdb.set_trace()`
+3. Execute relevant `comfy-cli` command
+
+
 ## Contact
 
 If you have any questions or need further assistance, please contact the project maintainer at [???](mailto:???@drip.art).

diff --git a/comfy_cli/cmdline.py b/comfy_cli/cmdline.py
@@ -219,6 +219,14 @@ def install(
     commit: Annotated[
         Optional[str], typer.Option(help="Specify commit hash for ComfyUI")
     ] = None,
+    fast_deps: Annotated[
+        Optional[bool],
+        typer.Option(
+            "--fast-deps",
+            show_default=False,
+            help="Use new fast dependency installer",
+        ),
+    ] = False,
 ):
     check_for_updates()
     checker = EnvChecker()
@@ -260,6 +268,7 @@ def install(
             plat=platform,
             skip_torch_or_directml=skip_torch_or_directml,
             skip_requirement=skip_requirement,
+            fast_deps=fast_deps,
         )
         print(f"ComfyUI is installed at: {comfy_path}")
         return None
@@ -331,6 +340,7 @@ def install(
         plat=platform,
         skip_torch_or_directml=skip_torch_or_directml,
         skip_requirement=skip_requirement,
+        fast_deps=fast_deps,
     )
 
     print(f"ComfyUI is installed at: {comfy_path}")

diff --git a/comfy_cli/command/custom_nodes/cm_cli_util.py b/comfy_cli/command/custom_nodes/cm_cli_util.py
@@ -9,12 +9,18 @@
 from rich import print
 
 from comfy_cli.config_manager import ConfigManager
+from comfy_cli.uv import DependencyCompiler
 from comfy_cli.workspace_manager import WorkspaceManager
 
 workspace_manager = WorkspaceManager()
 
+# set of commands that invalidate (ie require an update of) dependencies after they are run
+_dependency_cmds = {
+    'install',
+    'reinstall',
+}
 
-def execute_cm_cli(args, channel=None, mode=None) -> str | None:
+def execute_cm_cli(args, channel=None, fast_deps=False, mode=None) -> str | None:
     _config_manager = ConfigManager()
 
     workspace_path = workspace_manager.workspace_path
@@ -34,9 +40,13 @@ def execute_cm_cli(args, channel=None, mode=None) -> str | None:
         raise typer.Exit(code=1)
 
     cmd = [sys.executable, cm_cli_path] + args
+
     if channel is not None:
         cmd += ["--channel", channel]
 
+    if fast_deps:
+        cmd += ["--no-deps"]
+
     if mode is not None:
         cmd += ["--mode", mode]
 
@@ -54,6 +64,12 @@ def execute_cm_cli(args, channel=None, mode=None) -> str | None:
             cmd, env=new_env, check=True, capture_output=True, text=True
         )
         print(result.stdout)
+
+        if fast_deps and args[0] in _dependency_cmds:
+            # we're using the fast_deps behavior and just ran a command that invalidated the dependencies
+            depComp = DependencyCompiler(cwd=workspace_path)
+            depComp.install_comfy_deps()
+
         return result.stdout
     except subprocess.CalledProcessError as e:
         if e.returncode == 1:

diff --git a/comfy_cli/command/custom_nodes/command.py b/comfy_cli/command/custom_nodes/command.py
@@ -363,7 +363,7 @@ def show(
 
     validate_mode(mode)
 
-    execute_cm_cli(["show", arg], channel, mode)
+    execute_cm_cli(["show", arg], channel=channel, mode=mode)
 
 
 @app.command("simple-show", help="Show node list (simple mode)")
@@ -402,7 +402,7 @@ def simple_show(
 
     validate_mode(mode)
 
-    execute_cm_cli(["simple-show", arg], channel, mode)
+    execute_cm_cli(["simple-show", arg], channel=channel, mode=mode)
 
 
 # install, reinstall, uninstall
@@ -420,6 +420,14 @@ def install(
             autocompletion=channel_completer,
         ),
     ] = None,
+    fast_deps: Annotated[
+        Optional[bool],
+        typer.Option(
+            "--fast-deps",
+            show_default=False,
+            help="Use new fast dependency installer",
+        ),
+    ] = False,
     mode: str = typer.Option(
         None,
         help="[remote|local|cache]",
@@ -432,7 +440,7 @@ def install(
 
     validate_mode(mode)
 
-    execute_cm_cli(["install"] + nodes, channel, mode)
+    execute_cm_cli(["install"] + nodes, channel=channel, fast_deps=fast_deps, mode=mode)
 
 
 @app.command(help="Reinstall custom nodes")
@@ -449,6 +457,14 @@ def reinstall(
             autocompletion=channel_completer,
         ),
     ] = None,
+    fast_deps: Annotated[
+        Optional[bool],
+        typer.Option(
+            "--fast-deps",
+            show_default=False,
+            help="Use new fast dependency installer",
+        ),
+    ] = False,
     mode: str = typer.Option(
         None,
         help="[remote|local|cache]",
@@ -461,7 +477,7 @@ def reinstall(
 
     validate_mode(mode)
 
-    execute_cm_cli(["reinstall"] + nodes, channel, mode)
+    execute_cm_cli(["reinstall"] + nodes, channel=channel, fast_deps=fast_deps, mode=mode)
 
 
 @app.command(help="Uninstall custom nodes")
@@ -490,7 +506,7 @@ def uninstall(
 
     validate_mode(mode)
 
-    execute_cm_cli(["uninstall"] + nodes, channel, mode)
+    execute_cm_cli(["uninstall"] + nodes, channel=channel, mode=mode)
 
 
 def update_node_id_cache():
@@ -544,7 +560,7 @@ def update(
 ):
     validate_mode(mode)
 
-    execute_cm_cli(["update"] + nodes, channel, mode)
+    execute_cm_cli(["update"] + nodes, channel=channel, mode=mode)
 
     update_node_id_cache()
 
@@ -573,7 +589,7 @@ def disable(
 ):
     validate_mode(mode)
 
-    execute_cm_cli(["disable"] + nodes, channel, mode)
+    execute_cm_cli(["disable"] + nodes, channel=channel, mode=mode)
 
 
 @app.command(help="Enable custom nodes")
@@ -600,7 +616,7 @@ def enable(
 ):
     validate_mode(mode)
 
-    execute_cm_cli(["enable"] + nodes, channel, mode)
+    execute_cm_cli(["enable"] + nodes, channel=channel, mode=mode)
 
 
 @app.command(help="Fix dependencies of custom nodes")
@@ -627,7 +643,7 @@ def fix(
 ):
     validate_mode(mode)
 
-    execute_cm_cli(["fix"] + nodes, channel, mode)
+    execute_cm_cli(["fix"] + nodes, channel=channel, mode=mode)
 
 
 @app.command(
@@ -685,7 +701,7 @@ def install_deps(
     else:
         deps_file = os.path.abspath(os.path.expanduser(deps))
 
-    execute_cm_cli(["install-deps", deps_file], channel, mode)
+    execute_cm_cli(["install-deps", deps_file], channel=channel, mode=mode)
 
     if tmp_path is not None and os.path.exists(tmp_path):
         os.remove(tmp_path)

diff --git a/comfy_cli/command/install.py b/comfy_cli/command/install.py
@@ -9,6 +9,7 @@
 from comfy_cli import constants, ui, utils
 from comfy_cli.command.custom_nodes.command import update_node_id_cache
 from comfy_cli.constants import GPU_OPTION
+from comfy_cli.uv import DependencyCompiler
 from comfy_cli.workspace_manager import WorkspaceManager, check_comfy_repo
 
 workspace_manager = WorkspaceManager()
@@ -20,7 +21,7 @@ def get_os_details():
     return os_name, os_version
 
 
-def install_comfyui_dependencies(
+def pip_install_comfyui_dependencies(
     repo_dir,
     gpu: GPU_OPTION,
     plat: constants.OS,
@@ -150,7 +151,7 @@ def install_comfyui_dependencies(
 
 
 # install requirements for manager
-def install_manager_dependencies(repo_dir):
+def pip_install_manager_dependencies(repo_dir):
     os.chdir(os.path.join(repo_dir, "custom_nodes", "ComfyUI-Manager"))
     subprocess.run(
         [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], check=True
@@ -169,6 +170,7 @@ def execute(
     plat: constants.OS = None,
     skip_torch_or_directml: bool = False,
     skip_requirement: bool = False,
+    fast_deps: bool = False,
     *args,
     **kwargs,
 ):
@@ -192,7 +194,6 @@ def execute(
         if "@" in url:
             # clone specific branch
             url, branch = url.rsplit("@", 1)
-
             subprocess.run(["git", "clone", "-b", branch, url, repo_dir], check=True)
         else:
             subprocess.run(["git", "clone", url, repo_dir], check=True)
@@ -208,9 +209,10 @@ def execute(
         os.chdir(repo_dir)
         subprocess.run(["git", "checkout", commit], check=True)
 
-    install_comfyui_dependencies(
-        repo_dir, gpu, plat, cuda_version, skip_torch_or_directml, skip_requirement
-    )
+    if not fast_deps:
+        pip_install_comfyui_dependencies(
+            repo_dir, gpu, plat, cuda_version, skip_torch_or_directml, skip_requirement
+        )
 
     WorkspaceManager().set_recent_workspace(repo_dir)
     workspace_manager.setup_workspace_manager(specified_workspace=repo_dir)
@@ -224,8 +226,8 @@ def execute(
         manager_repo_dir = os.path.join(repo_dir, "custom_nodes", "ComfyUI-Manager")
 
         if os.path.exists(manager_repo_dir):
-            if restore:
-                install_manager_dependencies(repo_dir)
+            if restore and not fast_deps:
+                pip_install_manager_dependencies(repo_dir)
             else:
                 print(
                     f"Directory {manager_repo_dir} already exists. Skipping installation of ComfyUI-Manager.\nIf you want to restore dependencies, add the '--restore' option."
@@ -236,13 +238,18 @@ def execute(
             if "@" in manager_url:
                 # clone specific branch
                 manager_url, manager_branch = manager_url.rsplit("@", 1)
-
                 subprocess.run(["git", "clone", "-b", manager_branch, manager_url, manager_repo_dir], check=True)
             else:
                 subprocess.run(["git", "clone", manager_url, manager_repo_dir], check=True)
 
-            install_manager_dependencies(repo_dir)
+            if not fast_deps:
+                pip_install_manager_dependencies(repo_dir)
+
+    if fast_deps:
+        depComp = DependencyCompiler(cwd=repo_dir, gpu=gpu)
+        depComp.install_comfy_deps()
 
+    if not skip_manager:
         update_node_id_cache()
 
     os.chdir(repo_dir)

diff --git a/comfy_cli/ui.py b/comfy_cli/ui.py
@@ -36,9 +36,33 @@ def show_progress(iterable, total, description="Downloading..."):
 
 ChoiceType = Union[str, Choice, Dict[str, Any]]
 
+def prompt_autocomplete(
+    question: str,
+    choices: List[ChoiceType],
+    default: ChoiceType = "",
+    force_prompting: bool = False
+) -> Optional[ChoiceType]:
+    """
+    Asks a single select question using questionary and returns the selected response.
+
+    Args:
+        question (str): The question to display to the user.
+        choices (List[ChoiceType]): A list of choices the user can autocomplete from.
+        default (ChoiceType): Default choice.
+        force_prompting (bool): Whether to force prompting even if skip_prompting is set.
+
+    Returns:
+        Optional[ChoiceType]: The selected choice from the user, or None if skipping prompts.
+    """
+    if workspace_manager.skip_prompting and not force_prompting:
+        return None
+    return questionary.autocomplete(question, choices=choices, default=default).ask()
+
+
 def prompt_select(
     question: str,
     choices: List[ChoiceType],
+    default: ChoiceType = "",
     force_prompting: bool = False
 ) -> Optional[ChoiceType]:
     """
@@ -47,14 +71,15 @@ def prompt_select(
     Args:
         question (str): The question to display to the user.
         choices (List[ChoiceType]): A list of choices for the user to select from.
+        default (ChoiceType): Default choice.
         force_prompting (bool): Whether to force prompting even if skip_prompting is set.
 
     Returns:
         Optional[ChoiceType]: The selected choice from the user, or None if skipping prompts.
     """
     if workspace_manager.skip_prompting and not force_prompting:
         return None
-    return questionary.select(question, choices=choices).ask()
+    return questionary.select(question, choices=choices, default=default).ask()
 
 
 E = TypeVar('E', bound=Enum)