From 7e615419f40121d313d36037f50d3d7f44753535 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 12 Jul 2024 09:49:50 -0400 Subject: [PATCH 01/30] added uv script --- .gitignore | 2 + comfy_cli/uv.py | 274 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 276 insertions(+) create mode 100644 comfy_cli/uv.py diff --git a/.gitignore b/.gitignore index fffdff1..8ab7d13 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,8 @@ __pycache__/ .vscode/settings.json .idea/ .vscode/ +*.code-workspace +.history # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py new file mode 100644 index 0000000..f3ba32c --- /dev/null +++ b/comfy_cli/uv.py @@ -0,0 +1,274 @@ +import argparse +import os +from pathlib import Path +import subprocess +import sys +from textwrap import dedent +from typing import Any + +PathLike = os.PathLike[str] | str + +def _run(cmd: list[str], cwd: PathLike) -> subprocess.CompletedProcess[Any]: + return subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + ) + +def _check_call(cmd: list[str], cwd: PathLike): + """uses check_call to run pip, as reccomended by the pip maintainers. + see https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program""" + + subprocess.check_call(cmd, cwd=cwd) + +class DependencyCompiler: + rocmPytorchUrl = "https://download.pytorch.org/whl/rocm6.0" + nvidiaPytorchUrl = "https://download.pytorch.org/whl/cu121" + + overrideGpu = dedent(""" + # ensure usage of {gpu} version of pytorch + --extra-index-url {gpuUrl} + torch + torchsde + torchvision + """).strip() + + reqNames = [ + "requirements.txt", + "pyproject.toml", + "setup.cfg", + "setup.py", + ] + + @staticmethod + def findReqFiles(p: PathLike) -> list[Path]: + p = Path(p).absolute() + reqFiles: list[Path] = [] + for reqName in DependencyCompiler.reqNames: + reqFiles.extend(p.glob(reqName)) + return reqFiles + + @staticmethod + def compile( + cwd: PathLike, + reqFiles: list[PathLike], + override: PathLike | None = None, + out: PathLike | None = None, + index_strategy: str | None = "unsafe-best-match", + ) -> subprocess.CompletedProcess[Any]: + cmd = [ + sys.executable, + "-m", + "uv", + "pip", + "compile", + ] + + for reqFile in reqFiles: + cmd.append(str(reqFile)) + + # ensures that eg tqdm is latest version, even though an old tqdm is on the amd url + # see https://github.com/astral-sh/uv/blob/main/PIP_COMPATIBILITY.md#packages-that-exist-on-multiple-indexes and https://github.com/astral-sh/uv/issues/171 + if index_strategy is not None: + cmd.extend([ + "--index-strategy", + "unsafe-best-match", + ]) + + if override is not None: + cmd.extend([ + "--override", + str(override), + ]) + + if out is not None: + cmd.extend([ + "-o", + str(out), + ]) + + return _run(cmd, cwd) + + @staticmethod + def install( + cwd: PathLike, + reqFile: list[PathLike], + override: PathLike | None = None, + index_strategy: str | None = "unsafe-best-match", + dry: bool = False + ) -> subprocess.CompletedProcess[Any]: + cmd = [ + sys.executable, + "-m", + "uv", + "pip", + "install", + "-r", + str(reqFile), + ] + + if index_strategy is not None: + cmd.extend([ + "--index-strategy", + "unsafe-best-match", + ]) + + if override is not None: + cmd.extend([ + "--override", + str(override), + ]) + + if dry: + cmd.append("--dry-run") + + return _check_call(cmd, cwd) + + @staticmethod + def sync( + cwd: PathLike, + reqFile: list[PathLike], + extraUrl: str | None = None, + index_strategy: str | None = "unsafe-best-match", + dry: bool = False + ) -> subprocess.CompletedProcess[Any]: + cmd = [ + sys.executable, + "-m", + "uv", + "pip", + "sync", + str(reqFile), + ] + + if index_strategy is not None: + cmd.extend([ + "--index-strategy", + "unsafe-best-match", + ]) + + if extraUrl is not None: + cmd.extend([ + "--extra-index-url", + extraUrl, + ]) + + if dry: + cmd.append("--dry-run") + + return _check_call(cmd, cwd) + + def __init__( + self, + cwd: PathLike = ".", + extDirs: list[PathLike] = [], + gpu: str | None = None, + outName: str = "requirements.compiled", + ): + self.cwd = Path(cwd) + self.extDirs = [Path(extDir) for extDir in extDirs] if extDirs is not None else None + self.gpu = gpu + + self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == "nvidia" else DependencyCompiler.rocmPytorchUrl if self.gpu == "amd" else None + self.out = self.cwd / outName + self.override = self.cwd / "override.txt" + + self.coreReqFiles = DependencyCompiler.findReqFiles(self.cwd) + self.extReqFiles = [reqFile for extDir in self.extDirs for reqFile in DependencyCompiler.findReqFiles(extDir)] + + def makeOverride(self): + #clean up + self.override.unlink(missing_ok=True) + + with open(self.override, "w") as f: + if self.gpu is not None: + f.write(DependencyCompiler.overrideGpu.format(gpu=self.gpu, gpuUrl=self.gpuUrl)) + f.write("\n\n") + + coreOverride = DependencyCompiler.compile( + cwd=self.cwd, + reqFiles=self.coreReqFiles, + override=self.override + ) + + with open(self.override, "a") as f: + f.write("# ensure that core comfyui deps take precedence over any 3rd party extension deps\n") + for line in coreOverride.stdout: + f.write(line) + f.write("\n") + + def compileCorePlusExt(self): + #clean up + self.out.unlink(missing_ok=True) + + DependencyCompiler.compile( + cwd=self.cwd, + reqFiles=(self.coreReqFiles + self.extReqFiles), + override=self.override, + out=self.out, + ) + + def syncCorePlusExt(self): + # Appler.install( + # cwd=self.cwd, + # reqFile=self.out, + # override=self.override, + # dry=True, + # ) + + DependencyCompiler.sync( + cwd=self.cwd, + reqFile=self.out, + extraUrl=self.gpuUrl, + ) + + def handleOpencv(self): + """as per the opencv docs, you should only have exactly one opencv package. + headless is more suitable for comfy than the gui version, so remove gui if + headless is present. TODO: add support for contrib pkgs. see: https://github.com/opencv/opencv-python""" + + with open(self.out, "r") as f: + lines = f.readlines() + + guiFound, headlessFound = False, False + for line in lines: + if "opencv-python==" in line: + guiFound = True + elif "opencv-python-headless==" in line: + headlessFound = True + + if headlessFound and guiFound: + with open(self.out, "w") as f: + for line in lines: + if "opencv-python==" not in line: + f.write(line) + +def installComfyDeps(cwd: PathLike, gpu: str): + _check_call(["pip", "install", "uv"]) + + p = Path(cwd) + extDirs = [d for d in p.glob("custom_nodes/[!__pycache__]*") if d.is_dir()] + + appler = DependencyCompiler(cwd=cwd, extDirs=extDirs, gpu=gpu) + + appler.makeOverride() + appler.compileCorePlusExt() + appler.handleOpencv() + + appler.syncCorePlusExt() + +def parseArguments(): + # Create argument parser + parser = argparse.ArgumentParser() + + # Positional mandatory arguments + parser.add_argument("cwd", help="ComfyUI core directory to run in", type=str) + parser.add_argument("gpu", help="GPU type. amd, nvidia, or none", type=str, choices=["amd", "nvidia", "none"]) + + return parser.parse_args() + +if __name__ == "__main__": + args = parseArguments() + + installComfyDeps(cwd=args.cwd, gpu=args.gpu) From 3f0b106a4c710451115c225effdfdd2d789d5b61 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 12 Jul 2024 14:08:38 -0400 Subject: [PATCH 02/30] --fast-deps option now works with installing comfyui core and comfyui-manager --- comfy_cli/cmdline.py | 9 +++++++ comfy_cli/command/install.py | 21 +++++++++++----- comfy_cli/uv.py | 47 ++++++++++++++++-------------------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/comfy_cli/cmdline.py b/comfy_cli/cmdline.py index 32c3e39..8ed1a0f 100644 --- a/comfy_cli/cmdline.py +++ b/comfy_cli/cmdline.py @@ -219,6 +219,13 @@ def install( commit: Annotated[ Optional[str], typer.Option(help="Specify commit hash for ComfyUI") ] = None, + fast_deps: Annotated[ + Optional[bool], + typer.Option( + show_default=False, + help="Use new fast dependency installer", + ), + ] = False, ): check_for_updates() checker = EnvChecker() @@ -260,6 +267,7 @@ def install( plat=platform, skip_torch_or_directml=skip_torch_or_directml, skip_requirement=skip_requirement, + fast_deps=fast_deps, ) print(f"ComfyUI is installed at: {comfy_path}") return None @@ -331,6 +339,7 @@ def install( plat=platform, skip_torch_or_directml=skip_torch_or_directml, skip_requirement=skip_requirement, + fast_deps=fast_deps, ) print(f"ComfyUI is installed at: {comfy_path}") diff --git a/comfy_cli/command/install.py b/comfy_cli/command/install.py index 553046e..95c5c9b 100644 --- a/comfy_cli/command/install.py +++ b/comfy_cli/command/install.py @@ -9,6 +9,7 @@ from comfy_cli import constants, ui, utils from comfy_cli.command.custom_nodes.command import update_node_id_cache from comfy_cli.constants import GPU_OPTION +from comfy_cli.uv import fastInstallComfyDeps from comfy_cli.workspace_manager import WorkspaceManager, check_comfy_repo workspace_manager = WorkspaceManager() @@ -169,6 +170,7 @@ def execute( plat: constants.OS = None, skip_torch_or_directml: bool = False, skip_requirement: bool = False, + fast_deps: bool = False, *args, **kwargs, ): @@ -208,9 +210,10 @@ def execute( os.chdir(repo_dir) subprocess.run(["git", "checkout", commit], check=True) - install_comfyui_dependencies( - repo_dir, gpu, plat, cuda_version, skip_torch_or_directml, skip_requirement - ) + if not fast_deps: + install_comfyui_dependencies( + repo_dir, gpu, plat, cuda_version, skip_torch_or_directml, skip_requirement + ) WorkspaceManager().set_recent_workspace(repo_dir) workspace_manager.setup_workspace_manager(specified_workspace=repo_dir) @@ -224,7 +227,7 @@ def execute( manager_repo_dir = os.path.join(repo_dir, "custom_nodes", "ComfyUI-Manager") if os.path.exists(manager_repo_dir): - if restore: + if restore and not fast_deps: install_manager_dependencies(repo_dir) else: print( @@ -241,9 +244,15 @@ def execute( else: subprocess.run(["git", "clone", manager_url, manager_repo_dir], check=True) - install_manager_dependencies(repo_dir) + if not fast_deps: + install_manager_dependencies(repo_dir) + if not fast_deps: + update_node_id_cache() - update_node_id_cache() + if fast_deps: + fastInstallComfyDeps(cwd=repo_dir, gpu=gpu) + if not skip_manager: + update_node_id_cache() os.chdir(repo_dir) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index f3ba32c..88a3e4e 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -6,6 +6,8 @@ from textwrap import dedent from typing import Any +from comfy_cli.constants import GPU_OPTION + PathLike = os.PathLike[str] | str def _run(cmd: list[str], cwd: PathLike) -> subprocess.CompletedProcess[Any]: @@ -95,6 +97,7 @@ def install( cwd: PathLike, reqFile: list[PathLike], override: PathLike | None = None, + extraUrl: str | None = None, index_strategy: str | None = "unsafe-best-match", dry: bool = False ) -> subprocess.CompletedProcess[Any]: @@ -114,6 +117,12 @@ def install( "unsafe-best-match", ]) + if extraUrl is not None: + cmd.extend([ + "--extra-index-url", + extraUrl, + ]) + if override is not None: cmd.extend([ "--override", @@ -170,7 +179,7 @@ def __init__( self.extDirs = [Path(extDir) for extDir in extDirs] if extDirs is not None else None self.gpu = gpu - self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == "nvidia" else DependencyCompiler.rocmPytorchUrl if self.gpu == "amd" else None + self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == GPU_OPTION.NVIDIA else DependencyCompiler.rocmPytorchUrl if self.gpu == GPU_OPTION.AMD else None self.out = self.cwd / outName self.override = self.cwd / "override.txt" @@ -209,14 +218,15 @@ def compileCorePlusExt(self): out=self.out, ) - def syncCorePlusExt(self): - # Appler.install( - # cwd=self.cwd, - # reqFile=self.out, - # override=self.override, - # dry=True, - # ) + def installCorePlusExt(self): + DependencyCompiler.install( + cwd=self.cwd, + reqFile=self.out, + override=self.override, + extraUrl=self.gpuUrl, + ) + def syncCorePlusExt(self): DependencyCompiler.sync( cwd=self.cwd, reqFile=self.out, @@ -244,8 +254,8 @@ def handleOpencv(self): if "opencv-python==" not in line: f.write(line) -def installComfyDeps(cwd: PathLike, gpu: str): - _check_call(["pip", "install", "uv"]) +def fastInstallComfyDeps(cwd: PathLike, gpu: str): + _check_call(cmd=["pip", "install", "uv"], cwd=cwd) p = Path(cwd) extDirs = [d for d in p.glob("custom_nodes/[!__pycache__]*") if d.is_dir()] @@ -256,19 +266,4 @@ def installComfyDeps(cwd: PathLike, gpu: str): appler.compileCorePlusExt() appler.handleOpencv() - appler.syncCorePlusExt() - -def parseArguments(): - # Create argument parser - parser = argparse.ArgumentParser() - - # Positional mandatory arguments - parser.add_argument("cwd", help="ComfyUI core directory to run in", type=str) - parser.add_argument("gpu", help="GPU type. amd, nvidia, or none", type=str, choices=["amd", "nvidia", "none"]) - - return parser.parse_args() - -if __name__ == "__main__": - args = parseArguments() - - installComfyDeps(cwd=args.cwd, gpu=args.gpu) + appler.installCorePlusExt() From 4f08f724a2db3eea2f91dd1e79cf750de961be9d Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 19 Jul 2024 10:19:40 -0400 Subject: [PATCH 03/30] `comfy node install --fast-deps` is now supported --- comfy_cli/command/custom_nodes/cm_cli_util.py | 17 +++++++++- comfy_cli/command/custom_nodes/command.py | 34 +++++++++++++------ comfy_cli/uv.py | 2 +- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/comfy_cli/command/custom_nodes/cm_cli_util.py b/comfy_cli/command/custom_nodes/cm_cli_util.py index 1fcccda..dd39eb5 100644 --- a/comfy_cli/command/custom_nodes/cm_cli_util.py +++ b/comfy_cli/command/custom_nodes/cm_cli_util.py @@ -9,12 +9,18 @@ from rich import print from comfy_cli.config_manager import ConfigManager +from comfy_cli.uv import fastInstallComfyDeps from comfy_cli.workspace_manager import WorkspaceManager workspace_manager = WorkspaceManager() +# set of commands that invalidate (ie require an update of) dependencies after they are run +_dependency_cmds = { + 'install', + 'reinstall', +} -def execute_cm_cli(args, channel=None, mode=None) -> str | None: +def execute_cm_cli(args, channel=None, fast_deps=False, mode=None) -> str | None: _config_manager = ConfigManager() workspace_path = workspace_manager.workspace_path @@ -34,9 +40,13 @@ def execute_cm_cli(args, channel=None, mode=None) -> str | None: raise typer.Exit(code=1) cmd = [sys.executable, cm_cli_path] + args + if channel is not None: cmd += ["--channel", channel] + if fast_deps: + cmd += ["--no-deps"] + if mode is not None: cmd += ["--mode", mode] @@ -54,6 +64,11 @@ def execute_cm_cli(args, channel=None, mode=None) -> str | None: cmd, env=new_env, check=True, capture_output=True, text=True ) print(result.stdout) + + if fast_deps and args[0] in _dependency_cmds: + # we're using the fast_deps behavior and just ran a command that invalidated the dependencies + fastInstallComfyDeps(cwd=workspace_path) + return result.stdout except subprocess.CalledProcessError as e: if e.returncode == 1: diff --git a/comfy_cli/command/custom_nodes/command.py b/comfy_cli/command/custom_nodes/command.py index 218b796..235e78f 100644 --- a/comfy_cli/command/custom_nodes/command.py +++ b/comfy_cli/command/custom_nodes/command.py @@ -363,7 +363,7 @@ def show( validate_mode(mode) - execute_cm_cli(["show", arg], channel, mode) + execute_cm_cli(["show", arg], channel=channel, mode=mode) @app.command("simple-show", help="Show node list (simple mode)") @@ -402,7 +402,7 @@ def simple_show( validate_mode(mode) - execute_cm_cli(["simple-show", arg], channel, mode) + execute_cm_cli(["simple-show", arg], channel=channel, mode=mode) # install, reinstall, uninstall @@ -420,6 +420,13 @@ def install( autocompletion=channel_completer, ), ] = None, + fast_deps: Annotated[ + Optional[bool], + typer.Option( + show_default=False, + help="Use new fast dependency installer", + ), + ] = False, mode: str = typer.Option( None, help="[remote|local|cache]", @@ -432,7 +439,7 @@ def install( validate_mode(mode) - execute_cm_cli(["install"] + nodes, channel, mode) + execute_cm_cli(["install"] + nodes, channel=channel, fast_deps=fast_deps, mode=mode) @app.command(help="Reinstall custom nodes") @@ -449,6 +456,13 @@ def reinstall( autocompletion=channel_completer, ), ] = None, + fast_deps: Annotated[ + Optional[bool], + typer.Option( + show_default=False, + help="Use new fast dependency installer", + ), + ] = False, mode: str = typer.Option( None, help="[remote|local|cache]", @@ -461,7 +475,7 @@ def reinstall( validate_mode(mode) - execute_cm_cli(["reinstall"] + nodes, channel, mode) + execute_cm_cli(["reinstall"] + nodes, channel=channel, fast_deps=fast_deps, mode=mode) @app.command(help="Uninstall custom nodes") @@ -490,7 +504,7 @@ def uninstall( validate_mode(mode) - execute_cm_cli(["uninstall"] + nodes, channel, mode) + execute_cm_cli(["uninstall"] + nodes, channel=channel, mode=mode) def update_node_id_cache(): @@ -544,7 +558,7 @@ def update( ): validate_mode(mode) - execute_cm_cli(["update"] + nodes, channel, mode) + execute_cm_cli(["update"] + nodes, channel=channel, mode=mode) update_node_id_cache() @@ -573,7 +587,7 @@ def disable( ): validate_mode(mode) - execute_cm_cli(["disable"] + nodes, channel, mode) + execute_cm_cli(["disable"] + nodes, channel=channel, mode=mode) @app.command(help="Enable custom nodes") @@ -600,7 +614,7 @@ def enable( ): validate_mode(mode) - execute_cm_cli(["enable"] + nodes, channel, mode) + execute_cm_cli(["enable"] + nodes, channel=channel, mode=mode) @app.command(help="Fix dependencies of custom nodes") @@ -627,7 +641,7 @@ def fix( ): validate_mode(mode) - execute_cm_cli(["fix"] + nodes, channel, mode) + execute_cm_cli(["fix"] + nodes, channel=channel, mode=mode) @app.command( @@ -685,7 +699,7 @@ def install_deps( else: deps_file = os.path.abspath(os.path.expanduser(deps)) - execute_cm_cli(["install-deps", deps_file], channel, mode) + execute_cm_cli(["install-deps", deps_file], channel=channel, mode=mode) if tmp_path is not None and os.path.exists(tmp_path): os.remove(tmp_path) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 88a3e4e..a2f03c1 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -254,7 +254,7 @@ def handleOpencv(self): if "opencv-python==" not in line: f.write(line) -def fastInstallComfyDeps(cwd: PathLike, gpu: str): +def fastInstallComfyDeps(cwd: PathLike, gpu: str | None = None): _check_call(cmd=["pip", "install", "uv"], cwd=cwd) p = Path(cwd) From 3449db5adb4a6bf954cf93f62e0d014d2c5bdc96 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 19 Jul 2024 10:49:09 -0400 Subject: [PATCH 04/30] `--fast-deps` installs of core/nodes now auto-detect gpu if `torch` is already installed --- comfy_cli/uv.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index a2f03c1..8f106a1 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -1,4 +1,4 @@ -import argparse +from importlib import metadata import os from pathlib import Path import subprocess @@ -168,6 +168,22 @@ def sync( return _check_call(cmd, cwd) + @staticmethod + def resolveGpu(gpu: str | None): + if gpu is None: + try: + tver = metadata.version("torch") + if "+cu" in tver: + return GPU_OPTION.NVIDIA + elif "+rocm" in tver: + return GPU_OPTION.AMD + else: + return None + except metadata.PackageNotFoundError: + return None + else: + return gpu + def __init__( self, cwd: PathLike = ".", @@ -177,7 +193,7 @@ def __init__( ): self.cwd = Path(cwd) self.extDirs = [Path(extDir) for extDir in extDirs] if extDirs is not None else None - self.gpu = gpu + self.gpu = DependencyCompiler.resolveGpu(gpu) self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == GPU_OPTION.NVIDIA else DependencyCompiler.rocmPytorchUrl if self.gpu == GPU_OPTION.AMD else None self.out = self.cwd / outName From 66bcfdb48f4b177a0123c8018f80cd5c0204c0d7 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 19 Jul 2024 13:13:42 -0400 Subject: [PATCH 05/30] added section on developing comfy-cli and ComfyUI-Manager together --- DEV_README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/DEV_README.md b/DEV_README.md index fee7e17..aba94e3 100644 --- a/DEV_README.md +++ b/DEV_README.md @@ -83,6 +83,26 @@ def remove(name: str): - Use `rich` for all console output - For progress reporting, use either [`rich.progress`](https://rich.readthedocs.io/en/stable/progress.html) +## Develop comfy-cli and ComfyUI-Manager (cm-cli) together +### Make changes to both +1. Fork your own branches of `comfy-cli` and `ComfyUI-Manager`, make changes +2. Be sure to commit any changes to `ComfyUI-Manager` to a new branch, and push to remote + +### Try out changes to both +1. clone the changed branch of `comfy-cli`, then live install `comfy-cli`: + - `pip install -e comfy-cli` +2. Go to a test dir and run: + - `comfy --here install --manager-url=` +3. Run: + - `cd ComfyUI/custom_nodes/ComfyUI-Manager/ && git checkout && cd -` +4. Further changes can be pulled into these copies of the `comfy-cli` and `ComfyUI-Manager` repos + +### Debug both simultaneously +1. Follow instructions above to get working install with changes +2. Add breakpoints directly to code: `import ipdb; ipdb.set_trace()` +3. Execute relevant `comfy-cli` command + + ## Contact If you have any questions or need further assistance, please contact the project maintainer at [???](mailto:???@drip.art). From 2a57a70d2a226d2f736bf4d399a23966a97c8b4a Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 26 Jul 2024 12:33:30 -0400 Subject: [PATCH 06/30] clean up specification of --fast-deps option --- comfy_cli/cmdline.py | 1 + comfy_cli/command/custom_nodes/command.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/comfy_cli/cmdline.py b/comfy_cli/cmdline.py index 8ed1a0f..e13dc68 100644 --- a/comfy_cli/cmdline.py +++ b/comfy_cli/cmdline.py @@ -222,6 +222,7 @@ def install( fast_deps: Annotated[ Optional[bool], typer.Option( + "--fast-deps", show_default=False, help="Use new fast dependency installer", ), diff --git a/comfy_cli/command/custom_nodes/command.py b/comfy_cli/command/custom_nodes/command.py index 235e78f..823a1a2 100644 --- a/comfy_cli/command/custom_nodes/command.py +++ b/comfy_cli/command/custom_nodes/command.py @@ -423,6 +423,7 @@ def install( fast_deps: Annotated[ Optional[bool], typer.Option( + "--fast-deps", show_default=False, help="Use new fast dependency installer", ), @@ -459,6 +460,7 @@ def reinstall( fast_deps: Annotated[ Optional[bool], typer.Option( + "--fast-deps", show_default=False, help="Use new fast dependency installer", ), From 4cc42416551e8be4d1b42ddc56269f8284c777c7 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 26 Jul 2024 16:23:53 -0400 Subject: [PATCH 07/30] fix python 3.9 incompatible type hint syntax --- comfy_cli/uv.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 8f106a1..6c18016 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -4,11 +4,11 @@ import subprocess import sys from textwrap import dedent -from typing import Any +from typing import Any, Optional, Union from comfy_cli.constants import GPU_OPTION -PathLike = os.PathLike[str] | str +PathLike = Union[os.PathLike[str], str] def _run(cmd: list[str], cwd: PathLike) -> subprocess.CompletedProcess[Any]: return subprocess.run( @@ -55,9 +55,9 @@ def findReqFiles(p: PathLike) -> list[Path]: def compile( cwd: PathLike, reqFiles: list[PathLike], - override: PathLike | None = None, - out: PathLike | None = None, - index_strategy: str | None = "unsafe-best-match", + override: Optional[PathLike] = None, + out: Optional[PathLike] = None, + index_strategy: Optional[str] = "unsafe-best-match", ) -> subprocess.CompletedProcess[Any]: cmd = [ sys.executable, @@ -96,9 +96,9 @@ def compile( def install( cwd: PathLike, reqFile: list[PathLike], - override: PathLike | None = None, - extraUrl: str | None = None, - index_strategy: str | None = "unsafe-best-match", + override: Optional[PathLike] = None, + extraUrl: Optional[str] = None, + index_strategy: Optional[str] = "unsafe-best-match", dry: bool = False ) -> subprocess.CompletedProcess[Any]: cmd = [ @@ -138,8 +138,8 @@ def install( def sync( cwd: PathLike, reqFile: list[PathLike], - extraUrl: str | None = None, - index_strategy: str | None = "unsafe-best-match", + extraUrl: Optional[str] = None, + index_strategy: Optional[str] = "unsafe-best-match", dry: bool = False ) -> subprocess.CompletedProcess[Any]: cmd = [ @@ -169,7 +169,7 @@ def sync( return _check_call(cmd, cwd) @staticmethod - def resolveGpu(gpu: str | None): + def resolveGpu(gpu: Union[str, None]): if gpu is None: try: tver = metadata.version("torch") @@ -188,7 +188,7 @@ def __init__( self, cwd: PathLike = ".", extDirs: list[PathLike] = [], - gpu: str | None = None, + gpu: Union[str, None] = None, outName: str = "requirements.compiled", ): self.cwd = Path(cwd) @@ -270,7 +270,7 @@ def handleOpencv(self): if "opencv-python==" not in line: f.write(line) -def fastInstallComfyDeps(cwd: PathLike, gpu: str | None = None): +def fastInstallComfyDeps(cwd: PathLike, gpu: Optional[str] = None): _check_call(cmd=["pip", "install", "uv"], cwd=cwd) p = Path(cwd) From d0c271db2a945424b27d52db64e7778cd1c90c35 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 26 Jul 2024 19:04:14 -0400 Subject: [PATCH 08/30] picked lint --- .pylintrc | 1 + comfy_cli/uv.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 22987f1..761ec41 100644 --- a/.pylintrc +++ b/.pylintrc @@ -16,6 +16,7 @@ disable= W0212, # protected-access C0301, # line-too-long C0103, # invalid-name + W1510, # subprocess-run-check # TODO W3101, # missing timeout on request diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 6c18016..3302c5a 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -187,7 +187,7 @@ def resolveGpu(gpu: Union[str, None]): def __init__( self, cwd: PathLike = ".", - extDirs: list[PathLike] = [], + extDirs: Optional[list[PathLike]] = None, gpu: Union[str, None] = None, outName: str = "requirements.compiled", ): From 09f1883b92f1abeef2600e4dd5abfc7fd97ab5b6 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 01:06:38 -0400 Subject: [PATCH 09/30] post rebase cleanup --- comfy_cli/command/install.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/comfy_cli/command/install.py b/comfy_cli/command/install.py index 95c5c9b..cd00ff3 100644 --- a/comfy_cli/command/install.py +++ b/comfy_cli/command/install.py @@ -194,7 +194,6 @@ def execute( if "@" in url: # clone specific branch url, branch = url.rsplit("@", 1) - subprocess.run(["git", "clone", "-b", branch, url, repo_dir], check=True) else: subprocess.run(["git", "clone", url, repo_dir], check=True) @@ -239,20 +238,18 @@ def execute( if "@" in manager_url: # clone specific branch manager_url, manager_branch = manager_url.rsplit("@", 1) - subprocess.run(["git", "clone", "-b", manager_branch, manager_url, manager_repo_dir], check=True) else: subprocess.run(["git", "clone", manager_url, manager_repo_dir], check=True) if not fast_deps: install_manager_dependencies(repo_dir) - if not fast_deps: - update_node_id_cache() if fast_deps: fastInstallComfyDeps(cwd=repo_dir, gpu=gpu) - if not skip_manager: - update_node_id_cache() + + if not skip_manager: + update_node_id_cache() os.chdir(repo_dir) From 54f3d409612d179d07d5d26289193a3df9de6af3 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 02:15:49 -0400 Subject: [PATCH 10/30] make `DependencyCompiler` more flexible/testable --- comfy_cli/uv.py | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 3302c5a..33fe9cc 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -36,20 +36,20 @@ class DependencyCompiler: torchvision """).strip() - reqNames = [ + reqNames = { "requirements.txt", "pyproject.toml", "setup.cfg", "setup.py", - ] + } @staticmethod - def findReqFiles(p: PathLike) -> list[Path]: - p = Path(p).absolute() - reqFiles: list[Path] = [] - for reqName in DependencyCompiler.reqNames: - reqFiles.extend(p.glob(reqName)) - return reqFiles + def findReqFiles(*ders: PathLike) -> list[Path]: + return [file + for der in ders + for file in Path(der).absolute().iterdir() + if file.name in DependencyCompiler.reqNames + ] @staticmethod def compile( @@ -187,20 +187,28 @@ def resolveGpu(gpu: Union[str, None]): def __init__( self, cwd: PathLike = ".", - extDirs: Optional[list[PathLike]] = None, - gpu: Union[str, None] = None, + reqFilesCore: Optional[list[PathLike]] = None, + reqFilesExt: Optional[list[PathLike]] = None, + gpu: Optional[str] = None, outName: str = "requirements.compiled", ): self.cwd = Path(cwd) - self.extDirs = [Path(extDir) for extDir in extDirs] if extDirs is not None else None + self.reqFiles = [Path(reqFile) for reqFile in reqFilesExt] if reqFilesExt is not None else None self.gpu = DependencyCompiler.resolveGpu(gpu) self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == GPU_OPTION.NVIDIA else DependencyCompiler.rocmPytorchUrl if self.gpu == GPU_OPTION.AMD else None self.out = self.cwd / outName self.override = self.cwd / "override.txt" - self.coreReqFiles = DependencyCompiler.findReqFiles(self.cwd) - self.extReqFiles = [reqFile for extDir in self.extDirs for reqFile in DependencyCompiler.findReqFiles(extDir)] + self.reqFilesCore = reqFilesCore if reqFilesCore is not None else self.findCoreReqs() + self.reqFilesExt = reqFilesExt if reqFilesExt is not None else self.findExtReqs() + + def findCoreReqs(self): + return DependencyCompiler.findReqFiles(self.cwd) + + def findExtReqs(self): + extDirs = [d for d in self.cwd.glob("custom_nodes/[!__pycache__]*") if d.is_dir()] + return DependencyCompiler.findReqFiles(*extDirs) def makeOverride(self): #clean up @@ -213,7 +221,7 @@ def makeOverride(self): coreOverride = DependencyCompiler.compile( cwd=self.cwd, - reqFiles=self.coreReqFiles, + reqFiles=self.reqFilesCore, override=self.override ) @@ -229,7 +237,7 @@ def compileCorePlusExt(self): DependencyCompiler.compile( cwd=self.cwd, - reqFiles=(self.coreReqFiles + self.extReqFiles), + reqFiles=(self.reqFilesCore + self.reqFilesExt), override=self.override, out=self.out, ) @@ -273,10 +281,7 @@ def handleOpencv(self): def fastInstallComfyDeps(cwd: PathLike, gpu: Optional[str] = None): _check_call(cmd=["pip", "install", "uv"], cwd=cwd) - p = Path(cwd) - extDirs = [d for d in p.glob("custom_nodes/[!__pycache__]*") if d.is_dir()] - - appler = DependencyCompiler(cwd=cwd, extDirs=extDirs, gpu=gpu) + appler = DependencyCompiler(cwd=cwd, gpu=gpu) appler.makeOverride() appler.compileCorePlusExt() From de64035bac4af6ac246cb67f91542c0f1f035f56 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 04:41:50 -0400 Subject: [PATCH 11/30] fix routine for finding custom node dirs --- comfy_cli/uv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 33fe9cc..3992330 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -207,7 +207,7 @@ def findCoreReqs(self): return DependencyCompiler.findReqFiles(self.cwd) def findExtReqs(self): - extDirs = [d for d in self.cwd.glob("custom_nodes/[!__pycache__]*") if d.is_dir()] + extDirs = [d for d in (self.cwd / "custom_nodes").iterdir() if d.is_dir() and d.name != "__pycache__"] return DependencyCompiler.findReqFiles(*extDirs) def makeOverride(self): @@ -279,6 +279,7 @@ def handleOpencv(self): f.write(line) def fastInstallComfyDeps(cwd: PathLike, gpu: Optional[str] = None): + _check_call(cmd=["pip", "install", "-U", "pip"], cwd=cwd) _check_call(cmd=["pip", "install", "uv"], cwd=cwd) appler = DependencyCompiler(cwd=cwd, gpu=gpu) From 00e447e22a82133d29d2b85e9c3a3f3f1a4dd294 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 05:30:38 -0400 Subject: [PATCH 12/30] remove awkward `fastInstallComfyDeps` shim function --- comfy_cli/command/custom_nodes/cm_cli_util.py | 5 ++-- comfy_cli/command/install.py | 5 ++-- comfy_cli/uv.py | 26 ++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/comfy_cli/command/custom_nodes/cm_cli_util.py b/comfy_cli/command/custom_nodes/cm_cli_util.py index dd39eb5..0978213 100644 --- a/comfy_cli/command/custom_nodes/cm_cli_util.py +++ b/comfy_cli/command/custom_nodes/cm_cli_util.py @@ -9,7 +9,7 @@ from rich import print from comfy_cli.config_manager import ConfigManager -from comfy_cli.uv import fastInstallComfyDeps +from comfy_cli.uv import DependencyCompiler from comfy_cli.workspace_manager import WorkspaceManager workspace_manager = WorkspaceManager() @@ -67,7 +67,8 @@ def execute_cm_cli(args, channel=None, fast_deps=False, mode=None) -> str | None if fast_deps and args[0] in _dependency_cmds: # we're using the fast_deps behavior and just ran a command that invalidated the dependencies - fastInstallComfyDeps(cwd=workspace_path) + depComp = DependencyCompiler(cwd=workspace_path) + depComp.installComfyDeps() return result.stdout except subprocess.CalledProcessError as e: diff --git a/comfy_cli/command/install.py b/comfy_cli/command/install.py index cd00ff3..704309c 100644 --- a/comfy_cli/command/install.py +++ b/comfy_cli/command/install.py @@ -9,7 +9,7 @@ from comfy_cli import constants, ui, utils from comfy_cli.command.custom_nodes.command import update_node_id_cache from comfy_cli.constants import GPU_OPTION -from comfy_cli.uv import fastInstallComfyDeps +from comfy_cli.uv import DependencyCompiler from comfy_cli.workspace_manager import WorkspaceManager, check_comfy_repo workspace_manager = WorkspaceManager() @@ -246,7 +246,8 @@ def execute( install_manager_dependencies(repo_dir) if fast_deps: - fastInstallComfyDeps(cwd=repo_dir, gpu=gpu) + depComp = DependencyCompiler(cwd=repo_dir, gpu=gpu) + depComp.installComfyDeps() if not skip_manager: update_node_id_cache() diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 3992330..0753b63 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -1,6 +1,7 @@ from importlib import metadata import os from pathlib import Path +import shutil import subprocess import sys from textwrap import dedent @@ -18,7 +19,7 @@ def _run(cmd: list[str], cwd: PathLike) -> subprocess.CompletedProcess[Any]: text=True, ) -def _check_call(cmd: list[str], cwd: PathLike): +def _check_call(cmd: list[str], cwd: Optional[PathLike] = None): """uses check_call to run pip, as reccomended by the pip maintainers. see https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program""" @@ -51,6 +52,14 @@ def findReqFiles(*ders: PathLike) -> list[Path]: if file.name in DependencyCompiler.reqNames ] + @staticmethod + def installBuildDeps(): + """Use pip to install bare minimum requirements for uv to do its thing + """ + if shutil.which("uv") is None: + _check_call(cmd=["python", "-m", "pip", "install", "-U", "pip"]) + _check_call(cmd=["python", "-m", "pip", "install", "uv"]) + @staticmethod def compile( cwd: PathLike, @@ -278,14 +287,11 @@ def handleOpencv(self): if "opencv-python==" not in line: f.write(line) -def fastInstallComfyDeps(cwd: PathLike, gpu: Optional[str] = None): - _check_call(cmd=["pip", "install", "-U", "pip"], cwd=cwd) - _check_call(cmd=["pip", "install", "uv"], cwd=cwd) - - appler = DependencyCompiler(cwd=cwd, gpu=gpu) + def installComfyDeps(self): + DependencyCompiler.installBuildDeps() - appler.makeOverride() - appler.compileCorePlusExt() - appler.handleOpencv() + self.makeOverride() + self.compileCorePlusExt() + self.handleOpencv() - appler.installCorePlusExt() + self.installCorePlusExt() From 0773ed0b70141bf6893c8d24a4e179e05b564c03 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 05:33:06 -0400 Subject: [PATCH 13/30] mark `staticmethod`s with UpperCamelCase names --- comfy_cli/uv.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 0753b63..1778b05 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -45,7 +45,7 @@ class DependencyCompiler: } @staticmethod - def findReqFiles(*ders: PathLike) -> list[Path]: + def FindReqFiles(*ders: PathLike) -> list[Path]: return [file for der in ders for file in Path(der).absolute().iterdir() @@ -53,7 +53,7 @@ def findReqFiles(*ders: PathLike) -> list[Path]: ] @staticmethod - def installBuildDeps(): + def InstallBuildDeps(): """Use pip to install bare minimum requirements for uv to do its thing """ if shutil.which("uv") is None: @@ -61,7 +61,7 @@ def installBuildDeps(): _check_call(cmd=["python", "-m", "pip", "install", "uv"]) @staticmethod - def compile( + def Compile( cwd: PathLike, reqFiles: list[PathLike], override: Optional[PathLike] = None, @@ -102,7 +102,7 @@ def compile( return _run(cmd, cwd) @staticmethod - def install( + def Install( cwd: PathLike, reqFile: list[PathLike], override: Optional[PathLike] = None, @@ -144,7 +144,7 @@ def install( return _check_call(cmd, cwd) @staticmethod - def sync( + def Sync( cwd: PathLike, reqFile: list[PathLike], extraUrl: Optional[str] = None, @@ -178,7 +178,7 @@ def sync( return _check_call(cmd, cwd) @staticmethod - def resolveGpu(gpu: Union[str, None]): + def ResolveGpu(gpu: Union[str, None]): if gpu is None: try: tver = metadata.version("torch") @@ -203,7 +203,7 @@ def __init__( ): self.cwd = Path(cwd) self.reqFiles = [Path(reqFile) for reqFile in reqFilesExt] if reqFilesExt is not None else None - self.gpu = DependencyCompiler.resolveGpu(gpu) + self.gpu = DependencyCompiler.ResolveGpu(gpu) self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == GPU_OPTION.NVIDIA else DependencyCompiler.rocmPytorchUrl if self.gpu == GPU_OPTION.AMD else None self.out = self.cwd / outName @@ -213,11 +213,11 @@ def __init__( self.reqFilesExt = reqFilesExt if reqFilesExt is not None else self.findExtReqs() def findCoreReqs(self): - return DependencyCompiler.findReqFiles(self.cwd) + return DependencyCompiler.FindReqFiles(self.cwd) def findExtReqs(self): extDirs = [d for d in (self.cwd / "custom_nodes").iterdir() if d.is_dir() and d.name != "__pycache__"] - return DependencyCompiler.findReqFiles(*extDirs) + return DependencyCompiler.FindReqFiles(*extDirs) def makeOverride(self): #clean up @@ -228,7 +228,7 @@ def makeOverride(self): f.write(DependencyCompiler.overrideGpu.format(gpu=self.gpu, gpuUrl=self.gpuUrl)) f.write("\n\n") - coreOverride = DependencyCompiler.compile( + coreOverride = DependencyCompiler.Compile( cwd=self.cwd, reqFiles=self.reqFilesCore, override=self.override @@ -244,7 +244,7 @@ def compileCorePlusExt(self): #clean up self.out.unlink(missing_ok=True) - DependencyCompiler.compile( + DependencyCompiler.Compile( cwd=self.cwd, reqFiles=(self.reqFilesCore + self.reqFilesExt), override=self.override, @@ -252,7 +252,7 @@ def compileCorePlusExt(self): ) def installCorePlusExt(self): - DependencyCompiler.install( + DependencyCompiler.Install( cwd=self.cwd, reqFile=self.out, override=self.override, @@ -260,7 +260,7 @@ def installCorePlusExt(self): ) def syncCorePlusExt(self): - DependencyCompiler.sync( + DependencyCompiler.Sync( cwd=self.cwd, reqFile=self.out, extraUrl=self.gpuUrl, @@ -288,7 +288,7 @@ def handleOpencv(self): f.write(line) def installComfyDeps(self): - DependencyCompiler.installBuildDeps() + DependencyCompiler.InstallBuildDeps() self.makeOverride() self.compileCorePlusExt() From 5baac229fe5c81561088ef0b4d43597b2f7a623a Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 05:42:01 -0400 Subject: [PATCH 14/30] cleanup `DependencyCompiler.InstallBuildDeps` --- comfy_cli/uv.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 1778b05..5719ca6 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -57,8 +57,17 @@ def InstallBuildDeps(): """Use pip to install bare minimum requirements for uv to do its thing """ if shutil.which("uv") is None: - _check_call(cmd=["python", "-m", "pip", "install", "-U", "pip"]) - _check_call(cmd=["python", "-m", "pip", "install", "uv"]) + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--upgrade", + "pip", + "uv" + ] + + _check_call(cmd=cmd) @staticmethod def Compile( From a7866ad0056b07d212004a06ea7aa8f985e9f6bd Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 06:27:05 -0400 Subject: [PATCH 15/30] started adding tests for code in `uv.py` module --- tests/uv/mock_requirements/core_reqs.txt | 1 + tests/uv/mock_requirements/x_reqs.txt | 3 +++ tests/uv/mock_requirements/y_reqs.txt | 3 +++ tests/uv/test_uv.py | 14 ++++++++++++++ 4 files changed, 21 insertions(+) create mode 100644 tests/uv/mock_requirements/core_reqs.txt create mode 100644 tests/uv/mock_requirements/x_reqs.txt create mode 100644 tests/uv/mock_requirements/y_reqs.txt create mode 100644 tests/uv/test_uv.py diff --git a/tests/uv/mock_requirements/core_reqs.txt b/tests/uv/mock_requirements/core_reqs.txt new file mode 100644 index 0000000..d69c960 --- /dev/null +++ b/tests/uv/mock_requirements/core_reqs.txt @@ -0,0 +1 @@ +tqdm==4.66.4 diff --git a/tests/uv/mock_requirements/x_reqs.txt b/tests/uv/mock_requirements/x_reqs.txt new file mode 100644 index 0000000..ed08df4 --- /dev/null +++ b/tests/uv/mock_requirements/x_reqs.txt @@ -0,0 +1,3 @@ +numpy>=2.0.1 +sympy<=1.10.1 +tqdm==1.0 diff --git a/tests/uv/mock_requirements/y_reqs.txt b/tests/uv/mock_requirements/y_reqs.txt new file mode 100644 index 0000000..a1bd36a --- /dev/null +++ b/tests/uv/mock_requirements/y_reqs.txt @@ -0,0 +1,3 @@ +numpy<=1.5.0 +sympy>=1.13.1 +tqdm==2.0.0 diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py new file mode 100644 index 0000000..3b280bd --- /dev/null +++ b/tests/uv/test_uv.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from comfy_cli.uv import DependencyCompiler + +testsDir = Path(__file__).parent.resolve() +temp = testsDir / "temp" +temp.mkdir(exist_ok=True) +here = Path(__file__).resolve() + +depComp = DependencyCompiler( + cwd=temp, + reqFilesCore=[here / "mock_requirements/core_reqs.txt"], + reqFilesExt=[here / "mock_requirements/x_reqs.txt", here / "mock_requirements/y_reqs.txt"], +) From 643f4ae33fb31b380b848c64b2bb1815bbd3ac67 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 11:50:45 -0400 Subject: [PATCH 16/30] test_uv now runs, dies on mock extension vs extension dependency conflict --- .gitignore | 3 +++ tests/uv/test_uv.py | 21 ++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 8ab7d13..fadd0d0 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ share/python-wheels/ *.manifest *.spec +# temporary files created by tests +tests/temp + venv/ bisect_state.json diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index 3b280bd..45c23cc 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -2,13 +2,20 @@ from comfy_cli.uv import DependencyCompiler -testsDir = Path(__file__).parent.resolve() +hereDir = Path(__file__).parent.resolve() +testsDir = hereDir.parent.resolve() temp = testsDir / "temp" temp.mkdir(exist_ok=True) -here = Path(__file__).resolve() -depComp = DependencyCompiler( - cwd=temp, - reqFilesCore=[here / "mock_requirements/core_reqs.txt"], - reqFilesExt=[here / "mock_requirements/x_reqs.txt", here / "mock_requirements/y_reqs.txt"], -) +def test_compile(): + depComp = DependencyCompiler( + cwd=temp, + reqFilesCore=[hereDir/"mock_requirements/core_reqs.txt"], + reqFilesExt=[hereDir/"mock_requirements/x_reqs.txt", hereDir/"mock_requirements/y_reqs.txt"], + ) + + depComp.makeOverride() + depComp.compileCorePlusExt() + +if __name__ == "__main__": + test_compile() From 73396cee2caeb6975df7f7276d88462641abe311 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 2 Aug 2024 11:51:39 -0400 Subject: [PATCH 17/30] `uv` subprocess calls now print cmd/output on error - output is currently unsatisfactory in the case of a dependency conflict - doesn't print the names of conflicting top-level packages. The uv maintainers have expressed interest in fixing this: https://github.com/astral-sh/uv/issues/1854 --- comfy_cli/uv.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 5719ca6..e9f87b5 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -12,12 +12,22 @@ PathLike = Union[os.PathLike[str], str] def _run(cmd: list[str], cwd: PathLike) -> subprocess.CompletedProcess[Any]: - return subprocess.run( - cmd, - cwd=cwd, - capture_output=True, - text=True, - ) + try: + return subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + check=True + ) + + except subprocess.CalledProcessError as e: + print(e.__class__.__name__) + print(e) + print(f"STDOUT:\n{e.stdout}") + print(f"STDERR:\n{e.stderr}") + + raise RuntimeError def _check_call(cmd: list[str], cwd: Optional[PathLike] = None): """uses check_call to run pip, as reccomended by the pip maintainers. @@ -237,7 +247,7 @@ def makeOverride(self): f.write(DependencyCompiler.overrideGpu.format(gpu=self.gpu, gpuUrl=self.gpuUrl)) f.write("\n\n") - coreOverride = DependencyCompiler.Compile( + completed = DependencyCompiler.Compile( cwd=self.cwd, reqFiles=self.reqFilesCore, override=self.override @@ -245,7 +255,7 @@ def makeOverride(self): with open(self.override, "a") as f: f.write("# ensure that core comfyui deps take precedence over any 3rd party extension deps\n") - for line in coreOverride.stdout: + for line in completed.stdout: f.write(line) f.write("\n") From 17e2e472f387d490fcaf08d803ef97f96fa2f6fb Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 9 Aug 2024 10:14:47 -0400 Subject: [PATCH 18/30] add parsing of `uv compile` conflict error --- comfy_cli/uv.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index e9f87b5..e34ebb1 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -1,11 +1,12 @@ from importlib import metadata import os from pathlib import Path +import re import shutil import subprocess import sys from textwrap import dedent -from typing import Any, Optional, Union +from typing import Any, Optional, Union, cast from comfy_cli.constants import GPU_OPTION @@ -35,6 +36,21 @@ def _check_call(cmd: list[str], cwd: Optional[PathLike] = None): subprocess.check_call(cmd, cwd=cwd) +_reqNameRe: re.Pattern[str] = re.compile(r"require\s([\w-]+)") + +def _reqReClosure(name: str) -> re.Pattern[str]: + return re.compile(rf"({name}\S+)") + +def parseUvCompileError(err: str) -> list[str]: + if reqNameMatch := _reqNameRe.search(err): + reqName = reqNameMatch[1] + else: + raise ValueError + + reqRe = _reqReClosure(reqName) + + return reqName, cast(list[str], reqRe.findall(err)) + class DependencyCompiler: rocmPytorchUrl = "https://download.pytorch.org/whl/rocm6.0" nvidiaPytorchUrl = "https://download.pytorch.org/whl/cu121" From c5d696b97573276610188d90851c1be456ca27db Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 9 Aug 2024 10:26:11 -0400 Subject: [PATCH 19/30] add `resolve_strategy="ask"` option to `DependencyCompiler.Compile` - will manually prompt user for resolution in case of extension-vs-extension dependency conflict, using `ui.prompt_select` --- comfy_cli/uv.py | 52 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index e34ebb1..5f44dfb 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -9,26 +9,18 @@ from typing import Any, Optional, Union, cast from comfy_cli.constants import GPU_OPTION +from comfy_cli.ui import prompt_select PathLike = Union[os.PathLike[str], str] def _run(cmd: list[str], cwd: PathLike) -> subprocess.CompletedProcess[Any]: - try: - return subprocess.run( - cmd, - cwd=cwd, - capture_output=True, - text=True, - check=True - ) - - except subprocess.CalledProcessError as e: - print(e.__class__.__name__) - print(e) - print(f"STDOUT:\n{e.stdout}") - print(f"STDERR:\n{e.stderr}") - - raise RuntimeError + return subprocess.run( + cmd, + cwd=cwd, + capture_output=True, + text=True, + check=True + ) def _check_call(cmd: list[str], cwd: Optional[PathLike] = None): """uses check_call to run pip, as reccomended by the pip maintainers. @@ -102,6 +94,7 @@ def Compile( override: Optional[PathLike] = None, out: Optional[PathLike] = None, index_strategy: Optional[str] = "unsafe-best-match", + resolve_strategy: Optional[str] = None, ) -> subprocess.CompletedProcess[Any]: cmd = [ sys.executable, @@ -134,7 +127,31 @@ def Compile( str(out), ]) - return _run(cmd, cwd) + try: + return _run(cmd, cwd) + except subprocess.CalledProcessError as e: + if resolve_strategy == "ask": + name, reqs = parseUvCompileError(e.stderr) + vers = [req.split(name)[1].strip(",") for req in reqs] + + ver = prompt_select("Please manually select one of the conflicting requirements (or latest):", vers + ["latest"]) + + if ver == "latest": + req = name + else: + req = name + ver + + e.req = req + raise e + elif resolve_strategy is not None: + raise ValueError + + print(e.__class__.__name__) + print(e) + print(f"STDOUT:\n{e.stdout}") + print(f"STDERR:\n{e.stderr}") + + raise e @staticmethod def Install( @@ -284,6 +301,7 @@ def compileCorePlusExt(self): reqFiles=(self.reqFilesCore + self.reqFilesExt), override=self.override, out=self.out, + resolve_strategy="ask", ) def installCorePlusExt(self): From 2efb3a7accc9b6b380500bae0faa0878b0448682 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 9 Aug 2024 12:55:57 -0400 Subject: [PATCH 20/30] basic implementation of ext-vs-ext conflict resolution via manual user input now functional - needs better output --- comfy_cli/uv.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 5f44dfb..4d4de69 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -296,13 +296,23 @@ def compileCorePlusExt(self): #clean up self.out.unlink(missing_ok=True) - DependencyCompiler.Compile( - cwd=self.cwd, - reqFiles=(self.reqFilesCore + self.reqFilesExt), - override=self.override, - out=self.out, - resolve_strategy="ask", - ) + while True: + try: + DependencyCompiler.Compile( + cwd=self.cwd, + reqFiles=(self.reqFilesCore + self.reqFilesExt), + override=self.override, + out=self.out, + resolve_strategy="ask", + ) + + break + except subprocess.CalledProcessError as e: + if hasattr(e, "req"): + with open(self.override, "a") as f: + f.write(e.req + "\n") + else: + raise ValueError def installCorePlusExt(self): DependencyCompiler.Install( From c92fc857a353c9f92051c23bfef0feeac0865d21 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 9 Aug 2024 13:05:53 -0400 Subject: [PATCH 21/30] improved feedback/UX of manual dependency conflict resolution --- comfy_cli/uv.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 4d4de69..d61ceb5 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -34,6 +34,10 @@ def _reqReClosure(name: str) -> re.Pattern[str]: return re.compile(rf"({name}\S+)") def parseUvCompileError(err: str) -> list[str]: + """takes in stderr from a run of `uv pip compile` that failed due to requirement conflict and spits out + a tuple of (reqiurement_name, requirement_spec_in_conflict_a, requirement_spec_in_conflict_b). Will probably + fail for stderr produced from other kinds of errors + """ if reqNameMatch := _reqNameRe.search(err): reqName = reqNameMatch[1] else: @@ -130,11 +134,16 @@ def Compile( try: return _run(cmd, cwd) except subprocess.CalledProcessError as e: + print(e.__class__.__name__) + print(e) + print(f"STDOUT:\n{e.stdout}") + print(f"STDERR:\n{e.stderr}") + if resolve_strategy == "ask": name, reqs = parseUvCompileError(e.stderr) vers = [req.split(name)[1].strip(",") for req in reqs] - ver = prompt_select("Please manually select one of the conflicting requirements (or latest):", vers + ["latest"]) + ver = prompt_select("Please pick one of the conflicting version specs (or pick latest):", vers + ["latest"]) if ver == "latest": req = name @@ -142,15 +151,10 @@ def Compile( req = name + ver e.req = req - raise e elif resolve_strategy is not None: + # no other resolve_strategy options implemented yet raise ValueError - print(e.__class__.__name__) - print(e) - print(f"STDOUT:\n{e.stdout}") - print(f"STDERR:\n{e.stderr}") - raise e @staticmethod From 699505e2828faf2cf67ceece0fb5b3aba363ae06 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 9 Aug 2024 13:41:03 -0400 Subject: [PATCH 22/30] fixup user input for conflict resolution; fixup test-uv paths --- .gitignore | 2 +- comfy_cli/ui.py | 27 ++++++++++++++++++++++++++- comfy_cli/uv.py | 8 ++++++-- tests/uv/mock_requirements/y_reqs.txt | 2 +- tests/uv/test_uv.py | 6 ++++-- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index fadd0d0..61e15be 100644 --- a/.gitignore +++ b/.gitignore @@ -48,7 +48,7 @@ share/python-wheels/ *.spec # temporary files created by tests -tests/temp +tests/temp/ venv/ diff --git a/comfy_cli/ui.py b/comfy_cli/ui.py index c79fc23..77054b0 100644 --- a/comfy_cli/ui.py +++ b/comfy_cli/ui.py @@ -36,9 +36,33 @@ def show_progress(iterable, total, description="Downloading..."): ChoiceType = Union[str, Choice, Dict[str, Any]] +def prompt_autocomplete( + question: str, + choices: List[ChoiceType], + default: ChoiceType = "", + force_prompting: bool = False +) -> Optional[ChoiceType]: + """ + Asks a single select question using questionary and returns the selected response. + + Args: + question (str): The question to display to the user. + choices (List[ChoiceType]): A list of choices the user can autocomplete from. + default (ChoiceType): Default choice. + force_prompting (bool): Whether to force prompting even if skip_prompting is set. + + Returns: + Optional[ChoiceType]: The selected choice from the user, or None if skipping prompts. + """ + if workspace_manager.skip_prompting and not force_prompting: + return None + return questionary.autocomplete(question, choices=choices, default=default).ask() + + def prompt_select( question: str, choices: List[ChoiceType], + default: ChoiceType = "", force_prompting: bool = False ) -> Optional[ChoiceType]: """ @@ -47,6 +71,7 @@ def prompt_select( Args: question (str): The question to display to the user. choices (List[ChoiceType]): A list of choices for the user to select from. + default (ChoiceType): Default choice. force_prompting (bool): Whether to force prompting even if skip_prompting is set. Returns: @@ -54,7 +79,7 @@ def prompt_select( """ if workspace_manager.skip_prompting and not force_prompting: return None - return questionary.select(question, choices=choices).ask() + return questionary.select(question, choices=choices, default=default).ask() E = TypeVar('E', bound=Enum) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index d61ceb5..548e5af 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -143,7 +143,11 @@ def Compile( name, reqs = parseUvCompileError(e.stderr) vers = [req.split(name)[1].strip(",") for req in reqs] - ver = prompt_select("Please pick one of the conflicting version specs (or pick latest):", vers + ["latest"]) + ver = prompt_select( + "Please pick one of the conflicting version specs (or pick latest):", + choices=vers + ["latest"], + default=vers[0], + ) if ver == "latest": req = name @@ -316,7 +320,7 @@ def compileCorePlusExt(self): with open(self.override, "a") as f: f.write(e.req + "\n") else: - raise ValueError + raise AttributeError def installCorePlusExt(self): DependencyCompiler.Install( diff --git a/tests/uv/mock_requirements/y_reqs.txt b/tests/uv/mock_requirements/y_reqs.txt index a1bd36a..018fa39 100644 --- a/tests/uv/mock_requirements/y_reqs.txt +++ b/tests/uv/mock_requirements/y_reqs.txt @@ -1,3 +1,3 @@ numpy<=1.5.0 -sympy>=1.13.1 +sympy>=1.13.0 tqdm==2.0.0 diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index 45c23cc..d696599 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -1,11 +1,13 @@ from pathlib import Path +import shutil from comfy_cli.uv import DependencyCompiler hereDir = Path(__file__).parent.resolve() testsDir = hereDir.parent.resolve() -temp = testsDir / "temp" -temp.mkdir(exist_ok=True) +temp = testsDir / "temp" / "test_uv" +shutil.rmtree(temp, ignore_errors=True) +temp.mkdir(exist_ok=True, parents=True) def test_compile(): depComp = DependencyCompiler( From 2825212d1e5aa1c6aebc44bd26d315034d8fc691 Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 9 Aug 2024 14:38:45 -0400 Subject: [PATCH 23/30] `test_compile` in `test_uv` now works/runs correctly via pytest - still needs an actual assert at the end --- comfy_cli/uv.py | 4 ++-- tests/uv/mock_requirements/x_reqs.txt | 2 +- tests/uv/test_uv.py | 13 ++++++++++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 548e5af..44b1252 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -9,7 +9,7 @@ from typing import Any, Optional, Union, cast from comfy_cli.constants import GPU_OPTION -from comfy_cli.ui import prompt_select +from comfy_cli import ui PathLike = Union[os.PathLike[str], str] @@ -143,7 +143,7 @@ def Compile( name, reqs = parseUvCompileError(e.stderr) vers = [req.split(name)[1].strip(",") for req in reqs] - ver = prompt_select( + ver = ui.prompt_select( "Please pick one of the conflicting version specs (or pick latest):", choices=vers + ["latest"], default=vers[0], diff --git a/tests/uv/mock_requirements/x_reqs.txt b/tests/uv/mock_requirements/x_reqs.txt index ed08df4..a22c58d 100644 --- a/tests/uv/mock_requirements/x_reqs.txt +++ b/tests/uv/mock_requirements/x_reqs.txt @@ -1,3 +1,3 @@ -numpy>=2.0.1 +numpy>=2.0.0 sympy<=1.10.1 tqdm==1.0 diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index d696599..205c5af 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -1,7 +1,10 @@ +from itertools import cycle from pathlib import Path +import pytest import shutil from comfy_cli.uv import DependencyCompiler +from comfy_cli import ui hereDir = Path(__file__).parent.resolve() testsDir = hereDir.parent.resolve() @@ -9,7 +12,15 @@ shutil.rmtree(temp, ignore_errors=True) temp.mkdir(exist_ok=True, parents=True) -def test_compile(): +@pytest.fixture +def mock_prompt_select(monkeypatch): + mockChoices = [">=1.13.0", ">=2.0.0"] + def _mock_prompt_select(*args, **kwargs): + return mockChoices.pop(0) + + monkeypatch.setattr(ui, "prompt_select", _mock_prompt_select) + +def test_compile(mock_prompt_select): depComp = DependencyCompiler( cwd=temp, reqFilesCore=[hereDir/"mock_requirements/core_reqs.txt"], From 4295769a488fad59e61907c5c324a0a9e4e84e4d Mon Sep 17 00:00:00 2001 From: telamonian Date: Mon, 12 Aug 2024 22:58:59 -0400 Subject: [PATCH 24/30] added `assert` to `test_compile`. Might still be a bit fragile --- .../mock_requirements/requirements.compiled | 20 +++++++++++++++++++ tests/uv/test_uv.py | 15 ++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 tests/uv/mock_requirements/requirements.compiled diff --git a/tests/uv/mock_requirements/requirements.compiled b/tests/uv/mock_requirements/requirements.compiled new file mode 100644 index 0000000..70d906c --- /dev/null +++ b/tests/uv/mock_requirements/requirements.compiled @@ -0,0 +1,20 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile /home/tel/git/comfy-cli/tests/uv/mock_requirements/core_reqs.txt /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt --index-strategy unsafe-best-match --override /home/tel/git/comfy-cli/tests/temp/test_uv/override.txt -o /home/tel/git/comfy-cli/tests/temp/test_uv/requirements.compiled +mpmath==1.3.0 + # via sympy +numpy==2.0.1 + # via + # --override override.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt +sympy==1.13.2 + # via + # --override override.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt +tqdm==4.66.4 + # via + # --override override.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/core_reqs.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index 205c5af..99b7d32 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -1,4 +1,3 @@ -from itertools import cycle from pathlib import Path import pytest import shutil @@ -7,8 +6,11 @@ from comfy_cli import ui hereDir = Path(__file__).parent.resolve() +reqsDir = hereDir/"mock_requirements" + +# set up a temp dir to write files to testsDir = hereDir.parent.resolve() -temp = testsDir / "temp" / "test_uv" +temp = testsDir/"temp"/"test_uv" shutil.rmtree(temp, ignore_errors=True) temp.mkdir(exist_ok=True, parents=True) @@ -23,12 +25,13 @@ def _mock_prompt_select(*args, **kwargs): def test_compile(mock_prompt_select): depComp = DependencyCompiler( cwd=temp, - reqFilesCore=[hereDir/"mock_requirements/core_reqs.txt"], - reqFilesExt=[hereDir/"mock_requirements/x_reqs.txt", hereDir/"mock_requirements/y_reqs.txt"], + reqFilesCore=[reqsDir/"core_reqs.txt"], + reqFilesExt=[reqsDir/"x_reqs.txt", reqsDir/"y_reqs.txt"], ) depComp.makeOverride() depComp.compileCorePlusExt() -if __name__ == "__main__": - test_compile() + with open(reqsDir/"requirements.compiled", "r") as known, open(temp/"requirements.compiled", "r") as test: + knownLines, testLines = known.readlines(), test.readlines() + assert knownLines == testLines From 98b6238d071c2ac02874bf0b554b930656dc1d30 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 15 Aug 2024 15:29:18 -0400 Subject: [PATCH 25/30] made `test_compile` more robust --- tests/uv/mock_requirements/requirements.compiled | 8 +++++--- tests/uv/mock_requirements/y_reqs.txt | 1 + tests/uv/test_uv.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/uv/mock_requirements/requirements.compiled b/tests/uv/mock_requirements/requirements.compiled index 70d906c..2ff3d84 100644 --- a/tests/uv/mock_requirements/requirements.compiled +++ b/tests/uv/mock_requirements/requirements.compiled @@ -1,13 +1,15 @@ # This file was autogenerated by uv via the following command: # uv pip compile /home/tel/git/comfy-cli/tests/uv/mock_requirements/core_reqs.txt /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt --index-strategy unsafe-best-match --override /home/tel/git/comfy-cli/tests/temp/test_uv/override.txt -o /home/tel/git/comfy-cli/tests/temp/test_uv/requirements.compiled mpmath==1.3.0 - # via sympy -numpy==2.0.1 + # via + # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt + # sympy +numpy==2.0.0 # via # --override override.txt # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/y_reqs.txt -sympy==1.13.2 +sympy==1.13.0 # via # --override override.txt # -r /home/tel/git/comfy-cli/tests/uv/mock_requirements/x_reqs.txt diff --git a/tests/uv/mock_requirements/y_reqs.txt b/tests/uv/mock_requirements/y_reqs.txt index 018fa39..a3e0604 100644 --- a/tests/uv/mock_requirements/y_reqs.txt +++ b/tests/uv/mock_requirements/y_reqs.txt @@ -1,3 +1,4 @@ +mpmath==1.3.0 numpy<=1.5.0 sympy>=1.13.0 tqdm==2.0.0 diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index 99b7d32..f0a003e 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -16,7 +16,7 @@ @pytest.fixture def mock_prompt_select(monkeypatch): - mockChoices = [">=1.13.0", ">=2.0.0"] + mockChoices = ["==1.13.0", "==2.0.0"] def _mock_prompt_select(*args, **kwargs): return mockChoices.pop(0) From 5c9b379e201ce2411d4cdf3db4b3ad23c965bdc2 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 15 Aug 2024 18:04:34 -0400 Subject: [PATCH 26/30] small type hint fix --- comfy_cli/uv.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index 44b1252..c4729ed 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -33,9 +33,9 @@ def _check_call(cmd: list[str], cwd: Optional[PathLike] = None): def _reqReClosure(name: str) -> re.Pattern[str]: return re.compile(rf"({name}\S+)") -def parseUvCompileError(err: str) -> list[str]: +def parseUvCompileError(err: str) -> tuple[str, list[str]]: """takes in stderr from a run of `uv pip compile` that failed due to requirement conflict and spits out - a tuple of (reqiurement_name, requirement_spec_in_conflict_a, requirement_spec_in_conflict_b). Will probably + a tuple of (reqiurement_name, [requirement_spec_in_conflict_a, requirement_spec_in_conflict_b]). Will probably fail for stderr produced from other kinds of errors """ if reqNameMatch := _reqNameRe.search(err): From 5a81f655e150ba37df8e0feada1c55581f7c5488 Mon Sep 17 00:00:00 2001 From: telamonian Date: Thu, 15 Aug 2024 18:06:25 -0400 Subject: [PATCH 27/30] mark old py dep install funcs with `pip_` prefix --- comfy_cli/command/install.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/comfy_cli/command/install.py b/comfy_cli/command/install.py index 704309c..73dc7d1 100644 --- a/comfy_cli/command/install.py +++ b/comfy_cli/command/install.py @@ -21,7 +21,7 @@ def get_os_details(): return os_name, os_version -def install_comfyui_dependencies( +def pip_install_comfyui_dependencies( repo_dir, gpu: GPU_OPTION, plat: constants.OS, @@ -151,7 +151,7 @@ def install_comfyui_dependencies( # install requirements for manager -def install_manager_dependencies(repo_dir): +def pip_install_manager_dependencies(repo_dir): os.chdir(os.path.join(repo_dir, "custom_nodes", "ComfyUI-Manager")) subprocess.run( [sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], check=True @@ -210,7 +210,7 @@ def execute( subprocess.run(["git", "checkout", commit], check=True) if not fast_deps: - install_comfyui_dependencies( + pip_install_comfyui_dependencies( repo_dir, gpu, plat, cuda_version, skip_torch_or_directml, skip_requirement ) @@ -227,7 +227,7 @@ def execute( if os.path.exists(manager_repo_dir): if restore and not fast_deps: - install_manager_dependencies(repo_dir) + pip_install_manager_dependencies(repo_dir) else: print( f"Directory {manager_repo_dir} already exists. Skipping installation of ComfyUI-Manager.\nIf you want to restore dependencies, add the '--restore' option." @@ -243,7 +243,7 @@ def execute( subprocess.run(["git", "clone", manager_url, manager_repo_dir], check=True) if not fast_deps: - install_manager_dependencies(repo_dir) + pip_install_manager_dependencies(repo_dir) if fast_deps: depComp = DependencyCompiler(cwd=repo_dir, gpu=gpu) From 41de4c5b194d54c6ed43acf487c66cf7d5e7db7a Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 16 Aug 2024 01:20:14 -0400 Subject: [PATCH 28/30] in `uv.py`, make func/methods `snake_case`, static methods `Snake_Case` --- comfy_cli/command/custom_nodes/cm_cli_util.py | 2 +- comfy_cli/command/install.py | 2 +- comfy_cli/uv.py | 54 +++++++++---------- tests/uv/test_uv.py | 4 +- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/comfy_cli/command/custom_nodes/cm_cli_util.py b/comfy_cli/command/custom_nodes/cm_cli_util.py index 0978213..c17c944 100644 --- a/comfy_cli/command/custom_nodes/cm_cli_util.py +++ b/comfy_cli/command/custom_nodes/cm_cli_util.py @@ -68,7 +68,7 @@ def execute_cm_cli(args, channel=None, fast_deps=False, mode=None) -> str | None if fast_deps and args[0] in _dependency_cmds: # we're using the fast_deps behavior and just ran a command that invalidated the dependencies depComp = DependencyCompiler(cwd=workspace_path) - depComp.installComfyDeps() + depComp.install_comfy_deps() return result.stdout except subprocess.CalledProcessError as e: diff --git a/comfy_cli/command/install.py b/comfy_cli/command/install.py index 73dc7d1..f4c8f0d 100644 --- a/comfy_cli/command/install.py +++ b/comfy_cli/command/install.py @@ -247,7 +247,7 @@ def execute( if fast_deps: depComp = DependencyCompiler(cwd=repo_dir, gpu=gpu) - depComp.installComfyDeps() + depComp.install_comfy_deps() if not skip_manager: update_node_id_cache() diff --git a/comfy_cli/uv.py b/comfy_cli/uv.py index c4729ed..4f0a66a 100644 --- a/comfy_cli/uv.py +++ b/comfy_cli/uv.py @@ -28,22 +28,22 @@ def _check_call(cmd: list[str], cwd: Optional[PathLike] = None): subprocess.check_call(cmd, cwd=cwd) -_reqNameRe: re.Pattern[str] = re.compile(r"require\s([\w-]+)") +_req_name_re: re.Pattern[str] = re.compile(r"require\s([\w-]+)") -def _reqReClosure(name: str) -> re.Pattern[str]: +def _req_re_closure(name: str) -> re.Pattern[str]: return re.compile(rf"({name}\S+)") -def parseUvCompileError(err: str) -> tuple[str, list[str]]: +def parse_uv_compile_error(err: str) -> tuple[str, list[str]]: """takes in stderr from a run of `uv pip compile` that failed due to requirement conflict and spits out a tuple of (reqiurement_name, [requirement_spec_in_conflict_a, requirement_spec_in_conflict_b]). Will probably fail for stderr produced from other kinds of errors """ - if reqNameMatch := _reqNameRe.search(err): + if reqNameMatch := _req_name_re.search(err): reqName = reqNameMatch[1] else: raise ValueError - reqRe = _reqReClosure(reqName) + reqRe = _req_re_closure(reqName) return reqName, cast(list[str], reqRe.findall(err)) @@ -67,7 +67,7 @@ class DependencyCompiler: } @staticmethod - def FindReqFiles(*ders: PathLike) -> list[Path]: + def Find_Req_Files(*ders: PathLike) -> list[Path]: return [file for der in ders for file in Path(der).absolute().iterdir() @@ -75,7 +75,7 @@ def FindReqFiles(*ders: PathLike) -> list[Path]: ] @staticmethod - def InstallBuildDeps(): + def Install_Build_Deps(): """Use pip to install bare minimum requirements for uv to do its thing """ if shutil.which("uv") is None: @@ -140,7 +140,7 @@ def Compile( print(f"STDERR:\n{e.stderr}") if resolve_strategy == "ask": - name, reqs = parseUvCompileError(e.stderr) + name, reqs = parse_uv_compile_error(e.stderr) vers = [req.split(name)[1].strip(",") for req in reqs] ver = ui.prompt_select( @@ -238,7 +238,7 @@ def Sync( return _check_call(cmd, cwd) @staticmethod - def ResolveGpu(gpu: Union[str, None]): + def Resolve_Gpu(gpu: Union[str, None]): if gpu is None: try: tver = metadata.version("torch") @@ -263,23 +263,23 @@ def __init__( ): self.cwd = Path(cwd) self.reqFiles = [Path(reqFile) for reqFile in reqFilesExt] if reqFilesExt is not None else None - self.gpu = DependencyCompiler.ResolveGpu(gpu) + self.gpu = DependencyCompiler.Resolve_Gpu(gpu) self.gpuUrl = DependencyCompiler.nvidiaPytorchUrl if self.gpu == GPU_OPTION.NVIDIA else DependencyCompiler.rocmPytorchUrl if self.gpu == GPU_OPTION.AMD else None self.out = self.cwd / outName self.override = self.cwd / "override.txt" - self.reqFilesCore = reqFilesCore if reqFilesCore is not None else self.findCoreReqs() - self.reqFilesExt = reqFilesExt if reqFilesExt is not None else self.findExtReqs() + self.reqFilesCore = reqFilesCore if reqFilesCore is not None else self.find_core_reqs() + self.reqFilesExt = reqFilesExt if reqFilesExt is not None else self.find_ext_reqs() - def findCoreReqs(self): - return DependencyCompiler.FindReqFiles(self.cwd) + def find_core_reqs(self): + return DependencyCompiler.Find_Req_Files(self.cwd) - def findExtReqs(self): + def find_ext_reqs(self): extDirs = [d for d in (self.cwd / "custom_nodes").iterdir() if d.is_dir() and d.name != "__pycache__"] - return DependencyCompiler.FindReqFiles(*extDirs) + return DependencyCompiler.Find_Req_Files(*extDirs) - def makeOverride(self): + def make_override(self): #clean up self.override.unlink(missing_ok=True) @@ -300,7 +300,7 @@ def makeOverride(self): f.write(line) f.write("\n") - def compileCorePlusExt(self): + def compile_core_plus_ext(self): #clean up self.out.unlink(missing_ok=True) @@ -322,7 +322,7 @@ def compileCorePlusExt(self): else: raise AttributeError - def installCorePlusExt(self): + def install_core_plus_ext(self): DependencyCompiler.Install( cwd=self.cwd, reqFile=self.out, @@ -330,14 +330,14 @@ def installCorePlusExt(self): extraUrl=self.gpuUrl, ) - def syncCorePlusExt(self): + def sync_core_plus_ext(self): DependencyCompiler.Sync( cwd=self.cwd, reqFile=self.out, extraUrl=self.gpuUrl, ) - def handleOpencv(self): + def handle_opencv(self): """as per the opencv docs, you should only have exactly one opencv package. headless is more suitable for comfy than the gui version, so remove gui if headless is present. TODO: add support for contrib pkgs. see: https://github.com/opencv/opencv-python""" @@ -358,11 +358,11 @@ def handleOpencv(self): if "opencv-python==" not in line: f.write(line) - def installComfyDeps(self): - DependencyCompiler.InstallBuildDeps() + def install_comfy_deps(self): + DependencyCompiler.Install_Build_Deps() - self.makeOverride() - self.compileCorePlusExt() - self.handleOpencv() + self.make_override() + self.compile_core_plus_ext() + self.handle_opencv() - self.installCorePlusExt() + self.install_core_plus_ext() diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index f0a003e..9bb3010 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -29,8 +29,8 @@ def test_compile(mock_prompt_select): reqFilesExt=[reqsDir/"x_reqs.txt", reqsDir/"y_reqs.txt"], ) - depComp.makeOverride() - depComp.compileCorePlusExt() + depComp.make_override() + depComp.compile_core_plus_ext() with open(reqsDir/"requirements.compiled", "r") as known, open(temp/"requirements.compiled", "r") as test: knownLines, testLines = known.readlines(), test.readlines() From c87cb3d21d253da0b09ff88ed4e18e58fdb8ad2e Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 16 Aug 2024 01:38:16 -0400 Subject: [PATCH 29/30] ci test fix --- tests/uv/test_uv.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/uv/test_uv.py b/tests/uv/test_uv.py index 9bb3010..e7d5e07 100644 --- a/tests/uv/test_uv.py +++ b/tests/uv/test_uv.py @@ -29,9 +29,14 @@ def test_compile(mock_prompt_select): reqFilesExt=[reqsDir/"x_reqs.txt", reqsDir/"y_reqs.txt"], ) + DependencyCompiler.Install_Build_Deps() depComp.make_override() depComp.compile_core_plus_ext() with open(reqsDir/"requirements.compiled", "r") as known, open(temp/"requirements.compiled", "r") as test: - knownLines, testLines = known.readlines(), test.readlines() + # compare all non-commented lines in generated file vs reference file + knownLines, testLines = [ + [line for line in known.readlines() if line.strip()[0]!="#"], + [line for line in test.readlines() if line.strip()[0]!="#"], + ] assert knownLines == testLines From ce674002ecfc126b9babd5dc5e066e52cc3eb3eb Mon Sep 17 00:00:00 2001 From: telamonian Date: Fri, 16 Aug 2024 01:53:44 -0400 Subject: [PATCH 30/30] disable nuisance lint rule (W0707) --- .pylintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.pylintrc b/.pylintrc index 761ec41..a31e900 100644 --- a/.pylintrc +++ b/.pylintrc @@ -17,6 +17,7 @@ disable= C0301, # line-too-long C0103, # invalid-name W1510, # subprocess-run-check + W0707, # raise-missing-from # TODO W3101, # missing timeout on request