From 02ead0cd008b5b57fbb1f76e1022cc0a70529abf Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 09:25:14 -0400 Subject: [PATCH 01/16] Support python 3.12 --- conda.recipe/meta.yaml | 7 +++++-- environment.yml | 2 +- pyproject.toml | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 7272e40..293b8d0 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -29,11 +29,10 @@ build: requirements: build: - - python 3.11 + - python 3.12 - hatchling - setuptools - run: - python {{ project['requires-python'] }} @@ -41,6 +40,10 @@ requirements: - {{ dep.lower() }} {% endfor %} +whl2conda: + something: + - foo + - bar test: source_files: diff --git a/environment.yml b/environment.yml index 6287f49..cff9512 100644 --- a/environment.yml +++ b/environment.yml @@ -3,7 +3,7 @@ channels: - conda-forge - garpy # for mkdocstrings-python-xref dependencies: - - python >=3.8,<3.12 + - python >=3.8,<3.13 # runtime - conda-package-handling >=2.2,<3.0 - platformdirs >=3.10 diff --git a/pyproject.toml b/pyproject.toml index 4130fcd..90b1ae7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] keywords = ["conda", "wheel", "build"] requires-python = ">=3.8" From e3ead7597af132a13f00e927078cfae03bf82f95 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 09:26:10 -0400 Subject: [PATCH 02/16] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 0a55949..c32d8b3 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -9,7 +9,7 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.8", "3.10", "3.11"] + python-version: ["3.8", "3.10", "3.12"] steps: - uses: actions/checkout@v3 - uses: goanpeca/setup-miniconda@v2.2.0 From 52017220ee678e79decba258c2d31aba337c80f6 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 11:05:33 -0400 Subject: [PATCH 03/16] Change order of conda index arguments --- src/whl2conda/cli/install.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/whl2conda/cli/install.py b/src/whl2conda/cli/install.py index 5a62265..07884d8 100644 --- a/src/whl2conda/cli/install.py +++ b/src/whl2conda/cli/install.py @@ -221,9 +221,8 @@ def conda_bld_install(parsed: InstallArgs, subdir: str): shutil.copyfile( parsed.package_file, subdir_path.joinpath(parsed.package_file.name) ) - # TODO: use conda-index instead subprocess.check_call( - ["conda", "index", str(conda_bld_path), "--subdir", subdir] + ["conda", "index", "--subdir", subdir, str(conda_bld_path)] ) From 3021a4bdfe0f0b20720b4abb79e3c867ef5a97f3 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 11:51:18 -0400 Subject: [PATCH 04/16] Relax black version for python 3.12 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index cff9512..40222a3 100644 --- a/environment.yml +++ b/environment.yml @@ -21,7 +21,7 @@ dependencies: - pytest-cov >=4.1.0,<5.0 - types-pyyaml >=6.0 # documentation - - black >=23.7 + - black >=22.6 - mike >=1.1,<2.0 - mkdocs >=1.5,<2.0 - mkdocstrings-python >=1.3,<2.0 From e9cca49239a4fa23a08d22966ad6c5925e3d0d46 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 12:38:03 -0400 Subject: [PATCH 05/16] Make createdev uses python 3.12 --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13498fa..19248c5 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,8 @@ help: DEV_INSTALL := $(CONDA_RUN) pip install -e . --no-deps --no-build-isolation createdev: - conda env create -f environment.yml -n $(DEV_ENV) --yes + conda create -n $(DEV_ENV) python=3.12 --yes + conda env update -f environment.yml -n $(DEV_ENV) $(MAKE) dev-install updatedev: From 5cb5bb7bb8568bddbb4727c9915785c1e01d9c75 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 12:38:45 -0400 Subject: [PATCH 06/16] `whl2conda build` help description --- src/whl2conda/cli/build.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/whl2conda/cli/build.py b/src/whl2conda/cli/build.py index d2feaf7..823a180 100644 --- a/src/whl2conda/cli/build.py +++ b/src/whl2conda/cli/build.py @@ -56,15 +56,14 @@ def build_main( parser = argparse.ArgumentParser( description=dedent( """ - Compare the content of two conda packages - - This will unpack each conda packaeg into temporary - directories, normalize the layout of files in the - input directory to minimize line differences, and - will run the specified diff tool and arguments. - - This can be used to compare packages generated using - this tool against those created using conda-build. + Build a conda package from a pure python wheel. + + This command is limited drop-in replacement for `conda build`. + It requires that the conda recipe has a build script entry + of the form `pip install` or `pip wheel`. + + This is an experimental feature and is still under active + change and development. """ ), formatter_class=argparse.RawTextHelpFormatter, From 52bf5615f19fb8296e35d16bd12d265e00ddc2f6 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 12:39:05 -0400 Subject: [PATCH 07/16] Reuse get_conda_bld_path --- src/whl2conda/cli/install.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/whl2conda/cli/install.py b/src/whl2conda/cli/install.py index 07884d8..57fcdad 100644 --- a/src/whl2conda/cli/install.py +++ b/src/whl2conda/cli/install.py @@ -29,7 +29,7 @@ from conda_package_handling.api import extract as extract_conda_pkg -from .common import dedent, existing_path, add_markdown_help +from .common import dedent, existing_path, add_markdown_help, get_conda_bld_path __all__ = ["install_main"] @@ -202,17 +202,7 @@ def install_main( def conda_bld_install(parsed: InstallArgs, subdir: str): """Install package into conda-bld directory""" - config = json.loads( - subprocess.check_output( - ["conda", "config", "--show", "--json"], - encoding="utf8", - ) - ) - conda_bld = config.get("bld_path") or config.get("croot") - if not conda_bld: # pragma: no cover - # this is extremely unlikely to ever occur - raise LookupError("Cannot find conda-bld location") - conda_bld_path = Path(conda_bld) + conda_bld_path = get_conda_bld_path() subdir_path = conda_bld_path.joinpath(subdir) # e.g. noarch/ print(f"Installing {parsed.package_file} into {subdir_path}") From f0d89dcfb976ed3a59b30549f60cf8e1877a2cfe Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 12:39:21 -0400 Subject: [PATCH 08/16] Fix pylint for python 3.12 --- environment.yml | 5 +---- pyproject.toml | 4 ++++ test/api/test_converter.py | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index 40222a3..dd70c98 100644 --- a/environment.yml +++ b/environment.yml @@ -16,7 +16,7 @@ dependencies: - make >=4.3 # testing - mypy >=1.5,<2.0 - - pylint >=2.17,<3.0 + - pylint >=2.17 - pytest >=7.4,<8.0 - pytest-cov >=4.1.0,<5.0 - types-pyyaml >=6.0 @@ -34,6 +34,3 @@ dependencies: - pip >=23.2.1 - pip: - mkdocs-awesome-pages-plugin >=2.9.2 - - - diff --git a/pyproject.toml b/pyproject.toml index 90b1ae7..909c1ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,9 @@ max-returns = 6 max-statements = 50 min-public-methods = 1 +[tool.pylint.similarities] +min-similarity-lines = 6 + [tool.pylint.variables] allowed-redefined-builtins = ["help", "license"] @@ -106,6 +109,7 @@ disable = [ "locally-disabled", "raw-checker-failed", "suppressed-message", + "too-many-arguments", "trailing-newlines", "unspecified-encoding", "use-dict-literal", diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 3ae01e1..3a293ba 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -480,6 +480,7 @@ def test_overwrite_prompt( prompts: Iterator[str] = iter(()) responses: Iterator[str] = iter(()) + # pylint: disable=duplicate-code def fake_input(prompt: str) -> str: expected_prompt = next(prompts) assert re.search( From c44adb34048f2806179120516b4d93dad00ed6d6 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 12:55:07 -0400 Subject: [PATCH 09/16] install conda-index in base environment in CI runs --- .github/workflows/python-package-conda.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index c32d8b3..888c366 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -23,6 +23,9 @@ jobs: condarc-file: github-condarc.yml auto-activate-base: true use-mamba: true + - name: install conda-index + run: | + conda install -n base conda-index --yes - name: Dev install whl2conda run: | conda run -n whl2conda-dev pip install -e . --no-deps --no-build-isolation From 76d568329d0ae58afbdf7ac0e868202567f207b0 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:26:35 -0400 Subject: [PATCH 10/16] change miniforge variant in CI script --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 888c366..bd84cf7 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v3 - uses: goanpeca/setup-miniconda@v2.2.0 with: - miniforge-variant: Mambaforge-pypy3 + miniforge-version: latest conda-build-version: ">=3.26" environment-file: environment.yml activate-environment: whl2conda-dev From be4d50e6b5f361923f9dcf73df16778d21d2fa3f Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:34:46 -0400 Subject: [PATCH 11/16] Fix pylint for python 3.12 --- src/whl2conda/cli/install.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/whl2conda/cli/install.py b/src/whl2conda/cli/install.py index 57fcdad..6a9a58c 100644 --- a/src/whl2conda/cli/install.py +++ b/src/whl2conda/cli/install.py @@ -211,8 +211,12 @@ def conda_bld_install(parsed: InstallArgs, subdir: str): shutil.copyfile( parsed.package_file, subdir_path.joinpath(parsed.package_file.name) ) + # subprocess.check_call( + # ["conda", "index", "--subdir", subdir, str(conda_bld_path)] + # ) + # Really just want subdir, but having problems in CI. See if this works: subprocess.check_call( - ["conda", "index", "--subdir", subdir, str(conda_bld_path)] + ["conda", "index", str(conda_bld_path)] ) From f4e705c4f03be71c41cccff4c49e17c7a2a4c0dd Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:47:32 -0400 Subject: [PATCH 12/16] Use conda 23.7.4 or greater in CI --- .github/workflows/python-package-conda.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index bd84cf7..f23f7d5 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -15,17 +15,20 @@ jobs: - uses: goanpeca/setup-miniconda@v2.2.0 with: miniforge-version: latest + conda-version: ">=23.7.4" conda-build-version: ">=3.26" environment-file: environment.yml activate-environment: whl2conda-dev python-version: ${{ matrix.python-version }} + auto-activate-base: true auto-update-conda: true condarc-file: github-condarc.yml auto-activate-base: true use-mamba: true - - name: install conda-index + - name: show conda versions run: | - conda install -n base conda-index --yes + conda list -n base conda + conda list -n base mamba - name: Dev install whl2conda run: | conda run -n whl2conda-dev pip install -e . --no-deps --no-build-isolation From 8ea7acb2be3225cf72c1a55ee99de939531276e9 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:48:23 -0400 Subject: [PATCH 13/16] Update python-package-conda.yml --- .github/workflows/python-package-conda.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index f23f7d5..df11d26 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -23,7 +23,6 @@ jobs: auto-activate-base: true auto-update-conda: true condarc-file: github-condarc.yml - auto-activate-base: true use-mamba: true - name: show conda versions run: | From 1b28b3144370e08d1c81a23d4e1876c7e8e5976c Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:50:02 -0400 Subject: [PATCH 14/16] Turn of auto update conda only one of conda-version and auto-update-conda may be provided in CI --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index df11d26..8ab75eb 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -16,12 +16,12 @@ jobs: with: miniforge-version: latest conda-version: ">=23.7.4" + # auto-update-conda: true conda-build-version: ">=3.26" environment-file: environment.yml activate-environment: whl2conda-dev python-version: ${{ matrix.python-version }} auto-activate-base: true - auto-update-conda: true condarc-file: github-condarc.yml use-mamba: true - name: show conda versions From 8073645661444ecd553320df90427a6105326cb1 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:52:28 -0400 Subject: [PATCH 15/16] black reformat --- src/whl2conda/cli/install.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/whl2conda/cli/install.py b/src/whl2conda/cli/install.py index 6a9a58c..e3d26d4 100644 --- a/src/whl2conda/cli/install.py +++ b/src/whl2conda/cli/install.py @@ -215,9 +215,7 @@ def conda_bld_install(parsed: InstallArgs, subdir: str): # ["conda", "index", "--subdir", subdir, str(conda_bld_path)] # ) # Really just want subdir, but having problems in CI. See if this works: - subprocess.check_call( - ["conda", "index", str(conda_bld_path)] - ) + subprocess.check_call(["conda", "index", str(conda_bld_path)]) def conda_env_install(parsed: InstallArgs, dependencies: list[str]): From 3463d86b855f25e064ca34850347c9df1b18d03c Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 8 Oct 2023 13:59:12 -0400 Subject: [PATCH 16/16] Restore previous conda index call (problems appear to be with particular conda/conda-build versions installed in CI) --- src/whl2conda/cli/install.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/whl2conda/cli/install.py b/src/whl2conda/cli/install.py index e3d26d4..57fcdad 100644 --- a/src/whl2conda/cli/install.py +++ b/src/whl2conda/cli/install.py @@ -211,11 +211,9 @@ def conda_bld_install(parsed: InstallArgs, subdir: str): shutil.copyfile( parsed.package_file, subdir_path.joinpath(parsed.package_file.name) ) - # subprocess.check_call( - # ["conda", "index", "--subdir", subdir, str(conda_bld_path)] - # ) - # Really just want subdir, but having problems in CI. See if this works: - subprocess.check_call(["conda", "index", str(conda_bld_path)]) + subprocess.check_call( + ["conda", "index", "--subdir", subdir, str(conda_bld_path)] + ) def conda_env_install(parsed: InstallArgs, dependencies: list[str]):