From c0fb17419166e92869ac33e4c9ff8b58a5f7feec Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sun, 17 Sep 2023 14:34:25 -0400 Subject: [PATCH 1/7] Change declaration of extras in test setup package --- src/whl2conda/api/converter.py | 4 ++-- test/api/test_converter.py | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/whl2conda/api/converter.py b/src/whl2conda/api/converter.py index 7726f84..7aad0e5 100644 --- a/src/whl2conda/api/converter.py +++ b/src/whl2conda/api/converter.py @@ -614,9 +614,9 @@ def _copy_licenses(self, conda_info_dir: Path, wheel_md: MetadataFromWheel) -> N from_files = [wheel_info_dir / license_path.name] if not license_path.is_absolute(): from_files.insert(0, wheel_info_dir / license_path) - for from_file in filter( + for from_file in filter( # pragma: no branch lambda f: f.exists(), from_files - ): # pragma: no branch + ): to_file = to_license_dir / from_file.relative_to(wheel_info_dir) if not to_file.exists(): # pragma: no branch to_file.parent.mkdir(parents=True, exist_ok=True) diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 9e08268..8b6ee83 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -19,6 +19,8 @@ # standard import email +import logging +import re import shutil import subprocess import tempfile @@ -170,6 +172,7 @@ def _validate(self, wheel_path: Path, package_path: Path) -> None: wheel_path, package_path, std_renames=converter.std_renames, + extra=converter.extra_dependencies, keep_pip_dependencies=converter.keep_pip_dependencies, ) @@ -408,6 +411,12 @@ def test_simple_wheel( v1pkg = test_case(simple_wheel).build(CondaPackageFormat.V1) assert v1pkg.name.endswith(".tar.bz2") + treepkg = test_case(simple_wheel).build(CondaPackageFormat.TREE) + assert treepkg.is_dir() + with pytest.raises(FileExistsError): + test_case(simple_wheel).build(CondaPackageFormat.TREE) + test_case(simple_wheel, overwrite=True).build(CondaPackageFormat.TREE) + # Repack wheel with build number dest_dir = test_case.tmp_path / "number" subprocess.check_call( @@ -434,6 +443,34 @@ def test_simple_wheel( ).build() +def test_debug_log( + test_case: ConverterTestCaseFactory, + simple_wheel: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test debug logging during conversion""" + caplog.set_level("DEBUG") + + test_case( + simple_wheel, + extra_dependencies=["mypy"], + ).build() + + messages: list[str] = [] + for record in caplog.records: + if record.levelno == logging.DEBUG: + messages.append(record.message) + assert messages + debug_out = "\n".join(messages) + + assert re.search(r"Extracted.*METADATA", debug_out) + assert "Packaging info/about.json" in debug_out + assert re.search(r"Skipping extra dependency.*pylint", debug_out) + assert re.search(r"Dependency copied.*black", debug_out) + assert re.search(r"Dependency renamed.*numpy-quaternion.*quaternion", debug_out) + assert re.search(r"Dependency added.*mypy", debug_out) + + def test_poetry( test_case: ConverterTestCaseFactory, tmp_path: Path, @@ -481,6 +518,11 @@ def test_bad_wheels( with pytest.raises(Wheel2CondaError, match="unsupported wheel version"): test_case(bad_version_wheel).build() + case = test_case(bad_version_wheel) + case.converter.dry_run = True + with pytest.raises(Wheel2CondaError, match="unsupported wheel version"): + case.build() + # # impure wheel # From 137b6fae6d7a8ba4a008b2349129ebaf23f572dc Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Thu, 21 Sep 2023 08:09:18 -0400 Subject: [PATCH 2/7] Add a test case for manual renaming --- test/api/test_converter.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 8b6ee83..8d1c19d 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -172,6 +172,10 @@ def _validate(self, wheel_path: Path, package_path: Path) -> None: wheel_path, package_path, std_renames=converter.std_renames, + renamed={ + r.pattern.pattern : r.replacement + for r in self.dependency_rename + }, extra=converter.extra_dependencies, keep_pip_dependencies=converter.keep_pip_dependencies, ) @@ -442,6 +446,13 @@ def test_simple_wheel( overwrite=True, ).build() + test_case( + simple_wheel, + dependency_rename=[ + ( "numpy-quaternion", "quaternion2") + ], + overwrite=True, + ).build() def test_debug_log( test_case: ConverterTestCaseFactory, From 2edd64c1b9d3f0a391f246d3e1c6f9c1448290f7 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Thu, 21 Sep 2023 10:23:33 -0400 Subject: [PATCH 3/7] Refactor code for updating extra marker --- src/whl2conda/api/converter.py | 38 +++++++++++++-------- test/api/test_converter.py | 62 ++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 30 deletions(-) diff --git a/src/whl2conda/api/converter.py b/src/whl2conda/api/converter.py index 7aad0e5..0abad4f 100644 --- a/src/whl2conda/api/converter.py +++ b/src/whl2conda/api/converter.py @@ -91,6 +91,17 @@ class RequiresDistEntry: generic: bool = True """True if marker is empty or only contains an extra expression""" + def set_marker(self, marker: str) -> None: + """Set marker value and update extra_marker_name and generic values""" + self.marker = marker + self.generic = False + for pat in _extra_marker_re: + if m := pat.search(marker): + self.extra_marker_name = m.group("name") + if m.group(0) == marker: + self.generic = True + return + @classmethod def parse(cls, raw: str) -> RequiresDistEntry: """ @@ -108,14 +119,17 @@ def parse(cls, raw: str) -> RequiresDistEntry: if version := m.group("version"): entry.version = version if marker := m.group("marker"): - entry.marker = marker - entry.generic = False - for pat in _extra_marker_re: - if m := pat.search(marker): - entry.extra_marker_name = m.group("name") - if m.group(0) == marker: - entry.generic = True - break + entry.set_marker(marker) + return entry + + def with_extra(self, name: str) -> RequiresDistEntry: + """Returns copy of entry with an extra == '' clause to marker""" + marker = f"extra == '{name}'" + if self.marker: + marker = f"({self.marker}) and {marker}" + + entry = dataclasses.replace(self) + entry.set_marker(marker) return entry def __str__(self) -> str: @@ -677,13 +691,7 @@ def _parse_wheel_metadata(self, wheel_dir: Path) -> MetadataFromWheel: del md_msg["Requires-Dist"] for entry in requires: if not entry.extra_marker_name: - marker = entry.marker - extra_clause = "extra == 'original'" - if marker: - marker = f"({entry.marker}) and {extra_clause}" - else: - marker = extra_clause - entry = dataclasses.replace(entry, marker=marker) + entry = entry.with_extra('original') md_msg.add_header("Requires-Dist", str(entry)) md_msg.add_header("Provides-Extra", "original") wheel_md_file.write_text(md_msg.as_string()) diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 8d1c19d..764331c 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -25,6 +25,7 @@ import subprocess import tempfile from pathlib import Path +from time import sleep from typing import Generator, Optional, Sequence, Union # third party @@ -168,17 +169,17 @@ def _get_wheel(self) -> Path: def _validate(self, wheel_path: Path, package_path: Path) -> None: converter = self._converter assert converter is not None - self._validator.validate( - wheel_path, - package_path, - std_renames=converter.std_renames, - renamed={ - r.pattern.pattern : r.replacement - for r in self.dependency_rename - }, - extra=converter.extra_dependencies, - keep_pip_dependencies=converter.keep_pip_dependencies, - ) + if not converter.dry_run: + self._validator.validate( + wheel_path, + package_path, + std_renames=converter.std_renames, + renamed={ + r.pattern.pattern: r.replacement for r in self.dependency_rename + }, + extra=converter.extra_dependencies, + keep_pip_dependencies=converter.keep_pip_dependencies, + ) class ConverterTestCaseFactory: @@ -263,6 +264,15 @@ def check_dist_entry(entry: RequiresDistEntry) -> None: entry2 = RequiresDistEntry.parse(raw) assert entry == entry2 + if not entry.extra_marker_name: + entry_with_extra = entry.with_extra('original') + assert entry_with_extra != entry + assert entry_with_extra.extra_marker_name == 'original' + assert entry_with_extra.generic == entry.generic + assert entry_with_extra.name == entry.name + assert entry_with_extra.version == entry.version + assert entry.marker in entry_with_extra.marker + def test_requires_dist_entry() -> None: """Test RequiresDistEntry data structure""" @@ -363,6 +373,13 @@ def test_dependency_rename() -> None: # Converter test cases # +# TODO: test interactive ovewrite prompt +# TODO: test build number override +# TODO: test non-generic dependency warning +# TODO: test dropped dependency debug log +# TODO: test bad Requires-Dist entry (shouldn't happen in real life) +# TODO: test adding extra == original clause to non generic dist entry + def test_this(test_case: ConverterTestCaseFactory) -> None: """Test using this own project's wheel""" @@ -404,8 +421,24 @@ def test_simple_wheel( ) -> None: """Test converting wheel from 'simple' project""" + # Dry run should not create package + case = test_case(simple_wheel) + case.converter.dry_run = True + v2pkg_dr = case.build() + assert v2pkg_dr.suffix == ".conda" + assert not v2pkg_dr.exists() + + # Normal run v2pkg = test_case(simple_wheel).build() - assert v2pkg.suffix == ".conda" + assert v2pkg == v2pkg_dr + + # Do another dry run, show that old package not removed + mtime = v2pkg.stat().st_mtime_ns + sleep(0.01) + case = test_case(simple_wheel, overwrite=True) + case.converter.dry_run = True + assert case.build() == v2pkg + assert v2pkg.stat().st_mtime_ns == mtime with pytest.raises(FileExistsError): test_case(simple_wheel).build() @@ -448,12 +481,11 @@ def test_simple_wheel( test_case( simple_wheel, - dependency_rename=[ - ( "numpy-quaternion", "quaternion2") - ], + dependency_rename=[("numpy-quaternion", "quaternion2")], overwrite=True, ).build() + def test_debug_log( test_case: ConverterTestCaseFactory, simple_wheel: Path, From 368e1c212f0361fbfa6fd4fe7b9c0832ee598c0f Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Fri, 22 Sep 2023 08:37:03 -0400 Subject: [PATCH 4/7] Test logging dropped dependency --- test-projects/simple/pyproject.toml | 3 ++- test/api/test_converter.py | 36 ++++++++++++++++++----------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/test-projects/simple/pyproject.toml b/test-projects/simple/pyproject.toml index 4f5fb6a..bd4525a 100644 --- a/test-projects/simple/pyproject.toml +++ b/test-projects/simple/pyproject.toml @@ -24,7 +24,8 @@ classifiers = [ dependencies = [ "black[colorama] >=23.3", - "numpy-quaternion" + "numpy-quaternion", + "tables", ] [project.optional-dependencies] diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 764331c..13fc139 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -373,10 +373,9 @@ def test_dependency_rename() -> None: # Converter test cases # -# TODO: test interactive ovewrite prompt +# TODO: test interactive overwrite prompt # TODO: test build number override # TODO: test non-generic dependency warning -# TODO: test dropped dependency debug log # TODO: test bad Requires-Dist entry (shouldn't happen in real life) # TODO: test adding extra == original clause to non generic dist entry @@ -492,19 +491,31 @@ def test_debug_log( caplog: pytest.LogCaptureFixture, ) -> None: """Test debug logging during conversion""" - caplog.set_level("DEBUG") - - test_case( + case = test_case( simple_wheel, extra_dependencies=["mypy"], - ).build() + dependency_rename= [ + ("tables", "") + ], + overwrite=True, + ) + case.build() - messages: list[str] = [] - for record in caplog.records: - if record.levelno == logging.DEBUG: - messages.append(record.message) - assert messages - debug_out = "\n".join(messages) + def get_debug_out() -> str: + messages: list[str] = [] + for record in caplog.records: + if record.levelno == logging.DEBUG: + messages.append(record.message) + return "\n".join(messages) + + debug_out = get_debug_out() + assert not debug_out + + caplog.set_level("DEBUG") + + case.build() + + debug_out = get_debug_out() assert re.search(r"Extracted.*METADATA", debug_out) assert "Packaging info/about.json" in debug_out @@ -513,7 +524,6 @@ def test_debug_log( assert re.search(r"Dependency renamed.*numpy-quaternion.*quaternion", debug_out) assert re.search(r"Dependency added.*mypy", debug_out) - def test_poetry( test_case: ConverterTestCaseFactory, tmp_path: Path, From e7818f5acaefebfd70334d24e559dda8bb40e64f Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Fri, 22 Sep 2023 08:54:22 -0400 Subject: [PATCH 5/7] Move converter test fixture into its own module --- test/api/converter.py | 230 +++++++++++++++++++++++++++++++++++++ test/api/test_converter.py | 199 +------------------------------- test/api/test_external.py | 5 +- 3 files changed, 237 insertions(+), 197 deletions(-) create mode 100644 test/api/converter.py diff --git a/test/api/converter.py b/test/api/converter.py new file mode 100644 index 0000000..46ea57b --- /dev/null +++ b/test/api/converter.py @@ -0,0 +1,230 @@ +# Copyright 2023 Christopher Barber +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Test fixtures for the converter module +""" +from __future__ import annotations + +import shutil +import subprocess +import tempfile +from pathlib import Path +from typing import Union, Sequence, Optional, Generator + +import pytest + +from whl2conda.api.converter import DependencyRename, Wheel2CondaConverter +from whl2conda.cli.install import install_main +from whl2conda.impl.pyproject import CondaPackageFormat + +from ..api.validator import PackageValidator + +this_dir = Path(__file__).parent.absolute() +root_dir = this_dir.parent.parent +test_projects = root_dir / "test-projects" + + +class ConverterTestCase: + """ + Runner for a test case + """ + + wheel_src: Union[Path, str] + dependency_rename: Sequence[DependencyRename] + extra_dependencies: Sequence[str] + overwrite: bool + package_name: str + tmp_dir: Path + project_dir: Path + out_dir: Path + pip_downloads: Path + was_run: bool = False + + _converter: Optional[Wheel2CondaConverter] = None + _validator_dir: Path + _validator: PackageValidator + + @property + def converter(self) -> Wheel2CondaConverter: + """Converter instance, constructed on demand.""" + if self._converter is None: + self._converter = Wheel2CondaConverter(self._get_wheel(), self.out_dir) + assert self._converter is not None + return self._converter + + def __init__( + self, + wheel_src: Union[Path, str], + *, + tmp_dir: Path, + project_dir: Path, + package_name: str = "", + dependency_rename: Sequence[tuple[str, str]] = (), + extra_dependencies: Sequence[str] = (), + overwrite: bool = False, + ) -> None: + if not str(wheel_src).startswith("pypi:"): + wheel_src = Path(wheel_src) + assert wheel_src.exists() + self.wheel_src = wheel_src + self.dependency_rename = tuple( + DependencyRename.from_strings(*dr) for dr in dependency_rename + ) + self.extra_dependencies = tuple(extra_dependencies) + self.overwrite = overwrite + self.tmp_dir = tmp_dir + self.project_dir = project_dir + self.package_name = package_name + assert tmp_dir.is_dir() + self.out_dir = self.tmp_dir.joinpath("out") + self.pip_downloads = self.tmp_dir / "pip-downloads" + self.pip_downloads.mkdir(exist_ok=True) + self._validator_dir = self.tmp_dir.joinpath("validator") + if self._validator_dir.exists(): + shutil.rmtree(self._validator_dir) + self._validator_dir.mkdir() + self._validator = PackageValidator(self._validator_dir) + + def build(self, out_format: CondaPackageFormat = CondaPackageFormat.V2) -> Path: + """Run the build test case""" + self.was_run = True + wheel_path = self._get_wheel() + package_path = self._convert(out_format=out_format) + self._validate(wheel_path, package_path) + return package_path + + def install(self, pkg_file: Path) -> Path: + """Install conda package file into new conda environment in test-env/ subdir""" + test_env = self.tmp_dir.joinpath("test-env") + install_main([str(pkg_file), "-p", str(test_env), "--yes", "--create"]) + return test_env + + def _convert(self, *, out_format: CondaPackageFormat) -> Path: + converter = self.converter + converter.dependency_rename = list(self.dependency_rename) + converter.extra_dependencies = list(self.extra_dependencies) + converter.package_name = self.package_name + converter.overwrite = self.overwrite + converter.out_format = out_format + self._converter = converter + return converter.convert() + + def _get_wheel(self) -> Path: + if isinstance(self.wheel_src, Path): + return self.wheel_src + + assert str(self.wheel_src).startswith("pypi:") + spec = str(self.wheel_src)[5:] + + with tempfile.TemporaryDirectory(dir=self.pip_downloads) as tmpdir: + download_dir = Path(tmpdir) + try: + subprocess.check_call( + ["pip", "download", spec, "--no-deps", "-d", str(download_dir)] + ) + except subprocess.CalledProcessError as ex: + pytest.skip(f"Cannot download {spec} from pypi: {ex}") + downloaded_wheel = next(download_dir.glob("*.whl")) + target_wheel = self.pip_downloads / downloaded_wheel.name + if target_wheel.exists(): + target_wheel.unlink() + shutil.copyfile(downloaded_wheel, target_wheel) + + return target_wheel + + def _validate(self, wheel_path: Path, package_path: Path) -> None: + converter = self._converter + assert converter is not None + if not converter.dry_run: + self._validator.validate( + wheel_path, + package_path, + std_renames=converter.std_renames, + renamed={ + r.pattern.pattern: r.replacement for r in self.dependency_rename + }, + extra=converter.extra_dependencies, + keep_pip_dependencies=converter.keep_pip_dependencies, + ) + + +class ConverterTestCaseFactory: + """ + Factory for generating test case runners + """ + + tmp_path_factory: pytest.TempPathFactory + tmp_path: Path + project_dir: Path + _cases: list[ConverterTestCase] + + def __init__(self, tmp_path_factory: pytest.TempPathFactory) -> None: + self.tmp_path_factory = tmp_path_factory + self.tmp_path = tmp_path_factory.mktemp("converter-test-cases-") + orig_project_dir = root_dir.joinpath("test-projects") + self.project_dir = self.tmp_path.joinpath("projects") + shutil.copytree(orig_project_dir, self.project_dir, dirs_exist_ok=True) + self._cases = [] + + def __call__( + self, + wheel_src: Union[Path, str], + *, + package_name: str = "", + dependency_rename: Sequence[tuple[str, str]] = (), + extra_dependencies: Sequence[str] = (), + overwrite: bool = False, + ) -> ConverterTestCase: + case = ConverterTestCase( + wheel_src, + tmp_dir=self.tmp_path, + package_name=package_name, + project_dir=self.project_dir, + dependency_rename=dependency_rename, + extra_dependencies=extra_dependencies, + overwrite=overwrite, + ) + self._cases.append(case) + return case + + def teardown(self) -> None: + """Make sure all test cases were actually run""" + for i, case in enumerate(self._cases): + assert case.was_run, f"Test case #{i} was not run" + + +@pytest.fixture +def test_case( + tmp_path_factory: pytest.TempPathFactory, +) -> Generator[ConverterTestCaseFactory, None, None]: + """ + Yields a TestCaseFactory for creating test cases + """ + factory = ConverterTestCaseFactory(tmp_path_factory) + yield factory + factory.teardown() diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 13fc139..77adcf0 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -21,12 +21,9 @@ import email import logging import re -import shutil import subprocess -import tempfile from pathlib import Path from time import sleep -from typing import Generator, Optional, Sequence, Union # third party import pytest @@ -34,15 +31,14 @@ # this package from whl2conda.api.converter import ( - Wheel2CondaConverter, CondaPackageFormat, DependencyRename, RequiresDistEntry, Wheel2CondaError, ) from whl2conda.cli.convert import do_build_wheel -from whl2conda.cli.install import install_main -from .validator import PackageValidator +from .converter import ConverterTestCaseFactory +from .converter import test_case # pylint: disable=unused-import from ..test_packages import ( # pylint: disable=unused-import setup_wheel, @@ -53,192 +49,6 @@ root_dir = this_dir.parent.parent test_projects = root_dir / "test-projects" -# -# Converter test fixture -# - - -class ConverterTestCase: - """ - Runner for a test case - """ - - wheel_src: Union[Path, str] - dependency_rename: Sequence[DependencyRename] - extra_dependencies: Sequence[str] - overwrite: bool - package_name: str - tmp_dir: Path - project_dir: Path - out_dir: Path - pip_downloads: Path - was_run: bool = False - - _converter: Optional[Wheel2CondaConverter] = None - _validator_dir: Path - _validator: PackageValidator - - @property - def converter(self) -> Wheel2CondaConverter: - """Converter instance, constructed on demand.""" - if self._converter is None: - self._converter = Wheel2CondaConverter(self._get_wheel(), self.out_dir) - assert self._converter is not None - return self._converter - - def __init__( - self, - wheel_src: Union[Path, str], - *, - tmp_dir: Path, - project_dir: Path, - package_name: str = "", - dependency_rename: Sequence[tuple[str, str]] = (), - extra_dependencies: Sequence[str] = (), - overwrite: bool = False, - ) -> None: - if not str(wheel_src).startswith("pypi:"): - wheel_src = Path(wheel_src) - assert wheel_src.exists() - self.wheel_src = wheel_src - self.dependency_rename = tuple( - DependencyRename.from_strings(*dr) for dr in dependency_rename - ) - self.extra_dependencies = tuple(extra_dependencies) - self.overwrite = overwrite - self.tmp_dir = tmp_dir - self.project_dir = project_dir - self.package_name = package_name - assert tmp_dir.is_dir() - self.out_dir = self.tmp_dir.joinpath("out") - self.pip_downloads = self.tmp_dir / "pip-downloads" - self.pip_downloads.mkdir(exist_ok=True) - self._validator_dir = self.tmp_dir.joinpath("validator") - if self._validator_dir.exists(): - shutil.rmtree(self._validator_dir) - self._validator_dir.mkdir() - self._validator = PackageValidator(self._validator_dir) - - def build(self, out_format: CondaPackageFormat = CondaPackageFormat.V2) -> Path: - """Run the build test case""" - self.was_run = True - wheel_path = self._get_wheel() - package_path = self._convert(out_format=out_format) - self._validate(wheel_path, package_path) - return package_path - - def install(self, pkg_file: Path) -> Path: - """Install conda package file into new conda environment in test-env/ subdir""" - test_env = self.tmp_dir.joinpath("test-env") - install_main([str(pkg_file), "-p", str(test_env), "--yes", "--create"]) - return test_env - - def _convert(self, *, out_format: CondaPackageFormat) -> Path: - converter = self.converter - converter.dependency_rename = list(self.dependency_rename) - converter.extra_dependencies = list(self.extra_dependencies) - converter.package_name = self.package_name - converter.overwrite = self.overwrite - converter.out_format = out_format - self._converter = converter - return converter.convert() - - def _get_wheel(self) -> Path: - if isinstance(self.wheel_src, Path): - return self.wheel_src - - assert str(self.wheel_src).startswith("pypi:") - spec = str(self.wheel_src)[5:] - - with tempfile.TemporaryDirectory(dir=self.pip_downloads) as tmpdir: - download_dir = Path(tmpdir) - try: - subprocess.check_call( - ["pip", "download", spec, "--no-deps", "-d", str(download_dir)] - ) - except subprocess.CalledProcessError as ex: - pytest.skip(f"Cannot download {spec} from pypi: {ex}") - downloaded_wheel = next(download_dir.glob("*.whl")) - target_wheel = self.pip_downloads / downloaded_wheel.name - if target_wheel.exists(): - target_wheel.unlink() - shutil.copyfile(downloaded_wheel, target_wheel) - - return target_wheel - - def _validate(self, wheel_path: Path, package_path: Path) -> None: - converter = self._converter - assert converter is not None - if not converter.dry_run: - self._validator.validate( - wheel_path, - package_path, - std_renames=converter.std_renames, - renamed={ - r.pattern.pattern: r.replacement for r in self.dependency_rename - }, - extra=converter.extra_dependencies, - keep_pip_dependencies=converter.keep_pip_dependencies, - ) - - -class ConverterTestCaseFactory: - """ - Factory for generating test case runners - """ - - tmp_path_factory: pytest.TempPathFactory - tmp_path: Path - project_dir: Path - _cases: list[ConverterTestCase] - - def __init__(self, tmp_path_factory: pytest.TempPathFactory) -> None: - self.tmp_path_factory = tmp_path_factory - self.tmp_path = tmp_path_factory.mktemp("converter-test-cases-") - orig_project_dir = root_dir.joinpath("test-projects") - self.project_dir = self.tmp_path.joinpath("projects") - shutil.copytree(orig_project_dir, self.project_dir, dirs_exist_ok=True) - self._cases = [] - - def __call__( - self, - wheel_src: Union[Path, str], - *, - package_name: str = "", - dependency_rename: Sequence[tuple[str, str]] = (), - extra_dependencies: Sequence[str] = (), - overwrite: bool = False, - ) -> ConverterTestCase: - case = ConverterTestCase( - wheel_src, - tmp_dir=self.tmp_path, - package_name=package_name, - project_dir=self.project_dir, - dependency_rename=dependency_rename, - extra_dependencies=extra_dependencies, - overwrite=overwrite, - ) - self._cases.append(case) - return case - - def teardown(self) -> None: - """Make sure all test cases were actually run""" - for i, case in enumerate(self._cases): - assert case.was_run, f"Test case #{i} was not run" - - -@pytest.fixture -def test_case( - tmp_path_factory: pytest.TempPathFactory, -) -> Generator[ConverterTestCaseFactory, None, None]: - """ - Yields a TestCaseFactory for creating test cases - """ - factory = ConverterTestCaseFactory(tmp_path_factory) - yield factory - factory.teardown() - - # pylint: disable=redefined-outer-name # @@ -494,9 +304,7 @@ def test_debug_log( case = test_case( simple_wheel, extra_dependencies=["mypy"], - dependency_rename= [ - ("tables", "") - ], + dependency_rename=[("tables", "")], overwrite=True, ) case.build() @@ -524,6 +332,7 @@ def get_debug_out() -> str: assert re.search(r"Dependency renamed.*numpy-quaternion.*quaternion", debug_out) assert re.search(r"Dependency added.*mypy", debug_out) + def test_poetry( test_case: ConverterTestCaseFactory, tmp_path: Path, diff --git a/test/api/test_external.py b/test/api/test_external.py index e4b1036..fd10760 100644 --- a/test/api/test_external.py +++ b/test/api/test_external.py @@ -24,8 +24,9 @@ from whl2conda.api.converter import Wheel2CondaError -from .test_converter import ConverterTestCaseFactory -from .test_converter import test_case # pylint: disable=unused-import +from .converter import ConverterTestCaseFactory +from .converter import test_case # pylint: disable=unused-import + # pylint: disable=redefined-outer-name From 84dd50e83537a33a401bfb853fc0cfb3880afd88 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Fri, 22 Sep 2023 09:18:55 -0400 Subject: [PATCH 6/7] Restrict coverage to src/whl2conda dir --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9c04f99..bf2a694 100644 --- a/Makefile +++ b/Makefile @@ -105,10 +105,10 @@ pytest: test: pytest coverage: - $(CONDA_RUN) pytest -s --cov=src --cov-report=json --cov-report=term test + $(CONDA_RUN) pytest -s --cov=src/whl2conda --cov-report=json --cov-report=term test slow-coverage: - $(CONDA_RUN) pytest -s --cov=src --cov-report=json --cov-report=term test --run-slow + $(CONDA_RUN) pytest -s --cov=src/whl2conda --cov-report=json --cov-report=term test --run-slow htmlcov/index.html: .coverage $(CONDA_RUN) coverage html From 483f2aaf0509cd5048d526e90f7aba9a430d4566 Mon Sep 17 00:00:00 2001 From: Christopher Barber Date: Sat, 23 Sep 2023 12:04:03 -0400 Subject: [PATCH 7/7] 100% test coverage --- test-projects/markers/markers/__init__.py | 14 ++++ test-projects/markers/pyproject.toml | 17 ++++ test/api/converter.py | 1 + test/api/test_converter.py | 95 +++++++++++++++++++++-- test/api/validator.py | 19 +++-- test/test_packages.py | 17 ++++ 6 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 test-projects/markers/markers/__init__.py create mode 100644 test-projects/markers/pyproject.toml diff --git a/test-projects/markers/markers/__init__.py b/test-projects/markers/markers/__init__.py new file mode 100644 index 0000000..a64a240 --- /dev/null +++ b/test-projects/markers/markers/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023 Christopher Barber +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/test-projects/markers/pyproject.toml b/test-projects/markers/pyproject.toml new file mode 100644 index 0000000..eb19d6f --- /dev/null +++ b/test-projects/markers/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "markers" +version = "2023.9.23" +description = "test dependencies with environment markers" +requires-python = ">=3.8" + +# https://peps.python.org/pep-0508/#environment-markers +dependencies = [ + "typing-extensions ; python_version < '3.9'", + "ntfsdump >=2.4 ; platform_system == 'Windows'", + "atomacos ; platform_system == 'Darwin'", + "pytest" +] diff --git a/test/api/converter.py b/test/api/converter.py index 46ea57b..9f1889e 100644 --- a/test/api/converter.py +++ b/test/api/converter.py @@ -170,6 +170,7 @@ def _validate(self, wheel_path: Path, package_path: Path) -> None: }, extra=converter.extra_dependencies, keep_pip_dependencies=converter.keep_pip_dependencies, + build_number=converter.build_number, ) diff --git a/test/api/test_converter.py b/test/api/test_converter.py index 77adcf0..3ae01e1 100644 --- a/test/api/test_converter.py +++ b/test/api/test_converter.py @@ -24,6 +24,7 @@ import subprocess from pathlib import Path from time import sleep +from typing import Iterator # third party import pytest @@ -41,6 +42,7 @@ from .converter import test_case # pylint: disable=unused-import from ..test_packages import ( # pylint: disable=unused-import + markers_wheel, setup_wheel, simple_wheel, ) @@ -183,12 +185,6 @@ def test_dependency_rename() -> None: # Converter test cases # -# TODO: test interactive overwrite prompt -# TODO: test build number override -# TODO: test non-generic dependency warning -# TODO: test bad Requires-Dist entry (shouldn't happen in real life) -# TODO: test adding extra == original clause to non generic dist entry - def test_this(test_case: ConverterTestCaseFactory) -> None: """Test using this own project's wheel""" @@ -288,6 +284,13 @@ def test_simple_wheel( overwrite=True, ).build() + case = test_case( + build42whl, + overwrite=True, + ) + case.converter.build_number = 23 + case.build() + test_case( simple_wheel, dependency_rename=[("numpy-quaternion", "quaternion2")], @@ -333,6 +336,50 @@ def get_debug_out() -> str: assert re.search(r"Dependency added.*mypy", debug_out) +def test_warnings( + test_case: ConverterTestCaseFactory, + markers_wheel: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """ + Test conversion warnings + """ + + def get_warn_out() -> str: + messages: list[str] = [] + for record in caplog.records: + if record.levelno == logging.WARNING: + messages.append(record.message) + return "\n".join(messages) + + test_case(markers_wheel).build() + + warn_out = get_warn_out() + assert re.search( + r"Skipping.*with.*marker.*typing-extensions ; python_version < '3.9'", warn_out + ) + assert re.search(r"Skipping.*ntfsdump", warn_out) + assert re.search(r"Skipping.*atomacos", warn_out) + + # Make wheel with bad Requires-Dist entries + wheel = WheelFile(markers_wheel) + bad_wheel_dir = test_case.tmp_path / "bad-wheel" + wheel.extractall(bad_wheel_dir) + distinfo_dir = next(bad_wheel_dir.glob("*.dist-info")) + metadata_file = distinfo_dir / "METADATA" + contents = metadata_file.read_text("utf8") + # Add bogus !!! to Requires-Dist entries with markers + contents = re.sub(r"Requires-Dist:(.*);", r"Requires-Dist:!!!\1;", contents) + metadata_file.write_text(contents) + bad_wheel_file = bad_wheel_dir / markers_wheel.name + with WheelFile(str(bad_wheel_file), "w") as wf: + wf.write_files(str(bad_wheel_dir)) + + test_case(bad_wheel_file, overwrite=True).build() + warn_out = get_warn_out() + print(warn_out) + + def test_poetry( test_case: ConverterTestCaseFactory, tmp_path: Path, @@ -420,3 +467,39 @@ def test_bad_wheels( with pytest.raises(Wheel2CondaError, match="unsupported metadata version"): test_case(bad_md_version_wheel).build() + + +def test_overwrite_prompt( + test_case: ConverterTestCaseFactory, + simple_wheel: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """ + Test interactive prompting for overwrite. + """ + prompts: Iterator[str] = iter(()) + responses: Iterator[str] = iter(()) + + def fake_input(prompt: str) -> str: + expected_prompt = next(prompts) + assert re.search( + expected_prompt, prompt + ), f"'{expected_prompt}' does not match prompt '{prompt}'" + return next(responses) + + monkeypatch.setattr("builtins.input", fake_input) + + case = test_case(simple_wheel) + case.converter.interactive = False + case.build() + + case.converter.interactive = True + prompts = iter(["Overwrite?"]) + responses = iter(["no"]) + with pytest.raises(FileExistsError): + case.build() + + case.converter.interactive = True + prompts = iter(["Overwrite?"]) + responses = iter(["yes"]) + case.build() diff --git a/test/api/validator.py b/test/api/validator.py index 1f21c94..7e474a5 100644 --- a/test/api/validator.py +++ b/test/api/validator.py @@ -53,6 +53,7 @@ class PackageValidator: _std_renames: dict[str, Any] _extra_dependencies: Sequence[str] _keep_pip_dependencies: bool = False + _build_number: int | None = None def __init__(self, tmp_dir: Path) -> None: self.tmp_dir = tmp_dir @@ -70,6 +71,7 @@ def validate( std_renames: Optional[dict[str, str]] = None, extra: Sequence[str] = (), keep_pip_dependencies: bool = False, + build_number: int | None = None, ) -> None: """Validate conda package against wheel from which it was generated""" self._override_name = name @@ -77,6 +79,7 @@ def validate( self._std_renames = std_renames or {} self._extra_dependencies = extra self._keep_pip_dependencies = keep_pip_dependencies + self._build_number = build_number wheel_dir = self._unpack_wheel(wheel) self._unpack_package(conda_pkg) @@ -276,10 +279,13 @@ def _validate_index(self, info_dir: Path) -> None: assert index["arch"] is None assert index['build'] == 'py_0' - try: - build_number = int(wheel_md.get("build", 0)) - except ValueError: - build_number = 0 + if self._build_number is not None: + build_number = self._build_number + else: + try: + build_number = int(wheel_md.get("build", 0)) + except ValueError: + build_number = 0 assert index['build_number'] == build_number assert index["platform"] is None @@ -298,7 +304,10 @@ def _validate_dependencies(self, dependencies: Sequence[str]) -> None: if python_ver := wheel_md.get("requires-python"): expected_depends.add(f"python {python_ver}") for dep in wheel_md.get("requires-dist", []): - entry = RequiresDistEntry.parse(dep) + try: + entry = RequiresDistEntry.parse(dep) + except SyntaxError: + continue if entry.marker: continue name = entry.name diff --git a/test/test_packages.py b/test/test_packages.py index ef0e021..260d27c 100644 --- a/test/test_packages.py +++ b/test/test_packages.py @@ -24,6 +24,7 @@ from whl2conda.cli.convert import convert_main, do_build_wheel __all__ = [ + "markers_wheel", "project_dir", "setup_wheel", "simple_conda_package", @@ -33,6 +34,7 @@ this_dir = Path(__file__).parent.absolute() root_dir = this_dir.parent project_dir = root_dir.joinpath("test-projects") +markers_project = project_dir.joinpath("markers") simple_project = project_dir.joinpath("simple") setup_project = project_dir.joinpath("setup") @@ -71,6 +73,21 @@ def simple_conda_package( yield list(simple_wheel.parent.glob("*.conda"))[0] +@pytest.fixture(scope="session") +def markers_wheel( + tmp_path_factory: pytest.TempPathFactory, +) -> Generator[Path, None, None]: + """Provides pip wheel for "markers" test project""" + distdir = tmp_path_factory.mktemp("dist") + yield do_build_wheel( + markers_project, + distdir, + no_deps=True, + no_build_isolation=True, + capture_output=True, + ) + + @pytest.fixture(scope="session") def setup_wheel( tmp_path_factory: pytest.TempPathFactory,