Merge pull request #94 from zuzukin/diff

Diff

CHANGELOG.md

@@ -1,5 +1,16 @@
 # whl2conda changes
 
+## [23.8.10] - 2023-9-16 (*prerelease*)
+
+* Fix generation of entry points
+* Adjust metadata generation
+* Add `whl2conda diff` subcommand
+
+## [23.8.9] - 2023-9-14 (*prerelease*)
+
+* Support `python -m whl2conda`
+* Fix issue with license copying
+
 ## [23.8.8] - 2023-9-10 (*prerelease*)
 
 * hide pip build wheel output with `--quiet`

conda.recipe/meta.yaml

@@ -0,0 +1,74 @@
+{% set version = load_file_data('src/whl2conda/VERSION', 'yaml') %}
+{% set project = load_file_data('pyproject.toml')['project'] %}
+
+
+package:
+  name: whl2conda
+  version: {{ version }}
+
+source:
+  - path: ../pyproject.toml
+  - path: ../src
+    folder: src
+  - path: ../LICENSE.md
+  - path: ../README.md
+  - path: ../test
+    folder: test
+
+build:
+  noarch: python
+  script: pip install . -vv --no-deps --no-build-isolation
+
+  entry_points:
+    {% for script in project['scripts'] %}
+    - '{{ script }}={{ project['scripts'][script] }}'
+    {% endfor %}
+
+requirements:
+  build:
+    - python 3.11
+    - hatchling
+    - setuptools
+
+
+  run:
+    - python {{ project['requires-python'] }}
+
+    {% for dep in project['dependencies'] %}
+    - {{ dep.lower() }}
+    {% endfor %}
+
+
+test:
+  source_files:
+    - test
+  requires:
+    - pytest
+  imports:
+    - whl2conda.api
+  commands:
+    - pytest test
+
+about:
+  home: {{ project['urls']['homepage']}}
+  readme: {{ project['readme'] }}
+  summary: {{ project['summary'] }}
+  description: {{ project['description'] }}
+  keywords:
+    {% for keyword in project['keywords'] %}
+    - '{{ keyword }}'
+    {% endfor %}
+  dev_url: {{ project['urls']['repository'] }}
+  doc_url: {{ project['urls']['documentation'] }}
+  license: {{ project['license'] }}
+  license_file: {{ project['license-files']['paths'][0] }}
+
+extra:
+  authors:
+    {% for author in project['authors'] %}
+    - '{{ author['name'] }} <{{ author['email'] }}>'
+    {% endfor %}
+  classifiers:
+    {% for classifier in project['classifiers'] %}
+    - '{{ classifier }}'
+    {% endfor %}

doc/limitations.md

@@ -38,26 +38,13 @@ whl2conda mywheel-1.2.3-py3-none-any.whl -D foo -A 'foo >=1.2.3,1.2.*'
 This will be fixed in a future release 
 (see [issue 84](https://github.com/zuzukin/whl2conda/issues/84)).
 
-## whl2conda install does not create entry points
+## Wheel data directories not supported
 
-The `whl2conda install` command depends on `conda install` to
-install the package file, and that apparently does not create
-entry points declared in the package. For instance, if you
-have a package that normally provides a command line program
-`mycli`, that will not be available after this install.
+Wheels with `*.data` directies are not fully supported.
+Any such data directories will not be copied.
 
-If your package has a suitable main module, then you should still
-be able to invoke your program using `python -m`, e.g.:
-
-```bash
-python -m mycli
-```
-
-This will be fixed in a future release (see [issue #88](https://github.com/zuzukin/whl2conda/issues/88)).
-
-Note that this is only an issue with this installation technique.
-There is no problem with packages built by this tool that are
-installed from an indexed channel.
+This will be addressed in a future release
+(see [issue 91](https://github.com/zuzukin/whl2conda/issues/91))
 
 ## Cannot convert from sdist
 

doc/links.md

@@ -7,6 +7,8 @@
 
 ## Wheels:
 
+* [Binary distribution format](https://packaging.python.org/en/latest/specifications/binary-distribution-format/)
+* [Recording installed packages](https://packaging.python.org/en/latest/specifications/recording-installed-packages/)
 * [Current meta-data specification](https://packaging.python.org/en/latest/specifications/core-metadata/)
 * [Wheel 1.0 file format (PEP 427)](https://peps.python.org/pep-0427/)
 * [Wheel metadata 1.0 (PEP 241)](https://peps.python.org/pep-0241/)

doc/testing.md

@@ -35,3 +35,39 @@ this using:
 ```bash
 $ whl2conda install mypackage-1.2.3-py_0.conda --conda-bld
 ```
+
+## Comparing packages
+
+You may wish to compare generated packages against those generated
+by conda-build in order both to understand what this tool is doing
+and to verify that nothing important is missing. You can do this
+using the `whl2conda diff` command with your favorite directory
+diff tool. This will unpack the packages into temporary directories,
+normalize the metadata files to minimize mismatches and run the
+specified diff tool with the given arguments.
+
+For instance,
+
+```bash
+$ whl2conda diff \
+   dist/mypackage-1.2.3-py_0.conda \
+   ~/miniforge3/conda-bld/noarch/mypackage-1.2.3-py_90.tar.bz2 \
+   kdiff3
+```
+
+Note that some differences are expected in the `info/` directory,
+specifically:
+
+* packages generated with whl2conda will not have copy of the recipe
+   or test directory
+* the about.json file may differ
+* the timestamp will be different in the `index.json` file
+* the `paths.json` file should reflect any files that differ
+
+There are also expected to be changes in the `site-packages/*dist-info/`
+for the package:
+
+* the `INSTALLER` file will contain `whl2conda` instead of `conda`
+* the `Requires-Dist` entries in `METADATA` will be modified to add
+    `; extra = 'original'`
+

mkdocs.yml

@@ -20,6 +20,7 @@ nav:
         - wl2conda: cli/whl2conda.md
         - whl2conda config: cli/whl2conda-config.md
         - whl2conda convert: cli/whl2conda-convert.md
+        - whl2conda diff: cli/whl2conda-diff.md
         - whl2conda install: cli/whl2conda-install.md
       - API (<i>not stable!</i>):
           - api/converter.md

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "whl2conda"
-description = "build conda packages directly from pure python wheels"
+summary = "build conda packages directly from pure python wheels"
 readme = "README.md"
 dynamic = ["version"]
 license = "Apache-2.0"
@@ -31,6 +31,7 @@ dependencies = [
 ]
 
 [project.urls]
+homepage = "https://github.com/analog-cbarber/whl2conda"
 repository = "https://github.com/analog-cbarber/whl2conda"
 documentation = "https://zuzukin.github.io/whl2conda/"
 

src/whl2conda/VERSION

@@ -1 +1 @@
-23.8.9
+23.8.10

src/whl2conda/__main__.py

@@ -17,5 +17,5 @@
 """
 from .cli.main import main
 
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     main()

src/whl2conda/api/converter.py

@@ -134,20 +134,13 @@ class Wheel2CondaError(RuntimeError):
     """Errors from Wheel2CondaConverter"""
 
 
-class NonNoneDict(dict):
+def non_none_dict(**kwargs: Any) -> dict[str, Any]:
     """dict that drops keys with None values"""
-
-    def __init__(self, **kwargs: Any):
-        super().__init__()
-        for k, v in kwargs.items():
-            self[k] = v
-
-    def __setitem__(self, key: str, val: Any) -> None:
-        if val is None:
-            if key in self:
-                del self[key]
-        else:
-            super().__setitem__(key, val)
+    d = dict()
+    for k, v in kwargs.items():
+        if v is not None:
+            d[k] = v
+    return d
 
 
 @dataclass
@@ -307,13 +300,16 @@ def convert(self) -> Path:
             conda_dependencies = self._compute_conda_dependencies(wheel_md.dependencies)
 
             # Write conda info files
+            # TODO - copy readme file into info
+            #  must be one of README, README.md or README.rst
             self._copy_licenses(conda_info_dir, wheel_md)
             self._write_about(conda_info_dir, wheel_md.md)
             self._write_hash_input(conda_info_dir)
             self._write_files_list(conda_info_dir, rel_files)
             self._write_index(conda_info_dir, wheel_md, conda_dependencies)
             self._write_link_file(conda_info_dir, wheel_md.wheel_info_dir)
             self._write_paths_file(conda_dir, rel_files)
+            self._write_git_file(conda_info_dir)
 
             conda_pkg_path = self._conda_package_path(
                 wheel_md.package_name, wheel_md.version
@@ -367,6 +363,13 @@ def _write_conda_package(self, conda_dir: Path, conda_pkg_path: Path) -> Path:
 
         return conda_pkg_path
 
+    def _write_git_file(self, conda_info_dir: Path) -> None:
+        """Write empty git file"""
+        # python wheels don't have this concept, but conda-build
+        # will write an empty git file if there are no git sources,
+        # so we follow suit:
+        conda_info_dir.joinpath("git").write_bytes(b'')
+
     def _write_paths_file(self, conda_dir: Path, rel_files: Sequence[str]) -> None:
         # info/paths.json - paths with SHA256 do we really need this?
         conda_paths_file = conda_dir.joinpath("info", "paths.json")
@@ -398,14 +401,17 @@ def _write_link_file(self, conda_info_dir: Path, wheel_info_dir: Path) -> None:
                 if section_name in wheel_entry_points:
                     if section := wheel_entry_points[section_name]:
                         console_scripts.extend(f"{k}={v}" for k, v in section.items())
+        noarch_dict: dict[str, Any] = dict(type="python")
+        if console_scripts:
+            noarch_dict["entry_points"] = console_scripts
         conda_link_file.write_text(
             json.dumps(
                 dict(
-                    noarch=dict(type="python"),
-                    entry_points=console_scripts,
+                    noarch=noarch_dict,
                     package_metadata_version=1,
                 ),
                 indent=2,
+                sort_keys=True,
             )
         )
 
@@ -458,16 +464,35 @@ def _write_hash_input(self, conda_info_dir: Path) -> None:
         conda_hash_input_file = conda_info_dir.joinpath("hash_input.json")
         conda_hash_input_file.write_text(json.dumps({}, indent=2))
 
+    # pylint: disable=too-many-locals
     def _write_about(self, conda_info_dir: Path, md: dict[str, Any]) -> None:
         # * info/about.json
+        #
+        # Note that the supported fields in the about section are not
+        # well documented, but conda-build will only copy fields from
+        # its approved list, which can be found in the FIELDS datastructure
+        # in the conda_build.metadata module. This currently includes:
+        #
+        #   URLS: home, dev_url, doc_url, doc_source_url
+        #   Text: license, summary, description, license_family
+        #   Lists: tags, keyword
+        #   Paths in source tree: license-file, prelink_message, readme
+        #
+        # conda-build also adds conda-build-version and conda-version fields.
+
         license = md.get("license-expression") or md.get("license")
         conda_about_file = conda_info_dir.joinpath("about.json")
         # TODO description can come from METADATA message body
         #   then need to also use content type. It doesn't seem
         #   that conda-forge packages include this in the info/
         doc_url: Optional[str] = None
         dev_url: Optional[str] = None
-        extra: dict[str, Any] = {}
+        extra: dict[str, Any] = non_none_dict(
+            author=md.get("author"),
+            classifiers=md.get("classifier"),
+            maintainer=md.get("maintainer"),
+            whl2conda_version=__version__,
+        )
         for urlline in md.get("project-url", ()):
             urlparts = re.split(r"\s*,\s*", urlline, maxsplit=1)
             if len(urlparts) > 1:
@@ -479,22 +504,25 @@ def _write_about(self, conda_info_dir: Path, md: dict[str, Any]) -> None:
                     dev_url = urlparts[1]
                 if key and url:
                     extra[key] = url
-        for key in ["author", "maintainer", "author-email", "maintainer-email"]:
+        for key in ["author-email", "maintainer-email"]:
             val = md.get(key)
             if val:
-                extra[key] = val
+                author_key = key.split("-", maxsplit=1)[0] + "s"
+                extra[author_key] = val.split(",")
         if license_files := md.get("license-file"):
             extra["license_files"] = list(license_files)
+        if keywords := md.get("keywords"):
+            keyword_list = keywords.split(",")
+        else:
+            keyword_list = None
         conda_about_file.write_text(
             json.dumps(
-                NonNoneDict(
+                non_none_dict(
                     description=md.get("description"),
                     summary=md.get("summary"),
                     license=license or None,
-                    classifiers=md.get("classifier"),
-                    keywords=md.get("keywords"),
+                    keywords=keyword_list,
                     home=md.get("home-page"),
-                    whl2conda_version=__version__,
                     dev_url=dev_url,
                     doc_url=doc_url,
                     extra=extra or None,
@@ -554,6 +582,12 @@ def _copy_site_packages(self, wheel_dir: Path, conda_dir: Path) -> list[str]:
         conda_info_dir = conda_dir.joinpath("info")
         conda_info_dir.mkdir()
         shutil.copytree(wheel_dir, conda_site_packages, dirs_exist_ok=True)
+        assert self.wheel_md is not None
+        dist_info_dir = conda_site_packages / self.wheel_md.wheel_info_dir.name
+        installer_file = dist_info_dir / "INSTALLER"
+        installer_file.write_text("whl2conda")
+        requested_file = dist_info_dir / "REQUESTED"
+        requested_file.write_text("")
         rel_files = list(
             str(f.relative_to(conda_dir))
             for f in conda_site_packages.glob("**/*")
@@ -662,6 +696,12 @@ def _parse_wheel_metadata(self, wheel_dir: Path) -> MetadataFromWheel:
         self.package_name = package_name
         version = md.get("version")
 
+        # RECORD_file = wheel_info_dir / "RECORD"
+        # TODO: strip __pycache__ entries from RECORD
+        # TODO: add INSTALLER and REQUESTED to RECORD
+        # TODO: add direct_url to wheel and to RECORD
+        # RECORD line format: <path>,sha256=<hash>,<len>
+
         python_version: str = str(md.get("requires-python", ""))
         if python_version:
             requires.append(RequiresDistEntry("python", version=python_version))