From 47e62a1e527abb44bc007a0adda766a02010d788 Mon Sep 17 00:00:00 2001 From: StardustDL Date: Sun, 28 Jan 2024 22:14:21 +0800 Subject: [PATCH] fix diff bug --- .github/workflows/ci.yml | 37 ++++---- README.md | 118 ++++++++++++++++---------- src/aexpy/__main__.py | 25 ++++-- src/aexpy/diffing/differs/checkers.py | 3 +- src/aexpy/diffing/differs/default.py | 2 +- src/aexpy/preprocessing/download.py | 2 +- src/aexpy/preprocessing/wheel.py | 17 +++- 7 files changed, 127 insertions(+), 77 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abd7b885..4717257e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ defaults: jobs: package: runs-on: ubuntu-latest + env: + PYTHONUTF8: 1 steps: - name: Checkout uses: actions/checkout@v4 @@ -48,7 +50,7 @@ jobs: - name: Test Difference continue-on-error: true run: | - aexpy -vvv diff ./cache/api1.json ./cache/api1.json ./cache/diff.json + aexpy -vvv diff ./cache/api1.json ./cache/api2.json ./cache/diff.json - name: Test Report continue-on-error: true run: | @@ -69,6 +71,8 @@ jobs: path: ./cache image: runs-on: ubuntu-latest + env: + PYTHONUTF8: 1 steps: - name: Checkout uses: actions/checkout@v4 @@ -91,30 +95,30 @@ jobs: mkdir -p ./cache - name: Test Preprocess run: | - docker run -v ${{ github.workspace }}/cache:/data -vvv preprocess -r -p generator-oj-problem@0.0.1 /data /data/distribution1.json - docker run -v ${{ github.workspace }}/cache:/data -vvv preprocess -r -p generator-oj-problem@0.0.2 /data /data/distribution2.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv preprocess -r -p generator-oj-problem@0.0.1 /data /data/distribution1.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv preprocess -r -p generator-oj-problem@0.0.2 /data /data/distribution2.json - name: Test Extraction continue-on-error: true run: | - docker run -v ${{ github.workspace }}/cache:/data -vvv extract /data/distribution1.json /data/api1.json - docker run -v ${{ github.workspace }}/cache:/data -vvv extract /data/distribution2.json /data/api2.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution1.json /data/api1.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution2.json /data/api2.json - name: Test Difference continue-on-error: true run: | - docker run -v ${{ github.workspace }}/cache:/data -vvv diff /data/api1.json /data/api1.json /data/diff.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv diff /data/api1.json /data/api2.json /data/diff.json - name: Test Report continue-on-error: true run: | - docker run -v ${{ github.workspace }}/cache:/data -vvv report /data/diff.json /data/report.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv report /data/diff.json /data/report.json - name: Test View continue-on-error: true run: | - docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/distribution1.json - docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/distribution2.json - docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/api1.json - docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/api2.json - docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/diff.json - docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/report.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/distribution1.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/distribution2.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api1.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api2.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/diff.json + docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/report.json - name: Upload results uses: actions/upload-artifact@v4 with: @@ -122,6 +126,8 @@ jobs: path: ./cache docs: runs-on: ubuntu-latest + env: + PYTHONUTF8: 1 steps: - name: Checkout uses: actions/checkout@v4 @@ -146,6 +152,8 @@ jobs: if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' || github.event_name == 'release' }} needs: [image, docs, package] runs-on: ubuntu-latest + env: + PYTHONUTF8: 1 steps: - name: Download artifacts uses: actions/download-artifact@v4 @@ -162,14 +170,13 @@ jobs: args: deploy --dir=./docs/dist --prod secrets: '["NETLIFY_AUTH_TOKEN", "NETLIFY_SITE_ID"]' - name: Download package artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: package path: ./dist - name: Deploy packages if: ${{ github.event_name == 'release' }} env: - PYTHONUTF8: 1 TWINE_USERNAME: '__token__' TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | diff --git a/README.md b/README.md index c7bff187..b4b8e761 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ AexPy also provides a framework to process Python packages, extract APIs, and de ## Features - Preprocessing - - Build packages and get source code. + - Download packages and get source code, or use existing code base. - Count package file sizes and lines of code. - Read package metadata and detect top modules. - Extracting @@ -51,103 +51,127 @@ AexPy also provides a framework to process Python packages, extract APIs, and de - Grade changes by their severities. - Reporting - Generate a human-readable report for API change detection results. -- Batching - - Process packages and releases in batch. - Framework - Customize processors and implementation details. - Process Python packages in AexPy's general pipeline with logging and caching. - Generate portable data in JSON for API descriptions, changes, and so on. - - Execute processing and view data by AexPy's command-line / RESTful APIs / front-end. + - Execute processing and view data by AexPy's command-line, with stdin/stdout supported. ## Install -We recommend using our Docker image for running AexPy. Other distributions may suffer from environment errors. +We provide the Python package on PyPI. Use pip to install the package. + +```sh +python -m pip install --upgrade aexpy + +aexpy --help +``` + +> Please ensure your Python interpreter works in [UTF-8 mode](https://peps.python.org/pep-0540/). + +We also provide the Docker image for running AexPy, to avoid environment errors. ```sh docker pull stardustdl/aexpy:latest +docker run stardustdl/aexpy:latest --help + # or the image from the main branch docker pull stardustdl/aexpy:main ``` -> To run the original package instead of the image, please ensure your Python interpreter works in [UTF-8 mode](https://peps.python.org/pep-0540/). +## Quick Start -## Usage - -### Front-end - -AexPy provides a convenient frontend for exploring APIs and changes. Use the following command to start the server, and then access the front-end at `http://localhost:8008` in browser. +Diff generator-oj-problem v0.0.1 and v0.0.2, output report content to `report.txt`, while saving API descriptions to `cache/api1.json` and `cache/api2.json` ```sh -docker run -p 8008:8008 stardustdl/aexpy:latest serve +mkdir -p cache +aexpy preprocess -r -p generator-oj-problem@0.0.2 ./cache - | aexpy extract - ./cache/api2.json +aexpy diff ./cache/api1.json ./cache/api2.json - | aexpy report - - | aexpy view - > report.txt ``` -The front-end depends on the AexPy's RESTful APIs at the endpoint `/api`. - -### Command-line +## Usage -Use the following command to detect changes between v1.0 and v2.0 of a package named demo: +1. Preprocess a distribution for a package release ```sh -docker run stardustdl/aexpy:latest report demo@1.0:2.0 +# download the package wheel and unpack into ./cache +# output the distribution file to ./cache/distribution.json +aexpy preprocess -r -p generator-oj-problem@0.0.1 ./cache ./cache/distribution.json +# or output the distribution file to stdout +aexpy preprocess -r -p generator-oj-problem@0.0.1 ./cache - + +# use existing wheel file +aexpy preprocess -w ./cache/generator_oj_problem-0.0.1-py3-none-any.whl ./cache/distribution.json + +# use existing unpacked wheel directory, auto load metadata from .dist-info directory +aexpy preprocess -d ./cache/generator_oj_problem-0.0.1-py3-none-any ./cache/distribution.json -# e.g. detect API changes between jinja2 v3.1.1 and v3.1.2 -docker run stardustdl/aexpy:latest report jinja2@3.1.1:3.1.2 +# use existing source code directory, given the package's name, version, and top-level modules +aexpy preprocess ./cache/generator_oj_problem-0.0.1-py3-none-any ./cache/distribution.json -p generator-oj-problem@0.0.1 -m generator_oj_problem ``` -Use the following command to extract API information of v1.0 of a package named demo: +2. Extract the API description from a distribution ```sh -docker run stardustdl/aexpy:latest extract demo@1.0 - -# e.g. extract APIs from click v8.1.3 -docker run stardustdl/aexpy:latest extract click@8.1.3 +aexpy extract ./cache/distribution.json ./cache/api.json +# or input the distribution file from stdin +# (this feature is also supported in other commands) +aexpy extract - ./cache/api.json +# or output the api description file to stdout +aexpy extract ./cache/distribution.json - ``` -For all available commands, use the following command: +3. Diff two API descriptions and detect changes ```sh -docker run stardustdl/aexpy:latest --help +aexpy diff ./cache/api1.json ./cache/api2.json ./cache/diff.json ``` -## Advanced Tools +4. Generate report from detect changes -### Batching +```sh +aexpy report ./cache/diff.json ./cache/report.json +``` -AexPy supports processing all available versions of a package in batch. +5. View produced data ```sh -aexpy batch coxbuild +aexpy view ./cache/distribution1.json +aexpy view ./cache/distribution2.json +aexpy view ./cache/api1.json +aexpy view ./cache/api2.json +aexpy view ./cache/diff.json +aexpy view ./cache/report.json ``` +> The docker image keeps the same command-line interface, +> only need a volume mapping to `/data` for file access. +> +> ```sh +> docker run -v $pwd/cache:/data aexpy/aexpy extract /data/distribution.json /data/api.json +> ``` + +## Advanced Tools + ### Logging -The processing may cost time, you can use multiple `-v` for verbose logs. +The processing may cost time, you can use multiple `-v` for verbose logs (which are outputed to stderr). ```sh docker run aexpy:latest -vvv extract click@8.1.3 ``` -### Data +### Interactive -You can mount cache directory to `/data` to save the processed data. AexPy will use the cache data if it exists, and produce results in JSON format under the cache directory. +Add `-i` or `--interact` to enable interactive mode, every command will create an interactive Python shell after finishing processing. Here are some useful variable you could use in the interactive Python shell. -```sh -docker run -v /path/to/cache:/data aexpy:latest extract click@8.1.3 +- `result`: The produced data object +- `context`: The producing context, use `exception` to access the exception if failing to process -cat /path/to/cache/extracting/types/click/8.1.3.json -``` +> Feel free to use `locals()` and `dir()` to explore the interactive environment. ### Pipeline -AexPy has four stages in its pipeline, use the following commands to run the corresponding stage. - -```sh -aexpy preprocess coxbuild@0.0.1 -aexpy extract coxbuild@0.0.1 -aexpy diff coxbuild@0.0.1:0.0.2 -aexpy report coxbuild@0.0.1:0.0.2 -``` - -The four stages are loosely coupled. The adjacent stages transfer data by JSON, defined in [models](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/models/) directory. You can easily write your own implementation for every stage, and combine your implementation into the pipeline. See [third](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/third/) directory for an example on how to implement stages and integrate other tools. \ No newline at end of file +AexPy has four loosely-coupled stages in its pipeline. The adjacent stages transfer data by JSON, defined in [models](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/models/) directory. You can easily write your own implementation for every stage, and combine your implementation into the pipeline. See [third](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/third/) directory for an example on how to implement stages and integrate other tools. \ No newline at end of file diff --git a/src/aexpy/__main__.py b/src/aexpy/__main__.py index 27ca784d..ae684d55 100644 --- a/src/aexpy/__main__.py +++ b/src/aexpy/__main__.py @@ -1,11 +1,13 @@ import code +import json import logging import pathlib from pathlib import Path import sys -from typing import IO, Literal +from typing import IO, Annotated, Literal import click +from pydantic import Field from aexpy.models import ProduceState from aexpy.caching import ( @@ -310,6 +312,9 @@ def report(difference: IO[str], report: IO[str]): result = context.product StreamWriterProduceCache(report).save(result, context.log) + print(result.overview(), file=sys.stderr) + print(f"\n{result.content}", file=sys.stderr) + if FLAG_interact: code.interact(banner="", local=locals()) @@ -324,18 +329,24 @@ def view(file: IO[str]): Supports distribution, api-description, api-difference, and report file (in json format). """ - from pydantic import TypeAdapter - cache = StreamReaderProduceCache(file) try: - result = TypeAdapter( - Distribution | ApiDescription | ApiDifference | Report - ).validate_json(cache.raw()) + data = json.loads(cache.raw()) + if "release" in data: + result = Distribution.model_validate(data) + elif "distribution" in data: + result = ApiDescription.model_validate(data) + elif "entries" in data: + result = ApiDifference.model_validate(data) + else: + result = Report.model_validate(data) except Exception as ex: assert False, f"Failed to load data: {ex}" - print(result.overview(), file=sys.stderr) + print(result.overview()) + if isinstance(result, Report): + print(f"\n{result.content}") if FLAG_interact: code.interact(banner="", local=locals()) diff --git a/src/aexpy/diffing/differs/checkers.py b/src/aexpy/diffing/differs/checkers.py index 22202a01..10061a10 100644 --- a/src/aexpy/diffing/differs/checkers.py +++ b/src/aexpy/diffing/differs/checkers.py @@ -1,4 +1,3 @@ -import dataclasses from dataclasses import dataclass, field import functools from typing import Any, Callable, Literal, TypeVar, Type, cast, TypeGuard, overload @@ -67,7 +66,7 @@ def __call__( result = self.checker(old, new, oldCollection, newCollection) return ( [ - dataclasses.replace(entry, kind=self.kind, old=old, new=new) + entry.model_copy(update={"kind": self.kind, "old": old, "new": new}) for entry in result ] if result diff --git a/src/aexpy/diffing/differs/default.py b/src/aexpy/diffing/differs/default.py index 7c5a2f15..1d5573d3 100644 --- a/src/aexpy/diffing/differs/default.py +++ b/src/aexpy/diffing/differs/default.py @@ -55,7 +55,7 @@ def _processEntry( result = [] for constraint in self.constraints: try: - done: "list[DiffEntry]" = constraint( + done: list[DiffEntry] = constraint( old, new, oldDescription, newDescription ) if done: diff --git a/src/aexpy/preprocessing/download.py b/src/aexpy/preprocessing/download.py index 7e23eb73..0e2a31a9 100644 --- a/src/aexpy/preprocessing/download.py +++ b/src/aexpy/preprocessing/download.py @@ -43,7 +43,7 @@ def check(s: str): ) for item in glob(".whl"): - logger.info(f"Remove downloaded {item}.") + logger.warning(f"Remove downloaded {item}.") os.remove(item) for pyversion in PYVERSIONS: diff --git a/src/aexpy/preprocessing/wheel.py b/src/aexpy/preprocessing/wheel.py index 7e98731f..32781dca 100644 --- a/src/aexpy/preprocessing/wheel.py +++ b/src/aexpy/preprocessing/wheel.py @@ -1,3 +1,4 @@ +import shutil from typing import override import zipfile from dataclasses import dataclass, field @@ -81,7 +82,7 @@ def pyversion(self) -> str | None: @classmethod def fromdir(cls, path: Path, project: str = ""): - distinfoDir = list(path.glob(f"{project}*.dist-info")) + distinfoDir = list(path.glob(f"{project.replace("-", "_")}*.dist-info")) if len(distinfoDir) == 0: return None distinfoDir = distinfoDir[0] @@ -124,10 +125,13 @@ def preprocess(self, product): ), "No wheel file provided." targetDir = self.cacheDir / product.wheelFile.stem - assert ( - not targetDir.exists() - ), f"The unpacked directory has been existed: {targetDir}" + if targetDir.exists(): + self.logger.warning(f"Remove unpacked directory {targetDir}") + shutil.rmtree(targetDir) + + self.logger.info(f"Unpacking {product.wheelFile} to {targetDir}") unpackWheel(product.wheelFile, targetDir) + self.logger.info(f"Unpacked {product.wheelFile} to {targetDir}") product.rootPath = targetDir @@ -136,7 +140,12 @@ class WheelMetadataPreprocessor(Preprocessor): def preprocess(self, product): assert product.rootPath, "No root path provided." + self.logger.info(f"Load dist-info from {product.rootPath} for {product.release.project}") + distInfo = DistInfo.fromdir(product.rootPath, product.release.project) + + self.logger.info(f"Loaded dist-info from {product.rootPath}: {distInfo}") + if distInfo: if distInfo.pyversion: if not product.pyversion: