fix diff bug

StardustDL · Jan 28, 2024 · 47e62a1 · 47e62a1
1 parent 95a3d5c
commit 47e62a1
Show file tree

Hide file tree

Showing 7 changed files with 127 additions and 77 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,8 @@ defaults:
 jobs:
   package:
     runs-on: ubuntu-latest
+    env:
+      PYTHONUTF8: 1
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -48,7 +50,7 @@ jobs:
     - name: Test Difference
       continue-on-error: true
       run: |
-        aexpy -vvv diff ./cache/api1.json ./cache/api1.json ./cache/diff.json
+        aexpy -vvv diff ./cache/api1.json ./cache/api2.json ./cache/diff.json
     - name: Test Report
       continue-on-error: true
       run: |
@@ -69,6 +71,8 @@ jobs:
         path: ./cache
   image:
     runs-on: ubuntu-latest
+    env:
+      PYTHONUTF8: 1
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -91,37 +95,39 @@ jobs:
         mkdir -p ./cache
     - name: Test Preprocess
       run: |
-        docker run -v ${{ github.workspace }}/cache:/data -vvv preprocess -r -p generator-oj-problem@0.0.1 /data /data/distribution1.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv preprocess -r -p generator-oj-problem@0.0.2 /data /data/distribution2.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv preprocess -r -p generator-oj-problem@0.0.1 /data /data/distribution1.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv preprocess -r -p generator-oj-problem@0.0.2 /data /data/distribution2.json
     - name: Test Extraction
       continue-on-error: true
       run: |
-        docker run -v ${{ github.workspace }}/cache:/data -vvv extract /data/distribution1.json /data/api1.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv extract /data/distribution2.json /data/api2.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution1.json /data/api1.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution2.json /data/api2.json
     - name: Test Difference
       continue-on-error: true
       run: |
-        docker run -v ${{ github.workspace }}/cache:/data -vvv diff /data/api1.json /data/api1.json /data/diff.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv diff /data/api1.json /data/api2.json /data/diff.json
     - name: Test Report
       continue-on-error: true
       run: |
-        docker run -v ${{ github.workspace }}/cache:/data -vvv report /data/diff.json /data/report.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv report /data/diff.json /data/report.json
     - name: Test View
       continue-on-error: true
       run: |
-        docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/distribution1.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/distribution2.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/api1.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/api2.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/diff.json
-        docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/report.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/distribution1.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/distribution2.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api1.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api2.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/diff.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/report.json
     - name: Upload results
       uses: actions/upload-artifact@v4
       with:
         name: image-cache
         path: ./cache
   docs:
     runs-on: ubuntu-latest
+    env:
+      PYTHONUTF8: 1
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -146,6 +152,8 @@ jobs:
     if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' || github.event_name == 'release' }}
     needs: [image, docs, package]
     runs-on: ubuntu-latest
+    env:
+      PYTHONUTF8: 1
     steps:
     - name: Download artifacts
       uses: actions/download-artifact@v4
@@ -162,14 +170,13 @@ jobs:
         args: deploy --dir=./docs/dist --prod
         secrets: '["NETLIFY_AUTH_TOKEN", "NETLIFY_SITE_ID"]'
     - name: Download package artifacts
-      uses: actions/download-artifact@v3
+      uses: actions/download-artifact@v4
       with:
         name: package
         path: ./dist
     - name: Deploy packages
       if: ${{ github.event_name == 'release' }}
       env:
-        PYTHONUTF8: 1
         TWINE_USERNAME: '__token__'
         TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
       run: |

diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ AexPy also provides a framework to process Python packages, extract APIs, and de
 ## Features
 
 - Preprocessing
-  - Build packages and get source code.
+  - Download packages and get source code, or use existing code base.
   - Count package file sizes and lines of code.
   - Read package metadata and detect top modules.
 - Extracting
@@ -51,103 +51,127 @@ AexPy also provides a framework to process Python packages, extract APIs, and de
   - Grade changes by their severities.
 - Reporting
   - Generate a human-readable report for API change detection results.
-- Batching
-  - Process packages and releases in batch.
 - Framework
   - Customize processors and implementation details.
   - Process Python packages in AexPy's general pipeline with logging and caching.
   - Generate portable data in JSON for API descriptions, changes, and so on.
-  - Execute processing and view data by AexPy's command-line / RESTful APIs / front-end.
+  - Execute processing and view data by AexPy's command-line, with stdin/stdout supported.
 
 ## Install
 
-We recommend using our Docker image for running AexPy. Other distributions may suffer from environment errors.
+We provide the Python package on PyPI. Use pip to install the package.
+
+```sh
+python -m pip install --upgrade aexpy
+
+aexpy --help
+```
+
+> Please ensure your Python interpreter works in [UTF-8 mode](https://peps.python.org/pep-0540/).
+
+We also provide the Docker image for running AexPy, to avoid environment errors.
 
 ```sh
 docker pull stardustdl/aexpy:latest
 
+docker run stardustdl/aexpy:latest --help
+
 # or the image from the main branch
 
 docker pull stardustdl/aexpy:main
 ```
 
-> To run the original package instead of the image, please ensure your Python interpreter works in [UTF-8 mode](https://peps.python.org/pep-0540/).
+## Quick Start
 
-## Usage
-
-### Front-end
-
-AexPy provides a convenient frontend for exploring APIs and changes. Use the following command to start the server, and then access the front-end at `http://localhost:8008` in browser.
+Diff generator-oj-problem v0.0.1 and v0.0.2, output report content to `report.txt`, while saving API descriptions to `cache/api1.json` and `cache/api2.json`
 
 ```sh
-docker run -p 8008:8008 stardustdl/aexpy:latest serve
+mkdir -p cache
+aexpy preprocess -r -p generator-oj-problem@0.0.2 ./cache - | aexpy extract - ./cache/api2.json
+aexpy diff ./cache/api1.json ./cache/api2.json - | aexpy report - - | aexpy view - > report.txt
 ```
 
-The front-end depends on the AexPy's RESTful APIs at the endpoint `/api`.
-
-### Command-line
+## Usage
 
-Use the following command to detect changes between v1.0 and v2.0 of a package named demo:
+1. Preprocess a distribution for a package release
 
 ```sh
-docker run stardustdl/aexpy:latest report demo@1.0:2.0
+# download the package wheel and unpack into ./cache
+# output the distribution file to ./cache/distribution.json
+aexpy preprocess -r -p generator-oj-problem@0.0.1 ./cache ./cache/distribution.json
+# or output the distribution file to stdout
+aexpy preprocess -r -p generator-oj-problem@0.0.1 ./cache -
+
+# use existing wheel file
+aexpy preprocess -w ./cache/generator_oj_problem-0.0.1-py3-none-any.whl ./cache/distribution.json
+
+# use existing unpacked wheel directory, auto load metadata from .dist-info directory
+aexpy preprocess -d ./cache/generator_oj_problem-0.0.1-py3-none-any ./cache/distribution.json
 
-# e.g. detect API changes between jinja2 v3.1.1 and v3.1.2
-docker run stardustdl/aexpy:latest report jinja2@3.1.1:3.1.2
+# use existing source code directory, given the package's name, version, and top-level modules
+aexpy preprocess ./cache/generator_oj_problem-0.0.1-py3-none-any ./cache/distribution.json -p generator-oj-problem@0.0.1 -m generator_oj_problem
 ```
 
-Use the following command to extract API information of v1.0 of a package named demo:
+2. Extract the API description from a distribution
 
 ```sh
-docker run stardustdl/aexpy:latest extract demo@1.0
-
-# e.g. extract APIs from click v8.1.3
-docker run stardustdl/aexpy:latest extract click@8.1.3
+aexpy extract ./cache/distribution.json ./cache/api.json
+# or input the distribution file from stdin 
+# (this feature is also supported in other commands)
+aexpy extract - ./cache/api.json
+# or output the api description file to stdout
+aexpy extract ./cache/distribution.json -
 ```
 
-For all available commands, use the following command:
+3. Diff two API descriptions and detect changes
 
 ```sh
-docker run stardustdl/aexpy:latest --help
+aexpy diff ./cache/api1.json ./cache/api2.json ./cache/diff.json
 ```
 
-## Advanced Tools
+4. Generate report from detect changes
 
-### Batching
+```sh
+aexpy report ./cache/diff.json ./cache/report.json
+```
 
-AexPy supports processing all available versions of a package in batch.
+5. View produced data
 
 ```sh
-aexpy batch coxbuild
+aexpy view ./cache/distribution1.json
+aexpy view ./cache/distribution2.json
+aexpy view ./cache/api1.json
+aexpy view ./cache/api2.json
+aexpy view ./cache/diff.json
+aexpy view ./cache/report.json
 ```
 
+> The docker image keeps the same command-line interface,
+> only need a volume mapping to `/data` for file access.
+>
+> ```sh
+> docker run -v $pwd/cache:/data aexpy/aexpy extract /data/distribution.json /data/api.json
+> ```
+
+## Advanced Tools
+
 ### Logging
 
-The processing may cost time, you can use multiple `-v` for verbose logs.
+The processing may cost time, you can use multiple `-v` for verbose logs (which are outputed to stderr).
 
 ```sh
 docker run aexpy:latest -vvv extract click@8.1.3
 ```
 
-### Data
+### Interactive
 
-You can mount cache directory to `/data` to save the processed data. AexPy will use the cache data if it exists, and produce results in JSON format under the cache directory.
+Add `-i` or `--interact` to enable interactive mode, every command will create an interactive Python shell after finishing processing. Here are some useful variable you could use in the interactive Python shell.
 
-```sh
-docker run -v /path/to/cache:/data aexpy:latest extract click@8.1.3
+- `result`: The produced data object
+- `context`: The producing context, use `exception` to access the exception if failing to process
 
-cat /path/to/cache/extracting/types/click/8.1.3.json
-```
+> Feel free to use `locals()` and `dir()` to explore the interactive environment.
 
 ### Pipeline
 
-AexPy has four stages in its pipeline, use the following commands to run the corresponding stage.
-
-```sh
-aexpy preprocess coxbuild@0.0.1
-aexpy extract coxbuild@0.0.1
-aexpy diff coxbuild@0.0.1:0.0.2
-aexpy report coxbuild@0.0.1:0.0.2
-```
-
-The four stages are loosely coupled. The adjacent stages transfer data by JSON, defined in [models](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/models/) directory. You can easily write your own implementation for every stage, and combine your implementation into the pipeline. See [third](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/third/) directory for an example on how to implement stages and integrate other tools.
+AexPy has four loosely-coupled stages in its pipeline. The adjacent stages transfer data by JSON, defined in [models](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/models/) directory. You can easily write your own implementation for every stage, and combine your implementation into the pipeline. See [third](https://github.com/StardustDL/aexpy/blob/main/src/aexpy/third/) directory for an example on how to implement stages and integrate other tools.
diff --git a/src/aexpy/__main__.py b/src/aexpy/__main__.py
@@ -1,11 +1,13 @@
 import code
+import json
 import logging
 import pathlib
 from pathlib import Path
 import sys
-from typing import IO, Literal
+from typing import IO, Annotated, Literal
 
 import click
+from pydantic import Field
 
 from aexpy.models import ProduceState
 from aexpy.caching import (
@@ -310,6 +312,9 @@ def report(difference: IO[str], report: IO[str]):
     result = context.product
     StreamWriterProduceCache(report).save(result, context.log)
 
+    print(result.overview(), file=sys.stderr)
+    print(f"\n{result.content}", file=sys.stderr)
+
     if FLAG_interact:
         code.interact(banner="", local=locals())
 
@@ -324,18 +329,24 @@ def view(file: IO[str]):
     Supports distribution, api-description, api-difference, and report file (in json format).
     """
 
-    from pydantic import TypeAdapter
-
     cache = StreamReaderProduceCache(file)
 
     try:
-        result = TypeAdapter(
-            Distribution | ApiDescription | ApiDifference | Report
-        ).validate_json(cache.raw())
+        data = json.loads(cache.raw())
+        if "release" in data:
+            result = Distribution.model_validate(data)
+        elif "distribution" in data:
+            result = ApiDescription.model_validate(data)
+        elif "entries" in data:
+            result = ApiDifference.model_validate(data)
+        else:
+            result = Report.model_validate(data)
     except Exception as ex:
         assert False, f"Failed to load data: {ex}"
 
-    print(result.overview(), file=sys.stderr)
+    print(result.overview())
+    if isinstance(result, Report):
+        print(f"\n{result.content}")
 
     if FLAG_interact:
         code.interact(banner="", local=locals())

diff --git a/src/aexpy/diffing/differs/checkers.py b/src/aexpy/diffing/differs/checkers.py
@@ -1,4 +1,3 @@
-import dataclasses
 from dataclasses import dataclass, field
 import functools
 from typing import Any, Callable, Literal, TypeVar, Type, cast, TypeGuard, overload
@@ -67,7 +66,7 @@ def __call__(
         result = self.checker(old, new, oldCollection, newCollection)
         return (
             [
-                dataclasses.replace(entry, kind=self.kind, old=old, new=new)
+                entry.model_copy(update={"kind": self.kind, "old": old, "new": new})
                 for entry in result
             ]
             if result

diff --git a/src/aexpy/diffing/differs/default.py b/src/aexpy/diffing/differs/default.py
@@ -55,7 +55,7 @@ def _processEntry(
         result = []
         for constraint in self.constraints:
             try:
-                done: "list[DiffEntry]" = constraint(
+                done: list[DiffEntry] = constraint(
                     old, new, oldDescription, newDescription
                 )
                 if done:

diff --git a/src/aexpy/preprocessing/download.py b/src/aexpy/preprocessing/download.py
@@ -43,7 +43,7 @@ def check(s: str):
         )
 
     for item in glob(".whl"):
-        logger.info(f"Remove downloaded {item}.")
+        logger.warning(f"Remove downloaded {item}.")
         os.remove(item)
 
     for pyversion in PYVERSIONS: