support custom env for extraction

.github/workflows/ci.yml

@@ -47,6 +47,10 @@ jobs:
       run: |
         aexpy -vvv extract ./cache/distribution1.json ./cache/api1.json
         aexpy -vvv extract ./cache/distribution2.json ./cache/api2.json
+    - name: Test Extraction in Env
+      continue-on-error: false
+      run: |
+        aexpy -vvv extract ./cache/distribution1.json ./cache/api3.json -e -
     - name: Test Difference
       continue-on-error: false
       run: |
@@ -62,6 +66,7 @@ jobs:
         aexpy -vvv view ./cache/distribution2.json
         aexpy -vvv view ./cache/api1.json
         aexpy -vvv view ./cache/api2.json
+        aexpy -vvv view ./cache/api3.json
         aexpy -vvv view ./cache/diff.json
         aexpy -vvv view ./cache/report.json
     - name: Upload results
@@ -102,6 +107,10 @@ jobs:
       run: |
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution1.json /data/api1.json
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution2.json /data/api2.json
+    - name: Test Extraction in Env
+      continue-on-error: false
+      run: |
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract /data/distribution1.json /data/api3.json -e -
     - name: Test Difference
       continue-on-error: false
       run: |
@@ -117,6 +126,7 @@ jobs:
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/distribution2.json
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api1.json
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api2.json
+        docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/api3.json
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/diff.json
         docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv view /data/report.json
     - name: Upload results

README.md

@@ -141,9 +141,13 @@ aexpy preprocess ./cache/generator_oj_problem-0.0.1-py3-none-any ./cache/distrib
 
 Extract the API description from a distribution.
 
-> AexPy would dynamically import the target module to detect all available APIs,
-> so please ensure all dependencies have been installed in the current Python environment,
-> or specify the `dependencies` field in the distribution, and AexPy will install them into current Python environment
+AexPy would dynamically import the target module to detect all available APIs. So please ensure all dependencies have been installed in the extraction environment, or specify the `dependencies` field in the distribution, and AexPy will install them into the extraction environment.
+
+Use option `-e`, `--env` to specify a conda env name as the extraction environment.
+
+- Keep empty (default) for using the current Python environment (as same as AexPy).
+- Set to `-` to let AexPy create a temporary conda environment that matches the distribution's pyverion field.
+- Set to other valeues, indicates a concrete existed conda environment name.
 
 ```sh
 aexpy extract ./cache/distribution.json ./cache/api.json
@@ -152,6 +156,11 @@ aexpy extract ./cache/distribution.json ./cache/api.json
 aexpy extract - ./cache/api.json
 # or output the api description file to stdout
 aexpy extract ./cache/distribution.json -
+
+# Use a conda env named demo-env
+aexpy extract ./cache/distribution.json - -e demo-env
+# Create a temporary conda env
+aexpy extract ./cache/distribution.json - -e -
 ```
 
 ### Diff

src/aexpy/__main__.py

@@ -150,16 +150,25 @@ def preprocess(
     """Preprocess and generate a package distribution file.
 
     DISTRIBUTION describes the output package distribution file (in json format, use `-` for stdout).
+
     PATH describes the target path for each mode:
+
     mode=src, PATH points to the directory that contains the package code directory
+
     mode=dist, PATH points to the directory that contains the package code directory and the .dist-info directory
+
     mode=wheel, PATH points to the '.whl' file, which will be unpacked to the same directory as the file
+
     mode=release, PATH points to the target directory for downloading and unpacking
 
     Examples:
+
     aexpy preprocess -p aexpy@0.1.0 -r ./temp -
+
     aexpy preprocess -w ./temp/aexpy-0.1.0.whl -
+
     aexpy preprocess -d ./temp/aexpy-0.1.0 -
+
     aexpy preprocess ./temp/aexpy-0.1.0 -
     """
     from .models import Distribution
@@ -235,31 +244,38 @@ def preprocess(
 @main.command()
 @click.argument("distribution", type=click.File("r"))
 @click.argument("description", type=click.File("w"))
-@click.option("-e", "--env", type=str, default="", help="Conda env name, keep empty to use current environment.")
+@click.option("-e", "--env", type=str, default="", help="Conda env name, empty for current environment, - for new temp environment.")
 def extract(distribution: IO[str], description: IO[str], env: str = ""):
     """Extract the API in a distribution.
 
     DISTRIBUTION describes the input package distribution file (in json format, use `-` for stdin).
+
     DESCRIPTION describes the output API description file (in json format, use `-` for stdout).
 
     Examples:
+
     aexpy extract ./distribution1.json ./api1.json
     """
 
     data = StreamReaderProduceCache(distribution).data(Distribution)
     with produce(ApiDescription(distribution=data)) as context:
         from .extracting.default import DefaultExtractor
 
-        if env:
-            from .extracting.environment import ExtractorEnvironment
-            eenv = ExtractorEnvironment(env, context.logger)
+        if env == "":
+            from .environments import CurrentEnvironment, SingleExecutionEnvironmentBuilder
+            envBuilder = SingleExecutionEnvironmentBuilder(CurrentEnvironment(context.logger), context.logger)
+        elif env == "-":
+            from .extracting.environment import getExtractorEnvironmentBuilder
+            envBuilder = getExtractorEnvironmentBuilder(context.logger)
         else:
-            from .environments import CurrentEnvironment
-            eenv = CurrentEnvironment(context.logger)
+            from .environments import SingleExecutionEnvironmentBuilder
+            from .extracting.environment import getExtractorEnvironment
+            envBuilder = SingleExecutionEnvironmentBuilder(getExtractorEnvironment(env, context.logger), context.logger)
 
-        extractor = DefaultExtractor(env=eenv, logger=context.logger)
-        context.use(extractor)
-        extractor.extract(data, context.product)
+        with envBuilder.use(data.pyversion, context.logger) as eenv:
+            extractor = DefaultExtractor(env=eenv, logger=context.logger)
+            context.use(extractor)
+            extractor.extract(data, context.product)
 
     result = context.product
     StreamWriterProduceCache(description).save(result, context.log)
@@ -280,10 +296,13 @@ def diff(old: IO[str], new: IO[str], difference: IO[str]):
     """Diff the API description and find all changes.
 
     OLD describes the input API description file of the old distribution (in json format, use `-` for stdin).
+
     NEW describes the input API description file of the new distribution (in json format, use `-` for stdin).
+
     DIFFERENCE describes the output API difference file (in json format, use `-` for stdout).
 
     Examples:
+
     aexpy diff ./api1.json ./api2.json ./changes.json
     """
     oldData = StreamReaderProduceCache(old).data(ApiDescription)
@@ -316,9 +335,11 @@ def report(difference: IO[str], report: IO[str]):
     """Generate a report for the API difference file.
 
     DIFFERENCE describes the input API difference file (in json format, use `-` for stdin).
+
     REPORT describes the output report file (in json format, use `-` for stdout).
 
     Examples:
+
     aexpy report ./changes.json ./report.json
     """
 

src/aexpy/environments/__init__.py

@@ -1,6 +1,36 @@
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
 import subprocess
 import sys
 import logging
+from typing import override
+
+
+class ExecutionEnvironmentRunner:
+    def __init__(self, commandPrefix: str = "", pythonName: str = "python", **options) -> None:
+        self.commandPrefix = commandPrefix
+        self.pythonName = pythonName
+        self.options = options
+
+    def run(self, command: str, **kwargs) -> subprocess.CompletedProcess:
+        """Run a command in the environment."""
+
+        return subprocess.run(f"{self.commandPrefix} {command}", **kwargs, **self.options)
+
+    def runPython(self, command: str, **kwargs) -> subprocess.CompletedProcess:
+        """Run a command in the environment."""
+
+        return subprocess.run(f"{self.commandPrefix} {self.pythonName} {command}", **kwargs, **self.options)
+
+    def runText(self, command: str, **kwargs) -> subprocess.CompletedProcess[str]:
+        """Run a command in the environment."""
+
+        return subprocess.run(f"{self.commandPrefix} {command}", **kwargs, **self.options, capture_output=True, text=True)
+
+    def runPythonText(self, command: str, **kwargs) -> subprocess.CompletedProcess[str]:
+        """Run a command in the environment."""
+
+        return subprocess.run(f"{self.commandPrefix} {self.pythonName} {command}", **kwargs, **self.options, capture_output=True, text=True)
 
 
 class ExecutionEnvironment:
@@ -12,32 +42,80 @@ def __init__(
         self.logger = logger or logging.getLogger("exe-env")
         """Python version of the environment."""
 
-    def run(self, command: str, **kwargs) -> subprocess.CompletedProcess:
-        """Run a command in the environment."""
+    def runner(self):
+        return ExecutionEnvironmentRunner()
 
-        return subprocess.run(command, **kwargs)
+    def __enter__(self):
+        self.logger.info(f"Enter the environment: {self=}")
+        return self.runner()
 
-    def runPython(self, command: str, **kwargs) -> subprocess.CompletedProcess:
-        """Run a command in the environment."""
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.logger.info(f"Exit the environment: {self=}")
 
-        return subprocess.run(f"python {command}", **kwargs)
 
-    def __enter__(self):
-        return self.run, self.runPython
+class ExecutionEnvironmentBuilder[T: ExecutionEnvironment](ABC):
+    """Builder to create environment that runs extractor code."""
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __init__(
+        self, logger: logging.Logger | None = None
+    ) -> None:
+        self.logger = logger or logging.getLogger("exe-env-builder")
+
+    @abstractmethod
+    def build(self, pyversion: str = "3.12", logger: logging.Logger | None = None) -> T:
         pass
 
+    @abstractmethod
+    def clean(self, env: T):
+        pass
+
+    @contextmanager
+    def use(self, pyversion: str = "3.12", logger: logging.Logger | None = None):
+        logger = logger or self.logger.getChild("sub-env")
+        self.logger.info(f"Build env {pyversion=}")
+        try:
+            env = self.build(pyversion=pyversion, logger=logger)
+        except Exception as ex:
+            self.logger.error(f"Failed to create env {pyversion=}", exc_info=ex)
+            raise
+        self.logger.info(f"Built env {pyversion=}, {env=}")
+
+        self.logger.info(f"Use env {pyversion=}, {env=}")
+        try:
+            yield env
+        except Exception as ex:
+            self.logger.error(f"Error occurs when using env {env=}", exc_info=ex)
+            raise
+        finally:
+            self.logger.info(f"Used env {pyversion=}, {env=}")
+            self.logger.info(f"Clean env {pyversion=}, {env=}")
+            try:
+                self.clean(env)
+            except Exception as ex:
+                self.logger.error(f"Failed to clean env {pyversion=}, {env=}", exc_info=ex)
+                raise
+            self.logger.info(f"Cleaned env {pyversion=}, {env=}")
+
 
 class CurrentEnvironment(ExecutionEnvironment):
     """Use the same environment for extractor."""
 
-    def run(self, command: str, **kwargs):
-        """Run a command in the environment."""
+    @override
+    def runner(self):
+        return ExecutionEnvironmentRunner(pythonName=sys.executable)
 
-        return subprocess.run(command, **kwargs, shell=True)
 
-    def runPython(self, command: str, **kwargs):
-        """Run a command in the environment."""
+class SingleExecutionEnvironmentBuilder[T: ExecutionEnvironment](ExecutionEnvironmentBuilder[T]):
+    def __init__(
+        self, env: T, logger: logging.Logger | None = None
+    ) -> None:
+        super().__init__(logger=logger)
+        self.env = env
+
+    @override
+    def build(self, pyversion = "3.12", logger = None):
+        return self.env
 
-        return self.run(f"{sys.executable} {command}", **kwargs)
+    @override
+    def clean(self, env: T):
+        pass