From a3e17bb0d8f8dcf998d074663680bf4340eef2be Mon Sep 17 00:00:00 2001 From: StardustDL Date: Sun, 28 Jan 2024 18:51:32 +0800 Subject: [PATCH] update docker build --- .github/dependabot.yml | 7 +- .github/workflows/ci.yml | 246 +++++++++++++++++++++------------------ Dockerfile | 38 +----- buildcox.py | 53 --------- 4 files changed, 136 insertions(+), 208 deletions(-) delete mode 100644 buildcox.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 6bec7450..cf554a8c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,12 +1,7 @@ version: 2 updates: - package-ecosystem: pip - directory: "/src" - schedule: - interval: daily - open-pull-requests-limit: 1 - - package-ecosystem: npm - directory: "/src/web" + directory: "/" schedule: interval: daily open-pull-requests-limit: 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6812f553..abd7b885 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - - uses: conda-incubator/setup-miniconda@v2 + - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: "3.12" @@ -27,87 +27,104 @@ jobs: python -m pip install build python -m build - name: Upload package artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: package path: ./dist - name: Prepare run: | + python -m pip install ./dist/*.whl + aexpy --help mkdir -p ./cache - name: Test Preprocess run: | - cd src - python -u -m aexpy -vvv preprocess -r -p generator-oj-problem@0.0.1 ../cache ../cache/distribution1.json - python -u -m aexpy -vvv preprocess -r -p generator-oj-problem@0.0.2 ../cache ../cache/distribution2.json + aexpy -vvv preprocess -r -p generator-oj-problem@0.0.1 ./cache ./cache/distribution1.json + aexpy -vvv preprocess -r -p generator-oj-problem@0.0.2 ./cache ./cache/distribution2.json - name: Test Extraction continue-on-error: true run: | - cd src - python -u -m aexpy -vvv extract ../cache/distribution1.json ../cache/api1.json - python -u -m aexpy -vvv extract ../cache/distribution2.json ../cache/api2.json + aexpy -vvv extract ./cache/distribution1.json ./cache/api1.json + aexpy -vvv extract ./cache/distribution2.json ./cache/api2.json - name: Test Difference continue-on-error: true run: | - cd src - python -u -m aexpy -vvv diff ../cache/api1.json ../cache/api1.json ../cache/diff.json + aexpy -vvv diff ./cache/api1.json ./cache/api1.json ./cache/diff.json - name: Test Report continue-on-error: true run: | - cd src - python -u -m aexpy -vvv report ../cache/diff.json ../cache/report.json + aexpy -vvv report ./cache/diff.json ./cache/report.json - name: Test View continue-on-error: true run: | - cd src - python -u -m aexpy -vvv view ../cache/distribution1.json - python -u -m aexpy -vvv view ../cache/distribution2.json - python -u -m aexpy -vvv view ../cache/api1.json - python -u -m aexpy -vvv view ../cache/api2.json - python -u -m aexpy -vvv view ../cache/diff.json - python -u -m aexpy -vvv view ../cache/report.json + aexpy -vvv view ./cache/distribution1.json + aexpy -vvv view ./cache/distribution2.json + aexpy -vvv view ./cache/api1.json + aexpy -vvv view ./cache/api2.json + aexpy -vvv view ./cache/diff.json + aexpy -vvv view ./cache/report.json - name: Upload results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: package-cache path: ./cache - # image: - # runs-on: ubuntu-latest - # steps: - # - name: Checkout - # uses: actions/checkout@v3 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v4 - # with: - # python-version: '3.12' - # architecture: 'x64' - # - name: Setup coxbuild - # run: pip install coxbuild - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v2 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - # - name: Build - # run: | - # docker build -t aexpy/aexpy . - # mkdir cache - # docker run aexpy/aexpy --help - # - name: Test Extraction - # run: docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv extract click@8.1.3 - # - name: Test Report 1 - # run: docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv report jinja2@3.1.1:3.1.2 - # - name: Test Report 2 - # run: docker run -v ${{ github.workspace }}/cache:/data aexpy/aexpy -vvv report coxbuild@0.1.5:0.1.6 - # - name: Upload results - # uses: actions/upload-artifact@v3 - # with: - # name: image-cache - # path: ./cache + image: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v4 + with: + python-version: '3.12' + architecture: 'x64' + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build + run: | + docker build -t aexpy/aexpy . + - name: Prepare + run: | + docker run aexpy/aexpy --help + mkdir -p ./cache + - name: Test Preprocess + run: | + docker run -v ${{ github.workspace }}/cache:/data -vvv preprocess -r -p generator-oj-problem@0.0.1 /data /data/distribution1.json + docker run -v ${{ github.workspace }}/cache:/data -vvv preprocess -r -p generator-oj-problem@0.0.2 /data /data/distribution2.json + - name: Test Extraction + continue-on-error: true + run: | + docker run -v ${{ github.workspace }}/cache:/data -vvv extract /data/distribution1.json /data/api1.json + docker run -v ${{ github.workspace }}/cache:/data -vvv extract /data/distribution2.json /data/api2.json + - name: Test Difference + continue-on-error: true + run: | + docker run -v ${{ github.workspace }}/cache:/data -vvv diff /data/api1.json /data/api1.json /data/diff.json + - name: Test Report + continue-on-error: true + run: | + docker run -v ${{ github.workspace }}/cache:/data -vvv report /data/diff.json /data/report.json + - name: Test View + continue-on-error: true + run: | + docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/distribution1.json + docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/distribution2.json + docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/api1.json + docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/api2.json + docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/diff.json + docker run -v ${{ github.workspace }}/cache:/data -vvv view /data/report.json + - name: Upload results + uses: actions/upload-artifact@v4 + with: + name: image-cache + path: ./cache docs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false - uses: actions/setup-python@v4 @@ -121,65 +138,66 @@ jobs: cd docs paperead build - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 + with: + name: docs + path: ./docs/dist + deploy: + if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' || github.event_name == 'release' }} + needs: [image, docs, package] + runs-on: ubuntu-latest + steps: + - name: Download artifacts + uses: actions/download-artifact@v4 with: name: docs path: ./docs/dist - # deploy: - # if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' || github.event_name == 'release' }} - # needs: [image, docs, package] - # runs-on: ubuntu-latest - # steps: - # - name: Download artifacts - # uses: actions/download-artifact@v3 - # with: - # name: docs - # path: ./docs/dist - # - name: Deploy docs to netlify - # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - # uses: netlify/actions/cli@master - # env: - # NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} - # NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} - # with: - # args: deploy --dir=./docs/dist --prod - # secrets: '["NETLIFY_AUTH_TOKEN", "NETLIFY_SITE_ID"]' - # - name: Download package artifacts - # uses: actions/download-artifact@v3 - # with: - # name: package - # path: ./dist - # - name: Deploy packages - # if: ${{ github.event_name == 'release' }} - # env: - # PYTHONUTF8: 1 - # TWINE_USERNAME: '__token__' - # TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - # run: | - # python -m pip install --upgrade build twine - # python -m twine upload --skip-existing --repository pypi "dist/*" - # - name: Set up QEMU - # uses: docker/setup-qemu-action@v2 - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v2 - # - name: Login to DockerHub - # uses: docker/login-action@v2 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # - name: Image Metadata - # id: meta - # uses: docker/metadata-action@v4 - # with: - # images: stardustdl/aexpy - # - name: Set BUILD_DATE - # run: echo "BUILD_DATE=$(date -Ins)" >> $GITHUB_ENV - # - name: Deploy image - # uses: docker/build-push-action@v3 - # with: - # push: ${{ github.event_name == 'push' }} - # tags: ${{ steps.meta.outputs.tags }} - # labels: ${{ steps.meta.outputs.labels }} - # build-args: | - # GIT_COMMIT=${{ github.sha }} - # BUILD_DATE=${{ env.BUILD_DATE }} \ No newline at end of file + - name: Deploy docs to netlify + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + uses: netlify/actions/cli@master + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} + NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }} + with: + args: deploy --dir=./docs/dist --prod + secrets: '["NETLIFY_AUTH_TOKEN", "NETLIFY_SITE_ID"]' + - name: Download package artifacts + uses: actions/download-artifact@v3 + with: + name: package + path: ./dist + - name: Deploy packages + if: ${{ github.event_name == 'release' }} + env: + PYTHONUTF8: 1 + TWINE_USERNAME: '__token__' + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + python -m pip install --upgrade twine + python -m twine upload --skip-existing --repository pypi "dist/*" + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Image Metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: stardustdl/aexpy + - name: Set BUILD_DATE + run: echo "BUILD_DATE=$(date -Ins)" >> $GITHUB_ENV + - name: Deploy image + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + GIT_COMMIT=${{ github.sha }} + BUILD_DATE=${{ env.BUILD_DATE }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 236c8b26..40035099 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,50 +1,18 @@ -FROM node:16 - -WORKDIR /app/web - -COPY ./src/web/package*.json /app/web/ - -RUN npm --registry=https://registry.npmmirror.com ci - -COPY ./src/web /app/web - -RUN npm run build - FROM continuumio/miniconda3 ENV PYTHONUTF8=1 -ENV RUN_IN_DOCKER=1 - -ENV AEXPY_CACHE=/data -ENV AEXPY_CONFIG=/config/config.yml WORKDIR /app -EXPOSE 8008 VOLUME [ "/data" ] +RUN conda create -n main python=3.12 -qy +COPY ./pyproject.toml /app/pyproject.toml - - - - - - -RUN conda create -n main python=3.10 -qy && \ - conda create -n py39 python=3.9 -qy && \ - conda create -n py38 python=3.8 -qy && \ - conda create -n py37 python=3.7 -qy - -COPY ./src/requirements.txt /app/requirements.txt - -RUN [ "conda", "run", "-n", "main", "--no-capture-output", "python", "-u", "-m", "pip", "install", "-r", "/app/requirements.txt" ] +RUN [ "conda", "run", "-n", "main", "--no-capture-output", "python", "-u", "-m", "pip", "install", "-r", "/app/pyproject.toml" ] COPY ./src/aexpy /app/aexpy -RUN [ "conda", "run", "-n", "main", "--no-capture-output", "python", "-u", "-m", "aexpy", "initialize"] - -COPY --from=0 /app/web/dist /app/aexpy/serving/server/wwwroot - ARG GIT_COMMIT ENV GIT_COMMIT=${GIT_COMMIT:-unknown} diff --git a/buildcox.py b/buildcox.py deleted file mode 100644 index 28f57245..00000000 --- a/buildcox.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -from coxbuild.schema import task, group, named, run, depend, ext, withExecutionState, ExecutionState, withProject, ProjectSettings -from coxbuild.extensions.python import format as pyformat, package as pypackage - - -ext("file://build/build.py") -ext("file://build/data.py") -ext("file://build/test.py") -ext("file://build/exps.py") - - -@depend(pyformat.restore) -@withProject -@task -async def format(project: "ProjectSettings"): - await pyformat.autopep8(project.src.joinpath('aexpy')) - await pyformat.isort(project.src.joinpath('aexpy')) - - -@depend(pypackage.restore) -@task -def restore(): - pass - - -@named("run") -@withExecutionState -@task -def runexe(executionState: "ExecutionState"): - run(["python", "-u", "-m", "aexpy", "-c", "../../aexpy-exps", - *executionState.unmatchedTasks], cwd=Path("src")) - - -@named("done") -@withExecutionState -@task -def done(executionState: "ExecutionState"): - run(["python", "-u", "-m", "aexpy", "-c", "../results", "-C", - *executionState.unmatchedTasks], cwd=Path("src")) - - -@named("serve:docker") -@task -def serve_docker(): - run(["docker", "run", "--rm", "-d", - "-p", "50036:8008", - "-v", "/var/run/docker.sock:/var/run/docker.sock", - "-v", "/home/test/liang/aexpy-exps:/data", - "-m", "20g", - "--name", "aexpy", - "aexpy/aexpy", "-vv", - "serve", "-d", - "-u", "star", "-P", "truth"])