diff --git a/.editorconfig b/.editorconfig index d6eafe8d8f..7e5ce6236a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -50,3 +50,9 @@ indent_size = 2 [ietf/**.html] insert_final_newline = false + +# Settings for Kubernetes yaml +# --------------------------------------------------------- +# Use 2-space indents +[k8s/**.yaml] +indent_size = 2 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ba25ff55e..99c6c9ab34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,15 +17,15 @@ on: default: true required: true type: boolean - sandboxNoDbRefresh: - description: 'Sandbox Disable Daily DB Refresh' + deploy: + description: 'Deploy to Staging / Prod' default: false required: true type: boolean - legacySandbox: - description: 'Deploy to Legacy Sandbox' + sandboxNoDbRefresh: + description: 'Sandbox Disable Daily DB Refresh' default: false - required: false + required: true type: boolean skiptests: description: 'Skip Tests' @@ -161,7 +161,7 @@ jobs: - name: Download a Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.1.4 + uses: actions/download-artifact@v4.1.7 with: name: coverage @@ -220,7 +220,7 @@ jobs: .devcontainer .github .vscode - helm + k8s playwright svn-history docker-compose.yml @@ -323,7 +323,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.25.0 + uses: slackapi/slack-github-action@v1.26.0 with: channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} payload: | @@ -346,7 +346,7 @@ jobs: SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }} - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v1.25.0 + uses: slackapi/slack-github-action@v1.26.0 with: channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }} payload: | @@ -385,7 +385,7 @@ jobs: - uses: actions/checkout@v4 - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.4 + uses: actions/download-artifact@v4.1.7 with: name: release-${{ env.PKG_VERSION }} @@ -407,62 +407,39 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | docker image prune -a -f - - legacySandbox: - name: Deploy to Legacy Sandbox - if: ${{ !failure() && !cancelled() && github.event.inputs.legacySandbox == 'true' }} + + # ----------------------------------------------------------------- + # STAGING + # ----------------------------------------------------------------- + staging: + name: Deploy to Staging + if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }} needs: [prepare, release] - runs-on: [self-hosted, legacy-sandbox-server] + runs-on: ubuntu-latest environment: - name: legacy-sandbox - url: "https://sandbox.ietf.org" + name: staging env: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.4 - with: - name: release-${{ env.PKG_VERSION }} - path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} + - name: Deploy to staging + run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}' - - name: Extract Release - env: - DEBIAN_FRONTEND: noninteractive - working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} - run: | - echo "Extracting release tarball..." - tar xzf release.tar.gz - echo "Deleting release tarball..." - rm -rf release.tar.gz - - - name: Setup Environment - env: - DEBIAN_FRONTEND: noninteractive - working-directory: /a/www/ietf-datatracker/main.dev.${{ github.run_number }} - run: | - echo "Copying settings from previous deploy..." - cp ../web/ietf/settings_local.py ietf/ - rsync -a ../web/test/ test/ - echo "Installing Python dependencies..." - python3.9 -mvenv env - source env/bin/activate - pip install -r requirements.txt - pip freeze > frozen-requirements.txt - echo "Collecting static..." - ietf/manage.py collectstatic - echo "Running checks..." - ietf/manage.py check - - - name: Update Docker Containers - env: - DEBIAN_FRONTEND: noninteractive - working-directory: /a/docker/datatracker - run: | - echo "Pulling latest docker images..." - docker image tag ghcr.io/ietf-tools/datatracker-celery:latest datatracker-celery-fallback - docker image tag ghcr.io/ietf-tools/datatracker-mq:latest datatracker-mq-fallback - docker-compose pull - # echo "Shutting down containers..." - # docker-compose down -t 300 - + # ----------------------------------------------------------------- + # PROD + # ----------------------------------------------------------------- + prod: + name: Deploy to Production + if: ${{ !failure() && !cancelled() && github.event.inputs.deploy == 'true' }} + needs: [staging] + runs-on: ubuntu-latest + environment: + name: production + env: + PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} + + steps: + - name: Deploy to production + run: | + curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${{ secrets.GH_INFRA_K8S_TOKEN }}" ${{ secrets.GHA_K8S_DEPLOY_API }} -d '{"ref":"main", "inputs": { "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}" }}' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dbac84cda5..f6d54b14bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -59,7 +59,7 @@ jobs: path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v4.2.0 + uses: codecov/codecov-action@v4.3.1 with: files: coverage.xml diff --git a/bin/daily b/bin/daily index 8211e1e237..6adb16794d 100755 --- a/bin/daily +++ b/bin/daily @@ -17,16 +17,10 @@ cd $DTDIR/ logger -p user.info -t cron "Running $DTDIR/bin/daily" -# Run the hourly jobs first -$DTDIR/bin/hourly - # Set up the virtual environment source $DTDIR/env/bin/activate -# Update our information about the current version of some commands we use -$DTDIR/ietf/manage.py update_external_command_info - # Get IANA-registered yang models #YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR') # Hardcode the rsync target to avoid any unwanted deletes: @@ -43,9 +37,5 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0 # Re-run yang checks on active documents $DTDIR/ietf/manage.py run_yang_model_checks -v0 -# Expire last calls -# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner: -$DTDIR/ietf/bin/expire-last-calls - # Purge older PersonApiKeyEvents $DTDIR/ietf/manage.py purge_old_personal_api_key_events 14 diff --git a/bin/dump-to-names-json b/bin/dump-to-names-json index 9c7dfac07d..20d4e0f951 100644 --- a/bin/dump-to-names-json +++ b/bin/dump-to-names-json @@ -10,7 +10,7 @@ set -x ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \ - mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \ + mailtrigger.MailTrigger mailtrigger.Recipient name \ group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \ | jq --sort-keys "sort_by(.model, .pk)" \ | jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json diff --git a/bin/hourly b/bin/hourly deleted file mode 100755 index 9478bec119..0000000000 --- a/bin/hourly +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -# Hourly datatracker jobs -# -# This script is expected to be triggered by cron from -# /etc/cron.d/datatracker -export LANG=en_US.UTF-8 -export PYTHONIOENCODING=utf-8 - -# Make sure we stop if something goes wrong: -program=${0##*/} -trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR - -DTDIR=/a/www/ietf-datatracker/web -cd $DTDIR/ - -# Set up the virtual environment -source $DTDIR/env/bin/activate - -logger -p user.info -t cron "Running $DTDIR/bin/hourly" - -# Generate some static files -ID=/a/ietfdata/doc/draft/repository -DERIVED=/a/ietfdata/derived -DOWNLOAD=/a/www/www6s/download - -$DTDIR/ietf/manage.py generate_idnits2_rfc_status -$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted - -CHARTER=/a/www/ietf-ftp/charter -wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt -wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt - -# Regenerate the last week of bibxml-ids -$DTDIR/ietf/manage.py generate_draft_bibxml_files - -# Create and update group wikis -#$DTDIR/ietf/manage.py create_group_wikis - -# exit 0 diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index 2ffce35495..12f73300dd 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,17 +1,39 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:latest -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -COPY . . -COPY ./dev/build/start.sh ./start.sh -RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt -RUN chmod +x start.sh && \ - chmod +x docker/scripts/app-create-dirs.sh && \ - sh ./docker/scripts/app-create-dirs.sh - -VOLUME [ "/assets" ] - -EXPOSE 8000 - -CMD ["./start.sh"] \ No newline at end of file +FROM ghcr.io/ietf-tools/datatracker-app-base:latest +LABEL maintainer="IETF Tools Team " + +ENV DEBIAN_FRONTEND=noninteractive + +# uid 498 = wwwrun and gid 496 = www on ietfa +RUN groupadd -g 1000 datatracker && \ + useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker + +RUN apt-get purge -y imagemagick imagemagick-6-common + +# Install libreoffice (needed via PPT2PDF_COMMAND) +RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \ + apt-get update && \ + apt-get -qyt bullseye-backports install libreoffice-nogui + +COPY . . +COPY ./dev/build/start.sh ./start.sh +COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh +COPY ./dev/build/celery-start.sh ./celery-start.sh + +RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \ + echo '# empty' > ietf/settings_local.py && \ + ietf/manage.py patch_libraries && \ + rm -f ietf/settings_local.py + +RUN chmod +x start.sh && \ + chmod +x datatracker-start.sh && \ + chmod +x celery-start.sh && \ + chmod +x docker/scripts/app-create-dirs.sh && \ + sh ./docker/scripts/app-create-dirs.sh + +RUN mkdir -p /a + +VOLUME [ "/a" ] + +EXPOSE 8000 + +CMD ["./start.sh"] diff --git a/dev/build/celery-start.sh b/dev/build/celery-start.sh new file mode 100644 index 0000000000..c229defa2f --- /dev/null +++ b/dev/build/celery-start.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# +# Run a celery worker +# +echo "Running Datatracker checks..." +./ietf/manage.py check + +cleanup () { + # Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit. + if [[ -n "${celery_pid}" ]]; then + echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..." + kill -TERM "${celery_pid}" + wait "${celery_pid}" + fi +} + +trap 'trap "" TERM; cleanup' TERM + +# start celery in the background so we can trap the TERM signal +celery "$@" & +celery_pid=$! +wait "${celery_pid}" diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh new file mode 100644 index 0000000000..390b46af86 --- /dev/null +++ b/dev/build/datatracker-start.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo "Running Datatracker checks..." +./ietf/manage.py check + +echo "Running Datatracker migrations..." +./ietf/manage.py migrate --settings=settings_local + +echo "Starting Datatracker..." + +gunicorn \ + --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ + --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ + --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ + --bind :8000 \ + --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ + ietf.wsgi:application diff --git a/dev/build/start.sh b/dev/build/start.sh index ef64ca7b30..f20d264beb 100644 --- a/dev/build/start.sh +++ b/dev/build/start.sh @@ -1,10 +1,20 @@ #!/bin/bash - -echo "Running Datatracker checks..." -./ietf/manage.py check - -echo "Running Datatracker migrations..." -./ietf/manage.py migrate --settings=settings_local - -echo "Starting Datatracker..." -./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local +# +# Environment config: +# +# CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker) +# +case "${CONTAINER_ROLE:-datatracker}" in + datatracker) + exec ./datatracker-start.sh + ;; + celery) + exec ./celery-start.sh --app=ietf worker + ;; + beat) + exec ./celery-start.sh --app=ietf beat + ;; + *) + echo "Unknown role '${CONTAINER_ROLE}'" + exit 255 +esac diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 2912c98fdf..4aee7ba1ed 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -3,7 +3,7 @@ import Docker from 'dockerode' import path from 'path' import fs from 'fs-extra' -import tar from 'tar' +import * as tar from 'tar' import yargs from 'yargs/yargs' import { hideBin } from 'yargs/helpers' import slugify from 'slugify' diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index 6e6bd88e9b..550f8f072e 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -11,7 +11,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^6.2.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { @@ -23,6 +23,115 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -53,6 +162,11 @@ "safer-buffer": "~2.1.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -80,6 +194,14 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", @@ -137,6 +259,19 @@ "node": ">=10.0.0" } }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -180,6 +315,11 @@ "node": ">= 8.0" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -201,6 +341,21 @@ "node": ">=6" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -219,17 +374,6 @@ "node": ">=14.14" } }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -238,6 +382,27 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -275,6 +440,28 @@ "node": ">=8" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -286,38 +473,60 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "dependencies": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==", + "engines": { + "node": ">=16 || 14 >=14.17" } }, "node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" }, "engines": { - "node": ">= 8" + "node": ">= 18" } }, "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", "bin": { - "mkdirp": "bin/cmd.js" + "mkdirp": "dist/cjs/src/bin.js" }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mkdirp-classic": { @@ -366,6 +575,29 @@ "wrappy": "1" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -396,6 +628,23 @@ "node": ">=0.10.0" } }, + "node_modules/rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -420,6 +669,36 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/slugify": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", @@ -471,6 +750,20 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -482,20 +775,32 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.0", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/tar-fs": { @@ -558,19 +863,11 @@ } }, "node_modules/tar/node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "engines": { - "node": ">=10" - } - }, - "node_modules/tar/node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", "engines": { - "node": ">=8" + "node": ">=18" } }, "node_modules/tweetnacl": { @@ -591,6 +888,20 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -607,6 +918,23 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -621,9 +949,12 @@ } }, "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "engines": { + "node": ">=18" + } }, "node_modules/yargs": { "version": "17.7.2", @@ -657,6 +988,78 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "requires": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + }, + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "requires": { + "ansi-regex": "^6.0.1" + } + }, + "wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "requires": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + } + } + } + }, + "@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "requires": { + "minipass": "^7.0.4" + } + }, + "@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true + }, "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -678,6 +1081,11 @@ "safer-buffer": "~2.1.0" } }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -691,6 +1099,14 @@ "tweetnacl": "^0.14.3" } }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "buildcheck": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", @@ -735,6 +1151,16 @@ "nan": "^2.17.0" } }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -764,6 +1190,11 @@ "tar-fs": "~2.0.1" } }, + "eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -782,6 +1213,15 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, + "foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "requires": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + } + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -797,19 +1237,23 @@ "universalify": "^2.0.0" } }, - "fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "requires": { - "minipass": "^3.0.0" - } - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "requires": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + } + }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -830,6 +1274,20 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "requires": { + "@isaacs/cliui": "^8.0.2", + "@pkgjs/parseargs": "^0.11.0" + } + }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -839,27 +1297,37 @@ "universalify": "^2.0.0" } }, - "minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" + }, + "minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "requires": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" } }, + "minipass": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==" + }, "minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "requires": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" } }, "mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" }, "mkdirp-classic": { "version": "0.5.3", @@ -895,6 +1363,20 @@ "wrappy": "1" } }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "requires": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + } + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -919,6 +1401,14 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, + "rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "requires": { + "glob": "^10.3.7" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -929,6 +1419,24 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + }, "slugify": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", @@ -968,6 +1476,16 @@ "strip-ansi": "^6.0.1" } }, + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -976,28 +1494,31 @@ "ansi-regex": "^5.0.1" } }, + "strip-ansi-cjs": { + "version": "npm:strip-ansi@6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, "tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "requires": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.0", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "dependencies": { "chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" - }, - "minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==" } } }, @@ -1060,6 +1581,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1070,6 +1599,16 @@ "strip-ansi": "^6.0.0" } }, + "wrap-ansi-cjs": { + "version": "npm:wrap-ansi@7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1081,9 +1620,9 @@ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==" }, "yargs": { "version": "17.7.2", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index 01c2cc3d7e..6b134e5405 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -7,7 +7,7 @@ "nanoid": "5.0.7", "nanoid-dictionary": "5.0.0-beta.1", "slugify": "1.6.6", - "tar": "^6.2.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json index d4583ec051..d97649b39e 100644 --- a/dev/diff/package-lock.json +++ b/dev/diff/package-lock.json @@ -17,7 +17,7 @@ "lodash-es": "^4.17.21", "luxon": "^3.4.4", "pretty-bytes": "^6.1.1", - "tar": "^6.2.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { @@ -29,6 +29,115 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "dependencies": { + "minipass": "^7.0.4" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@sindresorhus/is": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz", @@ -123,6 +232,11 @@ "safer-buffer": "~2.1.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -150,6 +264,14 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -331,6 +453,19 @@ "node": ">=10.0.0" } }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -491,6 +626,32 @@ "pend": "~1.2.0" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data-encoder": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", @@ -517,17 +678,6 @@ "node": ">=14.14" } }, - "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -547,6 +697,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/got": { "version": "13.0.0", "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", @@ -625,6 +796,28 @@ "node": ">=8" } }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -858,6 +1051,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "engines": { + "node": "14 || >=16.14" + } + }, "node_modules/luxon": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", @@ -885,38 +1086,52 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "node_modules/minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "dependencies": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" }, "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==", + "engines": { + "node": ">=16 || 14 >=14.17" } }, "node_modules/minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "dependencies": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" }, "engines": { - "node": ">= 8" + "node": ">= 18" } }, "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", "bin": { - "mkdirp": "bin/cmd.js" + "mkdirp": "dist/cjs/src/bin.js" }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/mkdirp-classic": { @@ -976,6 +1191,29 @@ "node": ">=12.20" } }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -1072,6 +1310,23 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "node_modules/rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "dependencies": { + "glob": "^10.3.7" + }, + "bin": { + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1096,6 +1351,25 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -1181,6 +1455,20 @@ "node": ">=8" } }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1192,20 +1480,32 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "dependencies": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.0", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" } }, "node_modules/tar-fs": { @@ -1268,19 +1568,11 @@ } }, "node_modules/tar/node_modules/chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "engines": { - "node": ">=10" - } - }, - "node_modules/tar/node_modules/minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==", "engines": { - "node": ">=8" + "node": ">=18" } }, "node_modules/tweetnacl": { @@ -1312,6 +1604,20 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1328,6 +1634,23 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1342,9 +1665,12 @@ } }, "node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==", + "engines": { + "node": ">=18" + } }, "node_modules/yargs": { "version": "17.7.2", @@ -1387,6 +1713,78 @@ "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" }, + "@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "requires": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" + }, + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "requires": { + "ansi-regex": "^6.0.1" + } + }, + "wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "requires": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + } + } + } + }, + "@isaacs/fs-minipass": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", + "integrity": "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==", + "requires": { + "minipass": "^7.0.4" + } + }, + "@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true + }, "@sindresorhus/is": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz", @@ -1454,6 +1852,11 @@ "safer-buffer": "~2.1.0" } }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1467,6 +1870,14 @@ "tweetnacl": "^0.14.3" } }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "requires": { + "balanced-match": "^1.0.0" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -1592,6 +2003,16 @@ "nan": "^2.17.0" } }, + "cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, "debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -1707,6 +2128,22 @@ "pend": "~1.2.0" } }, + "foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "requires": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "dependencies": { + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" + } + } + }, "form-data-encoder": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", @@ -1727,14 +2164,6 @@ "universalify": "^2.0.0" } }, - "fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "requires": { - "minipass": "^3.0.0" - } - }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -1745,6 +2174,18 @@ "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==" }, + "glob": { + "version": "10.3.12", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", + "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", + "requires": { + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.6", + "minimatch": "^9.0.1", + "minipass": "^7.0.4", + "path-scurry": "^1.10.2" + } + }, "got": { "version": "13.0.0", "resolved": "https://registry.npmjs.org/got/-/got-13.0.0.tgz", @@ -1797,6 +2238,20 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, + "isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "requires": { + "@isaacs/cliui": "^8.0.2", + "@pkgjs/parseargs": "^0.11.0" + } + }, "json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -1949,6 +2404,11 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz", "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==" }, + "lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" + }, "luxon": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.4.4.tgz", @@ -1964,27 +2424,32 @@ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz", "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==" }, - "minipass": { - "version": "3.3.4", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", - "integrity": "sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==", + "minimatch": { + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "requires": { - "yallist": "^4.0.0" + "brace-expansion": "^2.0.1" } }, + "minipass": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.0.tgz", + "integrity": "sha512-oGZRv2OT1lO2UF1zUcwdTb3wqUwI0kBGTgt/T7OdSj6M6N5m3o5uPf0AIW6lVxGGoiWUR7e2AwTE+xiwK8WQig==" + }, "minizlib": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", - "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", + "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", "requires": { - "minipass": "^3.0.0", - "yallist": "^4.0.0" + "minipass": "^7.0.4", + "rimraf": "^5.0.5" } }, "mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", + "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" }, "mkdirp-classic": { "version": "0.5.3", @@ -2028,6 +2493,20 @@ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==" }, + "path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" + }, + "path-scurry": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", + "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", + "requires": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + } + }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -2094,6 +2573,14 @@ "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" }, + "rimraf": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", + "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", + "requires": { + "glob": "^10.3.7" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2104,6 +2591,19 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" + }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -2164,6 +2664,16 @@ "strip-ansi": "^6.0.1" } }, + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -2172,28 +2682,31 @@ "ansi-regex": "^5.0.1" } }, + "strip-ansi-cjs": { + "version": "npm:strip-ansi@6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, "tar": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", - "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.1.0.tgz", + "integrity": "sha512-ENhg4W6BmjYxl8GTaE7/h99f0aXiSWv4kikRZ9n2/JRxypZniE84ILZqimAhxxX7Zb8Px6pFdheW3EeHfhnXQQ==", "requires": { - "chownr": "^2.0.0", - "fs-minipass": "^2.0.0", - "minipass": "^5.0.0", - "minizlib": "^2.1.1", - "mkdirp": "^1.0.3", - "yallist": "^4.0.0" + "@isaacs/fs-minipass": "^4.0.0", + "chownr": "^3.0.0", + "minipass": "^7.1.0", + "minizlib": "^3.0.1", + "mkdirp": "^3.0.1", + "yallist": "^5.0.0" }, "dependencies": { "chownr": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", - "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" - }, - "minipass": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", - "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==" + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", + "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==" } } }, @@ -2261,6 +2774,14 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "requires": { + "isexe": "^2.0.0" + } + }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -2271,6 +2792,16 @@ "strip-ansi": "^6.0.0" } }, + "wrap-ansi-cjs": { + "version": "npm:wrap-ansi@7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -2282,9 +2813,9 @@ "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz", + "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==" }, "yargs": { "version": "17.7.2", diff --git a/dev/diff/package.json b/dev/diff/package.json index 99661c3846..dcc6e9eb7b 100644 --- a/dev/diff/package.json +++ b/dev/diff/package.json @@ -13,7 +13,7 @@ "lodash-es": "^4.17.21", "luxon": "^3.4.4", "pretty-bytes": "^6.1.1", - "tar": "^6.2.1", + "tar": "^7.1.0", "yargs": "^17.7.2" }, "engines": { diff --git a/helm/.helmignore b/helm/.helmignore deleted file mode 100644 index 2252590f2e..0000000000 --- a/helm/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ \ No newline at end of file diff --git a/helm/Chart.yaml b/helm/Chart.yaml deleted file mode 100644 index 1c19834c06..0000000000 --- a/helm/Chart.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v2 -name: datatracker -description: The day-to-day front-end to the IETF database for people who work on IETF standards. -home: https://datatracker.ietf.org -sources: - - https://github.com/ietf-tools/datatracker -maintainers: - - name: IETF Tools Team - email: tools-discuss@ietf.org - url: https://github.com/ietf-tools - -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.0.0" \ No newline at end of file diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl deleted file mode 100644 index 071e9b824c..0000000000 --- a/helm/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* - Expand the name of the chart. - */}} -{{- define "datatracker.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "datatracker.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "datatracker.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "datatracker.labels" -}} -helm.sh/chart: {{ include "datatracker.chart" . }} -{{ include "datatracker.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "datatracker.selectorLabels" -}} -app.kubernetes.io/name: {{ include "datatracker.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "datatracker.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "datatracker.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml deleted file mode 100644 index b47c41a970..0000000000 --- a/helm/templates/deployment.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} - selector: - matchLabels: - {{- include "datatracker.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "datatracker.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "datatracker.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ default "latest" .Values.image.tag }}" - imagePullPolicy: {{ default "IfNotPresent" .Values.image.imagePullPolicy }} - env: - {{- if .Values.env }} - {{- toYaml .Values.env | nindent 12 }} - {{- end }} - {{- with .Values.volumeMounts }} - volumeMounts: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - livenessProbe: - {{- toYaml .Values.livenessProbe | nindent 12 }} - readinessProbe: - {{- toYaml .Values.readinessProbe | nindent 12 }} - startupProbe: - {{- toYaml .Values.startupProbe | nindent 12 }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.volumes }} - volumes: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/helm/templates/hpa.yaml b/helm/templates/hpa.yaml deleted file mode 100644 index 518f7e23ab..0000000000 --- a/helm/templates/hpa.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "datatracker.fullname" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "datatracker.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - target: - type: Utilization - averageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/templates/ingress.yaml b/helm/templates/ingress.yaml deleted file mode 100644 index 8d9258cd83..0000000000 --- a/helm/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "datatracker.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} \ No newline at end of file diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml deleted file mode 100644 index f1bdca0ad2..0000000000 --- a/helm/templates/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{include "datatracker.fullname" .}} - labels: {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.service.annotations }} - annotations: - {{- range $key, $value := . }} - {{ $key }}: {{ $value | quote }} - {{- end }} - {{- end }} -spec: - type: {{.Values.service.type}} - ports: - - port: {{ default "80" .Values.service.port}} - targetPort: http - protocol: TCP - name: http - selector: {{- include "datatracker.selectorLabels" . | nindent 4}} \ No newline at end of file diff --git a/helm/templates/serviceaccount.yaml b/helm/templates/serviceaccount.yaml deleted file mode 100644 index 475fcd51f7..0000000000 --- a/helm/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "datatracker.serviceAccountName" . }} - labels: - {{- include "datatracker.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end -}} \ No newline at end of file diff --git a/helm/values.yaml b/helm/values.yaml deleted file mode 100644 index 92efbce9dd..0000000000 --- a/helm/values.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Default values for datatracker. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: "ghcr.io/ietf-tools/datatracker" - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - # tag: "v1.1.0" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Automatically mount a ServiceAccount's API credentials? - automount: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -livenessProbe: - httpGet: - path: /healthz - port: http - -readinessProbe: - httpGet: - path: /healthz - port: http - -startupProbe: - initialDelaySeconds: 15 - periodSeconds: 5 - timeoutSeconds: 5 - successThreshold: 1 - failureThreshold: 60 - httpGet: - path: /healthz - port: http - -podAnnotations: {} -podLabels: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: datatracker.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -# Additional volumes on the output Deployment definition. -volumes: [] -# - name: foo -# secret: -# secretName: mysecret -# optional: false - -# Additional volumeMounts on the output Deployment definition. -volumeMounts: [] -# - name: foo -# mountPath: "/etc/foo" -# readOnly: true - -nodeSelector: {} - -tolerations: [] - -affinity: {} \ No newline at end of file diff --git a/ietf/bin/expire-last-calls b/ietf/bin/expire-last-calls deleted file mode 100755 index 83b565e192..0000000000 --- a/ietf/bin/expire-last-calls +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python - -# This script requires that the proper virtual python environment has been -# invoked before start - -import os -import sys -import syslog - -# boilerplate -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) -sys.path = [ basedir ] + sys.path -os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" - -virtualenv_activation = os.path.join(basedir, "env", "bin", "activate_this.py") -if os.path.exists(virtualenv_activation): - execfile(virtualenv_activation, dict(__file__=virtualenv_activation)) - -syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER) - -import django -django.setup() - -# ---------------------------------------------------------------------- - -from ietf.doc.lastcall import get_expired_last_calls, expire_last_call - -drafts = get_expired_last_calls() -for doc in drafts: - try: - expire_last_call(doc) - syslog.syslog("Expired last call for %s (id=%s)" % (doc.file_tag(), doc.pk)) - except Exception as e: - syslog.syslog(syslog.LOG_ERR, "ERROR: Failed to expire last call for %s (id=%s)" % (doc.file_tag(), doc.pk)) diff --git a/ietf/bin/rfc-editor-index-updates b/ietf/bin/rfc-editor-index-updates deleted file mode 100755 index c3e8f1f462..0000000000 --- a/ietf/bin/rfc-editor-index-updates +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -# This script requires that the proper virtual python environment has been -# invoked before start - -import datetime -import io -import os -import requests -import sys -import syslog -import traceback - -# boilerplate -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) -sys.path = [ basedir ] + sys.path -os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" - -# Before invoking django -syslog.openlog(os.path.basename(__file__), syslog.LOG_PID, syslog.LOG_USER) - -import django -django.setup() - -from django.conf import settings -from optparse import OptionParser -from django.core.mail import mail_admins - -from ietf.doc.utils import rebuild_reference_relations -from ietf.utils.log import log -from ietf.utils.pipe import pipe -from ietf.utils.timezone import date_today - -import ietf.sync.rfceditor - - -parser = OptionParser() -parser.add_option("-d", dest="skip_date", - help="To speed up processing skip RFCs published before this date (default is one year ago)", metavar="YYYY-MM-DD") - -options, args = parser.parse_args() - -skip_date = date_today() - datetime.timedelta(days=365) -if options.skip_date: - skip_date = datetime.datetime.strptime(options.skip_date, "%Y-%m-%d").date() - -log("Updating document metadata from RFC index going back to %s, from %s" % (skip_date, settings.RFC_EDITOR_INDEX_URL)) - - -try: - response = requests.get( - settings.RFC_EDITOR_INDEX_URL, - timeout=30, # seconds - ) -except requests.Timeout as exc: - log(f'GET request timed out retrieving RFC editor index: {exc}') - sys.exit(1) - - -rfc_index_xml = response.text -index_data = ietf.sync.rfceditor.parse_index(io.StringIO(rfc_index_xml)) - -try: - response = requests.get( - settings.RFC_EDITOR_ERRATA_JSON_URL, - timeout=30, # seconds - ) -except requests.Timeout as exc: - log(f'GET request timed out retrieving RFC editor errata: {exc}') - sys.exit(1) -errata_data = response.json() - -if len(index_data) < ietf.sync.rfceditor.MIN_INDEX_RESULTS: - log("Not enough index entries, only %s" % len(index_data)) - sys.exit(1) - -if len(errata_data) < ietf.sync.rfceditor.MIN_ERRATA_RESULTS: - log("Not enough errata entries, only %s" % len(errata_data)) - sys.exit(1) - -new_rfcs = [] -for rfc_number, changes, doc, rfc_published in ietf.sync.rfceditor.update_docs_from_rfc_index(index_data, errata_data, skip_older_than_date=skip_date): - if rfc_published: - new_rfcs.append(doc) - - for c in changes: - log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) - -sys.exit(0) - -# This can be called while processing a notifying POST from the RFC Editor -# Spawn a child to sync the rfcs and calculate new reference relationships -# so that the POST - -newpid = os.fork() - -if newpid == 0: - try: - pipe("%s -a %s %s" % (settings.RSYNC_BINARY,settings.RFC_TEXT_RSYNC_SOURCE,settings.RFC_PATH)) - for rfc in new_rfcs: - rebuild_reference_relations(rfc) - log("Updated references for %s"%rfc.name) - except: - subject = "Exception in updating references for new rfcs: %s : %s" % (sys.exc_info()[0],sys.exc_info()[1]) - msg = "%s\n%s\n----\n%s"%(sys.exc_info()[0],sys.exc_info()[1],traceback.format_tb(sys.exc_info()[2])) - mail_admins(subject,msg,fail_silently=True) - log(subject) - os._exit(0) -else: - sys.exit(0) diff --git a/ietf/bin/rfc-editor-queue-updates b/ietf/bin/rfc-editor-queue-updates deleted file mode 100755 index b441e50ebc..0000000000 --- a/ietf/bin/rfc-editor-queue-updates +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -import io -import os -import requests -import sys - -# boilerplate -basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) -sys.path = [ basedir ] + sys.path -os.environ["DJANGO_SETTINGS_MODULE"] = "ietf.settings" - -import django -django.setup() - -from django.conf import settings - -from ietf.sync.rfceditor import parse_queue, MIN_QUEUE_RESULTS, update_drafts_from_queue -from ietf.utils.log import log - -log("Updating RFC Editor queue states from %s" % settings.RFC_EDITOR_QUEUE_URL) - -try: - response = requests.get( - settings.RFC_EDITOR_QUEUE_URL, - timeout=30, # seconds - ) -except requests.Timeout as exc: - log(f'GET request timed out retrieving RFC editor queue: {exc}') - sys.exit(1) -drafts, warnings = parse_queue(io.StringIO(response.text)) -for w in warnings: - log(u"Warning: %s" % w) - -if len(drafts) < MIN_QUEUE_RESULTS: - log("Not enough results, only %s" % len(drafts)) - sys.exit(1) - -changed, warnings = update_drafts_from_queue(drafts) -for w in warnings: - log(u"Warning: %s" % w) - -for c in changed: - log(u"Updated %s" % c) diff --git a/ietf/celeryapp.py b/ietf/celeryapp.py index b36f134636..fda89c30be 100644 --- a/ietf/celeryapp.py +++ b/ietf/celeryapp.py @@ -1,14 +1,20 @@ import os import scout_apm.celery -from celery import Celery +import celery from scout_apm.api import Config +# Disable celery's internal logging configuration, we set it up via Django +@celery.signals.setup_logging.connect +def on_setup_logging(**kwargs): + pass + + # Set the default Django settings module for the 'celery' program os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ietf.settings') -app = Celery('ietf') +app = celery.Celery('ietf') # Using a string here means the worker doesn't have to serialize # the configuration object to child processes. @@ -17,10 +23,13 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # Turn on Scout APM celery instrumentation if configured in the environment -scout_key = os.environ.get("SCOUT_KEY", "") -scout_name = os.environ.get("SCOUT_NAME", "") -scout_core_agent_socket_path = os.environ.get("SCOUT_CORE_AGENT_SOCKET_PATH", "tcp://scoutapm:6590") -if scout_key and scout_name: +scout_key = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if scout_key is not None: + scout_name = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") + scout_core_agent_socket_path = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), + ) Config.set( key=scout_key, name=scout_name, diff --git a/ietf/doc/management/commands/generate_draft_bibxml_files.py b/ietf/doc/management/commands/generate_draft_bibxml_files.py deleted file mode 100644 index eda67c401b..0000000000 --- a/ietf/doc/management/commands/generate_draft_bibxml_files.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import datetime -import io -import os -import re -import sys - -from django.conf import settings -from django.core.management.base import BaseCommand -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.doc.models import NewRevisionDocEvent -from ietf.doc.utils import bibxml_for_draft - -DEFAULT_DAYS = 7 - -class Command(BaseCommand): - help = ('Generate draft bibxml files for xml2rfc references, placing them in the ' - 'directory configured in settings.BIBXML_BASE_PATH: %s. ' - 'By default, generate files as needed for new Internet-Draft revisions from the ' - 'last %s days.' % (settings.BIBXML_BASE_PATH, DEFAULT_DAYS)) - - def add_arguments(self, parser): - parser.add_argument('--all', action='store_true', default=False, help="Process all documents, not only recent submissions") - parser.add_argument('--days', type=int, default=DEFAULT_DAYS, help="Look submissions from the last DAYS days, instead of %s" % DEFAULT_DAYS) - - def say(self, msg): - if self.verbosity > 0: - sys.stdout.write(msg) - sys.stdout.write('\n') - - def note(self, msg): - if self.verbosity > 1: - sys.stdout.write(msg) - sys.stdout.write('\n') - - def mutter(self, msg): - if self.verbosity > 2: - sys.stdout.write(msg) - sys.stdout.write('\n') - - def write(self, fn, new): - # normalize new - new = re.sub(r'\r\n?', r'\n', new) - try: - with io.open(fn, encoding='utf-8') as f: - old = f.read() - except IOError: - old = "" - if old.strip() != new.strip(): - self.note('Writing %s' % os.path.basename(fn)) - with io.open(fn, "w", encoding='utf-8') as f: - f.write(new) - - def handle(self, *args, **options): - self.verbosity = options.get("verbosity", 1) - process_all = options.get("all") - days = options.get("days") - # - bibxmldir = os.path.join(settings.BIBXML_BASE_PATH, 'bibxml-ids') - if not os.path.exists(bibxmldir): - os.makedirs(bibxmldir) - # - if process_all: - doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft') - else: - start = timezone.now() - datetime.timedelta(days=days) - doc_events = NewRevisionDocEvent.objects.filter(type='new_revision', doc__type_id='draft', time__gte=start) - doc_events = doc_events.order_by('time') - - for e in doc_events: - self.mutter('%s %s' % (e.time, e.doc.name)) - try: - doc = e.doc - bibxml = bibxml_for_draft(doc, e.rev) - ref_rev_file_name = os.path.join(bibxmldir, 'reference.I-D.%s-%s.xml' % (doc.name, e.rev)) - self.write(ref_rev_file_name, bibxml) - except Exception as ee: - sys.stderr.write('\n%s-%s: %s\n' % (doc.name, doc.rev, ee)) diff --git a/ietf/doc/management/commands/generate_idnits2_rfc_status.py b/ietf/doc/management/commands/generate_idnits2_rfc_status.py deleted file mode 100644 index 45be188018..0000000000 --- a/ietf/doc/management/commands/generate_idnits2_rfc_status.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright The IETF Trust 2021 All Rights Reserved - -import os - -from django.conf import settings -from django.core.management.base import BaseCommand - -from ietf.doc.utils import generate_idnits2_rfc_status -from ietf.utils.log import log - -class Command(BaseCommand): - help = ('Generate the rfc_status blob used by idnits2') - - def handle(self, *args, **options): - filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfc-status') - blob = generate_idnits2_rfc_status() - try: - bytes = blob.encode('utf-8') - with open(filename,'wb') as f: - f.write(bytes) - except Exception as e: - log('failed to write idnits2-rfc-status: '+str(e)) - raise e diff --git a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py b/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py deleted file mode 100644 index 8bd122e87e..0000000000 --- a/ietf/doc/management/commands/generate_idnits2_rfcs_obsoleted.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright The IETF Trust 2021 All Rights Reserved - -import os - -from django.conf import settings -from django.core.management.base import BaseCommand - -from ietf.doc.utils import generate_idnits2_rfcs_obsoleted -from ietf.utils.log import log - -class Command(BaseCommand): - help = ('Generate the rfcs-obsoleted file used by idnits2') - - def handle(self, *args, **options): - filename=os.path.join(settings.DERIVED_DIR,'idnits2-rfcs-obsoleted') - blob = generate_idnits2_rfcs_obsoleted() - try: - bytes = blob.encode('utf-8') - with open(filename,'wb') as f: - f.write(bytes) - except Exception as e: - log('failed to write idnits2-rfcs-obsoleted: '+str(e)) - raise e diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index a2e83e9e26..209db035a4 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -6,6 +6,10 @@ import debug # pyflakes:ignore from celery import shared_task +from pathlib import Path + +from django.conf import settings +from django.utils import timezone from ietf.utils import log from ietf.utils.timezone import datetime_today @@ -20,7 +24,14 @@ get_soon_to_expire_drafts, send_expire_warning_for_draft, ) -from .models import Document +from .lastcall import get_expired_last_calls, expire_last_call +from .models import Document, NewRevisionDocEvent +from .utils import ( + generate_idnits2_rfc_status, + generate_idnits2_rfcs_obsoleted, + update_or_create_draft_bibxml_file, + ensure_draft_bibxml_path_exists, +) @shared_task @@ -54,3 +65,55 @@ def expire_ids_task(): def notify_expirations_task(notify_days=14): for doc in get_soon_to_expire_drafts(notify_days): send_expire_warning_for_draft(doc) + + +@shared_task +def expire_last_calls_task(): + for doc in get_expired_last_calls(): + try: + expire_last_call(doc) + except Exception: + log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})") + else: + log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})") + + +@shared_task +def generate_idnits2_rfc_status_task(): + outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" + blob = generate_idnits2_rfc_status() + try: + outpath.write_text(blob, encoding="utf8") + except Exception as e: + log.log(f"failed to write idnits2-rfc-status: {e}") + + +@shared_task +def generate_idnits2_rfcs_obsoleted_task(): + outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" + blob = generate_idnits2_rfcs_obsoleted() + try: + outpath.write_text(blob, encoding="utf8") + except Exception as e: + log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") + + +@shared_task +def generate_draft_bibxml_files_task(days=7, process_all=False): + """Generate bibxml files for recently updated docs + + If process_all is False (the default), processes only docs with new revisions + in the last specified number of days. + """ + ensure_draft_bibxml_path_exists() + doc_events = NewRevisionDocEvent.objects.filter( + type="new_revision", + doc__type_id="draft", + ).order_by("time") + if not process_all: + doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days)) + for event in doc_events: + try: + update_or_create_draft_bibxml_file(event.doc, event.rev) + except Exception as err: + log.log(f"Error generating bibxml for {event.doc.name}-{event.rev}: {err}") diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 4c200a4700..35c79ac28d 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -906,13 +906,17 @@ def mtime(path): """Returns a datetime object representing mtime given a pathlib Path object""" return datetime.datetime.fromtimestamp(path.stat().st_mtime).astimezone(ZoneInfo(settings.TIME_ZONE)) +@register.filter +def mtime_is_epoch(path): + return path.stat().st_mtime == 0 + @register.filter def url_for_path(path): """Consructs a 'best' URL for web access to the given pathlib Path object. Assumes that the path is into the Internet-Draft archive or the proceedings. """ - if path.match(f"{settings.AGENDA_PATH}/**/*"): + if Path(settings.AGENDA_PATH) in path.parents: return ( f"https://www.ietf.org/proceedings/{path.relative_to(settings.AGENDA_PATH)}" ) diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index 4f5492e6cb..e6a50937a5 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -20,7 +20,6 @@ from collections import defaultdict from zoneinfo import ZoneInfo -from django.core.management import call_command from django.urls import reverse as urlreverse from django.conf import settings from django.forms import Form @@ -45,7 +44,14 @@ StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField -from ietf.doc.utils import create_ballot_if_not_open, investigate_fragment, uppercase_std_abbreviated_name, DraftAliasGenerator +from ietf.doc.utils import ( + create_ballot_if_not_open, + investigate_fragment, + uppercase_std_abbreviated_name, + DraftAliasGenerator, + generate_idnits2_rfc_status, + generate_idnits2_rfcs_obsoleted, +) from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory from ietf.ipr.factories import HolderIprDisclosureFactory @@ -2831,32 +2837,40 @@ def test_markdown_and_text(self): class Idnits2SupportTests(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] - def test_obsoleted(self): + def test_generate_idnits2_rfcs_obsoleted(self): rfc = WgRfcFactory(rfc_number=1001) WgRfcFactory(rfc_number=1003,relations=[('obs',rfc)]) rfc = WgRfcFactory(rfc_number=1005) WgRfcFactory(rfc_number=1007,relations=[('obs',rfc)]) + blob = generate_idnits2_rfcs_obsoleted() + self.assertEqual(blob, b'1001 1003\n1005 1007\n'.decode("utf8")) + def test_obsoleted(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 404) - call_command('generate_idnits2_rfcs_obsoleted') + # value written is arbitrary, expect it to be passed through + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").write_bytes(b'1001 1003\n1005 1007\n') url = urlreverse('ietf.doc.views_doc.idnits2_rfcs_obsoleted') r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertEqual(r.content, b'1001 1003\n1005 1007\n') - def test_rfc_status(self): + def test_generate_idnits2_rfc_status(self): for slug in ('bcp', 'ds', 'exp', 'hist', 'inf', 'std', 'ps', 'unkn'): WgRfcFactory(std_level_id=slug) + blob = generate_idnits2_rfc_status().replace("\n", "") + self.assertEqual(blob[6312-1], "O") + + def test_rfc_status(self): url = urlreverse('ietf.doc.views_doc.idnits2_rfc_status') r = self.client.get(url) self.assertEqual(r.status_code,404) - call_command('generate_idnits2_rfc_status') + # value written is arbitrary, expect it to be passed through + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").write_bytes(b'1001 1003\n1005 1007\n') r = self.client.get(url) self.assertEqual(r.status_code,200) - blob = unicontent(r).replace('\n','') - self.assertEqual(blob[6312-1],'O') + self.assertEqual(r.content, b'1001 1003\n1005 1007\n') def test_idnits2_state(self): rfc = WgRfcFactory() diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 931ed438dc..51a8556e69 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,15 +1,28 @@ # Copyright The IETF Trust 2024, All Rights Reserved +import datetime import mock +from pathlib import Path + +from django.conf import settings +from django.utils import timezone + from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from .factories import DocumentFactory -from .models import Document -from .tasks import expire_ids_task, notify_expirations_task - +from .factories import DocumentFactory, NewRevisionDocEventFactory +from .models import Document, NewRevisionDocEvent +from .tasks import ( + expire_ids_task, + expire_last_calls_task, + generate_draft_bibxml_files_task, + generate_idnits2_rfcs_obsoleted_task, + generate_idnits2_rfc_status_task, + notify_expirations_task, +) class TaskTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["DERIVED_DIR"] @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -35,10 +48,10 @@ def test_expire_ids_task( Document.objects.filter(pk=doc.pk), Document.objects.filter(pk=other_doc.pk), ] - + # call task expire_ids_task() - + # check results self.assertTrue(in_draft_expire_freeze_mock.called) self.assertEqual(expirable_drafts_mock.call_count, 2) @@ -50,7 +63,7 @@ def test_expire_ids_task( # test that an exception is raised in_draft_expire_freeze_mock.side_effect = RuntimeError - with self.assertRaises(RuntimeError):( + with self.assertRaises(RuntimeError): ( expire_ids_task()) @mock.patch("ietf.doc.tasks.send_expire_warning_for_draft") @@ -61,3 +74,129 @@ def test_notify_expirations_task(self, get_drafts_mock, send_warning_mock): notify_expirations_task() self.assertEqual(send_warning_mock.call_count, 1) self.assertEqual(send_warning_mock.call_args[0], ("sentinel",)) + + @mock.patch("ietf.doc.tasks.expire_last_call") + @mock.patch("ietf.doc.tasks.get_expired_last_calls") + def test_expire_last_calls_task(self, mock_get_expired, mock_expire): + docs = DocumentFactory.create_batch(3) + mock_get_expired.return_value = docs + expire_last_calls_task() + self.assertTrue(mock_get_expired.called) + self.assertEqual(mock_expire.call_count, 3) + self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) + self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) + self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) + + # Check that it runs even if exceptions occur + mock_get_expired.reset_mock() + mock_expire.reset_mock() + mock_expire.side_effect = ValueError + expire_last_calls_task() + self.assertTrue(mock_get_expired.called) + self.assertEqual(mock_expire.call_count, 3) + self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) + self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) + self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfc_status") + def test_generate_idnits2_rfc_status_task(self, mock_generate): + mock_generate.return_value = "dåtå" + generate_idnits2_rfc_status_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual( + "dåtå".encode("utf8"), + (Path(settings.DERIVED_DIR) / "idnits2-rfc-status").read_bytes(), + ) + + @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") + def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): + mock_generate.return_value = "dåtå" + generate_idnits2_rfcs_obsoleted_task() + self.assertEqual(mock_generate.call_count, 1) + self.assertEqual( + "dåtå".encode("utf8"), + (Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted").read_bytes(), + ) + + @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") + @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") + def test_generate_draft_bibxml_files_task(self, mock_create, mock_ensure_path): + now = timezone.now() + very_old_event = NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=1000), rev="17" + ) + old_event = NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=8), rev="03" + ) + young_event = NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=6), rev="06" + ) + # a couple that should always be ignored + NewRevisionDocEventFactory( + time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc" # not a draft + ) + NewRevisionDocEventFactory( + type="changed_document", # not a "new_revision" type + time=now - datetime.timedelta(days=6), + rev="09", + doc__type_id="rfc", + ) + + # Get rid of the "00" events created by the factories -- they're just noise for this test + NewRevisionDocEvent.objects.filter(rev="00").delete() + + # default args - look back 7 days + generate_draft_bibxml_files_task() + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, [mock.call(young_event.doc, young_event.rev)] + ) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # shorter lookback + generate_draft_bibxml_files_task(days=5) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual(mock_create.call_args_list, []) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # longer lookback + generate_draft_bibxml_files_task(days=9) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, + [ + mock.call(young_event.doc, young_event.rev), + mock.call(old_event.doc, old_event.rev), + ], + ) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # everything + generate_draft_bibxml_files_task(process_all=True) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, + [ + mock.call(young_event.doc, young_event.rev), + mock.call(old_event.doc, old_event.rev), + mock.call(very_old_event.doc, very_old_event.rev), + ], + ) + mock_create.reset_mock() + mock_ensure_path.reset_mock() + + # everything should still be tried, even if there's an exception + mock_create.side_effect = RuntimeError + generate_draft_bibxml_files_task(process_all=True) + self.assertTrue(mock_ensure_path.called) + self.assertCountEqual( + mock_create.call_args_list, + [ + mock.call(young_event.doc, young_event.rev), + mock.call(old_event.doc, old_event.rev), + mock.call(very_old_event.doc, very_old_event.rev), + ], + ) diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index 248ac345af..f610fe3d76 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -2,8 +2,10 @@ import datetime import debug # pyflakes:ignore -from unittest.mock import patch +from pathlib import Path +from unittest.mock import call, patch +from django.conf import settings from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone @@ -16,7 +18,8 @@ from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, - fuzzy_find_documents, rebuild_reference_relations, build_file_urls) + fuzzy_find_documents, rebuild_reference_relations, build_file_urls, + ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file) from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -484,3 +487,49 @@ def test_xml_and_plaintext(self, mock_init, mock_get_refs, mock_plaintext_init): (self.updated.name, 'updates'), ] ) + + +class DraftBibxmlTests(TestCase): + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ["BIBXML_BASE_PATH"] + + def test_ensure_draft_bibxml_path_exists(self): + expected = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" + self.assertFalse(expected.exists()) + ensure_draft_bibxml_path_exists() + self.assertTrue(expected.is_dir()) # false if does not exist or is not dir + + @patch("ietf.doc.utils.bibxml_for_draft", return_value="This\ris\nmy\r\nbibxml") + def test_create_draft_bibxml_file(self, mock): + bibxml_path = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" + bibxml_path.mkdir(exist_ok=False) # expect to start with a clean slate + + doc = DocumentFactory() + ref_path = bibxml_path / f"reference.I-D.{doc.name}-26.xml" # we're pretending it's rev 26 + + update_or_create_draft_bibxml_file(doc, "26") + self.assertEqual(mock.call_count, 1) + self.assertEqual(mock.call_args, call(doc, "26")) + self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + @patch("ietf.doc.utils.bibxml_for_draft", return_value="This\ris\nmy\r\nbibxml") + def test_update_draft_bibxml_file(self, mock): + bibxml_path = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" + bibxml_path.mkdir(exist_ok=False) # expect to start with a clean slate + + doc = DocumentFactory() + ref_path = bibxml_path / f"reference.I-D.{doc.name}-26.xml" # we're pretending it's rev 26 + ref_path.write_text("Old data") + + # should replace it + update_or_create_draft_bibxml_file(doc, "26") + self.assertEqual(mock.call_count, 1) + self.assertEqual(mock.call_args, call(doc, "26")) + self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + # should leave it alone if it differs only by leading/trailing whitespace + mock.reset_mock() + mock.return_value = " \n This\nis\nmy\nbibxml " + update_or_create_draft_bibxml_file(doc, "26") + self.assertEqual(mock.call_count, 1) + self.assertEqual(mock.call_args, call(doc, "26")) + self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 8ec3985b8b..c3d7552f24 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -1388,14 +1388,18 @@ def investigate_fragment(name_fragment): can_verify = set() for root in [settings.INTERNET_DRAFT_PATH, settings.INTERNET_DRAFT_ARCHIVE_DIR]: can_verify.update(list(Path(root).glob(f"*{name_fragment}*"))) - + archive_verifiable_names = set([p.name for p in can_verify]) + # Can also verify drafts in proceedings directories can_verify.update(list(Path(settings.AGENDA_PATH).glob(f"**/*{name_fragment}*"))) # N.B. This reflects the assumption that the internet draft archive dir is in the # a directory with other collections (at /a/ietfdata/draft/collections as this is written) - unverifiable_collections = set( + unverifiable_collections = set([ + p for p in Path(settings.INTERNET_DRAFT_ARCHIVE_DIR).parent.glob(f"**/*{name_fragment}*") - ) + if p.name not in archive_verifiable_names + ]) + unverifiable_collections.difference_update(can_verify) expected_names = set([p.name for p in can_verify.union(unverifiable_collections)]) @@ -1409,3 +1413,20 @@ def investigate_fragment(name_fragment): unverifiable_collections=unverifiable_collections, unexpected=unexpected, ) + + +def update_or_create_draft_bibxml_file(doc, rev): + log.assertion("doc.type_id == 'draft'") + normalized_bibxml = re.sub(r"\r\n?", r"\n", bibxml_for_draft(doc, rev)) + ref_rev_file_path = Path(settings.BIBXML_BASE_PATH) / "bibxml-ids" / f"reference.I-D.{doc.name}-{rev}.xml" + try: + existing_bibxml = ref_rev_file_path.read_text(encoding="utf8") + except IOError: + existing_bibxml = "" + if normalized_bibxml.strip() != existing_bibxml.strip(): + log.log(f"Writing {ref_rev_file_path}") + ref_rev_file_path.write_text(normalized_bibxml, encoding="utf8") + + +def ensure_draft_bibxml_path_exists(): + (Path(settings.BIBXML_BASE_PATH) / "bibxml-ids").mkdir(exist_ok=True) diff --git a/ietf/doc/views_charter.py b/ietf/doc/views_charter.py index d44a675f68..f8748d2126 100644 --- a/ietf/doc/views_charter.py +++ b/ietf/doc/views_charter.py @@ -37,8 +37,8 @@ from ietf.doc.mails import email_state_changed, email_charter_internal_review from ietf.group.mails import email_admin_re_charter from ietf.group.models import Group, ChangeStateGroupEvent, MilestoneGroupEvent -from ietf.group.utils import save_group_in_history, save_milestone_in_history, can_manage_all_groups_of_type -from ietf.group.views import fill_in_charter_info +from ietf.group.utils import save_group_in_history, save_milestone_in_history, can_manage_all_groups_of_type, \ + fill_in_charter_info from ietf.ietfauth.utils import has_role, role_required from ietf.name.models import GroupStateName from ietf.person.models import Person diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py new file mode 100644 index 0000000000..f7717616d1 --- /dev/null +++ b/ietf/group/tasks.py @@ -0,0 +1,61 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +# +# Celery task definitions +# +import shutil + +from celery import shared_task +from pathlib import Path + +from django.conf import settings +from django.template.loader import render_to_string + +from ietf.utils import log + +from .models import Group +from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles + + +@shared_task +def generate_wg_charters_files_task(): + areas = Group.objects.filter(type="area", state="active").order_by("name") + groups = ( + Group.objects.filter(type="wg", state="active") + .exclude(parent=None) + .order_by("acronym") + ) + for group in groups: + fill_in_charter_info(group) + fill_in_wg_roles(group) + fill_in_wg_drafts(group) + for area in areas: + area.groups = [g for g in groups if g.parent_id == area.pk] + charter_path = Path(settings.CHARTER_PATH) + charters_file = charter_path / "1wg-charters.txt" + charters_file.write_text( + render_to_string("group/1wg-charters.txt", {"areas": areas}), + encoding="utf8", + ) + charters_by_acronym_file = charter_path / "1wg-charters-by-acronym.txt" + charters_by_acronym_file.write_text( + render_to_string("group/1wg-charters-by-acronym.txt", {"groups": groups}), + encoding="utf8", + ) + + charter_copy_dest = getattr(settings, "CHARTER_COPY_PATH", None) + if charter_copy_dest is not None: + if not Path(charter_copy_dest).is_dir(): + log.log( + f"Error copying 1wg-charter files to {charter_copy_dest}: it does not exist or is not a directory" + ) + else: + try: + shutil.copy2(charters_file, charter_copy_dest) + except IOError as err: + log.log(f"Error copying {charters_file} to {charter_copy_dest}: {err}") + try: + shutil.copy2(charters_by_acronym_file, charter_copy_dest) + except IOError as err: + log.log( + f"Error copying {charters_by_acronym_file} to {charter_copy_dest}: {err}" + ) diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index 6ecac7d347..777671db9d 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -17,6 +17,7 @@ from django.conf import settings from django.test import RequestFactory +from django.test.utils import override_settings from django.urls import reverse as urlreverse from django.urls import NoReverseMatch from django.utils import timezone @@ -34,6 +35,7 @@ DatedGroupMilestoneFactory, DatelessGroupMilestoneFactory) from ietf.group.forms import GroupForm from ietf.group.models import Group, GroupEvent, GroupMilestone, GroupStateTransitions, Role +from ietf.group.tasks import generate_wg_charters_files_task from ietf.group.utils import save_group_in_history, setup_default_community_list_for_group from ietf.meeting.factories import SessionFactory from ietf.name.models import DocTagName, GroupStateName, GroupTypeName, ExtResourceName, RoleName @@ -56,7 +58,7 @@ def pklist(docs): return [ str(doc.pk) for doc in docs.all() ] class GroupPagesTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CHARTER_PATH'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['CHARTER_PATH', 'CHARTER_COPY_PATH'] def test_active_groups(self): area = GroupFactory.create(type_id='area') @@ -117,10 +119,6 @@ def test_wg_summaries(self): chair = Email.objects.filter(role__group=group, role__name="chair")[0] - ( - Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" - ).write_text("This is a charter.") - url = urlreverse('ietf.group.views.wg_summary_area', kwargs=dict(group_type="wg")) r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -136,23 +134,125 @@ def test_wg_summaries(self): self.assertContains(r, group.name) self.assertContains(r, chair.address) - url = urlreverse('ietf.group.views.wg_charters', kwargs=dict(group_type="wg")) + def test_wg_charters(self): + # file does not exist = 404 + url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="wg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # should return expected file with expected encoding + wg_path = Path(settings.CHARTER_PATH) / "1wg-charters.txt" + wg_path.write_text("This is a charters file with an é") r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertContains(r, group.acronym) - self.assertContains(r, group.name) - self.assertContains(r, group.ad_role().person.plain_name()) - self.assertContains(r, chair.address) - self.assertContains(r, "This is a charter.") + self.assertEqual(r.charset, "UTF-8") + self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é") - url = urlreverse('ietf.group.views.wg_charters_by_acronym', kwargs=dict(group_type="wg")) + # non-wg request = 404 even if the file exists + url = urlreverse("ietf.group.views.wg_charters", kwargs=dict(group_type="rg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + def test_wg_charters_by_acronym(self): + url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="wg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + wg_path = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" + wg_path.write_text("This is a charters file with an é") r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertContains(r, group.acronym) - self.assertContains(r, group.name) - self.assertContains(r, group.ad_role().person.plain_name()) - self.assertContains(r, chair.address) - self.assertContains(r, "This is a charter.") + self.assertEqual(r.charset, "UTF-8") + self.assertEqual(r.content.decode("utf8"), "This is a charters file with an é") + + # non-wg request = 404 even if the file exists + url = urlreverse("ietf.group.views.wg_charters_by_acronym", kwargs=dict(group_type="rg")) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + def test_generate_wg_charters_files_task(self): + group = CharterFactory( + group__type_id="wg", group__parent=GroupFactory(type_id="area") + ).group + RoleFactory(group=group, name_id="chair", person=PersonFactory()) + RoleFactory(group=group, name_id="ad", person=PersonFactory()) + chair = Email.objects.filter(role__group=group, role__name="chair")[0] + ( + Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" + ).write_text("This is a charter.") + + generate_wg_charters_files_task() + wg_charters_contents = (Path(settings.CHARTER_PATH) / "1wg-charters.txt").read_text( + encoding="utf8" + ) + self.assertIn(group.acronym, wg_charters_contents) + self.assertIn(group.name, wg_charters_contents) + self.assertIn(group.ad_role().person.plain_name(), wg_charters_contents) + self.assertIn(chair.address, wg_charters_contents) + self.assertIn("This is a charter.", wg_charters_contents) + wg_charters_copy = ( + Path(settings.CHARTER_COPY_PATH) / "1wg-charters.txt" + ).read_text(encoding="utf8") + self.assertEqual(wg_charters_copy, wg_charters_contents) + + wg_charters_by_acronym_contents = ( + Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" + ).read_text(encoding="utf8") + self.assertIn(group.acronym, wg_charters_by_acronym_contents) + self.assertIn(group.name, wg_charters_by_acronym_contents) + self.assertIn(group.ad_role().person.plain_name(), wg_charters_by_acronym_contents) + self.assertIn(chair.address, wg_charters_by_acronym_contents) + self.assertIn("This is a charter.", wg_charters_by_acronym_contents) + wg_charters_by_acronymcopy = ( + Path(settings.CHARTER_COPY_PATH) / "1wg-charters-by-acronym.txt" + ).read_text(encoding="utf8") + self.assertEqual(wg_charters_by_acronymcopy, wg_charters_by_acronym_contents) + + def test_generate_wg_charters_files_task_without_copy(self): + """Test disabling charter file copying + + Note that these tests mostly check that errors are not encountered. Because they unset + the CHARTER_COPY_PATH or set it to a non-directory destination, it's not clear where to + look to see whether the files were (incorrectly) copied somewhere. + """ + group = CharterFactory( + group__type_id="wg", group__parent=GroupFactory(type_id="area") + ).group + ( + Path(settings.CHARTER_PATH) / f"{group.charter.name}-{group.charter.rev}.txt" + ).write_text("This is a charter.") + + # No directory set + with override_settings(): + del settings.CHARTER_COPY_PATH + generate_wg_charters_files_task() + # n.b., CHARTER_COPY_PATH is set again outside the with block + self.assertTrue((Path(settings.CHARTER_PATH) / "1wg-charters.txt").exists()) + self.assertFalse((Path(settings.CHARTER_COPY_PATH) / "1wg-charters.txt").exists()) + self.assertTrue( + (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + self.assertFalse( + (Path(settings.CHARTER_COPY_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + (Path(settings.CHARTER_PATH) / "1wg-charters.txt").unlink() + (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").unlink() + + # Set to a file, not a directory + not_a_dir = Path(settings.CHARTER_COPY_PATH) / "not-a-dir.txt" + not_a_dir.write_text("Not a dir") + with override_settings(CHARTER_COPY_PATH=str(not_a_dir)): + generate_wg_charters_files_task() + # n.b., CHARTER_COPY_PATH is set again outside the with block + self.assertTrue((Path(settings.CHARTER_PATH) / "1wg-charters.txt").exists()) + self.assertFalse((Path(settings.CHARTER_COPY_PATH) / "1wg-charters.txt").exists()) + self.assertTrue( + (Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + self.assertFalse( + (Path(settings.CHARTER_COPY_PATH) / "1wg-charters-by-acronym.txt").exists() + ) + self.assertEqual(not_a_dir.read_text(), "Not a dir") def test_chartering_groups(self): group = CharterFactory(group__type_id='wg',group__parent=GroupFactory(type_id='area'),states=[('charter','intrev')]).group diff --git a/ietf/group/utils.py b/ietf/group/utils.py index f9c2aa15bf..51696eb39b 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -15,13 +15,13 @@ from ietf.community.models import CommunityList, SearchRule from ietf.community.utils import reset_name_contains_index_for_rule, can_manage_community_list -from ietf.doc.models import Document, State +from ietf.doc.models import Document, State, RelatedDocument from ietf.group.models import Group, RoleHistory, Role, GroupFeatures, GroupEvent from ietf.ietfauth.utils import has_role from ietf.name.models import GroupTypeName, RoleName from ietf.person.models import Email from ietf.review.utils import can_manage_review_requests_for_team -from ietf.utils import log +from ietf.utils import log, markdown from ietf.utils.history import get_history_object_for, copy_many_to_many_for_history from ietf.doc.templatetags.ietf_filters import is_valid_url from functools import reduce @@ -450,3 +450,68 @@ def role_holder_emails(): address__startswith="unknown-email-" ) return emails.filter(person__role__in=roles).distinct() + + +def fill_in_charter_info(group, include_drafts=False): + group.areadirector = getattr(group.ad_role(),'email',None) + + personnel = {} + for r in Role.objects.filter(group=group).order_by('person__name').select_related("email", "person", "name"): + if r.name_id not in personnel: + personnel[r.name_id] = [] + personnel[r.name_id].append(r) + + if group.parent and group.parent.type_id == "area" and group.ad_role() and "ad" not in personnel: + ad_roles = list(Role.objects.filter(group=group.parent, name="ad", person=group.ad_role().person)) + if ad_roles: + personnel["ad"] = ad_roles + + group.personnel = [] + for role_name_slug, roles in personnel.items(): + label = roles[0].name.name + if len(roles) > 1: + if label.endswith("y"): + label = label[:-1] + "ies" + else: + label += "s" + + group.personnel.append((role_name_slug, label, roles)) + + group.personnel.sort(key=lambda t: t[2][0].name.order) + + milestone_state = "charter" if group.state_id == "proposed" else "active" + group.milestones = group.groupmilestone_set.filter(state=milestone_state) + if group.uses_milestone_dates: + group.milestones = group.milestones.order_by('resolved', 'due') + else: + group.milestones = group.milestones.order_by('resolved', 'order') + + if group.charter: + group.charter_text = get_charter_text(group) + else: + group.charter_text = "Not chartered yet." + group.charter_html = markdown.markdown(group.charter_text) + + +def fill_in_wg_roles(group): + def get_roles(slug, default): + for role_slug, label, roles in group.personnel: + if slug == role_slug: + return roles + return default + + group.chairs = get_roles("chair", []) + ads = get_roles("ad", []) + group.areadirector = ads[0] if ads else None + group.techadvisors = get_roles("techadv", []) + group.editors = get_roles("editor", []) + group.secretaries = get_roles("secr", []) + + +def fill_in_wg_drafts(group): + group.drafts = Document.objects.filter(type_id="draft", group=group).order_by("name") + group.rfcs = Document.objects.filter(type_id="rfc", group=group).order_by("rfc_number") + for rfc in group.rfcs: + # TODO: remote_field? + rfc.remote_field = RelatedDocument.objects.filter(source=rfc,relationship_id__in=['obs','updates']).distinct() + rfc.invrel = RelatedDocument.objects.filter(target=rfc,relationship_id__in=['obs','updates']).distinct() diff --git a/ietf/group/views.py b/ietf/group/views.py index 636871d901..7ad6e5bf02 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -41,9 +41,10 @@ import math import re import json +import types from collections import OrderedDict, defaultdict -import types +from pathlib import Path from simple_history.utils import update_change_reason from django import forms @@ -75,12 +76,12 @@ from ietf.group.mails import email_admin_re_charter, email_personnel_change, email_comment from ietf.group.models import ( Group, Role, GroupEvent, GroupStateTransitions, ChangeStateGroupEvent, GroupFeatures, AppealArtifact ) -from ietf.group.utils import (get_charter_text, can_manage_all_groups_of_type, +from ietf.group.utils import (can_manage_all_groups_of_type, milestone_reviewer_for_group_type, can_provide_status_update, can_manage_materials, group_attribute_change_desc, construct_group_menu_context, get_group_materials, save_group_in_history, can_manage_group, update_role_set, - get_group_or_404, setup_default_community_list_for_group, ) + get_group_or_404, setup_default_community_list_for_group, fill_in_charter_info) # from ietf.ietfauth.utils import has_role, is_authorized_in_group from ietf.mailtrigger.utils import gather_relevant_expansions @@ -132,70 +133,9 @@ def roles(group, role_name): return Role.objects.filter(group=group, name=role_name).select_related("email", "person") -def fill_in_charter_info(group, include_drafts=False): - group.areadirector = getattr(group.ad_role(),'email',None) - - personnel = {} - for r in Role.objects.filter(group=group).order_by('person__name').select_related("email", "person", "name"): - if r.name_id not in personnel: - personnel[r.name_id] = [] - personnel[r.name_id].append(r) - - if group.parent and group.parent.type_id == "area" and group.ad_role() and "ad" not in personnel: - ad_roles = list(Role.objects.filter(group=group.parent, name="ad", person=group.ad_role().person)) - if ad_roles: - personnel["ad"] = ad_roles - - group.personnel = [] - for role_name_slug, roles in personnel.items(): - label = roles[0].name.name - if len(roles) > 1: - if label.endswith("y"): - label = label[:-1] + "ies" - else: - label += "s" - - group.personnel.append((role_name_slug, label, roles)) - - group.personnel.sort(key=lambda t: t[2][0].name.order) - - milestone_state = "charter" if group.state_id == "proposed" else "active" - group.milestones = group.groupmilestone_set.filter(state=milestone_state) - if group.uses_milestone_dates: - group.milestones = group.milestones.order_by('resolved', 'due') - else: - group.milestones = group.milestones.order_by('resolved', 'order') - - if group.charter: - group.charter_text = get_charter_text(group) - else: - group.charter_text = "Not chartered yet." - group.charter_html = markdown.markdown(group.charter_text) - def extract_last_name(role): return role.person.name_parts()[3] -def fill_in_wg_roles(group): - def get_roles(slug, default): - for role_slug, label, roles in group.personnel: - if slug == role_slug: - return roles - return default - - group.chairs = get_roles("chair", []) - ads = get_roles("ad", []) - group.areadirector = ads[0] if ads else None - group.techadvisors = get_roles("techadv", []) - group.editors = get_roles("editor", []) - group.secretaries = get_roles("secr", []) - -def fill_in_wg_drafts(group): - group.drafts = Document.objects.filter(type_id="draft", group=group).order_by("name") - group.rfcs = Document.objects.filter(type_id="rfc", group=group).order_by("rfc_number") - for rfc in group.rfcs: - # TODO: remote_field? - rfc.remote_field = RelatedDocument.objects.filter(source=rfc,relationship_id__in=['obs','updates']).distinct() - rfc.invrel = RelatedDocument.objects.filter(target=rfc,relationship_id__in=['obs','updates']).distinct() def check_group_email_aliases(): pattern = re.compile(r'expand-(.*?)(-\w+)@.*? +(.*)$') @@ -241,34 +181,28 @@ def wg_summary_acronym(request, group_type): 'groups': groups }, content_type='text/plain; charset=UTF-8') -@cache_page ( 60 * 60, cache="slowpages" ) + def wg_charters(request, group_type): if group_type != "wg": raise Http404 - areas = Group.objects.filter(type="area", state="active").order_by("name") - for area in areas: - area.groups = Group.objects.filter(parent=area, type="wg", state="active").order_by("name") - for group in area.groups: - fill_in_charter_info(group) - fill_in_wg_roles(group) - fill_in_wg_drafts(group) - return render(request, 'group/1wg-charters.txt', - { 'areas': areas }, - content_type='text/plain; charset=UTF-8') + fpath = Path(settings.CHARTER_PATH) / "1wg-charters.txt" + try: + content = fpath.read_bytes() + except IOError: + raise Http404 + return HttpResponse(content, content_type="text/plain; charset=UTF-8") + -@cache_page ( 60 * 60, cache="slowpages" ) def wg_charters_by_acronym(request, group_type): if group_type != "wg": raise Http404 + fpath = Path(settings.CHARTER_PATH) / "1wg-charters-by-acronym.txt" + try: + content = fpath.read_bytes() + except IOError: + raise Http404 + return HttpResponse(content, content_type="text/plain; charset=UTF-8") - groups = Group.objects.filter(type="wg", state="active").exclude(parent=None).order_by("acronym") - for group in groups: - fill_in_charter_info(group) - fill_in_wg_roles(group) - fill_in_wg_drafts(group) - return render(request, 'group/1wg-charters-by-acronym.txt', - { 'groups': groups }, - content_type='text/plain; charset=UTF-8') def active_groups(request, group_type=None): diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 8d44cb7c17..a60d3b010a 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -949,9 +949,9 @@ def post_process(doc): Does post processing on uploaded file. - Convert PPT to PDF ''' - if is_powerpoint(doc) and hasattr(settings, 'SECR_PPT2PDF_COMMAND'): + if is_powerpoint(doc) and hasattr(settings, 'PPT2PDF_COMMAND'): try: - cmd = list(settings.SECR_PPT2PDF_COMMAND) # Don't operate on the list actually in settings + cmd = list(settings.PPT2PDF_COMMAND) # Don't operate on the list actually in settings cmd.append(doc.get_file_path()) # outdir cmd.append(os.path.join(doc.get_file_path(), doc.uploaded_filename)) # filename subprocess.check_call(cmd) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 35679dcaa1..913c6c987e 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -16789,49 +16789,5 @@ }, "model": "stats.countryalias", "pk": 303 - }, - { - "fields": { - "command": "xym", - "switch": "--version", - "time": "2024-03-21T07:06:23.405Z", - "used": true, - "version": "xym 0.7.0" - }, - "model": "utils.versioninfo", - "pk": 1 - }, - { - "fields": { - "command": "pyang", - "switch": "--version", - "time": "2024-03-21T07:06:23.755Z", - "used": true, - "version": "pyang 2.6.0" - }, - "model": "utils.versioninfo", - "pk": 2 - }, - { - "fields": { - "command": "yanglint", - "switch": "--version", - "time": "2024-03-21T07:06:23.773Z", - "used": true, - "version": "yanglint SO 1.9.2" - }, - "model": "utils.versioninfo", - "pk": 3 - }, - { - "fields": { - "command": "xml2rfc", - "switch": "--version", - "time": "2024-03-21T07:06:24.609Z", - "used": true, - "version": "xml2rfc 3.20.1" - }, - "model": "utils.versioninfo", - "pk": 4 } ] diff --git a/ietf/name/management/commands/generate_name_fixture.py b/ietf/name/management/commands/generate_name_fixture.py index bbf33e600e..ef30e54c73 100644 --- a/ietf/name/management/commands/generate_name_fixture.py +++ b/ietf/name/management/commands/generate_name_fixture.py @@ -77,7 +77,6 @@ def output(seq): from ietf.mailtrigger.models import MailTrigger, Recipient from ietf.meeting.models import BusinessConstraint from ietf.stats.models import CountryAlias - from ietf.utils.models import VersionInfo # Grab all ietf.name.models for n in dir(ietf.name.models): @@ -87,7 +86,7 @@ def output(seq): model_objects[model_name(item)] = list(item.objects.all().order_by('pk')) for m in ( BallotType, State, StateType, GroupFeatures, MailTrigger, Recipient, - CountryAlias, VersionInfo, BusinessConstraint ): + CountryAlias, BusinessConstraint ): model_objects[model_name(m)] = list(m.objects.all().order_by('pk')) for m in ( DBTemplate, ): diff --git a/ietf/nomcom/management/commands/send_reminders.py b/ietf/nomcom/management/commands/send_reminders.py deleted file mode 100644 index bc10425430..0000000000 --- a/ietf/nomcom/management/commands/send_reminders.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import syslog - -from django.core.management.base import BaseCommand - -from ietf.nomcom.models import NomCom, NomineePosition -from ietf.nomcom.utils import send_accept_reminder_to_nominee,send_questionnaire_reminder_to_nominee -from ietf.utils.timezone import date_today - - -def log(message): - syslog.syslog(message) - -def is_time_to_send(nomcom,send_date,nomination_date): - if nomcom.reminder_interval: - days_passed = (send_date - nomination_date).days - return days_passed > 0 and days_passed % nomcom.reminder_interval == 0 - else: - return bool(nomcom.reminderdates_set.filter(date=send_date)) - -class Command(BaseCommand): - help = ("Send acceptance and questionnaire reminders to nominees") - - def handle(self, *args, **options): - for nomcom in NomCom.objects.filter(group__state__slug='active'): - nps = NomineePosition.objects.filter(nominee__nomcom=nomcom,nominee__duplicated__isnull=True) - for nominee_position in nps.pending(): - if is_time_to_send(nomcom, date_today(), nominee_position.time.date()): - send_accept_reminder_to_nominee(nominee_position) - log('Sent accept reminder to %s' % nominee_position.nominee.email.address) - for nominee_position in nps.accepted().without_questionnaire_response(): - if is_time_to_send(nomcom, date_today(), nominee_position.time.date()): - send_questionnaire_reminder_to_nominee(nominee_position) - log('Sent questionnaire reminder to %s' % nominee_position.nominee.email.address) diff --git a/ietf/nomcom/tasks.py b/ietf/nomcom/tasks.py new file mode 100644 index 0000000000..3d063a6b26 --- /dev/null +++ b/ietf/nomcom/tasks.py @@ -0,0 +1,10 @@ +# Copyright The IETF Trust 2024, All Rights Reserved + +from celery import shared_task + +from .utils import send_reminders + + +@shared_task +def send_nomcom_reminders_task(): + send_reminders() diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 9a615c91d1..8f94cc7fc5 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -40,14 +40,14 @@ NomineePositionStateName, Feedback, FeedbackTypeName, \ Nomination, FeedbackLastSeen, TopicFeedbackLastSeen, ReminderDates, \ NomCom -from ietf.nomcom.management.commands.send_reminders import Command, is_time_to_send from ietf.nomcom.factories import NomComFactory, FeedbackFactory, TopicFactory, \ nomcom_kwargs_for_year, provide_private_key_to_test_client, \ key +from ietf.nomcom.tasks import send_nomcom_reminders_task from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \ get_hash_nominee_position, is_eligible, list_eligible, \ get_eligibility_date, suggest_affiliation, ingest_feedback_email, \ - decorate_volunteers_with_qualifications + decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.stats.models import MeetingRegistration @@ -1207,36 +1207,41 @@ def tearDown(self): teardown_test_public_keys_dir(self) super().tearDown() - def test_is_time_to_send(self): + def test_is_time_to_send_reminder(self): self.nomcom.reminder_interval = 4 today = date_today() - self.assertTrue(is_time_to_send(self.nomcom,today+datetime.timedelta(days=4),today)) + self.assertTrue( + _is_time_to_send_reminder(self.nomcom, today + datetime.timedelta(days=4), today) + ) for delta in range(4): - self.assertFalse(is_time_to_send(self.nomcom,today+datetime.timedelta(days=delta),today)) + self.assertFalse( + _is_time_to_send_reminder( + self.nomcom, today + datetime.timedelta(days=delta), today + ) + ) self.nomcom.reminder_interval = None - self.assertFalse(is_time_to_send(self.nomcom,today,today)) + self.assertFalse(_is_time_to_send_reminder(self.nomcom, today, today)) self.nomcom.reminderdates_set.create(date=today) - self.assertTrue(is_time_to_send(self.nomcom,today,today)) + self.assertTrue(_is_time_to_send_reminder(self.nomcom, today, today)) - def test_command(self): - c = Command() - messages_before=len(outbox) + def test_send_reminders(self): + messages_before = len(outbox) self.nomcom.reminder_interval = 3 self.nomcom.save() - c.handle(None,None) + send_reminders() self.assertEqual(len(outbox), messages_before + 2) self.assertIn('nominee1@example.org', outbox[-1]['To']) self.assertIn('please complete', outbox[-1]['Subject']) self.assertIn('nominee1@example.org', outbox[-2]['To']) self.assertIn('please accept', outbox[-2]['Subject']) - messages_before=len(outbox) + messages_before = len(outbox) self.nomcom.reminder_interval = 4 self.nomcom.save() - c.handle(None,None) + send_reminders() self.assertEqual(len(outbox), messages_before + 1) self.assertIn('nominee2@example.org', outbox[-1]['To']) self.assertIn('please accept', outbox[-1]['Subject']) - + def test_remind_accept_view(self): url = reverse('ietf.nomcom.views.send_reminder_mail', kwargs={'year': NOMCOM_YEAR,'type':'accept'}) login_testing_unauthorized(self, CHAIR_USER, url) @@ -3048,3 +3053,10 @@ def test_reclassify_feedback_unrelated(self): self.assertEqual(fb.type_id, 'junk') self.assertEqual(Feedback.objects.filter(type='read').count(), 0) self.assertEqual(Feedback.objects.filter(type='junk').count(), 1) + + +class TaskTests(TestCase): + @mock.patch("ietf.nomcom.tasks.send_reminders") + def test_send_nomcom_reminders_task(self, mock_send): + send_nomcom_reminders_task() + self.assertEqual(mock_send.call_count, 1) diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index 53e775deb9..ab155ef1d5 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -747,3 +747,27 @@ def ingest_feedback_email(message: bytes, year: int): email_original_message=message, ) from err log("Received nomcom email from %s" % feedback.author) + + +def _is_time_to_send_reminder(nomcom, send_date, nomination_date): + if nomcom.reminder_interval: + days_passed = (send_date - nomination_date).days + return days_passed > 0 and days_passed % nomcom.reminder_interval == 0 + else: + return bool(nomcom.reminderdates_set.filter(date=send_date)) + + +def send_reminders(): + from .models import NomCom, NomineePosition + for nomcom in NomCom.objects.filter(group__state__slug="active"): + nps = NomineePosition.objects.filter( + nominee__nomcom=nomcom, nominee__duplicated__isnull=True + ) + for nominee_position in nps.pending(): + if _is_time_to_send_reminder(nomcom, date_today(), nominee_position.time.date()): + send_accept_reminder_to_nominee(nominee_position) + log(f"Sent accept reminder to {nominee_position.nominee.email.address}") + for nominee_position in nps.accepted().without_questionnaire_response(): + if _is_time_to_send_reminder(nomcom, date_today(), nominee_position.time.date()): + send_questionnaire_reminder_to_nominee(nominee_position) + log(f"Sent questionnaire reminder to {nominee_position.nominee.email.address}") diff --git a/ietf/secr/meetings/tests.py b/ietf/secr/meetings/tests.py index 0e51ff8ca2..a241d2b5eb 100644 --- a/ietf/secr/meetings/tests.py +++ b/ietf/secr/meetings/tests.py @@ -3,14 +3,11 @@ import datetime -import os -import shutil from pyquery import PyQuery import debug # pyflakes:ignore -from django.conf import settings from django.urls import reverse from django.utils import timezone @@ -27,24 +24,6 @@ class SecrMeetingTestCase(TestCase): settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['AGENDA_PATH'] - def setUp(self): - super().setUp() - self.bluesheet_dir = self.tempdir('bluesheet') - self.bluesheet_path = os.path.join(self.bluesheet_dir,'blue_sheet.rtf') - self.saved_secr_blue_sheet_path = settings.SECR_BLUE_SHEET_PATH - settings.SECR_BLUE_SHEET_PATH = self.bluesheet_path - - # n.b., the bluesheet upload relies on SECR_PROCEEDINGS_DIR being the same - # as AGENDA_PATH. This is probably a bug, but may not be worth fixing if - # the secr app is on the way out. - self.saved_secr_proceedings_dir = settings.SECR_PROCEEDINGS_DIR - settings.SECR_PROCEEDINGS_DIR = settings.AGENDA_PATH - - def tearDown(self): - settings.SECR_PROCEEDINGS_DIR = self.saved_secr_proceedings_dir - settings.SECR_BLUE_SHEET_PATH = self.saved_secr_blue_sheet_path - shutil.rmtree(self.bluesheet_dir) - super().tearDown() def test_main(self): "Main Test" @@ -416,4 +395,4 @@ def test_get_times(self): times = get_times(meeting,day) values = [ x[0] for x in times ] self.assertTrue(times) - self.assertTrue(timeslot.time.strftime('%H%M') in values) \ No newline at end of file + self.assertTrue(timeslot.time.strftime('%H%M') in values) diff --git a/ietf/settings.py b/ietf/settings.py index cd8c0700a1..8bb264bd67 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -125,6 +125,10 @@ # In the future (relative to 4.2), the default will become 'django.db.models.BigAutoField.' DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +# OIDC configuration +_SITE_URL = os.environ.get("OIDC_SITE_URL", None) +if _SITE_URL is not None: + SITE_URL = _SITE_URL if SERVER_MODE == 'production': MEDIA_ROOT = '/a/www/www6s/lib/dt/media/' @@ -248,13 +252,21 @@ def skip_unreadable_post(record): 'level': 'INFO', }, 'django.security': { - 'handlers': ['debug_console', ], + 'handlers': ['debug_console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'celery': { + 'handlers': ['debug_console'], 'level': 'INFO', }, - 'oidc_provider': { - 'handlers': ['debug_console', ], - 'level': 'DEBUG', - }, }, # # No logger filters @@ -263,14 +275,7 @@ def skip_unreadable_post(record): 'console': { 'level': 'DEBUG', 'class': 'logging.StreamHandler', - 'formatter': 'plain', - }, - 'syslog': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'facility': 'user', - 'formatter': 'plain', - 'address': '/dev/log', + 'formatter': 'json', }, 'debug_console': { # Active only when DEBUG=True @@ -325,18 +330,12 @@ def skip_unreadable_post(record): 'style': '{', 'format': '{levelname}: {name}:{lineno}: {message}', }, + 'json' : { + '()': 'pythonjsonlogger.jsonlogger.JsonFormatter' + } }, } -# This should be overridden by settings_local for any logger where debug (or -# other) custom log settings are wanted. Use "ietf/manage.py showloggers -l" -# to show registered loggers. The content here should match the levels above -# and is shown as an example: -UTILS_LOGGER_LEVELS: Dict[str, str] = { -# 'django': 'INFO', -# 'django.server': 'INFO', -} - # End logging # ------------------------------------------------------------------------ @@ -671,6 +670,7 @@ def skip_unreadable_post(record): INTERNET_DRAFT_PDF_PATH = '/a/www/ietf-datatracker/pdf/' RFC_PATH = '/a/www/ietf-ftp/rfc/' CHARTER_PATH = '/a/ietfdata/doc/charter/' +CHARTER_COPY_PATH = '/a/www/ietf-ftp/ietf' # copy 1wg-charters files here if set BOFREQ_PATH = '/a/ietfdata/doc/bofreq/' CONFLICT_REVIEW_PATH = '/a/ietfdata/doc/conflict-review' STATUS_CHANGE_PATH = '/a/ietfdata/doc/status-change' @@ -978,7 +978,7 @@ def skip_unreadable_post(record): DAYS_TO_EXPIRE_REGISTRATION_LINK = 3 MINUTES_TO_EXPIRE_RESET_PASSWORD_LINK = 60 HTPASSWD_COMMAND = "/usr/bin/htpasswd" -HTPASSWD_FILE = "/www/htpasswd" +HTPASSWD_FILE = "/a/www/htpasswd" # Generation of pdf files GHOSTSCRIPT_COMMAND = "/usr/bin/gs" @@ -989,12 +989,11 @@ def skip_unreadable_post(record): # Timezone files for iCalendar TZDATA_ICS_PATH = BASE_DIR + '/../vzic/zoneinfo/' -SECR_BLUE_SHEET_PATH = '/a/www/ietf-datatracker/documents/blue_sheet.rtf' -SECR_BLUE_SHEET_URL = IDTRACKER_BASE_URL + '/documents/blue_sheet.rtf' -SECR_INTERIM_LISTING_DIR = '/a/www/www6/meeting/interim' -SECR_MAX_UPLOAD_SIZE = 40960000 -SECR_PROCEEDINGS_DIR = '/a/www/www6s/proceedings/' -SECR_PPT2PDF_COMMAND = ['/usr/bin/soffice','--headless','--convert-to','pdf:writer_globaldocument_pdf_Export','--outdir'] +DATATRACKER_MAX_UPLOAD_SIZE = 40960000 +PPT2PDF_COMMAND = [ + "/usr/bin/soffice", "--headless", "--convert-to", "pdf:writer_globaldocument_pdf_Export", "--outdir" +] + STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/' PROCEEDINGS_VERSION_CHANGES = [ 0, # version 1 @@ -1206,81 +1205,83 @@ def skip_unreadable_post(record): MIDDLEWARE += DEV_MIDDLEWARE TEMPLATES[0]['OPTIONS']['context_processors'] += DEV_TEMPLATE_CONTEXT_PROCESSORS -if 'CACHES' not in locals(): - if SERVER_MODE == 'production': +if "CACHES" not in locals(): + if SERVER_MODE == "production": + MEMCACHED_HOST = os.environ.get("MEMCACHED_SERVICE_HOST", "127.0.0.1") + MEMCACHED_PORT = os.environ.get("MEMCACHED_SERVICE_PORT", "11211") CACHES = { - 'default': { - 'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', - 'LOCATION': '127.0.0.1:11211', - 'VERSION': __version__, - 'KEY_PREFIX': 'ietf:dt', - 'KEY_FUNCTION': lambda key, key_prefix, version: ( + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + "KEY_FUNCTION": lambda key, key_prefix, version: ( f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), }, - 'sessions': { - 'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', - 'LOCATION': '127.0.0.1:11211', + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", # No release-specific VERSION setting. - 'KEY_PREFIX': 'ietf:dt', + "KEY_PREFIX": "ietf:dt", }, - 'htmlized': { - 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/a/cache/datatracker/htmlized', - 'OPTIONS': { - 'MAX_ENTRIES': 100000, # 100,000 + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 }, }, - 'pdfized': { - 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/a/cache/datatracker/pdfized', - 'OPTIONS': { - 'MAX_ENTRIES': 100000, # 100,000 + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 }, }, - 'slowpages': { - 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/a/cache/datatracker/slowpages', - 'OPTIONS': { - 'MAX_ENTRIES': 5000, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, }, }, } else: CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + "default": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", #'BACKEND': 'ietf.utils.cache.LenientMemcacheCache', #'LOCATION': '127.0.0.1:11211', #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'VERSION': __version__, - 'KEY_PREFIX': 'ietf:dt', + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", }, - 'sessions': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + "sessions": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", }, - 'htmlized': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + "htmlized": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/var/cache/datatracker/htmlized', - 'OPTIONS': { - 'MAX_ENTRIES': 1000, + "LOCATION": "/var/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 1000, }, }, - 'pdfized': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + "pdfized": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/var/cache/datatracker/pdfized', - 'OPTIONS': { - 'MAX_ENTRIES': 1000, + "LOCATION": "/var/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 1000, }, }, - 'slowpages': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + "slowpages": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', - 'LOCATION': '/var/cache/datatracker/', - 'OPTIONS': { - 'MAX_ENTRIES': 5000, + "LOCATION": "/var/cache/datatracker/", + "OPTIONS": { + "MAX_ENTRIES": 5000, }, }, } diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 3f69f0ae38..024512a8db 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -60,3 +60,36 @@ def __getitem__(self, item): TEMPLATES[0]['OPTIONS']['context_processors'] = [ p for p in TEMPLATES[0]['OPTIONS']['context_processors'] if not p in DEV_TEMPLATE_CONTEXT_PROCESSORS ] # pyflakes:ignore REQUEST_PROFILE_STORE_ANONYMOUS_SESSIONS = False + +# Override loggers with a safer set in case things go to the log during testing. Specifically, +# make sure there are no syslog loggers that might send things to a real syslog. +LOGGING["loggers"] = { # pyflakes:ignore + 'django': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'django.request': { + 'handlers': ['debug_console'], + 'level': 'ERROR', + }, + 'django.server': { + 'handlers': ['django.server'], + 'level': 'INFO', + }, + 'django.security': { + 'handlers': ['debug_console', ], + 'level': 'INFO', + }, + 'oidc_provider': { + 'handlers': ['debug_console', ], + 'level': 'DEBUG', + }, + 'datatracker': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, + 'celery': { + 'handlers': ['debug_console'], + 'level': 'INFO', + }, +} diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 5822f155f5..d29e2a2355 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -14,8 +14,8 @@ import debug # pyflakes:ignore +from ietf.utils import tool_version from ietf.utils.log import log, assertion -from ietf.utils.models import VersionInfo from ietf.utils.pipe import pipe from ietf.utils.test_runner import set_coverage_checking @@ -177,8 +177,10 @@ def check_file_txt(self, path): model_list = list(set(model_list)) command = "xym" - cmd_version = VersionInfo.objects.get(command=command).version - message = "%s:\n%s\n\n" % (cmd_version, out.replace('\n\n','\n').strip() if code == 0 else err) + message = "{version}:\n{output}\n\n".format( + version=tool_version[command], + output=out.replace('\n\n', '\n').strip() if code == 0 else err, + ) results.append({ "name": name, @@ -209,7 +211,6 @@ def check_file_txt(self, path): # pyang cmd_template = settings.SUBMIT_PYANG_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(libs=modpath, model=path) venv_path = os.environ.get('VIRTUAL_ENV') or os.path.join(os.getcwd(), 'env') venv_bin = os.path.join(venv_path, 'bin') @@ -238,14 +239,17 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) # yanglint set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): cmd_template = settings.SUBMIT_YANGLINT_COMMAND command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd_version = VersionInfo.objects.get(command=command).version cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) @@ -264,7 +268,11 @@ def check_file_txt(self, path): except ValueError: pass #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "%s: %s:\n%s\n" % (cmd_version, cmd_template, out+"No validation errors\n" if (code == 0 and len(err) == 0) else out+err) + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) set_coverage_checking(True) else: errors += 1 @@ -293,4 +301,4 @@ def check_file_txt(self, path): items = [ e for res in results for e in res["items"] ] info['items'] = items info['code']['yang'] = model_list - return passed, message, errors, warnings, info \ No newline at end of file + return passed, message, errors, warnings, info diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 08b898c13a..58a47aef8b 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -49,9 +49,9 @@ from ietf.submit.forms import SubmissionBaseUploadForm, SubmissionAutoUploadForm from ietf.submit.models import Submission, Preapproval, SubmissionExtResource from ietf.submit.tasks import cancel_stale_submissions, process_and_accept_uploaded_submission_task +from ietf.utils import tool_version from ietf.utils.accesstoken import generate_access_token from ietf.utils.mail import outbox, get_payload_text -from ietf.utils.models import VersionInfo from ietf.utils.test_utils import login_testing_unauthorized, TestCase from ietf.utils.timezone import date_today from ietf.utils.draft import PlaintextDraft @@ -1854,7 +1854,7 @@ def test_submit_invalid_yang(self): # m = q('#yang-validation-message').text() for command in ['xym', 'pyang', 'yanglint']: - version = VersionInfo.objects.get(command=command).version + version = tool_version[command] if command != 'yanglint' or (settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY)): self.assertIn(version, m) self.assertIn("draft-yang-testing-invalid-00.txt", m) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index bc1218601f..53e23d7913 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -13,6 +13,7 @@ from ietf.sync import iana from ietf.sync import rfceditor +from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue from ietf.utils import log from ietf.utils.timezone import date_today @@ -70,6 +71,33 @@ def rfc_editor_index_update_task(full_index=False): log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) +@shared_task +def rfc_editor_queue_updates_task(): + log.log(f"Updating RFC Editor queue states from {settings.RFC_EDITOR_QUEUE_URL}") + try: + response = requests.get( + settings.RFC_EDITOR_QUEUE_URL, + timeout=30, # seconds + ) + except requests.Timeout as exc: + log.log(f"GET request timed out retrieving RFC editor queue: {exc}") + return # failed + drafts, warnings = parse_queue(io.StringIO(response.text)) + for w in warnings: + log.log(f"Warning: {w}") + + if len(drafts) < MIN_QUEUE_RESULTS: + log.log("Not enough results, only %s" % len(drafts)) + return # failed + + changed, warnings = update_drafts_from_queue(drafts) + for w in warnings: + log.log(f"Warning: {w}") + + for c in changed: + log.log(f"Updated {c}") + + @shared_task def iana_changes_update_task(): # compensate to avoid we ask for something that happened now and then diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index db56190954..b0cdf863f0 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -886,6 +886,36 @@ def json(self): tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") + @mock.patch("ietf.sync.tasks.update_drafts_from_queue") + @mock.patch("ietf.sync.tasks.parse_queue") + def test_rfc_editor_queue_updates_task(self, mock_parse, mock_update): + # test a request timeout + self.requests_mock.get("https://rfc-editor.example.com/queue/", exc=requests.exceptions.Timeout) + tasks.rfc_editor_queue_updates_task() + self.assertFalse(mock_parse.called) + self.assertFalse(mock_update.called) + + # now return a value rather than an exception + self.requests_mock.get("https://rfc-editor.example.com/queue/", text="the response") + + # mock returning < MIN_QUEUE_RESULTS values - treated as an error, so no update takes place + mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS - 1)], ["a warning"]) + tasks.rfc_editor_queue_updates_task() + self.assertEqual(mock_parse.call_count, 1) + self.assertEqual(mock_parse.call_args[0][0].read(), "the response") + self.assertFalse(mock_update.called) + mock_parse.reset_mock() + + # mock returning +. MIN_QUEUE_RESULTS - should succeed + mock_parse.return_value = ([n for n in range(rfceditor.MIN_QUEUE_RESULTS)], ["a warning"]) + mock_update.return_value = ([1,2,3], ["another warning"]) + tasks.rfc_editor_queue_updates_task() + self.assertEqual(mock_parse.call_count, 1) + self.assertEqual(mock_parse.call_args[0][0].read(), "the response") + self.assertEqual(mock_update.call_count, 1) + self.assertEqual(mock_update.call_args, mock.call([n for n in range(rfceditor.MIN_QUEUE_RESULTS)])) + @override_settings(IANA_SYNC_CHANGES_URL="https://iana.example.com/sync/") @mock.patch("ietf.sync.tasks.iana.update_history_with_changes") @mock.patch("ietf.sync.tasks.iana.parse_changes_json") diff --git a/ietf/sync/views.py b/ietf/sync/views.py index 788e982f7c..da407e1efd 100644 --- a/ietf/sync/views.py +++ b/ietf/sync/views.py @@ -2,7 +2,6 @@ # -*- coding: utf-8 -*- import datetime -import subprocess import os import json @@ -79,30 +78,18 @@ def notify(request, org, notification): raise Http404 if request.method == "POST": - def runscript(name): - python = os.path.join(os.path.dirname(settings.BASE_DIR), "env", "bin", "python") - cmd = [python, os.path.join(SYNC_BIN_PATH, name)] - cmdstring = " ".join(cmd) - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, err = p.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - if p.returncode: - log("Subprocess error %s when running '%s': %s %s" % (p.returncode, cmd, err, out)) - raise subprocess.CalledProcessError(p.returncode, cmdstring, "\n".join([err, out])) - if notification == "index": log("Queuing RFC Editor index sync from notify view POST") tasks.rfc_editor_index_update_task.delay() + elif notification == "queue": + log("Queuing RFC Editor queue sync from notify view POST") + tasks.rfc_editor_queue_updates_task.delay() elif notification == "changes": log("Queuing IANA changes sync from notify view POST") tasks.iana_changes_update_task.delay() elif notification == "protocols": log("Queuing IANA protocols sync from notify view POST") tasks.iana_protocols_update_task.delay() - elif notification == "queue": - log("Running sync script from notify view POST") - runscript("rfc-editor-queue-updates") return HttpResponse("OK", content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) diff --git a/ietf/templates/doc/investigate.html b/ietf/templates/doc/investigate.html index 80b004c838..bdcf644406 100644 --- a/ietf/templates/doc/investigate.html +++ b/ietf/templates/doc/investigate.html @@ -23,12 +23,24 @@

These can be authenticated

Name Last Modified On Link + Source {% for path in results.can_verify %} {% with url=path|url_for_path %} - {{path.name}}{{path|mtime|date:"DATETIME_FORMAT"}}{{url}} + + {{path.name}} + + {% if path|mtime_is_epoch %} + Timestamp has been lost (is Unix Epoch) + {% else %} + {{path|mtime|date:"DATETIME_FORMAT"}} + {% endif %} + + {{url}} + {{path}} + {% endwith %} {% endfor %} @@ -53,7 +65,13 @@

These are in the archive, but cannot be authenticated

{% with url=path|url_for_path %} {{path.name}} - {{path|mtime|date:"DATETIME_FORMAT"}} + + {% if path|mtime_is_epoch %} + Timestamp has been lost (is Unix Epoch) + {% else %} + {{path|mtime|date:"DATETIME_FORMAT"}} + {% endif %} + {{url}} {{path}} @@ -77,7 +95,13 @@

These are unexpected and we do not know what their origin is. These cannot b {% with url=path|url_for_path %} {{path.name}} - {{path|mtime|date:"DATETIME_FORMAT"}} + + {% if path|mtime_is_epoch %} + Timestamp has been lost (is Unix Epoch) + {% else %} + {{path|mtime|date:"DATETIME_FORMAT"}} + {% endif %} + {{url}} {% endwith %} diff --git a/ietf/templates/group/group_entry_with_charter.txt b/ietf/templates/group/group_entry_with_charter.txt index 50a40a9afd..846cc395b8 100644 --- a/ietf/templates/group/group_entry_with_charter.txt +++ b/ietf/templates/group/group_entry_with_charter.txt @@ -37,7 +37,7 @@ Goals and Milestones: {% for milestone in group.milestones %} {% if milestone.resolved %}{{ milestone.resolved }} {% else %}{{ milestone.due|date:"M Y" }}{% endif %} - {{ milestone.desc }} {% endfor %} Internet-Drafts: -{% for document in group.drafts %} - {{ document.title }} [{{ document.name }}-{{ document.rev }}] ({{ document.pages }} pages) +{% for document in group.drafts %} - {{ document.title|clean_whitespace }} [{{ document.name }}-{{ document.rev }}] ({{ document.pages }} pages) {% endfor %} {% if group.rfcs %}Requests for Comments: {% for document in group.rfcs %} {{ document.name.upper }}: {{ document.title}} ({{ document.pages }} pages){% for r in document.rel %} diff --git a/ietf/utils/__init__.py b/ietf/utils/__init__.py index 7f1df97602..fbe55eb043 100644 --- a/ietf/utils/__init__.py +++ b/ietf/utils/__init__.py @@ -1 +1,29 @@ -# Copyright The IETF Trust 2007, All Rights Reserved +# Copyright The IETF Trust 2007-2024, All Rights Reserved +import subprocess + + +class _ToolVersionManager: + _known = [ + "pyang", + "xml2rfc", + "xym", + "yanglint", + ] + _versions: dict[str, str] = dict() + + def __getitem__(self, item): + if item not in self._known: + return "Unknown" + elif item not in self._versions: + try: + self._versions[item] = subprocess.run( + [item, "--version"], + capture_output=True, + check=True, + ).stdout.decode().strip() + except subprocess.CalledProcessError: + return "Unknown" + return self._versions[item] + + +tool_version = _ToolVersionManager() diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index fa1ebb7081..6c1c8726e1 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -5,8 +5,6 @@ from django.contrib import admin from django.utils.encoding import force_str -from ietf.utils.models import VersionInfo - def name(obj): if hasattr(obj, 'abbrev'): return obj.abbrev() @@ -58,8 +56,3 @@ class DumpInfoAdmin(admin.ModelAdmin): list_display = ['date', 'host', 'tz'] list_filter = ['date'] admin.site.register(DumpInfo, DumpInfoAdmin) - -class VersionInfoAdmin(admin.ModelAdmin): - list_display = ['command', 'switch', 'version', 'time', ] -admin.site.register(VersionInfo, VersionInfoAdmin) - diff --git a/ietf/utils/log.py b/ietf/utils/log.py index d5a54e5516..2a068ade9a 100644 --- a/ietf/utils/log.py +++ b/ietf/utils/log.py @@ -9,37 +9,10 @@ import os.path import traceback -from typing import Callable # pyflakes:ignore - -try: - import syslog - logfunc = syslog.syslog # type: Callable -except ImportError: # import syslog will fail on Windows boxes - logging.basicConfig(filename='tracker.log',level=logging.INFO) - logfunc = logging.info - pass - from django.conf import settings import debug # pyflakes:ignore -formatter = logging.Formatter('{levelname}: {name}:{lineno}: {message}', style='{') -for name, level in settings.UTILS_LOGGER_LEVELS.items(): - logger = logging.getLogger(name) - if not logger.hasHandlers(): - debug.say(' Adding handlers to logger %s' % logger.name) - - handlers = [ - logging.StreamHandler(), - logging.handlers.SysLogHandler(address='/dev/log', - facility=logging.handlers.SysLogHandler.LOG_USER), - ] - for h in handlers: - h.setFormatter(formatter) - h.setLevel(level) - logger.addHandler(h) - debug.say(" Setting %s logging level to %s" % (logger.name, level)) - logger.setLevel(level) def getclass(frame): cls = None @@ -56,20 +29,9 @@ def getcaller(): return (pmodule, pclass, pfunction, pfile, pline) def log(msg, e=None): - "Uses syslog by preference. Logs the given calling point and message." - global logfunc - def _flushfunc(): - pass - _logfunc = logfunc - if settings.SERVER_MODE == 'test': - if getattr(settings, 'show_logging', False) is True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) - else: + "Logs the given calling point and message to the logging framework's datatracker handler at severity INFO" + if settings.SERVER_MODE == 'test' and not getattr(settings, 'show_logging',False): return - elif settings.DEBUG == True: - _logfunc = debug.say - _flushfunc = sys.stdout.flush # pyflakes:ignore (intentional redefinition) if not isinstance(msg, str): msg = msg.encode('unicode_escape') try: @@ -82,11 +44,8 @@ def _flushfunc(): where = " in " + func + "()" except IndexError: file, line, where = "/", 0, "" - _flushfunc() - _logfunc("ietf%s(%d)%s: %s" % (file, line, where, msg)) - -logger = logging.getLogger('django') + logging.getLogger("datatracker").info(msg=msg, extra = {"file":file, "line":line, "where":where}) def exc_parts(): @@ -124,6 +83,7 @@ def assertion(statement, state=True, note=None): This acts like an assertion. It uses the django logger in order to send the failed assertion and a backtrace as for an internal server error. """ + logger = logging.getLogger("django") # Note this is a change - before this would have gone to "django" frame = inspect.currentframe().f_back value = eval(statement, frame.f_globals, frame.f_locals) if bool(value) != bool(state): @@ -148,6 +108,7 @@ def assertion(statement, state=True, note=None): def unreachable(date="(unknown)"): "Raises an assertion or sends traceback to admins if executed." + logger = logging.getLogger("django") frame = inspect.currentframe().f_back if settings.DEBUG is True or settings.SERVER_MODE == 'test': raise AssertionError("Arrived at code in %s() which was marked unreachable on %s." % (frame.f_code.co_name, date)) diff --git a/ietf/utils/management/commands/patch_libraries.py b/ietf/utils/management/commands/patch_libraries.py new file mode 100644 index 0000000000..d9ae11097b --- /dev/null +++ b/ietf/utils/management/commands/patch_libraries.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +import django + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from pathlib import Path + +from ietf.utils import patch + + +class Command(BaseCommand): + """Apply IETF patches to libraries""" + requires_system_checks = tuple() + + def handle(self, *args, **options): + library_path = Path(django.__file__).parent.parent + top_dir = Path(settings.BASE_DIR).parent + + # All patches in settings.CHECKS_LIBRARY_PATCHES_TO_APPLY must have a + # relative file path starting from the site-packages dir, e.g. + # 'django/db/models/fields/__init__.py' + for patch_file in settings.CHECKS_LIBRARY_PATCHES_TO_APPLY: + patch_set = patch.fromfile(top_dir / Path(patch_file)) + if not patch_set: + raise CommandError(f"Could not parse patch file '{patch_file}'") + if not patch_set.apply(root=bytes(library_path)): + raise CommandError(f"Could not apply the patch from '{patch_file}'") + if patch_set.already_patched: + self.stdout.write(f"Patch from '{patch_file}' was already applied") + else: + self.stdout.write(f"Applied the patch from '{patch_file}'") diff --git a/ietf/utils/management/commands/periodic_tasks.py b/ietf/utils/management/commands/periodic_tasks.py index e359382839..792eb0068b 100644 --- a/ietf/utils/management/commands/periodic_tasks.py +++ b/ietf/utils/management/commands/periodic_tasks.py @@ -141,6 +141,16 @@ def create_default_tasks(self): ), ) + PeriodicTask.objects.get_or_create( + name="Expire Last Calls", + task="ietf.doc.tasks.expire_last_calls_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["daily"], + description="Move docs whose last call has expired to their next states", + ), + ) + PeriodicTask.objects.get_or_create( name="Sync with IANA changes", task="ietf.sync.tasks.iana_changes_update_task", @@ -181,6 +191,56 @@ def create_default_tasks(self): ) ) + PeriodicTask.objects.get_or_create( + name="Generate idnits2 rfcs-obsoleted blob", + task="ietf.doc.tasks.generate_idnits2_rfcs_obsoleted_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate the rfcs-obsoleted file used by idnits", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Generate idnits2 rfc-status blob", + task="ietf.doc.tasks.generate_idnits2_rfc_status_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate the rfc_status blob used by idnits", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Send NomCom reminders", + task="ietf.nomcom.tasks.send_nomcom_reminders_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["daily"], + description="Send acceptance and questionnaire reminders to nominees", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Generate WG charter files", + task="ietf.group.tasks.generate_wg_charters_files_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Update 1wg-charters.txt and 1wg-charters-by-acronym.txt", + ), + ) + + PeriodicTask.objects.get_or_create( + name="Generate I-D bibxml files", + task="ietf.doc.tasks.generate_draft_bibxml_files_task", + defaults=dict( + enabled=False, + crontab=self.crontabs["hourly"], + description="Generate draft bibxml files for the last week's drafts", + ), + ) + def show_tasks(self): for label, crontab in self.crontabs.items(): tasks = PeriodicTask.objects.filter(crontab=crontab).order_by( diff --git a/ietf/utils/management/commands/showloggers.py b/ietf/utils/management/commands/showloggers.py index 3de9db0c06..b79da9ce26 100644 --- a/ietf/utils/management/commands/showloggers.py +++ b/ietf/utils/management/commands/showloggers.py @@ -11,18 +11,7 @@ import debug # pyflakes:ignore class Command(BaseCommand): - """ - Display a list or tree representation of python loggers. - - Add a UTILS_LOGGER_LEVELS setting in settings_local.py to configure - non-default logging levels for any registered logger, for instance: - - UTILS_LOGGER_LEVELS = { - 'oicd_provider': 'DEBUG', - 'urllib3.connection': 'DEBUG', - } - - """ + """Display a list or tree representation of python loggers""" help = dedent(__doc__).strip() diff --git a/ietf/utils/management/commands/update_external_command_info.py b/ietf/utils/management/commands/update_external_command_info.py deleted file mode 100644 index e9e24f000d..0000000000 --- a/ietf/utils/management/commands/update_external_command_info.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright The IETF Trust 2017-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import sys - -from textwrap import dedent - -from django.core.management.base import BaseCommand - -import debug # pyflakes:ignore - -from ietf.utils.models import VersionInfo -from ietf.utils.pipe import pipe - -class Command(BaseCommand): - """ - Update the version information for external commands used by the datatracker. - - Iterates through the entries in the VersionInfo table, runs the relevant - command, and updates the version string with the result. - - """ - - help = dedent(__doc__).strip() - - def handle(self, *filenames, **options): - for c in VersionInfo.objects.filter(used=True): - cmd = "%s %s" % (c.command, c.switch) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code != 0: - sys.stderr.write("Command '%s' returned %s: \n%s\n%s\n" % (cmd, code, out, err)) - else: - c.version = (out.strip()+'\n'+err.strip()).strip() - if options.get('verbosity', 1) > 1: - sys.stdout.write( - "Command: %s\n" - " Version: %s\n" % (cmd, c.version)) - c.save() diff --git a/ietf/utils/migrations/0002_delete_versioninfo.py b/ietf/utils/migrations/0002_delete_versioninfo.py new file mode 100644 index 0000000000..2835bb017b --- /dev/null +++ b/ietf/utils/migrations/0002_delete_versioninfo.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.11 on 2024-05-03 21:03 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("utils", "0001_initial"), + ] + + operations = [ + migrations.DeleteModel( + name="VersionInfo", + ), + ] diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 0915537fd8..21af5766e9 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -9,15 +9,6 @@ class DumpInfo(models.Model): host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') -class VersionInfo(models.Model): - time = models.DateTimeField(auto_now=True) - command = models.CharField(max_length=32) - switch = models.CharField(max_length=16) - version = models.CharField(max_length=64) - used = models.BooleanField(default=True) - class Meta: - verbose_name_plural = 'VersionInfo' - class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 6d61c5e2ed..1252cfef14 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -12,7 +12,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo, VersionInfo +from ietf.utils.models import DumpInfo class UserResource(ModelResource): @@ -43,21 +43,3 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) - - -class VersionInfoResource(ModelResource): - class Meta: - queryset = VersionInfo.objects.all() - serializer = api.Serializer() - cache = SimpleCache() - #resource_name = 'versioninfo' - ordering = ['id', ] - filtering = { - "id": ALL, - "time": ALL, - "command": ALL, - "switch": ALL, - "version": ALL, - "used": ALL, - } -api.utils.register(VersionInfoResource()) diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 9642a2877c..8fe989df99 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -60,6 +60,7 @@ def __ne__(self, other): validate_regular_expression_string = RegexStringValidator() + def validate_file_size(file, missing_ok=False): try: size = file.size @@ -69,8 +70,14 @@ def validate_file_size(file, missing_ok=False): else: raise - if size > settings.SECR_MAX_UPLOAD_SIZE: - raise ValidationError('Please keep filesize under %s. Requested upload size was %s' % (filesizeformat(settings.SECR_MAX_UPLOAD_SIZE), filesizeformat(file.size))) + if size > settings.DATATRACKER_MAX_UPLOAD_SIZE: + raise ValidationError( + "Please keep filesize under {}. Requested upload size was {}".format( + filesizeformat(settings.DATATRACKER_MAX_UPLOAD_SIZE), + filesizeformat(file.size) + ) + ) + def validate_mime_type(file, valid, missing_ok=False): try: diff --git a/k8s/beat.yaml b/k8s/beat.yaml new file mode 100644 index 0000000000..3400b21cbf --- /dev/null +++ b/k8s/beat.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: beat +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: beat + strategy: + type: Recreate + template: + metadata: + labels: + app: beat + spec: + securityContext: + runAsNonRoot: true + containers: + - name: beat + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "beat" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/k8s/celery.yaml b/k8s/celery.yaml new file mode 100644 index 0000000000..407f21e66d --- /dev/null +++ b/k8s/celery.yaml @@ -0,0 +1,80 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: celery + strategy: + type: Recreate + template: + metadata: + labels: + app: celery + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Celery Container + # ----------------------------------------------------- + - name: celery + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "celery" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml new file mode 100644 index 0000000000..7ca92ba99e --- /dev/null +++ b/k8s/datatracker.yaml @@ -0,0 +1,94 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datatracker +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: datatracker + strategy: + type: Recreate + template: + metadata: + labels: + app: datatracker + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # ScoutAPM Container + # ----------------------------------------------------- + - name: scoutapm + image: "scoutapp/scoutapm:version-1.4.0" + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - "sh" + - "-c" + - "./core-agent probe --tcp 0.0.0.0:6590 | grep -q 'Agent found'" + securityContext: + readOnlyRootFilesystem: true + runAsUser: 65534 # "nobody" user by default + runAsGroup: 65534 # "nogroup" group by default + # ----------------------------------------------------- + # Datatracker Container + # ----------------------------------------------------- + - name: datatracker + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + ports: + - containerPort: 8000 + name: http + protocol: TCP + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "datatracker" + envFrom: + - configMapRef: + name: django-config + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + volumes: + # To be overriden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: datatracker +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app: datatracker diff --git a/k8s/django-config.yaml b/k8s/django-config.yaml new file mode 100644 index 0000000000..9ece6fa9ee --- /dev/null +++ b/k8s/django-config.yaml @@ -0,0 +1,79 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: django-config +data: + # n.b., these are debug values / non-secret secrets + DATATRACKER_SERVER_MODE: "development" # development for staging, production for production + DATATRACKER_ADMINS: |- + Robert Sparks + Ryan Cross + Kesara Rathnayake + Jennifer Richards + Nicolas Giard + DATATRACKER_ALLOWED_HOSTS: ".ietf.org" # newline-separated list also allowed + # DATATRACKER_DATATRACKER_DEBUG: "false" + + # DB access details - needs to be filled in + # DATATRACKER_DB_HOST: "db" + # DATATRACKER_DB_PORT: "5432" + # DATATRACKER_DB_NAME: "datatracker" + # DATATRACKER_DB_USER: "django" # secret + # DATATRACKER_DB_PASS: "RkTkDPFnKpko" # secret + + DATATRACKER_DJANGO_SECRET_KEY: "PDwXboUq!=hPjnrtG2=ge#N$Dwy+wn@uivrugwpic8mxyPfHk" # secret + + # Set this to point testing / staging at the production statics server until we + # sort that out + # DATATRACKER_STATIC_URL: "https://static.ietf.org/dt/12.10.0/" + + # DATATRACKER_EMAIL_DEBUG: "true" + + # Outgoing email details + # DATATRACKER_EMAIL_HOST: "localhost" # defaults to localhost + # DATATRACKER_EMAIL_PORT: "2025" # defaults to 2025 + + # The value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret + + DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret + DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret + DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret + DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret + + # API key configuration + DATATRACKER_API_KEY_TYPE: "ES265" + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PUBLIC_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFVCTElDIEtFWS0tLS0tCk1Ga3dFd1lIS29aSXpqMENBUVlJS + 29aSXpqMERBUWNEUWdBRXFWb2pzYW9mREpTY3VNSk4rdHNodW15Tk01TUUKZ2Fyel + ZQcWtWb3ZtRjZ5RTdJSi9kdjRGY1YrUUtDdEovck9TOGUzNlk4WkFFVll1dWtoZXM + weVoxdz09Ci0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo= + # secret - value here is the default from settings.py (i.e., not actually secret) + DATATRACKER_API_PRIVATE_KEY_PEM_B64: |- + Ci0tLS0tQkVHSU4gUFJJVkFURSBLRVktLS0tLQpNSUdIQWdFQU1CTUdCeXFHU000O + UFnRUdDQ3FHU000OUF3RUhCRzB3YXdJQkFRUWdvSTZMSmtvcEtxOFhySGk5ClFxR1 + F2RTRBODNURllqcUx6KzhnVUxZZWNzcWhSQU5DQUFTcFdpT3hxaDhNbEp5NHdrMzY + yeUc2Ykkwemt3U0IKcXZOVStxUldpK1lYcklUc2duOTIvZ1Z4WDVBb0swbitzNUx4 + N2ZwanhrQVJWaTY2U0Y2elRKblgKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo= + + # DATATRACKER_MEETECHO_API_BASE: "https://meetings.conf.meetecho.com/api/v1/" + DATATRACKER_MEETECHO_CLIENT_ID: "this-is-the-meetecho-client-id" # secret + DATATRACKER_MEETECHO_CLIENT_SECRET: "this-is-the-meetecho-client-secret" # secret + + # DATATRACKER_MATOMO_SITE_ID: "7" # must be present to enable Matomo + # DATATRACKER_MATOMO_DOMAIN_PATH: "analytics.ietf.org" + + CELERY_PASSWORD: "this-is-a-secret" # secret + + # Only one of these may be set + # DATATRACKER_APP_API_TOKENS_JSON_B64: "e30K" # secret + # DATATRACKER_APP_API_TOKENS_JSON: "{}" # secret + + # use this to override default - one entry per line + # DATATRACKER_CSRF_TRUSTED_ORIGINS: |- + # https://datatracker.staging.ietf.org + + # Scout configuration + DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" + DATATRACKER_SCOUT_NAME: "StagingDatatracker" diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml new file mode 100644 index 0000000000..e618bb630d --- /dev/null +++ b/k8s/kustomization.yaml @@ -0,0 +1,13 @@ +namespace: datatracker +namePrefix: dt- +configMapGenerator: + - name: files-cfgmap + files: + - settings_local.py +resources: + - beat.yaml + - celery.yaml + - datatracker.yaml + - django-config.yaml + - memcached.yaml + - rabbitmq.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml new file mode 100644 index 0000000000..e94066c9e0 --- /dev/null +++ b/k8s/memcached.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: memcached +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: memcached + template: + metadata: + labels: + app: memcached + spec: + securityContext: + runAsNonRoot: true + containers: + - image: "quay.io/prometheus/memcached-exporter:v0.14.3" + imagePullPolicy: IfNotPresent + name: memcached-exporter + ports: + - name: metrics + containerPort: 9150 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 65534 # nobody + runAsGroup: 65534 # nobody + - image: "memcached:1.6-alpine" + imagePullPolicy: IfNotPresent + args: ["-m", "1024"] + name: memcached + ports: + - name: memcached + containerPort: 11211 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # memcached image sets up uid/gid 11211 + runAsUser: 11211 + runAsGroup: 11211 + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 +--- +apiVersion: v1 +kind: Service +metadata: + name: memcached + annotations: + k8s.grafana.com/scrape: "true" # this is not a bool + k8s.grafana.com/metrics.portName: "metrics" +spec: + type: ClusterIP + ports: + - port: 11211 + targetPort: memcached + protocol: TCP + name: memcached + - port: 9150 + targetPort: metrics + protocol: TCP + name: metrics + selector: + app: memcached diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml new file mode 100644 index 0000000000..132ca79ded --- /dev/null +++ b/k8s/rabbitmq.yaml @@ -0,0 +1,175 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + securityContext: + runAsNonRoot: true + initContainers: + # ----------------------------------------------------- + # Init RabbitMQ data + # ----------------------------------------------------- + - name: init-rabbitmq + image: busybox:stable + command: + - "sh" + - "-c" + - "mkdir -p -m700 /mnt/rabbitmq && chown 100:101 /mnt/rabbitmq" + securityContext: + runAsNonRoot: false + runAsUser: 0 + readOnlyRootFilesystem: true + volumeMounts: + - name: "rabbitmq-data" + mountPath: "/mnt" + containers: + # ----------------------------------------------------- + # RabbitMQ Container + # ----------------------------------------------------- + - image: "ghcr.io/ietf-tools/datatracker-mq:3.12-alpine" + imagePullPolicy: Always + name: rabbitmq + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + volumeMounts: + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + subPath: "rabbitmq" + - name: rabbitmq-tmp + mountPath: /tmp + - name: rabbitmq-config + mountPath: "/etc/rabbitmq" + env: + - name: "CELERY_PASSWORD" + value: "this-is-a-secret" + livenessProbe: + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + periodSeconds: 30 + timeoutSeconds: 5 + startupProbe: + initialDelaySeconds: 15 + periodSeconds: 5 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 60 + exec: + command: ["rabbitmq-diagnostics", "-q", "ping"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + # rabbitmq image sets up uid/gid 100/101 + runAsUser: 100 + runAsGroup: 101 + volumes: + - name: rabbitmq-tmp + emptyDir: + sizeLimit: "50Mi" + - name: rabbitmq-config + configMap: + name: "rabbitmq-configmap" + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 30 + volumeClaimTemplates: + - metadata: + name: rabbitmq-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + # storageClassName: "" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-configmap +data: + definitions.json: |- + { + "permissions": [ + { + "configure": ".*", + "read": ".*", + "user": "datatracker", + "vhost": "dt", + "write": ".*" + } + ], + "users": [ + { + "hashing_algorithm": "rabbit_password_hashing_sha256", + "limits": {}, + "name": "datatracker", + "password_hash": "HJxcItcpXtBN+R/CH7dUelfKBOvdUs3AWo82SBw2yLMSguzb", + "tags": [] + } + ], + "vhosts": [ + { + "limits": [], + "metadata": { + "description": "", + "tags": [] + }, + "name": "dt" + } + ] + } + rabbitmq.conf: |- + # prevent guest from logging in over tcp + loopback_users.guest = true + + # load saved definitions + load_definitions = /etc/rabbitmq/definitions.json + + # Ensure that enough disk is available to flush to disk. To do this, need to limit the + # memory available to the container to something reasonable. See + # https://www.rabbitmq.com/production-checklist.html#monitoring-and-resource-usage + # for recommendations. + + # 1-1.5 times the memory available to the container is adequate for disk limit + disk_free_limit.absolute = 6000MB + + # This should be ~40% of the memory available to the container. Use an + # absolute number because relative will be proprtional to the full machine + # memory. + vm_memory_high_watermark.absolute = 1600MB + + # Logging + log.file = false + log.console = true + log.console.level = info + log.console.formatter = json +--- +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + type: ClusterIP + clusterIP: None # headless service + ports: + - port: 5672 + targetPort: amqp + protocol: TCP + name: amqp + selector: + app: rabbitmq diff --git a/k8s/settings_local.py b/k8s/settings_local.py new file mode 100644 index 0000000000..3c77551a9d --- /dev/null +++ b/k8s/settings_local.py @@ -0,0 +1,264 @@ +# Copyright The IETF Trust 2007-2024, All Rights Reserved +# -*- coding: utf-8 -*- + +from base64 import b64decode +from email.utils import parseaddr +import json + +from ietf import __release_hash__ +from ietf.settings import * # pyflakes:ignore + + +def _multiline_to_list(s): + """Helper to split at newlines and conver to list""" + return [item.strip() for item in s.split("\n")] + + +# Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! +SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") + +# Secrets +_SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) +if _SECRET_KEY is not None: + SECRET_KEY = _SECRET_KEY +else: + raise RuntimeError("DATATRACKER_DJANGO_SECRET_KEY must be set") + +_NOMCOM_APP_SECRET_B64 = os.environ.get("DATATRACKER_NOMCOM_APP_SECRET_B64", None) +if _NOMCOM_APP_SECRET_B64 is not None: + NOMCOM_APP_SECRET = b64decode(_NOMCOM_APP_SECRET_B64) +else: + raise RuntimeError("DATATRACKER_NOMCOM_APP_SECRET_B64 must be set") + +_IANA_SYNC_PASSWORD = os.environ.get("DATATRACKER_IANA_SYNC_PASSWORD", None) +if _IANA_SYNC_PASSWORD is not None: + IANA_SYNC_PASSWORD = _IANA_SYNC_PASSWORD +else: + raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") + +_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) +if _RFC_EDITOR_SYNC_PASSWORD is not None: + RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") +else: + raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") + +_YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) +if _YOUTUBE_API_KEY is not None: + YOUTUBE_API_KEY = _YOUTUBE_API_KEY +else: + raise RuntimeError("DATATRACKER_YOUTUBE_API_KEY must be set") + +_GITHUB_BACKUP_API_KEY = os.environ.get("DATATRACKER_GITHUB_BACKUP_API_KEY", None) +if _GITHUB_BACKUP_API_KEY is not None: + GITHUB_BACKUP_API_KEY = _GITHUB_BACKUP_API_KEY +else: + raise RuntimeError("DATATRACKER_GITHUB_BACKUP_API_KEY must be set") + +_API_KEY_TYPE = os.environ.get("DATATRACKER_API_KEY_TYPE", None) +if _API_KEY_TYPE is not None: + API_KEY_TYPE = _API_KEY_TYPE +else: + raise RuntimeError("DATATRACKER_API_KEY_TYPE must be set") + +_API_PUBLIC_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PUBLIC_KEY_PEM_B64", None) +if _API_PUBLIC_KEY_PEM_B64 is not None: + API_PUBLIC_KEY_PEM = b64decode(_API_PUBLIC_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PUBLIC_KEY_PEM_B64 must be set") + +_API_PRIVATE_KEY_PEM_B64 = os.environ.get("DATATRACKER_API_PRIVATE_KEY_PEM_B64", None) +if _API_PRIVATE_KEY_PEM_B64 is not None: + API_PRIVATE_KEY_PEM = b64decode(_API_PRIVATE_KEY_PEM_B64) +else: + raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") + +# Set DEBUG if DATATRACKER_DEBUG env var is the word "true" +DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" + +# DATATRACKER_ALLOWED_HOSTS env var is a comma-separated list of allowed hosts +_allowed_hosts_str = os.environ.get("DATATRACKER_ALLOWED_HOSTS", None) +if _allowed_hosts_str is not None: + ALLOWED_HOSTS = _multiline_to_list(_allowed_hosts_str) + +DATABASES = { + "default": { + "HOST": os.environ.get("DATATRACKER_DB_HOST", "db"), + "PORT": os.environ.get("DATATRACKER_DB_PORT", "5432"), + "NAME": os.environ.get("DATATRACKER_DB_NAME", "datatracker"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("DATATRACKER_DB_USER", "django"), + "PASSWORD": os.environ.get("DATATRACKER_DB_PASS", ""), + "OPTIONS": json.loads(os.environ.get("DATATRACKER_DB_OPTS_JSON", "{}")), + }, +} + +# DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr +_admins_str = os.environ.get("DATATRACKER_ADMINS", None) +if _admins_str is not None: + ADMINS = [parseaddr(admin) for admin in _multiline_to_list(_admins_str)] +else: + raise RuntimeError("DATATRACKER_ADMINS must be set") + +USING_DEBUG_EMAIL_SERVER = os.environ.get("DATATRACKER_EMAIL_DEBUG", "false").lower() == "true" +EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") +EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) + +_celery_password = os.environ.get("CELERY_PASSWORD", None) +if _celery_password is None: + raise RuntimeError("CELERY_PASSWORD must be set") +CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), + password=_celery_password, + queue=os.environ.get("RABBITMQ_QUEUE", "dt") +) + +IANA_SYNC_USERNAME = "ietfsync" +IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" +IANA_SYNC_PROTOCOLS_URL = "http://www.iana.org/protocols/" + +RFC_EDITOR_NOTIFICATION_URL = "http://www.rfc-editor.org/parser/parser.php" + +STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/?apikey=redacted' + +#FIRST_CUTOFF_DAYS = 12 +#SECOND_CUTOFF_DAYS = 12 +#SUBMISSION_CUTOFF_DAYS = 26 +#SUBMISSION_CORRECTION_DAYS = 57 +MEETING_MATERIALS_SUBMISSION_CUTOFF_DAYS = 26 +MEETING_MATERIALS_SUBMISSION_CORRECTION_DAYS = 54 + +HTPASSWD_COMMAND = "/usr/bin/htpasswd2" + +_MEETECHO_CLIENT_ID = os.environ.get("DATATRACKER_MEETECHO_CLIENT_ID", None) +_MEETECHO_CLIENT_SECRET = os.environ.get("DATATRACKER_MEETECHO_CLIENT_SECRET", None) +if _MEETECHO_CLIENT_ID is not None and _MEETECHO_CLIENT_SECRET is not None: + MEETECHO_API_CONFIG = { + "api_base": os.environ.get( + "DATATRACKER_MEETECHO_API_BASE", + "https://meetings.conf.meetecho.com/api/v1/", + ), + "client_id": _MEETECHO_CLIENT_ID, + "client_secret": _MEETECHO_CLIENT_SECRET, + "request_timeout": 3.01, # python-requests doc recommend slightly > a multiple of 3 seconds + } +else: + raise RuntimeError( + "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" + ) + +# For APP_API_TOKENS, ccept either base64-encoded JSON or raw JSON, but not both +if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ: + if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ: + raise RuntimeError( + "Only one of DATATRACKER_APP_API_TOKENS_JSON and DATATRACKER_APP_API_TOKENS_JSON_B64 may be set" + ) + _APP_API_TOKENS_JSON = b64decode(os.environ.get("DATATRACKER_APP_API_TOKENS_JSON_B64")) +else: + _APP_API_TOKENS_JSON = os.environ.get("DATATRACKER_APP_API_TOKENS_JSON", None) + +if _APP_API_TOKENS_JSON is not None: + APP_API_TOKENS = json.loads(_APP_API_TOKENS_JSON) +else: + APP_API_TOKENS = {} + +EMAIL_COPY_TO = "" + +# Until we teach the datatracker to look beyond cloudflare for this check +IDSUBMIT_MAX_DAILY_SAME_SUBMITTER = 5000 + +# Leave DATATRACKER_MATOMO_SITE_ID unset to disable Matomo reporting +if "DATATRACKER_MATOMO_SITE_ID" in os.environ: + MATOMO_DOMAIN_PATH = os.environ.get("DATATRACKER_MATOMO_DOMAIN_PATH", "analytics.ietf.org") + MATOMO_SITE_ID = os.environ.get("DATATRACKER_MATOMO_SITE_ID") + MATOMO_DISABLE_COOKIES = True + +# Leave DATATRACKER_SCOUT_KEY unset to disable Scout APM agent +_SCOUT_KEY = os.environ.get("DATATRACKER_SCOUT_KEY", None) +if _SCOUT_KEY is not None: + if SERVER_MODE == "production": + PROD_PRE_APPS = ["scout_apm.django", ] + else: + DEV_PRE_APPS = ["scout_apm.django", ] + SCOUT_MONITOR = True + SCOUT_KEY = _SCOUT_KEY + SCOUT_NAME = os.environ.get("DATATRACKER_SCOUT_NAME", "Datatracker") + SCOUT_ERRORS_ENABLED = True + SCOUT_SHUTDOWN_MESSAGE_ENABLED = False + SCOUT_CORE_AGENT_SOCKET_PATH = "tcp://{host}:{port}".format( + host=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_HOST", "localhost"), + port=os.environ.get("DATATRACKER_SCOUT_CORE_AGENT_PORT", "6590"), + ) + SCOUT_CORE_AGENT_DOWNLOAD = False + SCOUT_CORE_AGENT_LAUNCH = False + SCOUT_REVISION_SHA = __release_hash__[:7] + +# Path to the email alias lists. Used by ietf.utils.aliases +DRAFT_ALIASES_PATH = "/a/postfix/draft-aliases" +DRAFT_VIRTUAL_PATH = "/a/postfix/draft-virtual" +GROUP_ALIASES_PATH = "/a/postfix/group-aliases" +GROUP_VIRTUAL_PATH = "/a/postfix/group-virtual" + +STATIC_URL = os.environ.get("DATATRACKER_STATIC_URL", None) +if STATIC_URL is None: + from ietf import __version__ + STATIC_URL = f"https://static.ietf.org/dt/{__version__}/" + +# Set these to the same as "production" in settings.py, whether production mode or not +MEDIA_ROOT = "/a/www/www6s/lib/dt/media/" +MEDIA_URL = "https://www.ietf.org/lib/dt/media/" +PHOTOS_DIRNAME = "photo" +PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME + +# Normally only set for debug, but needed until we have a real FS +DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') + +# Binaries that are different in the docker image +DE_GFM_BINARY = "/usr/local/bin/de-gfm" +IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" + +# Duplicating production cache from settings.py and using it whether we're in production mode or not +MEMCACHED_HOST = os.environ.get("DT_MEMCACHED_SERVICE_HOST", "127.0.0.1") +MEMCACHED_PORT = os.environ.get("DT_MEMCACHED_SERVICE_PORT", "11211") +from ietf import __version__ +CACHES = { + "default": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + "VERSION": __version__, + "KEY_PREFIX": "ietf:dt", + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "sessions": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt", + }, + "htmlized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/htmlized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "pdfized": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/pdfized", + "OPTIONS": { + "MAX_ENTRIES": 100000, # 100,000 + }, + }, + "slowpages": { + "BACKEND": "django.core.cache.backends.filebased.FileBasedCache", + "LOCATION": "/a/cache/datatracker/slowpages", + "OPTIONS": { + "MAX_ENTRIES": 5000, + }, + }, +} + +_csrf_trusted_origins_str = os.environ.get("DATATRACKER_CSRF_TRUSTED_ORIGINS") +if _csrf_trusted_origins_str is not None: + CSRF_TRUSTED_ORIGINS = _multiline_to_list(_csrf_trusted_origins_str) diff --git a/requirements.txt b/requirements.txt index 9231e400a7..8187c1cebf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,7 +20,7 @@ django-markup>=1.5 # Limited use - need to reconcile against direct use of ma django-oidc-provider>=0.8.1 # 0.8 dropped Django 2 support django-referrer-policy>=1.0 django-simple-history>=3.0.0 -django-stubs>=4.2.7 # The django-stubs version used determines the the mypy version indicated below +django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below django-tastypie>=0.14.5 # Version must be locked in sync with version of Django django-vite>=2.0.2,<3 django-widget-tweaks>=1.4.12 @@ -53,10 +53,12 @@ pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not pyquery>=1.4.3 python-dateutil>=2.8.2 types-python-dateutil>=2.8.2 +python-json-logger>=2.0.7 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache python-mimeparse>=1.6 # from TastyPie pytz==2022.2.1 # Pinned as changes need to be vetted for their effect on Meeting fields +types-pytz==2022.2.1 # match pytz version requests>=2.31.0 types-requests>=2.27.1 requests-mock>=1.9.3