Skip to content

Commit

Permalink
Merge remote-tracking branch 'ietf-tools/main' into feat/clarity
Browse files Browse the repository at this point in the history
  • Loading branch information
rjsparks committed May 16, 2024
2 parents 3411938 + ffb9eb1 commit c495f29
Show file tree
Hide file tree
Showing 72 changed files with 2,963 additions and 1,377 deletions.
6 changes: 6 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,9 @@ indent_size = 2

[ietf/**.html]
insert_final_newline = false

# Settings for Kubernetes yaml
# ---------------------------------------------------------
# Use 2-space indents
[k8s/**.yaml]
indent_size = 2
12 changes: 6 additions & 6 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ jobs:

- name: Download a Coverage Results
if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }}
uses: actions/download-artifact@v4.1.4
uses: actions/download-artifact@v4.1.7
with:
name: coverage

Expand Down Expand Up @@ -220,7 +220,7 @@ jobs:
.devcontainer
.github
.vscode
helm
k8s
playwright
svn-history
docker-compose.yml
Expand Down Expand Up @@ -323,7 +323,7 @@ jobs:
steps:
- name: Notify on Slack (Success)
if: ${{ !contains(join(needs.*.result, ','), 'failure') }}
uses: slackapi/slack-github-action@v1.25.0
uses: slackapi/slack-github-action@v1.26.0
with:
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
payload: |
Expand All @@ -346,7 +346,7 @@ jobs:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_GH_BOT }}
- name: Notify on Slack (Failure)
if: ${{ contains(join(needs.*.result, ','), 'failure') }}
uses: slackapi/slack-github-action@v1.25.0
uses: slackapi/slack-github-action@v1.26.0
with:
channel-id: ${{ secrets.SLACK_GH_BUILDS_CHANNEL_ID }}
payload: |
Expand Down Expand Up @@ -385,7 +385,7 @@ jobs:
- uses: actions/checkout@v4

- name: Download a Release Artifact
uses: actions/download-artifact@v4.1.4
uses: actions/download-artifact@v4.1.7
with:
name: release-${{ env.PKG_VERSION }}

Expand Down Expand Up @@ -421,7 +421,7 @@ jobs:

steps:
- name: Download a Release Artifact
uses: actions/download-artifact@v4.1.4
uses: actions/download-artifact@v4.1.7
with:
name: release-${{ env.PKG_VERSION }}
path: /a/www/ietf-datatracker/main.dev.${{ github.run_number }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
path: geckodriver.log

- name: Upload Coverage Results to Codecov
uses: codecov/codecov-action@v4.2.0
uses: codecov/codecov-action@v4.3.1
with:
files: coverage.xml

Expand Down
7 changes: 0 additions & 7 deletions bin/daily
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@ $DTDIR/bin/hourly
source $DTDIR/env/bin/activate


# Update our information about the current version of some commands we use
$DTDIR/ietf/manage.py update_external_command_info

# Get IANA-registered yang models
#YANG_IANA_DIR=$(python -c 'import ietf.settings; print ietf.settings.SUBMIT_YANG_IANA_MODEL_DIR')
# Hardcode the rsync target to avoid any unwanted deletes:
Expand All @@ -43,9 +40,5 @@ $DTDIR/ietf/manage.py populate_yang_model_dirs -v0
# Re-run yang checks on active documents
$DTDIR/ietf/manage.py run_yang_model_checks -v0

# Expire last calls
# Enable when removed from /a/www/ietf-datatracker/scripts/Cron-runner:
$DTDIR/ietf/bin/expire-last-calls

# Purge older PersonApiKeyEvents
$DTDIR/ietf/manage.py purge_old_personal_api_key_events 14
2 changes: 1 addition & 1 deletion bin/dump-to-names-json
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

set -x
ietf/manage.py dumpdata --indent 1 doc.State doc.BallotType doc.StateType \
mailtrigger.MailTrigger mailtrigger.Recipient name utils.VersionInfo \
mailtrigger.MailTrigger mailtrigger.Recipient name \
group.GroupFeatures stats.CountryAlias dbtemplate.DBTemplate \
| jq --sort-keys "sort_by(.model, .pk)" \
| jq '[.[] | select(.model!="dbtemplate.dbtemplate" or .pk==354)]' > ietf/name/fixtures/names.json
21 changes: 0 additions & 21 deletions bin/hourly
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,15 @@
# This script is expected to be triggered by cron from
# /etc/cron.d/datatracker
export LANG=en_US.UTF-8
export PYTHONIOENCODING=utf-8

# Make sure we stop if something goes wrong:
program=${0##*/}
trap 'echo "$program($LINENO): Command failed with error code $? ([$$] $0 $*)"; exit 1' ERR

DTDIR=/a/www/ietf-datatracker/web
cd $DTDIR/

# Set up the virtual environment
source $DTDIR/env/bin/activate

logger -p user.info -t cron "Running $DTDIR/bin/hourly"

# Generate some static files
ID=/a/ietfdata/doc/draft/repository
DERIVED=/a/ietfdata/derived
DOWNLOAD=/a/www/www6s/download

$DTDIR/ietf/manage.py generate_idnits2_rfc_status
$DTDIR/ietf/manage.py generate_idnits2_rfcs_obsoleted

CHARTER=/a/www/ietf-ftp/charter
wget -q https://datatracker.ietf.org/wg/1wg-charters-by-acronym.txt -O $CHARTER/1wg-charters-by-acronym.txt
wget -q https://datatracker.ietf.org/wg/1wg-charters.txt -O $CHARTER/1wg-charters.txt

# Regenerate the last week of bibxml-ids
$DTDIR/ietf/manage.py generate_draft_bibxml_files

# Create and update group wikis
#$DTDIR/ietf/manage.py create_group_wikis

# exit 0
56 changes: 39 additions & 17 deletions dev/build/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,17 +1,39 @@
FROM ghcr.io/ietf-tools/datatracker-app-base:latest
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"

ENV DEBIAN_FRONTEND=noninteractive

COPY . .
COPY ./dev/build/start.sh ./start.sh
RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt
RUN chmod +x start.sh && \
chmod +x docker/scripts/app-create-dirs.sh && \
sh ./docker/scripts/app-create-dirs.sh

VOLUME [ "/assets" ]

EXPOSE 8000

CMD ["./start.sh"]
FROM ghcr.io/ietf-tools/datatracker-app-base:latest
LABEL maintainer="IETF Tools Team <tools-discuss@ietf.org>"

ENV DEBIAN_FRONTEND=noninteractive

# uid 498 = wwwrun and gid 496 = www on ietfa
RUN groupadd -g 1000 datatracker && \
useradd -c "Datatracker User" -u 1000 -g datatracker -m -s /bin/false datatracker

RUN apt-get purge -y imagemagick imagemagick-6-common

# Install libreoffice (needed via PPT2PDF_COMMAND)
RUN echo "deb http://deb.debian.org/debian bullseye-backports main" > /etc/apt/sources.list.d/bullseye-backports.list && \
apt-get update && \
apt-get -qyt bullseye-backports install libreoffice-nogui

COPY . .
COPY ./dev/build/start.sh ./start.sh
COPY ./dev/build/datatracker-start.sh ./datatracker-start.sh
COPY ./dev/build/celery-start.sh ./celery-start.sh

RUN pip3 --disable-pip-version-check --no-cache-dir install -r requirements.txt && \
echo '# empty' > ietf/settings_local.py && \
ietf/manage.py patch_libraries && \
rm -f ietf/settings_local.py

RUN chmod +x start.sh && \
chmod +x datatracker-start.sh && \
chmod +x celery-start.sh && \
chmod +x docker/scripts/app-create-dirs.sh && \
sh ./docker/scripts/app-create-dirs.sh

RUN mkdir -p /a

VOLUME [ "/a" ]

EXPOSE 8000

CMD ["./start.sh"]
22 changes: 22 additions & 0 deletions dev/build/celery-start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/bin/bash
#
# Run a celery worker
#
echo "Running Datatracker checks..."
./ietf/manage.py check

cleanup () {
# Cleanly terminate the celery app by sending it a TERM, then waiting for it to exit.
if [[ -n "${celery_pid}" ]]; then
echo "Gracefully terminating celery worker. This may take a few minutes if tasks are in progress..."
kill -TERM "${celery_pid}"
wait "${celery_pid}"
fi
}

trap 'trap "" TERM; cleanup' TERM

# start celery in the background so we can trap the TERM signal
celery "$@" &
celery_pid=$!
wait "${celery_pid}"
17 changes: 17 additions & 0 deletions dev/build/datatracker-start.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

echo "Running Datatracker checks..."
./ietf/manage.py check

echo "Running Datatracker migrations..."
./ietf/manage.py migrate --settings=settings_local

echo "Starting Datatracker..."

gunicorn \
--workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \
--max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \
--timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \
--bind :8000 \
--log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \
ietf.wsgi:application
28 changes: 19 additions & 9 deletions dev/build/start.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
#!/bin/bash

echo "Running Datatracker checks..."
./ietf/manage.py check

echo "Running Datatracker migrations..."
./ietf/manage.py migrate --settings=settings_local

echo "Starting Datatracker..."
./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local
#
# Environment config:
#
# CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker)
#
case "${CONTAINER_ROLE:-datatracker}" in
datatracker)
exec ./datatracker-start.sh
;;
celery)
exec ./celery-start.sh --app=ietf worker
;;
beat)
exec ./celery-start.sh --app=ietf beat
;;
*)
echo "Unknown role '${CONTAINER_ROLE}'"
exit 255
esac
Loading

0 comments on commit c495f29

Please sign in to comment.