Add initial docker build and documentation

.env

@@ -0,0 +1 @@
+COMPOSE_FILE=docker/docker-compose.yml

.gitignore

@@ -21,3 +21,5 @@ syntax: glob
 __pycache__
 _build
 twisted/plugins/dropin.cache
+
+.venv

docker/.env

@@ -0,0 +1,3 @@
+APP_PATH=/opt/irrd
+DATA_PATH=/opt/irrd_data
+CONFIG_FILE=docker/irrd.yaml

docker/Dockerfile

@@ -0,0 +1,50 @@
+FROM ubuntu:24.04
+
+# Install requirements
+RUN DEBIAN_FRONTEND=noninteractive apt-get update
+RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
+ca-certificates build-essential git gpg libenchant-2-2 libpq-dev netcat-traditional python3-dev pipx wget
+RUN apt-get clean && \
+rm -rf /var/lib/apt/lists/*
+
+ARG APP_PATH
+ARG CONFIG_FILE
+
+# BASH is needed for Poetry to work at build time
+SHELL ["/bin/bash", "-c"]
+
+# Setup poetry
+RUN pipx install poetry
+# Persist poetry path to $PATH in ~/.bashrc for run-time availability
+RUN pipx ensurepath
+# Add poetry path to current $PATH for build-time availability
+ENV PATH="/root/.local/bin:${PATH}"
+
+# Setup irrd
+WORKDIR $APP_PATH/
+COPY ./pyproject.toml $APP_PATH/pyproject.toml
+COPY ./poetry.lock $APP_PATH/poetry.lock
+# "poetry config virtualenvs.create false" will install the poetry dependencies
+# globally, not in a venv. This would be ideal, it allows all apps to access the
+# dependencies. This is not supported by pipx though, it will throw an error
+# that this clashes with any dependencies installed by the system package
+# manager.
+#
+# So we need to install poetry dep's in a venv. This can be problematic because
+# IRRd forks threads/processes and they don't run in the same poetry venv with
+# all the poetry managed dep's. To resolve this:
+# Install poetry dep's in a venv (default) but force the venv to be in the
+# project directory (non-default), so that they the venv is in a consistent
+# location ($APP_PATH/.venv). Then we can make the venv python bin our default
+# python bin, and all applications and threads/forks will have access to the
+# poetry dep's.
+RUN poetry config virtualenvs.create true && \
+poetry config virtualenvs.in-project true && \
+poetry install -vv --no-interaction --with=dev,docs
+
+# Copy a config file into the container with the relevant settings to access the
+# other containers (postgres and redis).
+# This can be overrider by the build argument.
+COPY $CONFIG_FILE /etc/irrd.yaml
+
+RUN useradd irrd

docker/docker-compose.yml

@@ -0,0 +1,58 @@
+services:
+  irrd:
+    build:
+      context: ../
+      dockerfile: docker/Dockerfile
+      args:
+        - APP_PATH=$APP_PATH
+        - CONFIG_FILE=$CONFIG_FILE
+    ports:
+      - "8080:8080"
+      - "8043:8043"
+    volumes:
+      - ../:$APP_PATH/:rw
+    entrypoint:
+      # A custom entry point is needed to init the DB before starting IRRd
+      - $APP_PATH/docker/irrd_init.sh
+      - "$APP_PATH"
+    environment:
+      - APP_PATH=$APP_PATH
+      - DATA_PATH=$DATA_PATH
+    depends_on:
+      - postgresql
+      - redis
+      - test-postgresql
+    restart: always
+  postgresql:
+    image: postgres:15-alpine
+    environment:
+      PGUSER: irrd
+      POSTGRES_USER: irrd
+      POSTGRES_PASSWORD: irrd
+    ports:
+      - "5432:5432"
+    volumes:
+      - ../docker/psql_init.sh:/docker-entrypoint-initdb.d/psql_init.sh:rw
+    # Need to change/increase these values from the defaults because they are too low when loading in data from RIRs
+    command: postgres -c random_page_cost=1 -c work_mem=50000 -c shared_buffers=102400 -c max_connections=100 -c checkpoint_timeout=86400 -c max_wal_size=2147483647
+    restart: always
+  redis:
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    command: redis-server --maxmemory 0
+    restart: always
+  # An instance of Postgres used for pytest
+  test-postgresql:
+    image: postgres:15-alpine
+    environment:
+      PGUSER: root
+      POSTGRES_DB: irrd_test
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_PASSWORD: secret
+      POSTGRES_USER: root
+    ports:
+      - "5433:5432"
+    # Default is off, needed for test_object_writing_and_status_checking()
+    command: postgres -c track_commit_timestamp=on
+    restart: always

docker/import_data.sh

@@ -0,0 +1,149 @@
+#!/bin/bash
+
+# This script can be used to import data for testing.
+# This is for use when running IRRd in Docker.
+
+set -eu
+
+function download() {
+    local URL="$1"
+    # wget returns 1 if file exists and using -nc
+    wget -nc -O "${DATA_PATH}/$(basename "$URL")" "${URL}" || true
+}
+
+mkdir -p "${DATA_PATH}"
+cd "${DATA_PATH}/"
+
+download "https://ftp.afrinic.net/pub/dbase/afrinic.db.gz"
+download "https://ftp.afrinic.net/pub/dbase/AFRINIC.CURRENTSERIAL"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.as-block.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.as-set.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.domain.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.filter-set.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.inet-rtr.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.irt.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.key-cert.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.limerick.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.mntner.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.peering-set.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.role.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.route-set.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.route.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.route6.gz"
+download "ftp://ftp.apnic.net/apnic/whois/apnic.db.rtr-set.gz"
+download "ftp://ftp.apnic.net/pub/apnic/whois/APNIC.CURRENTSERIAL"
+download "https://ftp.arin.net/pub/rr/arin.db.gz"
+download "https://ftp.arin.net/pub/rr/ARIN.CURRENTSERIAL"
+download "https://irr.lacnic.net/lacnic.db.gz"
+download "https://irr.lacnic.net/LACNIC.CURRENTSERIAL"
+download "ftp://ftp.radb.net/radb/dbase/radb.db.gz"
+download "ftp://ftp.radb.net/radb/dbase/RADB.CURRENTSERIAL"
+download "https://ftp.ripe.net/ripe/dbase/split/ripe.db.as-set.gz"
+download "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz"
+download "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route-set.gz"
+download "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz"
+download "https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz"
+download "https://ftp.ripe.net/ripe/dbase/RIPE.CURRENTSERIAL"
+
+gunzip -kf "${DATA_PATH}"/*.gz
+
+# Don't add the config below more than once / don't clobber existing config
+if ! grep -E "sources_default:|sources:" /etc/irrd.yaml
+then
+
+  echo "
+
+sources_default:
+    - AFRINIC
+    - APNIC
+    - ARIN
+    - LACNIC
+    - RADB
+    - RIPE
+
+sources:
+    AFRINIC:
+      authoritative: false
+      keep_journal: false
+      import_source: file://${DATA_PATH}/afrinic.db.gz
+      import_serial_source: file://${DATA_PATH}/AFRINIC.CURRENTSERIAL
+    APNIC:
+      authoritative: false
+      keep_journal: false
+      import_source:
+        - file://${DATA_PATH}/apnic.db.as-block.gz
+        - file://${DATA_PATH}/apnic.db.as-set.gz
+        - file://${DATA_PATH}/apnic.db.aut-num.gz
+        - file://${DATA_PATH}/apnic.db.domain.gz
+        - file://${DATA_PATH}/apnic.db.filter-set.gz
+        - file://${DATA_PATH}/apnic.db.inet-rtr.gz
+        - file://${DATA_PATH}/apnic.db.inet6num.gz
+        - file://${DATA_PATH}/apnic.db.inetnum.gz
+        - file://${DATA_PATH}/apnic.db.irt.gz
+        - file://${DATA_PATH}/apnic.db.key-cert.gz
+        - file://${DATA_PATH}/apnic.db.limerick.gz
+        - file://${DATA_PATH}/apnic.db.mntner.gz
+        - file://${DATA_PATH}/apnic.db.organisation.gz
+        - file://${DATA_PATH}/apnic.db.peering-set.gz
+        - file://${DATA_PATH}/apnic.db.role.gz
+        - file://${DATA_PATH}/apnic.db.route-set.gz
+        - file://${DATA_PATH}/apnic.db.route.gz
+        - file://${DATA_PATH}/apnic.db.route6.gz
+        - file://${DATA_PATH}/apnic.db.rtr-set.gz
+      import_serial_source: file://${DATA_PATH}/APNIC.CURRENTSERIAL
+    ARIN:
+      authoritative: false
+      keep_journal: false
+      import_source: file://${DATA_PATH}/arin.db.gz
+      import_serial_source: file://${DATA_PATH}/ARIN.CURRENTSERIAL
+    LACNIC:
+      authoritative: false
+      keep_journal: false
+      import_source: file://${DATA_PATH}/lacnic.db.gz
+      import_serial_source: file://${DATA_PATH}/LACNIC.CURRENTSERIAL
+    RADB:
+        authoritative: false
+        keep_journal: false
+        import_serial_source: file://${DATA_PATH}/RADB.CURRENTSERIAL
+        import_source:
+        - file://${DATA_PATH}/radb.db.gz
+    RIPE:
+        authoritative: false
+        keep_journal: false
+        import_serial_source: file:///opt/irrd_data/RIPE.CURRENTSERIAL
+        import_source:
+        - file://${DATA_PATH}/ripe.db.as-set.gz
+        - file://${DATA_PATH}/ripe.db.aut-num.gz
+        - file://${DATA_PATH}/ripe.db.route.gz
+        - file://${DATA_PATH}/ripe.db.route6.gz
+        - file://${DATA_PATH}/ripe.db.route-set.gz
+
+" >> /etc/irrd.yaml
+
+fi
+
+# Trigger a config import by irrd so that the new data sources are defined
+kill -s SIGHUP "$(cat /var/run/irrd.pid)"
+
+# Wait for irrd to restart
+sleep 2
+
+# Import the new data, each import runs single-threaded so spin them all up together
+for IRRDB in "AFRINIC afrinic.db" "APNIC apnic.db" "ARIN arin.db" "LACNIC lacnic.db" "RADB radb.db" "RIPE ripe.db"
+do
+  set -- $IRRDB # $1 = "AFRINC", $2 = "afrinic.db"
+  for FILE_NAME in $(find $DATA_PATH/ -not -name "*.gz" -name "${2}*")
+  do
+    "${APP_PATH}/.venv/bin/python3" "${APP_PATH}/irrd/scripts/load_database.py"  \
+    --source "$1" --serial "$(cat "${DATA_PATH}/$1.CURRENTSERIAL")" "${FILE_NAME}" &
+  done
+done
+
+# Wait for all imports to finish
+echo -e "\nWaiting for data import to completed...\n"
+wait $(jobs -p)
+echo -e "\nData import completed"

docker/irrd.yaml

@@ -0,0 +1,62 @@
+irrd:
+    database_url: 'postgresql://irrd:irrd@postgresql:5432/irrd'
+    redis_url: 'redis://redis'
+    piddir: /var/run/
+    user: irrd
+    group: irrd
+
+    access_lists:
+        allow_all:
+            - '::0'
+            - '0.0.0.0/0'
+
+    server:
+        http:
+            status_access_list: allow_all
+            interface: '::0'
+            port: 8080
+            url: "https://localhost/"
+        whois:
+            interface: '::0'
+            port: 8043
+
+    auth:
+        gnupg_keyring: /tmp
+
+    email:
+        from: example@example.com
+        smtp: localhost
+
+    log:
+        level: DEBUG
+
+    route_object_preference:
+        update_timer: 3600
+
+    rpki:
+        roa_source: https://rpki.gin.ntt.net/api/export.json
+        roa_import_timer: 3600
+        pseudo_irr_remarks: |
+            This AS{asn} route object represents routing data retrieved
+            from the RPKI. This route object is the result of an automated
+            RPKI-to-IRR conversion process performed by IRRd.
+        notify_invalid_subject: route(6) objects in {sources_str} marked RPKI invalid
+        notify_invalid_header: |
+            This is to notify that {object_count} route(6) objects for which you are a
+            contact have been marked as RPKI invalid. This concerns
+            objects in the {sources_str} database.
+            You have received this message because your e-mail address is
+            listed in one or more of the tech-c or admin-c contacts, on
+            the maintainer(s) for these route objects.
+            The {object_count} route(6) objects listed below have been validated using
+            RPKI origin validation, and found to be invalid. This means that
+            these objects are no longer visible on the IRRd instance that
+            sent this e-mail.
+            This may affect routing filters based on queries to this IRRd
+            instance. It is also no longer possible to modify these objects.
+            To resolve this situation, create or modify ROA objects that
+            result in these route(6) being valid, or not_found. If this
+            happens, the route(6) objects will return to being visible.
+            You may also delete these objects if they are no longer
+            relevant.
+

docker/irrd_init.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -eu
+
+# When making code changes and irrd restarts due to a traceback (i.e., an un-graceful restart),
+# we need to clean up the PID file
+rm -rf /var/run/irrd.pid
+
+# Wait for Postgres to start otherwise irrd sometimes starts faster and crashes
+while true
+do
+  if ! nc -z postgresql 5432
+  then
+    echo "Postgres not reachable yet..."
+    sleep 1
+  else
+    echo "Postgres is reachable!"
+    break
+  fi
+done
+
+# Use the poetry venv to run Python with all required deps
+cd "$APP_PATH/"
+export PATH="${APP_PATH}/.venv/bin:${PATH}"
+python3 "${APP_PATH}/irrd/scripts/database_upgrade.py"
+python3 "${APP_PATH}/irrd/daemon/main.py" --foreground

docker/psql_init.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 <<-EOSQL
+    CREATE DATABASE irrd;
+    ALTER DATABASE irrd OWNER TO irrd;
+    CREATE ROLE irrd WITH LOGIN ENCRYPTED PASSWORD 'irrd';
+    GRANT ALL PRIVILEGES ON DATABASE irrd TO irrd;
+    \c irrd
+    CREATE EXTENSION IF NOT EXISTS pgcrypto;
+EOSQL