Merge branch 'develop'

dbca-wa · Sep 9, 2024 · 8a3c787 · 8a3c787
2 parents 00a90f4 + 5773977
commit 8a3c787
Show file tree

Hide file tree

Showing 13 changed files with 66 additions and 64 deletions.
diff --git a/.ahoy.yml b/.ahoy.yml
@@ -14,7 +14,7 @@ entrypoint:
 commands:
   init:
     usage: Initialise the codebase on first-time setup (ahoy init)
-    cmd : |
+    cmd: |
       echo "Creating project variables."
       cp .env.dbca .env
 
@@ -142,7 +142,7 @@ commands:
 
   import-db:
     usage: Import the DB from an environment into your local dev environment | ahoy import-db ENVIRONMENT
-    cmd: |    
+    cmd: |
       source ./scripts/messages.sh
       #
       # Check the correct arguments are supplied
@@ -167,7 +167,6 @@ commands:
 
       success "Database import successful"
 
-
   reindex-solr:
     usage: Re-runs the Solr index, e.g. after running `db-import` command
     cmd: ahoy run "ckan search-index rebuild"
@@ -201,29 +200,19 @@ commands:
         echo "AZURE_FILE_SHARE_URL or AZURE_FILE_SHARE_SAS_TOKEN is not set"
         exit 1
       fi
-      
+
       echo "Downloading ckan_logs from:$AZURE_FILE_SHARE_URL"
-      
-      LOG_FILE="ckan.log"
-      echo "Downloading $LOG_FILE"
-      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
-      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
 
       LOG_FILE="ckan-worker.log"
       echo "Downloading $LOG_FILE"
       FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
       curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
 
-      LOG_FILE="ckan-cron-jobs.log"
-      echo "Downloading $LOG_FILE"
-      FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
-      curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
-      
       LOG_FILE="nginx_access.log"
       echo "Downloading $LOG_FILE"      
       FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
       curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
-      
+
       LOG_FILE="nginx_error.log"
       echo "Downloading $LOG_FILE"
       FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
@@ -233,5 +222,5 @@ commands:
       echo "Downloading $LOG_FILE"
       FULL_URL="$AZURE_FILE_SHARE_URL/ckan_logs/$LOG_FILE?$AZURE_FILE_SHARE_SAS_TOKEN"
       curl -L -o dbca/ckan_logs/$LOG_FILE "$FULL_URL"
-      
-      echo "CKAN logs downloaded to dbca/ckan_logs"
+
+      echo "CKAN logs downloaded to dbca/ckan_logs"
diff --git a/.env.dbca b/.env.dbca
@@ -81,6 +81,8 @@ CKAN_SERVER_NAME=localhost
 ## ckanext-doi ##
 CKANEXT__DOI__ACCOUNT_NAME=
 CKANEXT__DOI__ACCOUNT_PASSWORD=
+CKANEXT__DOI__PREFIX=
+CKANEXT__DOI__TEST_MODE=True
 
 ## ckanext-spatial ##
 CKANEXT__SPATIAL__COMMON_MAP__APIKEY=

diff --git a/ckan/Dockerfile b/ckan/Dockerfile
@@ -25,7 +25,7 @@ RUN for d in $APP_DIR/patches/*; do \
 ## DBCA specific configuration ##
 
 # ckanext-saml2auth dependencies
-RUN apk add xmlsec
+RUN apk add xmlsec-dev
 COPY saml ${APP_DIR}/saml
 
 # Override the default start_ckan.sh script
@@ -37,3 +37,11 @@ COPY config/*.ini ${APP_DIR}/config/
 
 # Override default CKAN config file to use dbca.ini
 ENV CKAN_INI=${APP_DIR}/config/dbca.ini
+
+# Set the permissions to the ckan user
+RUN chown -R ckan:ckan $APP_DIR && \
+    chown -R ckan:ckan /usr/lib/python3.10/site-packages/ && \
+    chown -R ckan:ckan /usr/bin/
+
+# Set the default user to ckan
+USER ckan
diff --git a/ckan/Dockerfile.dev b/ckan/Dockerfile.dev
@@ -58,7 +58,7 @@ RUN for d in $APP_DIR/patches/*; do \
 ## DBCA specific configuration ##
 
 # ckanext-saml2auth dependencies
-RUN apk add xmlsec
+RUN apk add xmlsec-dev
 COPY saml ${APP_DIR}/saml
 
 # Override the default dbca_start_ckan_development.sh script
@@ -70,3 +70,11 @@ COPY config/*.ini ${APP_DIR}/config/
 
 # Override default CKAN config file to use dbca.ini
 ENV CKAN_INI=${APP_DIR}/config/dbca.ini
+
+# Set the permissions to the ckan user
+RUN chown -R ckan:ckan $APP_DIR && \
+    chown -R ckan:ckan /usr/lib/python3.10/site-packages/ && \
+    chown -R ckan:ckan /usr/bin/
+
+# Set the default user to ckan
+USER ckan
diff --git a/ckan/Dockerfile.worker b/ckan/Dockerfile.worker
@@ -10,9 +10,17 @@ RUN ckan config-tool $CKAN_INI "ckan.plugins = ${CKAN__PLUGINS}"
 COPY supervisor/ckan_*.conf /etc/supervisord.d
 COPY supervisor/supervisord.conf /etc/supervisord.conf
 
-## Cron jobs config
+## Cron jobs scripts
 COPY setup/dbca_logs_maintenance.sql $APP_DIR
-RUN chmod -x $APP_DIR/dbca_logs_maintenance.sql
-COPY setup/dbca_ckan_cron_jobs $APP_DIR
-RUN chmod -x $APP_DIR/dbca_ckan_cron_jobs
-RUN crontab -u ckan $APP_DIR/dbca_ckan_cron_jobs
+
+# Set the default user to root temporarily to update file and folder permissions
+USER root
+# Set the permissions to the ckan user
+RUN chown -R ckan:ckan $APP_DIR && \
+    chown -R ckan:ckan /usr/lib/python3.10/site-packages/ && \
+    chown -R ckan:ckan /usr/bin/ && \
+    chown -R ckan:ckan /etc/supervisord.d && \
+    chmod -x $APP_DIR/dbca_logs_maintenance.sql
+
+# Set the default user to ckan
+USER ckan
diff --git a/ckan/docker-entrypoint.d/02_setup_dbca.sh b/ckan/docker-entrypoint.d/02_setup_dbca.sh
@@ -6,19 +6,20 @@ if [ ! -f /tmp/container_ready ]; then
     export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2)
     echo "CKAN__PLUGINS: $CKAN__PLUGINS"
 
-    ## Create logs folder/files and set permissions
+    ## Create logs folder/file
     mkdir -p $APP_DIR/logs
     touch $APP_DIR/logs/ckan-worker.log
-    chown -R ckan:ckan $APP_DIR/logs
+    touch $APP_DIR/logs/supervisord.log
 
-    ## Create webassets folder and set permissions
+    ## Create webassets folder
     mkdir -p $APP_DIR/webassets
-    chown -R ckan:ckan $APP_DIR/webassets
-    su ckan -c "ckan -c $CKAN_INI asset build"
+    ckan -c $CKAN_INI asset build
 
-    ## Create archive folder and set permissions
+    ## Create archive folder
     mkdir -p $CKAN_STORAGE_PATH/archiver
-    chown -R ckan:ckan $CKAN_STORAGE_PATH/archiver
+
+    ## Create resources folder
+    mkdir -p $CKAN_STORAGE_PATH/resources
 
     if [[ $CKAN__PLUGINS == *"xloader"* ]]; then
         CKAN_INI=$APP_DIR/ckan.ini
@@ -56,10 +57,6 @@ if [ ! -f /tmp/container_ready ]; then
         ckan -c $CKAN_INI dbca load_spatial_data
     fi
 
-    # if [[ $CKAN__PLUGINS == *"harvest"* ]]; then
-    #     ckan -c $CKAN_INI db upgrade -p harvest
-    # fi
-
     # Set the container as ready so the startup scripts are not run again
     touch /tmp/container_ready
 fi
diff --git a/ckan/setup/dbca_ckan_cron_jobs b/ckan/setup/dbca_ckan_cron_jobs
@@ -1,13 +1,7 @@
 # Crontab for CKAN cron jobs
-# Midnight Rebuild Solr Index
-0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini search-index rebuild >> /srv/app/logs/ckan-cron-jobs.log 2>&1
-# Example cron job runs the harvester run command every 15 mins
-#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run
 # 8am task to schedule embargo datasets to public visibility
-0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets >> /srv/app/logs/ckan-cron-jobs.log 2>&1
-# 12:30am report generation for archiver broken links
-30 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate >> /srv/app/logs/ckan-cron-jobs.log 2>&1
+0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets
 # 8am task to send CKAN email notifications
-0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini notify send_emails >> /srv/app/logs/ckan-cron-jobs.log 2>&1
+0 8 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini notify send_emails
 # Midnight dbca logs maintenance
-0 0 * * * psql $CKAN_SQLALCHEMY_URL -f /srv/app/dbca_logs_maintenance.sql >> /srv/app/logs/ckan-cron-jobs.log 2>&1
+0 0 * * * psql $CKAN_SQLALCHEMY_URL -f /srv/app/dbca_logs_maintenance.sql
diff --git a/ckan/setup/dbca_requirements.sh b/ckan/setup/dbca_requirements.sh
@@ -10,10 +10,6 @@ pip3 install -r ${SRC_DIR}/ckanext-archiver/requirements.txt
 pip3 install -e git+https://github.com/ckan/ckanext-dcat.git@v1.5.1#egg=ckanext-dcat
 pip3 install -r ${SRC_DIR}/ckanext-dcat/requirements.txt
 
-# Harvester
-pip3 install -e git+https://github.com/ckan/ckanext-harvest.git@v1.5.6#egg=ckanext-harvest
-pip3 install -r ${SRC_DIR}/ckanext-harvest/requirements.txt
-
 # Hierarchy
 pip3 install -e git+https://github.com/ckan/ckanext-hierarchy.git@v1.2.1#egg=ckanext-hierarchy
 pip3 install -r ${SRC_DIR}/ckanext-hierarchy/requirements.txt

diff --git a/ckan/setup/dbca_start_ckan.sh.override b/ckan/setup/dbca_start_ckan.sh.override
@@ -41,13 +41,11 @@ then
     # if CKAN_WORKER is set to true, start the background services
     if [ "$CKAN_WORKER" = "true" ]
     then
-        # Start crond
-        /usr/sbin/crond
         # Start supervisord
-        su ckan -c "supervisord --configuration /etc/supervisord.conf"
+        supervisord --configuration /etc/supervisord.conf
     else
         # Start uwsgi
-        su ckan -c 'uwsgi -i $APP_DIR/config/uwsgi.ini'
+        uwsgi -i $APP_DIR/config/uwsgi.ini
     fi
 else
   echo "[prerun] failed...not starting CKAN."

diff --git a/ckan/setup/dbca_start_ckan_development.sh.override b/ckan/setup/dbca_start_ckan_development.sh.override
@@ -98,15 +98,13 @@ export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=
 # if CKAN_WORKER is set to true, start the background services
 if [ "$CKAN_WORKER" = "true" ]
 then
-    # Start crond
-    /usr/sbin/crond
     # Start supervisord
-    su ckan -c "supervisord --configuration /etc/supervisord.conf"
+    supervisord --configuration /etc/supervisord.conf
 else
     # Start the development server as the ckan user with automatic reload
     if [ "$USE_HTTPS_FOR_DEV" = true ] ; then
-        su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0 -C unsafe.cert -K unsafe.key"
+        ckan -c $CKAN_INI run -H 0.0.0.0 -C unsafe.cert -K unsafe.key
     else
-        su ckan -c "/usr/bin/ckan -c $CKAN_INI run -H 0.0.0.0"
+        ckan -c $CKAN_INI run -H 0.0.0.0
     fi
 fi
diff --git a/ckan/supervisor/supervisord.conf b/ckan/supervisor/supervisord.conf
@@ -12,6 +12,7 @@ pidfile = /tmp/supervisord.pid
 nodaemon = true
 umask = 022
 identifier = supervisor
+user = ckan
 
 [supervisorctl]
 serverurl = unix:///tmp/supervisor.sock

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -5,11 +5,11 @@ volumes:
   pip_cache:
   site_packages:
   vscode_server:
-  ckan_logs:
 
 services:
 
   ckan-dev:
+    user: ckan:ckan
     build:
       context: ckan/
       dockerfile: Dockerfile.dev
@@ -37,12 +37,13 @@ services:
       - site_packages:/usr/lib/python3.10/site-packages
       - vscode_server:/root/.vscode-server
       - ./ckan/config:/srv/app/config
-      - ckan_logs:/srv/app/logs
+      - ./logs:/srv/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
 
   ckan-dev-worker:
+    user: ckan:ckan
     build:
       context: ckan/
       dockerfile: Dockerfile.worker
@@ -69,7 +70,7 @@ services:
       - pip_cache:/root/.cache/pip
       - site_packages:/usr/lib/python3.10/site-packages
       - vscode_server:/root/.vscode-server
-      - ckan_logs:/srv/app/logs
+      - ./logs:/srv/app/logs
       - ./spatial_data:/srv/app/spatial_data
     restart: unless-stopped
     healthcheck:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -4,7 +4,7 @@ volumes:
   solr_data:
   pip_cache:
   site_packages:
-  ckan_logs:
+
 
 services:
 
@@ -24,9 +24,10 @@ services:
       - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}"
     volumes:
       - ckan_storage:/var/lib/ckan
-      - ckan_logs:/srv/app/logs
+      - ./logs:/srv/app/logs
 
   ckan:
+    user: ckan:ckan
     platform: linux/amd64
     build:
       context: ckan/
@@ -53,12 +54,13 @@ services:
       - ckan_storage:/var/lib/ckan
       - pip_cache:/root/.cache/pip
       - site_packages:/usr/lib/python3.10/site-packages
-      - ckan_logs:/srv/app/logs
+      - ./logs:/srv/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "curl", "-I", "http://localhost:5000"]
 
   ckan-worker:
+    user: ckan:ckan
     platform: linux/amd64
     build:
       context: ckan/
@@ -86,7 +88,7 @@ services:
         condition: service_started
     volumes:
       - ckan_storage:/var/lib/ckan
-      - ckan_logs:/srv/app/logs
+      - ./logs:/srv/app/logs
       - ./spatial_data:/srv/app/spatial_data
     restart: unless-stopped
     healthcheck: