Updated project to use log files for ckan, ckan-worker, ckan-cron-job…

…s, nginx

ckan/Dockerfile

@@ -24,6 +24,12 @@ RUN for d in $APP_DIR/patches/*; do \
 
 ## DBCA specific configuration ##
 
+## Create logs folder
+RUN mkdir -p $APP_DIR/logs && \
+    touch $APP_DIR/logs/ckan.log && \
+    chown -R ckan:ckan $APP_DIR/logs && \
+    chmod -R 755 $APP_DIR/logs
+
 # Override the default start_ckan.sh script
 COPY setup/dbca_start_ckan.sh.override ${APP_DIR}/start_ckan.sh
 RUN chmod +x ${APP_DIR}/start_ckan.sh

ckan/Dockerfile.dev

@@ -57,6 +57,12 @@ RUN for d in $APP_DIR/patches/*; do \
 
 ## DBCA specific configuration ##
 
+## Create logs folder
+RUN mkdir -p $APP_DIR/logs && \
+    touch $APP_DIR/logs/ckan.log && \
+    chown -R ckan:ckan $APP_DIR/logs && \
+    chmod -R 755 $APP_DIR/logs
+
 # Override the default dbca_start_ckan_development.sh script
 COPY setup/dbca_start_ckan_development.sh.override ${APP_DIR}/start_ckan_development.sh
 RUN chmod +x ${APP_DIR}/start_ckan_development.sh

ckan/Dockerfile.worker

@@ -1,12 +1,6 @@
 ARG CKAN_IMAGE
 FROM ${CKAN_IMAGE}
 
-## Create logs folder
-RUN mkdir -p $APP_DIR/logs && \
-    touch $APP_DIR/logs/ckan-cron-jobs.log && \
-    chown -R ckan:ckan $APP_DIR/logs && \
-    chmod -R 755 $APP_DIR/logs
-
 ## Supervisor config
 COPY supervisor/*.conf /etc/supervisord.d
 

ckan/config/dbca.ini

@@ -57,30 +57,30 @@ ckanext.qa.qsv_bin=/usr/local/bin/qsv
 keys = root, ckan, ckanext, werkzeug
 
 [handlers]
-keys = console
+keys = console, fileHandler
 
 [formatters]
 keys = generic
 
 [logger_root]
 level = WARNING
-handlers = console
+handlers = console, fileHandler
 
 [logger_werkzeug]
 level = WARNING
-handlers = console
+handlers = console, fileHandler
 qualname = werkzeug
 propagate = 0
 
 [logger_ckan]
 level = INFO
-handlers = console
+handlers = console, fileHandler
 qualname = ckan
 propagate = 0
 
 [logger_ckanext]
 level = DEBUG
-handlers = console
+handlers = console, fileHandler
 qualname = ckanext
 propagate = 0
 
@@ -90,5 +90,11 @@ args = (sys.stderr,)
 level = NOTSET
 formatter = generic
 
+[handler_fileHandler]
+class = handlers.TimedRotatingFileHandler
+args = ('/srv/app/logs/ckan.log', 'midnight', 1, 0)
+level = NOTSET
+formatter = generic
+
 [formatter_generic]
 format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s

ckan/setup/dbca_ckan_cron_jobs

@@ -1,7 +1,7 @@
 # Crontab for CKAN cron jobs
 # Example cron job runs the harvester run command every 15 mins
-#*/15 * * * * /usr/bin/ckan -c /srv/app/ckan.ini harvester run >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
+#*/15 * * * * /usr/bin/ckan -c /srv/app/config/dbca.ini harvester run
 # Midnight task to schedule embargo datasets to public visibility
-0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca/ckan.ini dbca scheduled_datasets >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
+0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini dbca scheduled_datasets
 # Midnight report generation for archiver broken links
-0 0 * * * /usr/bin/ckan -c /srv/app/ckan.ini report generate >> $APP_DIR/logs/ckan-cron-jobs.log 2>&1
+0 0 * * * /usr/bin/ckan -c /srv/app/config/dbca.ini report generate

ckan/supervisor/ckan_cron_jobs.conf

@@ -9,7 +9,10 @@
 [program:ckan-cron-jobs]
 
 ; Use the full paths to the virtualenv and your configuration file here.
-command=/usr/sbin/crond -f -L /srv/app/logs/ckan-cron-jobs.log
+# This configuration sets up a cron job for CKAN.
+# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini file.
+# It then executes the crond command with the -f flag to run in the foreground.
+command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/sbin/crond -f"
 
 
 ; User the worker runs as.
@@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d
 
 
 ; Log files.
-stdout_logfile=/srv/app/logs/ckan-crons-jobs.stdout.log
-stderr_logfile=/srv/app/logs/ckan-crons-jobs.stderr.log
+stderr_logfile=/srv/app/logs/ckan-crons-jobs.log
 
 
 ; Make sure that the worker is started on system start and automatically

ckan/supervisor/ckan_worker_bulk.conf

@@ -9,6 +9,9 @@
 [program:ckan-worker-bulk]
 
 ; Use the full paths to the virtualenv and your configuration file here.
+# This configuration file specifies the command to run for the CKAN worker bulk process.
+# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file.
+# It then executes the ckan command with the specified configuration file and runs the jobs worker bulk command.
 command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker bulk"
 
 
@@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d
 
 
 ; Log files.
-stdout_logfile=/srv/app/logs/ckan-worker.stdout.log
-stderr_logfile=/srv/app/logs/ckan-worker.stderr.log
+stderr_logfile=/srv/app/logs/ckan-worker.log
 
 
 ; Make sure that the worker is started on system start and automatically

ckan/supervisor/ckan_worker_default.conf

@@ -9,6 +9,9 @@
 [program:ckan-worker-default]
 
 ; Use the full paths to the virtualenv and your configuration file here.
+# This configuration file specifies the command to run for the CKAN worker bulk process.
+# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file.
+# It then executes the ckan command with the specified configuration file and runs the jobs worker command.
 command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker"
 
 
@@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d
 
 
 ; Log files.
-stdout_logfile=/srv/app/logs/ckan-worker.stdout.log
-stderr_logfile=/srv/app/logs/ckan-worker.stderr.log
+stderr_logfile=/srv/app/logs/ckan-worker.log
 
 
 ; Make sure that the worker is started on system start and automatically

ckan/supervisor/ckan_worker_priority.conf

@@ -9,6 +9,9 @@
 [program:ckan-worker-priority]
 
 ; Use the full paths to the virtualenv and your configuration file here.
+# This configuration file specifies the command to run for the CKAN worker bulk process.
+# The command exports the CKAN__PLUGINS environment variable by extracting the value from the dbca.ini configuration file.
+# It then executes the ckan command with the specified configuration file and runs the jobs worker priority command.
 command=/bin/bash -c "export CKAN__PLUGINS=$(grep '^ckan\.plugins' $APP_DIR/config/dbca.ini | cut -d'=' -f2); exec /usr/bin/ckan -c /srv/app/config/dbca.ini jobs worker priority"
 
 
@@ -23,8 +26,7 @@ process_name=%(program_name)s-%(process_num)02d
 
 
 ; Log files.
-stdout_logfile=/srv/app/logs/ckan-worker.stdout.log
-stderr_logfile=/srv/app/logs/ckan-worker.stderr.log
+stderr_logfile=/srv/app/logs/ckan-worker.log
 
 
 ; Make sure that the worker is started on system start and automatically

docker-compose.dev.yml

@@ -4,6 +4,7 @@ volumes:
   ckan_storage:
   pg_data:
   solr_data:
+  ckan_logs:
 
 services:
 
@@ -32,6 +33,7 @@ services:
       - ckan_storage:/var/lib/ckan
       - ./src:/srv/app/src_extensions
       - ./ckan/config:/srv/app/config
+      - ckan_logs:/srv/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
@@ -62,6 +64,8 @@ services:
     volumes:
       - ckan_storage:/var/lib/ckan
       - ./src:/srv/app/src_extensions
+      - ./ckan/config:/srv/app/config
+      - ckan_logs:/srv/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]

docker-compose.yml

@@ -5,6 +5,7 @@ volumes:
   ckan_storage:
   pg_data:
   solr_data:
+  ckan_logs:
 
 services:
 
@@ -25,6 +26,7 @@ services:
       - "0.0.0.0:${NGINX_SSLPORT_HOST}:${NGINX_SSLPORT}"
     volumes:
       - ckan_storage:/var/lib/ckan
+      - ckan_logs:/srv/app/logs
 
   ckan:
     container_name: ${CKAN_CONTAINER_NAME}
@@ -53,6 +55,7 @@ services:
         condition: service_healthy
     volumes:
       - ckan_storage:/var/lib/ckan
+      - ckan_logs:/srv/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]
@@ -87,6 +90,7 @@ services:
         condition: service_started
     volumes:
       - ckan_storage:/var/lib/ckan
+      - ckan_logs:/srv/app/logs
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "wget", "-qO", "/dev/null", "http://localhost:5000"]

nginx/setup/default.conf

@@ -19,7 +19,10 @@ server {
     # ssl_session_cache dfine in stream and http
     ssl_session_tickets off;
 
-    #access_log  /var/log/nginx/host.access.log  main;
+    access_log /srv/app/logs/nginx_access.log main;
+    access_log /var/log/nginx/access.log main;
+    error_log /srv/app/logs/nginx_error.log error;
+    error_log /var/log/nginx/errror.log error;
 
     location / {
         proxy_pass http://ckan:5000/;

nginx/setup/default.conf.template

@@ -19,7 +19,10 @@ server {
     # ssl_session_cache dfine in stream and http
     ssl_session_tickets off;
 
-    #access_log  /var/log/nginx/host.access.log  main;
+    access_log /srv/app/logs/nginx_access.log main;
+    access_log /var/log/nginx/access.log main;
+    error_log /srv/app/logs/nginx_error.log error;
+    error_log /var/log/nginx/errror.log error;
 
     location / {
         proxy_pass http://${CKAN_CONTAINER_NAME}:${CKAN_PORT}/;