Merge pull request #146 from mjanez/ckan-2.9.12

Align with ckan/ckan-docker, update ckanext-schemingdcat and add CKAN 2.9.12 backport version

.env.example

@@ -5,21 +5,28 @@ APP_DIR=/srv/app
 
 # Host Ports
 CKAN_PORT_HOST=5000
-NGINX_PORT_HOST=81
-NGINX_SSLPORT_HOST=8443
-APACHE_PORT_HOST=81
+# Common proxy server for Apache or NGINX.
+# Change all the PROXY_SERVER_PORT_HOST to PROXY_SERVER_HTTPS_PORT_HOST if you can use HTTPS instead of HTTP (Only for NGINX and not development compose)
+PROXY_SERVER_PORT_HOST=81
+PROXY_SERVER_HTTPS_PORT_HOST=8443
+NGINX_PORT_HOST=${PROXY_SERVER_PORT_HOST}
+NGINX_SSLPORT_HOST=${PROXY_SERVER_HTTPS_PORT_HOST}
+APACHE_PORT_HOST=${PROXY_SERVER_PORT_HOST}
 PYCSW_PORT_HOST=8000
 
 # Solr
 SOLR_IMAGE_VERSION=2.9-solr9-spatial
 SOLR_PORT=8983
-CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan
-TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/ckan
+SOLR_CKAN_DATABASE=ckan
+CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE}
+TEST_CKAN_SOLR_URL=http://solr:${SOLR_PORT}/solr/${SOLR_CKAN_DATABASE}
 
 # Redis
 REDIS_VERSION=7-alpine
-CKAN_REDIS_URL=redis://redis:6379/1
-TEST_CKAN_REDIS_URL=redis://redis:6379/1
+REDIS_PORT=6379
+REDIS_CKAN_DATABASE=1
+CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE}
+TEST_CKAN_REDIS_URL=redis://redis:${REDIS_PORT}/${REDIS_CKAN_DATABASE}
 
 # NGINX
 NGINX_PORT=80
@@ -32,15 +39,16 @@ APACHE_PORT=80
 APACHE_LOG_DIR=/var/log/apache
 
 #NGINX/APACHE
-## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`.
+## Check CKAN__ROOT_PATH and CKANEXT__DCAT__BASE_URI and CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI. If you don't need to use domain locations, it is better to use the nginx configuration. Leave blank or use the root `/`. If you dont need PROXY_SERVER_PORT_HOST (e.g. is 80), edit the PROXY_SERVER_URL and remove :${PROXY_SERVER_PORT_HOST}
 PROXY_SERVER_NAME=localhost
+PROXY_SERVER_URL=http://${PROXY_SERVER_NAME}:${PROXY_SERVER_PORT_HOST}
 PROXY_CKAN_LOCATION=/catalog
 PROXY_PYCSW_LOCATION=/csw
 
 # pycsw
 PYCSW_PORT=8000
-CKAN_URL=http://localhost:81/catalog
-PYCSW_URL=http://localhost:81/csw
+CKAN_URL=${PROXY_SERVER_URL}${PROXY_CKAN_LOCATION}
+PYCSW_URL=${PROXY_SERVER_URL}${PROXY_PYCSW_LOCATION}
 # SCHEMAS: ckan2pycsw/model/dataset.py - Dataset type
 PYCSW_CKAN_SCHEMA=iso19139_geodcatap
 PYCSW_OUPUT_SCHEMA=iso19139_inspire
@@ -63,14 +71,16 @@ CKAN_DB=ckandb
 DATASTORE_READONLY_USER=datastore_ro
 DATASTORE_READONLY_PASSWORD=datastore
 DATASTORE_DB=datastore
-CKAN_SQLALCHEMY_URL=postgresql://ckandbuser:ckandbpassword@db/ckandb
-CKAN_DATASTORE_WRITE_URL=postgresql://ckandbuser:ckandbpassword@db/datastore
-CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore
+CKAN_SQLALCHEMY_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB}
+CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB}
+CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_DB}
 
 # Test database connections
-TEST_CKAN_SQLALCHEMY_URL=postgres://ckan:ckan@db/ckan_test
-TEST_CKAN_DATASTORE_WRITE_URL=postgresql://ckan:ckan@db/datastore_test
-TEST_CKAN_DATASTORE_READ_URL=postgresql://datastore_ro:datastore@db/datastore_test
+CKAN_TEST_DB=ckan_test
+DATASTORE_TEST_DB=datastore_test
+TEST_CKAN_SQLALCHEMY_URL=postgres://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_TEST_DB}
+TEST_CKAN_DATASTORE_WRITE_URL=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB}
+TEST_CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_READONLY_USER}:${DATASTORE_READONLY_PASSWORD}@${POSTGRES_HOST}/${DATASTORE_TEST_DB}
 
 # Dev settings
 USE_HTTPS_FOR_DEV=false
@@ -81,11 +91,12 @@ CKAN_DEV_COMPOSE_SERVICE=ckan-dev
 CKAN_VERSION=2.9.11
 CKAN_SITE_ID=default
 # CKAN_SITE_URL = http:/ or https:/ + PROXY_SERVER_NAME. Optionally the APACHE_HOST_PORT if different from 80 
-CKAN_SITE_URL=http://localhost:81
-CKAN__ROOT_PATH=/catalog/{{LANG}}
-CKAN_PORT=5000
-CKAN__FAVICON=/catalog/base/images/ckan.ico
+CKAN_SITE_URL=${PROXY_SERVER_URL}
+CKAN__ROOT_PATH=${PROXY_CKAN_LOCATION}/{{LANG}}
+CKAN__FAVICON=${PROXY_CKAN_LOCATION}/base/images/ckan.ico
 CKAN__SITE_LOGO=/images/default/ckan-logo.png
+# Custom licenses that are aligned with DCAT-AP
+CKAN___LICENSES_GROUP_URL=https://raw.githubusercontent.com/mjanez/ckanext-schemingdcat/main/ckanext/schemingdcat/public/static/licenses.json
 CKAN___BEAKER__SESSION__SECRET=CHANGE_ME
 # See https://docs.ckan.org/en/latest/maintaining/configuration.html#api-token-settings
 CKAN___API_TOKEN__JWT__ENCODE__SECRET=string:CHANGE_ME
@@ -99,7 +110,7 @@ CKAN_SMTP_SERVER=smtp.corporateict.domain:25
 CKAN_SMTP_STARTTLS=True
 CKAN_SMTP_USER=user
 CKAN_SMTP_PASSWORD=pass
-CKAN_SMTP_MAIL_FROM=ckan@localhost
+CKAN_SMTP_MAIL_FROM=ckan@${PROXY_SERVER_NAME}
 ## Customize which text formats the text_view plugin will show
 CKAN__PREVIEW__JSON_FORMATS="json jsonld"
 # html htm rdf+xml owl+xml xml n3 n-triples turtle plain atom csv tsv rss txt json
@@ -127,24 +138,24 @@ CKAN__LOCALE_ORDER="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru
 CKAN__LOCALES_OFFERED="en es pt_BR ja it cs_CZ ca fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv"
 
 # Extensions
-CKAN__PLUGINS="envvars stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent"
+CKAN__PLUGINS="stats image_view text_view recline_view webpage_view resourcedictionary datastore xloader spatial_metadata spatial_query spatial_harvest_metadata_api csw_harvester waf_harvester doc_harvester resource_proxy geo_view geojson_view wmts_view shp_view dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface schemingdcat_datasets schemingdcat_groups schemingdcat_organizations schemingdcat schemingdcat_ckan_harvester schemingdcat_xls_harvester harvest pdf_view pages fluent envvars"
 
 # ckanext-harvest
 CKAN__HARVEST__MQ__TYPE=redis
 CKAN__HARVEST__MQ__HOSTNAME=redis
-CKAN__HARVEST__MQ__PORT=6379
-CKAN__HARVEST__MQ__REDIS_DB=1
+CKAN__HARVEST__MQ__PORT=${REDIS_PORT}
+CKAN__HARVEST__MQ__REDIS_DB=${REDIS_CKAN_DATABASE}
 # Clean-up mechanism for the harvest log table. The default is 30 days.
 CKAN__HARVEST__LOG_TIMEFRAME=40
 
 # ckanext-xloader
 CKANEXT__XLOADER__API_TOKEN=api_token
-CKANEXT__XLOADER__JOBS__DB_URI=postgresql://ckandbuser:ckandbpassword@db/ckandb
+CKANEXT__XLOADER__JOBS__DB_URI=postgresql://${CKAN_DB_USER}:${CKAN_DB_PASSWORD}@${POSTGRES_HOST}/${CKAN_DB}
 
 # ckanext-dcat
 CKANEXT__DCAT__BASE_URI=${CKAN_URL}
-# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=euro_dcat_ap,spain_dcat
-CKANEXT__DCAT__RDF_PROFILES='euro_dcat_ap_2'
+# Default profile(s). Instead of this envvar, it's possible to specify all the profile(s) availables to be used for serialization using the profiles parameter: http://localhost:5000/catalog.xml?profiles=eu_dcat_ap_2,es_dcat
+CKANEXT__DCAT__RDF_PROFILES='eu_dcat_ap_2 eu_dcat_ap_scheming'
 # The custom endpoint **must** start with a forward slash (`/`) and contain the `{_format}` placeholder. The endpoint is added to the CKAN_SITE_URL and CKAN__ROOT_PATH, example: http://localhost:5000/catalog/catalog.rdf
 CKANEXT__DCAT__DEFAULT_CATALOG_ENDPOINT='/catalog.{_format}'
 
@@ -165,9 +176,9 @@ CKANEXT__GEOVIEW__SHP_VIEWER__ENCODING=UTF-8
 ## CSW Endpoint for spatial metadata
 CKANEXT__SCHEMINGDCAT_GEOMETADATA_BASE_URI=${PYCSW_URL}
 ## Scheming: setup_scheming.sh
-CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcatap/geodcatap_dataset.yaml"
-CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_group.json"
-CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcatap/geodcatap_org.json"
+CKANEXT__SCHEMINGDCAT_DATASET_SCHEMA="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_2.yaml ckanext.schemingdcat:schemas/resources/dcat_3_document.yaml"
+CKANEXT__SCHEMINGDCAT_GROUP_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_group.json"
+CKANEXT__SCHEMINGDCAT_ORGANIZATION_SCHEMAS="ckanext.schemingdcat:schemas/geodcat_ap/eu_geodcat_ap_org.json"
 CKANEXT__SCHEMINGDCAT_PRESETS="ckanext.schemingdcat:schemas/default_presets.json ckanext.fluent:presets.json"
 ## Facets: setup_scheming.sh
 CKANEXT__SCHEMINGDCAT_FACET_LIST="dataset_scope theme groups theme_eu dcat_type groups publisher_name publisher_type spatial_uri owner_org res_format frequency tags tag_uri conforms_to"
@@ -177,15 +188,12 @@ CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_ICON="theme"
 CKANEXT__SCHEMINGDCAT_DEFAULT_PACKAGE_ITEM_SHOW_SPATIAL=True
 CKANEXT__SCHEMINGDCAT_SHOW_METADATA_TEMPLATES_TOOLBAR=False
 CKANEXT__METADATA_TEMPLATES_SEARCH_IDENTIFIER="schemingdcat_xls-template"
+CKANEXT__SCHEMINGDCAT_ENDPOINTS_YAML="endpoints.yaml"
 
 # ckanext-pages
 CKANEXT__PAGES__ALOW_HTML=False
 CKANEXT__PAGES__ORGANIZATION=True
 CKANEXT__PAGES__GROUP=True
 CKANEXT__PAGES__ABOUT_MENU=False
 CKANEXT__PAGES__GROUP_MENU=True
-CKANEXT__PAGES__ORGANIZATION_MENU=True
-
-# WIP: ckanext-sparql_interface
-CKANEXT__SPARQL__ENDPOINT_URL=https://dbpedia.org/sparql
-CKANEXT__SPARQL__HIDE_ENDPOINT_URL=False
+CKANEXT__PAGES__ORGANIZATION_MENU=True

.github/workflows/docker-build.yml

@@ -14,9 +14,9 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  TAG: ghcr.io/${{ github.repository }}:${{ github.head_ref }}
   CONTEXT: .
   BRANCH: ${{ github.head_ref }}
+  VERSION: ${{ github.head_ref }}
   DOCKERFILE_PATH: /ckan
   DOCKERFILE: Dockerfile
 
@@ -43,21 +43,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Extract tag version from branch name
+        id: extract_tag_version
+        run: echo "VERSION=$(echo ${{ github.head_ref }} | sed 's/^ckan-//')" >> $GITHUB_ENV
+
       - name: Extract Docker metadata
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           labels: |
             org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md
-            org.opencontainers.image.version=${{ env.BRANCH }}
+            org.opencontainers.image.version=${{ env.VERSION }}
+          annotations: |
+            org.opencontainers.image.description=This image contains CKAN based on a Docker Compose deployment. The container includes CKAN along with its dependencies and configurations for spatial data support.
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: true
-          tags: ${{ env.TAG }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
           context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}
           file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
 
@@ -68,14 +76,14 @@ jobs:
           no-fail: true
 
       - name: Run Trivy container image vulnerability scanner
-        uses: aquasecurity/trivy-action@0.18.0
+        uses: aquasecurity/trivy-action@0.24.0
         with:
-          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }}
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           format: sarif
           output: trivy-results.sarif
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         if: always()
         with:
           sarif_file: trivy-results.sarif

.github/workflows/docker-manual.yml

@@ -5,9 +5,9 @@ on: workflow_dispatch
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
-  TAG: ghcr.io/${{ github.repository }}:${{ github.ref_name }}
   CONTEXT: .
   BRANCH: ${{ github.ref_name }}
+  VERSION: ${{ github.ref_name }}
   DOCKERFILE_PATH: /ckan
   DOCKERFILE: Dockerfile
 
@@ -33,21 +33,29 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Extract tag version from branch name
+        id: extract_tag_version
+        run: echo "VERSION=$(echo ${{ github.head_ref }} | sed 's/^ckan-//')" >> $GITHUB_ENV
+
       - name: Extract Docker metadata
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           labels: |
             org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md
-            org.opencontainers.image.version=${{ env.BRANCH }}
+            org.opencontainers.image.version=${{ env.VERSION }}
+          annotations: |
+            org.opencontainers.image.description=This image contains CKAN based on a Docker Compose deployment. The container includes CKAN along with its dependencies and configurations for spatial data support.
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: true
-          tags: ${{ env.TAG }}
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
           context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}
           file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
 
@@ -58,14 +66,14 @@ jobs:
           no-fail: true
 
       - name: Run Trivy container image vulnerability scanner
-        uses: aquasecurity/trivy-action@0.18.0
+        uses: aquasecurity/trivy-action@0.24.0
         with:
-          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.BRANCH }}
+          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           format: sarif
           output: trivy-results.sarif
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         if: always()
         with:
           sarif_file: trivy-results.sarif

.github/workflows/docker-master.yml

@@ -9,6 +9,8 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
   CONTEXT: .
+  BRANCH: master
+  VERSION: master
   DOCKERFILE_PATH: /ckan
   DOCKERFILE: Dockerfile
 
@@ -29,32 +31,36 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Get highest ckan branch excluding -dev
-        id: getbranch
-        run: echo "VERSION=$(git branch -r | grep -o 'ckan-[0-9]*\.[0-9]*\.[0-9]*[^-dev]$' | sort -V | tail -n 1)" >> $GITHUB_ENV
-
       - name: Login to registry
         uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Get highest ckan branch excluding -dev
+        id: getbranch
+        run: echo "VERSION=$(git branch -r | grep -o 'ckan-[0-9]*\.[0-9]*\.[0-9]*[^-dev]$' | sort -V | tail -n 1)" >> $GITHUB_ENV
+
       - name: Extract Docker metadata
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           labels: |
-            org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/master/README.md
+            org.opencontainers.image.documentation=https://github.com/${{ github.repository }}/blob/${{ env.BRANCH }}/README.md
             org.opencontainers.image.version=${{ env.VERSION }}
+          annotations: |
+            org.opencontainers.image.description=This image contains CKAN based on a Docker Compose deployment. The container includes CKAN along with its dependencies and configurations for spatial data support.
+            org.opencontainers.image.source=https://github.com/${{ github.repository }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: true
           tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           labels: ${{ steps.meta.outputs.labels }}
+          annotations: ${{ steps.meta.outputs.annotations }}
           context: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}
           file: ${{ env.CONTEXT }}${{ env.DOCKERFILE_PATH }}/${{ env.DOCKERFILE }}
 
@@ -65,14 +71,14 @@ jobs:
           no-fail: true
 
       - name: Run Trivy container image vulnerability scanner
-        uses: aquasecurity/trivy-action@0.18.0
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.VERSION }}
           format: sarif
           output: trivy-results.sarif
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         if: always()
         with:
           sarif_file: trivy-results.sarif