diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index b6a03cc8d..000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,144 +0,0 @@ -#!groovy - -def tryStep(String message, Closure block, Closure tearDown = null) { - try { - block() - } - catch (Throwable t) { - slackSend message: "${env.JOB_NAME}: ${message} failure ${env.BUILD_URL}", channel: '#ci-channel', color: 'danger' - throw t - } - finally { - if (tearDown) { - tearDown() - } - } -} - -node { - stage("Checkout") { - checkout scm - } - - stage('Test') { - // Get a fresh project name to prevent conflicts between concurrent builds. - gitHash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() - project = 'dso_api_' + gitHash - tryStep "test", { - sh "docker-compose -p ${project} -f src/.jenkins/test/docker-compose.yml build --pull && " + - "docker-compose -p ${project} -f src/.jenkins/test/docker-compose.yml run -u root --rm test" - }, { - sh "docker-compose -p ${project} -f src/.jenkins/test/docker-compose.yml down" - } - } - - // The rebuilding likely reuses the build cache from docker-compose. - stage("Build API image") { - tryStep "build", { - docker.withRegistry("${DOCKER_REGISTRY_HOST}",'docker_registry_auth') { - def image = docker.build("datapunt/dataservices/dso-api:${env.BUILD_NUMBER}", "src") - image.push() - } - } - } -} - - -String BRANCH = "${env.BRANCH_NAME}" - -if (BRANCH == "master") { - node { - stage('Push acceptance image') { - tryStep "image tagging", { - docker.withRegistry("${DOCKER_REGISTRY_HOST}",'docker_registry_auth') { - def image = docker.image("datapunt/dataservices/dso-api:${env.BUILD_NUMBER}") - image.pull() - image.push("acceptance") - } - } - } - } - - node { - stage("Deploy to ACC") { - tryStep "deployment", { - build job: 'Subtask_Openstack_Playbook', - parameters: [ - [$class: 'StringParameterValue', name: 'INVENTORY', value: 'acceptance'], - [$class: 'StringParameterValue', name: 'PLAYBOOK', value: 'deploy-dataservices-mdbrole.yml'], - ] - } - tryStep "deployment", { - build job: 'Subtask_Openstack_Playbook', - parameters: [ - [$class: 'StringParameterValue', name: 'INVENTORY', value: 'acceptance'], - [$class: 'StringParameterValue', name: 'PLAYBOOK', value: 'deploy.yml'], - [$class: 'StringParameterValue', name: 'PLAYBOOKPARAMS', value: "-e cmdb_id=app_dso-api"] - ] - } - } - } - - node { - stage('OWASP vulnerability scan') { - // Get a fresh project name to prevent conflicts between concurrent builds. - gitHash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() - project = 'owasp_check_' + gitHash - - tryStep "owasp vulnerability check", { - sh "docker-compose -p ${project} -f src/.jenkins/owasp_vulnerability_scan/docker-compose.yml build --pull && " + - "docker-compose -p ${project} -f src/.jenkins/owasp_vulnerability_scan/docker-compose.yml run -u root --rm test" - }, { - sh "docker-compose -p ${project} -f src/.jenkins/owasp_vulnerability_scan/docker-compose.yml down" - } - } - } - /* stage('OpenAPI specification check') { */ - /* tryStep "openapi specification check", { */ - /* sh "docker-compose -p openapi_validator -f src/.jenkins/openapi_validator/docker-compose.yml build --pull && " + */ - /* "docker-compose -p openapi_validator -f src/.jenkins/openapi_validator/docker-compose.yml run -u root --rm --entrypoint='' test /var/run_validator.sh" */ - /* }, { */ - /* sh "docker-compose -p openapi_validator -f src/.jenkins/openapi_validator/docker-compose.yml down" */ - /* } */ - /* } */ - /* } */ - - stage('Waiting for approval') { - slackSend channel: '#ci-channel', color: 'warning', message: 'DSO-API is waiting for Production Release - please confirm' - input "Deploy to Production?" - } - - node { - stage('Push production image') { - tryStep "image tagging", { - docker.withRegistry("${DOCKER_REGISTRY_HOST}",'docker_registry_auth') { - def image = docker.image("datapunt/dataservices/dso-api:${env.BUILD_NUMBER}") - image.pull() - image.push("production") - image.push("latest") - } - } - } - } - - node { - stage("Deploy") { - tryStep "deployment", { - build job: 'Subtask_Openstack_Playbook', - parameters: [ - [$class: 'StringParameterValue', name: 'INVENTORY', value: 'production'], - [$class: 'StringParameterValue', name: 'PLAYBOOK', value: 'deploy-dataservices-mdbrole.yml'], - ] - } - tryStep "deployment", { - build job: 'Subtask_Openstack_Playbook', - parameters: [ - [$class: 'StringParameterValue', name: 'INVENTORY', value: 'production'], - [$class: 'StringParameterValue', name: 'PLAYBOOK', value: 'deploy.yml'], - [$class: 'StringParameterValue', name: 'PLAYBOOKPARAMS', value: "-e cmdb_id=app_dso-api"] - ] - } - } - } - -} diff --git a/src/.jenkins/docker-migrate.sh b/src/.jenkins/docker-migrate.sh deleted file mode 100755 index 70465663d..000000000 --- a/src/.jenkins/docker-migrate.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -u # crash on missing env variables -set -e # stop on any error - -echo "Migrating db" -export DATABASE_URL=$DATABASE_URL_WRITE -yes yes | INITIALIZE_DYNAMIC_VIEWSETS=0 python ./manage.py migrate --noinput diff --git a/src/.jenkins/docker-wait.sh b/src/.jenkins/docker-wait.sh deleted file mode 100755 index bd3e6b41b..000000000 --- a/src/.jenkins/docker-wait.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -u # crash on missing env variables -set -e # stop on any error - -# wait for postgres -while ! nc -z database 5432 -do - echo "Waiting for PostgreSQL..." - sleep 2 -done diff --git a/src/.jenkins/openapi_validator/Dockerfile b/src/.jenkins/openapi_validator/Dockerfile deleted file mode 100644 index 5c3903466..000000000 --- a/src/.jenkins/openapi_validator/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM stoplight/spectral - -COPY run_validator.sh /var/run_validator.sh - - -ENTRYPOINT [ "/var/run_validator.sh" ] diff --git a/src/.jenkins/openapi_validator/docker-compose.yml b/src/.jenkins/openapi_validator/docker-compose.yml deleted file mode 100644 index 614358b21..000000000 --- a/src/.jenkins/openapi_validator/docker-compose.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: "3.0" -services: - - test: - build: . - command: /var/run_validator.sh diff --git a/src/.jenkins/openapi_validator/run_validator.sh b/src/.jenkins/openapi_validator/run_validator.sh deleted file mode 100755 index bab21230e..000000000 --- a/src/.jenkins/openapi_validator/run_validator.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - - -host=${OPENAPI_HOST:-"https://acc.api.data.amsterdam.nl"} -apk add curl -set -e - -echo "$host/v1/openapi.yaml" -curl $host/v1/openapi.yaml --output /tmp/openapi.yaml - -spectral lint /tmp/openapi.yaml -s oas3-unused-components-schema -s oas3-server-trailing-slash -s operation-tag-defined -s path-keys-no-trailing-slash -s operation-parameters -s openapi-tags -s no-\$ref-siblings -# -s oas3-unused-components-schema \ # WARNING: Unused compontents in schema? -# -s oas3-server-trailing-slash \ # WARNING: trailing slash present in server spec -# -s operation-tag-defined \ # WARNING: TAGs not defined -# -s path-keys-no-trailing-slash \ # WARNING: DSO API ends with trailing slashes -# -s openapi-tags \ # WARNING: missing OpenAPI tags -# -s operation-parameters \ # WARNING: A parameter in this operation already exposes the same combination of `name` and `in` values. -# -s no-$ref-siblings # ERROR: $ref cannot be placed next to any other properties , Geometry Fields diff --git a/src/.jenkins/owasp_vulnerability_scan/__init__.py b/src/.jenkins/owasp_vulnerability_scan/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/.jenkins/owasp_vulnerability_scan/docker-compose.yml b/src/.jenkins/owasp_vulnerability_scan/docker-compose.yml deleted file mode 100644 index 5df2af62f..000000000 --- a/src/.jenkins/owasp_vulnerability_scan/docker-compose.yml +++ /dev/null @@ -1,78 +0,0 @@ -# All commented out lines in this file are needed for debugging puproses. -version: "3.0" -services: - owasp: - # Using version 2.9.0 as version 2.10.0 has issues with Selenium. - image: owasp/zap2docker-stable:2.9.0 - ports: - - "8090:8090" - networks: - - owasp - # links: - # - web - command: zap.sh -daemon -port 8090 -host 0.0.0.0 -config api.disablekey=true -config api.addrs.addr.name=.* -config api.addrs.addr.regex=true - - test: - build: ../.. - links: - - owasp - # - web - networks: - - owasp - environment: - UWSGI_HTTP_SOCKET: ":8000" - UWSGI_MODULE: "dso_api.wsgi" - UWSGI_CALLABLE: "application" - UWSGI_MASTER: 1 - UWSGI_STATIC_MAP: "/dso_api/static=/static" - SECRET_KEY: insecure - DATABASE_URL: "postgres://dataservices:insecure@database/dataservices" - command: > - bash -c ".jenkins/owasp_vulnerability_scan/owasp_scan.sh" - # volumes: - # - ../../:/app - - # Local development goodies. - # database: - # image: amsterdam/postgres11 - # ports: - # - "5415:5432" - # networks: - # - owasp - # environment: - # POSTGRES_DB: dataservices - # POSTGRES_USER: dataservices - # POSTGRES_PASSWORD: insecure - # volumes: - # - "~/.ssh/datapunt.key:/root/.ssh/datapunt.key" - # - .:/host - # extra_hosts: - # admin.data.amsterdam.nl: 10.243.16.4 - # - # web: - # build: - # context: ../../ - # args: - # PIP_REQUIREMENTS: requirements_dev.txt - # ports: - # - "8000:8000" - # links: - # - database - # networks: - # - owasp - # environment: - # UWSGI_HTTP_SOCKET: ":8000" - # UWSGI_MODULE: "dso_api.wsgi" - # UWSGI_CALLABLE: "application" - # UWSGI_MASTER: 1 - # UWSGI_STATIC_MAP: "/dso_api/static=/static" - # SECRET_KEY: insecure - # DATABASE_URL: "postgres://dataservices:insecure@database/dataservices" - # DJANGO_DEBUG: 1 - # volumes: - # - ../../:/app - # - ./docs/build/html:/docs-static - # command: uwsgi --py-auto-reload=1 - -networks: - owasp: diff --git a/src/.jenkins/owasp_vulnerability_scan/docker-wait.sh b/src/.jenkins/owasp_vulnerability_scan/docker-wait.sh deleted file mode 100755 index fcf80e5ae..000000000 --- a/src/.jenkins/owasp_vulnerability_scan/docker-wait.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -u # crash on missing env variables -set -e # stop on any error - -# wait for owasp -while ! nc -z owasp 8090 -do - echo "Waiting for ZAP (zeds attack proxy)..." - sleep 2 -done - -echo "OWASP running...start OWASP check scan" \ No newline at end of file diff --git a/src/.jenkins/owasp_vulnerability_scan/owasp_scan.py b/src/.jenkins/owasp_vulnerability_scan/owasp_scan.py deleted file mode 100644 index 19199eb93..000000000 --- a/src/.jenkins/owasp_vulnerability_scan/owasp_scan.py +++ /dev/null @@ -1,69 +0,0 @@ -import time -from pprint import pprint - -from zapv2 import ZAPv2 - - -def test_owasp_check(): - """Prove that there are no HIGH risk vulnerabilities detected""" - # The URL of the application to be tested - target = "https://acc.api.data.amsterdam.nl/v1/fietspaaltjes/" - - # setup ZAP connection and point to ZAP proxy (a docker container) - # You can also access the ZAP GUI (and envoke scans or get data) - # by browsing to localhost:8090 - zap = ZAPv2( - proxies={"http": "http://owasp:8090", "https": "https://owasp:8090"}, - ) - - print("Stopping all current scans") - zap.ascan.stop_all_scans() - - # Exclude static from being scanned - zap.ascan.clear_excluded_from_scan() - zap.ascan.exclude_from_scan(regex=r".*/v1/(?!fietspaaltjes).*") - zap.spider.clear_excluded_from_scan() - zap.spider.exclude_from_scan(regex=r".*/v1/(?!fietspaaltjes).*") - print(f"Excluded: {zap.ascan.excluded_from_scan}") - print(f"Excluded from spider: {zap.spider.excluded_from_scan}") - - # start spider to get URL's - print(f"Spidering target {target}") - # The scan returns a scan id to support concurrent scanning - # zap.spider.set_option_max_children = 0 - # zap.spider.set_option_max_depth = 0 - scanID = zap.spider.scan(target) - run_spider(zap, scanID) - - # with active scanning, attacks are simulated on the target - print(f"Active Scanning target {target}") - zap.core.set_mode("ATTACK") - scanID = zap.ascan.scan(target) - while int(zap.ascan.status(scanID)) < 100: - # Loop until the scanner has finished - print(f"Scan progress: {zap.ascan.status(scanID)}, scan id: {scanID}") - time.sleep(5) - print("Active Scan completed") - - # print vulnerabilities found by the scanning - print("Hosts: {}".format(", ".join(zap.core.hosts))) - print("Alerts: ") - alerts = zap.alert.alerts(baseurl=target) - pprint(alerts) - print("Total number of alerts: " + str(len(alerts))) - - if any("High" in a["risk"] for a in alerts): - raise Exception("high risk alert found") - - -def run_spider(zap, scanID): - if len(zap.spider.all_urls): - print("Skip spider action") - else: - while int(zap.spider.status(scanID)) < 100: - # Poll the status until it completes - print(f"Spider progress: {zap.spider.status(scanID)}, scan id: {scanID}") - time.sleep(1) - - results = "\n".join(zap.spider.results(scanID)) - print(f"Spider found:\n{results}") diff --git a/src/.jenkins/owasp_vulnerability_scan/owasp_scan.sh b/src/.jenkins/owasp_vulnerability_scan/owasp_scan.sh deleted file mode 100755 index 97ceb6f72..000000000 --- a/src/.jenkins/owasp_vulnerability_scan/owasp_scan.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -u # crash on missing env -set -e # stop on any error - -echo "Waiting for OWASP" -source .jenkins/owasp_vulnerability_scan/docker-wait.sh - -# echo "Running OWASP tests" -DJANGO_DEBUG=false pytest --disable-warnings -s .jenkins/owasp_vulnerability_scan/owasp_scan.py diff --git a/src/.jenkins/test/docker-compose.yml b/src/.jenkins/test/docker-compose.yml deleted file mode 100644 index bcb20e7cb..000000000 --- a/src/.jenkins/test/docker-compose.yml +++ /dev/null @@ -1,24 +0,0 @@ -version: "3.0" -services: - database: - image: amsterdam/postgres11 - environment: - POSTGRES_DB: dataservices - POSTGRES_USER: dataservices - POSTGRES_PASSWORD: insecure - - test: - build: ../.. - links: - - database - environment: - UWSGI_HTTP_SOCKET: ":8000" - UWSGI_MODULE: "dso_api.wsgi" - UWSGI_CALLABLE: "application" - UWSGI_MASTER: 1 - UWSGI_STATIC_MAP: "/dso_api/static=/static" - SECRET_KEY: insecure - DATABASE_URL: "postgres://dataservices:insecure@database/dataservices" - DATABASE_URL_WRITE: "postgres://dataservices:insecure@database/dataservices" - command: > - bash -c ".jenkins/test/test.sh" diff --git a/src/.jenkins/test/test.sh b/src/.jenkins/test/test.sh deleted file mode 100755 index f84f7a2a4..000000000 --- a/src/.jenkins/test/test.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -u # crash on missing env -set -e # stop on any error - -echo "Waiting for db" -source .jenkins/docker-wait.sh -source .jenkins/docker-migrate.sh - -echo "Running style checks" -flake8 --config=.flake8 ./dso_api - -# echo "Running unit tests" -DJANGO_DEBUG=false pytest --nomigrations -vs --ds=tests.settings --show-capture=no - -echo "Running bandit" -bandit --format screen --exclude src/tests --recursive src