diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2797fa6b..7cde8402 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,4 +12,18 @@ updates: interval: weekly time: "09:00" timezone: "Europe/Berlin" +- package-ecosystem: gomod + directory: /acceptance-tests + target-branch: "maintenance" + schedule: + interval: weekly + time: "09:00" + timezone: "Europe/Berlin" +- package-ecosystem: pip + directory: /ci/scripts + target-branch: "maintenance" + schedule: + interval: weekly + time: "09:00" + timezone: "Europe/Berlin" diff --git a/ci/README.md b/ci/README.md index 826d5dd4..fb534a91 100644 --- a/ci/README.md +++ b/ci/README.md @@ -87,7 +87,7 @@ The pipeline `haproxy-boshrelease` is used to build, verify and release this BOS New pipeline steps should be added without modifying existing steps or resources, or in a separate pipeline altogether. -A pipeline can be uploaded to concourse via the [`upload-to-concourse.sh`](upload-to-concourse.sh) script. This script requires the data in `source.me`, which can be found in the team's Vault. +A pipeline can be uploaded manually to concourse via the [`upload-to-concourse.sh`](upload-to-concourse.sh) script. This script requires the data in `source.me`, which can be found in the team's Vault. ### Testing new Pipeline Steps in a Branch diff --git a/ci/pipeline-haproxy-maintenance.yml b/ci/pipeline-haproxy-maintenance.yml new file mode 100644 index 00000000..b4f1d6b1 --- /dev/null +++ b/ci/pipeline-haproxy-maintenance.yml @@ -0,0 +1,525 @@ +--- + +groups: + - name: haproxy-boshrelease-maintenance + jobs: + - unit-tests + - unit-tests-pr + - acceptance-tests + - acceptance-tests-pr + - pre + - rc + - shipit + - patch + - minor + - major + - autobump-dependencies + +jobs: + - name: unit-tests + public: true + serial: true + plan: + - do: + - get: git-previous-release + trigger: true + - task: lint + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - { name: git-previous-release } + caches: + - path: git-previous-release/vendor/cache + - path: git-previous-release/.bundle + run: + path: ./git-previous-release/ci/scripts/lint + args: [] + params: + REPO_ROOT: git-previous-release + - task: unit-tests + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - { name: git-previous-release } + + caches: + - path: git-previous-release/vendor/cache + - path: git-previous-release/.bundle + run: + path: ./git-previous-release/ci/scripts/unit-tests + args: [] + params: + REPO_ROOT: git-previous-release + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : unit-tests job failed" + + - name: unit-tests-pr + public: true + serial: true + plan: + - do: + - { get: git-pull-requests-previous-release, trigger: true, version: every } + - put: git-pull-requests-previous-release + params: + path: git-pull-requests-previous-release + status: pending + context: unit-tests + - task: lint + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - { name: git-pull-requests-previous-release } + caches: + - path: git-pull-requests-previous-release/vendor/cache + - path: git-pull-requests-previous-release/.bundle + run: + path: ./git-pull-requests-previous-release/ci/scripts/lint + args: [] + params: + REPO_ROOT: git-pull-requests-previous-release + - task: unit-tests + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - { name: git-pull-requests-previous-release } + caches: + - path: git-pull-requests-previous-release/vendor/cache + - path: git-pull-requests-previous-release/.bundle + run: + path: ./git-pull-requests-previous-release/ci/scripts/unit-tests + args: [] + params: + REPO_ROOT: git-pull-requests-previous-release + on_success: + put: git-pull-requests-previous-release + params: + path: git-pull-requests-previous-release + status: success + context: unit-tests + on_failure: + put: git-pull-requests-previous-release + params: + path: git-pull-requests-previous-release + status: failure + context: unit-tests + + + - name: acceptance-tests + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: git-previous-release, trigger: true, passed: [unit-tests] } + - { get: stemcell } + - { get: stemcell-bionic } + - { get: bpm } + - task: acceptance-tests + privileged: true + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - { name: git-previous-release } + - { name: stemcell } + - { name: stemcell-bionic } + - { name: bpm } + run: + path: ./git-previous-release/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git-previous-release + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : acceptance tests failed" + + - name: acceptance-tests-pr + public: true + serial: true + plan: + - do: + - { get: git-pull-requests-previous-release, trigger: true, version: every, passed: [unit-tests-pr] } + - { get: stemcell } + - { get: stemcell-bionic } + - { get: bpm } + - put: git-pull-requests-previous-release + params: + path: git-pull-requests-previous-release + status: pending + context: acceptance-tests + - task: acceptance-tests + privileged: true + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - { name: git-pull-requests-previous-release } + - { name: stemcell } + - { name: stemcell-bionic } + - { name: bpm } + run: + path: ./git-pull-requests-previous-release/ci/scripts/acceptance-tests + args: [] + params: + REPO_ROOT: git-pull-requests-previous-release + on_success: + put: git-pull-requests-previous-release + params: + path: git-pull-requests-previous-release + status: success + context: acceptance-tests + on_failure: + put: git-pull-requests-previous-release + params: + path: git-pull-requests-previous-release + status: failure + context: acceptance-tests + + - name: pre + public: true + serial: true + plan: + - do: + - get: git-previous-release + passed: + - acceptance-tests + trigger: true + - get: version + trigger: true + - task: release-notes + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + run: + path: sh + args: + - -ce + - | + cd git-previous-release + if [ -f ci/release_notes.md ]; then + echo "###### RELEASE NOTES ###############" + echo + cat ci/release_notes.md + echo + echo "########################################" + echo + else + echo "NO RELEASE NOTES HAVE BEEN WRITTEN" + echo "You *might* want to do that before" + echo "hitting (+) on that shipit job..." + echo + fi + inputs: + - name: git-previous-release + #on_failure: + # put: notify + # params: + # channel: "#haproxy-boshrelease" + # username: ci-bot + # icon_url: "((slack.icon))" + # text: release candidate job 'pre' failed (which is unusual). + # ok: false + + - name: rc + public: true + plan: + - do: + - in_parallel: + - { get: git-previous-release, trigger: true, passed: [pre] } + - { get: version, trigger: false, params: {pre: rc} } + - put: version + params: {file: version/number} + #on_failure: + # put: notify + # params: + # channel: "#haproxy-boshrelease" + # username: ci-bot + # icon_url: "((slack.icon))" + # text: "((slack.fail_url)) haproxy-boshrelease : rc job failed" + + - name: patch + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: patch} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : patch job failed" + + - name: minor + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: minor} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : minor job failed" + + - name: major + public: true + plan: + - do: + - { get: version, trigger: false, params: {bump: major} } + - { put: version, params: {file: version/number} } + on_failure: + put: notify + params: + channel: "#haproxy-boshrelease" + username: ci-bot + icon_url: "((slack.icon))" + text: "((slack.fail_url)) haproxy-boshrelease : major job failed" + + - name: shipit + public: true + serial: true + plan: + - do: + - in_parallel: + - { get: version, passed: [rc], params: {bump: final} } + - { get: git-previous-release, passed: [rc] } + - task: release + config: + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + inputs: + - name: version + - name: git-previous-release + outputs: + - name: gh + - name: pushme + - name: notifications + run: + path: ./git-previous-release/ci/scripts/shipit + args: [] + params: + REPO_ROOT: git-previous-release + VERSION_FROM: version/number + RELEASE_ROOT: gh + REPO_OUT: pushme + NOTIFICATION_OUT: notifications + BRANCH: maintenance + GITHUB_OWNER: cloudfoundry + + GCP_SERVICE_KEY: ((gcp.service_key)) + + - put: git-previous-release + params: + rebase: true + repository: pushme/git-previous-release + - put: blobstore + params: + file: "gh/artifacts/haproxy-*.tgz" + - put: github + params: + name: gh/name + tag: gh/tag + body: gh/notes.md + globs: [gh/artifacts/*] + # - put: notify + # params: + # channel: "#haproxy-boshrelease" + # username: ci-bot + # icon_url: "((slack.icon))" + # text_file: notifications/message + # on_failure: + # put: notify + # params: + # channel: "#haproxy-boshrelease" + # username: ci-bot + # icon_url: "((slack.icon))" + # text: "((slack.fail_url)) haproxy-boshrelease : shipit job failed" + - name: autobump-dependencies + public: true + serial: true + plan: + - do: + - get: daily + trigger: true + - get: git-previous-release + - task: autobump-dependencies + config: + inputs: + - name: git-previous-release + platform: linux + image_resource: + type: docker-image + source: + repository: iacbox.common.repositories.cloud.sap/haproxy-boshrelease-testflight + tag: latest + username: ((docker.username)) + password: ((docker.password)) + run: + dir: git-previous-release + path: /usr/bin/python3 + args: ["ci/scripts/autobump-dependencies.py"] + params: + REPO_ROOT: git-previous-release + PR_BASE: maintenance + PR_ORG: cloudfoundry + PR_LABEL: run-ci + + GCP_SERVICE_KEY: ((gcp.service_key)) + GITHUB_COM_TOKEN: ((github.access_token)) + +resource_types: + - name: slack-notification + type: docker-image + source: + repository: cfcommunity/slack-notification-resource + + - name: pull-request + type: docker-image + source: + repository: teliaoss/github-pr-resource + + # FIXME: Need to use latest version of this resource due to + # https://github.com/concourse/github-release-resource/issues/108 + # https://github.com/concourse/github-release-resource/pull/107 + # Until Concourse is updated to 7.5.0+ + - name: github-release-alt + type: registry-image + source: + repository: concourse/github-release-resource + + - name: gcs + type: docker-image + source: + repository: frodenas/gcs-resource + +resources: + - name: git-previous-release + type: git + source: + uri: git@github.com:cloudfoundry/haproxy-boshrelease.git + branch: maintenance + private_key: ((github.private_key)) + + - name: git-pull-requests-previous-release + type: pull-request + source: + access_token: ((github.access_token)) + repository: cloudfoundry/haproxy-boshrelease + base_branch: maintenance + labels: [run-ci] + + - name: stemcell-bionic + type: bosh-io-stemcell + source: + name: bosh-warden-boshlite-ubuntu-bionic-go_agent + + - name: stemcell + type: bosh-io-stemcell + source: + name: bosh-warden-boshlite-ubuntu-jammy-go_agent + + - name: bpm + type: bosh-io-release + source: + repository: cloudfoundry/bpm-release + + - name: version + type: semver + source : + driver: gcs + bucket: haproxy-boshrelease-maintenance + key: version + json_key: ((gcp.service_key)) + initial_version: "11.17.0" + + - name: notify + type: slack-notification + source: + url: ((slack.webhook)) + + - name: github + type: github-release-alt + source: + user: cloudfoundry + repository: haproxy-boshrelease + access_token: ((github.access_token)) + + - name: blobstore + type: gcs + source: + bucket: haproxy-boshrelease-maintenance + json_key: ((gcp.service_key)) + regexp: haproxy-([0-9\.]+).tgz + + - name: daily + type: time + source: + start: 7:00 AM + stop: 8:00 AM + location: Europe/Berlin + interval: 24h diff --git a/ci/scripts/shipit b/ci/scripts/shipit index f28be9a4..1ae3ecef 100755 --- a/ci/scripts/shipit +++ b/ci/scripts/shipit @@ -65,14 +65,29 @@ YAML header "Pulling in any git submodules..." git submodule update --init --recursive --force + +version() { + # extract the version variable $1 from the packaging script $2 (default 'haproxy') + pattern='s/VERSION=(.*)(\s?#.*)/\1/p' + package=${2:-haproxy} + # extract version and remove all spaces + sed -n -E "${pattern//VERSION/${1:?}}" "${REPO_ROOT}/packages/${package}/packaging" | sed 's/ *//g' +} + +HAPROXY_VERSION=$(version HAPROXY_VERSION) +LUA_VERSION=$(version LUA_VERSION) +SOCAT_VERSION=$(version SOCAT_VERSION) +PCRE_VERSION=$(version PCRE_VERSION) +KEEPALIVED_VERSION=$(version KEEPALIVED_VERSION keepalived) + header "Create final release..." -bosh -n create-release --final --version "${VERSION}" -bosh -n create-release "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.yml" \ - --tarball "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.tgz" +bosh -n create-release --final --version "${VERSION}+${HAPROXY_VERSION}" +bosh -n create-release "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}+${HAPROXY_VERSION}.yml" \ + --tarball "releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}+${HAPROXY_VERSION}.tgz" cd - # SC2155 discourages variable assignment and export in the same line. -RELEASE_TGZ=${REPO_ROOT}/releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}.tgz +RELEASE_TGZ=${REPO_ROOT}/releases/${RELEASE_NAME}/${RELEASE_NAME}-${VERSION}+${HAPROXY_VERSION}.tgz # shellcheck disable=SC2155 export SHA1=$(sha1sum "${RELEASE_TGZ}" | head -n1 | awk '{print $1}') echo "SHA1=${SHA1}" @@ -83,25 +98,11 @@ echo "SHA256=${SHA256}" mkdir -p "${RELEASE_ROOT}/artifacts" -echo "v${VERSION}" > "${RELEASE_ROOT}/tag" -echo "v${VERSION}" > "${RELEASE_ROOT}/name" -mv "${REPO_ROOT}"/releases/*/*-"${VERSION}".tgz "${RELEASE_ROOT}/artifacts" +echo "v${VERSION}+${HAPROXY_VERSION}" > "${RELEASE_ROOT}/tag" +echo "v${VERSION}+${HAPROXY_VERSION}" > "${RELEASE_ROOT}/name" +mv "${REPO_ROOT}"/releases/*/*-"${VERSION}+${HAPROXY_VERSION}".tgz "${RELEASE_ROOT}/artifacts" mv "${REPO_ROOT}/ci/release_notes.md" "${RELEASE_ROOT}/notes.md" -version() { - # extract the version variable $1 from the packaging script $2 (default 'haproxy') - pattern='s/VERSION=(.*)(\s?#.*)/\1/p' - package=${2:-haproxy} - # extract version and remove all spaces - sed -n -E "${pattern//VERSION/${1:?}}" "${REPO_ROOT}/packages/${package}/packaging" | sed 's/ *//g' -} - -HAPROXY_VERSION=$(version HAPROXY_VERSION) -LUA_VERSION=$(version LUA_VERSION) -SOCAT_VERSION=$(version SOCAT_VERSION) -PCRE_VERSION=$(version PCRE_VERSION) -KEEPALIVED_VERSION=$(version KEEPALIVED_VERSION keepalived) - cat >> "${RELEASE_ROOT}/notes.md" < "${RELEASE_ROOT}/notification" < New ${RELEASE_NAME} v${VERSION} released! + New ${RELEASE_NAME} v${VERSION}+${HAPROXY_VERSION} released! EOF header "Update git repo with final release..." @@ -142,12 +143,12 @@ fi pushd "${REPO_ROOT}" for MANIFEST_PATH in manifests/*.yml; do - "${DIR}/update-manifest" "${GITHUB_OWNER}" "${RELEASE_NAME}" "${VERSION}" "${SHA1}" "${MANIFEST_PATH}" + "${DIR}/update-manifest" "${GITHUB_OWNER}" "${RELEASE_NAME}" "${VERSION}+${HAPROXY_VERSION}" "${SHA1}" "${MANIFEST_PATH}" done git merge --no-edit "${BRANCH}" git add -A git status - git commit -m "release v${VERSION}" + git commit -m "release v${VERSION}+${HAPROXY_VERSION}" # After creating a final release we will also create a dev release patches from haproxy-patches directory echo "- haproxy/patches.tar.gz" >> packages/haproxy/spec @@ -156,7 +157,7 @@ pushd "${REPO_ROOT}" bosh upload-blobs bosh -n create-release --force --version "${VERSION}-patched" \ - --tarball "../${RELEASE_NAME}_patched-${VERSION}.tgz" + --tarball "../${RELEASE_NAME}_patched-${VERSION}+${HAPROXY_VERSION}.tgz" # Undo changes to repo from creating dev release git clean -df @@ -165,7 +166,7 @@ popd mv "${RELEASE_NAME}_patched-${VERSION}.tgz" "${RELEASE_ROOT}/artifacts" -PATCHED_RELEASE_TGZ=${RELEASE_ROOT}/artifacts/${RELEASE_NAME}_patched-${VERSION}.tgz +PATCHED_RELEASE_TGZ=${RELEASE_ROOT}/artifacts/${RELEASE_NAME}_patched-${VERSION}+${HAPROXY_VERSION}.tgz # shellcheck disable=SC2155 export PATCHED_SHA1=$(sha1sum "${PATCHED_RELEASE_TGZ}" | head -n1 | awk '{print $1}') echo "PATCHED_SHA1=${PATCHED_SHA1}" @@ -180,7 +181,7 @@ cat >> "${RELEASE_ROOT}/notes.md" < "${NOTIFICATION_OUT:-notifications}/message" <. +New ${RELEASE_NAME} v${VERSION}+${HAPROXY_VERSION} released. . EOS diff --git a/ci/upload-to-concourse.sh b/ci/upload-to-concourse.sh index b8a891b6..24130e03 100755 --- a/ci/upload-to-concourse.sh +++ b/ci/upload-to-concourse.sh @@ -32,5 +32,8 @@ fly -t "$CONCOURSE_TARGET" login -c "$CONCOURSE_URL" -u "$CONCOURSE_USER" -p "$C fly -t "$CONCOURSE_TARGET" validate-pipeline -c pipeline.yml fly -t "$CONCOURSE_TARGET" set-pipeline -p haproxy-boshrelease -c pipeline.yml --load-vars-from vars.yml fly -t "$CONCOURSE_TARGET" expose-pipeline -p haproxy-boshrelease +fly -t "$CONCOURSE_TARGET" validate-pipeline -c pipeline-haproxy-maintenance.yml +fly -t "$CONCOURSE_TARGET" set-pipeline -p haproxy-boshrelease-maintenance -c pipeline-haproxy-maintenance.yml --load-vars-from vars.yml +fly -t "$CONCOURSE_TARGET" expose-pipeline -p haproxy-boshrelease-maintenance echo "Done." diff --git a/config/final.yml b/config/final.yml index 0f50458f..8d10cca6 100644 --- a/config/final.yml +++ b/config/final.yml @@ -3,4 +3,4 @@ name: haproxy blobstore: provider: gcs options: - bucket_name: haproxy-boshrelease + bucket_name: haproxy-boshrelease-maintenance