From bed21a34d3e9216c7193c1a7e179418e553a9f48 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 16 Dec 2022 11:29:02 +0100
Subject: [PATCH 01/23] Add extra example argument to whirl to prepare for
 python packaging

---
 .github/workflows/whirl-ci.yml | 62 +++++++++++++++++++++++++++-------
 whirl                          | 27 +++++++++++----
 2 files changed, 70 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/whirl-ci.yml b/.github/workflows/whirl-ci.yml
index 509d863..d6f71d8 100644
--- a/.github/workflows/whirl-ci.yml
+++ b/.github/workflows/whirl-ci.yml
@@ -25,7 +25,18 @@ jobs:
       - uses: actions/checkout@v2
       - id: setdirs # Give it an id to handle to get step outputs in the outputs key above
         run: echo "::set-output name=dir::$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')"
-        # Define step output named dir base on ls command transformed to JSON thanks to jq
+        # Define step output named dir based on ls command transformed to JSON thanks to jq
+
+  examples: # Job that list subdirectories of ./examples
+    runs-on: ubuntu-latest
+    outputs:
+      # generate output name example by using inner step output
+      example: ${{ steps.setexamples.outputs.example }}
+    steps:
+      - uses: actions/checkout@v2
+      - id: setexamples # Give it an id to handle to get step outputs in the outputs key above
+      run: echo "::set-output name=example::$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)\//\1/g' | jq -R -s -c 'split("\n")[:-1]')"
+      # Define step output named example based on ls command transformed to JSON thanks to jq
 
   whirl-ci-default-envs:
     needs: [directories]
@@ -34,27 +45,54 @@ jobs:
       fail-fast: false
       max-parallel: 4
       matrix:
-        example: ${{ fromJson(needs.directories.outputs.dir) }}
+        example_dir: ${{ fromJson(needs.directories.outputs.dir) }}
         python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.2"]
+        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
         exclude:
           # Needs more memory than available on the runner
-          - example: ./examples/dbt-spark-example
-          - example: ./examples/spark-delta-sharing
-          - example: ./examples/spark-s3-to-hive
+          - example_dir: ./examples/dbt-spark-example
+          - example_dir: ./examples/spark-delta-sharing
+          - example_dir: ./examples/spark-s3-to-hive
           # Exclude failing dbt runs
-          - example: ./examples/dbt-example
+          - example_dir: ./examples/dbt-example
     env:
       PYTHON_VERSION: ${{ matrix.python_version }}
       AIRFLOW_VERSION: ${{ matrix.airflow_version }}
     steps:
       - uses: actions/checkout@v2
-      - name: Run whirl CI ${{ matrix.example }}
-        working-directory: ${{ matrix.example }}
+      - name: Run whirl CI ${{ matrix.example_dir }}
+        working-directory: ${{ matrix.example_dir }}
         run: |
-          echo Run Ci for example ${{ matrix.example }}
+          echo Run Ci from example directory ${{ matrix.example_dir }}
           ../../whirl ci
 
+  whirl-ci-default-envs-from-root-dir:
+    needs: [examples]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        example: ${{ fromJson(needs.examples.outputs.example) }}
+        python_version: ["3.8", "3.9"]
+        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
+        exclude:
+          # Needs more memory than available on the runner
+          - example: dbt-spark-example
+          - example: spark-delta-sharing
+          - example: spark-s3-to-hive
+          # Exclude failing dbt runs
+          - example: dbt-example
+    env:
+      PYTHON_VERSION: ${{ matrix.python_version }}
+      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run whirl CI example ${{ matrix.example }}
+        run: |
+          echo Run Ci for example ${{ matrix.example }}
+          ./whirl -x ${{ matrix.example }} ci
+
 
   whirl-ci-extra-env-spark-s3-to-postgres:
     runs-on: ubuntu-latest
@@ -72,7 +110,7 @@ jobs:
       max-parallel: 4
       matrix:
         python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.2"]
+        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: ${{ matrix.python_version }}
@@ -91,7 +129,7 @@ jobs:
       max-parallel: 4
       matrix:
         python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.2"]
+        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
     runs-on: ubuntu-latest
     env:
       PYTHON_VERSION: ${{ matrix.python_version }}
diff --git a/whirl b/whirl
index 52760d1..9d364f4 100755
--- a/whirl
+++ b/whirl
@@ -21,10 +21,6 @@ function export_environment_vars() {
   # shellcheck disable=SC2034
   DOCKER_CONTEXT_FOLDER=${SCRIPT_DIR}/docker
   # shellcheck disable=SC2034
-  DAG_FOLDER=$(pwd)
-  # shellcheck disable=SC2034
-  PROJECTNAME=$(basename "${DAG_FOLDER}")
-  # shellcheck disable=SC2034
   WHIRL_INITIATOR=$(whoami)
   # shellcheck disable=SC2034
   WHIRL_SETUP_FOLDER=/etc/airflow/whirl.setup.d
@@ -41,6 +37,17 @@ function export_environment_vars() {
     . "${SCRIPT_DIR}/.whirl.env"
   fi
 
+  # determine whether to use the example set at the commandline or
+  # in the current folder
+  if [ -z "${WHIRL_EXAMPLE_ARG}" ]; then
+    # shellcheck disable=SC2034
+    DAG_FOLDER=$(pwd)
+  else
+    DAG_FOLDER=${SCRIPT_DIR}/examples/${WHIRL_EXAMPLE_ARG}
+  fi
+  # shellcheck disable=SC2034
+  PROJECTNAME=$(basename "${DAG_FOLDER}")
+
   # determine whether to use the environment set at the commandline or
   # in the DAG FOLDER .whirl.env
   if [ -z "${WHIRL_ENVIRONMENT_ARG}" ]; then
@@ -113,7 +120,7 @@ function export_environment_vars() {
 }
 
 detect_potential_dag() {
-  test "$(find . -type f -name '*.py' -o -name '*.zip' | wc -l)" -gt 0
+  test "$(find "${DAG_FOLDER}" -type f -name '*.py' -maxdepth 1 -o -name '*.zip' | wc -l)" -gt 0
 }
 
 check_next_dagrun_scheduled_today() {
@@ -354,6 +361,11 @@ function read_arguments() {
   do
       key="${1}"
       case ${key} in
+      -x|--example)
+          WHIRL_EXAMPLE_ARG="${2}"
+          shift # past argument
+          shift # past value
+          ;;
       -e|--environment)
           WHIRL_ENVIRONMENT_ARG="${2}"
           shift # past argument
@@ -401,8 +413,8 @@ function read_arguments() {
 function main() {
   read_arguments "$@"
 
+  export_environment_vars
   if detect_potential_dag; then
-    export_environment_vars
 
     if [ -z "${LOGS}" ]; then
       if [ -z "${STOP}" ]; then
@@ -414,7 +426,8 @@ function main() {
       logs
     fi
   else
-    echo "No .py or .zip files found that may contain an Apache Airflow DAG"
+    echo "No .py or .zip files found in ${DAG_FOLDER} that may contain an Apache Airflow DAG"
+    echo "did you correctly specify the example directory?"
   fi
 }
 

From a97f98d2a852206fff2aa3c81c9a374068f0f182 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 16 Dec 2022 11:32:35 +0100
Subject: [PATCH 02/23] Add extra example argument to whirl to prepare for
 python packaging

---
 .github/workflows/whirl-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/whirl-ci.yml b/.github/workflows/whirl-ci.yml
index d6f71d8..27f378f 100644
--- a/.github/workflows/whirl-ci.yml
+++ b/.github/workflows/whirl-ci.yml
@@ -35,8 +35,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - id: setexamples # Give it an id to handle to get step outputs in the outputs key above
-      run: echo "::set-output name=example::$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)\//\1/g' | jq -R -s -c 'split("\n")[:-1]')"
-      # Define step output named example based on ls command transformed to JSON thanks to jq
+        run: echo "::set-output name=example::$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)\//\1/g' | jq -R -s -c 'split("\n")[:-1]')"
+        # Define step output named example based on ls command transformed to JSON thanks to jq
 
   whirl-ci-default-envs:
     needs: [directories]

From b38151a93802fbc7187925136b30aa8cd607d831 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 16 Dec 2022 11:52:12 +0100
Subject: [PATCH 03/23] Convert set-output to new environment based github
 actions solution

---
 .github/workflows/whirl-ci.yml | 149 +++++++++++++++++----------------
 1 file changed, 75 insertions(+), 74 deletions(-)

diff --git a/.github/workflows/whirl-ci.yml b/.github/workflows/whirl-ci.yml
index 27f378f..09ba53f 100644
--- a/.github/workflows/whirl-ci.yml
+++ b/.github/workflows/whirl-ci.yml
@@ -24,7 +24,8 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - id: setdirs # Give it an id to handle to get step outputs in the outputs key above
-        run: echo "::set-output name=dir::$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')"
+        # run: echo "::set-output name=dir::$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')"
+        run: echo "dir=$(ls -d ./examples/* | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
         # Define step output named dir based on ls command transformed to JSON thanks to jq
 
   examples: # Job that list subdirectories of ./examples
@@ -35,36 +36,36 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - id: setexamples # Give it an id to handle to get step outputs in the outputs key above
-        run: echo "::set-output name=example::$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)\//\1/g' | jq -R -s -c 'split("\n")[:-1]')"
+        run: echo "example=$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)/\1/g' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
         # Define step output named example based on ls command transformed to JSON thanks to jq
 
-  whirl-ci-default-envs:
-    needs: [directories]
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      max-parallel: 4
-      matrix:
-        example_dir: ${{ fromJson(needs.directories.outputs.dir) }}
-        python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
-        exclude:
-          # Needs more memory than available on the runner
-          - example_dir: ./examples/dbt-spark-example
-          - example_dir: ./examples/spark-delta-sharing
-          - example_dir: ./examples/spark-s3-to-hive
-          # Exclude failing dbt runs
-          - example_dir: ./examples/dbt-example
-    env:
-      PYTHON_VERSION: ${{ matrix.python_version }}
-      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run whirl CI ${{ matrix.example_dir }}
-        working-directory: ${{ matrix.example_dir }}
-        run: |
-          echo Run Ci from example directory ${{ matrix.example_dir }}
-          ../../whirl ci
+  # whirl-ci-default-envs:
+  #   needs: [directories]
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     fail-fast: false
+  #     max-parallel: 4
+  #     matrix:
+  #       example_dir: ${{ fromJson(needs.directories.outputs.dir) }}
+  #       python_version: ["3.8", "3.9"]
+  #       airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
+  #       exclude:
+  #         # Needs more memory than available on the runner
+  #         - example_dir: ./examples/dbt-spark-example
+  #         - example_dir: ./examples/spark-delta-sharing
+  #         - example_dir: ./examples/spark-s3-to-hive
+  #         # Exclude failing dbt runs
+  #         - example_dir: ./examples/dbt-example
+  #   env:
+  #     PYTHON_VERSION: ${{ matrix.python_version }}
+  #     AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run whirl CI ${{ matrix.example_dir }}
+  #       working-directory: ${{ matrix.example_dir }}
+  #       run: |
+  #         echo Run Ci from example directory ${{ matrix.example_dir }}
+  #         ../../whirl ci
 
   whirl-ci-default-envs-from-root-dir:
     needs: [examples]
@@ -94,50 +95,50 @@ jobs:
           ./whirl -x ${{ matrix.example }} ci
 
 
-  whirl-ci-extra-env-spark-s3-to-postgres:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run whirl CI non-default env
-        working-directory: ./examples/spark-s3-to-postgres/
-        run: |
-          echo Run Ci for example spark-s3-to-postgres with non-default env
-          ../../whirl ci -e postgres-s3-spark
+  # whirl-ci-extra-env-spark-s3-to-postgres:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run whirl CI non-default env
+  #       working-directory: ./examples/spark-s3-to-postgres/
+  #       run: |
+  #         echo Run Ci for example spark-s3-to-postgres with non-default env
+  #         ../../whirl ci -e postgres-s3-spark
 
-  whirl-ci-extra-env-api-to-s3:
-    strategy:
-      fail-fast: false
-      max-parallel: 4
-      matrix:
-        python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
-    runs-on: ubuntu-latest
-    env:
-      PYTHON_VERSION: ${{ matrix.python_version }}
-      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run whirl CI api-to-s3 on k8s executor
-        working-directory: ./examples/api-to-s3/
-        run: |
-          echo Run Ci for example api-to-s3 with k8s executor env
-          ../../whirl ci -e api-python-s3-k8s
+  # whirl-ci-extra-env-api-to-s3:
+  #   strategy:
+  #     fail-fast: false
+  #     max-parallel: 4
+  #     matrix:
+  #       python_version: ["3.8", "3.9"]
+  #       airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     PYTHON_VERSION: ${{ matrix.python_version }}
+  #     AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run whirl CI api-to-s3 on k8s executor
+  #       working-directory: ./examples/api-to-s3/
+  #       run: |
+  #         echo Run Ci for example api-to-s3 with k8s executor env
+  #         ../../whirl ci -e api-python-s3-k8s
 
-  whirl-ci-extra-env-ha-scheduler:
-    strategy:
-      fail-fast: false
-      max-parallel: 4
-      matrix:
-        python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
-    runs-on: ubuntu-latest
-    env:
-      PYTHON_VERSION: ${{ matrix.python_version }}
-      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
-    steps:
-      - uses: actions/checkout@v2
-      - name: Run whirl CI ha-scheduler env
-        working-directory: ./examples/external-airflow-db/
-        run: |
-          echo Run Ci for example external-airflow-db with ha-scheduler env
-          ../../whirl ci -e ha-scheduler
+  # whirl-ci-extra-env-ha-scheduler:
+  #   strategy:
+  #     fail-fast: false
+  #     max-parallel: 4
+  #     matrix:
+  #       python_version: ["3.8", "3.9"]
+  #       airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
+  #   runs-on: ubuntu-latest
+  #   env:
+  #     PYTHON_VERSION: ${{ matrix.python_version }}
+  #     AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Run whirl CI ha-scheduler env
+  #       working-directory: ./examples/external-airflow-db/
+  #       run: |
+  #         echo Run Ci for example external-airflow-db with ha-scheduler env
+  #         ../../whirl ci -e ha-scheduler

From b2740cd39a4348dbb61adfac5554895bae2cef8e Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 16 Dec 2022 12:19:22 +0100
Subject: [PATCH 04/23] Enable 2 airflow versions max.

---
 .github/workflows/whirl-ci.yml | 146 ++++++++++++++++-----------------
 1 file changed, 73 insertions(+), 73 deletions(-)

diff --git a/.github/workflows/whirl-ci.yml b/.github/workflows/whirl-ci.yml
index 09ba53f..2cef48b 100644
--- a/.github/workflows/whirl-ci.yml
+++ b/.github/workflows/whirl-ci.yml
@@ -39,33 +39,33 @@ jobs:
         run: echo "example=$(ls -d ./examples/* | sed -r 's/\.\/examples\/(.*)/\1/g' | jq -R -s -c 'split("\n")[:-1]')" >> $GITHUB_OUTPUT
         # Define step output named example based on ls command transformed to JSON thanks to jq
 
-  # whirl-ci-default-envs:
-  #   needs: [directories]
-  #   runs-on: ubuntu-latest
-  #   strategy:
-  #     fail-fast: false
-  #     max-parallel: 4
-  #     matrix:
-  #       example_dir: ${{ fromJson(needs.directories.outputs.dir) }}
-  #       python_version: ["3.8", "3.9"]
-  #       airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
-  #       exclude:
-  #         # Needs more memory than available on the runner
-  #         - example_dir: ./examples/dbt-spark-example
-  #         - example_dir: ./examples/spark-delta-sharing
-  #         - example_dir: ./examples/spark-s3-to-hive
-  #         # Exclude failing dbt runs
-  #         - example_dir: ./examples/dbt-example
-  #   env:
-  #     PYTHON_VERSION: ${{ matrix.python_version }}
-  #     AIRFLOW_VERSION: ${{ matrix.airflow_version }}
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run whirl CI ${{ matrix.example_dir }}
-  #       working-directory: ${{ matrix.example_dir }}
-  #       run: |
-  #         echo Run Ci from example directory ${{ matrix.example_dir }}
-  #         ../../whirl ci
+  whirl-ci-default-envs:
+    needs: [directories]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        example_dir: ${{ fromJson(needs.directories.outputs.dir) }}
+        python_version: ["3.8", "3.9"]
+        airflow_version: ["2.2.5", "2.5.0"]
+        exclude:
+          # Needs more memory than available on the runner
+          - example_dir: ./examples/dbt-spark-example
+          - example_dir: ./examples/spark-delta-sharing
+          - example_dir: ./examples/spark-s3-to-hive
+          # Exclude failing dbt runs
+          - example_dir: ./examples/dbt-example
+    env:
+      PYTHON_VERSION: ${{ matrix.python_version }}
+      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run whirl CI ${{ matrix.example_dir }}
+        working-directory: ${{ matrix.example_dir }}
+        run: |
+          echo Run Ci from example directory ${{ matrix.example_dir }}
+          ../../whirl ci
 
   whirl-ci-default-envs-from-root-dir:
     needs: [examples]
@@ -76,7 +76,7 @@ jobs:
       matrix:
         example: ${{ fromJson(needs.examples.outputs.example) }}
         python_version: ["3.8", "3.9"]
-        airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
+        airflow_version: ["2.2.5", "2.5.0"]
         exclude:
           # Needs more memory than available on the runner
           - example: dbt-spark-example
@@ -95,50 +95,50 @@ jobs:
           ./whirl -x ${{ matrix.example }} ci
 
 
-  # whirl-ci-extra-env-spark-s3-to-postgres:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run whirl CI non-default env
-  #       working-directory: ./examples/spark-s3-to-postgres/
-  #       run: |
-  #         echo Run Ci for example spark-s3-to-postgres with non-default env
-  #         ../../whirl ci -e postgres-s3-spark
+  whirl-ci-extra-env-spark-s3-to-postgres:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run whirl CI non-default env
+        working-directory: ./examples/spark-s3-to-postgres/
+        run: |
+          echo Run Ci for example spark-s3-to-postgres with non-default env
+          ../../whirl ci -e postgres-s3-spark
 
-  # whirl-ci-extra-env-api-to-s3:
-  #   strategy:
-  #     fail-fast: false
-  #     max-parallel: 4
-  #     matrix:
-  #       python_version: ["3.8", "3.9"]
-  #       airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     PYTHON_VERSION: ${{ matrix.python_version }}
-  #     AIRFLOW_VERSION: ${{ matrix.airflow_version }}
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run whirl CI api-to-s3 on k8s executor
-  #       working-directory: ./examples/api-to-s3/
-  #       run: |
-  #         echo Run Ci for example api-to-s3 with k8s executor env
-  #         ../../whirl ci -e api-python-s3-k8s
+  whirl-ci-extra-env-api-to-s3:
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python_version: ["3.8", "3.9"]
+        airflow_version: ["2.2.5", "2.5.0"]
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ matrix.python_version }}
+      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run whirl CI api-to-s3 on k8s executor
+        working-directory: ./examples/api-to-s3/
+        run: |
+          echo Run Ci for example api-to-s3 with k8s executor env
+          ../../whirl ci -e api-python-s3-k8s
 
-  # whirl-ci-extra-env-ha-scheduler:
-  #   strategy:
-  #     fail-fast: false
-  #     max-parallel: 4
-  #     matrix:
-  #       python_version: ["3.8", "3.9"]
-  #       airflow_version: ["2.2.5", "2.3.4", "2.4.3", "2.5.0"]
-  #   runs-on: ubuntu-latest
-  #   env:
-  #     PYTHON_VERSION: ${{ matrix.python_version }}
-  #     AIRFLOW_VERSION: ${{ matrix.airflow_version }}
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #     - name: Run whirl CI ha-scheduler env
-  #       working-directory: ./examples/external-airflow-db/
-  #       run: |
-  #         echo Run Ci for example external-airflow-db with ha-scheduler env
-  #         ../../whirl ci -e ha-scheduler
+  whirl-ci-extra-env-ha-scheduler:
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python_version: ["3.8", "3.9"]
+        airflow_version: ["2.2.5", "2.5.0"]
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: ${{ matrix.python_version }}
+      AIRFLOW_VERSION: ${{ matrix.airflow_version }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run whirl CI ha-scheduler env
+        working-directory: ./examples/external-airflow-db/
+        run: |
+          echo Run Ci for example external-airflow-db with ha-scheduler env
+          ../../whirl ci -e ha-scheduler

From fb21964e0323efd562ac3cb1f177264f66e5e6a5 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 16 Dec 2022 14:20:02 +0100
Subject: [PATCH 05/23] Fix ci errors due to bumped airflow version

---
 envs/dbt-example/.whirl.env                      | 2 ++
 envs/postgres-s3-external-spark/.whirl.env       | 2 ++
 envs/postgres-s3-spark/.whirl.env                | 2 ++
 envs/s3-external-spark-hive/.whirl.env           | 2 ++
 envs/s3-spark-delta-sharing-minio/.whirl.env     | 2 ++
 envs/s3-spark-delta-sharing-riverbank/.whirl.env | 2 ++
 envs/s3-spark-delta-sharing/.whirl.env           | 2 ++
 envs/sftp-mysql-example/.whirl.env               | 2 +-
 examples/dbt-example/.whirl.env                  | 2 +-
 examples/dbt-spark-example/.whirl.env            | 2 +-
 examples/spark-delta-sharing/.whirl.env          | 2 +-
 examples/spark-s3-to-hive/.whirl.env             | 3 +--
 examples/spark-s3-to-postgres/.whirl.env         | 3 ++-
 13 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/envs/dbt-example/.whirl.env b/envs/dbt-example/.whirl.env
index d6f0e46..bb73a6f 100644
--- a/envs/dbt-example/.whirl.env
+++ b/envs/dbt-example/.whirl.env
@@ -22,3 +22,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/postgres-s3-external-spark/.whirl.env b/envs/postgres-s3-external-spark/.whirl.env
index 244ab75..9111fad 100644
--- a/envs/postgres-s3-external-spark/.whirl.env
+++ b/envs/postgres-s3-external-spark/.whirl.env
@@ -20,3 +20,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/postgres-s3-spark/.whirl.env b/envs/postgres-s3-spark/.whirl.env
index 244ab75..9111fad 100644
--- a/envs/postgres-s3-spark/.whirl.env
+++ b/envs/postgres-s3-spark/.whirl.env
@@ -20,3 +20,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/s3-external-spark-hive/.whirl.env b/envs/s3-external-spark-hive/.whirl.env
index e2196be..e82f248 100644
--- a/envs/s3-external-spark-hive/.whirl.env
+++ b/envs/s3-external-spark-hive/.whirl.env
@@ -15,3 +15,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/s3-spark-delta-sharing-minio/.whirl.env b/envs/s3-spark-delta-sharing-minio/.whirl.env
index 4e396a8..0128d7f 100644
--- a/envs/s3-spark-delta-sharing-minio/.whirl.env
+++ b/envs/s3-spark-delta-sharing-minio/.whirl.env
@@ -14,3 +14,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/s3-spark-delta-sharing-riverbank/.whirl.env b/envs/s3-spark-delta-sharing-riverbank/.whirl.env
index 5393700..20be87b 100644
--- a/envs/s3-spark-delta-sharing-riverbank/.whirl.env
+++ b/envs/s3-spark-delta-sharing-riverbank/.whirl.env
@@ -21,3 +21,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/s3-spark-delta-sharing/.whirl.env b/envs/s3-spark-delta-sharing/.whirl.env
index 1f67c40..5fcedbd 100644
--- a/envs/s3-spark-delta-sharing/.whirl.env
+++ b/envs/s3-spark-delta-sharing/.whirl.env
@@ -14,3 +14,5 @@ AIRFLOW__CORE__EXPOSE_CONFIG=True
 AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
 AIRFLOW__CORE__LOAD_EXAMPLES=False
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+
+MINIMAL_AIRFLOW_VERSION=2.3.0
\ No newline at end of file
diff --git a/envs/sftp-mysql-example/.whirl.env b/envs/sftp-mysql-example/.whirl.env
index 89b99c0..13175af 100644
--- a/envs/sftp-mysql-example/.whirl.env
+++ b/envs/sftp-mysql-example/.whirl.env
@@ -1,4 +1,4 @@
-MOCK_DATA_FOLDER=$(pwd)/mock-data
+MOCK_DATA_FOLDER=${DAG_FOLDER}/mock-data
 
 
 # Airflow variables
diff --git a/examples/dbt-example/.whirl.env b/examples/dbt-example/.whirl.env
index d9cbc60..591dd77 100644
--- a/examples/dbt-example/.whirl.env
+++ b/examples/dbt-example/.whirl.env
@@ -1,2 +1,2 @@
 WHIRL_ENVIRONMENT=dbt-example
-MOCK_DATA_FOLDER=$(pwd)/mock-data
+MOCK_DATA_FOLDER=${DAG_FOLDER}/mock-data
diff --git a/examples/dbt-spark-example/.whirl.env b/examples/dbt-spark-example/.whirl.env
index 9c28928..20064c5 100644
--- a/examples/dbt-spark-example/.whirl.env
+++ b/examples/dbt-spark-example/.whirl.env
@@ -1,2 +1,2 @@
 WHIRL_ENVIRONMENT=s3-external-spark-hive
-MOCK_DATA_FOLDER=$(pwd)/mock-data
+MOCK_DATA_FOLDER=${DAG_FOLDER}/mock-data
diff --git a/examples/spark-delta-sharing/.whirl.env b/examples/spark-delta-sharing/.whirl.env
index 1aa43ae..46d5cad 100644
--- a/examples/spark-delta-sharing/.whirl.env
+++ b/examples/spark-delta-sharing/.whirl.env
@@ -1,3 +1,3 @@
 WHIRL_ENVIRONMENT=s3-spark-delta-sharing
-MOCK_DATA_FOLDER=$(pwd)/mock-data
+MOCK_DATA_FOLDER=${DAG_FOLDER}/mock-data
 
diff --git a/examples/spark-s3-to-hive/.whirl.env b/examples/spark-s3-to-hive/.whirl.env
index 1de1646..20064c5 100644
--- a/examples/spark-s3-to-hive/.whirl.env
+++ b/examples/spark-s3-to-hive/.whirl.env
@@ -1,3 +1,2 @@
 WHIRL_ENVIRONMENT=s3-external-spark-hive
-MOCK_DATA_FOLDER=$(pwd)/mock-data
-
+MOCK_DATA_FOLDER=${DAG_FOLDER}/mock-data
diff --git a/examples/spark-s3-to-postgres/.whirl.env b/examples/spark-s3-to-postgres/.whirl.env
index f04b1a6..610f0e5 100644
--- a/examples/spark-s3-to-postgres/.whirl.env
+++ b/examples/spark-s3-to-postgres/.whirl.env
@@ -1,2 +1,3 @@
 WHIRL_ENVIRONMENT=postgres-s3-external-spark
-MOCK_DATA_FOLDER=$(pwd)/mock-data
+MOCK_DATA_FOLDER=${DAG_FOLDER}/mock-data
+MINIMAL_AIRFLOW_VERSION=2.3.0

From 406837b79a470d03f9a643d7b3f025543ada0d9d Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 11:38:02 +0100
Subject: [PATCH 06/23] Add export-ignore

---
 .gitattributes | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitattributes b/.gitattributes
index 1921125..a3acbb5 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,4 +1,9 @@
 *        text=auto
 *.sh     text eol=lf
 *.bat    text eol=crlf
-*.cmd    text eol=crlf
\ No newline at end of file
+*.cmd    text eol=crlf
+
+.gitattributes	export-ignore
+.gitignore		export-ignore
+.github			export-ignore
+logo	 		export-ignore
\ No newline at end of file

From 651940c13bbacc825e8612323cefc0110ddbe619 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 11:40:05 +0100
Subject: [PATCH 07/23] Add export-ignore

---
 .gitattributes | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitattributes b/.gitattributes
index a3acbb5..40ad1d0 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -3,7 +3,8 @@
 *.bat    text eol=crlf
 *.cmd    text eol=crlf
 
-.gitattributes	export-ignore
-.gitignore		export-ignore
-.github			export-ignore
-logo	 		export-ignore
\ No newline at end of file
+.gitattributes  export-ignore
+.gitignore      export-ignore
+.github         export-ignore
+.gitkeep        export-ignore
+logo            export-ignore

From 4e2f7ddca40897f18ec546b62927b6a26db3e35b Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 11:41:42 +0100
Subject: [PATCH 08/23] Add export-ignore

---
 .gitattributes | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitattributes b/.gitattributes
index 40ad1d0..0423074 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -8,3 +8,6 @@
 .github         export-ignore
 .gitkeep        export-ignore
 logo            export-ignore
+
+#Non finished examples
+examples/dbt**  export-ignore

From 5b15f479bb588827f6a9363628896de2698a1253 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 11:42:52 +0100
Subject: [PATCH 09/23] Add export-ignore

---
 .gitattributes | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitattributes b/.gitattributes
index 0423074..18684aa 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -9,5 +9,6 @@
 .gitkeep        export-ignore
 logo            export-ignore
 
-#Non finished examples
+#Non finished examples and their envs
 examples/dbt**  export-ignore
+envs/dbt**      export-ignore

From 8057c6ac7b5383f4439df9415254412301cf017b Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 11:44:01 +0100
Subject: [PATCH 10/23] Add export-ignore

---
 .gitattributes | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitattributes b/.gitattributes
index 18684aa..551897c 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -7,6 +7,7 @@
 .gitignore      export-ignore
 .github         export-ignore
 .gitkeep        export-ignore
+.editorconfig   export-ignore
 logo            export-ignore
 
 #Non finished examples and their envs

From a3307eeda0c71b94a0bc88fa4ec8e1320998df68 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 14:18:06 +0100
Subject: [PATCH 11/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 .github/workflows/whirl-release.yml

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
new file mode 100644
index 0000000..70ce840
--- /dev/null
+++ b/.github/workflows/whirl-release.yml
@@ -0,0 +1,22 @@
+name: Publish minimal release archive
+on:
+  release:
+    types: [created]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run git-archive command to create a release artifact
+        uses: actions/checkout@v3
+        run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tgz HEAD
+
+      - name: upload linux artifact
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: ./whirl-release.tgz
+          asset_name: Whirl release
+          asset_content_type: application/gzip
\ No newline at end of file

From 486b8b02d18b463dafb636890a135ead13fc0a9c Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 14:21:23 +0100
Subject: [PATCH 12/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 70ce840..3f3b18a 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -7,8 +7,9 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v2
+
       - name: Run git-archive command to create a release artifact
-        uses: actions/checkout@v3
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tgz HEAD
 
       - name: upload linux artifact

From 9fd440befa7fbe2881b23216e6892648fbc6de16 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 14:32:04 +0100
Subject: [PATCH 13/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 3f3b18a..91a7e1e 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -12,6 +12,9 @@ jobs:
       - name: Run git-archive command to create a release artifact
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tgz HEAD
 
+      - name: Test existance
+        run: ls -la
+
       - name: upload linux artifact
         uses: actions/upload-release-asset@v1
         env:

From b837049ce3f7aef9db6ad4ec10cbc59a9e2a5648 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 14:47:56 +0100
Subject: [PATCH 14/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 91a7e1e..5e63dc1 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -1,6 +1,8 @@
 name: Publish minimal release archive
 on:
   release:
+    permissions:
+      contents: write  # for actions/upload-release-asset to upload release asset
     types: [created]
 
 jobs:

From 2f16f05005e1c9597e6d65bceed3491da8d8cf3a Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 15:06:06 +0100
Subject: [PATCH 15/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 7 ++-----
 README.md                           | 8 +++++++-
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 5e63dc1..cb77cd8 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -14,15 +14,12 @@ jobs:
       - name: Run git-archive command to create a release artifact
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tgz HEAD
 
-      - name: Test existance
-        run: ls -la
-
       - name: upload linux artifact
         uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ github.token }}
         with:
           upload_url: ${{ github.event.release.upload_url }}
-          asset_path: ./whirl-release.tgz
-          asset_name: Whirl release
+          asset_path: whirl-release.tgz
+          asset_name: Whirl-release
           asset_content_type: application/gzip
\ No newline at end of file
diff --git a/README.md b/README.md
index 9940294..291fee5 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ NOTE: _whirl_ is not intended to replace proper (unit) testing of the logic you
 
 _whirl_ relies on [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/). Make sure you have it installed. If using _Docker for Mac_ or _Windows_ ensure that you have configured it with sufficient RAM (8GB or more recommended) for running all your containers.
 
-When you want to use _whirl_ in your CI pipeline (currently work in progress), you need to have `jq` installed. For example, with Homebrew:
+When you want to use _whirl_ in your CI pipeline, you need to have `jq` installed. For example, with Homebrew:
 
 ```bash
 brew install jq
@@ -28,6 +28,8 @@ As of January 2021, Whirl uses Airflow 2.x.x as the default version. A specific
 
 ## Getting Started
 
+### Development
+
 Clone this repository:
 
 ```
@@ -38,6 +40,10 @@ For ease of use you can add the base directory to your `PATH` environment variab
 export PATH=<target directory of whirl>:${PATH}
 ```
 
+### Use the release
+
+Download the release artifact
+
 ## Usage
 
 The `whirl` script is used to perform all actions.

From 24b8b7521fa7e6add7878d8af188ea68869f551a Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 15:09:29 +0100
Subject: [PATCH 16/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index cb77cd8..5b9c97c 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -3,7 +3,7 @@ on:
   release:
     permissions:
       contents: write  # for actions/upload-release-asset to upload release asset
-    types: [created]
+    types: [published]
 
 jobs:
   deploy:

From 3371b998aa9abd7a500ca0b516434f99d69e1e3b Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 15:24:09 +0100
Subject: [PATCH 17/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 5b9c97c..4afe030 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -15,11 +15,12 @@ jobs:
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tgz HEAD
 
       - name: upload linux artifact
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ github.event.release.upload_url }}
-          asset_path: whirl-release.tgz
-          asset_name: Whirl-release
-          asset_content_type: application/gzip
\ No newline at end of file
+        # uses: actions/upload-release-asset@v1
+        # env:
+        #   GITHUB_TOKEN: ${{ github.token }}
+        # with:
+        #   upload_url: ${{ github.event.release.upload_url }}
+        #   asset_path: whirl-release.tgz
+        #   asset_name: Whirl-release
+        #   asset_content_type: application/gzip
+        run: gh release upload ${{ github.event.release.tag_name }} whirl-release.tgz --clobber
\ No newline at end of file

From cc3574c65fa40e3210d0c7742d4565c65ffc0c3b Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Thu, 22 Dec 2022 15:30:51 +0100
Subject: [PATCH 18/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 4afe030..7353a41 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -16,8 +16,8 @@ jobs:
 
       - name: upload linux artifact
         # uses: actions/upload-release-asset@v1
-        # env:
-        #   GITHUB_TOKEN: ${{ github.token }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
         # with:
         #   upload_url: ${{ github.event.release.upload_url }}
         #   asset_path: whirl-release.tgz

From 72ccd1c5c4d031eb2adc00f31687a360ffec6c0d Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 23 Dec 2022 09:57:37 +0100
Subject: [PATCH 19/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 7353a41..e5b74e1 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Run git-archive command to create a release artifact
-        run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tgz HEAD
+        run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tar.gz HEAD
 
       - name: upload linux artifact
         # uses: actions/upload-release-asset@v1
@@ -23,4 +23,8 @@ jobs:
         #   asset_path: whirl-release.tgz
         #   asset_name: Whirl-release
         #   asset_content_type: application/gzip
-        run: gh release upload ${{ github.event.release.tag_name }} whirl-release.tgz --clobber
\ No newline at end of file
+        run: |
+          ls -la
+          gh auth status
+          gh release view ${{ github.event.release.tag_name }}
+          gh release upload ${{ github.event.release.tag_name }} whirl-release.tar.gz --clobber
\ No newline at end of file

From ab1d91c160cb939693cfb7c78567dc6b62e2ad86 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 23 Dec 2022 10:04:57 +0100
Subject: [PATCH 20/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index e5b74e1..c9d0144 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -1,13 +1,13 @@
 name: Publish minimal release archive
 on:
   release:
-    permissions:
-      contents: write  # for actions/upload-release-asset to upload release asset
     types: [published]
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    permissions: write-all
+      # contents: write  # for actions/upload-release-asset to upload release asset
     steps:
       - uses: actions/checkout@v2
 

From b4437d1f797392dd94fb43b9b2db019654a95792 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 23 Dec 2022 10:09:20 +0100
Subject: [PATCH 21/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index c9d0144..8f43675 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -6,8 +6,8 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
-    permissions: write-all
-      # contents: write  # for actions/upload-release-asset to upload release asset
+    permissions:
+      contents: write  # for actions/upload-release-asset to upload release asset
     steps:
       - uses: actions/checkout@v2
 
@@ -15,16 +15,16 @@ jobs:
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tar.gz HEAD
 
       - name: upload linux artifact
-        # uses: actions/upload-release-asset@v1
+        uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        # with:
-        #   upload_url: ${{ github.event.release.upload_url }}
-        #   asset_path: whirl-release.tgz
-        #   asset_name: Whirl-release
-        #   asset_content_type: application/gzip
-        run: |
-          ls -la
-          gh auth status
-          gh release view ${{ github.event.release.tag_name }}
-          gh release upload ${{ github.event.release.tag_name }} whirl-release.tar.gz --clobber
\ No newline at end of file
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: whirl-release.tar.gz
+          asset_name: Whirl-release
+          asset_content_type: application/gzip
+        # run: |
+        #   ls -la
+        #   gh auth status
+        #   gh release view ${{ github.event.release.tag_name }}
+        #   gh release upload ${{ github.event.release.tag_name }} whirl-release.tar.gz --clobber
\ No newline at end of file

From ebe2e0a83f918192cca215b2e848de7366ef2ec1 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 23 Dec 2022 10:25:49 +0100
Subject: [PATCH 22/23] Add release workflow

---
 .github/workflows/whirl-release.yml | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 8f43675..71bf5cb 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -15,16 +15,7 @@ jobs:
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tar.gz HEAD
 
       - name: upload linux artifact
-        uses: actions/upload-release-asset@v1
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ github.event.release.upload_url }}
-          asset_path: whirl-release.tar.gz
-          asset_name: Whirl-release
-          asset_content_type: application/gzip
-        # run: |
-        #   ls -la
-        #   gh auth status
-        #   gh release view ${{ github.event.release.tag_name }}
-        #   gh release upload ${{ github.event.release.tag_name }} whirl-release.tar.gz --clobber
\ No newline at end of file
+        run: |
+          gh release upload ${{ github.event.release.tag_name }} whirl-release.tar.gz --clobber
\ No newline at end of file

From 2315b3835451fa019b8ef9001f7861bf4b735952 Mon Sep 17 00:00:00 2001
From: Kris Geusebroek <krisgeus@gmail.com>
Date: Fri, 23 Dec 2022 10:58:06 +0100
Subject: [PATCH 23/23] Add release workflow

---
 .github/workflows/whirl-release.yml |  4 ++--
 README.md                           | 32 +++++++++++++++++++++++------
 whirl                               |  1 +
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/whirl-release.yml b/.github/workflows/whirl-release.yml
index 71bf5cb..f8d5acc 100644
--- a/.github/workflows/whirl-release.yml
+++ b/.github/workflows/whirl-release.yml
@@ -7,14 +7,14 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     permissions:
-      contents: write  # for actions/upload-release-asset to upload release asset
+      contents: write  # for upload release asset
     steps:
       - uses: actions/checkout@v2
 
       - name: Run git-archive command to create a release artifact
         run: git archive --format=tar.gz --prefix=whirl/ --output=whirl-release.tar.gz HEAD
 
-      - name: upload linux artifact
+      - name: upload Whirl release artifact
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: |
diff --git a/README.md b/README.md
index 291fee5..c9ee228 100644
--- a/README.md
+++ b/README.md
@@ -42,7 +42,18 @@ export PATH=<target directory of whirl>:${PATH}
 
 ### Use the release
 
-Download the release artifact
+Download the [latest Whirl release artifact](https://github.com/godatadriven/whirl/releases/download/latest/whirl-release.tar.gz)
+
+Extract the file (for example into `/usr/local/opt`)
+
+```bash
+tar -xvzf whirl-release.tar.gz -C /usr/local/opt
+```
+
+Make sure the whirl script is available on your path
+```bash
+export PATH=/usr/local/opt/whirl:$PATH
+```
 
 ## Usage
 
@@ -57,10 +68,14 @@ $ whirl --help
 
 #### Starting whirl
 
-The default action is to start the DAG in your current directory. It expects an environment to be configured. You can pass this as a command line argument or you can configure it in a `.whirl.env` file. (See [Configuring environment variables](#configuring-environment-variables).) The environment refers to a directory with the same name in the `envs` directory located near the _whirl_ script.
+The default action is to start the DAG in your current directory.
+
+With the `[-x example]` commandline argument you can run whirl from anywhere and tell whirl which example dag to run. The example refers to a directory with the same name in the `examples` directory located near the _whirl_ script.
+
+Whirl expects an environment to be configured. You can pass this as a command line argument `[-e environment]` or you can configure it as environment variable `WHIRL_ENVIRONMENT` in a `.whirl.env` file. (See [Configuring environment variables](#configuring-environment-variables).) The environment refers to a directory with the same name in the `envs` directory located near the _whirl_ script.
 
 ```bash
-$ whirl [start] [-d <directory>] [-e <environment>]
+$ whirl [-x example] [-e <environment>] [start]
 ```
 
 Specifying the `start` command line argument is a more explicit way to start _whirl_.
@@ -68,7 +83,7 @@ Specifying the `start` command line argument is a more explicit way to start _wh
 #### Stopping whirl
 
 ```bash
-$ whirl stop [-d <directory>] [-e <environment>]
+$ whirl  [-x example] [-e <environment>] stop
 ```
 Stops the configured environment.
 
@@ -140,8 +155,6 @@ Each example contains it's own README file to explain the specifics of that exam
 
 #### Generic running of examples
 
-From within the example directory the `whirl` command can be executed. 
-
 To run a example:
 
 ```bash
@@ -150,6 +163,13 @@ $ cd ./examples/<example-dag-directory>
 $ whirl -e <environment to use>
 ```
 
+or
+```bash
+$
+# Note: here we pass the whirl environment as a command-line argument. It can also be configured with the WHIRL_ENVIRONMENT variable
+$ whirl -x <example to run> -e <environment to use>
+```
+
 Open your browser to [http://localhost:5000](http://localhost:5000) to access the Airflow UI. Manually enable the DAG and watch the pipeline run to successful completion.
 
 
diff --git a/whirl b/whirl
index 9d364f4..327863f 100755
--- a/whirl
+++ b/whirl
@@ -347,6 +347,7 @@ logs() {
 usage() {
   echo "usage: ${BASH_SOURCE[0]} [-h|--help] [-e|--environment env] [start|stop|ci]"
   echo "  -h|--help                          display usage"
+  echo "  -x|--example example               specify example to run"
   echo "  -e|--environment environment       specify environment to use"
   echo "  -d|--directory environment_folder  specify the folder that contains the environments (defaults to SCRIPT_DIR)"
   echo "  -l|--logs servicename              tail the logs of the service"