diff --git a/.github/workflows/notebook-test-suite.yml b/.github/workflows/notebook-test-suite.yml new file mode 100644 index 0000000..02e494f --- /dev/null +++ b/.github/workflows/notebook-test-suite.yml @@ -0,0 +1,177 @@ +# This workflow runs Jupyter notebook based regression test suites. +# It can be run manually run on github Actions or invoked through +# the github Action workflow api. +# When invoked through the github Action workflow api, a event type needs +# to be provided. The event type should be either the name of one of the +# Harmony services or 'all'. +# The notebook test suites associated with Harmony service matching the event +# will be executed, or all the notebook test suites will be exectued when 'all' +# is provided which is triggered when a Harmonny server deployment occurs. +# +# Required settings in the repository: +# Environments (exactly the following strings): +# - prod +# - uat +# +# Secrets in each environment: +# - EDL_USER +# - EDL_PASSWORD +# +# Note: A workflow_call event can only have inputs of type boolean, number or +# string. +name: Run Jupyter notebook based test suite +run-name: ${{ github.event.action || inputs.service-name }} service regression tests + +on: + workflow_call: + inputs: + harmony-environment: + required: true + type: string + docker-image-tag: + required: false + type: string + default: 'latest' + + workflow_dispatch: + inputs: + harmony-environment: + description: "Select the Harmony environment to run the test in" + required: true + type: choice + options: + - uat + - production + service-name: + description: "Select the service to run tests against" + required: true + type: choice + options: + - geoloco + - harmony + - harmony-gdal-adapter + # - harmony-netcdf-to-zarr # Commented out because needs AWS credentials + - harmony-regression + - harmony-regridder + - harmony-service-example + - hoss + - hybig + - query-cmr + - subset-band-name + - swath-projector + - trajectory-subsetter + docker-image-tag: + description: "Optional Docker image tag to use for the tests" + required: false + type: string + default: 'latest' + + repository_dispatch: + types: [all,geoloco,harmony-gdal-adapter,harmony-netcdf-to-zarr,harmony-service-example,hoss,hybig,net2cog,swath-projector,trajectory-subsetter] + inputs: + harmony-environment: + required: true + type: choice + options: + - uat + - prod + docker-image-tag: + required: false + type: string + default: 'latest' + +jobs: + base: + environment: ${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }} + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.create-matrix.outputs.matrix }} + harmony_host_url: ${{ env.HARMONY_HOST_URL }} + run_url: ${{ env.RUN_URL }} + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Get the URL of the current workflow run + run: echo "RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV + + - name: Set HARMONY_HOST_URL + run: | + if [ "${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}" == "uat" ]; then + echo "HARMONY_HOST_URL=https://harmony.uat.earthdata.nasa.gov" + echo "HARMONY_HOST_URL=https://harmony.uat.earthdata.nasa.gov" >> $GITHUB_ENV + elif [ "${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}" == "prod" ]; then + echo "HARMONY_HOST_URL=https://harmony.earthdata.nasa.gov" + echo "HARMONY_HOST_URL=https://harmony.earthdata.nasa.gov" >> $GITHUB_ENV + else + echo "HARMONY_HOST_URL=https://harmony.sit.earthdata.nasa.gov" + echo "HARMONY_HOST_URL=https://harmony.sit.earthdata.nasa.gov" >> $GITHUB_ENV + fi + + - name: Create matrix configuration from Environment Variable + id: create-matrix + run: | + # Use the event type to select the appropriate service list + SERVICES_TESTS=$(cat ./config/services_tests_config_${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}.json | jq -r '."${{ github.event.action || inputs.service-name }}"') + # Split the selected service list into an array format and set it as the matrix + MATRIX=$(echo ",$SERVICES_TESTS" | tr ',' '\n' | awk '{$1=$1;print}' | jq -c -R '[inputs] | {service: .}') + echo "matrix=$MATRIX" >> $GITHUB_OUTPUT + + test: + needs: base + environment: ${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }} + runs-on: ubuntu-latest + outputs: + run_url: ${{ needs.base.outputs.run_url }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.base.outputs.matrix) }} + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Pull Docker image + run: docker pull ghcr.io/nasa/regression-tests-${{ matrix.service }}:${{ github.event.client_payload.docker-image-tag || 'latest' }} + + - name: Execute notebook + id: test-step + working-directory: ./test + env: + EDL_USER: ${{ secrets.EDL_USER }} + EDL_PASSWORD: ${{ secrets.EDL_PASSWORD }} + HARMONY_HOST_URL: ${{ needs.base.outputs.harmony_host_url }} + run: ./run_notebooks.sh ${{ matrix.service }} + + - name: Save notebook as an artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: notebook-output-${{ matrix.service }} + path: test/output/${{ matrix.service }}/Results.ipynb + + email: + needs: test + if: always() + environment: ${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }} + runs-on: ubuntu-latest + steps: + - name: Get TO_EMAIL + run: | + SERVICE_EMAILS=$(echo '${{ vars.SERVICES_EMAILS_CONFIG }}' | jq -r '."${{ github.event.action || inputs.service-name }}"') + if [ "$SERVICE_EMAILS" == "null" ]; then + SERVICE_EMAILS=${{ vars.TO_EMAIL_DEFAULT }} + fi + echo "TO_EMAIL=$SERVICE_EMAILS" >> $GITHUB_ENV + + - name: Send mail + uses: dawidd6/action-send-mail@v3 + with: + server_address: smtp.gmail.com + server_port: 465 + username: ${{ secrets.SMTP_USER }} + password: ${{ secrets.SMTP_PASSWORD }} + subject: ${{ github.event.action || inputs.service-name }} regression test result is ${{ needs.test.result }} + to: ${{ env.TO_EMAIL }} + from: ${{ secrets.FROM_EMAIL }} + body: ${{ github.event.action || inputs.service-name }} regression test result is ${{ needs.test.result }}! View the details at ${{ needs.test.outputs.run_url }}. diff --git a/README.md b/README.md index 97f3728..a65b2f7 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,21 @@ installed on your system, as it's used to manage some large files stored in GitHub. +## Running the Tests in GitHub: + +Each test suite can be individually invoked via a GitHub workflow. Navigate to +the [GitHub Actions tab](https://github.com/nasa/harmony-regression-tests/actions) +for this repository. Then select the "Run test suite" workflow from the lefthand +menu. On the right hand side, click the "Run workflow" dropdown, and select the +correct Docker image and Harmony environment. That should manually trigger the +workflow. + +The regression test GitHub actions can also be invoked through different event types +after a Harmony service is successfully deployed in Harmony or after a new version of +the Harmony server is deployed. +Note: Only the `latest` tag of the regression docker image will be used to run the +Jupyter notebook tests. + ## Running the Tests Locally Each test suite is run in a separate Docker container using a temporary Docker image @@ -122,6 +137,9 @@ environment before installing from the environment.yml. all_images=(
 )
    ```
 1. Update `script/test-in-bamboo.sh` to list the new suite name in `all_tests`.
+1. Update `config/services_tests_config_.json` to associate the new suite name
+   with a Harmony service and add it to the `all` list so that it will be run when
+   the associated Harmony service or Harmony server is deployed.
 
 With this in place, the new test suite should be able to be built and run:
 
diff --git a/config/services_tests_config_prod.json b/config/services_tests_config_prod.json
new file mode 100644
index 0000000..e4afcbf
--- /dev/null
+++ b/config/services_tests_config_prod.json
@@ -0,0 +1,21 @@
+{
+    "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter",
+    "batchee": "TBD",
+    "geoloco": "geoloco",
+    "giovanni-adapter": "TBD",
+    "harmony-gdal-adapter": "hga",
+    "harmony-netcdf-to-zarr": "n2z",
+    "harmony-regridder": "regridder",
+    "harmony-service-example": "harmony-regression",
+    "hoss": "hoss",
+    "hybig": "hybig",
+    "net2cog": "net2cog",
+    "podaac-concise": "TBD",
+    "podaac-l2-subsetter": "TBD",
+    "query-cmr": "geoloco",
+    "sds-maskfill": "TBD",
+    "stitchee": "TBD",
+    "subset-band-name": "subset-band-name",
+    "swath-projector": "swath-projector",
+    "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter"
+}
diff --git a/config/services_tests_config_uat.json b/config/services_tests_config_uat.json
new file mode 100644
index 0000000..75692fd
--- /dev/null
+++ b/config/services_tests_config_uat.json
@@ -0,0 +1,21 @@
+{
+    "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter",
+    "batchee": "TBD",
+    "geoloco": "geoloco",
+    "giovanni-adapter": "TBD",
+    "harmony-gdal-adapter": "hga",
+    "harmony-netcdf-to-zarr": "n2z",
+    "harmony-regridder": "regridder",
+    "harmony-service-example": "harmony-regression",
+    "hoss": "hoss, variable-subsetter",
+    "hybig": "hybig",
+    "net2cog": "net2cog",
+    "podaac-concise": "TBD",
+    "podaac-l2-subsetter": "TBD",
+    "query-cmr": "variable-subsetter",
+    "sds-maskfill": "TBD",
+    "stitchee": "TBD",
+    "subset-band-name": "subset-band-name",
+    "swath-projector": "swath-projector",
+    "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter"
+}