azure-pipelines.yml

# Grab variables from the specific variable group and
# determine sourceBranchName (avoids SourchBranchName=merge
# for PR)
variables:
  - group: 'Nutter Testing'
  - name: 'branchName'
    ${{ if startsWith(variables['Build.SourceBranch'], 'refs/heads/') }}:
      value: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ] 
    ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
      value: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]

trigger:
  batch: true
  branches:
    include:
    - '*'
  paths:
    exclude:
      - README.md
      - LICENSE
      - images
      - terraform
      - .github

#  tags:
#    include:
#      - v*.*
#      - prod

# This need an additional debugging
# pr:
#   branches:
#     include:
#       - master
#       - releases
#   paths:
#     exclude:
#       - README.md
#       - images
      
stages:
- stage: onPush
  condition: |
    and(
      ne(variables['Build.SourceBranch'], 'refs/heads/releases'),
      not(startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    )
  jobs:
  - job: onPushJob
    pool:
      vmImage: 'ubuntu-20.04'

    steps:
    - script: env | sort
      displayName: 'Environment / Context'

    - task: UsePythonVersion@0
      displayName: 'Use Python 3.8'
      inputs:
        versionSpec: 3.8

    - checkout: self
      displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'

    - script: |
        python -m pip install pip nutter
        # this is because of the old dependency inside Nutter
        python -m pip install --upgrade databricks-cli
      displayName: 'Install dependencies'

# https://docs.databricks.com/dev-tools/api/latest/repos.html
# this is simplification, and won't work with concurrent commits. Ideally it should be a
# separate repo for each commit
    - script: |
        echo "Checking out the $(branchName) branch"
        databricks repos update --path $(STAGING_DIRECTORY) --branch "$(branchName)"
      env:
        DATABRICKS_HOST: $(DATABRICKS_HOST)
        DATABRICKS_TOKEN: $(DATABRICKS_TOKEN)
      displayName: 'Update Staging project'

    - script: |
        nutter run "$(STAGING_DIRECTORY)/unit-tests/" --cluster_id $(CLUSTER_ID) --recursive --junit_report --timeout 500
      env:
        DATABRICKS_HOST: $(DATABRICKS_HOST)
        DATABRICKS_TOKEN: $(DATABRICKS_TOKEN)
      displayName: 'Execute Nutter tests'

    - task: PublishTestResults@2
      condition: succeededOrFailed()
      inputs:
        testResultsFormat: 'JUnit'
        testResultsFiles: '**/test-*.xml' 
        failTaskOnFailedTests: true

- stage: onRelease
  condition: |
    eq(variables['Build.SourceBranch'], 'refs/heads/releases')
  jobs:
  - job: onReleaseJob
    pool:
      vmImage: 'ubuntu-20.04'

    steps:
      - script: env | sort
        displayName: 'Environment / Context'

      - task: UsePythonVersion@0
        displayName: 'Use Python 3.8'
        inputs:
          versionSpec: 3.8

      - checkout: self
        persistCredentials: true
        clean: true
        displayName: 'Checkout & Build.Reason: $(Build.Reason) & Build.SourceBranchName: $(Build.SourceBranchName)'

      - script: |
          python -m pip install --upgrade pip nutter
          # this is because of the old dependency inside Nutter
          python -m pip install --upgrade databricks-cli
        displayName: 'Install dependencies'

      - script: |
          echo "Checking out the releases branch"
          databricks repos update --path $(STAGING_DIRECTORY) --branch "$(Build.SourceBranchName)"
        env:
          DATABRICKS_HOST: $(DATABRICKS_HOST)
          DATABRICKS_TOKEN: $(DATABRICKS_TOKEN)
        displayName: 'Update Staging repository'

# We can do a separate set of the tests for release branches
      - script: |
          nutter run "$(STAGING_DIRECTORY)/unit-tests/" --cluster_id $(CLUSTER_ID) --recursive --junit_report --timeout 500
        env:
          DATABRICKS_HOST: $(DATABRICKS_HOST)
          DATABRICKS_TOKEN: $(DATABRICKS_TOKEN)
        displayName: 'Execute Nutter tests on release'

      - task: PublishTestResults@2
        condition: succeededOrFailed()
        inputs:
          testResultsFormat: 'JUnit'
          testResultsFiles: '**/test-*.xml' 
          failTaskOnFailedTests: true