Merge pull request #654 from NVIDIA/branch-24.04

release 24.04 [skip ci]

.github/workflows/auto-merge.yml

@@ -18,7 +18,7 @@ name: auto-merge HEAD to BASE
 on:
   pull_request_target:
     branches:
-    - branch-24.02
+    - branch-24.04
     types: [closed]
 
 jobs:
@@ -27,16 +27,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
-          ref: branch-24.02 # force to fetch from latest upstream instead of PR ref
+          ref: branch-24.04 # force to fetch from latest upstream instead of PR ref
 
       - name: auto-merge job
         uses: ./.github/workflows/auto-merge
         env:
           OWNER: NVIDIA
           REPO_NAME: spark-rapids-ml
-          HEAD: branch-24.02
-          BASE: branch-24.04
+          HEAD: branch-24.04
+          BASE: branch-24.06
           AUTOMERGE_TOKEN: ${{ secrets.AUTOMERGE_TOKEN }} # use to merge PR
 

.github/workflows/blossom-ci.yml

@@ -44,8 +44,6 @@ jobs:
       GaryShen2008,\
       NvTimLiu,\
       YanxuanLiu,\
-      zhanga5,\
-      Er1cCheng,\
       ', format('{0},', github.actor)) && github.event.comment.body == 'build'
     steps:
       - name: Check if comment is issued by authorized person

.github/workflows/gcs-benchmark.yml

@@ -39,7 +39,7 @@ jobs:
       SERVICE_ACCOUNT: ${{ secrets.GCLOUD_SERVICE_ACCOUNT }}
       CLUSTER_NAME: github-spark-rapids-ml-${{github.run_number}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: run benchmark
         shell: bash

.github/workflows/signoff-check.yml

@@ -23,7 +23,7 @@ jobs:
   signoff-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: sigoff-check job
         uses: ./.github/workflows/signoff-check

README.md

@@ -35,16 +35,17 @@ The following table shows the currently supported algorithms.  The goal is to ex
 | Supported Algorithms   | Python | Scala |
 | :--------------------- | :----: | :---: |
 | CrossValidator         |   √    |       |
+| DBSCAN (*)             |   √    |       |
 | KMeans                 |   √    |       |
-| k-NN (*)               |   √    |       |
+| approx/exact k-NN (*)  |   √    |       |
 | LinearRegression       |   √    |       |
 | LogisticRegression     |   √    |       | 
 | PCA                    |   √    |   √   |
 | RandomForestClassifier |   √    |       |
 | RandomForestRegressor  |   √    |       |
 | UMAP (*)               |   √    |       |
 
-Note: Spark does not provide a k-Nearest Neighbors (k-NN) implementation, but it does have an [LSH-based Approximate Nearest Neighbor](https://spark.apache.org/docs/latest/ml-features.html#approximate-nearest-neighbor-search) implementation.   As an alternative to PCA, we also provide a Spark API for GPU accelerated Uniform Manifold Approximation and Projection (UMAP), a non-linear dimensionality reduction algorithm in the RAPIDS cuML library.
+Note: Spark does not provide a k-Nearest Neighbors (k-NN) implementation, but it does have an [LSH-based Approximate Nearest Neighbor](https://spark.apache.org/docs/latest/ml-features.html#approximate-nearest-neighbor-search) implementation.   As an alternative to PCA, we also provide a Spark API for GPU accelerated Uniform Manifold Approximation and Projection (UMAP), a non-linear dimensionality reduction algorithm in the RAPIDS cuML library.   As an alternative to KMeans, we also provide a Spark API for GPU accelerated Density-Based Spatial Clustering of Applications with Noise (DBSCAN), a density based clustering algorithm in the RAPIDS cuML library.
 
 ## Getting started
 

ci/Dockerfile

@@ -37,6 +37,6 @@ RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86
     && conda config --set solver libmamba
 
 # install cuML
-ARG CUML_VER=24.02
+ARG CUML_VER=24.04
 RUN conda install -y -c rapidsai -c conda-forge -c nvidia cuml=$CUML_VER python=3.9 cuda-version=11.8 \
     && conda clean --all -f -y

ci/Jenkinsfile.premerge

@@ -1,6 +1,6 @@
 #!/usr/local/env groovy
 /*
- * Copyright (c) 2023, NVIDIA CORPORATION.
+ * Copyright (c) 2023-2024, NVIDIA CORPORATION.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ pipeline {
     agent {
         kubernetes {
             label "premerge-init-${BUILD_TAG}"
-            cloud 'sc-ipp-blossom-prod'
+            cloud "${common.CLOUD_NAME}"
             yaml cpuImage
         }
     }
@@ -87,7 +87,7 @@ pipeline {
 
                     def title = githubHelper.getIssue().title
                     if (title ==~ /.*\[skip ci\].*/) {
-                        githubHelper.updateCommitStatus("$BUILD_URL", "Skipped", GitHubCommitState.SUCCESS)
+                        githubHelper.updateCommitStatus("", "Skipped", GitHubCommitState.SUCCESS)
                         currentBuild.result == "SUCCESS"
                         skipped = true
                         return
@@ -107,7 +107,7 @@ pipeline {
             agent {
                 kubernetes {
                     label "premerge-docker-${BUILD_TAG}"
-                    cloud 'sc-ipp-blossom-prod'
+                    cloud "${common.CLOUD_NAME}"
                     yaml pod.getDockerBuildYAML()
                     workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
                     customWorkspace "${CUSTOM_WORKSPACE}"
@@ -116,7 +116,7 @@ pipeline {
 
             steps {
                 script {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Running - preparing", GitHubCommitState.PENDING)
+                    githubHelper.updateCommitStatus("", "Running - preparing", GitHubCommitState.PENDING)
                     checkout(
                         changelog: false,
                         poll: true,
@@ -169,7 +169,7 @@ pipeline {
             agent {
                 kubernetes {
                     label "premerge-ci-${BUILD_TAG}"
-                    cloud 'sc-ipp-blossom-prod'
+                    cloud "${common.CLOUD_NAME}"
                     yaml pod.getGPUYAML("${IMAGE_PREMERGE}", "${env.GPU_RESOURCE}", '8', '32Gi')
                     workspaceVolume persistentVolumeClaimWorkspaceVolume(claimName: "${PVC}", readOnly: false)
                     customWorkspace "${CUSTOM_WORKSPACE}"
@@ -178,7 +178,7 @@ pipeline {
 
             steps {
                 script {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Running - tests", GitHubCommitState.PENDING)
+                    githubHelper.updateCommitStatus("", "Running - tests", GitHubCommitState.PENDING)
                     container('gpu') {
                         timeout(time: 2, unit: 'HOURS') { // step only timeout for test run
                             common.resolveIncompatibleDriverIssue(this)
@@ -198,14 +198,15 @@ pipeline {
                 }
 
                 if (currentBuild.currentResult == "SUCCESS") {
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Success", GitHubCommitState.SUCCESS)
+                    githubHelper.updateCommitStatus("", "Success", GitHubCommitState.SUCCESS)
                 } else {
                     // upload log only in case of build failure
-                    def guardWords = ["gitlab.*?\\.com", "urm.*?\\.com"]
+                    def guardWords = ["gitlab.*?\\.com", "urm.*?\\.com", "sc-ipp-*"]
                     guardWords.add("nvidia-smi(?s)(.*?)(?=git)") // hide GPU info
+                    guardWords.add("sc-ipp*") // hide cloud info
                     githubHelper.uploadLogs(this, env.JOB_NAME, env.BUILD_NUMBER, null, guardWords)
 
-                    githubHelper.updateCommitStatus("$BUILD_URL", "Fail", GitHubCommitState.FAILURE)
+                    githubHelper.updateCommitStatus("", "Fail", GitHubCommitState.FAILURE)
                 }
 
                 if (TEMP_IMAGE_BUILD) {

docker/Dockerfile.pip

@@ -18,7 +18,7 @@ ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04
 
 ARG PYSPARK_VERSION=3.3.1
-ARG RAPIDS_VERSION=24.2.0
+ARG RAPIDS_VERSION=24.4.0
 ARG ARCH=amd64
 #ARG ARCH=arm64
 # Install packages to build spark-rapids-ml

docker/Dockerfile.python

@@ -17,7 +17,7 @@
 ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
 
-ARG CUML_VERSION=24.02
+ARG CUML_VERSION=24.04
 
 # Install packages to build spark-rapids-ml
 RUN apt update -y \

docs/site/compatibility.md

@@ -11,16 +11,17 @@ The following table shows the currently supported algorithms.  The goal is to ex
 | Supported Algorithms   | Python | Scala |
 | :--------------------- | :----: | :---: |
 | CrossValidator         |   √    |       |
+| DBSCAN (*)             |   √    |       |
 | KMeans                 |   √    |       |
-| k-NN (*)               |   √    |       |
+| approx/exact k-NN (*)  |   √    |       |
 | LinearRegression       |   √    |       |
 | LogisticRegression     |   √    |       | 
 | PCA                    |   √    |   √   |
 | RandomForestClassifier |   √    |       |
 | RandomForestRegressor  |   √    |       |
 | UMAP (*)               |   √    |       |
 
-Note: Spark does not provide a k-Nearest Neighbors (k-NN) implementation, but it does have an [LSH-based Approximate Nearest Neighbor](https://spark.apache.org/docs/latest/ml-features.html#approximate-nearest-neighbor-search) implementation.   As an alternative to PCA, we also provide a Spark API for GPU accelerated Uniform Manifold Approximation and Projection (UMAP), a non-linear dimensionality reduction algorithm in the RAPIDS cuML library.
+Note: Spark does not provide a k-Nearest Neighbors (k-NN) implementation, but it does have an [LSH-based Approximate Nearest Neighbor](https://spark.apache.org/docs/latest/ml-features.html#approximate-nearest-neighbor-search) implementation.   As an alternative to PCA, we also provide a Spark API for GPU accelerated Uniform Manifold Approximation and Projection (UMAP), a non-linear dimensionality reduction algorithm in the RAPIDS cuML library.   As an alternative to KMeans, we also provide a Spark API for GPU accelerated Density-Based Spatial Clustering of Applications with Noise (DBSCAN), a density based clustering algorithm in the RAPIDS cuML library.
 
 
 ## Supported Versions

docs/source/conf.py

@@ -9,7 +9,7 @@
 project = 'spark-rapids-ml'
 copyright = '2024, NVIDIA'
 author = 'NVIDIA'
-release = '24.02.0'
+release = '24.04.0'
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

docs/source/spark_rapids_ml.rst

@@ -57,6 +57,8 @@ Clustering
     :template: autosummary/class_with_docs.rst
     :toctree: api
 
+    DBSCAN
+    DBSCANModel
     KMeans
     KMeansModel
 
@@ -85,6 +87,8 @@ Nearest Neighbors
     :template: autosummary/class_with_docs.rst
     :toctree: api
 
+    ApproximateNearestNeighbors
+    ApproximateNearestNeighborsModel
     NearestNeighbors
     NearestNeighborsModel
 
@@ -111,4 +115,4 @@ UMAP
     :toctree: api
 
     UMAP
-    UMAPModel
+    UMAPModel