From 2d2120399e8cb6aba0ad44dccdf8694a4ca7e56f Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 20 Jul 2023 15:11:56 +0200 Subject: [PATCH 01/56] fixes #63 - added some configs variables for stb and jacoco code coverage. --- build.sbt | 2 ++ project/Dependencies.scala | 4 +++- project/build.properties | 15 +++++++++++++++ project/plugins.sbt | 16 ++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index dc65d02cc..2fb2a0473 100644 --- a/build.sbt +++ b/build.sbt @@ -19,6 +19,8 @@ ThisBuild / organization := "za.co.absa" lazy val scala211 = "2.11.12" lazy val scala212 = "2.12.12" +lazy val spark2 = "2.4.7" +lazy val spark3 = "3.3.1" Test / parallelExecution := false diff --git a/project/Dependencies.scala b/project/Dependencies.scala index d5322d731..4174643b7 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -76,7 +76,9 @@ object Dependencies { Seq(sparkCore, sparkCommons, sparkCommonsTest, sparkSql, scalaTest, specs2core, typeSafeConfig) - }} + } + +} diff --git a/project/build.properties b/project/build.properties index 40b3b8e7b..7a0a1ed1d 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1,16 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + sbt.version=1.9.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index fc8188df7..e8e9c2e25 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -18,6 +18,22 @@ addSbtPlugin("com.earldouglas" % "xsbt-web-plugin" % "4.2.4") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") +// sbt-jacoco dependency downloading +lazy val ow2Version = "9.5" +lazy val jacocoVersion = "0.8.10-absa.1" +def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" +def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") +addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) +addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) +addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) +addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) +addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) +addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) + +addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.3" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.3/sbt-jacoco-3.4.1-absa.3.jar") +addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") From 14d646da1f860b62687b37e32f84c174d0d96a35 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jul 2023 08:13:54 +0200 Subject: [PATCH 02/56] fixes #63 - added more configs variables for stb --- build.sbt | 5 +++++ project/JacocoSetup.scala | 16 ++++++++++++++++ project/SparkVersionAxis.scala | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 project/JacocoSetup.scala create mode 100644 project/SparkVersionAxis.scala diff --git a/build.sbt b/build.sbt index 2fb2a0473..4c74fd1b0 100644 --- a/build.sbt +++ b/build.sbt @@ -13,8 +13,10 @@ * limitations under the License. */ +import sys.process._ import Dependencies._ +ThisBuild / name := "atum-service" ThisBuild / organization := "za.co.absa" lazy val scala211 = "2.11.12" @@ -22,6 +24,9 @@ lazy val scala212 = "2.12.12" lazy val spark2 = "2.4.7" lazy val spark3 = "3.3.1" +ThisBuild / crossScalaVersions := Seq(scala211, scala212) +ThisBuild / scalaVersion := scala211 + Test / parallelExecution := false Global / onChangedBuildSource := ReloadOnSourceChanges diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala new file mode 100644 index 000000000..c33cfced0 --- /dev/null +++ b/project/JacocoSetup.scala @@ -0,0 +1,16 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +object JacocoSetup diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala new file mode 100644 index 000000000..6db89d44e --- /dev/null +++ b/project/SparkVersionAxis.scala @@ -0,0 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +case class SparkVersionAxis(sparkVersion: String) + +case object SparkVersionAxis From 3eb3bc2f4c63d838b582ebec4dba48edad85086b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jul 2023 08:22:42 +0200 Subject: [PATCH 03/56] fixes #63 - added GitHub workflow Actions --- .github/CODEOWNERS | 1 + .github/workflows/build.yml | 43 +++++++++++++++++++++++++++++ .github/workflows/format_check.yml | 42 ++++++++++++++++++++++++++++ .github/workflows/licence_check.yml | 41 +++++++++++++++++++++++++++ .github/workflows/release.yml | 34 +++++++++++++++++++++++ 5 files changed, 161 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/format_check.yml create mode 100644 .github/workflows/licence_check.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..dc9d14583 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @Zejnilovic @dk1844 @benedeki @lsulak @Tebalelos diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..c6d504db6 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,43 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Build + +on: + push: + branches: [ main, develop, master ] + pull_request: + branches: [ master, develop ] + types: [ assigned, opened, synchronize, reopened, labeled ] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + scala: [2.11.12, 2.12.12] + name: Scala ${{matrix.scala}} + steps: + - name: Checkout code + uses: actions/checkout@v2 + - uses: coursier/cache-action@v5 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + - name: Build and run tests + run: sbt ++${{matrix.scala}} test doc diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml new file mode 100644 index 000000000..1f1375d6b --- /dev/null +++ b/.github/workflows/format_check.yml @@ -0,0 +1,42 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: FormatCheck + +on: + pull_request: + branches: [ master, develop ] + types: [ assigned, opened, synchronize, reopened, labeled ] + +jobs: + scalafmt: + runs-on: ubuntu-latest + name: Scalafmt Check + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + + - name: Run scalafmt And Print Diff + continue-on-error: true + run: sbt scalafmt scalafmtSbt && git diff --exit-code diff --git a/.github/workflows/licence_check.yml b/.github/workflows/licence_check.yml new file mode 100644 index 000000000..6d96c065d --- /dev/null +++ b/.github/workflows/licence_check.yml @@ -0,0 +1,41 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: License Check + +on: + push: + branches: [ main, develop, master ] + pull_request: + branches: [ master ] + types: [ assigned, opened, synchronize, reopened, labeled ] + +jobs: + license-test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + # note, that task "headerCheck" defaults to just "compile:headerCheck" - see https://github.com/sbt/sbt-header/issues/14 + - name: SBT src licence header check + run: sbt Compile/headerCheck + - name: SBT test licence header check + run: sbt Test/headerCheck + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..e15fbddb4 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,34 @@ +# +# Copyright 2022 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Release +on: + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.3.4 + with: + fetch-depth: 0 + - uses: olafurpg/setup-scala@v13 + - run: sbt ci-release + env: + PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} + PGP_SECRET: ${{ secrets.PGP_SECRET }} + SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} + SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} From 6ce13a788d8198aa3f7b2759eeb5051f06b4ee5a Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jul 2023 09:04:24 +0200 Subject: [PATCH 04/56] fixes #63 - added license --- .github/workflows/assign_project_to_issue.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/assign_project_to_issue.yml b/.github/workflows/assign_project_to_issue.yml index 023fc9ebc..a80ef642f 100644 --- a/.github/workflows/assign_project_to_issue.yml +++ b/.github/workflows/assign_project_to_issue.yml @@ -1,3 +1,18 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Auto Assign Issue to Project on: From 477668cd5e4a0fc59db94f83a2dd9ed522b4fdfc Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jul 2023 09:05:32 +0200 Subject: [PATCH 05/56] fixes #63 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6d504db6..40ccd2e45 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ name: Build on: push: - branches: [ main, develop, master ] + branches: [ develop, master ] pull_request: branches: [ master, develop ] types: [ assigned, opened, synchronize, reopened, labeled ] From b5eeb8132c56d597a6d7ba424fbf75457ba5b942 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 21 Jul 2023 12:39:17 +0200 Subject: [PATCH 06/56] fixes #63 --- build.sbt | 60 ++++++++++++++++++++++------------ project/Dependencies.scala | 42 ++++++++++++------------ project/SparkVersionAxis.scala | 39 ++++++++++++++++++++-- project/plugins.sbt | 1 + 4 files changed, 100 insertions(+), 42 deletions(-) diff --git a/build.sbt b/build.sbt index 4c74fd1b0..fade149b3 100644 --- a/build.sbt +++ b/build.sbt @@ -13,8 +13,8 @@ * limitations under the License. */ -import sys.process._ import Dependencies._ +import SparkVersionAxis._ ThisBuild / name := "atum-service" ThisBuild / organization := "za.co.absa" @@ -25,33 +25,53 @@ lazy val spark2 = "2.4.7" lazy val spark3 = "3.3.1" ThisBuild / crossScalaVersions := Seq(scala211, scala212) -ThisBuild / scalaVersion := scala211 - -Test / parallelExecution := false +ThisBuild / scalaVersion := scala212 Global / onChangedBuildSource := ReloadOnSourceChanges -lazy val root = (project in file(".")) +lazy val commonSettings = Seq( + libraryDependencies ++= commonDependencies, + scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), + javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"), + Test / parallelExecution := false +) + +lazy val parent = (project in file(".")) + .aggregate(atumServer.projectRefs ++ atumAgent.projectRefs: _*) + .settings( + name := "atum-service-parent", + publish / skip := true + ) + +lazy val atumAgent = (projectMatrix in file("agent")) .settings( - name := "atum-root", - javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint") + commonSettings ++ Seq( + name := "atum-agent", + libraryDependencies ++= Dependencies.agentDependencies(???), + scalafmtOnCompile := true + ) ) + .enablePlugins(ScalafmtPlugin) + .sparkRow(SparkVersionAxis(spark2), scalaVersions = Seq(scala211, scala212)) + .sparkRow(SparkVersionAxis(spark3), scalaVersions = Seq(scala212)) -lazy val server = project +lazy val atumServer = (projectMatrix in file("server")) .settings( - name := "atum-server", - scalaVersion := scala212, - libraryDependencies ++= Dependencies.serverDependencies, - webappWebInfClasses := true, - inheritJarManifest := true + commonSettings ++ Seq( + name := "atum-server", + libraryDependencies ++= Dependencies.serverDependencies, + webappWebInfClasses := true, + inheritJarManifest := true + ): _* ) .enablePlugins(TomcatPlugin) .enablePlugins(AutomateHeaderPlugin) + .jvmPlatform(scalaVersions = Seq(scala212)) -lazy val agent = project - .settings( - name := "atum-agent", - scalaVersion := scala212, - libraryDependencies ++= Dependencies.agentDependencies, - scalafmtOnCompile := true - ).enablePlugins(ScalafmtPlugin) +//lazy val agent = project +// .settings( +// name := "atum-agent", +// scalaVersion := scala212, +// libraryDependencies ++= Dependencies.agentDependencies, +// scalafmtOnCompile := true +// ).enablePlugins(ScalafmtPlugin) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 4174643b7..e8bbe6cb2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,8 +17,19 @@ import sbt._ object Dependencies { + private def getSparkVersionUpToMinor(sparkVersion: String): String = { + val pattern = "([0-9]+)\\.([0-9]+)\\.([0-9]+)".r + val pattern(major, minor, patch) = sparkVersion + s"$major.$minor" + } + + def commonDependencies: Seq[ModuleID] = Seq( + "org.scalatest" %% "scalatest" % "3.2.2" % Test, + "org.mockito" %% "mockito-scala" % "1.17.12" % Test + ) + + def serverDependencies: Seq[ModuleID] = { - val serverDependencies: Seq[ModuleID] = { val springVersion = "2.6.1" val springOrg = "org.springframework.boot" @@ -54,27 +65,18 @@ object Dependencies { } - val agentDependencies: Seq[ModuleID] = { - - val spark3Version = "3.3.2" - val scala212 = "2.12.12" - val scalatestVersion = "3.2.15" - val specs2Version = "4.19.2" - val typesafeConfigVersion = "1.4.2" - - lazy val sparkCore = "org.apache.spark" %% "spark-core" % spark3Version + def agentDependencies(sparkVersion: String): Seq[ModuleID] = { - lazy val sparkCommons = "za.co.absa" % "spark-commons-spark3.3_2.12" % "0.5.0" + val sparkMinorVersion = getSparkVersionUpToMinor(sparkVersion) - lazy val sparkCommonsTest = "za.co.absa" %% "spark-commons-test" % "0.5.0" % Test - - lazy val sparkSql ="org.apache.spark" %% "spark-sql" % spark3Version - lazy val scalaTest = "org.scalatest" %% "scalatest" % scalatestVersion % Test - lazy val specs2core = "org.specs2" %% "specs2-core" % specs2Version % Test - lazy val typeSafeConfig = "com.typesafe" % "config" % typesafeConfigVersion - - - Seq(sparkCore, sparkCommons, sparkCommonsTest, sparkSql, scalaTest, specs2core, typeSafeConfig) + Seq( + "org.apache.spark" %% "spark-core" % sparkVersion % Provided, + "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, + "com.typesafe" % "config" % "1.4.2", // to check if used + "za.co.absa" %% s"spark-commons-$sparkMinorVersion" % "0.6.0", + "za.co.absa" %% "spark-commons-test" % "0.6.0" % Test, + "org.specs2" %% "specs2-core" % "4.19.2" % Test, // to check + ) } diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 6db89d44e..24174854c 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -13,6 +13,41 @@ * limitations under the License. */ -case class SparkVersionAxis(sparkVersion: String) +//import Dependencies.sparkCommonsDependencies +import JacocoSetup.{jacocoProjectExcludes, jacocoSettings} +import sbt.Keys.{libraryDependencies, moduleName, name} +import sbt.{Def, VirtualAxis} +import sbt.internal.ProjectMatrix -case object SparkVersionAxis +case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { + val sparkVersionMinor: String = sparkVersion.split("\\.", 3).take(2).mkString(".") + override val directorySuffix = s"-spark${sparkVersionMinor}" + override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") +} + +object SparkVersionAxis { + private def camelCaseToLowerDashCase(origName: String): String = { + origName + .replaceAll("([A-Z])", "-$1") + .toLowerCase() + } + + implicit class ProjectExtension(val projectMatrix: ProjectMatrix) extends AnyVal { + + def sparkRow(sparkAxis: SparkVersionAxis, scalaVersions: Seq[String], settings: Def.SettingsDefinition*): ProjectMatrix = { + val sparkVersion = sparkAxis.sparkVersion + scalaVersions.foldLeft(projectMatrix)((currentProjectMatrix, scalaVersion) => + currentProjectMatrix.customRow( + scalaVersions = Seq(scalaVersion), + axisValues = Seq(sparkAxis, VirtualAxis.jvm), + _.settings( + moduleName := camelCaseToLowerDashCase(name.value + sparkAxis.directorySuffix), + libraryDependencies ++= sparkCommonsDependencies(sparkAxis.sparkVersion), + jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion), + jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion) + ).settings(settings: _*) + ) + ) + } + } +} diff --git a/project/plugins.sbt b/project/plugins.sbt index e8e9c2e25..1c394ee87 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -17,6 +17,7 @@ addSbtPlugin("com.earldouglas" % "xsbt-web-plugin" % "4.2.4") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") +addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") // sbt-jacoco dependency downloading lazy val ow2Version = "9.5" From 5e515f4e77d5a66a2d3da5f073a1804e3486847d Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Fri, 21 Jul 2023 23:45:19 +0200 Subject: [PATCH 07/56] * Fixed `ProjectMatrix` --- build.sbt | 2 -- project/Dependencies.scala | 18 +++++++++-------- project/SparkVersionAxis.scala | 21 +++++++++++--------- project/plugins.sbt | 36 +++++++++++++++++----------------- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/build.sbt b/build.sbt index fade149b3..d4850daa7 100644 --- a/build.sbt +++ b/build.sbt @@ -16,7 +16,6 @@ import Dependencies._ import SparkVersionAxis._ -ThisBuild / name := "atum-service" ThisBuild / organization := "za.co.absa" lazy val scala211 = "2.11.12" @@ -47,7 +46,6 @@ lazy val atumAgent = (projectMatrix in file("agent")) .settings( commonSettings ++ Seq( name := "atum-agent", - libraryDependencies ++= Dependencies.agentDependencies(???), scalafmtOnCompile := true ) ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index e8bbe6cb2..b071079ac 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -64,18 +64,20 @@ object Dependencies { ) } - def agentDependencies(sparkVersion: String): Seq[ModuleID] = { - val sparkMinorVersion = getSparkVersionUpToMinor(sparkVersion) + val typesafeVersion = "1.4.2" + val sparkCommonsVersion = "0.6.6" + val sparkMinorVersion = getSparkVersionUpToMinor(sparkVersion) + val specs2CoreVersion = "4.19.2" Seq( - "org.apache.spark" %% "spark-core" % sparkVersion % Provided, - "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, - "com.typesafe" % "config" % "1.4.2", // to check if used - "za.co.absa" %% s"spark-commons-$sparkMinorVersion" % "0.6.0", - "za.co.absa" %% "spark-commons-test" % "0.6.0" % Test, - "org.specs2" %% "specs2-core" % "4.19.2" % Test, // to check + "org.apache.spark" %% "spark-core" % sparkVersion % Provided, + "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, + "com.typesafe" % "config" % typesafeVersion, // to check if used + "za.co.absa" %% s"spark-commons-$sparkMinorVersion" % sparkCommonsVersion, + "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test, + "org.specs2" %% "specs2-core" % specs2CoreVersion % Test, // to check ) } diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 24174854c..5a570ba72 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -13,15 +13,18 @@ * limitations under the License. */ -//import Dependencies.sparkCommonsDependencies -import JacocoSetup.{jacocoProjectExcludes, jacocoSettings} -import sbt.Keys.{libraryDependencies, moduleName, name} -import sbt.{Def, VirtualAxis} +import sbt._ +import sbt.Keys._ import sbt.internal.ProjectMatrix +import Dependencies._ +//import JacocoSetup._ +//import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} + case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { - val sparkVersionMinor: String = sparkVersion.split("\\.", 3).take(2).mkString(".") - override val directorySuffix = s"-spark${sparkVersionMinor}" + + val sparkVersionMajor: String = sparkVersion.split("\\.", 2).take(1).mkString(".") + override val directorySuffix = s"-spark${sparkVersionMajor}" override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") } @@ -42,9 +45,9 @@ object SparkVersionAxis { axisValues = Seq(sparkAxis, VirtualAxis.jvm), _.settings( moduleName := camelCaseToLowerDashCase(name.value + sparkAxis.directorySuffix), - libraryDependencies ++= sparkCommonsDependencies(sparkAxis.sparkVersion), - jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion), - jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion) + libraryDependencies ++= agentDependencies(sparkAxis.sparkVersion), +// jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion), +// jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion) ).settings(settings: _*) ) ) diff --git a/project/plugins.sbt b/project/plugins.sbt index 1c394ee87..c23efee09 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -20,21 +20,21 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") // sbt-jacoco dependency downloading -lazy val ow2Version = "9.5" -lazy val jacocoVersion = "0.8.10-absa.1" - -def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" -def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" - -addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") -addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") - -addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) -addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) -addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) -addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) -addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) -addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) - -addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.3" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.3/sbt-jacoco-3.4.1-absa.3.jar") -addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") +//lazy val ow2Version = "9.5" +//lazy val jacocoVersion = "0.8.10-absa.1" +// +//def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" +//def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" +// +//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") +//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") +// +//addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) +//addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) +//addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) +//addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) +//addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) +//addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) +// +//addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.3" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.3/sbt-jacoco-3.4.1-absa.3.jar") +//addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") From baf028e4c7c54433630f505501ec7b5fa7ed6ce8 Mon Sep 17 00:00:00 2001 From: David Benedeki Date: Tue, 25 Jul 2023 12:54:21 +0200 Subject: [PATCH 08/56] * fixed bugs preventing project load and compilation --- project/Dependencies.scala | 29 +++++++++++-------- project/SparkVersionAxis.scala | 2 +- .../za/co/absa/atum/web/api/package.scala | 2 +- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index b071079ac..83ffe5086 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -17,10 +17,16 @@ import sbt._ object Dependencies { - private def getSparkVersionUpToMinor(sparkVersion: String): String = { - val pattern = "([0-9]+)\\.([0-9]+)\\.([0-9]+)".r - val pattern(major, minor, patch) = sparkVersion - s"$major.$minor" + private def limitVersion(version: String, parts: Int): String = { + version.split("\\.", parts + 1).take(parts).mkString(".") + } + + def getVersionUpToMinor(version: String): String = { + limitVersion(version, 2) + } + + def getVersionUpToMajor(version: String): String = { + limitVersion(version, 1) } def commonDependencies: Seq[ModuleID] = Seq( @@ -67,17 +73,16 @@ object Dependencies { def agentDependencies(sparkVersion: String): Seq[ModuleID] = { val typesafeVersion = "1.4.2" - val sparkCommonsVersion = "0.6.6" - val sparkMinorVersion = getSparkVersionUpToMinor(sparkVersion) + val sparkCommonsVersion = "0.6.0" + val sparkMinorVersion = getVersionUpToMinor(sparkVersion) val specs2CoreVersion = "4.19.2" Seq( - "org.apache.spark" %% "spark-core" % sparkVersion % Provided, - "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, - "com.typesafe" % "config" % typesafeVersion, // to check if used - "za.co.absa" %% s"spark-commons-$sparkMinorVersion" % sparkCommonsVersion, - "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test, - "org.specs2" %% "specs2-core" % specs2CoreVersion % Test, // to check + "org.apache.spark" %% "spark-core" % sparkVersion % Provided, + "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, + "com.typesafe" % "config" % typesafeVersion, //TODO to check if used + "za.co.absa" %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion, + "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test, ) } diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 5a570ba72..7bebe8c83 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -23,7 +23,7 @@ import Dependencies._ case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { - val sparkVersionMajor: String = sparkVersion.split("\\.", 2).take(1).mkString(".") + val sparkVersionMajor: String = Dependencies.getVersionUpToMajor(sparkVersion) override val directorySuffix = s"-spark${sparkVersionMajor}" override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") } diff --git a/server/src/main/scala/za/co/absa/atum/web/api/package.scala b/server/src/main/scala/za/co/absa/atum/web/api/package.scala index 3d198a87e..56c2934e8 100644 --- a/server/src/main/scala/za/co/absa/atum/web/api/package.scala +++ b/server/src/main/scala/za/co/absa/atum/web/api/package.scala @@ -19,8 +19,8 @@ package za.co.absa.atum.web import java.util.Optional import java.util.concurrent.CompletableFuture import scala.concurrent.Future - import scala.compat.java8.FutureConverters._ +import scala.language.implicitConversions package object api { From 4986ee3557a834a0fb2e4faf08827c743b716dba Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 25 Jul 2023 15:21:59 +0200 Subject: [PATCH 09/56] Fixes #63 - Emplementing jacoco setup --- project/JacocoSetup.scala | 17 +++++++++++++++- project/SparkVersionAxis.scala | 1 + project/plugins.sbt | 36 +++++++++++++++++----------------- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index c33cfced0..3cc8fea56 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -1,3 +1,4 @@ + /* * Copyright 2021 ABSA Group Limited * @@ -13,4 +14,18 @@ * limitations under the License. */ -object JacocoSetup + + +import com.github.sbt.jacoco.JacocoKeys.JacocoReportFormats +import com.github.sbt.jacoco.report.JacocoReportSettings + +import java.time.format.DateTimeFormatter +import java.time.{ZoneId, ZonedDateTime} + +object JacocoSetup { + + private val jacocoReportCommonSettings: JacocoReportSettings = JacocoReportSettings( + formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) + ) + +} diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 7bebe8c83..7876c9ef7 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -54,3 +54,4 @@ object SparkVersionAxis { } } } + diff --git a/project/plugins.sbt b/project/plugins.sbt index c23efee09..1c394ee87 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -20,21 +20,21 @@ addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") // sbt-jacoco dependency downloading -//lazy val ow2Version = "9.5" -//lazy val jacocoVersion = "0.8.10-absa.1" -// -//def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" -//def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" -// -//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") -//addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") -// -//addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) -//addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) -//addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) -//addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) -//addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) -//addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) -// -//addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.3" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.3/sbt-jacoco-3.4.1-absa.3.jar") -//addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") +lazy val ow2Version = "9.5" +lazy val jacocoVersion = "0.8.10-absa.1" + +def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" +def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" + +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.11/2.0/scala-arm_2.11-2.0.jar") +addSbtPlugin("com.jsuereth" %% "scala-arm" % "2.0" from "https://repo1.maven.org/maven2/com/jsuereth/scala-arm_2.12/2.0/scala-arm_2.12-2.0.jar") + +addSbtPlugin("za.co.absa.jacoco" % "report" % jacocoVersion from jacocoUrl("report")) +addSbtPlugin("za.co.absa.jacoco" % "core" % jacocoVersion from jacocoUrl("core")) +addSbtPlugin("za.co.absa.jacoco" % "agent" % jacocoVersion from jacocoUrl("agent")) +addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) +addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) +addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) + +addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.3" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.3/sbt-jacoco-3.4.1-absa.3.jar") +addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") From 5b45f6407657ddd81c3111389897413710a43850 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jul 2023 09:27:21 +0200 Subject: [PATCH 10/56] Fixes #63 - implementing jacoco setup --- build.sbt | 8 +------- project/JacocoSetup.scala | 16 +++++++++++++--- project/SparkVersionAxis.scala | 9 ++++----- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/build.sbt b/build.sbt index d4850daa7..f11717a0d 100644 --- a/build.sbt +++ b/build.sbt @@ -66,10 +66,4 @@ lazy val atumServer = (projectMatrix in file("server")) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(scala212)) -//lazy val agent = project -// .settings( -// name := "atum-agent", -// scalaVersion := scala212, -// libraryDependencies ++= Dependencies.agentDependencies, -// scalafmtOnCompile := true -// ).enablePlugins(ScalafmtPlugin) + diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 3cc8fea56..0807d0be4 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -1,4 +1,3 @@ - /* * Copyright 2021 ABSA Group Limited * @@ -14,8 +13,6 @@ * limitations under the License. */ - - import com.github.sbt.jacoco.JacocoKeys.JacocoReportFormats import com.github.sbt.jacoco.report.JacocoReportSettings @@ -28,4 +25,17 @@ object JacocoSetup { formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) ) + def jacocoSettings(sparkVersion: String, scalaVersion: String): JacocoReportSettings = { + val utcDateTime = ZonedDateTime.now.withZoneSameInstant(ZoneId.of("UTC")) + val now = s"as of ${DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm Z z").format(utcDateTime)}" + jacocoReportCommonSettings.withTitle(s"Jacoco Report on `spark-commons` for spark:$sparkVersion - scala:$scalaVersion [$now]") + } + + def jacocoProjectExcludes(sparkVersion: String, scalaVersion: String): Seq[String] = { + Seq( + "za.co.absa.atum.service.adapters.CallUdfAdapter", + "za.co.absa.atum.service.adapters.TransformAdapter" + ) + } + } diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 7876c9ef7..7f03e6543 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -17,9 +17,8 @@ import sbt._ import sbt.Keys._ import sbt.internal.ProjectMatrix import Dependencies._ -//import JacocoSetup._ -//import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} - +import JacocoSetup._ +import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { @@ -46,8 +45,8 @@ object SparkVersionAxis { _.settings( moduleName := camelCaseToLowerDashCase(name.value + sparkAxis.directorySuffix), libraryDependencies ++= agentDependencies(sparkAxis.sparkVersion), -// jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion), -// jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion) + jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion), + jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion) ).settings(settings: _*) ) ) From 51b9e1cc9bd1bbbd3e2816ca346abf9dca1c2e9a Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jul 2023 09:28:59 +0200 Subject: [PATCH 11/56] Fixes #63 - implementing jacoco setup --- project/SparkVersionAxis.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 7f03e6543..bccc5ae4e 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -23,7 +23,7 @@ import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { val sparkVersionMajor: String = Dependencies.getVersionUpToMajor(sparkVersion) - override val directorySuffix = s"-spark${sparkVersionMajor}" + override val directorySuffix = s"-spark$sparkVersionMajor" override val idSuffix: String = directorySuffix.replaceAll("""\W+""", "_") } From 15e9cdad4a815205358bcb70737236b20457a58f Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jul 2023 10:10:09 +0200 Subject: [PATCH 12/56] Fixes #63 - implementing jacoco setup --- project/JacocoSetup.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index 0807d0be4..b69e41acd 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -28,7 +28,7 @@ object JacocoSetup { def jacocoSettings(sparkVersion: String, scalaVersion: String): JacocoReportSettings = { val utcDateTime = ZonedDateTime.now.withZoneSameInstant(ZoneId.of("UTC")) val now = s"as of ${DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm Z z").format(utcDateTime)}" - jacocoReportCommonSettings.withTitle(s"Jacoco Report on `spark-commons` for spark:$sparkVersion - scala:$scalaVersion [$now]") + jacocoReportCommonSettings.withTitle(s"Jacoco Report on `atum-service` for spark:$sparkVersion - scala:$scalaVersion [$now]") } def jacocoProjectExcludes(sparkVersion: String, scalaVersion: String): Seq[String] = { From cad07a87b3ab2077b975587af215613591c3a55d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jul 2023 10:30:03 +0200 Subject: [PATCH 13/56] Fixes #63 - Fixing license --- version.sbt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/version.sbt b/version.sbt index a3a28d6cf..f65af45fa 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1,17 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + ThisBuild / version := "0.0.1-SNAPSHOT" From 6f280296a09a935662fc7eb0d5d3364405df12b5 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jul 2023 11:16:01 +0200 Subject: [PATCH 14/56] Fixes #63 - Fixing license --- .../scala/za/co/absa/atum/agent/AtumAgent.scala | 15 +++++++++++++++ .../scala/za/co/absa/atum/agent/AtumContext.scala | 15 +++++++++++++++ .../atum/agent/core/MeasurementProcessor.scala | 15 +++++++++++++++ .../co/absa/atum/agent/model/MeasureResult.scala | 15 +++++++++++++++ .../za/co/absa/atum/agent/model/Measurement.scala | 15 +++++++++++++++ .../za/co/absa/atum/agent/AtumContextSpec.scala | 15 +++++++++++++++ 6 files changed, 90 insertions(+) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index c63db3f63..c0da4e6af 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import za.co.absa.atum.agent.model.MeasureResult diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index bf39f85f8..c44c8d831 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala index e0a864734..5e685a4e7 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.core import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index d4154df7a..f8c09a7d0 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model case class MeasureResult(measurement: Measurement, result: String) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala index 27250ab79..536ebc387 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model import org.apache.spark.sql.functions._ diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala index 50644babc..4c31ec4ea 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import org.scalatest.flatspec.AnyFlatSpec From f857c6cc682bf4d6dc871c1acd2bf46fa5e6cc16 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 26 Jul 2023 12:15:54 +0200 Subject: [PATCH 15/56] Fixes #63 - removed license_check and CODEOWNERS files --- .github/CODEOWNERS | 1 - .github/workflows/licence_check.yml | 41 ------------------- .../atum/agent/model/MeasurementSpec.scala | 15 +++++++ 3 files changed, 15 insertions(+), 42 deletions(-) delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/workflows/licence_check.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index dc9d14583..000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -* @Zejnilovic @dk1844 @benedeki @lsulak @Tebalelos diff --git a/.github/workflows/licence_check.yml b/.github/workflows/licence_check.yml deleted file mode 100644 index 6d96c065d..000000000 --- a/.github/workflows/licence_check.yml +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2021 ABSA Group Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: License Check - -on: - push: - branches: [ main, develop, master ] - pull_request: - branches: [ master ] - types: [ assigned, opened, synchronize, reopened, labeled ] - -jobs: - license-test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Scala - uses: olafurpg/setup-scala@v10 - with: - java-version: "adopt@1.8" - # note, that task "headerCheck" defaults to just "compile:headerCheck" - see https://github.com/sbt/sbt-header/issues/14 - - name: SBT src licence header check - run: sbt Compile/headerCheck - - name: SBT test licence header check - run: sbt Test/headerCheck - diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala index b10057c91..b861397db 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala @@ -1,3 +1,18 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model import org.apache.spark.sql.{SQLContext, SQLImplicits, SparkSession} From 4fc19add191444e3340d5f681fee83294eb8cf9f Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Fri, 28 Jul 2023 15:27:43 +0200 Subject: [PATCH 16/56] Update project/plugins.sbt Co-authored-by: miroslavpojer --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 1c394ee87..4265a004e 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -21,7 +21,7 @@ addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") // sbt-jacoco dependency downloading lazy val ow2Version = "9.5" -lazy val jacocoVersion = "0.8.10-absa.1" +lazy val jacocoVersion = "0.8.11-absa.1" def jacocoUrl(artifactName: String): String = s"https://github.com/AbsaOSS/jacoco/releases/download/$jacocoVersion/org.jacoco.$artifactName-$jacocoVersion.jar" def ow2Url(artifactName: String): String = s"https://repo1.maven.org/maven2/org/ow2/asm/$artifactName/$ow2Version/$artifactName-$ow2Version.jar" From 061306261bd908e3aa822e1087988be11a84b7c3 Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Fri, 28 Jul 2023 15:27:54 +0200 Subject: [PATCH 17/56] Update project/plugins.sbt Co-authored-by: miroslavpojer --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 4265a004e..cb126e5dd 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -36,5 +36,5 @@ addSbtPlugin("org.ow2.asm" % "asm" % ow2Version from ow2Url("asm")) addSbtPlugin("org.ow2.asm" % "asm-commons" % ow2Version from ow2Url("asm-commons")) addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) -addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.3" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.3/sbt-jacoco-3.4.1-absa.3.jar") +addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") From 7d8e344ed0cedf8b717c2bd378fd420e6f6c8db4 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Sun, 30 Jul 2023 20:00:02 +0200 Subject: [PATCH 18/56] Fixes #63 - Added jacoco workflow --- .github/workflows/build.yml | 12 ++--- .github/workflows/jacoco_check.yml | 75 ++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/jacoco_check.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 40ccd2e45..ff833ea67 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,20 +17,16 @@ name: Build on: - push: - branches: [ develop, master ] pull_request: - branches: [ master, develop ] - types: [ assigned, opened, synchronize, reopened, labeled ] + branches: [ master ] + types: [ opened, synchronize, reopened ] jobs: test: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - scala: [2.11.12, 2.12.12] - name: Scala ${{matrix.scala}} + steps: - name: Checkout code uses: actions/checkout@v2 @@ -40,4 +36,4 @@ jobs: with: java-version: "adopt@1.8" - name: Build and run tests - run: sbt ++${{matrix.scala}} test doc + run: sbt test doc diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml new file mode 100644 index 000000000..e97f694ec --- /dev/null +++ b/.github/workflows/jacoco_check.yml @@ -0,0 +1,75 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: JaCoCo report + +on: + pull_request: + branches: [ master ] + types: [ opened, synchronize, reopened ] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + - name: Build and run tests + run: sbt jacoco + - name: Add coverage to PR for scala 2.11 and spark 2.4 + id: jacoco211 + uses: madrapps/jacoco-report@v1.3 + with: + paths: ${{ github.workspace }}/atum-service/target/spark2.4-jvm-2.11/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: 0.0 + min-coverage-changed-files: 80.0 + title: JaCoCo code coverage report - scala:2.11 - spark:2.4 + update-comment: true + - name: Get the Coverage info + run: | + echo "Total coverage ${{ steps.jacoco211.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco211.outputs.coverage-changed-files }}" + - name: Add coverage to PR for scala 2.12 and spark 3.3 + id: jacoco212 + uses: madrapps/jacoco-report@v1.3 + with: + paths: ${{ github.workspace }}/atum-service/target/spark3.3-jvm-2.12/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: 0.0 + min-coverage-changed-files: 80.0 + title: JaCoCo code coverage report - scala:2.12 - spark:3.3 + update-comment: true + - name: Get the Coverage info + run: | + echo "Total coverage ${{ steps.jacoco212.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco212.outputs.coverage-changed-files }}" + - name: Fail PR if changed files coverage is less than 80% + if: ${{ steps.jacoco211.outputs.coverage-changed-files < 80.0 }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('Changed files coverage is less than 80%!') + - name: Fail PR if changed files coverage is less than 80% + if: ${{ steps.jacoco212.outputs.coverage-changed-files < 80.0 }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('Changed files coverage is less than 80%!') From 74f762f49c6a3953a7e6a5dd8f85efc662bd6766 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Sun, 30 Jul 2023 20:42:04 +0200 Subject: [PATCH 19/56] Fixes #63 - Added dependent_items.yml --- .github/workflows/dependent_items.yml | 59 +++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/dependent_items.yml diff --git a/.github/workflows/dependent_items.yml b/.github/workflows/dependent_items.yml new file mode 100644 index 000000000..fd0349ceb --- /dev/null +++ b/.github/workflows/dependent_items.yml @@ -0,0 +1,59 @@ +name: Dependent Issues + +on: + issues: + types: + - opened + - edited + - closed + - reopened + pull_request: + types: + - opened + - edited + - closed + - reopened + # Makes sure we always add status check for PRs. Useful only if + # this action is required to pass before merging. Otherwise, it + # can be removed. + - synchronize + + # Schedule a daily check. Useful if you reference cross-repository + # issues or pull requests. Otherwise, it can be removed. + schedule: + - cron: '0 0 * * *' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: z0al/dependent-issues@v1.5.2 + env: + # (Required) The token to use to make API calls to GitHub. + GITHUB_TOKEN: ${{ secrets.PAT_REPO_PROJECT_DISCUSS }} + # (Optional) The token to use to make API calls to GitHub for remote repos. + GITHUB_READ_TOKEN: ${{ secrets.PAT_REPO_PROJECT_DISCUSS }} + + with: + # (Optional) The label to use to mark dependent issues + label: dependent + + # (Optional) Enable checking for dependencies in issues. + # Enable by setting the value to "on". Default "off" + check_issues: on + + # (Optional) Ignore dependabot PRs. + # Enable by setting the value to "on". Default "off" + ignore_dependabot: on + + # (Optional) A comma-separated list of keywords. Default + # "depends on, blocked by" + # keywords: depends on, blocked by + + # (Optional) A custom comment body. It supports `{{ dependencies }}` token. + comment: > + This item depends on: + + {{ dependencies }} + + By **[Dependent Issues](https://github.com/z0al/dependent-issues)**. From 4f963b94b80095c52123768bce4ad053d6d890e3 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 31 Jul 2023 11:10:29 +0200 Subject: [PATCH 20/56] Fixes #63 - Added dependent_items.yml --- project/Dependencies.scala | 2 +- project/plugins.sbt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 83ffe5086..9d737bf64 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -80,7 +80,7 @@ object Dependencies { Seq( "org.apache.spark" %% "spark-core" % sparkVersion % Provided, "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, - "com.typesafe" % "config" % typesafeVersion, //TODO to check if used + "com.typesafe" % "config" % typesafeVersion, //TODO - check if used "za.co.absa" %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion, "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test, ) diff --git a/project/plugins.sbt b/project/plugins.sbt index cb126e5dd..69dbdac16 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -38,3 +38,6 @@ addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") + +// To add release plugin + From 618ee833fb39f390eeb213e00deda7e7d20b10a7 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 31 Jul 2023 15:39:49 +0200 Subject: [PATCH 21/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- RELEASE.md | 23 +++++++++++++++++++++++ project/plugins.sbt | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 RELEASE.md diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index e97f694ec..cef60dbd0 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -51,7 +51,7 @@ jobs: id: jacoco212 uses: madrapps/jacoco-report@v1.3 with: - paths: ${{ github.workspace }}/atum-service/target/spark3.3-jvm-2.12/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/target/spark3.3-jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 0.0 min-coverage-changed-files: 80.0 diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..5e94dbc0b --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,23 @@ +# Release + +Releases of this library are currently handled by [sbt-ci-release](https://github.com/sbt/sbt-ci-release). +Please see its documentation for more details about how it works if you are interested to know more. + +The actual deployments are triggered manually by the maintainers of this repository, using `workflow_dispatch` event +trigger. + +Once changes from a PR were reviewed and merged into the master branch, follow these steps: +1. Create a new Git Tag and push it to the repository, to the master branch. For example, + if you want to release a version 0.4.0 (note that we are using [Semantic Versioning](https://semver.org/)): + + ```shell + git tag -a v0.4.0 -m "v0.4.0" + git push origin v0.4.0 + ``` + +2. In GitHub UI, go to the repository's **Actions** -> **Release** -> **Run workflow**, and under **Use workflow from** + use **Tags** and find the tag you created in the previous step. + + > **Important note**: don't run the workflow against the master branch, but against the tag. + > `sbt-ci-release` plugin won't be able to correctly find tag, and it will think that you are trying + > to do a snapshot release, not an actual release that should be synchronized with Maven Central. diff --git a/project/plugins.sbt b/project/plugins.sbt index 6fb697943..1fec9c35a 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -40,4 +40,5 @@ addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://gith addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") // To add release plugin +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") From 0b73dd62875f278ff35dbd1bb3baa27661a0d517 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Mon, 31 Jul 2023 15:45:47 +0200 Subject: [PATCH 22/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index cef60dbd0..2e73fd708 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -37,7 +37,7 @@ jobs: id: jacoco211 uses: madrapps/jacoco-report@v1.3 with: - paths: ${{ github.workspace }}/atum-service/target/spark2.4-jvm-2.11/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/target/spark2.4-jvm-2.11/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 0.0 min-coverage-changed-files: 80.0 From f69adb23cacc7dd1e0e811784b69cb5d4195f5e4 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 1 Aug 2023 15:03:49 +0200 Subject: [PATCH 23/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 99 ++++++++++++++---------------- 1 file changed, 46 insertions(+), 53 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 2e73fd708..7b5ff1b8d 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -1,4 +1,4 @@ -# + # Copyright 2021 ABSA Group Limited # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,57 +19,50 @@ name: JaCoCo report on: pull_request: branches: [ master ] - types: [ opened, synchronize, reopened ] + types: [ opened, edited, synchronize, reopened ] jobs: - test: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Scala - uses: olafurpg/setup-scala@v10 - with: - java-version: "adopt@1.8" - - name: Build and run tests - run: sbt jacoco - - name: Add coverage to PR for scala 2.11 and spark 2.4 - id: jacoco211 - uses: madrapps/jacoco-report@v1.3 - with: - paths: ${{ github.workspace }}/target/spark2.4-jvm-2.11/jacoco/report/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: 0.0 - min-coverage-changed-files: 80.0 - title: JaCoCo code coverage report - scala:2.11 - spark:2.4 - update-comment: true - - name: Get the Coverage info - run: | - echo "Total coverage ${{ steps.jacoco211.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco211.outputs.coverage-changed-files }}" - - name: Add coverage to PR for scala 2.12 and spark 3.3 - id: jacoco212 - uses: madrapps/jacoco-report@v1.3 - with: - paths: ${{ github.workspace }}/target/spark3.3-jvm-2.12/jacoco/report/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: 0.0 - min-coverage-changed-files: 80.0 - title: JaCoCo code coverage report - scala:2.12 - spark:3.3 - update-comment: true - - name: Get the Coverage info - run: | - echo "Total coverage ${{ steps.jacoco212.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco212.outputs.coverage-changed-files }}" - - name: Fail PR if changed files coverage is less than 80% - if: ${{ steps.jacoco211.outputs.coverage-changed-files < 80.0 }} - uses: actions/github-script@v6 - with: - script: | - core.setFailed('Changed files coverage is less than 80%!') - - name: Fail PR if changed files coverage is less than 80% - if: ${{ steps.jacoco212.outputs.coverage-changed-files < 80.0 }} - uses: actions/github-script@v6 - with: - script: | - core.setFailed('Changed files coverage is less than 80%!') + runs-on: ubuntu-latest + strategy: + matrix: + include: + # The project supports Scala 2.11, 2.12 + # # The CI runs all tests suites for all supported Scala versions at build.yml + # # The codebase for all Scala versions is the same, so the coverage is calculated only once + # # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and + # # vendor distributions. + - scala: 2.12.12 + scalaShort: "2.12" + overall: 80.0 + changed: 80.0 + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + - name: Build and run tests + run: sbt ++${{matrix.scala}} jacoco + - name: Add coverage to PR + id: jacoco + uses: madrapps/jacoco-report@v1.3 + with: + paths: ${{ github.workspace }}/target/scala-${{ matrix.scalaShot }}/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{matrix.overall }} + min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo code coverage report - scala ${{ matrix.scala }} + updated-coment: true + - name: Get the Coverage info + run: | + echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }} + - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% + if: ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') + + From 19577813841126ef55679c446dabd0a47c8d222a Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 1 Aug 2023 15:34:01 +0200 Subject: [PATCH 24/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 85 +++++++++++++++--------------- 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 7b5ff1b8d..497d370b6 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -22,47 +22,48 @@ on: types: [ opened, edited, synchronize, reopened ] jobs: - runs-on: ubuntu-latest - strategy: - matrix: - include: - # The project supports Scala 2.11, 2.12 - # # The CI runs all tests suites for all supported Scala versions at build.yml - # # The codebase for all Scala versions is the same, so the coverage is calculated only once - # # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and - # # vendor distributions. - - scala: 2.12.12 - scalaShort: "2.12" - overall: 80.0 - changed: 80.0 - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Scala - uses: olafurpg/setup-scala@v10 - with: - java-version: "adopt@1.8" - - name: Build and run tests - run: sbt ++${{matrix.scala}} jacoco - - name: Add coverage to PR - id: jacoco - uses: madrapps/jacoco-report@v1.3 - with: - paths: ${{ github.workspace }}/target/scala-${{ matrix.scalaShot }}/jacoco/report/jacoco.xml - token: ${{ secrets.GITHUB_TOKEN }} - min-coverage-overall: ${{matrix.overall }} - min-coverage-changed-file: ${{ matrix.changed }} - title: JaCoCo code coverage report - scala ${{ matrix.scala }} - updated-coment: true - - name: Get the Coverage info - run: | - echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }} - - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% - if: ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }} - uses: actions/github-script@v6 - with: - script: | - core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + # The project supports Scala 2.11, 2.12 + # # The CI runs all tests suites for all supported Scala versions at build.yml + # # The codebase for all Scala versions is the same, so the coverage is calculated only once + # # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and + # # vendor distributions. + - scala: 2.12.12 + scalaShort: "2.12" + overall: 80.0 + changed: 80.0 + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Setup Scala + uses: olafurpg/setup-scala@v10 + with: + java-version: "adopt@1.8" + - name: Build and run tests + run: sbt ++${{matrix.scala}} jacoco + - name: Add coverage to PR + id: jacoco + uses: madrapps/jacoco-report@v1.3 + with: + paths: ${{ github.workspace }}/target/scala-${{ matrix.scalaShot }}/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{matrix.overall }} + min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo code coverage report - scala ${{ matrix.scala }} + updated-comment: true + - name: Get the Coverage info + run: | + echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }} + - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% + if: ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }} + uses: actions/github-script@v6 + with: + script: | + core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') From 53a72bc88ff489a18dd682e1dd1e3515d685cab2 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 1 Aug 2023 15:42:28 +0200 Subject: [PATCH 25/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 497d370b6..4c93c9a04 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -49,7 +49,7 @@ jobs: id: jacoco uses: madrapps/jacoco-report@v1.3 with: - paths: ${{ github.workspace }}/target/scala-${{ matrix.scalaShot }}/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} From d6d66ba8b0f22a59aba206e64477d863a44ae25d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 1 Aug 2023 15:59:24 +0200 Subject: [PATCH 26/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- .../scala/za/co/absa/atum/agent/AtumAgent.scala | 15 --------------- .../scala/za/co/absa/atum/agent/AtumContext.scala | 15 --------------- .../atum/agent/core/MeasurementProcessor.scala | 15 --------------- .../co/absa/atum/agent/model/MeasureResult.scala | 15 --------------- .../za/co/absa/atum/agent/model/Measurement.scala | 15 --------------- .../za/co/absa/atum/agent/AtumContextSpec.scala | 15 --------------- .../absa/atum/agent/model/MeasurementSpec.scala | 15 --------------- 8 files changed, 1 insertion(+), 106 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 4c93c9a04..de437b78b 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -49,7 +49,7 @@ jobs: id: jacoco uses: madrapps/jacoco-report@v1.3 with: - paths: ${{ github.workspace }}/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index c0da4e6af..c63db3f63 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent import za.co.absa.atum.agent.model.MeasureResult diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index c44c8d831..bf39f85f8 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala index 5e685a4e7..e0a864734 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent.core import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index f8c09a7d0..d4154df7a 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent.model case class MeasureResult(measurement: Measurement, result: String) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala index 536ebc387..27250ab79 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent.model import org.apache.spark.sql.functions._ diff --git a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala index 4c31ec4ea..50644babc 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/AtumContextSpec.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent import org.scalatest.flatspec.AnyFlatSpec diff --git a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala index b861397db..b10057c91 100644 --- a/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala +++ b/agent/src/test/scala/za/co/absa/atum/agent/model/MeasurementSpec.scala @@ -1,18 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package za.co.absa.atum.agent.model import org.apache.spark.sql.{SQLContext, SQLImplicits, SparkSession} From 3e0314e22240161e3bcb07c9b9747d2285bbcbec Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 14:38:33 +0200 Subject: [PATCH 27/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 18 ++++++++++++++++-- build.sbt | 20 ++++++-------------- project/SparkVersionAxis.scala | 8 ++++---- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index de437b78b..9a608fdb6 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -45,16 +45,30 @@ jobs: java-version: "adopt@1.8" - name: Build and run tests run: sbt ++${{matrix.scala}} jacoco +# Agent and server module code coverage - name: Add coverage to PR id: jacoco uses: madrapps/jacoco-report@v1.3 with: - paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml + paths: | + ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml + ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} - title: JaCoCo code coverage report - scala ${{ matrix.scala }} + title: JaCoCo agent module code coverage report - scala ${{ matrix.scala }} updated-comment: true +# server module code coverage +# - name: Add coverage to PR +# id: jacoco +# uses: madrapps/jacoco-report@v1.3 +# with: +# paths: ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml +# token: ${{ secrets.GITHUB_TOKEN }} +# min-coverage-overall: ${{matrix.overall }} +# min-coverage-changed-file: ${{ matrix.changed }} +# title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} +# updated-comment: true - name: Get the Coverage info run: | echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" diff --git a/build.sbt b/build.sbt index f11717a0d..90b7a38bc 100644 --- a/build.sbt +++ b/build.sbt @@ -1,17 +1,3 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ import Dependencies._ import SparkVersionAxis._ @@ -66,4 +52,10 @@ lazy val atumServer = (projectMatrix in file("server")) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(scala212)) +//lazy val projectMatrix = sparkRow( +// sparkAxis = SparkVersionAxis(spark3), +// scalaVersions = Seq(scala212, scala211), +// modules = Seq(atumServer, atumAgent) +//) + diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index bccc5ae4e..652afd9c9 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -13,11 +13,11 @@ * limitations under the License. */ -import sbt._ -import sbt.Keys._ +import sbt.* +import sbt.Keys.* import sbt.internal.ProjectMatrix -import Dependencies._ -import JacocoSetup._ +import Dependencies.* +import JacocoSetup.* import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { From f0f218dd8e7f03014baaa27e1b644fa029bfcaa0 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Wed, 2 Aug 2023 15:13:31 +0200 Subject: [PATCH 28/56] * Proposal of adding Jacoco support for server module. --- build.sbt | 5 +++++ project/JacocoSetup.scala | 12 +++++++++--- project/SparkVersionAxis.scala | 12 ++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/build.sbt b/build.sbt index 90b7a38bc..f09b6473d 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,7 @@ import Dependencies._ import SparkVersionAxis._ +import JacocoSetup._ ThisBuild / organization := "za.co.absa" @@ -48,6 +49,10 @@ lazy val atumServer = (projectMatrix in file("server")) inheritJarManifest := true ): _* ) + .settings( + jacocoReportSettings := jacocoSettings( scalaVersion.value, "atum-server"), + jacocoExcludes := jacocoProjectExcludes() + ) .enablePlugins(TomcatPlugin) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(scala212)) diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index b69e41acd..db9b858b7 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -25,13 +25,19 @@ object JacocoSetup { formats = Seq(JacocoReportFormats.HTML, JacocoReportFormats.XML) ) - def jacocoSettings(sparkVersion: String, scalaVersion: String): JacocoReportSettings = { + def jacocoSettings(sparkVersion: String, scalaVersion: String, moduleName: String): JacocoReportSettings = { val utcDateTime = ZonedDateTime.now.withZoneSameInstant(ZoneId.of("UTC")) val now = s"as of ${DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm Z z").format(utcDateTime)}" - jacocoReportCommonSettings.withTitle(s"Jacoco Report on `atum-service` for spark:$sparkVersion - scala:$scalaVersion [$now]") + jacocoReportCommonSettings.withTitle(s"Jacoco Report on `$moduleName` for spark:$sparkVersion - scala:$scalaVersion [$now]") } - def jacocoProjectExcludes(sparkVersion: String, scalaVersion: String): Seq[String] = { + def jacocoSettings(scalaVersion: String, moduleName: String): JacocoReportSettings = { + val utcDateTime = ZonedDateTime.now.withZoneSameInstant(ZoneId.of("UTC")) + val now = s"as of ${DateTimeFormatter.ofPattern("yyyy-MM-dd hh:mm Z z").format(utcDateTime)}" + jacocoReportCommonSettings.withTitle(s"Jacoco Report on `$moduleName` for scala:$scalaVersion [$now]") + } + + def jacocoProjectExcludes(): Seq[String] = { Seq( "za.co.absa.atum.service.adapters.CallUdfAdapter", "za.co.absa.atum.service.adapters.TransformAdapter" diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index 652afd9c9..a0fa3cc5a 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -13,11 +13,11 @@ * limitations under the License. */ -import sbt.* -import sbt.Keys.* +import sbt.{Def, VirtualAxis} +import sbt.Keys.{libraryDependencies, moduleName} import sbt.internal.ProjectMatrix -import Dependencies.* -import JacocoSetup.* +import Dependencies.agentDependencies +import JacocoSetup.{jacocoProjectExcludes, jacocoSettings} import com.github.sbt.jacoco.JacocoKeys.{jacocoExcludes, jacocoReportSettings} case class SparkVersionAxis(sparkVersion: String) extends sbt.VirtualAxis.WeakAxis { @@ -45,8 +45,8 @@ object SparkVersionAxis { _.settings( moduleName := camelCaseToLowerDashCase(name.value + sparkAxis.directorySuffix), libraryDependencies ++= agentDependencies(sparkAxis.sparkVersion), - jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion), - jacocoExcludes := jacocoProjectExcludes(sparkVersion, scalaVersion) + jacocoReportSettings := jacocoSettings(sparkVersion, scalaVersion, "atum-agent"), + jacocoExcludes := jacocoProjectExcludes() ).settings(settings: _*) ) ) From 41d5b7fcdd76a5d9ba2602feefd54bcdf8dd8ecb Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 17:37:17 +0200 Subject: [PATCH 29/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 15 --------------- project/SparkVersionAxis.scala | 4 ++-- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 9a608fdb6..fe78934c8 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -1,19 +1,4 @@ -# Copyright 2021 ABSA Group Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - name: JaCoCo report on: diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index a0fa3cc5a..ff25a11f1 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -13,8 +13,8 @@ * limitations under the License. */ -import sbt.{Def, VirtualAxis} -import sbt.Keys.{libraryDependencies, moduleName} +import sbt.{Def, Keys, VirtualAxis} +import sbt.Keys.{libraryDependencies, moduleName, name} import sbt.internal.ProjectMatrix import Dependencies.agentDependencies import JacocoSetup.{jacocoProjectExcludes, jacocoSettings} From 14e6786f4511496a57e7b49064944e959aef802d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 18:13:16 +0200 Subject: [PATCH 30/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index fe78934c8..4ab528e6b 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -19,8 +19,8 @@ jobs: # # vendor distributions. - scala: 2.12.12 scalaShort: "2.12" - overall: 80.0 - changed: 80.0 + overall: 60.0 + changed: 60.0 steps: - name: Checkout code uses: actions/checkout@v2 From 8bd231aec360a3a79e433f61e7b7775641b7b89d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 18:34:47 +0200 Subject: [PATCH 31/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 4ab528e6b..f129a8aa1 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -35,9 +35,8 @@ jobs: id: jacoco uses: madrapps/jacoco-report@v1.3 with: - paths: | - ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml - ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml +# ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} From 9ce61560304b289e87f1ad70a2781dde1276bb6b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 18:51:18 +0200 Subject: [PATCH 32/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index f129a8aa1..75789dabf 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -58,7 +58,7 @@ jobs: echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }} - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% - if: ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }} + if: ${{ steps.jacoco.outputs.coverage-changed-files < 60.0 }} uses: actions/github-script@v6 with: script: | From e8f20f443c9be7201b2e56bde37f93e91b6e4fd1 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 18:59:45 +0200 Subject: [PATCH 33/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 75789dabf..8d0246b5f 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -5,7 +5,6 @@ on: pull_request: branches: [ master ] types: [ opened, edited, synchronize, reopened ] - jobs: test: runs-on: ubuntu-latest From 2a022d5f3085c489d17e4e4cf22d2888f4f8bc04 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 19:12:11 +0200 Subject: [PATCH 34/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 8d0246b5f..719cbc170 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -1,10 +1,10 @@ - name: JaCoCo report on: pull_request: branches: [ master ] types: [ opened, edited, synchronize, reopened ] + jobs: test: runs-on: ubuntu-latest @@ -29,29 +29,16 @@ jobs: java-version: "adopt@1.8" - name: Build and run tests run: sbt ++${{matrix.scala}} jacoco -# Agent and server module code coverage - name: Add coverage to PR id: jacoco uses: madrapps/jacoco-report@v1.3 with: paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml -# ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo agent module code coverage report - scala ${{ matrix.scala }} updated-comment: true -# server module code coverage -# - name: Add coverage to PR -# id: jacoco -# uses: madrapps/jacoco-report@v1.3 -# with: -# paths: ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml -# token: ${{ secrets.GITHUB_TOKEN }} -# min-coverage-overall: ${{matrix.overall }} -# min-coverage-changed-file: ${{ matrix.changed }} -# title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} -# updated-comment: true - name: Get the Coverage info run: | echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" @@ -62,5 +49,3 @@ jobs: with: script: | core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') - - From 27016851dc169950da3fc40bd6f925fba20489ff Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 19:19:48 +0200 Subject: [PATCH 35/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 719cbc170..9b07d02e1 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -42,7 +42,7 @@ jobs: - name: Get the Coverage info run: | echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }} + echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}" - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% if: ${{ steps.jacoco.outputs.coverage-changed-files < 60.0 }} uses: actions/github-script@v6 From c08bbbe61d885ff09924effbe7910a2533724671 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 19:25:52 +0200 Subject: [PATCH 36/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 9b07d02e1..075bf3339 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -33,7 +33,9 @@ jobs: id: jacoco uses: madrapps/jacoco-report@v1.3 with: - paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml + paths: | + ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml + ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} From 1d432d1c9c755a5b9ac5b055b462b52986f824b3 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 19:36:28 +0200 Subject: [PATCH 37/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 34 ++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 075bf3339..b840fd414 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -1,3 +1,19 @@ + +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: JaCoCo report on: @@ -18,8 +34,8 @@ jobs: # # vendor distributions. - scala: 2.12.12 scalaShort: "2.12" - overall: 60.0 - changed: 60.0 + overall: 80.0 + changed: 80.0 steps: - name: Checkout code uses: actions/checkout@v2 @@ -29,6 +45,7 @@ jobs: java-version: "adopt@1.8" - name: Build and run tests run: sbt ++${{matrix.scala}} jacoco +# Agent and server module code coverage - name: Add coverage to PR id: jacoco uses: madrapps/jacoco-report@v1.3 @@ -41,6 +58,17 @@ jobs: min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo agent module code coverage report - scala ${{ matrix.scala }} updated-comment: true +# server module code coverage + - name: Add coverage to PR + id: jacoco + uses: madrapps/jacoco-report@v1.3 + with: + paths: ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml + token: ${{ secrets.GITHUB_TOKEN }} + min-coverage-overall: ${{matrix.overall }} + min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} + updated-comment: true - name: Get the Coverage info run: | echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" @@ -51,3 +79,5 @@ jobs: with: script: | core.setFailed('Changed files coverage is less than ${{ matrix.changed }}%!') + + From 12229b36bf269bc46dfdd51e106897290269ba1d Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 19:50:27 +0200 Subject: [PATCH 38/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index b840fd414..acdd93186 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -45,9 +45,9 @@ jobs: java-version: "adopt@1.8" - name: Build and run tests run: sbt ++${{matrix.scala}} jacoco -# Agent and server module code coverage + # Agent and server module code coverage - name: Add coverage to PR - id: jacoco + id: jacoco-agent uses: madrapps/jacoco-report@v1.3 with: paths: | @@ -58,9 +58,9 @@ jobs: min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo agent module code coverage report - scala ${{ matrix.scala }} updated-comment: true -# server module code coverage + # server module code coverage - name: Add coverage to PR - id: jacoco + id: jacoco-server uses: madrapps/jacoco-report@v1.3 with: paths: ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml @@ -71,10 +71,14 @@ jobs: updated-comment: true - name: Get the Coverage info run: | - echo "Total coverage ${{ steps.jacoco.outputs.coverage-overall }}" - echo "Changed Files coverage ${{ steps.jacoco.outputs.coverage-changed-files }}" + echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco-agent.outputs.coverage-changed-files }}" + echo "Total sever module coverage ${{ steps.jacoco-server.outputs.coverage-overall }}" + echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% - if: ${{ steps.jacoco.outputs.coverage-changed-files < 60.0 }} + if: | + ${{ steps.jacoco.outputs.coverage-changed-files < 60.0 }} && + ${{ steps.jacoco-server.outputs.coverage-changed-files < 60.0 }} uses: actions/github-script@v6 with: script: | From ab2dbfff5ad349678451d562d47090b85df0a636 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 21:40:07 +0200 Subject: [PATCH 39/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index acdd93186..d2db642df 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -50,9 +50,8 @@ jobs: id: jacoco-agent uses: madrapps/jacoco-report@v1.3 with: - paths: | - ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml - ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml + name: agent-jacoco-report + paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} @@ -63,6 +62,7 @@ jobs: id: jacoco-server uses: madrapps/jacoco-report@v1.3 with: + name: server-jacoco-report paths: ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} From 3c2457d62d198d62a96610cffd6a5fadb64587d0 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 21:48:38 +0200 Subject: [PATCH 40/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index d2db642df..3cbb63b0c 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -34,8 +34,8 @@ jobs: # # vendor distributions. - scala: 2.12.12 scalaShort: "2.12" - overall: 80.0 - changed: 80.0 + overall: 60.0 + changed: 60.0 steps: - name: Checkout code uses: actions/checkout@v2 From 148c856d45c8d3dd7198c23dcf8c7650b5ad9088 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 2 Aug 2023 21:57:13 +0200 Subject: [PATCH 41/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 3cbb63b0c..8578f295f 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -34,8 +34,8 @@ jobs: # # vendor distributions. - scala: 2.12.12 scalaShort: "2.12" - overall: 60.0 - changed: 60.0 + overall: 80.0 + changed: 80.0 steps: - name: Checkout code uses: actions/checkout@v2 @@ -77,8 +77,8 @@ jobs: echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% if: | - ${{ steps.jacoco.outputs.coverage-changed-files < 60.0 }} && - ${{ steps.jacoco-server.outputs.coverage-changed-files < 60.0 }} + ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }} && + ${{ steps.jacoco-server.outputs.coverage-changed-files < 80.0 }} uses: actions/github-script@v6 with: script: | From a988abddd56205ee058854814b46bc5c38e96446 Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 3 Aug 2023 10:08:33 +0200 Subject: [PATCH 42/56] * Proposal for change. --- .github/workflows/jacoco_check.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 8578f295f..34c80a99d 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -55,7 +55,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} - title: JaCoCo agent module code coverage report - scala ${{ matrix.scala }} updated-comment: true # server module code coverage - name: Add coverage to PR @@ -67,7 +66,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} - title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} updated-comment: true - name: Get the Coverage info run: | @@ -77,7 +75,7 @@ jobs: echo "Changed Files coverage ${{ steps.jacoco-server.outputs.coverage-changed-files }}" - name: Fail PR if changed files coverage is less than ${{ matrix.changed }}% if: | - ${{ steps.jacoco.outputs.coverage-changed-files < 80.0 }} && + ${{ steps.jacoco-agent.outputs.coverage-changed-files < 80.0 }} && ${{ steps.jacoco-server.outputs.coverage-changed-files < 80.0 }} uses: actions/github-script@v6 with: From 095e735fceb647bc71749c668dbc425def083d2b Mon Sep 17 00:00:00 2001 From: miroslavpojer Date: Thu, 3 Aug 2023 10:16:02 +0200 Subject: [PATCH 43/56] * Return back the title with spark major version now. --- .github/workflows/jacoco_check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 34c80a99d..274b6c14a 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -55,6 +55,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo agent module code coverage report - spark:2 - scala ${{ matrix.scala }} updated-comment: true # server module code coverage - name: Add coverage to PR @@ -66,6 +67,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} + title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} updated-comment: true - name: Get the Coverage info run: | From 9000ad07b23ca1a9901dd7572355d1098124a3df Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 3 Aug 2023 17:09:20 +0200 Subject: [PATCH 44/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 17 +---------------- .github/workflows/license_check.yml | 2 +- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 274b6c14a..6939abb94 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -1,19 +1,3 @@ - -# Copyright 2021 ABSA Group Limited -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - name: JaCoCo report on: @@ -23,6 +7,7 @@ on: jobs: test: + name: Build and test runs-on: ubuntu-latest strategy: matrix: diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index 965dd287f..ec542a908 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -19,7 +19,7 @@ name: License Check on: pull_request: branches: [ master ] - types: [ opened, synchronize, reopened ] + types: [ opened, edited, synchronize, reopened ] jobs: license-test: From 3963bff7d87dd97c94975169ecb742a3a6fec163 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 3 Aug 2023 17:15:34 +0200 Subject: [PATCH 45/56] Fixes #63 - added license --- .../scala/za/co/absa/atum/agent/AtumAgent.scala | 16 ++++++++++++++++ .../za/co/absa/atum/agent/AtumContext.scala | 16 ++++++++++++++++ .../atum/agent/core/MeasurementProcessor.scala | 16 ++++++++++++++++ .../co/absa/atum/agent/model/MeasureResult.scala | 16 ++++++++++++++++ .../co/absa/atum/agent/model/Measurement.scala | 16 ++++++++++++++++ 5 files changed, 80 insertions(+) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala index c63db3f63..64e683723 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import za.co.absa.atum.agent.model.MeasureResult diff --git a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala index bf39f85f8..e1e4cab6d 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala index e0a864734..c21a45a61 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.core import org.apache.spark.sql.DataFrame diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala index d4154df7a..377294f9b 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model case class MeasureResult(measurement: Measurement, result: String) diff --git a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala index 27250ab79..c1a4df86f 100644 --- a/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala +++ b/agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala @@ -1,3 +1,19 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package za.co.absa.atum.agent.model import org.apache.spark.sql.functions._ From f481bc6ec99cc8400a41a049309ee98be65a74cb Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Thu, 3 Aug 2023 17:19:32 +0200 Subject: [PATCH 46/56] Update .github/workflows/jacoco_check.yml Co-authored-by: Ladislav Sulak --- .github/workflows/jacoco_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 6939abb94..5bc212628 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -36,7 +36,7 @@ jobs: uses: madrapps/jacoco-report@v1.3 with: name: agent-jacoco-report - paths: ${{ github.workspace }}/agent/target/spark2-jvm-2.12/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/agent/target/spark2-jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} From ce44d9667ae994d86bcf2c3075be64487fbe9666 Mon Sep 17 00:00:00 2001 From: TebaleloS <107194332+TebaleloS@users.noreply.github.com> Date: Thu, 3 Aug 2023 17:19:52 +0200 Subject: [PATCH 47/56] Update .github/workflows/jacoco_check.yml Co-authored-by: Ladislav Sulak --- .github/workflows/jacoco_check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 5bc212628..354bd857e 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -48,7 +48,7 @@ jobs: uses: madrapps/jacoco-report@v1.3 with: name: server-jacoco-report - paths: ${{ github.workspace }}/server/target/jvm-2.12/jacoco/report/jacoco.xml + paths: ${{ github.workspace }}/server/target/jvm-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} From a67dadef9bfa270051f0d1dc0b457c4ea8f9e87c Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 3 Aug 2023 17:55:30 +0200 Subject: [PATCH 48/56] Fixes #63 - update build.sbt --- .github/workflows/dependent_items.yml | 16 ++++++++++++ .github/workflows/jacoco_check.yml | 16 ++++++++++++ .github/workflows/release.yml | 2 +- build.sbt | 36 +++++++++++++++++++++------ project/JacocoSetup.scala | 5 +--- project/SparkVersionAxis.scala | 2 +- 6 files changed, 64 insertions(+), 13 deletions(-) diff --git a/.github/workflows/dependent_items.yml b/.github/workflows/dependent_items.yml index fd0349ceb..1d690f2ee 100644 --- a/.github/workflows/dependent_items.yml +++ b/.github/workflows/dependent_items.yml @@ -1,3 +1,19 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: Dependent Issues on: diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 6939abb94..90f13f483 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -1,3 +1,19 @@ +# +# Copyright 2021 ABSA Group Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + name: JaCoCo report on: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e15fbddb4..2f418b92c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ # -# Copyright 2022 ABSA Group Limited +# Copyright 2021 ABSA Group Limited # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/build.sbt b/build.sbt index f09b6473d..0cc2166ff 100644 --- a/build.sbt +++ b/build.sbt @@ -1,20 +1,44 @@ +/* + * Copyright 2021 ABSA Group Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ import Dependencies._ import SparkVersionAxis._ import JacocoSetup._ -ThisBuild / organization := "za.co.absa" +ThisBuild / organization := "za.co.absa.atum-service" lazy val scala211 = "2.11.12" -lazy val scala212 = "2.12.12" +lazy val scala212 = "2.12.17" lazy val spark2 = "2.4.7" lazy val spark3 = "3.3.1" ThisBuild / crossScalaVersions := Seq(scala211, scala212) ThisBuild / scalaVersion := scala212 +ThisBuild / versionScheme := Some("early-semver") + Global / onChangedBuildSource := ReloadOnSourceChanges +lazy val printScalaVersion = taskKey[Unit]("Print Scala versions for atum-service is being built for.") + +ThisBuild / printScalaVersion := { + val log = streams.value.log + log.info(s"Building with Scala ${scalaVersion.value}") +} + lazy val commonSettings = Seq( libraryDependencies ++= commonDependencies, scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"), @@ -33,6 +57,7 @@ lazy val atumAgent = (projectMatrix in file("agent")) .settings( commonSettings ++ Seq( name := "atum-agent", + (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value, scalafmtOnCompile := true ) ) @@ -45,6 +70,7 @@ lazy val atumServer = (projectMatrix in file("server")) commonSettings ++ Seq( name := "atum-server", libraryDependencies ++= Dependencies.serverDependencies, + (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value, webappWebInfClasses := true, inheritJarManifest := true ): _* @@ -57,10 +83,6 @@ lazy val atumServer = (projectMatrix in file("server")) .enablePlugins(AutomateHeaderPlugin) .jvmPlatform(scalaVersions = Seq(scala212)) -//lazy val projectMatrix = sparkRow( -// sparkAxis = SparkVersionAxis(spark3), -// scalaVersions = Seq(scala212, scala211), -// modules = Seq(atumServer, atumAgent) -//) + diff --git a/project/JacocoSetup.scala b/project/JacocoSetup.scala index db9b858b7..3395256af 100644 --- a/project/JacocoSetup.scala +++ b/project/JacocoSetup.scala @@ -38,10 +38,7 @@ object JacocoSetup { } def jacocoProjectExcludes(): Seq[String] = { - Seq( - "za.co.absa.atum.service.adapters.CallUdfAdapter", - "za.co.absa.atum.service.adapters.TransformAdapter" - ) + Seq() } } diff --git a/project/SparkVersionAxis.scala b/project/SparkVersionAxis.scala index ff25a11f1..e548e4057 100644 --- a/project/SparkVersionAxis.scala +++ b/project/SparkVersionAxis.scala @@ -13,7 +13,7 @@ * limitations under the License. */ -import sbt.{Def, Keys, VirtualAxis} +import sbt.{Def, VirtualAxis} import sbt.Keys.{libraryDependencies, moduleName, name} import sbt.internal.ProjectMatrix import Dependencies.agentDependencies From ff74223395f86954f44a91c8e9cdaa7f39472f41 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 3 Aug 2023 18:45:42 +0200 Subject: [PATCH 49/56] Fixes #63 - update publish.sbt --- publish.sbt | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/publish.sbt b/publish.sbt index 23943f3a5..68766db9a 100644 --- a/publish.sbt +++ b/publish.sbt @@ -14,6 +14,47 @@ * limitations under the License. */ +ThisBuild / scmInfo := Some( + ScmInfo( + browseUrl = url("https://github.com/AbsaOSS/atum-service/tree/master"), + connection = "scm:git:git://github.com/AbsaOSS/atum-service.git", + devConnection = "scm:git:ssh://github.com/AbsaOSS/atum-service.git" + ) +) + +ThisBuild / developers := List( + Developer( + id = "Zejnilovic", + name = "Saša Zejnilović", + email = "sasa.zejnilovic@absa.africa", + url = url("https://github.com/Zejnilovic") + ), + Developer( + id = "benedeki", + name = "David Benedeki", + email = "david.benedeki@absa.africa", + url = url("https://github.com/benedeki") + ), + Developer( + id = "miroslavpojer", + name = "Miroslav Pojer", + email = "miroslav.pojer@absa.africa", + url = url("https://github.com/miroslavpojer") + ), + Developer( + id = "lsulak", + name = "Ladislav Sulak", + email = "ladislav.sulak@absa.africa", + url = url("https://github.com/lsulak") + ), + Developer( + id = "TebaleloS", + name = "Tebalelo Sekhula", + email = "tebalelo.sekhula@absa.africa", + url = url("https://github.com/TebaleloS") + ) +) + ThisBuild / organizationName := "ABSA Group Limited" ThisBuild / organizationHomepage := Some(url("https://www.absa.africa")) From a0719313d1434e23281d7dcb287c0e8b4d0bbfd0 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 4 Aug 2023 08:19:55 +0200 Subject: [PATCH 50/56] Fixes #63 - update publish.sbt --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 0cc2166ff..7afc24a68 100644 --- a/build.sbt +++ b/build.sbt @@ -21,7 +21,7 @@ import JacocoSetup._ ThisBuild / organization := "za.co.absa.atum-service" lazy val scala211 = "2.11.12" -lazy val scala212 = "2.12.17" +lazy val scala212 = "2.12.12" lazy val spark2 = "2.4.7" lazy val spark3 = "3.3.1" From 69540df53265cfddfe5c2fc3667414a0c8fbb3f4 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Fri, 4 Aug 2023 13:04:54 +0200 Subject: [PATCH 51/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 +- project/Dependencies.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index df70a1117..553cc3f1c 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -28,7 +28,7 @@ jobs: strategy: matrix: include: - # The project supports Scala 2.11, 2.12 + # The project supports Scala 2.11, 2.12 and the default version of spark2 # # The CI runs all tests suites for all supported Scala versions at build.yml # # The codebase for all Scala versions is the same, so the coverage is calculated only once # # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 9d737bf64..623c982f2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -80,7 +80,7 @@ object Dependencies { Seq( "org.apache.spark" %% "spark-core" % sparkVersion % Provided, "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, - "com.typesafe" % "config" % typesafeVersion, //TODO - check if used + "com.typesafe" % "config" % typesafeVersion, "za.co.absa" %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion, "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test, ) From 064ab03b1fa167c65376e092586e60fc97e21bba Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 8 Aug 2023 10:35:25 +0200 Subject: [PATCH 52/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 8 ++++---- .github/workflows/license_check.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index 553cc3f1c..dce13f57b 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -29,10 +29,10 @@ jobs: matrix: include: # The project supports Scala 2.11, 2.12 and the default version of spark2 - # # The CI runs all tests suites for all supported Scala versions at build.yml - # # The codebase for all Scala versions is the same, so the coverage is calculated only once - # # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and - # # vendor distributions. + # The CI runs all tests suites for all supported Scala versions at build.yml + # The codebase for all Scala versions is the same, so the coverage is calculated only once + # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and + # vendor distributions. - scala: 2.12.12 scalaShort: "2.12" overall: 80.0 diff --git a/.github/workflows/license_check.yml b/.github/workflows/license_check.yml index ec542a908..965dd287f 100644 --- a/.github/workflows/license_check.yml +++ b/.github/workflows/license_check.yml @@ -19,7 +19,7 @@ name: License Check on: pull_request: branches: [ master ] - types: [ opened, edited, synchronize, reopened ] + types: [ opened, synchronize, reopened ] jobs: license-test: From fdb86e3b6131f6c8d69415682a1af60b82b76950 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Tue, 8 Aug 2023 13:11:42 +0200 Subject: [PATCH 53/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index dce13f57b..f33fd2f0d 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -46,7 +46,7 @@ jobs: java-version: "adopt@1.8" - name: Build and run tests run: sbt ++${{matrix.scala}} jacoco - # Agent and server module code coverage + # Agent module code coverage - name: Add coverage to PR id: jacoco-agent uses: madrapps/jacoco-report@v1.3 @@ -57,7 +57,6 @@ jobs: min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo agent module code coverage report - spark:2 - scala ${{ matrix.scala }} - updated-comment: true # server module code coverage - name: Add coverage to PR id: jacoco-server @@ -69,7 +68,6 @@ jobs: min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} - updated-comment: true - name: Get the Coverage info run: | echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" From d244a4b8c5e9f103a45bf71b4dab401683a4a7fc Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 9 Aug 2023 21:41:17 +0200 Subject: [PATCH 54/56] Fixes #63 --- .github/workflows/jacoco_check.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index f33fd2f0d..d9dd5f19d 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -57,6 +57,7 @@ jobs: min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo agent module code coverage report - spark:2 - scala ${{ matrix.scala }} + update-comment: true # server module code coverage - name: Add coverage to PR id: jacoco-server @@ -68,6 +69,7 @@ jobs: min-coverage-overall: ${{matrix.overall }} min-coverage-changed-file: ${{ matrix.changed }} title: JaCoCo server module code coverage report - scala ${{ matrix.scala }} + update-comment: true - name: Get the Coverage info run: | echo "Total agent module coverage ${{ steps.jacoco-agent.outputs.coverage-overall }}" From 723246d18e34eb74257fb89d2594b0cd6ca01f2b Mon Sep 17 00:00:00 2001 From: AB019TC Date: Thu, 10 Aug 2023 11:56:04 +0200 Subject: [PATCH 55/56] Closes #63 --- .github/workflows/jacoco_check.yml | 8 ++++---- version.sbt | 17 ----------------- 2 files changed, 4 insertions(+), 21 deletions(-) delete mode 100644 version.sbt diff --git a/.github/workflows/jacoco_check.yml b/.github/workflows/jacoco_check.yml index d9dd5f19d..398a32ddf 100644 --- a/.github/workflows/jacoco_check.yml +++ b/.github/workflows/jacoco_check.yml @@ -29,10 +29,10 @@ jobs: matrix: include: # The project supports Scala 2.11, 2.12 and the default version of spark2 - # The CI runs all tests suites for all supported Scala versions at build.yml - # The codebase for all Scala versions is the same, so the coverage is calculated only once - # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and - # vendor distributions. + # The CI runs all tests suites for all supported Scala versions at build.yml + # The codebase for all Scala versions is the same, so the coverage is calculated only once + # Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and + # vendor distributions. - scala: 2.12.12 scalaShort: "2.12" overall: 80.0 diff --git a/version.sbt b/version.sbt deleted file mode 100644 index f65af45fa..000000000 --- a/version.sbt +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright 2021 ABSA Group Limited - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -ThisBuild / version := "0.0.1-SNAPSHOT" From 26c3506b7cb54f69c630cbd0e90a86766fa3f230 Mon Sep 17 00:00:00 2001 From: AB019TC Date: Wed, 16 Aug 2023 12:49:16 +0200 Subject: [PATCH 56/56] Closes #63 - Added notes for jacoco coverage and realse in the readme file --- README.md | 15 +++++++++++++++ project/plugins.sbt | 6 ++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 640d93c46..66738d954 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,18 @@ See `agent/README.md`. ### Server `server/` An API under construction that communicates with AtumAgent and with the persisting storage. It also provides measure configuration to the `AtumAgent`. See `server/README.md`. + +## How to generate Code coverage report +```sbt +sbt jacoco +``` +Code coverage wil be generated on path: +``` +{project-root}/atum-service/target/spark{spark_version}-jvm-{scala_version}/jacoco/report/html +{project-root}/atum-service-test/target/jvm-{scala_version}/jacoco/report/html +``` + + +## How to Release + +Please see [this file](RELEASE.md) for more details. \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 1fec9c35a..be661ebd4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -19,6 +19,9 @@ addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.7.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") addSbtPlugin("com.eed3si9n" % "sbt-projectmatrix" % "0.9.1") +// To add release plugin +addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") + // sbt-jacoco dependency downloading lazy val ow2Version = "9.5" lazy val jacocoVersion = "0.8.11-absa.1" @@ -39,6 +42,5 @@ addSbtPlugin("org.ow2.asm" % "asm-tree" % ow2Version from ow2Url("asm-tree")) addSbtPlugin("za.co.absa.sbt" % "sbt-jacoco" % "3.4.1-absa.4" from "https://github.com/AbsaOSS/sbt-jacoco/releases/download/3.4.1-absa.4/sbt-jacoco-3.4.1-absa.4.jar") addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") -// To add release plugin -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.12") +