Merge branch 'master' into feature/#51-Make-the-server-executable

.github/workflows/assign_issue_to_project.yml

@@ -1,3 +1,18 @@
+#
+#   Copyright 2021 ABSA Group Limited
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
 name: Auto Assign Issue to Project
 
 on:

.github/workflows/dependent_items.yml

@@ -1,3 +1,19 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 name: Dependent Issues
 
 on:

README.md

@@ -10,3 +10,18 @@ See `agent/README.md`.
 ### Server `server/`
 An API under construction that communicates with AtumAgent and with the persisting storage. It also provides measure configuration to the `AtumAgent`.
 See `server/README.md`.
+
+## How to generate Code coverage report
+```sbt
+sbt jacoco
+```
+Code coverage wil be generated on path:
+```
+{project-root}/atum-service/target/spark{spark_version}-jvm-{scala_version}/jacoco/report/html
+{project-root}/atum-service-test/target/jvm-{scala_version}/jacoco/report/html
+```
+
+
+## How to Release
+
+Please see [this file](RELEASE.md) for more details.

agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent
 
 import za.co.absa.atum.agent.model.MeasureResult

agent/src/main/scala/za/co/absa/atum/agent/AtumContext.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent
 
 import org.apache.spark.sql.DataFrame

agent/src/main/scala/za/co/absa/atum/agent/core/MeasurementProcessor.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent.core
 
 import org.apache.spark.sql.DataFrame

agent/src/main/scala/za/co/absa/atum/agent/model/MeasureResult.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent.model
 
 case class MeasureResult(measurement: Measurement, result: String)

agent/src/main/scala/za/co/absa/atum/agent/model/Measurement.scala

@@ -1,3 +1,19 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package za.co.absa.atum.agent.model
 
 import org.apache.spark.sql.functions._

build.sbt

@@ -4,6 +4,7 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
+ *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
@@ -14,44 +15,76 @@
  */
 
 import Dependencies._
+import SparkVersionAxis._
+import JacocoSetup._
+import sbt.Keys.name
 
-ThisBuild / organization := "za.co.absa"
-
-enablePlugins(JettyPlugin)
+ThisBuild / organization := "za.co.absa.atum-service"
 
 lazy val scala211 = "2.11.12"
 lazy val scala212 = "2.12.12"
+lazy val spark2 = "2.4.7"
+lazy val spark3 = "3.3.1"
+
+ThisBuild / crossScalaVersions := Seq(scala211, scala212)
+ThisBuild / scalaVersion := scala212
 
-Test / parallelExecution := false
+ThisBuild / versionScheme := Some("early-semver")
 
 Global / onChangedBuildSource := ReloadOnSourceChanges
 
-lazy val root = (project in file("."))
-  .aggregate(server, agent)
+lazy val printScalaVersion = taskKey[Unit]("Print Scala versions for atum-service is being built for.")
+
+ThisBuild / printScalaVersion := {
+  val log = streams.value.log
+  log.info(s"Building with Scala ${scalaVersion.value}")
+}
+
+lazy val commonSettings = Seq(
+  libraryDependencies ++= commonDependencies,
+  scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature", "-Xfatal-warnings"),
+  javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint"),
+  Test / parallelExecution := false
+)
+
+lazy val parent = (project in file("."))
+  .aggregate(atumServer.projectRefs ++ atumAgent.projectRefs: _*)
   .settings(
-    name := "atum-root",
-    javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint")
+    name := "atum-service-parent",
+    publish / skip := true
   )
 
-lazy val server = (project in file("server"))
-  .enablePlugins(SbtWeb)
+lazy val atumAgent = (projectMatrix in file("agent"))
   .settings(
-    name         := "atum-server",
-    scalaVersion := scala212,
-    libraryDependencies ++= Dependencies.serverDependencies,
-    webappWebInfClasses := true,
-    inheritJarManifest  := true,
-    resourceDirectory in Compile := (webappResources in Compile).value,
-    artifactPath in (Compile, packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.war"
+    commonSettings ++ Seq(
+      name := "atum-agent",
+      (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value,
+      scalafmtOnCompile := true
+    )
+  )
+  .enablePlugins(ScalafmtPlugin)
+  .sparkRow(SparkVersionAxis(spark2), scalaVersions = Seq(scala211, scala212))
+  .sparkRow(SparkVersionAxis(spark3), scalaVersions = Seq(scala212))
+
+lazy val atumServer = (projectMatrix in file("server"))
+  .enablePlugins(AssemblyPlugin)
+  .settings(
+    commonSettings ++ Seq(
+      name := "atum-server",
+      libraryDependencies ++= Dependencies.serverDependencies,
+      (Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value,
+      packageBin := (assembly in  Compile).value,
+      artifactPath in (Compile, packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.war",
+//      assemblyOutputPath / assembly := baseDirectory.value / s"target/${name.value}-${version.value}.war",
+      webappWebInfClasses := true,
+      inheritJarManifest := true
+    ): _*
+  )
+  .settings(
+    jacocoReportSettings := jacocoSettings( scalaVersion.value, "atum-server"),
+    jacocoExcludes := jacocoProjectExcludes()
   )
-  .enablePlugins(JettyPlugin)
   .enablePlugins(TomcatPlugin)
   .enablePlugins(AutomateHeaderPlugin)
+  .jvmPlatform(scalaVersions = Seq(scala212))
 
-lazy val agent = (project in file("agent"))
-  .settings(
-    name         := "atum-agent",
-    scalaVersion := scala212,
-    libraryDependencies ++= Dependencies.agentDependencies,
-    scalafmtOnCompile := true
-  ).enablePlugins(ScalafmtPlugin)

project/Dependencies.scala

@@ -17,9 +17,25 @@
 import sbt._
 
 object Dependencies {
+  private def limitVersion(version: String, parts: Int): String = {
+    version.split("\\.", parts + 1).take(parts).mkString(".")
+  }
+
+  def getVersionUpToMinor(version: String): String = {
+    limitVersion(version, 2)
+  }
 
-  val serverDependencies: Seq[ModuleID] = {
-    val springVersion = "3.1.0"
+  def getVersionUpToMajor(version: String): String = {
+    limitVersion(version, 1)
+  }
+
+  def commonDependencies: Seq[ModuleID] = Seq(
+    "org.scalatest" %% "scalatest" % "3.2.2" % Test,
+    "org.mockito" %% "mockito-scala" % "1.17.12" % Test
+  )
+
+  def serverDependencies: Seq[ModuleID] = {
+    val springVersion = "2.6.1"
     val springOrg = "org.springframework.boot"
 
     lazy val scalaTest = "org.scalatest" %% "scalatest" % "3.2.15"
@@ -53,30 +69,24 @@ object Dependencies {
     )
   }
 
+  def agentDependencies(sparkVersion: String): Seq[ModuleID] = {
 
-  val agentDependencies: Seq[ModuleID] = {
-
-    val spark3Version = "3.3.2"
-    val scala212 = "2.12.12"
-    val scalatestVersion = "3.2.15"
-    val specs2Version = "4.19.2"
-    val typesafeConfigVersion = "1.4.2"
-
-    lazy val sparkCore = "org.apache.spark" %% "spark-core" % spark3Version
-
-    lazy val sparkCommons = "za.co.absa" % "spark-commons-spark3.3_2.12" % "0.5.0"
-
-    lazy val sparkCommonsTest = "za.co.absa" %% "spark-commons-test" % "0.5.0" % Test
-
-    lazy val sparkSql ="org.apache.spark" %% "spark-sql" %  spark3Version
-    lazy val scalaTest = "org.scalatest" %% "scalatest" % scalatestVersion % Test
-    lazy val specs2core = "org.specs2" %% "specs2-core" % specs2Version % Test
-    lazy val typeSafeConfig = "com.typesafe" % "config" % typesafeConfigVersion
+    val typesafeVersion     = "1.4.2"
+    val sparkCommonsVersion = "0.6.0"
+    val sparkMinorVersion   = getVersionUpToMinor(sparkVersion)
+    val specs2CoreVersion   = "4.19.2"
 
+    Seq(
+        "org.apache.spark" %% "spark-core"                               % sparkVersion % Provided,
+        "org.apache.spark" %% "spark-sql"                                % sparkVersion % Provided,
+        "com.typesafe"      % "config"                                   % typesafeVersion,
+        "za.co.absa"       %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion,
+        "za.co.absa"       %% "spark-commons-test"                       % sparkCommonsVersion % Test,
+    )
 
-    Seq(sparkCore, sparkCommons, sparkCommonsTest, sparkSql, scalaTest, specs2core, typeSafeConfig)
+  }
 
-  }}
+}
 
 
 

project/build.properties

@@ -1 +1,16 @@
+#
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 sbt.version=1.9.0