Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#77: scala2.13 support #78

Merged
merged 13 commits into from
Nov 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,13 @@ on:
jobs:
test:
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
scala: [2.11.12, 2.12.18, 2.13.11]

name: Scala ${{matrix.scala}}

steps:
- name: Checkout code
Expand All @@ -36,4 +41,4 @@ jobs:
with:
java-version: "adopt@1.8"
- name: Build and run tests
run: sbt test doc
run: sbt ++${{matrix.scala}} test doc
2 changes: 1 addition & 1 deletion .github/workflows/jacoco_check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ jobs:
# The codebase for all Scala versions is the same, so the coverage is calculated only once
# Scala 2.12 is chosen since it is supported by the most wide range of Spark versions and
# vendor distributions.
- scala: 2.12.12
- scala: 2.12.18
Zejnilovic marked this conversation as resolved.
Show resolved Hide resolved
scalaShort: "2.12"
overall: 80.0
changed: 80.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ class AtumContextTest extends AnyFlatSpec with Matchers {
val atumContext = new AtumContext(atumPartitions, mockAgent)
.addMeasure(RecordCount("letter"))

val spark = SparkSession.builder
val spark = SparkSession.builder()
.master("local")
.config("spark.driver.host", "localhost")
.config("spark.ui.enabled", "false")
Expand Down Expand Up @@ -134,7 +134,7 @@ class AtumContextTest extends AnyFlatSpec with Matchers {
implicit val atumContext: AtumContext = new AtumContext(atumPartitions, mockAgent)
.addMeasure(RecordCount("notImportantColumn"))

val spark = SparkSession.builder
val spark = SparkSession.builder()
.master("local")
.config("spark.driver.host", "localhost")
.config("spark.ui.enabled", "false")
Expand Down
13 changes: 5 additions & 8 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,8 @@ import sbt.Keys.name


ThisBuild / organization := "za.co.absa"
ThisBuild / name := "atum-service"

ThisBuild / scalaVersion := Versions.scala212 // default version
ThisBuild / crossScalaVersions := Versions.supportedScalaVersions

ThisBuild / versionScheme := Some("early-semver")

Expand Down Expand Up @@ -71,8 +69,7 @@ lazy val server = (projectMatrix in file("server"))
packageBin := (Compile / assembly).value,
artifactPath / (Compile / packageBin) := baseDirectory.value / s"target/${name.value}-${version.value}.war",
webappWebInfClasses := true,
inheritJarManifest := true,
mergeStrategy
inheritJarManifest := true
): _*
)
.settings(
Expand All @@ -82,7 +79,7 @@ lazy val server = (projectMatrix in file("server"))
.enablePlugins(AssemblyPlugin)
.enablePlugins(TomcatPlugin)
.enablePlugins(AutomateHeaderPlugin)
.jvmPlatform(scalaVersions = Versions.supportedScalaVersions)
.jvmPlatform(scalaVersions = Seq(Versions.serviceScalaVersion))
.dependsOn(model)

lazy val agent = (projectMatrix in file("agent"))
Expand All @@ -102,8 +99,8 @@ lazy val agent = (projectMatrix in file("agent"))
jacocoExcludes := jacocoProjectExcludes()
)
.sparkRow(SparkVersionAxis(Versions.spark2), scalaVersions = Seq(Versions.scala211, Versions.scala212))
.sparkRow(SparkVersionAxis(Versions.spark3), scalaVersions = Seq(Versions.scala212))
.jvmPlatform(scalaVersions = Versions.supportedScalaVersions)
.sparkRow(SparkVersionAxis(Versions.spark3), scalaVersions = Seq(Versions.scala212, Versions.scala213))
.jvmPlatform(scalaVersions = Versions.clientSupportedScalaVersions)
.dependsOn(model)

lazy val model = (projectMatrix in file("model"))
Expand All @@ -118,4 +115,4 @@ lazy val model = (projectMatrix in file("model"))
jacocoReportSettings := jacocoSettings(scalaVersion.value, "atum-agent: model"),
jacocoExcludes := jacocoProjectExcludes()
)
.jvmPlatform(scalaVersions = Versions.supportedScalaVersions)
.jvmPlatform(scalaVersions = Versions.clientSupportedScalaVersions)
15 changes: 8 additions & 7 deletions project/Dependencies.scala
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,11 @@ object Dependencies {
val spark3 = "3.3.2"

val scala211 = "2.11.12"
val scala212 = "2.12.12"
val supportedScalaVersions: Seq[String] = Seq(scala211, scala212)
val scala212 = "2.12.18"
val scala213 = "2.13.11"

val serviceScalaVersion: String = scala212
val clientSupportedScalaVersions: Seq[String] = Seq(scala211, scala212, scala213)

val scalatest = "3.2.15"
val scalaMockito = "1.17.12"
Expand All @@ -35,7 +38,7 @@ object Dependencies {
val specs2 = "4.10.0"
val typesafeConfig = "1.4.2"

val spring = "2.6.6"
val spring = "2.6.15"

val javaxServlet = "4.0.1"

Expand All @@ -51,7 +54,6 @@ object Dependencies {
val json4s_spark2 = "3.5.3"
val json4s_spark3 = "3.7.0-M11"

val slf4s = "1.7.25"
val logback = "1.2.3"
}

Expand All @@ -72,7 +74,8 @@ object Dependencies {
scalaVersion match {
case _ if scalaVersion.startsWith("2.11") => Versions.spark2
case _ if scalaVersion.startsWith("2.12") => Versions.spark3
case _ => throw new IllegalArgumentException("Only Scala 2.11 and 2.12 are currently supported.")
case _ if scalaVersion.startsWith("2.13") => Versions.spark3
case _ => throw new IllegalArgumentException("Only Scala 2.11, 2.12, and 2.13 are currently supported.")
}
}

Expand All @@ -95,7 +98,6 @@ object Dependencies {
lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % json4sVersion % Provided
lazy val json4sNative = "org.json4s" %% "json4s-native" % json4sVersion % Provided

lazy val slf4s = "org.slf4s" %% "slf4s-api" % Versions.slf4s
lazy val logback = "ch.qos.logback" % "logback-classic" % Versions.logback
lazy val scalatest = "org.scalatest" %% "scalatest" % Versions.scalatest % Test
lazy val mockito = "org.mockito" %% "mockito-scala" % Versions.scalaMockito % Test
Expand All @@ -106,7 +108,6 @@ object Dependencies {
json4sCore,
json4sJackson,
json4sNative,
slf4s,
logback,
scalatest,
mockito,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,9 @@ import za.co.absa.atum.server.model.{BaseApiModel, ControlMeasure, Flow, Partiti

import java.util.UUID
import java.util.concurrent.{ConcurrentHashMap, ConcurrentMap}
import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future

import scala.collection.JavaConverters._

object InMemoryApiModelDao {

Expand Down
Loading