#38 http client and config

.gitignore

@@ -67,6 +67,10 @@ _productMT
 _nestedStructs
 myTestCheckpoints
 
+**/main/resources/application.properties
+**/main/resources/application.conf
+**/main/resources/application.yaml
+**/main/resources/application.yml
 
 *.config
 *.keytab

agent/src/main/resources/reference.conf

@@ -0,0 +1,18 @@
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# dispatcher to be used (http or console)
+atum.dispatcher.type="http"
+
+# The REST API URI of the atum server
+#atum.dispatcher.http.url=

agent/src/main/scala/za/co/absa/atum/agent/AtumAgent.scala

@@ -15,25 +15,32 @@
  */
 
 package za.co.absa.atum.agent
-
+import com.typesafe.config.{Config, ConfigFactory}
+import za.co.absa.atum.agent.dispatcher.{ConsoleDispatcher, HttpDispatcher}
 import za.co.absa.atum.agent.AtumContext.AtumPartitions
 import za.co.absa.atum.agent.model.MeasureResult
 
 /**
- *  Place holder for the agent that communicate with the API.
+ * Place holder for the agent that communicate with the API.
  */
 class AtumAgent private() {
 
+  val config: Config = ConfigFactory.load()
+
+  private val dispatcher = config.getString("atum.dispatcher.type") match {
+    case "http" => new HttpDispatcher(config.getConfig("atum.dispatcher.http"))
+    case "console" => new ConsoleDispatcher
+    case dt => throw new UnsupportedOperationException(s"Unsupported dispatcher type: '$dt''")
+  }
+
   /**
    *  Sends a single `MeasureResult` to the AtumService API along with an extra data from a given `AtumContext`.
    *  @param checkpointKey
    *  @param atumContext
    *  @param measureResult
    */
   def publish(checkpointKey: String, atumContext: AtumContext, measureResult: MeasureResult): Unit =
-    println(
-      s"Enqueued measurement: ${Seq(checkpointKey, atumContext, measureResult).mkString(" || ")}"
-    )
+    dispatcher.publish(checkpointKey, atumContext, measureResult)
 
   /**
    *  Sends a single `MeasureResult` to the AtumService API. It doesn't involve AtumContext.
@@ -42,7 +49,7 @@ class AtumAgent private() {
    *  @param measureResult
    */
   def measurePublish(checkpointKey: String, measureResult: MeasureResult): Unit =
-    println(s"Enqueued measurement: $checkpointKey || $measureResult")
+    dispatcher.publish(checkpointKey, measureResult)
 
   /**
    *  Provides an AtumContext given a `AtumPartitions` instance. Retrieves the data from AtumService API.

agent/src/main/scala/za/co/absa/atum/agent/dispatcher/ConsoleDispatcher.scala

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.atum.agent.dispatcher
+
+import org.apache.spark.internal.Logging
+import za.co.absa.atum.agent.AtumContext
+import za.co.absa.atum.agent.model.MeasureResult
+
+/**
+ *  dispatcher useful for development, testing and debugging
+ */
+class ConsoleDispatcher extends Dispatcher with Logging {
+
+  logInfo("using console dispatcher")
+
+  override def publish(checkpointKey: String, measureResult: MeasureResult): Unit =
+    println(s"Publishing $checkpointKey $measureResult")
+
+  override def publish(checkpointKey: String, context: AtumContext, measureResult: MeasureResult): Unit =
+    println(s"Publishing $checkpointKey $context $measureResult")
+
+}

agent/src/main/scala/za/co/absa/atum/agent/dispatcher/Dispatcher.scala

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.atum.agent.dispatcher
+
+import za.co.absa.atum.agent.AtumContext
+import za.co.absa.atum.agent.model.MeasureResult
+
+trait Dispatcher {
+  def publish(checkpointKey: String, measureResult: MeasureResult): Unit
+
+  def publish(checkpointKey: String, context: AtumContext, measureResult: MeasureResult): Unit
+}

agent/src/main/scala/za/co/absa/atum/agent/dispatcher/HttpDispatcher.scala

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2021 ABSA Group Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package za.co.absa.atum.agent.dispatcher
+
+import com.typesafe.config.Config
+import org.apache.spark.internal.Logging
+import za.co.absa.atum.agent.model.MeasureResult
+import sttp.client3._
+import sttp.model.Uri
+import za.co.absa.atum.agent.AtumContext
+
+class HttpDispatcher(config: Config) extends Dispatcher with Logging {
+
+  private val serverUri = Uri.unsafeParse(config.getString("url"))
+  private val backend = HttpURLConnectionBackend()
+
+  logInfo("using http dispatcher")
+  logInfo(s"serverUri $serverUri")
+
+  override def publish(checkpointKey: String, measureResult: MeasureResult): Unit = {
+    basicRequest
+      .body(s"$checkpointKey $measureResult")
+      .post(serverUri)
+      .send(backend)
+  }
+
+  override def publish(checkpointKey: String, context: AtumContext, measureResult: MeasureResult): Unit = {
+    basicRequest
+      .body(s"$checkpointKey $context $measureResult")
+      .post(serverUri)
+      .send(backend)
+  }
+}

agent/src/test/resources/application.conf

@@ -0,0 +1,14 @@
+# Copyright 2021 ABSA Group Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+atum.dispatcher.type="console"

project/Dependencies.scala

@@ -60,8 +60,8 @@ object Dependencies {
       springBootTest % Test,
       springBootWeb,
       springBootConfiguration,
-      springBootTomcat /*% Provided*/,
-      servletApi /*% Provided*/,
+      springBootTomcat /*% Provided*/ ,
+      servletApi /*% Provided*/ ,
       springFoxSwagger,
       springFoxSwaggerUI,
       springFoxBoot,
@@ -72,22 +72,19 @@ object Dependencies {
 
   def agentDependencies(sparkVersion: String): Seq[ModuleID] = {
 
-    val typesafeVersion     = "1.4.2"
+    val typesafeVersion = "1.4.2"
     val sparkCommonsVersion = "0.6.0"
-    val sparkMinorVersion   = getVersionUpToMinor(sparkVersion)
-    val specs2CoreVersion   = "4.19.2"
+    val sparkMinorVersion = getVersionUpToMinor(sparkVersion)
+    val specs2CoreVersion = "4.19.2"
 
     Seq(
-        "org.apache.spark" %% "spark-core"                               % sparkVersion % Provided,
-        "org.apache.spark" %% "spark-sql"                                % sparkVersion % Provided,
-        "com.typesafe"      % "config"                                   % typesafeVersion,
-        "za.co.absa"       %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion,
-        "za.co.absa"       %% "spark-commons-test"                       % sparkCommonsVersion % Test,
+      "org.apache.spark" %% "spark-core" % sparkVersion % Provided,
+      "org.apache.spark" %% "spark-sql" % sparkVersion % Provided,
+      "com.typesafe" % "config" % typesafeVersion,
+      "za.co.absa" %% s"spark-commons-spark${sparkMinorVersion}" % sparkCommonsVersion,
+      "com.softwaremill.sttp.client3" %% "core" % "3.5.2",
+      "za.co.absa" %% "spark-commons-test" % sparkCommonsVersion % Test
     )
-
   }
 
 }
-
-
-