Merge pull request #783 from krgauraw/auto-creator-v2-fix

Issue IQ-682 fix: fix for Quml 1.1 objects

auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/functions/AutoCreatorFunction.scala

@@ -1,5 +1,6 @@
 package org.sunbird.job.autocreatorv2.functions
 
+import org.apache.commons.lang3.StringUtils
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
@@ -48,8 +49,11 @@ class AutoCreatorFunction(config: AutoCreatorV2Config, httpUtil: HttpUtil,
     if (event.isValid) {
       logger.info("Processing event for bulk approval operation having identifier : " + event.objectId)
       logger.debug("event edata : " + event.eData)
-      val definition: ObjectDefinition = defCache.getDefinition(event.objectType, config.schemaSupportVersionMap.getOrElse(event.objectType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath)
-      val obj: ObjectData = getObject(event.objectId, event.objectType, event.downloadUrl, event.repository)(config, httpUtil, definition)
+      val obj: ObjectData = getObject(event.objectId, event.objectType, event.downloadUrl, event.repository)(config, httpUtil, defCache)
+      val schemaVersion = obj.metadata.getOrElse("schemaVersion", "1.0").asInstanceOf[String]
+      val defVer: String = if (StringUtils.equalsIgnoreCase("1.1", schemaVersion)) schemaVersion else config.schemaSupportVersionMap.getOrElse(event.objectType.toLowerCase(), "1.0").asInstanceOf[String]
+      logger.info("Fetching Schema With Version " + defVer)
+      val definition: ObjectDefinition = defCache.getDefinition(event.objectType, defVer, config.definitionBasePath)
       logger.debug("Constructed the ObjectData for " + obj.identifier)
       val enObj = enrichMetadata(obj, event.metadata)(config)
       logger.info("Enriched metadata for " + enObj.identifier)

auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/AutoCreator.scala

@@ -14,11 +14,14 @@ trait AutoCreator extends ObjectUpdater with CollectionUpdater with HierarchyEnr
 
 	private[this] val logger = LoggerFactory.getLogger(classOf[AutoCreator])
 
-	def getObject(identifier: String, objType: String, downloadUrl: String, metaUrl: Option[String] = None)(implicit config: AutoCreatorV2Config, httpUtil: HttpUtil, objDef: ObjectDefinition): ObjectData = {
+	def getObject(identifier: String, objType: String, downloadUrl: String, metaUrl: Option[String] = None)(implicit config: AutoCreatorV2Config, httpUtil: HttpUtil, defCache: DefinitionCache): ObjectData = {
 		val extractPath = extractDataZip(identifier, downloadUrl)
 		val manifestData = getObjectDetails(identifier, extractPath, objType, metaUrl)
-		val metadata = manifestData.filterKeys(k => !(objDef.getRelationLabels().contains(k) || objDef.externalProperties.contains(k)))
-		val extData = manifestData.filterKeys(k => objDef.externalProperties.contains(k))
+		val schemaVersion = manifestData.getOrElse("schemaVersion", "").asInstanceOf[String]
+		val defVer = if (StringUtils.equalsIgnoreCase("1.1", schemaVersion)) schemaVersion else config.schemaSupportVersionMap.getOrElse(objType.toLowerCase(), "1.0").asInstanceOf[String]
+		val definition: ObjectDefinition = defCache.getDefinition(objType, defVer, config.definitionBasePath)
+		val metadata = manifestData.filterKeys(k => !(definition.getRelationLabels().contains(k) || definition.externalProperties.contains(k)))
+		val extData = manifestData.filterKeys(k => definition.externalProperties.contains(k))
 		val hierarchy = getHierarchy(extractPath, objType)(config)
 		val externalData = if (hierarchy.nonEmpty) extData ++ Map("hierarchy" -> hierarchy) else extData
 		new ObjectData(identifier, objType, metadata, Some(externalData), Some(hierarchy))
@@ -41,7 +44,8 @@ trait AutoCreator extends ObjectUpdater with CollectionUpdater with HierarchyEnr
       .find(p => StringUtils.equalsIgnoreCase(identifier, p.getOrElse("identifier", "").asInstanceOf[String])).getOrElse(Map())
     if (metaUrl.nonEmpty) {
       // TODO: deprecate setting "origin" after single sourcing refactoring.
-      val originData = s"""{\"identifier\": \"$identifier\",\"repository\":\"${metaUrl.head}\"}"""
+			val repoUrl = if(StringUtils.isNotBlank(metaUrl.head) && StringUtils.contains(metaUrl.head, "fields")) metaUrl.head.split("\\?")(0) else metaUrl.head
+      val originData = s"""{\"identifier\": \"$identifier\",\"repository\":\"${repoUrl}\"}"""
       val originDetails = Map[String, AnyRef]("origin" -> identifier, "originData" -> originData)
       val metadata = getMetaUrlData(metaUrl.head, objectType)(httpUtil) ++ originDetails
       manifestMetadata.++(metadata)
@@ -97,11 +101,14 @@ trait AutoCreator extends ObjectUpdater with CollectionUpdater with HierarchyEnr
 		children.flatMap(ch => {
 			logger.info("Processing Children Having Identifier : " + ch._1)
 			val objType = ch._2.asInstanceOf[Map[String, AnyRef]].getOrElse("objectType", "").asInstanceOf[String]
-			val definition: ObjectDefinition = defCache.getDefinition(objType, config.schemaSupportVersionMap.getOrElse(objType.toLowerCase(), "1.0").asInstanceOf[String], config.definitionBasePath)
+			val schemaVersion = ch._2.asInstanceOf[Map[String, AnyRef]].getOrElse("schemaVersion", "").asInstanceOf[String]
+			val defVer: String = if(StringUtils.equalsIgnoreCase("1.1", schemaVersion)) schemaVersion else config.schemaSupportVersionMap.getOrElse(objType.toLowerCase(), "1.0").asInstanceOf[String]
+			val definition: ObjectDefinition = defCache.getDefinition(objType, defVer, config.definitionBasePath)
 			val downloadUrl = ch._2.asInstanceOf[Map[String, AnyRef]].getOrElse("downloadUrl", "").asInstanceOf[String]
 			val props = definition.getSchemaProps() ++ definition.getExternalProps().keySet.toList
-			val repository = s"""${config.sourceBaseUrl}/${objType.toLowerCase}/v1/read/${ch._1}?fields=${props.mkString(",")}"""
-			val obj: ObjectData = getObject(ch._1, objType, downloadUrl, Some(repository))(config, httpUtil, definition)
+			val apiVer = if (StringUtils.equalsIgnoreCase("1.1", schemaVersion)) "v2" else "v1"
+			val repository = s"""${config.sourceBaseUrl}/${objType.toLowerCase}/${apiVer}/read/${ch._1}?fields=${props.mkString(",")}"""
+			val obj: ObjectData = getObject(ch._1, objType, downloadUrl, Some(repository))(config, httpUtil, defCache)
 			logger.debug("Graph metadata for " + obj.identifier + " : " + obj.metadata)
 			val enObj = enrichMetadata(obj, ch._2.asInstanceOf[Map[String, AnyRef]], overrideCloudProps = true)(config)
 			logger.debug("Enriched metadata for " + enObj.identifier + " : " + enObj.metadata)

auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/HierarchyEnricher.scala

@@ -57,7 +57,7 @@ trait HierarchyEnricher {
 	}
 
 	def getChildData(data: Map[String, AnyRef])(implicit config: AutoCreatorV2Config): Map[String, AnyRef] = {
-		val props = config.cloudProps ++ List("objectType")
+		val props = config.cloudProps ++ List("objectType","schemaVersion")
 		props.filter(x => data.get(x).nonEmpty).map(prop => (prop, data.getOrElse(prop, ""))).toMap
 	}
 

auto-creator-v2/src/main/scala/org/sunbird/job/autocreatorv2/helpers/ObjectUpdater.scala

@@ -17,7 +17,7 @@ trait ObjectUpdater {
 		val metadata = data - ("identifier", "objectType")
 		val metaQuery = metaDataQuery(metadata, objectDef)
 		val query = s"""MERGE (n:domain{IL_UNIQUE_ID:"$identifier"}) ON CREATE SET $metaQuery ON MATCH SET $metaQuery;"""
-		logger.debug("Graph Query: " + query)
+		logger.info("Graph Query: " + query)
 		val result: StatementResult = neo4JUtil.executeQuery(query)
 		if (null != result) {
 			logger.info("Object graph data stored successfully for " + identifier)
@@ -41,7 +41,7 @@ trait ObjectUpdater {
 				case _ => query.value(d._1, d._2)
 			}
 		})
-		logger.debug(s"Saving object external data for $identifier | Query : ${query.toString}")
+		logger.info(s"Saving object external data for $identifier | Query : ${query.toString}")
 		val result = cassandraUtil.upsert(query.toString)
 		if (result) {
 			logger.info(s"Object external data saved successfully for ${identifier}")

auto-creator-v2/src/test/scala/org/sunbird/job/autocreatorv2/spec/helper/AutoCreatorSpec.scala

@@ -58,7 +58,7 @@ class AutoCreatorSpec extends FlatSpec with BeforeAndAfterAll with Matchers with
 
 	"getObject" should "return a valid object" in {
 		val downloadUrl = "https://dockstorage.blob.core.windows.net/sunbird-content-dock/questionset/do_113244425048121344131/added1_1616751462043_do_113244425048121344131_1_SPINE.ecar"
-		val result = new TestAutoCreator().getObject("do_113244425048121344131", "QuestionSet", downloadUrl)(jobConfig, httpUtil, qsDefinition)
+		val result = new TestAutoCreator().getObject("do_113244425048121344131", "QuestionSet", downloadUrl)(jobConfig, httpUtil, defCache)
 		result.identifier shouldEqual "do_113244425048121344131"
 		result.metadata.nonEmpty shouldBe(true)
 	}

publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/EcarGeneratorSpec.scala

@@ -35,7 +35,7 @@ class EcarGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matchers {
   "Object Ecar Generator generateEcar" should "return a Map containing Packaging Type and its url after uploading it to cloud" in {
 
     val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")))
-    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
+    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "identifier" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
     val objData = new ObjectData("do_123", metadata, None, Some(hierarchy))
     val obj = new TestEcarGenerator()
     val result = obj.generateEcar(objData,List("SPINE"))
@@ -50,6 +50,6 @@ class TestEcarGenerator extends EcarGenerator {
     "src" -> "somepath/sunbird_1551961194254.jpeg",
     "baseUrl" -> "some_base_url"
   )
-  val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))
+  val testObj = List(Map("children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question"), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question")), "name" -> "Test QuestionSet", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "objectType" -> "QuestionSet", "identifier" -> "do_123", "status" -> "Live", "identifier" -> "do_123"), Map("identifier" -> "do_234", "name" -> "Children-1", "objectType" -> "Question", "media" -> ScalaJsonUtil.serialize(List(media))), Map("identifier" -> "do_345", "name" -> "Children-2", "objectType" -> "Question"))
   override def getDataForEcar(obj: ObjectData): Option[List[Map[String, AnyRef]]] = Some(testObj)
 }

publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ObjectEnrichmentSpec.scala

@@ -37,7 +37,7 @@ class ObjectEnrichmentSpec extends FlatSpec with BeforeAndAfterAll with Matchers
     when(mockNeo4JUtil.getNodesName(List("NCERT"))).thenReturn(Map("NCERT"-> "NCERT"))
 
     val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2")))
-    val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
+    val metadata = Map("identifier" -> "do_123", "targetFWIds" -> List("NCERT"), "boardIds" -> List("NCERT"), "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "IL_FUNC_OBJECT_TYPE" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
     val objData = new ObjectData("do_123", metadata, None, Some(hierarchy))
 
     val objectEnrichment = new TestObjectEnrichment()

publish-pipeline/publish-core/src/test/scala/org/sunbird/job/publish/spec/ThumbnailGeneratorSpec.scala

@@ -25,7 +25,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche
   "Object Thumbnail Generator generateThumbnail" should "add the thumbnail to ObjectData" in {
 
     val hierarchy = Map("identifier" -> "do_123", "children" -> List(Map("identifier" -> "do_234", "name" -> "Children-1"), Map("identifier" -> "do_345", "name" -> "Children-2")))
-    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
+    val metadata = Map("identifier" -> "do_123", "appIcon" -> "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png", "IL_UNIQUE_ID" -> "do_123", "objectType" -> "QuestionSet", "name" -> "Test QuestionSet", "status" -> "Live")
     val objData = new ObjectData("do_123", metadata, None, Some(hierarchy))
 
     val thumbnailGenerator = new TestThumbnailGenerator()
@@ -35,7 +35,7 @@ class ThumbnailGeneratorSpec extends FlatSpec with BeforeAndAfterAll with Matche
     resultMetadata.isEmpty should be(false)
     resultMetadata.getOrElse("posterImage", "").asInstanceOf[String].isEmpty should be(false)
     resultMetadata.getOrElse("appIcon", "").asInstanceOf[String].isEmpty should be(false)
-    resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.knowlg.sunbird.org/content/preview/assets/icons/avatar_anonymous.png"
+    resultMetadata.getOrElse("posterImage", "").asInstanceOf[String] shouldBe "https://dev.sunbirded.org/content/preview/assets/icons/avatar_anonymous.png"
     resultMetadata.getOrElse("appIcon", "").asInstanceOf[String] shouldBe "https://sunbirddev.blob.core.windows.net/sunbird-content-dev/questionset/do_123/artifact/avatar_anonymous.thumb.png"
   }