0.5.0 add watchdog option, suitable to be used in a cronjob

README.md

@@ -23,95 +23,19 @@ $ java -jar cf2rdf-x.y.z.jar generate-conf
 
 Edit `cf2rdf.conf` as needed.
 
-As command-line arguments for the regular execution, cf2rdf expects the desired steps to be performed.
-  
-The complete sequence, including registration looks like so:
+As command-line arguments for the regular execution, cf2rdf expects either the
+`watchdog` argument to check for current remote CF version and trigger conversion
+and registration in case of new version (based on comparison with latest processed file): 
 
 ```shell 
-$ java -jar cf2rdf-x.y.z.jar download convert register
-
-Downloading https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/46/src/cf-standard-name-table.xml
-            -> ./cf2rdf_output/cf-standard-name-table.xml
-
-Downloading http://vocab.nerc.ac.uk/collection/P07/current/
-            -> ./cf2rdf_output/nvs_P07.rdf
-
-
-Replaced %20 for space in the following rdf:resource IRIs from ./cf2rdf_output/nvs_P07.rdf
-	<owl:sameAs rdf:resource="http://mmisw.org/ont/cf/parameter/mole_fraction_of_chlorine dioxide_in_air"/>
-	<owl:sameAs rdf:resource="http://mmisw.org/ont/cf/parameter/mole_fraction_of_chlorine monoxide_in_air"/>
-	<owl:sameAs rdf:resource="http://mmisw.org/ont/cf/parameter/mole_fraction_of_dichlorine peroxide_in_air"/>
-	<owl:sameAs rdf:resource="http://mmisw.org/ont/cf/parameter/mole_fraction_of_hypochlorous acid_in_air"/>
-	<owl:sameAs rdf:resource="http://mmisw.org/ont/cf/parameter/rate_of_ hydroxyl_radical_destruction_due_to_reaction_with_nmvoc"/>
-
-[main] WARN org.apache.jena.riot - [line: 1, col: 121] {W119} A processing instruction is in RDF content. No processing was done.
-
-Summary: (saved in ./cf2rdf_output/cf-standard-name-table.conv-stats.txt)
-	cf2rdf conversion
-	input:  https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/46/src/cf-standard-name-table.xml
-	output: ./cf2rdf_output/cf-standard-name-table.rdf
-
-	vocabulary properties from input file:
-	 version_number: 46; last_modified: 2017-07-25T09:41:29Z
-
-	conversion stats:
-	numConcepts = 2890
-	numEntries = 2889
-	numWithNoCanonicalUnits = 9
-	numWithNoDefinitions = 26
-
-	Mapping ontology:
-	  mappingTermsAdded     = 2889
-	  mappingOutputFilename = ./cf2rdf_output/cfonmap.n3
-
+$ java -jar cf2rdf-x.y.z.jar watchdog
+```
 
-Registering http://mmisw.org/ont/cf/parameter - Climate and Forecast (CF) Standard Names (v.46)
-    - uploading...
-      POST http://localhost:8081/api/v0/ont/upload
-    - registering...
-      PUT http://localhost:8081/api/v0/ont {
-        "orgName":"mmi",
-        "name":"Climate and Forecast (CF) Standard Names (v.46)",
-        "uploadedFilename":"1504822022386._guess",
-        "uploadedFormat":"rdf",
-        "iri":"http://mmisw.org/ont/cf/parameter",
-        "status":"stable",
-        "visibility":"public",
-        "log":"reflect version number 46",
-        "userName":"carueda"
-      }
-      Result:
-      {
-        "uri":"http://mmisw.org/ont/cf/parameter",
-        "version":"20170907T150702",
-        "visibility":"public",
-        "status":"stable",
-        "updated":"2017-09-07T15:07:02Z"
-      }
+or the desired specific steps to be performed, for example:
+
+```shell 
+$ java -jar cf2rdf-x.y.z.jar download convert register
 
-Registering http://mmisw.org/ont/mmi/cfonmap - ORR-NVS CF standard name mapping (v.46)
-    - uploading...
-      POST http://localhost:8081/api/v0/ont/upload
-    - registering...
-      PUT http://localhost:8081/api/v0/ont {
-        "orgName":"mmi",
-        "name":"ORR-NVS CF standard name mapping (v.46)",
-        "uploadedFilename":"1504822022908._guess",
-        "uploadedFormat":"n3",
-        "iri":"http://mmisw.org/ont/mmi/cfonmap",
-        "status":"stable",
-        "visibility":"public",
-        "log":"reflect version number 46",
-        "userName":"carueda"
-      }
-      Result:
-      {
-        "uri":"http://mmisw.org/ont/mmi/cfonmap",
-        "version":"20170907T150703",
-        "visibility":"public",
-        "status":"stable",
-        "updated":"2017-09-07T15:07:03Z"
-      }
 ```
 
 The latest conversion report is [here](cf2rdf_output/cf-standard-name-table.conv-stats.txt).

build.sbt

@@ -1,4 +1,4 @@
-lazy val cf2rdfVersion = setVersion("0.4.0")
+lazy val cf2rdfVersion = setVersion("0.5.0")
 val scalaV             = "2.12.2"
 val cfgV               = "0.0.7"
 val scalajHttpV        = "2.3.0"

cf2rdf_output/cf-standard-name-table.conv-stats.txt

@@ -1,5 +1,5 @@
 cf2rdf conversion
-input:  https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/46/src/cf-standard-name-table.xml
+input:  https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/current/src/cf-standard-name-table.xml
 output: ./cf2rdf_output/cf-standard-name-table.rdf
 
 vocabulary properties from input file:

cf2rdf_output/cf-standard-name-table.rdf

@@ -20,10 +20,10 @@
     <omvm:hasResourceType>http://mmisw.org/ont/mmi/resourcetype/parameter</omvm:hasResourceType>
     <omv:description>Ontology representation of the Climate and Forecast (CF) standard names parameter vocabulary, which is intended for use with climate and forecast data in the atmosphere, surface and ocean domains. Every CF parameter is captured as a SKOS concept.</omv:description>
     <omv:name>Climate and Forecast (CF) Standard Names (v.46)</omv:name>
+    <omvm:origVocUri>https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/current/src/cf-standard-name-table.xml</omvm:origVocUri>
     <omv:reference>http://marinemetadata.org/orrcf</omv:reference>
     <omv:acronym>CF-standard-name</omv:acronym>
     <omv:keywords>NetCDF, CF, Climate and Forecast, self-describing, standard names, Canonical Units</omv:keywords>
-    <omvm:origVocUri>https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/46/src/cf-standard-name-table.xml</omvm:origVocUri>
     <omv:documentation>http://cfconventions.org/standard-names.html</omv:documentation>
   </owl:Ontology>
   <rdfs:Class rdf:about="http://www.w3.org/2004/02/skos/core#Concept"/>

changelog.md

@@ -1,3 +1,9 @@
+2017-09-07 0.5.0
+
+- add watchdog option, suitable to be used in a cronjob.
+  This checks for current remote CF version and triggers conversion/registration 
+  in case of new version, based on comparison with latest processed file.
+
 2017-09-07 0.4.0
 
 - cf2rdf can now register the generated ontologies

src/main/resources/params_template.conf

@@ -2,20 +2,16 @@
 # cf2rdf execution parameters.
 #
 # You can adjust any of the entries in this file but only the following
-# will typically need to be adjusted as needed:
-#     cfVersion
+# will typically need to be adjusted:
 #     orr.userName
 #     orr.password
 #
 
-# The CF version number to process
-cfVersion = 46
-
-# The template for the URL to retrieve the XML file
-xmlUrl = "https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/"${cfVersion}"/src/cf-standard-name-table.xml"
+# URL of current XML file
+xmlUrl = "https://raw.githubusercontent.com/cf-convention/cf-convention.github.io/master/Data/cf-standard-names/current/src/cf-standard-name-table.xml"
 
 # Output directory for generated files
-outDir = "cf2rdf_output"
+outDir = "./cf2rdf_output"
 
 # Name of local file for the download of xmlUrl
 destXml = ${outDir}"/cf-standard-name-table.xml"
@@ -53,7 +49,7 @@ orr {
   userName = ?
   password = ?
 
-  # Organization that owns the entries at the ORR instance
+  # Organization that owns the rdf.iri and mapping.iri entries at the ORR instance
   orgName = "mmi"
 
   # Visibility for the registration

src/main/resources/reference.conf

@@ -1 +1 @@
-cf2rdf.version = 0.4.0
+cf2rdf.version = 0.5.0

src/main/scala/org/mmisw/cf2rdf/Converter.scala

@@ -11,27 +11,23 @@ import scala.xml.Node
  *
  * @param xmlIn       Input XML
  */
-class Converter(xmlIn: Node) {
-
-  /** some general properties from the input */
-  val props: Map[String,String] = {
-    val keys = List("version_number", "last_modified") //, "institution", "contact")
-    (keys map (k ⇒ k -> (xmlIn \ k).text.trim)).toMap
-  }
+class Converter(xmlIn: Node, xmlProps: Map[String,String]) {
 
   /**
    * Does the conversion
    *
    * @return  Resulting Jena model
    */
   def convert: Model = {
-    val lastModifiedOpt = props.get("last_modified")
+    val lastModifiedOpt = xmlProps.get("last_modified")
 
     val namespace = cfg.rdf.iri + "/"
 
-    val M = new ModelConstructor(namespace, lastModifiedOpt)
+    val cfVersionOpt = xmlProps.get("version_number")
+
+    val M = new ModelConstructor(namespace, cfVersionOpt, lastModifiedOpt)
 
-    val mapper = new OrrNvsMapper(lastModifiedOpt)
+    val mapper = new OrrNvsMapper(cfVersionOpt, lastModifiedOpt)
 
     for (entry ← xmlIn \\ "entry") {
       stats.numEntries += 1

src/main/scala/org/mmisw/cf2rdf/ModelConstructor.scala

@@ -7,6 +7,7 @@ import org.apache.jena.vocabulary._
 import org.mmisw.orr.ont.vocabulary.{Omv, OmvMmi}
 
 class ModelConstructor(namespace: String,
+                       cfVersionOpt: Option[String],
                        lastModifiedOpt: Option[String]
                       ) {
 
@@ -32,8 +33,17 @@ class ModelConstructor(namespace: String,
 
   private val ontology = model.createOntology(cfg.rdf.iri)
 
-  ontology.addProperty(Omv.name,
-    s"Climate and Forecast (CF) Standard Names (v.${cfg.cfVersion})")
+  cfVersionOpt match {
+    case Some(cfVersion) ⇒
+      ontology.addProperty(Omv.name,
+        s"Climate and Forecast (CF) Standard Names (v.$cfVersion)")
+
+      ontology.addProperty(OmvMmi.origVocVersionId, cfVersion)
+
+    case None ⇒
+      ontology.addProperty(Omv.name,
+        s"Climate and Forecast (CF) Standard Names")
+  }
 
   ontology.addProperty(Omv.description,
     "Ontology representation of the Climate and Forecast (CF) standard names parameter vocabulary," +
@@ -61,7 +71,6 @@ class ModelConstructor(namespace: String,
     ontology.addProperty(Omv.creationDate, lm)
   }
 
-  ontology.addProperty(OmvMmi.origVocVersionId, cfg.cfVersion)
   ontology.addProperty(OmvMmi.origVocUri, cfg.xmlUrl)
 
   ontology.addProperty(OmvMmi.hasResourceType,

src/main/scala/org/mmisw/cf2rdf/OrrNvsMapper.scala

@@ -12,7 +12,7 @@ import org.mmisw.orr.ont.vocabulary.{Omv, OmvMmi}
 import scala.collection.JavaConverters._
 
 
-class OrrNvsMapper(lastModifiedOpt: Option[String]) {
+class OrrNvsMapper(cfVersionOpt: Option[String], lastModifiedOpt: Option[String]) {
 
   private val mapNamespace   = cfg.mapping.iri + "/"
   private val orrCfNamespace = cfg.rdf.iri + "/"
@@ -30,8 +30,15 @@ class OrrNvsMapper(lastModifiedOpt: Option[String]) {
 
   private val ontology = model.createOntology(cfg.mapping.iri)
 
-  ontology.addProperty(Omv.name,
-    s"ORR-NVS CF standard name mapping (v.${cfg.cfVersion})")
+  cfVersionOpt match {
+    case Some(cfVersion) ⇒
+      ontology.addProperty(Omv.name,
+        s"ORR-NVS CF standard name mapping (v.$cfVersion)")
+
+    case None ⇒
+      ontology.addProperty(Omv.name,
+        s"ORR-NVS CF standard name mapping")
+  }
 
   ontology.addProperty(Omv.description,
     "Uses skos:exactMatch to link the IRIs of the CF standard names between the" +

src/main/scala/org/mmisw/cf2rdf/Registerer.scala

@@ -12,7 +12,7 @@ import org.json4s.native.Serialization.writePretty
 import scalaj.http.{Http, HttpResponse, MultiPart}
 
 
-class Registerer(orr: OrrCfg) {
+class Registerer(orr: OrrCfg, cfVersion: String) {
   private implicit val jsonFormats: Formats = DefaultFormats ++ JodaTimeSerializers.all
 
   def registerOntologies(): Unit = {
@@ -22,16 +22,16 @@ class Registerer(orr: OrrCfg) {
 
   def registerRdf(): Unit = {
     register(cfg.rdf.iri,
-      s"Climate and Forecast (CF) Standard Names (v.${cfg.cfVersion})",
-      log = s"reflect version number ${cfg.cfVersion}",
+      s"Climate and Forecast (CF) Standard Names (v.$cfVersion)",
+      log = s"reflect version number $cfVersion",
       cfg.rdf.filename
     )
   }
 
   def registerMapping(): Unit = {
     register(cfg.mapping.iri,
-      s"ORR-NVS CF standard name mapping (v.${cfg.cfVersion})",
-      log = s"reflect version number ${cfg.cfVersion}",
+      s"ORR-NVS CF standard name mapping (v.$cfVersion)",
+      log = s"reflect version number $cfVersion",
       cfg.mapping.filename
     )
   }