Merge pull request #89 from RADAR-CNS/release-0.3

Release 0.3

.travis.yml

@@ -4,6 +4,15 @@ jdk:
 env:
   - TERM=dumb
 
+cache:
+  directories:
+    - $HOME/.gradle/caches/jars-1
+    - $HOME/.gradle/caches/jars-2
+    - $HOME/.gradle/caches/jars-3
+    - $HOME/.gradle/caches/modules-2/files-2.1/
+    - $HOME/.gradle/native
+    - $HOME/.gradle/wrapper
+
 deploy:
   - provider: releases
     api_key: ${GH_TOKEN}
@@ -27,6 +36,4 @@ deploy:
 
 before_install:
   - cd java-sdk
-
-before_deploy:
-  - pwd
+  - ./gradlew downloadDependencies

commons/kafka/aggregate_key.avsc

@@ -7,7 +7,7 @@
     {"name": "projectId", "type": ["null", "string"], "doc": "Project that the key belongs to.", "default": null},
     {"name": "userId", "type": "string", "doc": "User Identifier created during the enrolment."},
     {"name": "sourceId", "type": "string", "doc": "Unique identifier associated with the source."},
-    {"name": "start", "type": "long", "doc": "First timestamp in UNIX time contained in the time window."},
-    {"name": "end", "type": "long", "doc": "Last timestamp in UNIX time contained in the time window."}
+    {"name": "timeStart", "type": "double", "doc": "Time (seconds since the UNIX Epoch) of the time window start."},
+    {"name": "timeEnd", "type": "double", "doc": "Time (seconds since the UNIX Epoch) of the time window end."}
   ]
 }

commons/stream/aggregator/aggregate_list.avsc

@@ -0,0 +1,10 @@
+{
+  "namespace": "org.radarcns.stream.aggregator",
+  "type": "record",
+  "name": "AggregateList",
+  "doc": "A list of aggregated data, each item aggregating over a single field.",
+  "version": "1.0.0",
+  "fields": [
+    { "name": "fields", "type": {"type": "array", "items": "NumericAggregate" }, "doc": "Aggregates of each of the underlying elements." }
+  ]
+}

commons/stream/aggregator/double_aggregation.avsc → commons/stream/aggregator/numeric_aggregate.avsc

@@ -1,15 +1,16 @@
 {
   "namespace": "org.radarcns.stream.aggregator",
   "type": "record",
-  "name": "DoubleAggregation",
-  "doc": "Result of data aggregation.",
+  "name": "NumericAggregate",
+  "doc": "Basic statistics on a numeric value type.",
+  "version": "1.0.0",
   "fields": [
+    { "name": "name", "type": "string", "doc": "Name of the field that has been aggregated over." },
     { "name": "min", "type": "double", "doc": "State the minimum between accumulated values." },
     { "name": "max", "type": "double", "doc": "State the maximum between accumulated values." },
     { "name": "sum", "type": "double", "doc": "State the sum of accumulated values." },
-    { "name": "count", "type": "double", "doc": "Count the accumulated values." },
-    { "name": "avg", "type": ["null", "double"], "doc": "State the avg between accumulated values.", "default": null },
-    { "name": "quartile", "type": ["null", {"type": "array", "items": "double"}], "doc": "Quartile of accumulated values.", "default": null },
-    { "name": "iqr", "type": ["null", "double"], "doc": "State the interquartile range between accumulated values.", "default": null }
+    { "name": "count", "type": "int", "doc": "Count the accumulated values." },
+    { "name": "mean", "type": ["null", "double"], "doc": "State the arithmetic mean between accumulated values.", "default": null },
+    { "name": "quartile", "type": ["null", {"type": "array", "items": "double"}], "doc": "Quartile of accumulated values.", "default": null }
   ]
 }

commons/stream/aggregator/phone_usage_aggregation.avsc → commons/stream/aggregator/phone_usage_aggregate.avsc

@@ -1,8 +1,9 @@
 {
   "namespace": "org.radarcns.stream.aggregator",
   "type": "record",
-  "name": "PhoneUsageAggregation",
+  "name": "PhoneUsageAggregate",
   "doc": "Aggregate time and opening events for an app.",
+  "version": "1.0.0",
   "fields": [
     {"name": "packageName", "type": "string", "doc": "Package name of the app in use."},
     {"name": "durationInForeground", "type":"double", "doc":"Total time in milliseconds the app was in the foreground."},

java-sdk/README.md

@@ -9,13 +9,13 @@ repositories {
 
 dependencies {
     // Commons schemas (backend, passive remote monitoring app)
-    compile 'org.radarcns:radar-schemas-commons:0.2.2'
+    compile 'org.radarcns:radar-schemas-commons:0.3'
 
     // REST API schemas (REST API, testing)
-    compile 'org.radarcns:radar-schemas-restapi:0.2.2'
+    compile 'org.radarcns:radar-schemas-restapi:0.3'
 
     // Questionnaire schemas (active remote monitoring app)
-    compile 'org.radarcns:radar-schemas-questionnaire:0.2.2'
+    compile 'org.radarcns:radar-schemas-tools:0.3'
 }
 ```
 Usually, you only need to include the schemas you actually need in your dependencies.

java-sdk/build.gradle

@@ -17,7 +17,7 @@ subprojects {
     apply plugin: 'idea'
 
     // Configuration
-    version = '0.2.3'
+    version = '0.3'
     group = 'org.radarcns'
     ext.githubRepoName = 'RADAR-CNS/RADAR-Schemas'
 
@@ -130,7 +130,20 @@ subprojects {
     }
 }
 
+task downloadDependencies {
+    description "Pre-downloads *most* dependencies"
+    doLast {
+        configurations.getAsMap().each { name, config ->
+            println "Retrieving dependencies for $name"
+            try {
+                config.files
+            } catch (e) {
+                project.logger.info e.message // some cannot be resolved, silentlyish skip them
+            }
+        }
+    }
+}
+
 task wrapper(type: Wrapper) {
-    gradleVersion = '4.1'
-    distributionType 'all'
+    gradleVersion '4.4'
 }

java-sdk/gradle/wrapper/gradle-wrapper.properties

@@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-bin.zip

java-sdk/radar-schemas-tools/build.gradle

@@ -5,6 +5,8 @@ plugins {
     id 'application'
 }
 
+apply plugin: 'com.jfrog.artifactory'
+
 ext.artifactName = 'radar-schemas-tools'
 ext.description = 'RADAR Schemas specification and validation tools.'
 
@@ -36,9 +38,11 @@ dependencies {
     implementation group: 'org.eclipse.jetty', name: 'jetty-server', version: jettyVersion
     implementation group: 'org.eclipse.jetty', name: 'jetty-servlet', version: jettyVersion
     implementation group: 'org.glassfish.jersey.core', name: 'jersey-server', version: jerseyVersion
-    implementation group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion
-    implementation group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: jerseyVersion
 
+    // source catalogue service annotations
+    api group: 'org.glassfish.jersey.containers', name: 'jersey-container-servlet-core', version: jerseyVersion
+
+    runtimeOnly group: 'org.glassfish.jersey.media', name: 'jersey-media-json-jackson', version: jerseyVersion
     runtimeOnly group: 'org.slf4j', name: 'slf4j-simple', version: '1.7.25'
 
     testCompile group: 'junit', name: 'junit', version: junitVersion
@@ -140,3 +144,19 @@ bintray {
         }
     }
 }
+
+artifactory {
+    contextUrl = 'https://oss.jfrog.org/artifactory'
+    publish {
+        repository {
+            repoKey = 'oss-snapshot-local'
+            username = project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER')
+            password = project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY')
+            maven = true
+        }
+    }
+}
+
+artifactoryPublish {
+    publications('RadarCommonsPublication')
+}

java-sdk/radar-schemas-tools/src/main/java/org/radarcns/schema/registration/KafkaTopics.java

@@ -24,12 +24,19 @@
 
 import static org.radarcns.schema.CommandLineApp.matchTopic;
 
+/**
+ * Registers Kafka topics with Zookeeper.
+ */
 public class KafkaTopics implements Closeable {
     private static final Logger logger = LoggerFactory.getLogger(KafkaTopics.class);
     private static final int MAX_SLEEP = 32;
 
     private final ZkUtils zkUtils;
 
+    /**
+     * Create Kafka topics registration object with given Zookeeper.
+     * @param zookeeper comma-separated list of Zookeeper 'hostname:port'.
+     */
     public KafkaTopics(String zookeeper) {
         ZkClient zkClient = new ZkClient(zookeeper, 15_000, 10_000);
 
@@ -48,11 +55,58 @@ public Object deserialize(byte[] bytes) throws ZkMarshallingError {
         zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeper), false);
     }
 
+    /**
+     * Wait for brokers to become available. This uses a polling mechanism,
+     * waiting for at most 200 seconds.
+     * @param brokers number of brokers to wait for
+     * @return whether the brokers where available
+     * @throws InterruptedException
+     * @throws KeeperException
+     */
+    public boolean waitForBrokers(int brokers) throws InterruptedException, KeeperException {
+        boolean brokersAvailable = false;
+        int sleep = 2;
+        for (int tries = 0; tries < 10; tries++) {
+            int activeBrokers = getNumberOfBrokers();
+            brokersAvailable = activeBrokers >= brokers;
+            if (brokersAvailable) {
+                logger.info("Kafka brokers available. Starting topic creation.");
+                break;
+            }
+
+            if (tries < 9) {
+                logger.warn("Only {} out of {} Kafka brokers available. Waiting {} seconds.",
+                        activeBrokers, brokers, sleep);
+                Thread.sleep(sleep * 1000L);
+                sleep = Math.min(MAX_SLEEP, sleep * 2);
+            } else {
+                logger.error("Only {} out of {} Kafka brokers available."
+                                + " Failed to wait on all brokers.",
+                        activeBrokers, brokers, sleep);
+            }
+        }
+        return brokersAvailable;
+    }
+
+    /**
+     * Create all topics in a catalogue.
+     * @param catalogue source catalogue to extract topic names from
+     * @param partitions number of partitions per topic
+     * @param replication number of replicas for a topic
+     * @return whether the whole catalogue was registered
+     */
     public boolean createTopics(SourceCatalogue catalogue, int partitions, int replication) {
         return catalogue.getTopicNames()
                 .allMatch(topic -> createTopic(topic, partitions, replication));
     }
 
+    /**
+     * Create a single topic.
+     * @param topic name of the topic to create
+     * @param partitions number of partitions per topic
+     * @param replication number of replicas for a topic
+     * @return whether the topic was registered
+     */
     public boolean createTopic(String topic, int partitions, int replication) {
         Properties props = new Properties();
         try {
@@ -79,6 +133,9 @@ public void close() {
         zkUtils.close();
     }
 
+    /**
+     * Create a KafkaTopics command to register topics from the command line.
+     */
     public static SubCommand command() {
         return new KafkaTopicsCommand();
     }
@@ -103,7 +160,7 @@ public int execute(Namespace options, CommandLineApp app) {
             int partitions = options.getInt("partitions");
             String zookeeper = options.getString("zookeeper");
             try (KafkaTopics topics = new KafkaTopics(zookeeper)) {
-                if (!waitForBrokers(topics, brokers)) {
+                if (!topics.waitForBrokers(brokers)) {
                     logger.error("Kafka brokers not yet available. Aborting.");
                     return 1;
                 }
@@ -135,25 +192,6 @@ public int execute(Namespace options, CommandLineApp app) {
             }
         }
 
-        private boolean waitForBrokers(KafkaTopics topics, int brokers)
-                throws InterruptedException, KeeperException {
-            boolean brokersAvailable = false;
-            int sleep = 2;
-            for (int tries = 0; tries < 10; tries++) {
-                int activeBrokers = topics.getNumberOfBrokers();
-                brokersAvailable = activeBrokers >= brokers;
-                if (brokersAvailable) {
-                    logger.info("Kafka brokers available. Starting topic creation.");
-                    break;
-                }
-                logger.warn("Only {} out of {} Kafka brokers available. Waiting {} seconds.",
-                        activeBrokers, brokers, sleep);
-                Thread.sleep(sleep * 1000L);
-                sleep = Math.min(MAX_SLEEP, sleep * 2);
-            }
-            return brokersAvailable;
-        }
-
         @Override
         public void addParser(ArgumentParser parser) {
             parser.description("Create all topics that are missing on the Kafka server.");

specifications/stream/android_phone.yml

@@ -3,13 +3,13 @@ doc: Aggregation of Phone data
 master: .stream.phone.PhoneStreamMaster
 data:
   - input_topic: android_phone_acceleration
-    value_schema: .stream.aggregator.DoubleArrayAggregation
+    value_schema: .stream.aggregator.AggregateList
     windowed: true
   - input_topic: android_phone_usage_event
     value_schema: .passive.phone.PhoneUsageEvent
   - input_topic: android_phone_usage_event_output
-    value_schema: .stream.aggregator.PhoneUsageAggregation
+    value_schema: .stream.aggregator.PhoneUsageAggregate
     windowed: true
   - input_topic: android_phone_battery_level
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true

specifications/stream/biovotion_vsm1.yml

@@ -3,38 +3,38 @@ doc: Aggregation of Biovotion data
 master: .kafka.stream.biovotion.BiovotionVsm1Master
 data:
   - input_topic: android_biovotion_vsm1_acceleration
-    value_schema: .stream.aggregator.DoubleArrayAggregation
+    value_schema: .stream.aggregator.AggregateList
     windowed: true
   - input_topic: android_biovotion_vsm1_battery_level
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_blood_volume_pulse
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_energy
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_galvanic_skin_response
-    value_schema: .stream.aggregator.DoubleArrayAggregation
+    value_schema: .stream.aggregator.AggregateList
     windowed: true
   - input_topic: android_biovotion_vsm1_heartrate
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_heartrate_variability
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_led_current
-    value_schema: .stream.aggregator.DoubleArrayAggregation
+    value_schema: .stream.aggregator.AggregateList
     windowed: true
   - input_topic: android_biovotion_vsm1_ppg_raw
-    value_schema: .stream.aggregator.DoubleArrayAggregation
+    value_schema: .stream.aggregator.AggregateList
     windowed: true
   - input_topic: android_biovotion_vsm1_respiration_rate
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_oxygen_saturation
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true
   - input_topic: android_biovotion_vsm1_temperature
-    value_schema: .stream.aggregator.DoubleAggregation
+    value_schema: .stream.aggregator.NumericAggregate
     windowed: true