Merge pull request #63 from RADAR-CNS/v0.2_release

V0.2 release

.editorconfig

@@ -17,3 +17,7 @@ trim_trailing_whitespace = false
 # Gradle
 [*.gradle]
 indent_size = 4
+
+# Java
+[*.java]
+indent_size = 4

.gitignore

@@ -1,7 +1,10 @@
 .DS_Store
 build/
+out/
 .idea/
+*.iml
 .gradle/
-*.java
 .gradletasknamecache
 gradle.properties
+exchange.properties
+*.class

.travis.yml

@@ -5,19 +5,28 @@ env:
   - TERM=dumb
 
 deploy:
-  provider: releases
-  api_key: ${GH_TOKEN}
-  file_glob: true
-  file: "*/build/libs/*.jar"
-  skip_cleanup: true
-  on:
-    tags: true
+  - provider: releases
+    api_key: ${GH_TOKEN}
+    file_glob: true
+    file:
+      - "*/build/libs/*.jar"
+      - "radar-schemas-tools/build/distributions/*"
+    skip_cleanup: true
+    on:
+      tags: true
+  - provider: script
+    script: ./gradlew artifactoryPublish
+    skip_cleanup: true
+    on:
+      branch: dev
+  - provider: script
+    script: ./gradlew bintrayUpload
+    skip_cleanup: true
+    on:
+      tags: true
 
 before_install:
   - cd java-sdk
 
 before_deploy:
   - pwd
-
-after_deploy:
-  - ./gradlew bintrayUpload

Dockerfile

@@ -0,0 +1,27 @@
+FROM openjdk:8-jdk-alpine
+
+RUN mkdir -p /code/java-sdk
+WORKDIR /code/java-sdk
+
+COPY commons /code/commons
+COPY specifications /code/specifications
+COPY restapi /code/restapi
+COPY java-sdk/gradle /code/java-sdk/gradle
+COPY java-sdk/gradle.properties java-sdk/build.gradle java-sdk/settings.gradle  java-sdk/gradlew /code/java-sdk/
+RUN ./gradlew tasks
+COPY java-sdk/radar-schemas-commons/build.gradle /code/java-sdk/radar-schemas-commons/
+COPY java-sdk/radar-schemas-commons/src /code/java-sdk/radar-schemas-commons/src
+RUN ./gradlew :radar-schemas-commons:classes
+COPY java-sdk/radar-schemas-tools/build.gradle /code/java-sdk/radar-schemas-tools/
+COPY java-sdk/radar-schemas-tools/src /code/java-sdk/radar-schemas-tools/src
+RUN ./gradlew distTar && cd radar-schemas-tools/build/distributions && tar xzf radar-schemas-tools*.tar.gz
+
+FROM openjdk:8-jre-alpine
+
+COPY --from=0 /code/java-sdk/radar-schemas-tools/build/distributions/radar-schemas-tools-*/lib/* /usr/lib/
+COPY --from=0 /code/java-sdk/radar-schemas-tools/build/distributions/radar-schemas-tools-*/bin/radar-schemas-tools /usr/bin/
+
+WORKDIR /schemas
+VOLUME /schemas
+
+CMD ["radar-schemas-tools", "-h"]

README.md

@@ -2,9 +2,18 @@
 
 [![Build Status](https://travis-ci.org/RADAR-CNS/RADAR-Schemas.svg?branch=master)](https://travis-ci.org/RADAR-CNS/RADAR-Schemas)
 
-[Avro schemas](https://avro.apache.org/docs/1.8.1/spec.html) used in RADAR-CNS. The schemas are divided into three parts: `commons` for the passive remote monitoring application and the backend, `restapi` for the REST API, and `questionnaire` for the active remote monitoring application.
+[Avro schemas](https://avro.apache.org/docs/1.8.2/spec.html) used in RADAR-CNS. The schemas are organized as follows:
 
-Java SDKs for each of the components are provided in the `java-sdk` folder, see installation instructions there. They are automatically generated from the Avro schemas using the Avro 1.8.1 specification.
+- The `commons` directory contains all schemas used inside Kafka and data fed into Kafka.
+  - In the `active` subdirectory, add schemas for active data collection, like questionnaires or assignments.
+  - In the `catalogue` subdirectory, modify schemas for cataloguing data types.
+  - In the `kafka` subdirectory, add schemas used throughtout Kafka, like record keys.
+  - In the `monitor` subdirectory, add schemas for monitoring applications that gather data.
+  - In the `passive` subdirectory, add schemas for passive data collection, like wearables.
+  - In the `stream` subdirectory, add schemas used in Kafka Streams.
+- The `restapi` directory contains schemas used to get data from the RADAR-CNS REST API.
+- The `specifications` directory contains specifications of what data types are collected through which devices.
+- Java SDKs for each of the components are provided in the `java-sdk` folder, see installation instructions there. They are automatically generated from the Avro schemas using the Avro 1.8.2 specification.
 
 ## Contributing
 
@@ -15,12 +24,38 @@ In addition, schemas in the `commons` directory should follow the following guid
 - Try to avoid abbreviations in the field names and write out the field name instead.
 - There should be no need to add `value` at the end of a field name.
 - Enumerator items should be written in uppercase characters separated by underscores.
-- Add documentation (the `doc` property) to each schema, each field, and each enum. The documentation should show in text what is being measured, how, and what units or ranges are applicable. Abbreviations and acronyms in the documentation should be written out.
+- Add documentation (the `doc` property) to each schema, each field, and each enum. The documentation should show in text what is being measured, how, and what units or ranges are applicable. Abbreviations and acronyms in the documentation should be written out. Each doc property should start with a capital and end with a period.
 - Prefer a categorical specification (an Avro enum) over a free string if that categorization is expected to remain very stable. This disambiguates the possible values for analysis. If a field is expected to be extended outside this project or very often within this project, use a free string instead.
 - Prefer a flat record over a hierarchical record. This simplifies the organization of the data downstream, for example, when mapping to CSV.
 - Prefer written out fields to arrays. This simplifies the organization of the data downstream, for example, when mapping to CSV.
-- Give each schema a proper namespace, preferably `org.radarcns.deviceproducer` fully in lowercase, without any numbers, uppercase letters or symbols (except `.`). For the Empatica E4, the producer is Empatica, so the namespace is `org.radarcns.empatica`. For generic types, like a phone, Android Wear device or Android application, the namespace could just be `org.radarcns.phone`, `org.radarcns.wear`, or `org.radarcns.application`.
+- Give each schema a proper namespace, preferably `org.radarcns.passive.<vendor>` fully in lowercase, without any numbers, uppercase letters or symbols (except `.`). For the Empatica E4, the vendor is Empatica, so the namespace is `org.radarcns.passive.empatica`. For generic types, like a phone, Android Wear device or Android application, the namespace could just be `org.radarcns.passive.phone`, `org.radarcns.passive.wear`, or `org.radarcns.monitor.application`.
 - In the schema name, use upper camel case and name the device explicitly (for example, `EmpaticaE4Temperature`).
 - For fields that are inherent to a record, and will never be removed or renamed, no default value is needed. For all other fields:
   - if the type is an enum, use an `UNKNOWN` symbol as default value
   - otherwise, set the type to a union of `["null", <intended type>]` and set the default value to `null`.
+
+### Validation phase
+
+Avro schemas are automatically validated against RADAR-CNS guide lines while building. For more details, check [catalog validator](java-sdk/radar-schemas-tools).
+
+### Test setup
+
+The RADAR schema tools can be tested locally using Docker. To run the tools, first install Docker. Then run
+
+```shell
+docker-compose build
+docker-compose up -d zookeeper-1 kafka-1 schema-registry-1
+```
+Now you can run tools commands with
+```shell
+# usage
+docker-compose run --rm tools
+# validation
+docker-compose run --rm tools radar-schemas-tools validate
+# list topic information
+docker-compose run --rm tools radar-schemas-tools list
+# register schemas with the schema registry
+docker-compose run --rm tools radar-schemas-tools register http://schema-registry:8081
+# create topics with zookeeper
+docker-compose run --rm tools radar-schemas-tools create zookeeper-1:2181
+```

commons/README.md

@@ -0,0 +1,9 @@
+# Commons
+
+This folder is organised as follow
+- [active](active)
+- [kafka](kafka)
+- [monitor](monitor)
+- [passive](passive)
+
+WIP

commons/active/questionnaire/questionnaire.avsc

@@ -0,0 +1,24 @@
+{
+  "namespace": "org.radarcns.active.questionnaire",
+  "type": "record",
+  "name": "Questionnaire",
+  "doc": "General schema for questionnaire. Check the specification folder to see how the questionnaire has been defined. For each QuestionnaireType there is a JSON file reporting the questions set and possible answers if available.",
+  "fields": [
+    { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the questionnaire is submitted to the subject." },
+    { "name": "timeCompleted", "type": "double", "doc": "Timestamp in UTC (s) when subject marks the questionnaire as." },
+    { "name": "name", "type": "string", "doc": "Questionnaire names." },
+    { "name": "version", "type": "string", "doc": "It reports the questionnaire version stated in the JSON specification." },
+    { "name": "answers", "type": {
+      "type": "array",
+      "items": {
+        "name": "Answer",
+        "type": "record",
+        "doc": "Questionnaire answer.",
+         "fields": [
+           { "name": "value", "type": ["int", "string", "double"], "doc": "Subject answer." },
+           { "name": "startTime", "type": "double", "doc": "Timestamp in UTC (s) when the question is shown." },
+           { "name": "endTime", "type": "double", "doc": "Timestamp in UTC (s)  when the question is answered." }
+         ]
+       }}, "doc": "Answers list. The answers order must follow the questions order."}
+  ]
+}

commons/active/thincit/thinc_it_code_breaker.avsc

@@ -0,0 +1,32 @@
+{
+  "namespace": "org.radarcns.active.thincit",
+  "type": "record",
+  "name": "ThincItCodeBreaker",
+  "doc": "THINC-it Assessment derived from Digit Symbol Substitution Test. As many trials as can be completed within 2 minutes. Each trial is a symbol/number to be matched.",
+  "fields": [
+    { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the test is submitted to the subject." },
+    { "name": "timeCompleted", "type": "double", "doc": "Timestamp in UTC (s) when the subject completes the test." },
+    { "name": "score", "type": "int", "doc": "THINC-it index score - 0 to 4000." },
+    { "name": "completed", "type": "int", "doc": "The number of completed trials." },
+    { "name": "incorrect", "type": "int", "doc": "The number of incorrect responses." },
+    { "name": "longestTrial", "type": "int", "doc": "Longest trial completion time expressed in milliseconds." },
+    { "name": "iqrCompletion", "type": "int", "doc": "Mean of trial completion times within interquartile range." },
+    { "name": "inputType", "type": "ThincItInputType", "doc": "Dominant input type used by the subject to answer.", "default": "UNKNOWN" },
+    { "name": "quarterSummaryOne", "type":
+      {
+        "type": "record",
+        "name": "ThincItCodeBreakerQuarterSummary",
+        "doc": "Trial summary for CodeBreaker assessmen.",
+        "fields": [
+          { "name": "longestTrial", "type": "int", "doc": "Longest trial completion time expressed in milliseconds." },
+          { "name": "meanTime", "type": "int", "doc": "Mean trial completion time expressed in milliseconds." },
+          { "name": "incorrect", "type": "int", "doc": "The number of incorrect responses." },
+          { "name": "inputType", "type": "ThincItInputType", "doc": "Dominant input type used by the subject to answer.", "default": "UNKNOWN" }
+        ]
+      }, "doc": "Trial summary for CodeBreaker assessment Quarter 1."},
+    { "name": "quarterSummaryTwo", "type": "ThincItCodeBreakerQuarterSummary", "doc": "Trial summary for CodeBreaker assessment Quarter 2."},
+    { "name": "quarterSummaryThree", "type": "ThincItCodeBreakerQuarterSummary", "doc": "Trial summary for CodeBreaker assessment Quarter 3."},
+    { "name": "quarterSummaryFour", "type": "ThincItCodeBreakerQuarterSummary", "doc": "Trial summary for CodeBreaker assessment Quarter 4."},
+	  { "name": "appVersion", "type": "int", "doc": "App version." }
+  ]
+}

commons/active/thincit/thinc_it_input_type.avsc

@@ -0,0 +1,12 @@
+{
+  "namespace": "org.radarcns.active.thincit",
+  "type": "enum",
+  "name": "ThincItInputType",
+  "doc": "Possible input type that a subject can use to do an assignment.",
+  "symbols": [
+    "KEYBOARD",
+    "MOUSE",
+    "TOUCHSCREEN",
+    "UNKNOWN"
+   ]
+}

commons/active/thincit/thinc_it_spotter.avsc

@@ -0,0 +1,31 @@
+{
+  "namespace": "org.radarcns.active.thincit",
+  "type": "record",
+  "name": "ThincItSpotter",
+  "doc": "THINC-it Assessment derived from Choice Reaction Time test. 40 trials, or 2 minutes max. Each trial is the presentation of a left or right facing arrow.",
+  "fields": [
+    { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the test is submitted to the subject." },
+    { "name": "timeCompleted", "type": "double", "doc": "Timestamp in UTC (s) when the subject completes the test." },
+    { "name": "score", "type": "int", "doc": "THINC-it index score - 0 to 4000." },
+    { "name": "meanLatency", "type": "int", "doc": "Mean latency of correct responses expressed in milliseconds." },
+    { "name": "timeouts", "type": "int", "doc": "The number of times that a subject failed to input a response within the time limit." },
+    { "name": "anticipations", "type": "int", "doc": "The number of responses within 100ms of stimulus presentation." },
+    { "name": "incorrect", "type": "int", "doc": "The number of incorrect responses." },
+    { "name": "inputType", "type": "ThincItInputType", "doc": "Dominant input type used by the subject to answer.", "default": "UNKNOWN" },
+    { "name": "quarterSummaryOne", "type": {
+      "type": "record",
+      "name": "ThincItSpotterQuarterSummary",
+      "doc": "Trial summary for SymbolCheck assessment.",
+      "fields": [
+        { "name": "timeouts", "type": "int", "doc": "The number of times that a subject failed to input a response within the time limit." },
+        { "name": "anticipations", "type": "int", "doc": "The number of responses within 100ms of stimulus presentation." },
+        { "name": "incorrect", "type": "int", "doc": "The number of incorrect response." },
+        { "name": "inputType", "type": "ThincItInputType", "doc": "Dominant input type used by the subject to answer.", "default": "UNKNOWN" }
+      ]
+    } , "doc": "Trial summary for SymbolCheck assessment Quarter 1."},
+    { "name": "quarterSummaryTwo", "type": "ThincItSpotterQuarterSummary", "doc": "Trial summary for SymbolCheck assessment Quarter 2."},
+    { "name": "quarterSummaryThree", "type": "ThincItSpotterQuarterSummary", "doc": "Trial summary for SymbolCheck assessment Quarter 3."},
+    { "name": "quarterSummaryFour", "type": "ThincItSpotterQuarterSummary", "doc": "Trial summary for SymbolCheck assessment Quarter 4."},
+	  { "name": "appVersion", "type": "int", "doc": "App version." }
+  ]
+}

commons/active/thincit/thinc_it_symbol_check.avsc

@@ -0,0 +1,31 @@
+{
+  "namespace": "org.radarcns.active.thincit",
+  "type": "record",
+  "name": "ThincItSymbolCheck",
+  "doc": "THINC-it Assessment derived from NBack. 40 trials, or 2 minutes max. Each obscured symbol to be recalled is a trial.",
+  "fields": [
+    { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the test is submitted to the subject." },
+    { "name": "timeCompleted", "type": "double", "doc": "Timestamp in UTC (s) when the subject completes the test." },
+    { "name": "score", "type": "int", "doc": "THINC-it index score - 0 to 4000." },
+    { "name": "completed", "type": "int", "doc": "The number of completed trials." },
+    { "name": "correct", "type": "int", "doc": "The number of correct responses." },
+    { "name": "incorrect", "type": "int", "doc": "The number of incorrect responses." },
+    { "name": "timeouts", "type": "int", "doc": "The number of times that a subject failed to input a response within the time limit." },
+    { "name": "inputType", "type": "ThincItInputType", "doc": "Dominant input type used by the subject to answer.", "default": "UNKNOWN" },
+    { "name": "quarterSummaryOne", "type": {
+      "type": "record",
+      "name": "ThincItSymbolCheckQuarterSummary",
+      "doc": "Trial summary for SymbolCheck assessmen.",
+      "fields": [
+        { "name": "correct", "type": "int", "doc": "The amount of correct responses." },
+        { "name": "incorrect", "type": "int", "doc": "The number of incorrect responses." },
+        { "name": "timeouts", "type": "int", "doc": "The number of times that a subject failed to input a response within the time limit." },
+        { "name": "inputType", "type": "ThincItInputType", "doc": "Dominant input type used by the subject to answer.", "default": "UNKNOWN" }
+      ]
+    } , "doc": "Trial summary for SymbolCheck assessment Quarter 1."},
+    { "name": "quarterSummaryTwo", "type": "ThincItSymbolCheckQuarterSummary", "doc": "Trial summary for SymbolCheck assessment Quarter 2."},
+    { "name": "quarterSummaryThree", "type": "ThincItSymbolCheckQuarterSummary", "doc": "Trial summary for SymbolCheck assessment Quarter 3."},
+    { "name": "quarterSummaryFour", "type": "ThincItSymbolCheckQuarterSummary", "doc": "Trial summary for SymbolCheck assessment Quarter 4."},
+	  { "name": "appVersion", "type": "int", "doc": "App version." }
+  ]
+}

commons/active/thincit/thinc_it_trails.avsc

@@ -0,0 +1,35 @@
+{
+  "namespace": "org.radarcns.active.thincit",
+  "type": "record",
+  "name": "ThincItTrails",
+  "doc": "THINC-it Assessment derived from Trail Making B. 17 trials. Each trial is the transition from one circle to another.",
+  "fields": [
+    { "name": "time", "type": "double", "doc": "Timestamp in UTC (s) when the test is submitted to the subject." },
+    { "name": "timeCompleted", "type": "double", "doc": "Timestamp in UTC (s) when the subject completes the test." },
+    { "name": "score", "type": "int", "doc": "THINC-it index score - 0 to 4000." },
+    { "name": "duration", "type": "int", "doc": "Total time taken to complete the assessment in milliseconds." },
+    { "name": "layoutVersion", "type": "int", "doc": "Each play randomly select one of 40 pre-designed layouts." },
+    { "name": "wrongNode", "type": "int", "doc": "The number of times that the subject transitioned to an incorrect next circle before successful transition to correct next circle." },
+    { "name": "lostContact", "type": "int", "doc": "The number of times that the mouse button has been released before transitioning successfully to next circle." },
+    { "name": "longestTrial", "type": "int", "doc": "Longest trial completion time expressed in milliseconds." },
+    { "name": "iqrCompletion", "type": "int", "doc": "Mean of trial completion times within interquartile range." },
+    { "name": "intervalOne", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalTwo", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalThree", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalFour", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalFive", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalSix", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalSeven", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalEight", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalNine", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalTen", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalEleven", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalTwelve", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalThirteen", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalFourteen", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalFifteen", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalSixteen", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "intervalSeventeen", "type": "int", "doc": "Trial completion time in milliseconds." },
+    { "name": "appVersion", "type": "int", "doc": "App version." }
+  ]
+}