diff --git a/.gitignore b/.gitignore index 0df21acd..e194a2f7 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,7 @@ smtp.env /libs/ # Distribution -/radar-backend-* \ No newline at end of file +/radar-backend-* +/src/integrationTest/docker/etc/google-credentials.json + +temp \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 6524605c..c89e43df 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,8 +35,6 @@ script: - sudo docker-compose up -d zookeeper-1 kafka-1 schema-registry-1 && sleep 30 && sudo docker-compose run --rm integration-test - sudo docker-compose down - cd ../../.. -after_script: - - ./gradlew sendCoverageToCodacy deploy: provider: releases @@ -49,6 +47,3 @@ deploy: skip_cleanup: true on: tags: true - -after_deploy: - - ./gradlew bintrayUpload diff --git a/Dockerfile b/Dockerfile index 26e68b3c..dbda1adc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,44 +10,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM gradle:6.6.1-jdk11 as builder +FROM gradle:7.4-jdk17 as builder RUN mkdir /code WORKDIR /code -ENV GRADLE_OPTS=-Dorg.gradle.project.profile=prod \ +ENV GRADLE_OPTS="-Dorg.gradle.project.profile=prod -Djdk.lang.Process.launchMechanism=vfork" \ GRADLE_USER_HOME=/code/.gradlecache COPY ./gradle/profile.prod.gradle /code/gradle/ COPY ./build.gradle ./gradle.properties ./settings.gradle /code/ -RUN gradle downloadRuntimeDependencies +RUN gradle downloadRuntimeDependencies copyDependencies startScripts COPY ./src/ /code/src -RUN gradle distTar && \ - tar xf build/distributions/*.tar && \ - rm build/distributions/*.tar +RUN gradle jar -FROM openjdk:11-jre-slim +FROM azul/zulu-openjdk-alpine:17-jre-headless MAINTAINER Nivethika M , Joris Borgdorff , Yatharth Ranjan LABEL description="RADAR-CNS Backend streams and monitor" -RUN apt-get update && apt-get install -y --no-install-recommends \ - curl \ - wget \ - && rm -rf /var/lib/apt/lists/* +RUN apk add --no-cache curl + +RUN mkdir -p /var/lib/radar/data +RUN chown 101:101 /var/lib/radar/data -ENV KAFKA_REST_PROXY http://rest-proxy:8082 ENV KAFKA_SCHEMA_REGISTRY http://schema-registry:8081 ENV RADAR_BACKEND_CONFIG /etc/radar.yml -COPY --from=builder /code/radar-backend-*/bin/* /usr/bin/ -COPY --from=builder /code/radar-backend-*/lib/* /usr/lib/ +COPY --from=builder /code/build/third-party/* /usr/lib/ +COPY --from=builder /code/build/scripts/* /usr/bin/ +COPY --from=builder /code/build/libs/* /usr/lib/ # Load topics validator COPY ./src/main/docker/radar-backend-init /usr/bin +USER 101:101 + ENTRYPOINT ["radar-backend-init"] diff --git a/README.md b/README.md index faa45b80..9f6f4b63 100644 --- a/README.md +++ b/README.md @@ -3,23 +3,32 @@ [![Build Status](https://travis-ci.org/RADAR-base/RADAR-Backend.svg?branch=master)](https://travis-ci.org/RADAR-base/RADAR-Backend) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/e21c0c25f43e4676a3c69bae444101ca)](https://www.codacy.com/app/RADAR-base/RADAR-Backend?utm_source=github.com&utm_medium=referral&utm_content=RADAR-CNS/RADAR-Backend&utm_campaign=Badge_Grade) -RADAR-Backend is a Java application based on Confluent Platform to standardize, analyze and persist data collected by RADAR-CNS data sources. It supports the backend requirements of RADAR-CNS project. The data is produced and consumed in Apache Avro format using the schema stored inside the RADAR-CNS [schema repository](https://github.com/RADAR-base/RADAR-Schemas). - -RADAR-Backend provides an abstract layer to monitor and analyze streams of wearable data and write data to Hot or Cold storage. The Application Programming Interfaces (APIs) of RADAR-Backend makes the process of to integrating additional topics, wearable devices easier. It currently provides MongoDB as the Hot-storage and HDFS data store as the Cold-storage. They can be easily tuned using property files. The stream-monitors monitor topics and notify users (e.g. via emails) under given circumstances. +RADAR-Backend is a Java application based on Confluent Platform to standardize, analyze and persist +data collected by RADAR-CNS data sources. It supports the backend requirements of RADAR-CNS project. +The data is produced and consumed in Apache Avro format using the schema stored inside the +RADAR-CNS [schema repository](https://github.com/RADAR-base/RADAR-Schemas). + +RADAR-Backend provides an abstract layer to monitor and analyze streams of wearable data and write +data to Hot or Cold storage. The Application Programming Interfaces (APIs) of RADAR-Backend makes +the process of to integrating additional topics, wearable devices easier. It currently provides +MongoDB as the Hot-storage and HDFS data store as the Cold-storage. They can be easily tuned using +property files. The stream-monitors monitor topics and notify users (e.g. via emails) under given +circumstances. ## Dependencies The following are the prerequisites to run RADAR-Backend on your machine: - Java 8 -- [Confluent Platform 5.0.0](http://docs.confluent.io/5.0.0/installation.html) ( Running instances of Zookeeper, Kafka-broker(s), Schema-Registry and Kafka-REST-Proxy services ). +- [Confluent Platform 5.0.0](http://docs.confluent.io/5.0.0/installation.html) ( Running instances + of Zookeeper, Kafka-broker(s), Schema-Registry and Kafka-REST-Proxy services ). - SMTP server to send notifications from the monitors. ## Installation 1. Install the dependencies mentioned above. 2. Clone radar-backend repository. - + ```shell git clone https://github.com/RADAR-base/radar-backend.git ``` @@ -39,28 +48,39 @@ The following are the prerequisites to run RADAR-Backend on your machine: sudo mkdir -p /usr/local && \ sudo tar --strip-components 1 -C /usr/local xzf build/distributions/*.tar.gz ``` - + Now the backend is available as the `/usr/local/bin/radar-backend` script. ## Usage -The RADAR command-line has three subcommands: `stream`, `monitor` and `mock`. The `stream` command will start all streams, the `monitor command` will start all monitors, and the `mock` command will send mock data to the backend. Before any of these commands are issued, start the Confluent platform with the zookeeper, kafka, schema-registry and rest-proxy components. Put the `build/libs/radarbackend-1.0.jar` and `radar.yml` in the same folder, and then modify `radar.yml`: +The RADAR command-line has three subcommands: `stream`, `monitor` and `mock`. The `stream` command +will start all streams, the `monitor command` will start all monitors, and the `mock` command will +send mock data to the backend. Before any of these commands are issued, start the Confluent platform +with the zookeeper, kafka, schema-registry and rest-proxy components. Put +the `build/libs/radarbackend-1.0.jar` and `radar.yml` in the same folder, and then +modify `radar.yml`: ### RADAR-Backend streams -1. In `radar.yml`, Specify in which `mode` you want to run the application. There are two alternatives: `standalone` and `high_performance`. The `standalone` starts one thread for each streams without checking the priority, whereas the `high_performance` starts as many thread as the related priority value -2. If `auto.create.topics.enable` is `false` in your Kafka `server.properties`, before starting you must create the topics manually. The stream server will print what topics to create. -3. Run `radar-backend` with configured `radar.yml` and `stream` argument +1. In `radar.yml`, Specify in which `mode` you want to run the application. There are two + alternatives: `standalone` and `high_performance`. The `standalone` starts one thread for each + streams without checking the priority, whereas the `high_performance` starts as many thread as + the related priority value +2. If `auto.create.topics.enable` is `false` in your Kafka `server.properties`, before starting you + must create the topics manually. The stream server will print what topics to create. +3. Run `radar-backend` with configured `radar.yml` and `stream` argument ```shell radar-backend -c path/to/radar.yml stream ``` -The phone usage event stream uses an internal cache of 1 million elements, which may take about 50 MB of memory. Adjust `org.radarcns.stream.phone.PhoneUsageStream.MAX_CACHE_SIZE` to change it. +The phone usage event stream uses an internal cache of 1 million elements, which may take about 50 +MB of memory. Adjust `org.radarcns.stream.phone.PhoneUsageStream.MAX_CACHE_SIZE` to change it. ### RADAR-backend monitors -To get email notifications for Empatica E4 battery status, an email server without a password set up, for example on `localhost`. +To get email notifications for Empatica E4 battery status, an email server without a password set +up, for example on `localhost`. 1. For battery status monitor, configure the following @@ -108,8 +128,9 @@ To get email notifications for Empatica E4 battery status, an email server witho - android_empatica_e4_temperature ``` -3. For Source Statistics monitors, configure what source topics to monitor to output some basic output statistics (like last time seen) - +3. For Source Statistics monitors, configure what source topics to monitor to output some basic + output statistics (like last time seen) + ```yaml stream: statistics_monitors: @@ -142,15 +163,84 @@ To get email notifications for Empatica E4 battery status, an email server witho - android_phone_sms output_topic: source_statistics_radar_prmt ``` - + 3. Run `radar-backend` with configured `radar.yml` and `monitor` argument ```shell radar-backend -c path/to/radar.yml monitor ``` +### RADAR-backend Realtime Inference Consumers + +Realtime Consumers are consumers for the results from the invocation of a model from the realtime +analysis pipeline ( +see [model-builder](https://github.com/RADAR-base/model-builder/tree/dev/model-builder) +, [model-invocation-endpoint](https://github.com/RADAR-base/model-builder/tree/dev/model-invocation-endpoint) +and [ksqldb](https://github.com/RADAR-base/ksql-extras)). The results from the pipeline are in JSON +and the invocation is called by +a [KSQL function](https://github.com/RADAR-base/ksql-extras/blob/main/src/main/java/org/radarbase/ksql/udf/RestInferenceUdf.java) +which gets inference results then puts the results in a kafka topic. These consumers are meant to +consume data from this topic. + +3 terms used - + +1. `Action`: Any task to be performed is an action. This can be sending a notification or an email + or just logging something. +2. `Condition`: This is a predicate that will be evaluated on the incoming data. For instance, this + can be used to evaluate if the ML model run crossed a threshold of some sort. +3. `RealtimeInferenceConsumer`: This is the base consumer for realtime results topics. Each consumer + can be configured with a set of Conditions and Actions. If all the conditions evaluate to be + true, then all the actions are triggered. + +Currently, only one Condition is provided which evaluates +a [JSONPath](https://github.com/json-path/JsonPath) expression. This should be sufficient for most +simple use cases. More complex use cases can create concrete implementations of the `Condition` +interface or `ConditionBase` abstract class. Currently, the JsonPath expression is read from the +configuration file and hence is static for a particular consumer and topic. Later, we can also +provide this through AppConfig so it can be dynamic based on userId or projectId. + +The supported config (for instance an intervention using aRMT app) is of the format - + +```yaml +realtime-consumers: + - name: 'lstm-lung-study-consumer' # Name of the consumer + topic: 'lung_study_lstm_ad_inference' # Kafka topic to consume from (this should contain the inference results) + notify_errors: + email_addresses: + - 'admin@radar-base.org' + conditions: + - name: 'LocalJsonPathCondition' # Name of the condition + projects: [ 'radar-test' ] # Only evaluate for these projects + subjects: [ 'sub-1', 'sub-2' ] # Only evaluate for these subjects + properties: + jsonpath: '$[?(@.invocation_result.anomaly_detected == true)]' # JsonPath expression to evaluate + key: 'invocation_result' # Key that contains data to evaluate + actions: + - name: 'ActiveAppNotificationAction' # Name of the action + projects: [ 'radar-test' ] # Only execute for these projects + subjects: [ 'sub-1', 'sub-2' ] # Only execute for these subjects + properties: + questionnaire_name: 'ers' # Name of the questionnaire to trigger + time_of_day: '09:00:00' # Local user time of day to trigger at + default_timezone: 'Europe/London' # Default timezone to use for the time of day if not found in the appserver + appserver_base_url: 'http://localhost:8080/' # Base URL of the appserver + management_portal_token_url: 'http://localhost/managementportal/api/oauth/token' # URL to get the management portal token + client_id: 'realtime_consumer' # Client ID for the management portal + client_secret: 'secret' # Client secret for the management portal + metadata_key: 'invocation_result' # Key that contains the metdata to be forwarded to the aRMT app + - name: 'EmailUserAction' + projects: [ 'radar-test' ] + subjects: [ 'sub-1', 'sub-2' ] + properties: + email_addresses: [ 'admin@radar-base.org' ] +``` + +Note: The `properties` section is specific to each Action and Condition. Please take a look at the +condition and action docs for the keys supported. If the `projects` or `subjects` key is not +specified the action and condition will be used on all projects or subjects respectively. + ### Send mock data to the backend - + 1. Configure the REST proxy setting in `radar.yml`: ```yaml @@ -170,21 +260,24 @@ To get email notifications for Empatica E4 battery status, an email server witho value_schema: org.radarcns.passive.empatica.EmpaticaE4Acceleration ``` - Each value has a topic to send the data to, a file containing the data, a schema class for the key and a schema class for the value. Also create a CSV file for each of these entries: + Each value has a topic to send the data to, a file containing the data, a schema class for the + key and a schema class for the value. Also create a CSV file for each of these entries: ```csv userId,sourceId,time,timeReceived,acceleration a,b,14191933191.223,14191933193.223,[0.001;0.3222;0.6342] a,c,14191933194.223,14191933195.223,[0.13131;0.6241;0.2423] ``` - Note that for array entries, use brackets (`[` and `]`) to enclose the values and use `;` as a delimiter. + Note that for array entries, use brackets (`[` and `]`) to enclose the values and use `;` as a + delimiter. -3. To generate data on some `backend_mock_empatica_e4_<>` topic with a number of devices, run (substitute `` with the needed number of devices): +3. To generate data on some `backend_mock_empatica_e4_<>` topic with a number of devices, run ( + substitute `` with the needed number of devices): ```shell radar-backend -c path/to/radar.yml mock --devices ``` - Press `Ctrl-C` to stop. + Press `Ctrl-C` to stop. 4. To generate the file data configured in point 2, run @@ -192,11 +285,12 @@ To get email notifications for Empatica E4 battery status, an email server witho radar-backend -c path/to/radar.yml mock --file mock_data.yml ``` - The data sending will automatically be stopped. + The data sending will automatically be stopped. ### Docker image -The backend is [published to Docker Hub](https://hub.docker.com/r/radarcns/radar-backend-kafka). Mount a `/etc/radar.yml` file to configure either the streams or the monitor. +The backend is [published to Docker Hub](https://hub.docker.com/r/radarcns/radar-backend-kafka). +Mount a `/etc/radar.yml` file to configure either the streams or the monitor. This image requires the following environment variable: @@ -204,30 +298,42 @@ This image requires the following environment variable: - `KAFKA_SCHEMA_REGISTRY`: a valid Confluent Schema Registry. - `KAFKA_BROKERS`: number of brokers expected (default: 3). -For a complete use case scenario, check the RADAR-base `docker-compose` file available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) - +For a complete use case scenario, check the RADAR-base `docker-compose` file +available [here](https://github.com/RADAR-base/RADAR-Docker/blob/backend-integration/dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml) + ## Contributing -Code should be formatted using the [Google Java Code Style Guide](https://google.github.io/styleguide/javaguide.html). -If you want to contribute a feature or fix browse our [issues](https://github.com/RADAR-base/RADAR-Backend/issues), and please make a pull request. +Code should be formatted using +the [Google Java Code Style Guide](https://google.github.io/styleguide/javaguide.html). If you want +to contribute a feature or fix browse +our [issues](https://github.com/RADAR-base/RADAR-Backend/issues), and please make a pull request. -There are currently two APIs in RADAR-Backend: one for streaming data (RADAR-Stream) and one for monitoring topics (RADAR-Monitor). To contribute to those APIs, please mind the following. +There are currently two APIs in RADAR-Backend: one for streaming data (RADAR-Stream) and one for +monitoring topics (RADAR-Monitor). To contribute to those APIs, please mind the following. ### Extending RADAR-Stream -RADAR-Stream is a layer on top of Kafka streams. Topics are processed by streams in two phases. First, a group of sensor streams aggregates data of sensors into predefined time windows (e.g., 10 seconds). Next, internal topics aggregate and transforms data that has already been processed by an earlier stream. +RADAR-Stream is a layer on top of Kafka streams. Topics are processed by streams in two phases. +First, a group of sensor streams aggregates data of sensors into predefined time windows (e.g., 10 +seconds). Next, internal topics aggregate and transforms data that has already been processed by an +earlier stream. -KafkaStreams currently communicates using master-slave model. The [StreamMaster][1] defines the stream-master, while [StreamWorker][2] represents the stream-slave. The master-stream creates, starts and stops a list of stream-slaves registered with the corresponding master. While the classical Kafka Consumer requires two implementations to support standalone and group executions, the StreamWorker provides both behaviors with one implementation. +KafkaStreams currently communicates using master-slave model. The [StreamMaster][1] defines the +stream-master, while [StreamWorker][2] represents the stream-slave. The master-stream creates, +starts and stops a list of stream-slaves registered with the corresponding master. While the +classical Kafka Consumer requires two implementations to support standalone and group executions, +the StreamWorker provides both behaviors with one implementation. -To extend the RADAR-Stream API, follow these steps (see the `org.radarcns.passive.empatica` package as an example): +To extend the RADAR-Stream API, follow these steps (see the `org.radarcns.passive.empatica` package +as an example): - For each topic, create a [StreamWorker][2] or more conveniently extend [SensorStreamWorker][6]. - Add the stream topic to the `stream: streams: [{class: MyClass}]` configuration - #### Empatica E4 -Currently, RADAR-Backend provides implementation to stream, monitor, store Empatica E4 topics data produced by RADAR-AndroidApplication. It defines the following streams: +Currently, RADAR-Backend provides implementation to stream, monitor, store Empatica E4 topics data +produced by RADAR-AndroidApplication. It defines the following streams: - [E4Acceleration][15] aggregates data coming from accelerometer - [E4BatteryLevel][16] aggregates battery level information @@ -240,7 +346,10 @@ And one internal topic: - [E4HeartRate][19]: starting from the inter-beat-interval, this aggregator computes the heart rate -[DeviceTimestampExtractor][10] implements a [TimestampExtractor](http://docs.confluent.io/5.0.0/streams/javadocs/index.html) such that: given in input a generic Apache Avro object, it extracts a field named `timeReceived`. [DeviceTimestampExtractor][10] works with the entire set of sensor schemas currently available. +[DeviceTimestampExtractor][10] implements +a [TimestampExtractor](http://docs.confluent.io/5.0.0/streams/javadocs/index.html) such that: given +in input a generic Apache Avro object, it extracts a field named `timeReceived` +. [DeviceTimestampExtractor][10] works with the entire set of sensor schemas currently available. #### Android Phone @@ -249,7 +358,10 @@ categories for app usage events. ### Extending RADAR-Monitor -Monitors can be used to evaluate the status of a single stream, for example whether each device is still online, has acceptable values and is transmitting at an acceptable rate. To create a new monitor, extend [AbstractKafkaMonitor][3]. To use the monitor from the command-line, modify [KafkaMonitorFactory][4]. See [DisconnectMonitor][5] for an example. +Monitors can be used to evaluate the status of a single stream, for example whether each device is +still online, has acceptable values and is transmitting at an acceptable rate. To create a new +monitor, extend [AbstractKafkaMonitor][3]. To use the monitor from the command-line, +modify [KafkaMonitorFactory][4]. See [DisconnectMonitor][5] for an example. ### NOTE @@ -262,16 +374,29 @@ Monitors can be used to evaluate the status of a single stream, for example whet - the default log path is the jar folder [1]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/StreamMaster.java + [2]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/StreamWorker.java + [3]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java + [4]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/monitor/KafkaMonitorFactory.java + [5]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/monitor/DisconnectMonitor.java + [6]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/SensorStreamWorker.java + [10]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/DeviceTimestampExtractor.java + [15]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4Acceleration.java + [16]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4BatteryLevel.java + [17]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4BloodVolumePulse.java + [18]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4ElectroDermalActivity.java + [19]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4HeartRate.java + [20]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4InterBeatInterval.java + [21]: https://github.com/RADAR-base/RADAR-Backend/blob/master/src/main/java/org/radarcns/stream/empatica/E4Temperature.java diff --git a/build.gradle b/build.gradle index def5c5ab..fa59b429 100644 --- a/build.gradle +++ b/build.gradle @@ -2,6 +2,7 @@ plugins { // Apply the java plugin to add support for Java id 'java' id 'application' + id 'org.jetbrains.kotlin.jvm' version '1.6.10' } //---------------------------------------------------------------------------// @@ -14,10 +15,6 @@ version = '0.4.1-SNAPSHOT' mainClassName = 'org.radarcns.RadarBackend' applicationDefaultJvmArgs = ["-Dlog4j.configuration=log4j.properties"] - -targetCompatibility = '11.0' -sourceCompatibility = '11.0' - ext { moduleDescription = 'Kafka backend for processing device data.' githubRepoName = 'RADAR-Base/RADAR-Backend' @@ -27,23 +24,23 @@ ext { issueUrl = 'https://github.com/' + githubRepoName + '/issues' website = 'http://radar-base.org' - codacyVersion = '4.0.2' - confluentVersion = '5.5.1' - hamcrestVersion = '1.3' - kafkaVersion = '2.5.0' - jacksonVersion = '2.11.2' + confluentVersion = '7.0.1' + hamcrestVersion = '2.2' + kafkaVersion = '3.0.0' + jacksonVersion = '2.13.1' javaMailVersion = '1.6.2' - junitVersion = '4.12' + junitVersion = '4.13.1' findbugVersion = '3.0.2' commonsCliVersion = '1.4' - mockitoVersion = '3.5.11' - radarCommonsVersion = '0.13.0' - radarSchemasVersion = '0.5.14' + mockitoVersion = '4.3.1' + radarCommonsVersion = '0.14.1-SNAPSHOT' + radarSchemasVersion = '0.7.6' subethamailVersion = '3.1.7' - jsoupVersion = '1.13.1' - slf4jVersion = '1.7.30' - log4jVersion = '1.2.17' - avroVersion = '1.9.2' + jsoupVersion = '1.14.3' + slf4jVersion = '1.7.35' + reload4jVersion = '1.2.18.5' + avroVersion = '1.11.0' + jsonPathVersion = '2.7.0' } //---------------------------------------------------------------------------// @@ -53,14 +50,9 @@ ext { // In this section you declare where to find the dependencies of your project repositories { mavenCentral() - // Non-jcenter radar releases - maven { url 'http://dl.bintray.com/radar-cns/org.radarcns' } - maven { url 'http://dl.bintray.com/radar-base/org.radarbase' } // Kafka/confluent releases - maven { url 'http://packages.confluent.io/maven/' } - // For working with dev-branches - maven { url 'http://oss.jfrog.org/artifactory/oss-snapshot-local/' } - // Github code + maven { url 'https://packages.confluent.io/maven/' } + maven { url = "https://oss.sonatype.org/content/repositories/snapshots" } maven { url 'https://jitpack.io' } } @@ -69,11 +61,14 @@ dependencies { implementation group: 'org.radarbase', name: 'radar-commons', version: radarCommonsVersion implementation group: 'org.apache.avro', name: 'avro', version: avroVersion implementation group: 'org.radarbase', name: 'radar-commons-testing', version: radarCommonsVersion - implementation group: 'org.radarcns', name: 'radar-schemas-commons', version: radarSchemasVersion + implementation group: 'org.radarbase', name: 'radar-schemas-commons', version: radarSchemasVersion + implementation group: 'org.radarbase', name: 'oauth-client-util', version: '0.8.1' + implementation group: 'org.radarbase', name: 'radar-app-config-client', version: '0.4.1-SNAPSHOT' // Kafka streaming API implementation group: 'org.apache.kafka', name: 'kafka-streams', version: kafkaVersion implementation group: 'io.confluent', name: 'kafka-streams-avro-serde', version: confluentVersion + implementation group: 'io.confluent', name: 'kafka-json-serializer', version: confluentVersion // Nonnull annotation implementation group: 'com.google.code.findbugs' , name: 'jsr305' , version: findbugVersion @@ -84,6 +79,7 @@ dependencies { // Configuration @JsonProperty implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: jacksonVersion implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: jacksonVersion + implementation group: 'com.fasterxml.jackson.module', name: 'jackson-module-kotlin', version: jacksonVersion // Monitor mail sending implementation group: 'javax.mail', name: 'javax.mail-api', version: javaMailVersion @@ -92,14 +88,34 @@ dependencies { // JSoup html parser implementation group: 'org.jsoup', name: 'jsoup', version: jsoupVersion - runtimeOnly group: 'org.radarbase', name: 'radar-commons-unsafe', version: radarCommonsVersion + // JsonPath for evaluating json conditions dynamically. + implementation group: 'com.jayway.jsonpath', name: 'json-path', version: jsonPathVersion - runtimeOnly group: 'log4j', name: 'log4j', version: log4jVersion - runtimeOnly group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion + runtimeOnly group: 'ch.qos.reload4j', name: 'reload4j', version: reload4jVersion + runtimeOnly group: 'org.slf4j', name: 'slf4j-reload4j', version: slf4jVersion } if (!hasProperty('profile')) { ext.profile = 'dev' } +java { + toolchain { + languageVersion.set(JavaLanguageVersion.of(17)) + } +} + +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile) { + kotlinOptions { + jvmTarget = "17" + freeCompilerArgs = [ + "-Xjavac-arguments='-Xlint:unchecked -Xlint:deprecation'" + ] + } +} + +tasks.withType(JavaCompile) { + options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation" +} + apply from: "gradle/profile.${profile}.gradle" diff --git a/gradle.properties b/gradle.properties index a54c2e20..e69de29b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,2 +0,0 @@ -bintrayUser= -bintrayApiKey= \ No newline at end of file diff --git a/gradle/codacy.gradle b/gradle/codacy.gradle deleted file mode 100644 index 73f33abb..00000000 --- a/gradle/codacy.gradle +++ /dev/null @@ -1,32 +0,0 @@ -//---------------------------------------------------------------------------// -// Code coverage and codacy // -//---------------------------------------------------------------------------// - -apply plugin: 'jacoco' - -configurations { - codacy -} - -jacoco { - toolVersion = "0.8.3" -} - -dependencies { - codacy group: 'com.github.codacy', name: 'codacy-coverage-reporter', version: '4.0.1' -} - -jacocoTestReport { - reports { - xml.enabled true - csv.enabled false - html.enabled true - } - executionData test, integrationTest -} - -task sendCoverageToCodacy(type: JavaExec, dependsOn: jacocoTestReport) { - main = 'com.codacy.CodacyCoverageReporter' - classpath = configurations.codacy - args = ['report', '-l', 'Java', '-r', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml"] -} diff --git a/gradle/profile.dev.gradle b/gradle/profile.dev.gradle index 7113bfb3..575cdd06 100644 --- a/gradle/profile.dev.gradle +++ b/gradle/profile.dev.gradle @@ -1,5 +1,3 @@ apply from: 'gradle/test.gradle' -apply from: 'gradle/codacy.gradle' apply from: 'gradle/style.gradle' apply from: 'gradle/utilities.gradle' -apply from: 'gradle/publishing.gradle' diff --git a/gradle/profile.prod.gradle b/gradle/profile.prod.gradle index 2b15eeac..29287c56 100644 --- a/gradle/profile.prod.gradle +++ b/gradle/profile.prod.gradle @@ -6,6 +6,14 @@ task downloadRuntimeDependencies { } } +tasks.register("copyDependencies", Copy.class) { + from(configurations.named("runtimeClasspath").map { it.files }) + into("$buildDir/third-party/") + doLast { + println("Copied third-party runtime dependencies") + } +} + processResources { expand(version: version) } diff --git a/gradle/publishing.gradle b/gradle/publishing.gradle deleted file mode 100644 index fdaa1ab1..00000000 --- a/gradle/publishing.gradle +++ /dev/null @@ -1,119 +0,0 @@ -apply plugin: 'maven-publish' -apply plugin: 'com.jfrog.bintray' - -ext.sharedManifest = manifest { - attributes( - "Implementation-Title": rootProject.name, - "Implementation-Version": version) -} - -//---------------------------------------------------------------------------// -// Packaging // -//---------------------------------------------------------------------------// - -processResources { - expand(version: version) -} - -jar { - manifest { - from sharedManifest - attributes('Main-Class': mainClassName) - } -} - -tasks.withType(Tar){ - compression = Compression.GZIP - archiveExtension.set('tar.gz') -} - -// custom tasks for creating source/javadoc jars -task sourcesJar(type: Jar, dependsOn: classes) { - archiveClassifier.set('sources') - from sourceSets.main.allSource - manifest.from sharedManifest -} - -task javadocJar(type: Jar, dependsOn: javadoc) { - archiveClassifier.set('javadoc') - from javadoc.destinationDir - manifest.from sharedManifest -} - -// add javadoc/source jar tasks as artifacts -artifacts { - archives sourcesJar, javadocJar -} - -publishing { - publications { - mavenJar(MavenPublication) { - from components.java - artifact sourcesJar - artifact javadocJar - pom { - description = moduleDescription - url = githubUrl - - licenses { - license { - name = 'The Apache Software License, Version 2.0' - url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' - distribution = 'repo' - } - } - developers { - developer { - id = 'blootsvoets' - name = 'Joris Borgdorff' - email = 'joris@thehyve.nl' - organization = 'The Hyve' - } - developer { - id = 'nivemaham' - name = 'Nivethika Mahasivam' - email = 'nivethika@thehyve.nl' - organization = 'The Hyve' - } - } - issueManagement { - system = 'GitHub' - url = issueUrl - } - organization { - name = 'RADAR-Base' - url = website - } - scm { - connection = 'scm:git:' + githubUrl - url = githubUrl - } - } - } - } -} - -bintray { - user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER') - key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY') - override false - publications 'mavenJar' - pkg { - repo = project.group - name = rootProject.name - userOrg = 'radar-cns' - desc = moduleDescription - licenses = ['Apache-2.0'] - websiteUrl = website - issueTrackerUrl = issueUrl - vcsUrl = githubUrl - githubRepo = githubRepoName - githubReleaseNotesFile = 'README.md' - version { - name = project.version - desc = moduleDescription - vcsTag = System.getenv('TRAVIS_TAG') - released = new Date() - } - } -} diff --git a/gradle/test.gradle b/gradle/test.gradle index 51270ffc..cfa606b5 100644 --- a/gradle/test.gradle +++ b/gradle/test.gradle @@ -25,15 +25,15 @@ dependencies { // Testing testImplementation group: 'junit', name: 'junit', version: junitVersion testImplementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion - testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: hamcrestVersion + testImplementation group: 'org.hamcrest', name: 'hamcrest', version: hamcrestVersion // Mock mail server testImplementation group: 'org.subethamail', name: 'subethasmtp', version: subethamailVersion - testRuntimeOnly group: 'log4j', name: 'log4j', version: log4jVersion - testRuntimeOnly group: 'org.slf4j', name: 'slf4j-log4j12', version: slf4jVersion + testRuntimeOnly group: 'ch.qos.reload4j', name: 'reload4j', version: reload4jVersion + testRuntimeOnly group: 'org.slf4j', name: 'slf4j-reload4j', version: slf4jVersion - integrationTestImplementation group: 'org.radarcns', name: 'radar-schemas-tools', version: radarSchemasVersion + integrationTestImplementation group: 'org.radarbase', name: 'radar-schemas-registration', version: radarSchemasVersion } tasks.matching {it instanceof Test}.all { @@ -60,3 +60,7 @@ task integrationTest(type: Test) { } check.dependsOn integrationTestClasses + +tasks.named("processIntegrationTestResources", Copy.class) { + duplicatesStrategy = DuplicatesStrategy.INCLUDE +} diff --git a/gradle/utilities.gradle b/gradle/utilities.gradle index 545fbc01..4202cb84 100644 --- a/gradle/utilities.gradle +++ b/gradle/utilities.gradle @@ -21,5 +21,5 @@ idea { } wrapper { - gradleVersion '6.6.1' + gradleVersion '7.3.3' } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index e708b1c0..7454180f 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 12d38de6..2e6e5897 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 4f906e0c..1b6c7873 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ -#!/usr/bin/env sh +#!/bin/sh # -# Copyright 2015 the original author or authors. +# Copyright © 2015-2021 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,67 +17,101 @@ # ############################################################################## -## -## Gradle start up script for UN*X -## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# ############################################################################## # Attempt to set APP_HOME + # Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` +APP_BASE_NAME=${0##*/} # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" +MAX_FD=maximum warn () { echo "$*" -} +} >&2 die () { echo echo "$*" echo exit 1 -} +} >&2 # OS specific support (must be 'true' or 'false'). cygwin=false msys=false darwin=false nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACMD=$JAVA_HOME/jre/sh/java else - JAVACMD="$JAVA_HOME/bin/java" + JAVACMD=$JAVA_HOME/bin/java fi if [ ! -x "$JAVACMD" ] ; then die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME @@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the location of your Java installation." fi else - JAVACMD="java" + JAVACMD=java which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the @@ -106,80 +140,95 @@ location of your Java installation." fi # Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac fi -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. # For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) fi - i=`expr $i + 1` + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg done - case $i in - 0) set -- ;; - 1) set -- "$args0" ;; - 2) set -- "$args0" "$args1" ;; - 3) set -- "$args0" "$args1" "$args2" ;; - 4) set -- "$args0" "$args1" "$args2" "$args3" ;; - 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac fi -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=`save "$@"` +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' exec "$JAVACMD" "$@" diff --git a/radar.yml b/radar.yml index 1b4463af..050279f8 100644 --- a/radar.yml +++ b/radar.yml @@ -1,17 +1,11 @@ version: 1.0 released: 2016-11-27 -#============================== Zookeeper ==============================# -#List of Zookeeper instances -zookeeper: - - host: localhost - port: 2181 - #================================ Kafka ================================# #List of Kafka brokers broker: - - host: localhost - port: 9092 + - host: kafka-1 + port: 9092 #Kafka internal parameters @@ -45,35 +39,41 @@ stream: source_statistics: - name: Empatica E4 topics: - - android_empatica_e4_blood_volume_pulse_1min + - android_empatica_e4_blood_volume_pulse_1min output_topic: source_statistics_empatica_e4 - name: Biovotion VSM1 topics: - - android_biovotion_vsm1_acceleration_1min + - android_biovotion_vsm1_acceleration_1min output_topic: source_statistics_biovotion_vsm1 - name: RADAR pRMT topics: - - android_phone_acceleration_1min - - android_phone_bluetooth_devices - - android_phone_sms - - android_phone_call - - android_phone_contacts - - android_phone_usage_event - - android_phone_relative_location + - android_phone_acceleration_1min + - android_phone_bluetooth_devices + - android_phone_sms + - android_phone_call + - android_phone_contacts + - android_phone_usage_event + - android_phone_relative_location output_topic: source_statistics_android_phone #=========================== Schema Registry ===========================# #List of Schema Registry instances schema_registry: - - host: localhost - port: 8081 - protocol: http + - host: schema-registry-1 + port: 8081 + protocol: http rest_proxy: - host: radar-test.thehyve.net - port: 8082 - protocol: http + host: rest-proxy-1 + port: 8082 + protocol: http + +email_server: + host: localhost + port: 25 + user: no-reply@radarcns.org + #======================== Battery level monitor ========================# battery_monitor: @@ -85,9 +85,6 @@ battery_monitor: email_address: - radar@thehyve.nl level: LOW - email_host: localhost - email_port: 25 - email_user: no-reply@radarcns.org topics: - android_empatica_e4_battery_level @@ -100,12 +97,50 @@ disconnect_monitor: - project_id: s2 email_address: - radar@thehyve.nl - email_host: localhost - email_port: 25 - email_user: no-reply@radarcns.org topics: - android_empatica_e4_temperature timeout: 1800 # seconds after which a stream is set disconnected alert_repetitions: 2 # number of additional emails to send after the first -# persistence_path: /var/lib/radar/data +persistence_path: /tmp + +realtime_consumers: + - name: 'lstm-lung-study-consumer' + topic: 'REST_API_INFERENCE_LSTM_COPD' + conditions: + - name: 'LocalJsonPathCondition' + projects: ['p1'] + properties: + jsonpath: '$[?(@.invocation_result.anomaly_detected == true)]' + key: 'INFERENCE_RESULT_JSON' + + actions: + - name: 'ActiveAppNotificationAction' + projects: ['p1'] + properties: + questionnaire_name: 'ers' + time_of_day: '09:00:00' + default_timezone: 'Europe/London' + appserver_base_url: 'http://appserver:8080/' + management_portal_token_url: 'http://managementportal-app:8080/managementportal/oauth/token' + client_id: 'radar_appserver_client' + client_secret: 'secret' + metadata_key: 'INFERENCE_RESULT_JSON' + +#====================== Notification monitor=============================# +#intervention_monitor: +# #=========================App Server======================================# +# app_server_url: http://localhost:8080/appserver/api +# app_config_client: dynamint_ksql +# app_config_url: http://localhost:8090/appconfig/api +# auth: +# token_url: http://localhost:8090/managementportal/oauth/token +# clientId: radar_backend_client +# clientSecret: secret +# topic: interventions +# notify: +# - project_id: p1 +# email_address: +# - test@thehyve.nl +# protocol_directory: interventions + diff --git a/settings.gradle b/settings.gradle index 978147d9..e065bebd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,12 +1 @@ -pluginManagement { - buildscript { - repositories { - jcenter() - } - dependencies { - classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.5' - } - } -} - rootProject.name = 'radar-backend' diff --git a/src/integrationTest/docker/docker-compose.yml b/src/integrationTest/docker/docker-compose.yml index 9b3f8c31..7879f3f7 100644 --- a/src/integrationTest/docker/docker-compose.yml +++ b/src/integrationTest/docker/docker-compose.yml @@ -114,23 +114,133 @@ services: KAFKA_REST_HOST_NAME: rest-proxy-1 KAFKA_GROUP_MIN_SESSION_TIMEOUT_MS: 5000 + # #---------------------------------------------------------------------------# + # # Integration test # + # #---------------------------------------------------------------------------# + # integration-test: + # build: + # context: ../../.. + # dockerfile: src/integrationTest/docker/Dockerfile + # # Right now, only direct connections to kafka are tested + # depends_on: + # - kafka-1 + # - schema-registry-1 + # networks: + # - kafka + # - default + # command: + # - gradle + # - integrationTest + # volumes: + # - ../../../build/jacoco:/code/build/jacoco + # - ../../../build/reports:/code/build/reports + #---------------------------------------------------------------------------# - # Integration test # + # RADAR APPSERVER # #---------------------------------------------------------------------------# - integration-test: - build: - context: ../../.. - dockerfile: src/integrationTest/docker/Dockerfile - # Right now, only direct connections to kafka are tested + appserver: + image: radarbase/radar-appserver:1.3.0 + restart: always + ports: + - 8080:8080 + volumes: + - ./etc/radar_is.yml:/resources/radar_is.yml + - ./logs/:/var/log/radar/appserver/ + - ./etc/google-credentials.json:/resources/google-credentials.json depends_on: - - kafka-1 - - schema-registry-1 + - radarbase-postgresql + - managementportal-app + networks: + - kafka + environment: + JDK_JAVA_OPTIONS: -Xmx4G -Djava.security.egd=file:/dev/./urandom + SPRING_DATASOURCE_URL: jdbc:postgresql://radarbase-postgresql:5432/appserver + SPRING_DATASOURCE_USERNAME: radarcns + SPRING_DATASOURCE_PASSWORD: radarcns + SECURITY_RADAR_MANAGEMENTPORTAL_URL: http://localhost:8090/managementportal + GOOGLE_APPLICATION_CREDENTIALS: /resources/google-credentials.json + RADAR_ADMIN_USER: radar + RADAR_ADMIN_PASSWORD: appserver + SPRING_APPLICATION_JSON: '{"spring":{"boot":{"admin":{"client":{"url":"http://spring-boot-admin:1111","username":"radar","password":"appserver"}}}}}' + RADAR_IS_CONFIG_LOCATION: "/resources/radar_is.yml" + SPRING_BOOT_ADMIN_CLIENT_INSTANCE_NAME: radar-appserver + healthcheck: + # should give an unauthenticated response, rather than a 404 + test: [ "CMD-SHELL", "wget --spider localhost:8080/projects 2>&1 | grep -q 401 || exit 1" ] + interval: 1m30s + timeout: 5s + retries: 3 + + spring-boot-admin: + image: slydeveloper/spring-boot-admin:latest + ports: + - 8888:1111 + networks: + - kafka + environment: + SPRING_BOOT_ADMIN_USER_NAME: radar + SPRING_BOOT_ADMIN_USER_PASSWORD: appserver + SPRING_BOOT_ADMIN_TITLE: RADAR-appserver + SPRING_APPLICATION_JSON: '{"spring":{"boot":{"admin":{"username":"radar","password":"appserver","title":"RADAR-appserver"}}}}' + #---------------------------------------------------------------------------# + # Management Portal # + #---------------------------------------------------------------------------# + managementportal-app: + image: radarbase/management-portal:0.8.0 + depends_on: + - radarbase-postgresql + ports: + - 8090:8080 networks: - kafka - - default + environment: + SPRING_PROFILES_ACTIVE: prod + SPRING_DATASOURCE_URL: jdbc:postgresql://radarbase-postgresql:5432/managementportal + SPRING_DATASOURCE_USERNAME: radarcns + SPRING_DATASOURCE_PASSWORD: radarcns + MANAGEMENTPORTAL_FRONTEND_CLIENT_SECRET: "testMe" + MANAGEMENTPORTAL_COMMON_BASE_URL: http://localhost:8080/managementportal + MANAGEMENTPORTAL_COMMON_MANAGEMENT_PORTAL_BASE_URL: http://localhost:8080/managementportal + MANAGEMENTPORTAL_OAUTH_CLIENTS_FILE: /mp-includes/config/oauth_client_details.csv + MANAGEMENTPORTAL_CATALOGUE_SERVER_ENABLE_AUTO_IMPORT: 'false' + MANAGEMENTPORTAL_OAUTH_SIGNING_KEY_ALIAS: 'radarbase-managementportal-ec' + JAVA_OPTS: -Xmx256m # maximum heap size for the JVM running ManagementPortal, increase this as necessary + volumes: + - ./etc/mp-config/:/mp-includes/config + + radarbase-postgresql: + image: radarbase/radarbase-postgres:latest + ports: + - "5434:5432" + networks: + - kafka + environment: + - POSTGRES_USER=radarcns + - POSTGRES_PASSWORD=radarcns + - POSTGRES_MULTIPLE_DATABASES=managementportal,appserver + + + realtime-consumer: + image: radarbase/realtime-consumer:latest + build: ../../../ + networks: + - kafka + volumes: + - "../../../radar.yml:/etc/radar.yml" command: - - gradle - - integrationTest + - "realtime_consumers" + - "-c /etc/radar.yml" + environment: + KAFKA_SCHEMA_REGISTRY: http://schema-registry-1:8081 + + smtp: + image: namshi/smtp:latest + networks: + - kafka volumes: - - ../../../build/jacoco:/code/build/jacoco - - ../../../build/reports:/code/build/reports + - /var/spool/exim + restart: always + ports: + - "25:25" + env_file: + - ./etc/smtp.env diff --git a/src/integrationTest/docker/etc/mp-config/keystore.p12 b/src/integrationTest/docker/etc/mp-config/keystore.p12 new file mode 100644 index 00000000..f1cdeb82 Binary files /dev/null and b/src/integrationTest/docker/etc/mp-config/keystore.p12 differ diff --git a/src/integrationTest/docker/etc/mp-config/oauth_client_details.csv b/src/integrationTest/docker/etc/mp-config/oauth_client_details.csv new file mode 100644 index 00000000..728408d5 --- /dev/null +++ b/src/integrationTest/docker/etc/mp-config/oauth_client_details.csv @@ -0,0 +1,4 @@ +client_id;resource_ids;client_secret;scope;authorized_grant_types;redirect_uri;authorities;access_token_validity;refresh_token_validity;additional_information;autoapprove +pRMT;res_ManagementPortal,res_gateway;;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +aRMT;res_ManagementPortal,res_gateway;secret;MEASUREMENT.CREATE,SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;refresh_token,authorization_code;;;43200;7948800;{"dynamic_registration": true}; +radar_appserver_client;res_ManagementPortal,res_AppServer;secret;SUBJECT.UPDATE,SUBJECT.READ,PROJECT.READ,SOURCETYPE.READ,SOURCE.READ,SOURCETYPE.READ,SOURCEDATA.READ,USER.READ,ROLE.READ;client_credentials;;;43200;259200;{}; \ No newline at end of file diff --git a/src/integrationTest/docker/etc/radar_is.yml b/src/integrationTest/docker/etc/radar_is.yml new file mode 100644 index 00000000..026fcccf --- /dev/null +++ b/src/integrationTest/docker/etc/radar_is.yml @@ -0,0 +1,3 @@ +resourceName: res_AppServer +publicKeyEndpoints: + - http://managementportal-app:8080/managementportal/oauth/token_key \ No newline at end of file diff --git a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java index 0f9123cf..c1b02139 100644 --- a/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java +++ b/src/integrationTest/java/org/radarcns/integration/E4AggregatedAccelerationMonitor.java @@ -52,7 +52,7 @@ public E4AggregatedAccelerationMonitor(RadarPropertyHandler radar, String topic, } @Override - protected void evaluateRecord(ConsumerRecord records) { + protected void evaluateRecord(ConsumerRecord record) { // noop } diff --git a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java index a58cb622..b06bfa67 100644 --- a/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java +++ b/src/integrationTest/java/org/radarcns/integration/PhoneStreamTest.java @@ -67,8 +67,8 @@ import org.radarcns.passive.empatica.EmpaticaE4Temperature; import org.radarcns.passive.phone.PhoneUsageEvent; import org.radarcns.passive.phone.UsageEventType; -import org.radarcns.schema.registration.KafkaTopics; -import org.radarcns.schema.registration.SchemaRegistry; +import org.radarbase.schema.registration.KafkaTopics; +import org.radarbase.schema.registration.SchemaRegistry; import org.radarcns.util.RadarSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -108,7 +108,8 @@ public void setUp() throws IOException, ParseException, InterruptedException { } ConfigRadar props = propHandler.getRadarProperties(); - KafkaTopics topics = new KafkaTopics(props.getZookeeperPaths()); + Map kafkaConf = Map.of("bootstrap.servers", props.getBrokerPaths()); + KafkaTopics topics = new KafkaTopics(kafkaConf); int expectedBrokers = props.getBroker().size(); topics.initialize(expectedBrokers); diff --git a/src/integrationTest/resources/org/radarcns/kafka/radar.yml b/src/integrationTest/resources/org/radarcns/kafka/radar.yml index 85344e95..8d5281c5 100644 --- a/src/integrationTest/resources/org/radarcns/kafka/radar.yml +++ b/src/integrationTest/resources/org/radarcns/kafka/radar.yml @@ -1,13 +1,6 @@ version: 1.0 released: 2016-11-27 - -#============================== Zookeeper ==============================# -#List of Zookeeper instances -zookeeper: - - host: zookeeper-1 - port: 2181 - #================================ Kafka ================================# #List of Kafka brokers broker: diff --git a/src/main/docker/radar-backend-init b/src/main/docker/radar-backend-init index 46e66df1..7b97cde7 100755 --- a/src/main/docker/radar-backend-init +++ b/src/main/docker/radar-backend-init @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -e @@ -6,34 +6,13 @@ set -e echo "===> Waiting for RADAR-CNS topics ... " # Check if variables exist -if [ -n "$KAFKA_REST_PROXY" ] && [ -n "$KAFKA_SCHEMA_REGISTRY" ] && [ -n "$KAFKA_BROKERS" ]; then +if [ -n "$KAFKA_SCHEMA_REGISTRY" ]; then max_timeout=32 tries=10 timeout=1 while true; do - IFS=, read -r -a array <<< $(curl -s "${KAFKA_REST_PROXY}/brokers" | sed 's/^.*\[\(.*\)\]\}/\1/') - LENGTH=${#array[@]} - if [ "$LENGTH" -eq "$KAFKA_BROKERS" ]; then - echo "Kafka brokers available." - break - fi - tries=$((tries - 1)) - if [ $tries -eq 0 ]; then - echo "FAILED: KAFKA BROKERs NOT READY." - exit 5 - fi - echo "Kafka brokers or Kafka REST proxy not ready. Retrying in ${timeout} seconds." - sleep $timeout - if [ $timeout -lt $max_timeout ]; then - timeout=$((timeout * 2)) - fi - done - - tries=10 - timeout=1 - while true; do - if wget --spider -q "${KAFKA_SCHEMA_REGISTRY}/subjects" 2>/dev/null; then + if [ $(curl -s "${KAFKA_SCHEMA_REGISTRY}/subjects" | wc -c) -ge 2 ]; then break fi tries=$((tries - 1)) diff --git a/src/main/java/org/radarbase/appserver/client/AppServerData.kt b/src/main/java/org/radarbase/appserver/client/AppServerData.kt new file mode 100644 index 00000000..2956e021 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/AppServerData.kt @@ -0,0 +1,10 @@ +package org.radarbase.appserver.client + +data class AppServerData( + val ttlSeconds: Long, + val sourceId: String, + val sourceType: String = "aRMT", + val appPackage: String = "org.phidatalab.radar_armt", + val scheduledTime: String, + val dataMap: String, +) : AppServerMessageContents diff --git a/src/main/java/org/radarbase/appserver/client/AppServerMessageContents.kt b/src/main/java/org/radarbase/appserver/client/AppServerMessageContents.kt new file mode 100644 index 00000000..e8ba01c8 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/AppServerMessageContents.kt @@ -0,0 +1,3 @@ +package org.radarbase.appserver.client + +sealed interface AppServerMessageContents diff --git a/src/main/java/org/radarbase/appserver/client/AppServerNotification.kt b/src/main/java/org/radarbase/appserver/client/AppServerNotification.kt new file mode 100644 index 00000000..b4628e74 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/AppServerNotification.kt @@ -0,0 +1,13 @@ +package org.radarbase.appserver.client + +data class AppServerNotification( + val title: String, + val body: String, + val ttlSeconds: Long, + val sourceId: String, + val type: String, + val sourceType: String = "aRMT", + val appPackage: String = "org.phidatalab.radar_armt", + val scheduledTime: String, + val additionalData: String, +) : AppServerMessageContents diff --git a/src/main/java/org/radarbase/appserver/client/AppserverClient.kt b/src/main/java/org/radarbase/appserver/client/AppserverClient.kt new file mode 100644 index 00000000..e8975c5f --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/AppserverClient.kt @@ -0,0 +1,160 @@ +package org.radarbase.appserver.client + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import okhttp3.* +import okhttp3.Headers.Companion.headersOf +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import org.radarbase.exception.TokenException +import org.radarbase.oauth.OAuth2Client +import org.slf4j.LoggerFactory +import java.io.IOException + +/** + * Client to provide interface to the AppServer REST API. It uses the OAuth client library from + * management portal to authenticate requests. + */ +class AppserverClient(config: AppserverClientConfig) { + private val baseUrl: HttpUrl + private val httpClient: OkHttpClient = config.httpClient ?: OkHttpClient() + private val objectMapper: ObjectMapper = config.mapper ?: ObjectMapper() + private val oauthClient: OAuth2Client? + + constructor(build: AppserverClientConfig.() -> Unit) : this(AppserverClientConfig().apply(build)) + + init { + baseUrl = requireNotNull(config.appserverUrl) { "Appserver client needs base URL" } + + oauthClient = if (config.tokenUrl == null) { + logger.warn("MP Token url was not provided. Will use unauthenticated requests to appserver.") + null + } else { + val clientId = requireNotNull(config.clientId) { "Client ID for appserver client is not set." } + val clientSecret = requireNotNull(config.clientSecret) { "Client secret for appserver client is not set." } + OAuth2Client.Builder() + .credentials(clientId, clientSecret) + .endpoint(config.tokenUrl) + .scopes("SUBJECT.READ SUBJECT.UPDATE PROJECT.READ") + .httpClient(httpClient) + .build() + } + } + + @Throws(TokenException::class) + private fun headers(): Headers { + return if (oauthClient != null) { + headersOf("Authorization", "Bearer " + oauthClient.validToken.accessToken) + } else { + headersOf() + } + } + + @Throws(IOException::class) + fun createMessage( + projectId: String, + userId: String, + type: MessagingType, + contents: AppServerMessageContents, + ): Map { + val stringContents = objectMapper.writeValueAsString(contents) + return createMessage(projectId, userId, type, stringContents) + } + + @Throws(IOException::class) + fun createMessage( + projectId: String, + userId: String, + type: MessagingType, + contents: String, + ): Map { + val request: Request = try { + Request.Builder() + .post(contents.toRequestBody(APPLICATION_JSON)) + .url( + baseUrl.newBuilder() + .addEncodedPathSegment("projects") + .addPathSegment(projectId) + .addEncodedPathSegment("users") + .addPathSegment(userId) + .addEncodedPathSegment("messaging") + .addEncodedPathSegment(type.urlPart) + .build() + ) + .headers(headers()) + .build() + } catch (e: TokenException) { + throw IOException(e) + } + return httpClient.newCall(request).execute().use { handleResponse(request, it) } + } + + @Throws(IOException::class) + fun createScheduleForAssessment( + projectId: String, + userId: String, + body: String + ) { + val request: Request = try { + Request.Builder() + .put(body.toRequestBody(APPLICATION_JSON)) + .url( + baseUrl.newBuilder() + .addEncodedPathSegment("projects") + .addPathSegment(projectId) + .addEncodedPathSegment("users") + .addPathSegment(userId) + .addEncodedPathSegment("questionnaire") + .addEncodedPathSegment("schedule") + .build() + ) + .headers(headers()) + .build() + } catch (e: TokenException) { + throw IOException(e) + } + return httpClient.newCall(request).execute().use { handleResponse(request, it) } + } + + @Throws(IOException::class) + fun getUserDetails(projectId: String, userId: String): Map { + val request: Request = try { + Request.Builder() + .get() + .url( + baseUrl.newBuilder() + .addEncodedPathSegment("projects") + .addPathSegment(projectId) + .addEncodedPathSegment("users") + .addPathSegment(userId) + .build() + ) + .headers(headers()) + .build() + } catch (e: TokenException) { + throw IOException(e) + } + return httpClient.newCall(request).execute().use { handleResponse(request, it) } + } + + @Throws(IOException::class) + private fun handleResponse(request: Request, response: Response): Map { + val body = response.body?.string() + ?: throw IOException("Response body from appserver ${request.url} was null.") + + return when (response.code) { + 404 -> throw IOException("The Entity or URL ${request.url} was not found in the appserver: $body") + 200, 201 -> + if (body.isNotEmpty()) + objectMapper.readValue(body, object : TypeReference>() {}) + else emptyMap() + else -> throw IOException("There was an error requesting the appserver URL ${request.url}. ${response.code} - $body") + } + } + + companion object { + private val logger = LoggerFactory.getLogger(AppserverClient::class.java) + + private val APPLICATION_JSON = "application/json".toMediaType() + } +} diff --git a/src/main/java/org/radarbase/appserver/client/AppserverClientConfig.kt b/src/main/java/org/radarbase/appserver/client/AppserverClientConfig.kt new file mode 100644 index 00000000..b1b30a24 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/AppserverClientConfig.kt @@ -0,0 +1,43 @@ +package org.radarbase.appserver.client + +import com.fasterxml.jackson.databind.ObjectMapper +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.OkHttpClient +import java.net.MalformedURLException +import java.net.URL + +data class AppserverClientConfig( + var appserverUrl: HttpUrl? = null, + var tokenUrl: URL? = null, + var clientId: String? = null, + var clientSecret: String? = null, +) { + var httpClient: OkHttpClient? = null + set(value) { + field = value?.newBuilder()?.build() + } + var mapper: ObjectMapper? = null + + fun tokenUrl(url: String?) { + tokenUrl = if (url != null) { + try { + URL(url) + } catch (ex: MalformedURLException) { + throw IllegalArgumentException(ex) + } + } else { + null + } + } + + fun appserverUrl(url: String?) { + appserverUrl = url?.toHttpUrl() + } + + override fun toString(): String = "AppServerClientConfig(" + + "appserverUrl=$appserverUrl, " + + "tokenUrl=$tokenUrl, " + + "clientId=$clientId, " + + "clientSecret=$clientSecret)" +} diff --git a/src/main/java/org/radarbase/appserver/client/MessagingType.kt b/src/main/java/org/radarbase/appserver/client/MessagingType.kt new file mode 100644 index 00000000..f22b9f59 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/MessagingType.kt @@ -0,0 +1,5 @@ +package org.radarbase.appserver.client + +enum class MessagingType(val urlPart: String) { + NOTIFICATIONS("notifications"), DATA("data"), ALL("all"); +} diff --git a/src/main/java/org/radarbase/appserver/client/protocol/ClinicalProtocol.kt b/src/main/java/org/radarbase/appserver/client/protocol/ClinicalProtocol.kt new file mode 100644 index 00000000..b13a3def --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/ClinicalProtocol.kt @@ -0,0 +1,6 @@ +package org.radarbase.appserver.client.protocol + +data class ClinicalProtocol( + val repeatAfterClinicVisit: RepeatQuestionnaire, + val requiresInClinicCompletion: Boolean = true, +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/FileProtocolDirectory.kt b/src/main/java/org/radarbase/appserver/client/protocol/FileProtocolDirectory.kt new file mode 100644 index 00000000..414afb0e --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/FileProtocolDirectory.kt @@ -0,0 +1,54 @@ +package org.radarbase.appserver.client.protocol + +import com.fasterxml.jackson.databind.ObjectMapper +import org.slf4j.LoggerFactory +import java.nio.file.Files +import java.nio.file.Paths +import java.util.* +import java.util.stream.Collectors +import kotlin.collections.HashMap +import kotlin.streams.asSequence + +class FileProtocolDirectory( + protocolDir: String, + mapper: ObjectMapper, +): ProtocolDirectory { + private val protocols: Map + + init { + val jsonReader = mapper.readerFor(QuestionnaireTrigger::class.java) + + protocols = Files.walk(Paths.get(protocolDir)).use { pathStream -> + pathStream + .filter { path -> path.fileName.toString().endsWith(".json") } + .asSequence() + .mapNotNull { path -> + try { + Pair( + path.fileName.toString() + .lowercase() + .removeSuffix(".json"), + Files.newInputStream(path).use { + jsonReader.readValue(it) + }, + ) + } catch (ex: Exception) { + logger.error("Failed to read JSON protocol {}", path, ex) + null + } + } + .toMap(TreeMap()) + } + logger.info("From {} loaded protocols {}", protocolDir, protocols) + } + + override fun get( + projectId: String, + userId: String, + attributes: Map + ): QuestionnaireTrigger? = protocols[attributes["intervention"] ?: "default"] + + companion object { + private val logger = LoggerFactory.getLogger(FileProtocolDirectory::class.java) + } +} diff --git a/src/main/java/org/radarbase/appserver/client/protocol/MultiLingualText.kt b/src/main/java/org/radarbase/appserver/client/protocol/MultiLingualText.kt new file mode 100644 index 00000000..ea6d9f16 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/MultiLingualText.kt @@ -0,0 +1,10 @@ +package org.radarbase.appserver.client.protocol + +typealias MultiLingualText = Map + +/** + * Get the translation in the given language. + * @return non-blank translation or null if not available. + */ +fun MultiLingualText.translation(language: String): String? = + this[language]?.takeIf { it.isNotBlank() } diff --git a/src/main/java/org/radarbase/appserver/client/protocol/Notification.kt b/src/main/java/org/radarbase/appserver/client/protocol/Notification.kt new file mode 100644 index 00000000..25422487 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/Notification.kt @@ -0,0 +1,15 @@ +package org.radarbase.appserver.client.protocol + +data class Notification( + val title: MultiLingualText = mapOf( + "en" to defaultNotificationTitle, + ), + val text: MultiLingualText = mapOf( + "en" to defaultNotificationText, + ), +) { + companion object { + const val defaultNotificationTitle = "Questionnaire Time" + const val defaultNotificationText = "Urgent Questionnaire Pending. Please complete now." + } +} diff --git a/src/main/java/org/radarbase/appserver/client/protocol/ProtocolDirectory.kt b/src/main/java/org/radarbase/appserver/client/protocol/ProtocolDirectory.kt new file mode 100644 index 00000000..3abbbe16 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/ProtocolDirectory.kt @@ -0,0 +1,9 @@ +package org.radarbase.appserver.client.protocol + +interface ProtocolDirectory { + fun get( + projectId: String, + userId: String, + attributes: Map + ): QuestionnaireTrigger? +} diff --git a/src/main/java/org/radarbase/appserver/client/protocol/ProtocolDuration.kt b/src/main/java/org/radarbase/appserver/client/protocol/ProtocolDuration.kt new file mode 100644 index 00000000..4e876be7 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/ProtocolDuration.kt @@ -0,0 +1,6 @@ +package org.radarbase.appserver.client.protocol + +data class ProtocolDuration( + val amount: Long, + val unit: String = "min", +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/Questionnaire.kt b/src/main/java/org/radarbase/appserver/client/protocol/Questionnaire.kt new file mode 100644 index 00000000..f55f3681 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/Questionnaire.kt @@ -0,0 +1,7 @@ +package org.radarbase.appserver.client.protocol + +data class Questionnaire( + val avsc: String, + val name: String, + val repository: String, +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/QuestionnaireTrigger.kt b/src/main/java/org/radarbase/appserver/client/protocol/QuestionnaireTrigger.kt new file mode 100644 index 00000000..a62a00e7 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/QuestionnaireTrigger.kt @@ -0,0 +1,15 @@ +package org.radarbase.appserver.client.protocol + +import com.fasterxml.jackson.annotation.JsonIgnore +import org.radarcns.consumer.realtime.Grouping.Companion.objectMapper + +data class QuestionnaireTrigger( + @JsonIgnore + val singleProtocol: SingleProtocol, + @JsonIgnore + val metadataMap: Map = mapOf(), + + val action: String = "QUESTIONNAIRE_TRIGGER", + val questionnaire: String = objectMapper.writeValueAsString(singleProtocol), + val metadata: String = objectMapper.writeValueAsString(metadataMap), +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/Reminders.kt b/src/main/java/org/radarbase/appserver/client/protocol/Reminders.kt new file mode 100644 index 00000000..cc2b9576 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/Reminders.kt @@ -0,0 +1,7 @@ +package org.radarbase.appserver.client.protocol + +data class Reminders( + val repeat: Int = 0, + val amount: Int = 0, + val unit: String = "day", +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/RepeatQuestionnaire.kt b/src/main/java/org/radarbase/appserver/client/protocol/RepeatQuestionnaire.kt new file mode 100644 index 00000000..a6b3a714 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/RepeatQuestionnaire.kt @@ -0,0 +1,6 @@ +package org.radarbase.appserver.client.protocol + +data class RepeatQuestionnaire( + val unitsFromZero: List = listOf(), + val unit: String = "min", +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/SingleProtocol.kt b/src/main/java/org/radarbase/appserver/client/protocol/SingleProtocol.kt new file mode 100644 index 00000000..f1e81476 --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/SingleProtocol.kt @@ -0,0 +1,17 @@ +package org.radarbase.appserver.client.protocol + + +data class SingleProtocol( + val name: String, + val type: String = "scheduled", + val questionnaire: Questionnaire, + val protocol: SingleProtocolSchedule, + val showInCalendar: Boolean = true, + val showIntroduction: Boolean = false, + val estimatedCompletionTime: Int = 1, + val order: Int = 0, + val isDemo: Boolean = false, + val startText: MultiLingualText = mapOf(), + val endText: MultiLingualText = mapOf(), + val warn: MultiLingualText = mapOf(), +) diff --git a/src/main/java/org/radarbase/appserver/client/protocol/SingleProtocolSchedule.kt b/src/main/java/org/radarbase/appserver/client/protocol/SingleProtocolSchedule.kt new file mode 100644 index 00000000..3eeb9c3e --- /dev/null +++ b/src/main/java/org/radarbase/appserver/client/protocol/SingleProtocolSchedule.kt @@ -0,0 +1,21 @@ +package org.radarbase.appserver.client.protocol + +import com.fasterxml.jackson.annotation.JsonInclude + +data class SingleProtocolSchedule( + @JsonInclude(JsonInclude.Include.NON_NULL) + val clinicalProtocol: ClinicalProtocol? = null, + val completionWindow: ProtocolDuration = ProtocolDuration( + amount = 15, + unit = "min", + ), + val notification: Notification = Notification(), + val reminders: Reminders = Reminders(), + val repeatProtocol: ProtocolDuration = ProtocolDuration( + // a lot of years, it will not repeat. + amount = 9999999999L, + unit = "min", + ), + val repeatQuestionnaire: RepeatQuestionnaire? = null, + val referenceTimestamp: String? = null, +) diff --git a/src/main/java/org/radarcns/RadarBackend.java b/src/main/java/org/radarcns/RadarBackend.java index dd1ecd64..c481c1b5 100644 --- a/src/main/java/org/radarcns/RadarBackend.java +++ b/src/main/java/org/radarcns/RadarBackend.java @@ -23,6 +23,7 @@ import org.radarcns.config.RadarBackendOptions; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.config.SubCommand; +import org.radarcns.consumer.realtime.RealtimeInferenceConsumerFactory; import org.radarcns.monitor.KafkaMonitorFactory; import org.radarcns.producer.MockProducerCommand; import org.radarcns.stream.KafkaStreamFactory; @@ -76,6 +77,8 @@ public SubCommand createCommand() throws IOException { return new KafkaMonitorFactory(options, radarPropertyHandler).createMonitor(); case "mock": return new MockProducerCommand(options, radarPropertyHandler); + case "realtime_consumers": + return RealtimeInferenceConsumerFactory.createConsumersFor(radarPropertyHandler); default: throw new IllegalArgumentException("Unknown subcommand " + options.getSubCommand()); diff --git a/src/main/java/org/radarcns/config/AppServerConfig.kt b/src/main/java/org/radarcns/config/AppServerConfig.kt new file mode 100644 index 00000000..274ff912 --- /dev/null +++ b/src/main/java/org/radarcns/config/AppServerConfig.kt @@ -0,0 +1,26 @@ +package org.radarcns.config + +import com.fasterxml.jackson.annotation.JsonProperty + +data class AppServerConfig( + @JsonProperty("base_url") + val baseUrl: String, + @JsonProperty("token_url") + val tokenUrl: String, + @JsonProperty("client_id") + var clientId: String? = null, + @JsonProperty("client_secret") + var clientSecret: String? = null, +) { + fun withEnv(): AppServerConfig { + val envClientId = System.getenv("APP_SERVER_CLIENT_ID") + if (envClientId != null) { + clientId = envClientId + } + val envClientSecret = System.getenv("APP_SERVER_CLIENT_SECRET") + if (envClientSecret != null) { + clientSecret = envClientSecret + } + return this + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/config/ConfigRadar.java b/src/main/java/org/radarcns/config/ConfigRadar.java index e9b70c63..adc0ebfd 100644 --- a/src/main/java/org/radarcns/config/ConfigRadar.java +++ b/src/main/java/org/radarcns/config/ConfigRadar.java @@ -16,13 +16,21 @@ package org.radarcns.config; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; +import com.fasterxml.jackson.module.kotlin.KotlinModule; import java.util.Date; import java.util.List; import java.util.Map; import org.radarbase.config.ServerConfig; import org.radarbase.config.YamlConfigLoader; +import org.radarcns.config.monitor.BatteryMonitorConfig; +import org.radarcns.config.monitor.DisconnectMonitorConfig; +import org.radarcns.config.monitor.InterventionMonitorConfig; +import org.radarcns.config.realtime.RealtimeConsumerConfig; /** * POJO representing the yml file @@ -30,7 +38,6 @@ public class ConfigRadar { private Date released; private String version; - private List zookeeper; private List broker; @JsonProperty("schema_registry") private List schemaRegistry; @@ -42,6 +49,10 @@ public class ConfigRadar { private DisconnectMonitorConfig disconnectMonitor; @JsonProperty("statistics_monitors") private List statisticsMonitors; + @JsonProperty("intervention_monitor") + private InterventionMonitorConfig interventionMonitor; + @JsonProperty("realtime_consumers") + private List consumerConfigs; @JsonProperty("stream") private StreamConfig stream; @JsonProperty("persistence_path") @@ -51,6 +62,12 @@ public class ConfigRadar { @JsonProperty("build_version") private String buildVersion; + @JsonProperty("email_server") + private EmailServerConfig emailServerConfig; + + @JsonIgnore + private YamlConfigLoader loader; + public Date getReleased() { return released; } @@ -67,14 +84,6 @@ public void setVersion(String version) { this.version = version; } - public List getZookeeper() { - return zookeeper; - } - - public void setZookeeper(List zookeeper) { - this.zookeeper = zookeeper; - } - public List getBroker() { return broker; } @@ -99,13 +108,6 @@ public void setRestProxy(ServerConfig restProxy) { this.restProxy = restProxy; } - public String getZookeeperPaths() { - if (zookeeper == null) { - throw new IllegalStateException("'zookeeper' is not configured"); - } - return ServerConfig.getPaths(zookeeper); - } - public String getBrokerPaths() { if (broker == null) { throw new IllegalStateException("Kafka 'broker' is not configured"); @@ -161,9 +163,17 @@ public void setExtras(Map extras) { this.extras = extras; } + public void setConfigLoader(YamlConfigLoader loader) { + this.loader = loader; + } + @Override public String toString() { - return new YamlConfigLoader().prettyString(this); + if (loader != null) { + return loader.prettyString(this); + } else { + return "ConfigRadar(...)"; + } } public String getBuildVersion() { @@ -182,7 +192,31 @@ public void setStatisticsMonitors(List statisticsM this.statisticsMonitors = statisticsMonitors; } + public List getConsumerConfigs() { + return consumerConfigs; + } + + public void setConsumerConfigs(List consumerConfigs) { + this.consumerConfigs = consumerConfigs; + } + public StreamConfig getStream() { return stream; } + + public InterventionMonitorConfig getInterventionMonitor() { + return interventionMonitor; + } + + public void setInterventionMonitor(InterventionMonitorConfig notificationMonitor) { + this.interventionMonitor = notificationMonitor; + } + + public EmailServerConfig getEmailServerConfig() { + return emailServerConfig; + } + + public void setEmailServerConfig(EmailServerConfig emailServerConfig) { + this.emailServerConfig = emailServerConfig; + } } diff --git a/src/main/java/org/radarcns/config/EmailServerConfig.java b/src/main/java/org/radarcns/config/EmailServerConfig.java new file mode 100644 index 00000000..cfe5ae67 --- /dev/null +++ b/src/main/java/org/radarcns/config/EmailServerConfig.java @@ -0,0 +1,33 @@ +package org.radarcns.config; + +public class EmailServerConfig { + private String host; + + private int port; + + private String user; + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } +} diff --git a/src/main/java/org/radarcns/config/KafkaProperty.java b/src/main/java/org/radarcns/config/KafkaProperty.java index ce28cb3e..b1cf990e 100644 --- a/src/main/java/org/radarcns/config/KafkaProperty.java +++ b/src/main/java/org/radarcns/config/KafkaProperty.java @@ -16,7 +16,9 @@ package org.radarcns.config; -import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; +import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS; +import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; + import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; import java.util.Properties; import javax.annotation.Nonnull; @@ -46,9 +48,9 @@ public Properties getStreamProperties(@Nonnull String clientId, props.put(StreamsConfig.APPLICATION_ID_CONFIG, clientId); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, configRadar.getBrokerPaths()); - props.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, + props.put(SCHEMA_REGISTRY_URL_CONFIG, configRadar.getSchemaRegistryPaths()); - props.put(AbstractKafkaAvroSerDeConfig.AUTO_REGISTER_SCHEMAS, true); + props.put(AUTO_REGISTER_SCHEMAS, true); props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, SpecificAvroSerde.class); props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, diff --git a/src/main/java/org/radarcns/config/RadarPropertyHandler.java b/src/main/java/org/radarcns/config/RadarPropertyHandler.java index 797a63b7..a20270d5 100644 --- a/src/main/java/org/radarcns/config/RadarPropertyHandler.java +++ b/src/main/java/org/radarcns/config/RadarPropertyHandler.java @@ -17,6 +17,7 @@ package org.radarcns.config; import java.io.IOException; +import org.radarbase.config.YamlConfigLoader; import org.radarcns.util.PersistentStateStore; /** @@ -40,6 +41,8 @@ public String getParam() { ConfigRadar getRadarProperties(); + YamlConfigLoader getLoader(); + void load(String pathFile) throws IOException; boolean isLoaded(); diff --git a/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java b/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java index ee334a3c..66cfe68c 100644 --- a/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java +++ b/src/main/java/org/radarcns/config/RadarPropertyHandlerImpl.java @@ -19,6 +19,9 @@ import static org.radarbase.util.Strings.isNullOrEmpty; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; +import com.fasterxml.jackson.module.kotlin.KotlinModule; import java.io.IOException; import java.io.InputStream; import java.net.URISyntaxException; @@ -44,6 +47,18 @@ public class RadarPropertyHandlerImpl implements RadarPropertyHandler { private ConfigRadar properties; private KafkaProperty kafkaProperty; + private YamlConfigLoader loader; + + public RadarPropertyHandlerImpl() { + loader = new YamlConfigLoader(mapper -> { + mapper.registerModule(new KotlinModule.Builder() + .enable(KotlinFeature.NullIsSameAsDefault) + .enable(KotlinFeature.NullToEmptyCollection) + .enable(KotlinFeature.NullToEmptyMap) + .build()); + mapper.registerModule(new JavaTimeModule()); + }); + } @Override public ConfigRadar getRadarProperties() { @@ -54,6 +69,11 @@ public ConfigRadar getRadarProperties() { return properties; } + @Override + public YamlConfigLoader getLoader() { + return loader; + } + @Override public boolean isLoaded() { return properties != null; @@ -79,7 +99,8 @@ public void load(String pathFile) throws IOException { if (!Files.exists(file)) { throw new IllegalArgumentException("Config file " + file + " does not exist"); } - properties = new YamlConfigLoader().load(file, ConfigRadar.class); + properties = loader.load(file, ConfigRadar.class); + properties.setConfigLoader(loader); Properties buildProperties = new Properties(); try (InputStream in = getClass().getResourceAsStream("/build.properties")) { @@ -122,7 +143,7 @@ public KafkaProperty getKafkaProperties() { public PersistentStateStore getPersistentStateStore() throws IOException { if (getRadarProperties().getPersistencePath() != null) { Path persistenceDir = Paths.get(getRadarProperties().getPersistencePath()); - return new YamlPersistentStateStore(persistenceDir); + return new YamlPersistentStateStore(loader, persistenceDir); } else { return null; } diff --git a/src/main/java/org/radarcns/config/SingleStreamConfig.java b/src/main/java/org/radarcns/config/SingleStreamConfig.java index b3522c91..3c1a021a 100644 --- a/src/main/java/org/radarcns/config/SingleStreamConfig.java +++ b/src/main/java/org/radarcns/config/SingleStreamConfig.java @@ -7,8 +7,9 @@ import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; + +import org.radarbase.stream.TimeWindowMetadata; import org.radarcns.config.RadarPropertyHandler.Priority; -import org.radarcns.stream.TimeWindowMetadata; public class SingleStreamConfig { @JsonProperty("class") diff --git a/src/main/java/org/radarcns/config/StreamConfig.java b/src/main/java/org/radarcns/config/StreamConfig.java index 17cad645..39072034 100644 --- a/src/main/java/org/radarcns/config/StreamConfig.java +++ b/src/main/java/org/radarcns/config/StreamConfig.java @@ -15,8 +15,9 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; + +import org.radarbase.stream.TimeWindowMetadata; import org.radarcns.config.RadarPropertyHandler.Priority; -import org.radarcns.stream.TimeWindowMetadata; // POJO class @SuppressWarnings("PMD.ImmutableField") diff --git a/src/main/java/org/radarcns/config/monitor/AuthConfig.kt b/src/main/java/org/radarcns/config/monitor/AuthConfig.kt new file mode 100644 index 00000000..a1726e95 --- /dev/null +++ b/src/main/java/org/radarcns/config/monitor/AuthConfig.kt @@ -0,0 +1,25 @@ +package org.radarcns.config.monitor + +import com.fasterxml.jackson.annotation.JsonProperty + +data class AuthConfig( + @JsonProperty("clientId") + val clientId: String? = null, + @JsonProperty("clientSecret") + val clientSecret: String? = null, + @JsonProperty("token_url") + val tokenUrl: String, +) { + fun withEnv(): AuthConfig { + var result = this; + val envClientId = System.getenv("AUTH_CLIENT_ID") + if (envClientId != null) { + result = result.copy(clientId = envClientId) + } + val envClientSecret = System.getenv("AUTH_CLIENT_SECRET") + if (envClientSecret != null) { + result = result.copy(clientSecret = envClientSecret) + } + return result + } +} diff --git a/src/main/java/org/radarcns/config/BatteryMonitorConfig.java b/src/main/java/org/radarcns/config/monitor/BatteryMonitorConfig.java similarity index 96% rename from src/main/java/org/radarcns/config/BatteryMonitorConfig.java rename to src/main/java/org/radarcns/config/monitor/BatteryMonitorConfig.java index 04c384e7..4718bf44 100644 --- a/src/main/java/org/radarcns/config/BatteryMonitorConfig.java +++ b/src/main/java/org/radarcns/config/monitor/BatteryMonitorConfig.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarcns.config.monitor; /** * POJO representing a battery status monitor configuration. diff --git a/src/main/java/org/radarcns/config/monitor/Condition.java b/src/main/java/org/radarcns/config/monitor/Condition.java new file mode 100644 index 00000000..82c2b670 --- /dev/null +++ b/src/main/java/org/radarcns/config/monitor/Condition.java @@ -0,0 +1,44 @@ +package org.radarcns.config.monitor; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; + +public class Condition { + + // Json path string which will be evaluated to trigger a notification + @JsonProperty("json_path") + String jsonPathCondition; + + // trigger notification, if all more than one condition should be met + @JsonProperty("and") + List andConditions; + + // trigger notification, if one of the many conditions should be met + @JsonProperty("or") + List orConditions; + + public String getJsonPathCondition() { + return jsonPathCondition; + } + + public void setJsonPathCondition(String jsonPathCondition) { + this.jsonPathCondition = jsonPathCondition; + } + + public List getAndConditions() { + return andConditions; + } + + public void setAndConditions(List andConditions) { + this.andConditions = andConditions; + } + + public List getOrConditions() { + return orConditions; + } + + public void setOrConditions(List orConditions) { + this.orConditions = orConditions; + } +} diff --git a/src/main/java/org/radarcns/config/DisconnectMonitorConfig.java b/src/main/java/org/radarcns/config/monitor/DisconnectMonitorConfig.java similarity index 97% rename from src/main/java/org/radarcns/config/DisconnectMonitorConfig.java rename to src/main/java/org/radarcns/config/monitor/DisconnectMonitorConfig.java index c66e863f..9df01347 100644 --- a/src/main/java/org/radarcns/config/DisconnectMonitorConfig.java +++ b/src/main/java/org/radarcns/config/monitor/DisconnectMonitorConfig.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarcns.config.monitor; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/src/main/java/org/radarcns/config/NotifyConfig.java b/src/main/java/org/radarcns/config/monitor/EmailNotifyConfig.java similarity index 77% rename from src/main/java/org/radarcns/config/NotifyConfig.java rename to src/main/java/org/radarcns/config/monitor/EmailNotifyConfig.java index 61537ea2..e67fbe0e 100644 --- a/src/main/java/org/radarcns/config/NotifyConfig.java +++ b/src/main/java/org/radarcns/config/monitor/EmailNotifyConfig.java @@ -1,4 +1,4 @@ -package org.radarcns.config; +package org.radarcns.config.monitor; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -8,7 +8,7 @@ /** * POJO to store each email Notification configuration. */ -public class NotifyConfig { +public class EmailNotifyConfig { @JsonProperty("project_id") private String projectId; @@ -16,8 +16,8 @@ public class NotifyConfig { private List emailAddress; @JsonCreator - public NotifyConfig(@JsonProperty("project_id") String projectId, - @JsonProperty("email_address") List emailAddress) { + public EmailNotifyConfig(@JsonProperty("project_id") String projectId, + @JsonProperty("email_address") List emailAddress) { this.projectId = projectId; this.emailAddress = emailAddress; } diff --git a/src/main/java/org/radarcns/config/monitor/InterventionMonitorConfig.kt b/src/main/java/org/radarcns/config/monitor/InterventionMonitorConfig.kt new file mode 100644 index 00000000..71748f1c --- /dev/null +++ b/src/main/java/org/radarcns/config/monitor/InterventionMonitorConfig.kt @@ -0,0 +1,41 @@ +package org.radarcns.config.monitor + +import com.fasterxml.jackson.annotation.JsonProperty +import java.time.Duration + +data class InterventionMonitorConfig( + @JsonProperty("app_server_url") + val appServerUrl: String, + @JsonProperty("app_config_client") + val ksqlAppConfigClient: String, + @JsonProperty("app_config_url") + val appConfigUrl: String, + @JsonProperty("auth") + val authConfig: AuthConfig, + + @JsonProperty("notify") + val emailNotifyConfig: List = listOf(), + + // The list of intervention topics, which will be used to evaluate the conditions + val topics: List, + + // List of notification configs and corresponding conditions to trigger + @JsonProperty("ttl_margin") + val ttlMargin: Duration = Duration.ofMinutes(5), + val properties: Map = mapOf(), + val deadline: Duration = Duration.ofMinutes(15), + @JsonProperty("state_reset_interval") + val stateResetInterval: Duration = Duration.ofHours(24), + + @JsonProperty("threshold_adaptation") + val thresholdAdaptation: ThresholdAdaptationConfig = ThresholdAdaptationConfig(), + + @JsonProperty("max_interventions") + val maxInterventions: Int = 4, + + @JsonProperty("protocol_directory") + val protocolDirectory: String, + val defaultLanguage: String = "en", +) { + fun withEnv(): InterventionMonitorConfig = copy(authConfig = authConfig.withEnv()) +} diff --git a/src/main/java/org/radarcns/config/MonitorConfig.java b/src/main/java/org/radarcns/config/monitor/MonitorConfig.java similarity index 61% rename from src/main/java/org/radarcns/config/MonitorConfig.java rename to src/main/java/org/radarcns/config/monitor/MonitorConfig.java index 7e132db6..463d5eb6 100644 --- a/src/main/java/org/radarcns/config/MonitorConfig.java +++ b/src/main/java/org/radarcns/config/monitor/MonitorConfig.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.radarcns.config; +package org.radarcns.config.monitor; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; @@ -24,16 +24,7 @@ */ public class MonitorConfig { @JsonProperty("notify") - private List notifyConfig; - - @JsonProperty("email_host") - private String emailHost; - - @JsonProperty("email_port") - private int emailPort; - - @JsonProperty("email_user") - private String emailUser; + private List emailNotifyConfig; @JsonProperty("log_interval") private int logInterval = 1000; @@ -43,14 +34,6 @@ public class MonitorConfig { @JsonProperty("message") private String message = null; - public List getNotifyConfig() { - return notifyConfig; - } - - public void setNotifyConfig(List notifyConfig) { - this.notifyConfig = notifyConfig; - } - public List getTopics() { return topics; } @@ -59,30 +42,6 @@ public void setTopics(List topics) { this.topics = topics; } - public String getEmailHost() { - return emailHost; - } - - public void setEmailHost(String emailHost) { - this.emailHost = emailHost; - } - - public int getEmailPort() { - return emailPort; - } - - public void setEmailPort(int emailPort) { - this.emailPort = emailPort; - } - - public String getEmailUser() { - return emailUser; - } - - public void setEmailUser(String emailUser) { - this.emailUser = emailUser; - } - public int getLogInterval() { return logInterval; } @@ -98,4 +57,12 @@ public String getMessage() { public void setMessage(String message) { this.message = message; } + + public List getEmailNotifyConfig() { + return emailNotifyConfig; + } + + public void setEmailNotifyConfig(List emailNotifyConfig) { + this.emailNotifyConfig = emailNotifyConfig; + } } diff --git a/src/main/java/org/radarcns/config/monitor/PushNotificationConfig.kt b/src/main/java/org/radarcns/config/monitor/PushNotificationConfig.kt new file mode 100644 index 00000000..80a55938 --- /dev/null +++ b/src/main/java/org/radarcns/config/monitor/PushNotificationConfig.kt @@ -0,0 +1,10 @@ +package org.radarcns.config.monitor + +import java.time.Duration + +class PushNotificationConfig { + var project: String? = null + var condition: Condition? = null + var questionnaire: List? = null + var ttlMargin: Duration = Duration.ofMinutes(5) +} diff --git a/src/main/java/org/radarcns/config/monitor/ThresholdAdaptationConfig.kt b/src/main/java/org/radarcns/config/monitor/ThresholdAdaptationConfig.kt new file mode 100644 index 00000000..89b8d54c --- /dev/null +++ b/src/main/java/org/radarcns/config/monitor/ThresholdAdaptationConfig.kt @@ -0,0 +1,10 @@ +package org.radarcns.config.monitor + +import com.fasterxml.jackson.annotation.JsonProperty + +data class ThresholdAdaptationConfig( + @JsonProperty("adjust_value") + val adjustValue: Float = 0.01f, + @JsonProperty("optimal_interventions") + val optimalInterventions: Int = 3, +) diff --git a/src/main/java/org/radarcns/config/realtime/ActionConfig.kt b/src/main/java/org/radarcns/config/realtime/ActionConfig.kt new file mode 100644 index 00000000..6af96e8a --- /dev/null +++ b/src/main/java/org/radarcns/config/realtime/ActionConfig.kt @@ -0,0 +1,12 @@ +package org.radarcns.config.realtime + +data class ActionConfig( + override val name: String, + override val properties: Map? = null, + override val projects: List? = null, + override val subjects: List? = null, + override val projectIdField: String? = null, + override val subjectIdField: String? = null, + override val sourceIdField: String? = null, + override val timeField: String? = null, +): BaseConfig \ No newline at end of file diff --git a/src/main/java/org/radarcns/config/realtime/BaseConfig.kt b/src/main/java/org/radarcns/config/realtime/BaseConfig.kt new file mode 100644 index 00000000..c03b7038 --- /dev/null +++ b/src/main/java/org/radarcns/config/realtime/BaseConfig.kt @@ -0,0 +1,12 @@ +package org.radarcns.config.realtime + +interface BaseConfig { + val name: String + val properties: Map? + val projects: List? + val subjects: List? + val projectIdField: String? + val subjectIdField: String? + val sourceIdField: String? + val timeField: String? +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/config/realtime/ConditionConfig.kt b/src/main/java/org/radarcns/config/realtime/ConditionConfig.kt new file mode 100644 index 00000000..f50bab57 --- /dev/null +++ b/src/main/java/org/radarcns/config/realtime/ConditionConfig.kt @@ -0,0 +1,12 @@ +package org.radarcns.config.realtime + +data class ConditionConfig( + override val name: String, + override val properties: Map? = null, + override val projects: List? = null, + override val subjects: List? = null, + override val projectIdField: String? = null, + override val subjectIdField: String? = null, + override val sourceIdField: String? = null, + override val timeField: String? = null, +) : BaseConfig \ No newline at end of file diff --git a/src/main/java/org/radarcns/config/realtime/RealtimeConsumerConfig.kt b/src/main/java/org/radarcns/config/realtime/RealtimeConsumerConfig.kt new file mode 100644 index 00000000..b88c107e --- /dev/null +++ b/src/main/java/org/radarcns/config/realtime/RealtimeConsumerConfig.kt @@ -0,0 +1,21 @@ +package org.radarcns.config.realtime + +import com.fasterxml.jackson.annotation.JsonProperty + +data class RealtimeConsumerConfig( + val name: String, + val topic: String, + @JsonProperty("notify_errors") + val notifyErrorsEmails: NotifyErrorConfig? = null, + @JsonProperty("conditions") + val conditionConfigs: List, + @JsonProperty("actions") + val actionConfigs: List, + @JsonProperty("consumer_properties") + val consumerProperties: Map? = null, +) + +data class NotifyErrorConfig( + @JsonProperty("email_addresses") + val emailAddresses: List? = null, +) \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/Grouping.kt b/src/main/java/org/radarcns/consumer/realtime/Grouping.kt new file mode 100644 index 00000000..c38c57cb --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/Grouping.kt @@ -0,0 +1,88 @@ +package org.radarcns.consumer.realtime + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.KotlinFeature +import com.fasterxml.jackson.module.kotlin.KotlinModule +import org.apache.avro.generic.GenericRecord +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarcns.kafka.ObservationKey +import java.io.IOException + +interface Grouping { + val projects: List? + val subjects: List? + val projectIdField: String? + val subjectIdField: String? + val sourceIdField: String? + val timeField: String? + + @Throws(IOException::class) + fun evaluateProject(record: ConsumerRecord<*, *>?): Boolean { + return try { + val key = getObservationKey(record) + return key != null + && projects?.contains(key.projectId) ?: true + && subjects?.contains(key.userId) ?: true + } catch (ex: IllegalArgumentException) { + false + } + } + + fun getKeys(record: ConsumerRecord<*, *>?): ObservationKey? { + return getObservationKey(record) + } + + fun getTime(record: ConsumerRecord<*, *>?): Long { + return record?.value()?.let { v -> + val value = v as GenericRecord + val key = timeField + ?: TIME_VALUE_KEYS.find { k -> value.hasField(k) } + ?: throw IllegalArgumentException("No time found in key") + value[key].toString().toDouble().toLong() + } ?: throw IllegalArgumentException("Time could not be parsed from record") + } + + private fun getObservationKey(record: ConsumerRecord<*, *>?): ObservationKey? { + return record?.key()?.let { k -> + val key = k as GenericRecord + + val pidKey: String = projectIdField + ?: findKey(record, PROJECT_ID_KEYS) + ?: throw IllegalArgumentException("No project id found in key") + + val uidKey: String = subjectIdField + ?: findKey(record, USER_ID_KEYS) + ?: throw IllegalArgumentException("No user id found in key") + + val sidKey: String? = sourceIdField ?: findKey(record, SOURCE_ID_KEYS) + + val project = key[pidKey].toString() + val user = key[uidKey].toString() + val source = if (sidKey != null) key[sidKey].toString() else null + ObservationKey(project, user, source) + } + } + + companion object { + val PROJECT_ID_KEYS = arrayOf("projectId", "PROJECTID", "project_id", "PROJECT_ID") + val USER_ID_KEYS = arrayOf("userId", "USERID", "user_id", "USER_ID") + val SOURCE_ID_KEYS = arrayOf("sourceId", "SOURCEID", "source_id", "SOURCE_ID") + val TIME_VALUE_KEYS = arrayOf("time", "TIME") + + val objectMapper: ObjectMapper = ObjectMapper() + .registerModule(KotlinModule.Builder() + .enable(KotlinFeature.NullIsSameAsDefault) + .enable(KotlinFeature.NullToEmptyCollection) + .enable(KotlinFeature.NullToEmptyMap) + .build()) + .registerModule(JavaTimeModule()) + + fun findKey(record: ConsumerRecord<*, *>?, keys: Array): String? { + return record?.key()?.let { + val key = it as GenericRecord + keys.find { k -> key.hasField(k) } + } + } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/RealtimeInferenceConsumer.kt b/src/main/java/org/radarcns/consumer/realtime/RealtimeInferenceConsumer.kt new file mode 100644 index 00000000..5b1cff82 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/RealtimeInferenceConsumer.kt @@ -0,0 +1,209 @@ +package org.radarcns.consumer.realtime + +import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig +import io.confluent.kafka.serializers.KafkaAvroDeserializer +import org.apache.avro.generic.GenericRecord +import org.apache.kafka.clients.consumer.Consumer +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.clients.consumer.KafkaConsumer +import org.apache.kafka.common.KafkaException +import org.apache.kafka.common.errors.InterruptException +import org.apache.kafka.common.errors.SerializationException +import org.apache.kafka.common.errors.WakeupException +import org.radarbase.util.RollingTimeAverage +import org.radarcns.config.ConfigRadar +import org.radarcns.config.realtime.ActionConfig +import org.radarcns.config.realtime.ConditionConfig +import org.radarcns.config.realtime.RealtimeConsumerConfig +import org.radarcns.consumer.realtime.RealtimeInferenceConsumer +import org.radarcns.consumer.realtime.action.Action +import org.radarcns.consumer.realtime.action.ActionFactory.getActionFor +import org.radarcns.consumer.realtime.condition.Condition +import org.radarcns.consumer.realtime.condition.ConditionFactory.getConditionFor +import org.radarcns.monitor.KafkaMonitor +import org.radarcns.util.EmailSender +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Duration +import java.util.* + +/** + * The main Kafka Consumer class that runs a single consumer on any topic. The consumer evaluates + * each incoming record based on the [Condition]s provided in the config. If and only If all + * the conditions evaluate to true, only then all the configured [Action]s are fired. + * + * + * To be used with the model-invocation-endpoint and KSQL API_INFERENCE function to evaluate and + * take action on incoming results from realtime inference on data. + */ +class RealtimeInferenceConsumer( + groupId: String?, clientId: String, radar: ConfigRadar, consumerConfig: RealtimeConsumerConfig) : KafkaMonitor { + private val properties: Properties = Properties().apply { + setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId) + setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "${consumerConfig.name}-$clientId") + setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true") + val deserializer = KafkaAvroDeserializer::class.java.name + setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, deserializer) + setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, deserializer) + setProperty(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1001") + setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "15101") + setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "7500") + setProperty(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, radar.schemaRegistryPaths) + setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, radar.brokerPaths) + // Override with any properties specified for this consumer + consumerConfig.consumerProperties?.let { putAll(it) } + } + private val conditions: List = consumerConfig.conditionConfigs + .map { c: ConditionConfig -> + getConditionFor(c) + } + private val actions: List = consumerConfig.actionConfigs + .map { a: ActionConfig -> + getActionFor(radar, a) + } + + private val topic: String = consumerConfig.topic + private var done: Boolean = false + private var pollTimeout: Duration = Duration.ofDays(365) + private var consumer: Consumer? = null + private val notifyErrorEmailSender: EmailSender? + + init { + require(topic.isNotBlank()) { "Cannot start consumer without topic." } + require(!(conditions.isEmpty() || actions.isEmpty())) { + "At least one each of condition and action is necessary to run the consumer." + } + + notifyErrorEmailSender = consumerConfig.notifyErrorsEmails?.emailAddresses?.let { + EmailSender(radar.emailServerConfig, radar.emailServerConfig.user, it) + } + } + + @Throws(IOException::class, InterruptedException::class) + override fun start() { + consumer = KafkaConsumer(properties) + consumer?.subscribe(setOf(topic)) + logger.info("Consuming realtime inference topic {}", topic) + val ops = RollingTimeAverage(20000) + try { + while (!isShutdown) { + try { + val records = consumer?.poll(getPollTimeout()) ?: continue + ops.add(records.count().toDouble()) + for (record in records) { + if (conditions.evaluateAll(record)) { + // Only execute the actions if all the conditions are true + actions.executeAll(record) + } + } + } catch (ex: SerializationException) { + logger.warn("Failed to deserialize the record: {}", ex.message) + notifyErrorEmailSender?.notifyErrors(null, ex) + } catch (ex: WakeupException) { + logger.info("Consumer woke up") + notifyErrorEmailSender?.notifyErrors(null, ex) + } catch (ex: InterruptException) { + logger.info("Consumer was interrupted") + notifyErrorEmailSender?.notifyErrors(null, ex) + shutdown() + } catch (ex: KafkaException) { + logger.error("Kafka consumer gave exception", ex) + notifyErrorEmailSender?.notifyErrors(null, ex) + } catch (ex: Exception) { + logger.error("Consumer gave exception", ex) + notifyErrorEmailSender?.notifyErrors(null, ex) + } + } + } finally { + consumer?.close() + } + } + + private fun List.evaluateAll(record: ConsumerRecord): Boolean { + return this.all { c: Condition -> + try { + c.evaluate(record) + } catch (exc: IOException) { + logger.warn( + "I/O Error evaluating one of the conditions: {}. Will not continue.", + c.name, + exc) + notifyErrorEmailSender?.notifyErrors(record, exc, condition = c) + false + } catch (exc: Exception) { + logger.warn( + "Error evaluating one of the conditions: {}. Will not continue.", + c.name, + exc) + notifyErrorEmailSender?.notifyErrors(record, exc, condition = c) + false + } + } + } + + private fun List.executeAll(record: ConsumerRecord): List { + return this.map { a: Action -> + try { + a.run(record) + } catch (ex: IllegalArgumentException) { + logger.warn("Argument was not valid. Error executing action", ex) + notifyErrorEmailSender?.notifyErrors(record, ex, a) + false + } catch (ex: IOException) { + logger.warn("I/O Error executing action", ex) + notifyErrorEmailSender?.notifyErrors(record, ex, action = a) + false + } catch (ex: Exception) { + // Catch all exceptions so that we can continue to execute the other actions + logger.warn("Error executing action", ex) + notifyErrorEmailSender?.notifyErrors(record, ex, action = a) + false + } + } + } + + @Throws(IOException::class, InterruptedException::class) + override fun shutdown() { + logger.info("Shutting down consumer {}", javaClass.simpleName) + done = true + consumer?.wakeup() + } + + override fun isShutdown(): Boolean { + return done + } + + override fun getPollTimeout(): Duration { + return pollTimeout + } + + override fun setPollTimeout(duration: Duration) { + pollTimeout = duration + } + + companion object { + private val logger = LoggerFactory.getLogger(RealtimeInferenceConsumer::class.java) + + private fun EmailSender.notifyErrors( + record: ConsumerRecord?, + throwable: Throwable, + action: Action? = null, + condition: Condition? = null, + ) { + val prefix = when { + action != null -> "Action ${action.name} failed" + condition != null -> "Condition ${condition.name} failed" + else -> "Unknown error" + } + + val customSubject = "$prefix for ${record?.key()?.toString() ?: "Realtime Inference"}" + + val customBody = "$prefix for ${record?.key()?.toString()}:\n" + + "Error: ${throwable.message}\n\n" + + "Stacktrace: \n${throwable.stackTrace.joinToString("\n\t")}" + + sendEmail(customSubject, customBody) + } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/RealtimeInferenceConsumerFactory.kt b/src/main/java/org/radarcns/consumer/realtime/RealtimeInferenceConsumerFactory.kt new file mode 100644 index 00000000..ddb34c80 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/RealtimeInferenceConsumerFactory.kt @@ -0,0 +1,32 @@ +package org.radarcns.consumer.realtime + +import org.radarcns.config.ConfigRadar +import org.radarcns.config.RadarPropertyHandler +import org.radarcns.config.realtime.RealtimeConsumerConfig +import org.radarcns.monitor.CombinedKafkaMonitor +import org.radarcns.monitor.KafkaMonitor + +/** + * Factory class for [RealtimeInferenceConsumer]. There can be multiple consumers, each with + * its own set of [org.radarcns.consumer.realtime.condition.Condition]s and [ ]s. + */ +object RealtimeInferenceConsumerFactory { + @JvmStatic + fun createConsumersFor(handler: RadarPropertyHandler): KafkaMonitor { + return CombinedKafkaMonitor( + handler.radarProperties.consumerConfigs.stream() + .map { c: RealtimeConsumerConfig -> + createConsumer(handler.radarProperties, c) + } + ) + } + + private fun createConsumer( + radar: ConfigRadar, consumerConfig: RealtimeConsumerConfig): KafkaMonitor { + return RealtimeInferenceConsumer( + "realtime-group-${consumerConfig.topic}-${consumerConfig.name}", + "${consumerConfig.topic}-1", + radar, + consumerConfig) + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/Action.kt b/src/main/java/org/radarcns/consumer/realtime/action/Action.kt new file mode 100644 index 00000000..d8f46226 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/Action.kt @@ -0,0 +1,25 @@ +package org.radarcns.consumer.realtime.action + +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarcns.consumer.realtime.Grouping +import java.io.IOException + +/** + * An action can be defined as any process that needs to take place when data is received and all + * the [org.radarcns.consumer.realtime.condition.Condition]s have evaluated to true. It can be + * emailing someone or just logging something. + * + * + * See [ActiveAppNotificationAction], [EmailUserAction] + */ +interface Action : Grouping { + val name: String + + @Throws(IllegalArgumentException::class, IOException::class) + fun executeFor(record: ConsumerRecord<*, *>?): Boolean + + @Throws(IllegalArgumentException::class, IOException::class) + fun run(record: ConsumerRecord<*, *>?): Boolean { + return evaluateProject(record) && executeFor(record) + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/ActionBase.kt b/src/main/java/org/radarcns/consumer/realtime/action/ActionBase.kt new file mode 100644 index 00000000..bcbf8812 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/ActionBase.kt @@ -0,0 +1,13 @@ +package org.radarcns.consumer.realtime.action + +import org.radarcns.config.realtime.ActionConfig + +abstract class ActionBase( + val config: ActionConfig, + override val projects: List? = config.projects, + override val subjects: List? = config.subjects, + override val subjectIdField: String? = config.projectIdField, + override val projectIdField: String? = config.subjectIdField, + override val timeField: String? = config.timeField, + override val sourceIdField: String? = config.sourceIdField, +) : Action \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/ActionFactory.kt b/src/main/java/org/radarcns/consumer/realtime/action/ActionFactory.kt new file mode 100644 index 00000000..b6fad12e --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/ActionFactory.kt @@ -0,0 +1,46 @@ +package org.radarcns.consumer.realtime.action + +import org.radarcns.config.ConfigRadar +import org.radarcns.config.realtime.ActionConfig +import java.io.IOException +import java.net.MalformedURLException +import javax.mail.internet.AddressException + +/** + * Factory class for [Action]s. It instantiates actions based on the configuration provided + * for the given consumer. + */ +object ActionFactory { + @JvmStatic + fun getActionFor(config: ConfigRadar, actionConfig: ActionConfig): Action { + when (actionConfig.name) { + ActiveAppNotificationAction.NAME -> { + return try { + ActiveAppNotificationAction(actionConfig) + } catch (exc: MalformedURLException) { + throw IllegalArgumentException( + "The supplied url config was incorrect. Please check.", exc) + } + } + AppServerTriggerAction.NAME -> { + return try { + AppServerTriggerAction(actionConfig) + } catch (exc: MalformedURLException) { + throw IllegalArgumentException( + "The supplied url config was incorrect. Please check.", exc) + } + } + EmailUserAction.NAME -> { + return try { + EmailUserAction(actionConfig, config.emailServerConfig) + } catch (e: AddressException) { + throw IllegalArgumentException("The configuration was invalid. Please check.", e) + } catch (e: IOException) { + throw IllegalArgumentException("The configuration was invalid. Please check.", e) + } + } + else -> throw IllegalArgumentException( + "The specified action with name " + actionConfig.name + " is " + "not correct.") + } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/ActiveAppNotificationAction.kt b/src/main/java/org/radarcns/consumer/realtime/action/ActiveAppNotificationAction.kt new file mode 100644 index 00000000..34c4fcb0 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/ActiveAppNotificationAction.kt @@ -0,0 +1,200 @@ +package org.radarcns.consumer.realtime.action + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.avro.generic.GenericRecord +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarbase.appserver.client.AppserverClient +import org.radarbase.appserver.client.AppserverClientConfig +import org.radarbase.appserver.client.MessagingType +import org.radarcns.config.realtime.ActionConfig +import org.radarcns.consumer.realtime.Grouping.Companion.objectMapper +import org.radarcns.consumer.realtime.action.appserver.* +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit + +/** + * This action can be used to trigger a notification for the aRMT app and schedule a corresponding + * questionnaire for the user to fill out. This can also work as an intervention mechanism in some + * use-cases. + */ +class ActiveAppNotificationAction( + actionConfig: ActionConfig, + override val name: String = NAME, +) : ActionBase(actionConfig) { + private val parsedConfig: ActiveAppNotificationActionConfig + private val appserverClient: AppserverClient + + + init { + parsedConfig = ActiveAppNotificationActionConfig.fromMap(actionConfig.properties) + appserverClient = AppserverClient(parsedConfig.appserverClientConfig) + } + + @Throws(IllegalArgumentException::class, IOException::class) + override fun executeFor(record: ConsumerRecord<*, *>?): Boolean { + + logger.debug("Executing action for record: $record") + + val key = getKeys(record) ?: return false + + val timezone = getUserTimezone(key.projectId, key.userId) + logger.debug("Got user timezone") + + val timeStrategy: ScheduleTimeStrategy = if (!parsedConfig.timeOfDay.isNullOrEmpty()) { + // get timezone for the user and create the correct local time of the day + TimeOfDayStrategy(parsedConfig.timeOfDay, timezone, parsedConfig.jitterInMinutes?.let { + Duration.ofMinutes(it.toLong()) + }) + } else { + // no time of the day provided, schedule now. + SimpleTimeStrategy(parsedConfig.optionalDelayMinutes.toLong(), ChronoUnit.MINUTES) + } + + val metadata = try { + if (!parsedConfig.metadataKey.isNullOrEmpty()) { + val root = ObjectMapper() + .readTree( + (record?.value() as GenericRecord?) + ?.get(parsedConfig.metadataKey).toString() + ) + if (root.isArray) { + root.mapIndexed { k1, v1 -> + Pair("metadata-$k1", objectMapper.writeValueAsString(v1)) + }.toMap() + } else { + objectMapper.convertValue(root, object : TypeReference>() {}) + } + } else { + null + } + } catch (e: Exception) { + logger.error("Failed to parse metadata", e) + null + } + + logger.debug("Parsed metadata") + + + if (getTime(record) < Instant.now() + .minus(Duration.ofDays(parsedConfig.toleranceInDays.toLong())) + .toEpochMilli() + ) { + logger.info("Skipping notification for ${key.userId} because it is too late.") + return false + } + + logger.debug("Tolerance is ok") + + // create the notification in appserver + val contentProvider: NotificationContentProvider = ProtocolNotificationProvider( + name = parsedConfig.questionnaireName, + scheduledTime = timeStrategy.scheduledTime, + sourceId = key.sourceId, + metadata = metadata ?: emptyMap(), + userTimezone = timezone + ) + + logger.debug("Sending message to appserver: ${key.projectId}, ${key.userId}, ${parsedConfig.type}") + + val msgType = when (parsedConfig.type) { + MessagingType.NOTIFICATIONS -> { + appserverClient.createMessage( + key.projectId, key.userId, MessagingType.NOTIFICATIONS, contentProvider.notificationMessage + ) + "notification message" + } + MessagingType.DATA -> { + appserverClient.createMessage( + key.projectId, key.userId, MessagingType.DATA, contentProvider.dataMessage + ) + "data message" + } + MessagingType.ALL -> { + appserverClient.createMessage(key.projectId, key.userId, MessagingType.NOTIFICATIONS, contentProvider.notificationMessage) + appserverClient.createMessage(key.projectId, key.userId, MessagingType.DATA, contentProvider.dataMessage) + "notification and data message" + } + } + logger.info("Sent $msgType to appserver for ${key.userId}") + return true + } + + @Throws(IOException::class) + private fun getUserTimezone(project: String, user: String): String { + return (appserverClient.getUserDetails(project, user)["timezone"] + ?: parsedConfig.defaultTimeZone) as String + } + + companion object { + const val NAME = "ActiveAppNotificationAction" + private val logger = LoggerFactory.getLogger(ActiveAppNotificationAction::class.java) + } +} + +data class ActiveAppNotificationActionConfig( + @JsonProperty("appserver_base_url") + val appServerBaseUrl: String, + @JsonProperty("questionnaire_name") + val questionnaireName: String, + @JsonProperty("management_portal_token_url") + val mpTokenUrl: String, + @JsonProperty("message_type") + val type: MessagingType = MessagingType.NOTIFICATIONS, + @JsonProperty("client_id") + val clientId: String = "realtime_consumer", + @JsonProperty("client_secret") + val clientSecret: String = "secret", + @JsonProperty("time_of_day") + val timeOfDay: String? = null, + @JsonProperty("default_timezone") + val defaultTimeZone: String = "UTC", + @JsonProperty("metadata_key") + val metadataKey: String? = null, + @JsonProperty("tolerance_in_days") + val toleranceInDays: Int = 5, + @JsonProperty("optional_delay_minutes") + val optionalDelayMinutes: Int = 5, + @JsonProperty("jitter_in_minutes") + val jitterInMinutes: Int? = null, + @JsonIgnore + val appserverClientConfig: AppserverClientConfig = AppserverClientConfig( + clientId = clientId, + clientSecret = clientSecret, + ).apply { appserverUrl(appServerBaseUrl); tokenUrl(mpTokenUrl); this.mapper = objectMapper } +) { + init { + if (timeOfDay != null && !timeOfDay.matches("[0-9]{2}:[0-9]{2}:[0-9]{2}".toRegex())) { + throw IllegalArgumentException("timeOfDay must be in the format HH:mm:ss") + } + } + + companion object { + + fun fromMap(properties: Map?): ActiveAppNotificationActionConfig { + return properties?.let { map -> + + ActiveAppNotificationActionConfig( + appServerBaseUrl = requireNotNull(map["appserver_base_url"]) as String, + questionnaireName = requireNotNull(map["questionnaire_name"]) as String, + mpTokenUrl = requireNotNull(map["management_portal_token_url"]) as String, + type = MessagingType.valueOf(map["message_type"] as String? + ?: "NOTIFICATIONS"), + clientId = map["client_id"] as String? ?: "realtime_consumer", + clientSecret = map["client_secret"] as String? ?: "secret", + timeOfDay = map["time_of_day"] as String?, + defaultTimeZone = map["default_timezone"] as String? ?: "UTC", + metadataKey = map["metadata_key"] as String?, + toleranceInDays = map["tolerance_in_days"] as Int? ?: 5, + optionalDelayMinutes = map["optional_delay_minutes"] as Int? ?: 5, + jitterInMinutes = map["jitter_in_minutes"] as Int?, + ) + } ?: throw IllegalArgumentException("Missing required properties") + } + } +} diff --git a/src/main/java/org/radarcns/consumer/realtime/action/AppServerTriggerAction.kt b/src/main/java/org/radarcns/consumer/realtime/action/AppServerTriggerAction.kt new file mode 100644 index 00000000..0e770585 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/AppServerTriggerAction.kt @@ -0,0 +1,127 @@ +package org.radarcns.consumer.realtime.action + +import com.fasterxml.jackson.annotation.JsonIgnore +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.core.JacksonException +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarbase.appserver.client.AppserverClient +import org.radarbase.appserver.client.AppserverClientConfig +import org.radarcns.config.realtime.ActionConfig +import org.radarcns.consumer.realtime.Grouping.Companion.objectMapper +import org.slf4j.LoggerFactory +import java.io.File +import java.io.IOException +import java.time.* + +/** + * This action can be used to trigger a notification for the aRMT app and schedule a corresponding + * questionnaire for the user to fill out. This can also work as an intervention mechanism in some + * use-cases. + */ +class AppServerTriggerAction( + actionConfig: ActionConfig, + override val name: String = NAME, +) : ActionBase(actionConfig) { + private val parsedConfig: AppServerTriggerActionConfig = + AppServerTriggerActionConfig.fromMap(actionConfig.properties) + private val appserverClient: AppserverClient = AppserverClient(parsedConfig.appserverClientConfig) + + + @Throws(IllegalArgumentException::class, IOException::class) + override fun executeFor(record: ConsumerRecord<*, *>?): Boolean { + + logger.debug("Executing action $name for record: $record") + + val key = getKeys(record) ?: return false + + if (getTime(record) < Instant.now().minus(Duration.ofDays(parsedConfig.toleranceInDays.toLong())) + .toEpochMilli() + ) { + logger.info("Skipping notification for ${key.userId} because it is too late.") + return false + } + + appserverClient.createScheduleForAssessment( + projectId = key.projectId, userId = key.userId, body = parsedConfig.assessment.toString() + ) + + logger.info("Scheduled task on the appserver for ${key.userId}") + return true + } + + companion object { + const val NAME = "AppServerTriggerAction" + private val logger = LoggerFactory.getLogger(AppServerTriggerAction::class.java) + } +} + +data class AppServerTriggerActionConfig( + @JsonProperty("appserver_base_url") val appServerBaseUrl: String, + @JsonProperty("management_portal_token_url") val mpTokenUrl: String, + @JsonProperty("assessment_json") val assessmentJson: String?, + @JsonProperty("assessment_json_file") val assessmentJsonFile: String?, + @JsonProperty("client_id") val clientId: String = "realtime_consumer", + @JsonProperty("client_secret") val clientSecret: String = "secret", + @JsonProperty("assessment_read_method") val method: Methods = Methods.JSON_INLINE, + @JsonProperty("tolerance_in_days") val toleranceInDays: Int = 5, + @JsonIgnore val appserverClientConfig: AppserverClientConfig = AppserverClientConfig( + clientId = clientId, + clientSecret = clientSecret, + ).apply { appserverUrl(appServerBaseUrl); tokenUrl(mpTokenUrl); this.mapper = objectMapper } +) { + + @JsonIgnore + val assessment: JsonNode = when (method) { + Methods.JSON_INLINE -> { + if (assessmentJson.isNullOrEmpty()) { + throw IllegalArgumentException("assessment_json is required with method ${Methods.JSON_INLINE}") + } + validateJson(assessmentJson) + } + Methods.JSON_FILE -> { + if (assessmentJsonFile.isNullOrEmpty()) { + throw IllegalArgumentException("assessment_json_file is required with method ${Methods.JSON_FILE}") + } + validateJson(File(assessmentJsonFile).inputStream().readBytes().toString(Charsets.UTF_8)) + } + } + + private fun validateJson(json: String): JsonNode { + val node = try { + ObjectMapper().readTree(json) + } catch (e: JacksonException) { + throw IllegalArgumentException("Invalid JSON format.", e) + } + + if (!node.isObject) { + throw IllegalArgumentException("The provided assessment is not valid JSON format.") + } + + return node + } + + companion object { + + fun fromMap(properties: Map?): AppServerTriggerActionConfig { + return properties?.let { map -> + + AppServerTriggerActionConfig( + appServerBaseUrl = requireNotNull(map["appserver_base_url"]) as String, + mpTokenUrl = requireNotNull(map["management_portal_token_url"]) as String, + assessmentJson = map["assessment_json"] as String?, + assessmentJsonFile = map["assessment_json_file"] as String?, + clientId = map["client_id"] as String? ?: "realtime_consumer", + clientSecret = map["client_secret"] as String? ?: "secret", + method = Methods.valueOf(map["assessment_read_method"] as String? ?: "JSON_INLINE"), + toleranceInDays = map["tolerance_in_days"] as Int? ?: 5, + ) + } ?: throw IllegalArgumentException("Missing required properties") + } + } +} + +enum class Methods { + JSON_INLINE, JSON_FILE +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/EmailUserAction.kt b/src/main/java/org/radarcns/consumer/realtime/action/EmailUserAction.kt new file mode 100644 index 00000000..cc4faf4e --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/EmailUserAction.kt @@ -0,0 +1,67 @@ +package org.radarcns.consumer.realtime.action + +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarcns.config.EmailServerConfig +import org.radarcns.config.realtime.ActionConfig +import org.radarcns.consumer.realtime.action.EmailUserAction +import org.radarcns.util.EmailSender +import org.slf4j.LoggerFactory +import java.time.Instant +import javax.mail.MessagingException + +/** + * This action can be used to trigger an email to the user. Currently, it just notifies that the + * conditions evaluated to true and provides some context. This is useful for project admins but can + * be modified to also work as an intervention mechanism in some use-cases. + */ +class EmailUserAction( + actionConfig: ActionConfig, + emailServerConfig: EmailServerConfig?, + override val name: String = NAME, +) : ActionBase(actionConfig) { + + private val props = actionConfig.properties + + @Suppress("UNCHECKED_CAST") + private val emailSender: EmailSender = EmailSender( + emailServerConfig, + (props?.get("from") as String?) + ?: throw IllegalArgumentException("Missing 'from' property"), + props?.getOrDefault("email_addresses", ArrayList()) as List) + + private val customTitle: String? = props?.getOrDefault("title", null) as String? + private val customBody: String? = props?.getOrDefault("body", null) as String? + + override fun executeFor(record: ConsumerRecord<*, *>?): Boolean { + val key = getKeys(record) + + val title: String = if (customTitle.isNullOrEmpty()) { + "Conditions triggered the action $name for user" + + " ${key?.userId}) from topic ${record?.topic()}" + } else customTitle + + val body: String = if (customBody.isNullOrEmpty()) { + """ + Record: + ${record?.value()?.toString()} + + Timestamp: ${Instant.now()} + Key: ${key.toString()} + """.trimIndent() + } else customBody + + return try { + emailSender.sendEmail(title, body) + logger.info("Email sent to admin for project ${key?.projectId}, user ${key?.userId}") + true + } catch (e: MessagingException) { + logger.error("Error sending email", e) + false + } + } + + companion object { + const val NAME = "EmailUserAction" + private val logger = LoggerFactory.getLogger(EmailUserAction::class.java) + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/appserver/NotificationContentProvider.kt b/src/main/java/org/radarcns/consumer/realtime/action/appserver/NotificationContentProvider.kt new file mode 100644 index 00000000..6bcb7e99 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/appserver/NotificationContentProvider.kt @@ -0,0 +1,10 @@ +package org.radarcns.consumer.realtime.action.appserver + +/** + * Provides data and notification message content to create a message in the Appserver for scheduled + * delivery through FCM. + */ +interface NotificationContentProvider { + val dataMessage: String + val notificationMessage: String +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/appserver/ProtocolNotificationProvider.kt b/src/main/java/org/radarcns/consumer/realtime/action/appserver/ProtocolNotificationProvider.kt new file mode 100644 index 00000000..438418ae --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/appserver/ProtocolNotificationProvider.kt @@ -0,0 +1,141 @@ +package org.radarcns.consumer.realtime.action.appserver + +import org.radarbase.appserver.client.protocol.* +import org.radarcns.consumer.realtime.Grouping.Companion.objectMapper +import org.slf4j.LoggerFactory +import java.time.Duration +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter + +/** + * The content provides a questionnaire's protocol block in the message to be added to the appserver + * and for the aRMT app to parse and schedule the specified questionnaire. Supports both FCM + * Notification and Data Messages. + */ + +class ProtocolNotificationProvider( + val repo: String = REPO, + val order: Int = 0, + val name: String, + val avsc: AvscTypes = AvscTypes.QUESTIONNAIRE, + val repeatProtocolMinutes: Long = 9999999999L, // a lot of years, it will not repeat + val completionWindowMinutes: Long = 24 * 60L, // 1day + val metadata: Map = HashMap(), + val notificationTitle: String = "Questionnaire Time", + val notificationBody: String = "Urgent Questionnaire Pending. Please complete now.", + val ttlSeconds: Int = completionWindowMinutes.toInt() * 60, + val scheduledTime: Instant, + val sourceId: String? = null, + val appPackage: String = "org.phidatalab.radar_armt", + val referenceTimestamp: Instant = scheduledTime, + val userTimezone: String = "UTC", + val repeatQuestionnaireMinutes: Array = + arrayOf(referenceTimestamp.timeSinceMidnight(userTimezone)), +) : NotificationContentProvider { + + override val notificationMessage: String + override val dataMessage: String + + enum class AvscTypes(val type: String) { + QUESTIONNAIRE("questionnaire"), + NOTIFICATION("notification"); + + override fun toString(): String { + return type + } + } + + init { + + val questionnaire = SingleProtocol( + name = name, + order = order, + questionnaire = Questionnaire( + name = name, + avsc = avsc.toString(), + repository = repo, + ), + protocol = SingleProtocolSchedule( + completionWindow = ProtocolDuration( + amount = completionWindowMinutes, + unit = "min", + + ), + repeatProtocol = ProtocolDuration( + amount = repeatProtocolMinutes, + unit = "min", + + ), + repeatQuestionnaire = RepeatQuestionnaire( + unitsFromZero = repeatQuestionnaireMinutes.map { it.toInt() }.toList(), + unit = "min" + ), + referenceTimestamp = referenceTimestamp + .atZone(ZoneId.of(userTimezone)) + .format(DateTimeFormatter.ofPattern("dd-MM-yyyy:HH:mm")), + //.format(DateTimeFormatter.ISO_ZONED_DATE_TIME), + ), + ) + + val trigger = QuestionnaireTrigger( + singleProtocol = questionnaire, + metadataMap = metadata, + ) + + notificationMessage = String.format( + NOTIFICATION_TEMPLATE, + notificationTitle, + notificationBody, + ttlSeconds, + sourceId, + name, + appPackage, + scheduledTime, + objectMapper.writeValueAsString(trigger)) + dataMessage = String.format( + DATA_TEMPLATE, + ttlSeconds, + sourceId, + appPackage, + scheduledTime, + objectMapper.writeValueAsString(trigger)) + + logger.debug("Notification message: {}", notificationMessage) + logger.debug("Data message: {}", notificationMessage) + } + + companion object { + const val NOTIFICATION_TEMPLATE = ("{\n" + + "\t\"title\" : \"%s\",\n" + + "\t\"body\": \"%s\",\n" + + "\t\"ttlSeconds\": %d,\n" + + "\t\"sourceId\": \"%s\",\n" + + "\t\"type\": \"%s\",\n" + + "\t\"sourceType\": \"aRMT\",\n" + + "\t\"appPackage\": \"%s\",\n" + + "\t\"scheduledTime\": \"%s\",\n" + + "\t\"additionalData\": %s\n" + + " }") + + const val DATA_TEMPLATE = ("{\n" + + "\t\"ttlSeconds\": %d,\n" + + "\t\"sourceId\": \"%s\",\n" + + "\t\"sourceType\": \"aRMT\",\n" + + "\t\"appPackage\": \"%s\",\n" + + "\t\"scheduledTime\": \"%s\",\n" + + "\t\"priority\": \"HIGH\",\n" + + "\t\"dataMap\": %s\n" + + " }") + + private val logger = LoggerFactory.getLogger(ProtocolNotificationProvider::class.java) + private const val REPO = "https://raw.githubusercontent.com/RADAR-base/RADAR-REDCap-aRMT-Definitions/master/questionnaires" + + fun Instant.timeSinceMidnight(timezone: String = "UTC"): Long { + val zoned = atZone(ZoneId.of(timezone)) + val midnight = zoned.toLocalDate().atStartOfDay(zoned.zone).toInstant() + val duration = Duration.between(midnight, this) + return duration.toMinutes() + } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/appserver/ScheduleTimeStrategy.kt b/src/main/java/org/radarcns/consumer/realtime/action/appserver/ScheduleTimeStrategy.kt new file mode 100644 index 00000000..2e55d601 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/appserver/ScheduleTimeStrategy.kt @@ -0,0 +1,13 @@ +package org.radarcns.consumer.realtime.action.appserver + +import java.time.Instant + +/** + * The time calculation strategy for scheduling the notification via the [ ]. + * + * + * See [SimpleTimeStrategy], [TimeOfDayStrategy] + */ +interface ScheduleTimeStrategy { + val scheduledTime: Instant +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/appserver/SimpleTimeStrategy.kt b/src/main/java/org/radarcns/consumer/realtime/action/appserver/SimpleTimeStrategy.kt new file mode 100644 index 00000000..f0bf7810 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/appserver/SimpleTimeStrategy.kt @@ -0,0 +1,15 @@ +package org.radarcns.consumer.realtime.action.appserver + +import java.time.Duration +import java.time.Instant +import java.time.temporal.ChronoUnit + +/** Schedules the time based on current time with an optional added delay. */ +class SimpleTimeStrategy( + delay: Long, + unit: ChronoUnit?, + delayDuration: Duration = Duration.of(delay, unit), + reference: Instant = Instant.now(), +) : ScheduleTimeStrategy { + override val scheduledTime: Instant = reference.plus(delayDuration) +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/action/appserver/TimeOfDayStrategy.kt b/src/main/java/org/radarcns/consumer/realtime/action/appserver/TimeOfDayStrategy.kt new file mode 100644 index 00000000..7bb621fc --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/action/appserver/TimeOfDayStrategy.kt @@ -0,0 +1,42 @@ +package org.radarcns.consumer.realtime.action.appserver + +import java.time.Duration +import java.time.Instant +import java.time.LocalTime +import java.time.ZoneId +import java.time.format.DateTimeFormatter + +/** + * Schedules the time based on the next Time Of Day (eg- 09:00:00 means 9 am on the day) as + * configured in the configuration file. If the time of day specified has passed for the current + * day, it will schedule at the time of day the next day. + */ +class TimeOfDayStrategy( + private val timeOfDay: String, + private val timezone: String = "GMT", + private val jitter: Duration? = null, + private val reference: Instant = Instant.now(), +) : ScheduleTimeStrategy { + + // If time has already passed, schedule the next day + override val scheduledTime: Instant = getTimeOfDay() + + fun getTimeOfDay(): Instant { + val now = reference + val localDate = now.atZone(ZoneId.of(timezone)).toLocalDate() + var ldt = localDate.atTime(LocalTime.parse(timeOfDay, DateTimeFormatter.ISO_LOCAL_TIME)) + if (jitter != null) { + ldt = ldt.plus(jitter) + } + // If time has already passed, schedule the next day + if (ldt.isBefore(now.atZone(ZoneId.of(timezone)).toLocalDateTime())) { + ldt = ldt.plusDays(1) + } + + return ldt.atZone(ZoneId.of(timezone)).toInstant() + } + + init { + require(timeOfDay.isNotEmpty()) { "The time of day is not provided. Cannot use this strategy." } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/condition/Condition.kt b/src/main/java/org/radarcns/consumer/realtime/condition/Condition.kt new file mode 100644 index 00000000..67f9b4be --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/condition/Condition.kt @@ -0,0 +1,21 @@ +package org.radarcns.consumer.realtime.condition + +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarcns.consumer.realtime.Grouping +import java.io.IOException + +/** + * A condition can be defined as any predicate on the incoming data that must be true before the + * [org.radarcns.consumer.realtime.action.Action]s can be triggered. + */ +interface Condition : Grouping { + val name: String + + @Throws(IOException::class) + fun isTrueFor(record: ConsumerRecord<*, *>?): Boolean + + @Throws(IOException::class) + fun evaluate(record: ConsumerRecord<*, *>?): Boolean { + return evaluateProject(record) && isTrueFor(record) + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/condition/ConditionBase.kt b/src/main/java/org/radarcns/consumer/realtime/condition/ConditionBase.kt new file mode 100644 index 00000000..1b7ee72f --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/condition/ConditionBase.kt @@ -0,0 +1,13 @@ +package org.radarcns.consumer.realtime.condition + +import org.radarcns.config.realtime.ConditionConfig + +abstract class ConditionBase( + config: ConditionConfig, + override val projects: List? = config.projects, + override val subjects: List? = config.subjects, + override val subjectIdField: String? = config.projectIdField, + override val projectIdField: String? = config.subjectIdField, + override val timeField: String? = config.timeField, + override val sourceIdField: String? = config.sourceIdField, +) : Condition \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/condition/ConditionFactory.kt b/src/main/java/org/radarcns/consumer/realtime/condition/ConditionFactory.kt new file mode 100644 index 00000000..4fc34e75 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/condition/ConditionFactory.kt @@ -0,0 +1,18 @@ +package org.radarcns.consumer.realtime.condition + +import org.radarcns.config.realtime.ConditionConfig + +/** + * Factory class for [Condition]s. It instantiates conditions based on the configuration + * provided for the given consumer. + */ +object ConditionFactory { + @JvmStatic + fun getConditionFor(conditionConfig: ConditionConfig): Condition { + return when (conditionConfig.name) { + LocalJsonPathCondition.NAME -> LocalJsonPathCondition(conditionConfig) + else -> throw IllegalArgumentException( + "The specified condition with name " + conditionConfig.name + " is not correct.") + } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/condition/JsonPathCondition.kt b/src/main/java/org/radarcns/consumer/realtime/condition/JsonPathCondition.kt new file mode 100644 index 00000000..7d640d63 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/condition/JsonPathCondition.kt @@ -0,0 +1,46 @@ +package org.radarcns.consumer.realtime.condition + +import com.jayway.jsonpath.JsonPath +import org.apache.avro.generic.GenericRecord +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarcns.config.realtime.ConditionConfig +import org.slf4j.LoggerFactory +import java.io.IOException + +/** + * Uses https://github.com/json-path/JsonPath to evaluate json expressions directly in the record + * making this condition a generic one for simple use cases such as predicates and comparisons for a + * field in the json record. + */ +abstract class JsonPathCondition(config: ConditionConfig) : ConditionBase(config) { + @Throws(IOException::class) + protected fun evaluateJsonPath( + record: ConsumerRecord<*, *>, + jsonPath: String?, rootKey: String? = null): Boolean { + // JsonPath expressions always return a List. + val result: List<*>? = try { + + val valueToParse = if (rootKey != null) { + (record.value() as GenericRecord?)?.get(rootKey)?.toString() ?: return false + } else { + (record.value() as GenericRecord?)?.toString() ?: return false + } + + logger.debug("value: $valueToParse") + + JsonPath.parse(valueToParse).read(jsonPath) + } catch (exc: ClassCastException) { + throw IOException("The provided json path does not seem to contain an expression. Make sure it" + + " contains an expression. Docs: https://github.com/json-path/JsonPath", exc) + } + + logger.debug("JsonPath result: $result") + + // At least one result matches the condition + return !result.isNullOrEmpty() + } + + companion object { + private val logger = LoggerFactory.getLogger(JsonPathCondition::class.java) + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/consumer/realtime/condition/LocalJsonPathCondition.kt b/src/main/java/org/radarcns/consumer/realtime/condition/LocalJsonPathCondition.kt new file mode 100644 index 00000000..45b0fcc0 --- /dev/null +++ b/src/main/java/org/radarcns/consumer/realtime/condition/LocalJsonPathCondition.kt @@ -0,0 +1,35 @@ +package org.radarcns.consumer.realtime.condition + +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.radarcns.config.realtime.ConditionConfig +import java.io.IOException + +/** + * Reads the JsonPath specification from the configuration file, provided to this class in [ ]. + */ +class LocalJsonPathCondition( + conditionConfig: ConditionConfig, + override val name: String = NAME, +) : JsonPathCondition(conditionConfig) { + private val jsonPath: String? + private val rootKey: String? + + @Throws(IOException::class) + override fun isTrueFor(record: ConsumerRecord<*, *>?): Boolean { + return evaluateJsonPath(record!!, jsonPath, rootKey) + } + + companion object { + const val NAME = "LocalJsonPathCondition" + } + + init { + rootKey = conditionConfig.properties?.let { it.getOrDefault("key", null) as String? } + jsonPath = requireNotNull( + conditionConfig.properties?.let { it["jsonpath"] as String? } + ) { + ("The 'jsonpath' property needs to be specified when " + + "using the LocalJsonPathCondition.") + } + } +} \ No newline at end of file diff --git a/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java b/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java index aa9190bc..50ea2082 100644 --- a/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java +++ b/src/main/java/org/radarcns/monitor/AbstractKafkaMonitor.java @@ -16,7 +16,7 @@ package org.radarcns.monitor; -import static io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; +import static io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.CLIENT_ID_CONFIG; @@ -152,7 +152,7 @@ protected final void configure(Properties properties) { * Monitor a given topic until the {@link #isShutdown()} method returns true. * *

When a message is encountered that cannot be deserialized, - * {@link #handleSerializationException()} is called. + * {@link #handleSerializationException(SerializationException)} is called. */ @Override public void start() { @@ -170,7 +170,7 @@ public void start() { ops.add(records.count()); evaluateRecords(records); } catch (SerializationException ex) { - handleSerializationException(); + handleSerializationException(ex); } catch (WakeupException ex) { logger.info("Consumer woke up"); } catch (InterruptException ex) { @@ -193,27 +193,31 @@ public void start() { * *

The new position is not committed, so on failure of the client, the message must be * skipped again. + * @param ex */ // TODO: submit the message to another topic to indicate that it could not be deserialized. - protected void handleSerializationException() { - logger.error("Failed to deserialize message. Skipping message."); + protected void handleSerializationException( + SerializationException ex) { + logger.error("Failed to deserialize message. Skipping message.", ex); topics.parallelStream() .flatMap(t -> consumer.partitionsFor(t).stream()) .map(tp -> new TopicPartition(tp.topic(), tp.partition())) .filter(tp -> { + String tmpId = "-tmp-" + UUID.randomUUID(); Properties tmpProperties = new Properties(); tmpProperties.putAll(properties); + tmpProperties.setProperty(GROUP_ID_CONFIG, + properties.getProperty(GROUP_ID_CONFIG) + tmpId); tmpProperties.setProperty(CLIENT_ID_CONFIG, - properties.getProperty(CLIENT_ID_CONFIG) - + "-tmp-" + UUID.randomUUID().toString()); + properties.getProperty(CLIENT_ID_CONFIG) + tmpId); try (Consumer tmpConsumer = new KafkaConsumer<>(tmpProperties)) { tmpConsumer.assign(List.of(tp)); tmpConsumer.seek(tp, consumer.position(tp)); tmpConsumer.poll(Duration.ZERO); return false; - } catch (SerializationException ex) { - logger.error("Serialization error, skipping message", ex); + } catch (SerializationException ex2) { + logger.error("Serialization error, skipping message", ex2); return true; } }) @@ -221,7 +225,7 @@ protected void handleSerializationException() { } /** Evaluate a single record that the monitor receives by overriding this function. */ - protected abstract void evaluateRecord(ConsumerRecord records); + protected abstract void evaluateRecord(ConsumerRecord record); /** Evaluates the records that the monitor receives. */ protected void evaluateRecords(ConsumerRecords records) { diff --git a/src/main/java/org/radarcns/monitor/CombinedKafkaMonitor.java b/src/main/java/org/radarcns/monitor/CombinedKafkaMonitor.java index 99f03cda..9481aa06 100644 --- a/src/main/java/org/radarcns/monitor/CombinedKafkaMonitor.java +++ b/src/main/java/org/radarcns/monitor/CombinedKafkaMonitor.java @@ -43,7 +43,7 @@ public class CombinedKafkaMonitor implements KafkaMonitor { private IOException ioException; private InterruptedException interruptedException; - public CombinedKafkaMonitor(Stream monitors) { + public CombinedKafkaMonitor(Stream monitors) { this.monitors = Objects.requireNonNull(monitors) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/src/main/java/org/radarcns/monitor/DisconnectMonitor.java b/src/main/java/org/radarcns/monitor/DisconnectMonitor.java index 62a8ce22..b1718259 100644 --- a/src/main/java/org/radarcns/monitor/DisconnectMonitor.java +++ b/src/main/java/org/radarcns/monitor/DisconnectMonitor.java @@ -39,7 +39,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; -import org.radarcns.config.DisconnectMonitorConfig; +import org.radarcns.config.monitor.DisconnectMonitorConfig; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.DisconnectMonitor.DisconnectMonitorState; diff --git a/src/main/java/org/radarcns/monitor/KafkaMonitorFactory.java b/src/main/java/org/radarcns/monitor/KafkaMonitorFactory.java index 3ef1d600..6ac75052 100644 --- a/src/main/java/org/radarcns/monitor/KafkaMonitorFactory.java +++ b/src/main/java/org/radarcns/monitor/KafkaMonitorFactory.java @@ -22,11 +22,15 @@ import java.util.List; import java.util.Locale; import java.util.stream.Stream; -import org.radarcns.config.BatteryMonitorConfig; -import org.radarcns.config.DisconnectMonitorConfig; -import org.radarcns.config.MonitorConfig; +import org.radarcns.config.EmailServerConfig; +import org.radarcns.config.monitor.BatteryMonitorConfig; +import org.radarcns.config.monitor.DisconnectMonitorConfig; +import org.radarcns.config.monitor.EmailNotifyConfig; +import org.radarcns.config.monitor.InterventionMonitorConfig; +import org.radarcns.config.monitor.MonitorConfig; import org.radarcns.config.RadarBackendOptions; import org.radarcns.config.RadarPropertyHandler; +import org.radarcns.monitor.intervention.InterventionMonitor; import org.radarcns.util.EmailSenders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,11 +41,13 @@ public class KafkaMonitorFactory { private final RadarPropertyHandler properties; private final RadarBackendOptions options; + private final EmailServerConfig emailServer; public KafkaMonitorFactory(RadarBackendOptions options, RadarPropertyHandler properties) { this.options = options; this.properties = properties; + this.emailServer = properties.getRadarProperties().getEmailServerConfig(); } public KafkaMonitor createMonitor() throws IOException { @@ -61,9 +67,12 @@ public KafkaMonitor createMonitor() throws IOException { case "disconnect": monitor = createDisconnectMonitor(); break; + case "intervention": + monitor = createInterventionMonitor(); + break; case "all": monitor = new CombinedKafkaMonitor( - Stream.of(createDisconnectMonitor(), createBatteryLevelMonitor())); + Stream.of(createDisconnectMonitor(), createBatteryLevelMonitor(), createInterventionMonitor())); break; default: throw new IllegalArgumentException("Cannot create unknown monitor " + commandType); @@ -74,6 +83,17 @@ public KafkaMonitor createMonitor() throws IOException { return monitor; } + private KafkaMonitor createInterventionMonitor() throws IOException { + InterventionMonitorConfig config = properties.getRadarProperties().getInterventionMonitor(); + if (config == null) { + logger.warn("Notification monitor is not configured. Cannot start it."); + return null; + } + + EmailSenders senders = createSenders(config.getEmailNotifyConfig()); + return new InterventionMonitor(config.withEnv(), properties, senders); + } + private KafkaMonitor createBatteryLevelMonitor() throws IOException { BatteryMonitorConfig config = properties.getRadarProperties().getBatteryMonitor(); @@ -83,7 +103,7 @@ private KafkaMonitor createBatteryLevelMonitor() throws IOException { } BatteryLevelMonitor.Status minLevel = BatteryLevelMonitor.Status.CRITICAL; - EmailSenders senders = getSenders(config); + EmailSenders senders = createSenders(config.getEmailNotifyConfig()); Collection topics = getTopics(config, "android_empatica_e4_battery_level"); if (config.getLevel() != null) { @@ -108,15 +128,18 @@ private KafkaMonitor createDisconnectMonitor() logger.warn("Disconnect monitor is not configured. Cannot start it."); return null; } - EmailSenders senders = getSenders(config); + EmailSenders senders = createSenders(config.getEmailNotifyConfig()); Collection topics = getTopics(config, "android_empatica_e4_temperature"); return new DisconnectMonitor(properties, topics, "disconnect_monitor", senders); } - private EmailSenders getSenders(MonitorConfig config) throws IOException { - if (config != null && config.getNotifyConfig() != null) { - return EmailSenders.parseConfig(config); + private EmailSenders createSenders(List notifyConfig) throws IOException { + if (emailServer != null && notifyConfig != null) { + return EmailSenders.parseConfig(emailServer, notifyConfig); + } else { + logger.warn("Monitor does not have email configured. " + + "Will not send email notifications."); } return null; } diff --git a/src/main/java/org/radarcns/monitor/intervention/AppServerIntervention.kt b/src/main/java/org/radarcns/monitor/intervention/AppServerIntervention.kt new file mode 100644 index 00000000..52110f21 --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/AppServerIntervention.kt @@ -0,0 +1,126 @@ +package org.radarcns.monitor.intervention + +import com.fasterxml.jackson.databind.ObjectMapper +import okhttp3.OkHttpClient +import org.radarbase.appserver.client.AppServerData +import org.radarbase.appserver.client.AppServerNotification +import org.radarbase.appserver.client.AppserverClient +import org.radarbase.appserver.client.MessagingType +import org.radarbase.appserver.client.protocol.Notification.Companion.defaultNotificationText +import org.radarbase.appserver.client.protocol.Notification.Companion.defaultNotificationTitle +import org.radarbase.appserver.client.protocol.ProtocolDirectory +import org.radarbase.appserver.client.protocol.QuestionnaireTrigger +import org.radarbase.appserver.client.protocol.translation +import org.radarcns.config.monitor.AuthConfig +import org.slf4j.LoggerFactory +import java.time.Duration +import java.time.Instant + +class AppServerIntervention( + private val protocolDirectory: ProtocolDirectory, + appserverUrl: String, + private val defaultLanguage: String, + authConfig: AuthConfig, + httpClient: OkHttpClient, + mapper: ObjectMapper, +) { + private val appserverClient: AppserverClient + private val protocolWriter = mapper.writerFor(QuestionnaireTrigger::class.java) + private val notificationWriter = mapper.writerFor(AppServerNotification::class.java) + private val dataWriter = mapper.writerFor(AppServerData::class.java) + + init { + appserverClient = AppserverClient { + appserverUrl(appserverUrl) + tokenUrl(authConfig.tokenUrl) + clientId = authConfig.clientId + clientSecret = authConfig.clientSecret + this.httpClient = httpClient + this.mapper = mapper + } + } + + fun createMessages(intervention: InterventionRecord, ttl: Duration) { + val attributes = if (intervention.name != null) mapOf("intervention" to intervention.name) else emptyMap() + val protocol = protocolDirectory.get( + projectId = intervention.projectId, + userId = intervention.userId, + attributes = attributes, + ) ?: throw NoSuchElementException("No protocol found for $intervention") + + val body = protocolWriter.writeValueAsString(protocol) + + val ttlSeconds = ttl.toSeconds() + val notificationResponse = createNotificationMessage(intervention, ttlSeconds, protocol, body) + val dataResponse = createDataMessage(intervention, ttlSeconds, body) + + logger.debug("Created App Server message for notification {} and data {}", + notificationResponse, dataResponse) + } + + private fun createNotificationMessage( + intervention: InterventionRecord, + ttlSeconds: Long, + protocol: QuestionnaireTrigger, + body: String, + ): Map { + val notification = protocol.singleProtocol.protocol.notification + + val language = try { + (appserverClient + .getUserDetails(intervention.projectId, intervention.userId)["language"] + ?: defaultLanguage) as String + + } catch (e: Exception) { + logger.warn("Failed to get user language for ${intervention.userId}. Using default", e) + defaultLanguage + } + + val notificationTitle = notification.title.translation(language) ?: defaultNotificationTitle + val notificationText = notification.text.translation(language) ?: defaultNotificationText + + val notificationBody = notificationWriter.writeValueAsString( + AppServerNotification( + title = notificationTitle, + body = notificationText, + sourceId = intervention.sourceId, + type = protocol.singleProtocol.questionnaire.name, + ttlSeconds = ttlSeconds, + scheduledTime = Instant.now().toString(), + additionalData = body, + ) + ) + return appserverClient.createMessage( + projectId = intervention.projectId, + userId = intervention.userId, + type = MessagingType.NOTIFICATIONS, + contents = notificationBody, + ) + } + + private fun createDataMessage( + intervention: InterventionRecord, + ttlSeconds: Long, + body: String, + ): Map { + val now = Instant.now().toString() + val data = dataWriter.writeValueAsString( + AppServerData( + sourceId = intervention.sourceId, + ttlSeconds = ttlSeconds, + scheduledTime = now, + dataMap = body, + ) + ) + return appserverClient.createMessage( + projectId = intervention.projectId, + userId = intervention.userId, + type = MessagingType.DATA, + contents = data, + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(AppServerIntervention::class.java) + } +} diff --git a/src/main/java/org/radarcns/monitor/intervention/InterventionAppConfigState.kt b/src/main/java/org/radarcns/monitor/intervention/InterventionAppConfigState.kt new file mode 100644 index 00000000..5bbe48ac --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/InterventionAppConfigState.kt @@ -0,0 +1,5 @@ +package org.radarcns.monitor.intervention + +data class InterventionAppConfigState( + val threshold: Float = 3.0f, +) diff --git a/src/main/java/org/radarcns/monitor/intervention/InterventionExceptionEmailer.kt b/src/main/java/org/radarcns/monitor/intervention/InterventionExceptionEmailer.kt new file mode 100644 index 00000000..7e9d214c --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/InterventionExceptionEmailer.kt @@ -0,0 +1,70 @@ +package org.radarcns.monitor.intervention + +import org.radarcns.util.EmailSenders +import org.slf4j.LoggerFactory +import java.time.LocalDate +import java.time.ZoneOffset +import java.time.ZonedDateTime + +class InterventionExceptionEmailer( + private val emailSenders: EmailSenders, + private val state: InterventionMonitorState, +) { + fun emailExceptions() { + state.exceptions.forEach { (projectId, projectExceptions) -> + try { + val userMessages: List = projectExceptions.exceptions + .entries + .map { (userId, userExceptions) -> + val exceptionList = userExceptions.lines + val numLines = exceptionList.size + if (userExceptions.isTruncated) { + exceptionList.addFirst("...") + } + val exceptionString = + exceptionList.joinToString(separator = "") { "$exceptionPrefix$it" } + "user $userId - listing $numLines out of ${userExceptions.count} exceptions:$exceptionString" + } + + val totalCount = projectExceptions.exceptions.values.sumOf { it.count } + + val date = LocalDate.ofInstant(state.fromDate, ZoneOffset.UTC).toString() + val start = ZonedDateTime.ofInstant(state.fromDate, ZoneOffset.UTC) + val end = ZonedDateTime.ofInstant(state.nextMidnight(), ZoneOffset.UTC) + + val subject = "[RADAR-base $projectId] Errors in intervention algorithm on $date" + val message = """ + Hi, + + On $date, some errors occurred in the RADAR-base intervention algorithm. This message summarizes the errors occurred for the RADAR-base $projectId project from $start to $end. A total of $totalCount errors were counted. Below is a summary of the errors: + + ${userMessages.joinToString(separator = "\n\n")} + + This is an automated message from the RADAR-base platform. Please refer to your RADAR-base administrator for more information. + """.trimIndent() + + logger.info("Sending message for project {}: {}", projectId, message) + + val sender = emailSenders.getEmailSenderForProject(projectId) + + if (sender != null) { + sender.sendEmail(subject, message) + } else { + logger.warn( + "No email sender configured for project {}. Not sending exception message.", + projectId + ) + } + } catch (ex: Throwable) { + logger.error("Failed to send error notifications for project {} - {}: {}", + projectId, projectExceptions, ex.toString()) + } + } + } + + companion object { + private val logger = LoggerFactory.getLogger(InterventionExceptionEmailer::class.java) + + private const val exceptionPrefix = "\n - " + } +} diff --git a/src/main/java/org/radarcns/monitor/intervention/InterventionMonitor.kt b/src/main/java/org/radarcns/monitor/intervention/InterventionMonitor.kt new file mode 100644 index 00000000..a6aa4aca --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/InterventionMonitor.kt @@ -0,0 +1,266 @@ +package org.radarcns.monitor.intervention + +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.ObjectReader +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule +import com.fasterxml.jackson.module.kotlin.KotlinFeature +import com.fasterxml.jackson.module.kotlin.jsonMapper +import com.fasterxml.jackson.module.kotlin.kotlinModule +import okhttp3.OkHttpClient +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.consumer.ConsumerRecord +import org.apache.kafka.common.serialization.BytesDeserializer +import org.apache.kafka.common.utils.Bytes +import org.radarbase.appserver.client.protocol.FileProtocolDirectory +import org.radarcns.config.RadarPropertyHandler +import org.radarcns.config.monitor.InterventionMonitorConfig +import org.radarcns.monitor.AbstractKafkaMonitor +import org.radarcns.monitor.intervention.InterventionMonitorState.Companion.lastMidnight +import org.radarcns.util.EmailSenders +import org.slf4j.LoggerFactory +import java.time.Duration +import java.time.Instant +import java.util.* +import java.util.concurrent.* + +/** + * The main Kafka Consumer class that runs a single consumer on any topic. The consumer evaluates + * each incoming record based on the Conditions provided in the config. If and only If all + * the conditions evaluate to true, only then all the configured Actions are fired. + * + * To be used with the model-invocation-endpoint and KSQL API_INFERENCE function to evaluate and + * take action on incoming results from realtime inference on data. + */ +class InterventionMonitor( + private val config: InterventionMonitorConfig, + radar: RadarPropertyHandler, + emailSenders: EmailSenders?, +) : AbstractKafkaMonitor( + radar, + config.topics, + "intervention_monitors", + "1", + InterventionMonitorState(), +) { + private val queue: MutableMap> = HashMap() + private val executor: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor() + private val thresholdAdjust: ThresholdAdjustmentAlgorithm + private val appServerNotifications: AppServerIntervention + private var emailer: InterventionExceptionEmailer? + private val exceptionBatch: LinkedBlockingQueue = LinkedBlockingQueue() + private val keyReader: ObjectReader + private val valueReader: ObjectReader + + init { + configureConsumer() + + val httpClient = OkHttpClient() + val mapper = jsonMapper { + addModule(kotlinModule { + enable(KotlinFeature.NullIsSameAsDefault) + enable(KotlinFeature.NullToEmptyMap) + enable(KotlinFeature.NullToEmptyCollection) + }) + addModule(JavaTimeModule()) + disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + } + keyReader = mapper.readerFor(String::class.java) + valueReader = mapper.readerFor(RawInterventionRecord::class.java) + + appServerNotifications = AppServerIntervention( + protocolDirectory = FileProtocolDirectory(config.protocolDirectory, mapper), + defaultLanguage = config.defaultLanguage, + appserverUrl = config.appServerUrl, + authConfig = config.authConfig, + httpClient = httpClient, + mapper = mapper, + ) + + thresholdAdjust = ThresholdAdjustmentAlgorithm( + clientId = config.ksqlAppConfigClient, + state = state, + config = config.thresholdAdaptation, + appConfigUrl = config.appConfigUrl, + authConfig = config.authConfig, + httpClient = httpClient, + mapper = mapper, + ) + + emailer = if (emailSenders != null) InterventionExceptionEmailer( + emailSenders = emailSenders, + state = state, + ) else null + } + + private fun configureConsumer() { + val properties = Properties() + val deserializer: String = BytesDeserializer::class.java.name + properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, deserializer) + properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, deserializer) + configure(properties) + } + + override fun start() { + executor.execute { + if (state.fromDate.passedDuration() > config.stateResetInterval) { + emailer?.emailExceptions() + val lastMidnight = lastMidnight() + val numberOfIterations = lastMidnight.passedDuration().dividedBy(config.stateResetInterval) + state.reset(lastMidnight + config.stateResetInterval.multipliedBy(numberOfIterations)) + storeState() + } + + // At the next interval, update the threshold levels + val remainingInterval = (state.fromDate + config.stateResetInterval).pendingDuration() + executor.scheduleAtFixedRate( + { + logger.info("Updating intervention state") + try { + thresholdAdjust.updateThresholds() + emailer?.emailExceptions() + resetQueue() + state.reset(state.fromDate + config.stateResetInterval) + storeState() + } catch (ex: Throwable) { + logger.error("Failed to update thresholds", ex) + } + }, + remainingInterval.toMillis(), + config.stateResetInterval.toMillis(), + TimeUnit.MILLISECONDS, + ) + } + + super.start() + } + + override fun evaluateRecord(record: ConsumerRecord) { + logger.info("Evaluating {}", record) + val intervention = parseRecord(record) ?: return + + if (!intervention.exception.isNullOrEmpty()) { + addException(intervention) + return + } + + val interventionDeadline = intervention.timeCompleted + config.deadline + val interventionDeadlineTotal = interventionDeadline + config.ttlMargin + + executor.execute { + try { + logger.info("Scheduling intervention {}", intervention) + + queue.remove(intervention.queueKey) + ?.cancel(false) + + if (!intervention.decision) { + return@execute + } + + val timeBeforeDeadlineTotal = interventionDeadlineTotal.pendingDuration() + if (timeBeforeDeadlineTotal.isNegative) { + logger.info("For user {}, deadline for intervention {} has passed. Skipping.", + intervention.userId, interventionDeadline) + return@execute + } + if (intervention.isFinal) { + createAppMessages(intervention, timeBeforeDeadlineTotal) + } else { + queue[intervention.queueKey] = executor.schedule( + { + createAppMessages(intervention, interventionDeadlineTotal.pendingDuration()) + queue -= intervention.queueKey + }, + interventionDeadline.pendingDuration().toMillis(), + TimeUnit.MILLISECONDS, + ) + } + } catch (ex: Throwable) { + logger.error("Failed to process intervention $intervention", ex) + } + } + } + + private fun addException(intervention: InterventionRecord) { + logger.warn("Record has exception for {} - {}: {}", + intervention.projectId, intervention.userId, intervention.exception) + exceptionBatch += intervention + } + + override fun afterEvaluate() { + executor.execute { + try { + val localExceptions = mutableListOf() + exceptionBatch.drainTo(localExceptions) + localExceptions.forEach { state.addException(it) } + super.afterEvaluate() + } catch (ex: Throwable) { + logger.error("Failed to run intervention afterEvaluate", ex) + } + } + } + + private fun parseRecord(record: ConsumerRecord): InterventionRecord? { + val userId = try { + keyReader.readValue(record.key().get(), String::class.java) + } catch (ex: Exception) { + logger.error("Cannot map intervention record key from {}: {}", record.key(), ex.toString()) + return null + } + if (userId.isNullOrEmpty()) { + logger.error("Cannot map record without user ID") + return null + } + val intervention = try { + valueReader.readValue(record.value().get(), RawInterventionRecord::class.java) + ?.toInterventionRecord(userId) + } catch (ex: Exception) { + logger.error("Cannot map user {} intervention record value from {}: {}", userId, record.value(), ex.toString()) + return null + } + if (intervention == null) { + logger.error("Cannot map user {} null intervention record value from {}", userId, record.value()) + return null + } + + if (intervention.timeCompleted < state.fromDate) { + return null + } + return intervention + } + + private fun createAppMessages( + intervention: InterventionRecord, + ttl: Duration + ) { + val userInterventions = state[intervention] + if (!userInterventions.interventions.add(intervention.time)) { + logger.info("Already sent intervention for time point {}. Skipping.", intervention.time) + return + } + if (userInterventions.interventions.size > config.maxInterventions) { + logger.info("For user {}, number of interventions {} would exceed maximum {}. Skipping.", + intervention.userId, userInterventions.interventions.size, config.maxInterventions) + return + } else { + logger.info("Creating app notification for intervention {}", intervention.userId, intervention) + } + + appServerNotifications.createMessages(intervention, ttl) + } + + private fun resetQueue() { + queue.values.forEach { it.cancel(false) } + queue.clear() + } + + companion object { + private val logger = LoggerFactory.getLogger(InterventionMonitor::class.java) + + private fun Instant.pendingDuration(): Duration = Duration.between(Instant.now(), this) + private fun Instant.passedDuration(): Duration = Duration.between(this, Instant.now()) + + private val InterventionRecord.queueKey: String + get() = "$userId-$timeCompleted" + } +} diff --git a/src/main/java/org/radarcns/monitor/intervention/InterventionMonitorState.kt b/src/main/java/org/radarcns/monitor/intervention/InterventionMonitorState.kt new file mode 100644 index 00000000..d21b2599 --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/InterventionMonitorState.kt @@ -0,0 +1,70 @@ +package org.radarcns.monitor.intervention + +import com.fasterxml.jackson.annotation.JsonIgnore +import java.time.Instant +import java.time.LocalDate +import java.time.Period +import java.time.ZoneOffset +import java.util.LinkedList + +data class InterventionMonitorState( + var fromDate: Instant = lastMidnight(), + val counts: MutableMap = HashMap(), + val exceptions: MutableMap = HashMap() +) { + fun reset(midnight: Instant) { + this.fromDate = midnight + counts.clear() + exceptions.clear() + } + + operator fun get(intervention: InterventionRecord) = counts.computeIfAbsent(intervention.userId) { + InterventionCount(intervention.projectId) + } + + fun addException(intervention: InterventionRecord) { + require(!intervention.exception.isNullOrEmpty()) { "Missing exception in intervention" } + val userExceptions = exceptions + .computeIfAbsent(intervention.projectId) { ProjectExceptions() } + .exceptions + .computeIfAbsent(intervention.userId) { UserExceptions() } + userExceptions += intervention.exception + } + + fun nextMidnight(): Instant = (LocalDate.ofInstant(fromDate, ZoneOffset.UTC) + ONE_DAY) + .atStartOfDay(ZoneOffset.UTC) + .toInstant() + + data class InterventionCount( + val projectId: String, + val interventions: MutableSet = mutableSetOf(), + ) + + data class ProjectExceptions( + val exceptions: MutableMap = mutableMapOf(), + ) + + data class UserExceptions( + var count: Int = 0, + val lines: LinkedList = LinkedList() + ) { + @get:JsonIgnore + val isTruncated: Boolean + get() = count > EXCEPTION_SIZE + + operator fun plusAssign(exception: String) { + count += 1 + if (count > EXCEPTION_SIZE) lines.removeFirst() + lines += exception + } + } + + companion object { + fun lastMidnight(): Instant = LocalDate.now(ZoneOffset.UTC) + .atStartOfDay(ZoneOffset.UTC) + .toInstant() + + private val ONE_DAY = Period.ofDays(1) + private const val EXCEPTION_SIZE = 3 + } +} diff --git a/src/main/java/org/radarcns/monitor/intervention/InterventionRecord.kt b/src/main/java/org/radarcns/monitor/intervention/InterventionRecord.kt new file mode 100644 index 00000000..d5f4aae9 --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/InterventionRecord.kt @@ -0,0 +1,34 @@ +package org.radarcns.monitor.intervention + +import java.time.Instant + +data class InterventionRecord( + val projectId: String, + val userId: String, + val sourceId: String, + val time: Long, + val timeCompleted: Instant, + val isFinal: Boolean, + val decision: Boolean, + val name: String?, + val exception: String?, +) { + companion object { + fun Map.toInterventionRecord(userId: String): InterventionRecord { + val projectId = requireNotNull(this["PROJECTID"] as? String) { "Cannot map record of $userId without project ID." } + val intervention = requireNotNull(this["INTERVENTION"] as? Map<*, *>) { "Cannot map record without intervention for $projectId - $userId" } + + return InterventionRecord( + projectId = projectId, + userId = userId, + sourceId = this["SOURCEID"] as String, + time = (this["TIME"] as Number).toLong(), + timeCompleted = Instant.ofEpochMilli(((this["TIMECOMPLETED"] as Number).toDouble() * 1000.0).toLong()), + isFinal = this["ISFINAL"] as Boolean, + decision = intervention["DECISION"] as Boolean, + name = intervention["NAME"] as? String, + exception = intervention["EXCEPTION"] as? String, + ) + } + } +} diff --git a/src/main/java/org/radarcns/monitor/intervention/RawInterventionDecisionRecord.kt b/src/main/java/org/radarcns/monitor/intervention/RawInterventionDecisionRecord.kt new file mode 100644 index 00000000..34538254 --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/RawInterventionDecisionRecord.kt @@ -0,0 +1,12 @@ +package org.radarcns.monitor.intervention + +import com.fasterxml.jackson.annotation.JsonProperty + +data class RawInterventionDecisionRecord( + @JsonProperty("DECISION") + val decision: Boolean, + @JsonProperty("NAME") + val name: String? = null, + @JsonProperty("EXCEPTION") + val exception: String? = null, +) diff --git a/src/main/java/org/radarcns/monitor/intervention/RawInterventionRecord.kt b/src/main/java/org/radarcns/monitor/intervention/RawInterventionRecord.kt new file mode 100644 index 00000000..46163b26 --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/RawInterventionRecord.kt @@ -0,0 +1,32 @@ +package org.radarcns.monitor.intervention + +import com.fasterxml.jackson.annotation.JsonProperty +import java.time.Instant + +data class RawInterventionRecord( + @JsonProperty("PROJECTID") + val projectId: String, + @JsonProperty("SOURCEID") + val sourceId: String, + @JsonProperty("TIME") + val time: Double, + @JsonProperty("TIMECOMPLETED") + val timeCompleted: Double, + @JsonProperty("ISFINAL") + val isFinal: Boolean, + @JsonProperty("INTERVENTION") + val intervention: RawInterventionDecisionRecord, +) { + fun toInterventionRecord(userId: String): InterventionRecord = InterventionRecord( + projectId = projectId, + userId = userId, + sourceId = sourceId, + time = time.toLong(), + timeCompleted = Instant.ofEpochMilli((timeCompleted * 1000.0).toLong()), + isFinal = isFinal, + decision = intervention.decision, + name = intervention.name, + exception = intervention.exception, + ) +} + diff --git a/src/main/java/org/radarcns/monitor/intervention/ThresholdAdjustmentAlgorithm.kt b/src/main/java/org/radarcns/monitor/intervention/ThresholdAdjustmentAlgorithm.kt new file mode 100644 index 00000000..989298d7 --- /dev/null +++ b/src/main/java/org/radarcns/monitor/intervention/ThresholdAdjustmentAlgorithm.kt @@ -0,0 +1,80 @@ +package org.radarcns.monitor.intervention + +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import okhttp3.OkHttpClient +import org.radarbase.appconfig.client.AppConfigClient +import org.radarcns.config.monitor.AuthConfig +import org.radarcns.config.monitor.ThresholdAdaptationConfig +import org.slf4j.LoggerFactory + +class ThresholdAdjustmentAlgorithm( + private val clientId: String, + private val state: InterventionMonitorState, + private val config: ThresholdAdaptationConfig, + appConfigUrl: String, + authConfig: AuthConfig, + httpClient: OkHttpClient, + mapper: ObjectMapper, +) { + private var appConfigClient: AppConfigClient + + init { + appConfigClient = AppConfigClient(object : TypeReference() {}) { + appConfigUrl(appConfigUrl) + tokenUrl(authConfig.tokenUrl) + clientId = authConfig.clientId + clientSecret = authConfig.clientSecret + this.httpClient = httpClient + this.mapper = mapper + } + } + + fun updateThresholds() { + state.counts.forEach { (userId, counts) -> + try { + val interventionConfig = appConfigClient.getUserConfig( + projectId = counts.projectId, + userId = userId, + clientId = clientId, + ) + val newInterventionConfig = interventionConfig.copy( + threshold = calculateThreshold( + currentValue = interventionConfig.threshold, + numberOfInterventions = counts.interventions.size, + ) + ) + logger.info( + "Updating thresholds for user {} from {} to {}", + userId, interventionConfig, newInterventionConfig + ) + appConfigClient.setUserConfig( + projectId = counts.projectId, + userId = userId, + config = newInterventionConfig, + clientId = clientId, + ) + } catch (ex: Throwable) { + logger.error("Failed to update app config for {} - {}: {}", + counts.projectId, userId, ex.toString()) + } + } + } + + private fun calculateThreshold( + currentValue: Float, + numberOfInterventions: Int + ): Float { + return if (numberOfInterventions > config.optimalInterventions) { + currentValue - config.adjustValue + } else if (numberOfInterventions < config.optimalInterventions) { + currentValue + config.adjustValue + } else { + currentValue + } + } + + companion object { + private val logger = LoggerFactory.getLogger(ThresholdAdjustmentAlgorithm::class.java) + } +} diff --git a/src/main/java/org/radarcns/producer/MockProducerCommand.java b/src/main/java/org/radarcns/producer/MockProducerCommand.java index 4385f9e6..4d265548 100644 --- a/src/main/java/org/radarcns/producer/MockProducerCommand.java +++ b/src/main/java/org/radarcns/producer/MockProducerCommand.java @@ -43,7 +43,7 @@ public MockProducerCommand(RadarBackendOptions options, Path mockFile = options.getMockFile(); if (mockFile != null) { - MockConfig mockConfig = new YamlConfigLoader().load(mockFile, MockConfig.class); + MockConfig mockConfig = radarPropertyHandler.getLoader().load(mockFile, MockConfig.class); producerConfig.setData(mockConfig.getData()); } else { producerConfig.setNumberOfDevices(options.getNumMockDevices()); diff --git a/src/main/java/org/radarcns/stream/AbstractStreamWorker.java b/src/main/java/org/radarcns/stream/AbstractStreamWorker.java index a5c2de7b..f8a64438 100644 --- a/src/main/java/org/radarcns/stream/AbstractStreamWorker.java +++ b/src/main/java/org/radarcns/stream/AbstractStreamWorker.java @@ -1,6 +1,8 @@ package org.radarcns.stream; -import java.lang.Thread.UncaughtExceptionHandler; +import static org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_APPLICATION; +import static org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse.SHUTDOWN_CLIENT; + import java.time.Duration; import java.util.ArrayList; import java.util.List; @@ -8,6 +10,7 @@ import java.util.stream.Stream; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.errors.StreamsException; +import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; import org.radarbase.topic.KafkaTopic; import org.radarcns.config.ConfigRadar; import org.radarcns.config.KafkaProperty; @@ -16,7 +19,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractStreamWorker implements StreamWorker, UncaughtExceptionHandler { +public abstract class AbstractStreamWorker implements StreamWorker, + StreamsUncaughtExceptionHandler { private static final Logger logger = LoggerFactory.getLogger(AbstractStreamWorker.class); public static final String OUTPUT_LABEL = "_output"; @@ -166,15 +170,17 @@ protected void closeStreams() { * terminating due to an exception. */ @Override - public void uncaughtException(Thread t, Throwable e) { - logger.error("Thread {} has been terminated due to {}", t.getName(), e.getMessage(), e); + public StreamThreadExceptionResponse handle(Throwable e) { + logger.error("Stream worker has been terminated due to {}", e.getMessage(), e); closeStreams(); if (e instanceof StreamsException) { master.restartStream(this); + return SHUTDOWN_CLIENT; } else { master.notifyCrashedStream(getClass().getSimpleName()); + return SHUTDOWN_APPLICATION; } } } diff --git a/src/main/java/org/radarcns/stream/SensorStreamWorker.java b/src/main/java/org/radarcns/stream/SensorStreamWorker.java index 39042683..2fb9e2f0 100644 --- a/src/main/java/org/radarcns/stream/SensorStreamWorker.java +++ b/src/main/java/org/radarcns/stream/SensorStreamWorker.java @@ -57,7 +57,7 @@ * @param input value type. */ public abstract class SensorStreamWorker - extends AbstractStreamWorker implements Thread.UncaughtExceptionHandler { + extends AbstractStreamWorker { @SuppressWarnings("PMD.LoggerIsNotStaticFinal") private final Logger monitorLog; private Collection> monitors; diff --git a/src/main/java/org/radarcns/stream/StreamDefinition.java b/src/main/java/org/radarcns/stream/StreamDefinition.java index 0d0272a3..0bc273c1 100644 --- a/src/main/java/org/radarcns/stream/StreamDefinition.java +++ b/src/main/java/org/radarcns/stream/StreamDefinition.java @@ -51,7 +51,7 @@ public StreamDefinition(@Nonnull KafkaTopic input, @Nullable KafkaTopic output) */ public StreamDefinition(@Nonnull KafkaTopic input, @Nullable KafkaTopic output, @Nullable Duration window) { - this(input, output, window == null ? null : TimeWindows.of(window), + this(input, output, window == null ? null : TimeWindows.ofSizeWithNoGrace(window), TIME_WINDOW_COMMIT_INTERVAL_DEFAULT); } @@ -65,7 +65,7 @@ public StreamDefinition(@Nonnull KafkaTopic input, @Nullable KafkaTopic output, */ public StreamDefinition(@Nonnull KafkaTopic input, @Nullable KafkaTopic output, @Nullable Duration window, @Nonnull Duration commitIntervalMs) { - this(input, output, window == null ? null : TimeWindows.of(window), + this(input, output, window == null ? null : TimeWindows.ofSizeWithNoGrace(window), commitIntervalMs); } diff --git a/src/main/java/org/radarcns/stream/statistics/SourceStatisticsStream.java b/src/main/java/org/radarcns/stream/statistics/SourceStatisticsStream.java index 807f02c4..f582b78e 100644 --- a/src/main/java/org/radarcns/stream/statistics/SourceStatisticsStream.java +++ b/src/main/java/org/radarcns/stream/statistics/SourceStatisticsStream.java @@ -7,7 +7,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; +import io.confluent.kafka.serializers.AbstractKafkaSchemaSerDeConfig; import io.confluent.kafka.streams.serdes.avro.GenericAvroDeserializer; import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerializer; @@ -25,9 +25,10 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.processor.Cancellable; -import org.apache.kafka.streams.processor.Processor; -import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.PunctuationType; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.ProcessorContext; +import org.apache.kafka.streams.processor.api.Record; import org.apache.kafka.streams.state.KeyValueIterator; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; @@ -82,9 +83,9 @@ private Topology getTopology() { Topology topology = new Topology(); final Map serdeConfig = Map.of( - AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, - getStreamsConfig().get(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG), - AbstractKafkaAvroSerDeConfig.AUTO_REGISTER_SCHEMAS, + AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, + getStreamsConfig().get(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG), + AbstractKafkaSchemaSerDeConfig.AUTO_REGISTER_SCHEMAS, true); addSource("source", topology, serdeConfig); @@ -163,17 +164,16 @@ private Properties getStreamsConfig() { return settings; } - private class SourceStatisticsProcessor implements Processor { + private class SourceStatisticsProcessor implements + Processor { private KeyValueStore store; - private ProcessorContext context; + private ProcessorContext context; private Cancellable punctuateCancellor; private Duration localInterval = Duration.ZERO; - @SuppressWarnings("unchecked") @Override - public void init(ProcessorContext context) { - store = (KeyValueStore) context - .getStateStore("statistics"); + public void init(ProcessorContext context) { + store = context.getStateStore("statistics"); this.context = context; updatePunctuate(); } @@ -197,7 +197,8 @@ private void sendNew(long timestamp) { while (iterator.hasNext()) { KeyValue next = iterator.next(); if (!next.value.isSent) { - context.forward(next.key, next.value.sourceStatistics()); + context.forward(new Record<>( + next.key, next.value.sourceStatistics(), timestamp)); sent.add(new KeyValue<>(next.key, next.value.sentRecord())); } } @@ -211,7 +212,9 @@ private void sendNew(long timestamp) { @SuppressWarnings("PMD.AccessorMethodGeneration") @Override - public void process(GenericRecord genericKey, GenericRecord value) { + public void process(Record record) { + GenericRecord genericKey = record.key(); + GenericRecord value = record.value(); if (genericKey == null || value == null) { logger.error("Cannot process records without both a key and a value"); return; @@ -245,11 +248,6 @@ public void process(GenericRecord genericKey, GenericRecord value) { store.put(key, newStats); } } - - @Override - public void close() { - // do nothing - } } @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/src/main/java/org/radarcns/util/EmailSender.java b/src/main/java/org/radarcns/util/EmailSender.java index bdb496fc..a6686ea1 100644 --- a/src/main/java/org/radarcns/util/EmailSender.java +++ b/src/main/java/org/radarcns/util/EmailSender.java @@ -27,11 +27,16 @@ import javax.mail.internet.AddressException; import javax.mail.internet.InternetAddress; import javax.mail.internet.MimeMessage; +import org.radarcns.config.EmailServerConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Sends emails. */ public class EmailSender { + private static final Logger logger = LoggerFactory.getLogger(EmailSender.class); + private final InternetAddress from; private final InternetAddress[] recipients; private final Session session; @@ -44,8 +49,13 @@ public class EmailSender { * @param to list of recipients in the MIME To header * @throws IOException if a connection cannot be established with the email provider. */ - public EmailSender(String host, int port, @Nonnull String from, List to) + public EmailSender(EmailServerConfig config, @Nonnull String from, List to) throws IOException, AddressException { + this(createSession(config), from, to); + } + + public EmailSender(Session session, @Nonnull String from, List to) + throws AddressException { this.from = new InternetAddress(from); if (to == null || to.isEmpty()) { throw new AddressException("Cannot create email sender without recipients."); @@ -58,29 +68,40 @@ public EmailSender(String host, int port, @Nonnull String from, List to) } recipients[i] = new InternetAddress(addr); } + this.session = session; + } + + public static Session createSession(EmailServerConfig config) throws IOException { + String host = config.getHost(); + int port = config.getPort(); + + if (host == null || port <= 0) { + logger.error("Cannot configure email sender without hosts ({}), port ({})", + host, port); + return null; + } + Properties properties = new Properties(); // Get system properties properties.putAll(System.getProperties()); - if (host != null) { - // Setup mail server - properties.setProperty("mail.smtp.host", host); - } - if (port > 0) { - properties.setProperty("mail.smtp.port", String.valueOf(port)); - } + // Setup mail server + properties.setProperty("mail.smtp.host", host); + properties.setProperty("mail.smtp.port", String.valueOf(port)); - session = Session.getInstance(properties); + Session session = Session.getInstance(properties); try { Transport transport = session.getTransport("smtp"); transport.connect(); if (!transport.isConnected()) { - throw new IOException("Cannot connect to SMTP server " + host + ":" + port); + throw new IOException("Cannot connect to SMTP server " + + config.getHost() + ":" + config.getPort()); } } catch (MessagingException ex) { throw new IOException("Cannot instantiate SMTP server", ex); } + return session; } /** diff --git a/src/main/java/org/radarcns/util/EmailSenders.java b/src/main/java/org/radarcns/util/EmailSenders.java index e692117b..c4e2519f 100644 --- a/src/main/java/org/radarcns/util/EmailSenders.java +++ b/src/main/java/org/radarcns/util/EmailSenders.java @@ -2,10 +2,13 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; +import javax.mail.Session; import javax.mail.internet.AddressException; -import org.radarcns.config.MonitorConfig; -import org.radarcns.config.NotifyConfig; +import org.radarcns.config.EmailServerConfig; +import org.radarcns.config.monitor.MonitorConfig; +import org.radarcns.config.monitor.EmailNotifyConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,31 +30,31 @@ public EmailSenders(Map map) { * {@link EmailSender} to each project. A project can have a list of * associated email addresses. * - * @param config Configuration of the Monitor containing project + * @param serverConfig Email server configuration + * @param notifyConfigs Configuration of the Monitor containing project * and email address mapping * @throws IOException if a connection cannot be established with the email provider. */ + public static EmailSenders parseConfig(EmailServerConfig serverConfig, + List notifyConfigs) throws IOException { + String user = serverConfig.getUser(); - public static EmailSenders parseConfig(MonitorConfig config) throws IOException { - String host = config.getEmailHost(); - int port = config.getEmailPort(); - String user = config.getEmailUser(); - - if (host == null || user == null || port <= 0) { - logger.error("Cannot configure email sender without hosts ({}), port ({}) or user ({})", - host, user, port); - return new EmailSenders(Map.of()); + if (user == null) { + logger.error("Cannot configure email server without user"); + return null; } + Session session = EmailSender.createSession(serverConfig); + Map map = new HashMap<>(); - for (NotifyConfig notifyConfig : config.getNotifyConfig()) { + for (EmailNotifyConfig emailNotifyConfig : notifyConfigs) { try { - map.put(notifyConfig.getProjectId(), - new EmailSender(host, port, user, notifyConfig.getEmailAddress())); + map.put(emailNotifyConfig.getProjectId(), + new EmailSender(session, user, emailNotifyConfig.getEmailAddress())); } catch (AddressException e) { logger.error("Failed to add email sender for addresses {} and {}", - config.getEmailUser(), notifyConfig.getEmailAddress(), e); + user, emailNotifyConfig.getEmailAddress(), e); } } diff --git a/src/main/java/org/radarcns/util/PersistentStateStore.java b/src/main/java/org/radarcns/util/PersistentStateStore.java index cf88bdd1..7b16c326 100644 --- a/src/main/java/org/radarcns/util/PersistentStateStore.java +++ b/src/main/java/org/radarcns/util/PersistentStateStore.java @@ -9,7 +9,8 @@ * as a map key by serializing it to String. */ public interface PersistentStateStore { - /** Retrieve a state. The default is returned if no existing state is found. + /** + * Retrieve a state. The default is returned if no existing state is found. * * @param groupId Kafka group ID of a consumer or producer. * @param clientId Kafka client ID of a consumer or producer. diff --git a/src/main/java/org/radarcns/util/YamlPersistentStateStore.java b/src/main/java/org/radarcns/util/YamlPersistentStateStore.java index 3ee993f2..0d99d65a 100644 --- a/src/main/java/org/radarcns/util/YamlPersistentStateStore.java +++ b/src/main/java/org/radarcns/util/YamlPersistentStateStore.java @@ -16,6 +16,9 @@ package org.radarcns.util; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; +import com.fasterxml.jackson.module.kotlin.KotlinModule; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -37,14 +40,16 @@ public class YamlPersistentStateStore implements PersistentStateStore { /** * State store that creates files at given directory. The directory will be created if it * does not exist. + * + * @param loader * @param basePath path to a directory. * @throws IOException if the given directory is not writable for states. */ - public YamlPersistentStateStore(Path basePath) throws IOException { + public YamlPersistentStateStore(YamlConfigLoader loader, Path basePath) throws IOException { checkBasePath(basePath); this.basePath = basePath; - this.loader = new YamlConfigLoader(); + this.loader = loader; } /** diff --git a/src/test/java/org/radarcns/config/RadarPropertyHandlerTest.java b/src/test/java/org/radarcns/config/RadarPropertyHandlerTest.java index 8c8839f6..8c3b2553 100644 --- a/src/test/java/org/radarcns/config/RadarPropertyHandlerTest.java +++ b/src/test/java/org/radarcns/config/RadarPropertyHandlerTest.java @@ -19,16 +19,14 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasEntry; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException; import java.io.IOException; import java.lang.reflect.Field; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; /** * Created by nivethika on 19-12-16. @@ -39,22 +37,15 @@ public class RadarPropertyHandlerTest { @Before public void setUp() { - this.propertyHandler = new RadarPropertyHandlerImpl(); - } - @Rule - public ExpectedException exception = ExpectedException.none(); - @Test public void getInstanceEmptyProperties() throws NoSuchFieldException, IllegalAccessException, SecurityException { Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); properties.setAccessible(true); properties.set(this.propertyHandler,null); - exception.expect(IllegalStateException.class); - exception.expectMessage("Properties cannot be accessed without calling load() first"); - propertyHandler.getRadarProperties(); + assertThrows(IllegalStateException.class, () -> propertyHandler.getRadarProperties()); } @Test @@ -62,9 +53,8 @@ public void loadWithInvalidFilePath() throws Exception { Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); properties.setAccessible(true); properties.set(this.propertyHandler, null); - exception.expect(IOException.class); String invalidPath = "/usr/"; - propertyHandler.load(invalidPath); + assertThrows(IOException.class, () -> propertyHandler.load(invalidPath)); } @Test @@ -80,31 +70,24 @@ public void load() throws Exception { assertNotNull(properties.getReleased()); assertNotNull(properties.getSchemaRegistry()); assertNotNull(properties.getSchemaRegistryPaths()); - assertNotNull(properties.getZookeeper()); - assertNotNull(properties.getZookeeperPaths()); assertNotNull(properties.getVersion()); assertThat(properties.getExtras(), hasEntry("somethingother", "bla")); } @Test public void loadInvalidYaml() throws Exception { - exception.expect(UnrecognizedPropertyException.class); - propertyHandler.load("src/test/resources/config/invalidradar.yml"); + assertThrows(UnrecognizedPropertyException.class, () -> propertyHandler.load("src/test/resources/config/invalidradar.yml")); } @Test public void loadInvalidStreamPriority() throws Exception { - exception.expect(JsonMappingException.class); - propertyHandler.load("src/test/resources/config/invalid_stream_priority.yml"); + assertThrows(JsonMappingException.class, () -> propertyHandler.load("src/test/resources/config/invalid_stream_priority.yml")); } @Test public void loadWithInstance() throws Exception { - - exception.expect(IllegalStateException.class); - exception.expectMessage("Properties class has been already loaded"); propertyHandler.load("radar.yml"); - propertyHandler.load("again.yml"); + assertThrows(IllegalStateException.class, () -> propertyHandler.load("again.yml")); ConfigRadar propertiesS = propertyHandler.getRadarProperties(); assertNotNull(propertiesS); } @@ -114,20 +97,16 @@ public void getKafkaPropertiesBeforeLoad() throws IllegalAccessException, NoSuch Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); properties.setAccessible(true); properties.set(this.propertyHandler,null); - exception.expect(IllegalStateException.class); - exception.expectMessage("Properties cannot be accessed without calling load() first"); - KafkaProperty property =propertyHandler.getKafkaProperties(); - assertNull(property); + assertThrows(IllegalStateException.class, () -> propertyHandler.getKafkaProperties()); } @Test - public void getKafkaProperties() throws Exception - { + public void getKafkaProperties() throws Exception { Field properties = RadarPropertyHandlerImpl.class.getDeclaredField("properties"); properties.setAccessible(true); properties.set(propertyHandler,null); propertyHandler.load("radar.yml"); - KafkaProperty property =propertyHandler.getKafkaProperties(); + KafkaProperty property = propertyHandler.getKafkaProperties(); assertNotNull(property); } diff --git a/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java b/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java index e1996bf1..27061564 100644 --- a/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java +++ b/src/test/java/org/radarcns/monitor/BatteryLevelMonitorTest.java @@ -24,6 +24,9 @@ import static org.mockito.Mockito.verify; import static org.radarcns.monitor.BatteryLevelMonitor.Status.LOW; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; +import com.fasterxml.jackson.module.kotlin.KotlinModule; import java.io.File; import java.util.Collections; import java.util.Map; @@ -33,6 +36,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.radarbase.config.YamlConfigLoader; import org.radarcns.config.ConfigRadar; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.ObservationKey; @@ -50,8 +54,15 @@ public class BatteryLevelMonitorTest { private long offset; private long timeReceived; private int timesSent; - private EmailSenders senders; private EmailSender sender; + private final YamlConfigLoader loader = new YamlConfigLoader(mapper -> { + mapper.registerModule(new KotlinModule.Builder() + .enable(KotlinFeature.NullIsSameAsDefault) + .enable(KotlinFeature.NullToEmptyCollection) + .enable(KotlinFeature.NullToEmptyMap) + .build()); + mapper.registerModule(new JavaTimeModule()); + }); private static final String PROJECT_ID = "test"; @@ -62,7 +73,7 @@ public void evaluateRecord() throws Exception { timesSent = 0; sender = mock(EmailSender.class); - senders = new EmailSenders(Collections.singletonMap(PROJECT_ID, sender)); + EmailSenders senders = new EmailSenders(Collections.singletonMap(PROJECT_ID, sender)); ConfigRadar config = KafkaMonitorFactoryTest .getBatteryMonitorConfig(25252, folder); @@ -108,14 +119,14 @@ private void sendMessage(BatteryLevelMonitor monitor, float batteryLevel, boolea @Test public void retrieveState() throws Exception { File base = folder.newFolder(); - YamlPersistentStateStore stateStore = new YamlPersistentStateStore(base.toPath()); + YamlPersistentStateStore stateStore = new YamlPersistentStateStore(loader, base.toPath()); BatteryLevelState state = new BatteryLevelState(); ObservationKey key1 = new ObservationKey("test", "a", "b"); String keyString = stateStore.keyToString(key1); state.updateLevel(keyString, 0.1f); stateStore.storeState("one", "two", state); - YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(base.toPath()); + YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(loader, base.toPath()); BatteryLevelState state2 = stateStore2.retrieveState("one", "two", new BatteryLevelState()); Map values = state2.getLevels(); assertThat(values, hasEntry(keyString, 0.1f)); diff --git a/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java b/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java index c1e2822a..a2726206 100644 --- a/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java +++ b/src/test/java/org/radarcns/monitor/DisconnectMonitorTest.java @@ -26,6 +26,9 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.kotlin.KotlinFeature; +import com.fasterxml.jackson.module.kotlin.KotlinModule; import java.io.File; import java.time.Duration; import java.util.Collections; @@ -42,8 +45,9 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.radarbase.config.YamlConfigLoader; import org.radarcns.config.ConfigRadar; -import org.radarcns.config.DisconnectMonitorConfig; +import org.radarcns.config.monitor.DisconnectMonitorConfig; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.DisconnectMonitor.DisconnectMonitorState; @@ -64,6 +68,15 @@ public class DisconnectMonitorTest { private EmailSenders senders; private EmailSender sender; + private final YamlConfigLoader loader = new YamlConfigLoader(mapper -> { + mapper.registerModule(new KotlinModule.Builder() + .enable(KotlinFeature.NullIsSameAsDefault) + .enable(KotlinFeature.NullToEmptyCollection) + .enable(KotlinFeature.NullToEmptyMap) + .build()); + mapper.registerModule(new JavaTimeModule()); + }); + private static final String PROJECT_ID = "test"; @Before @@ -154,7 +167,7 @@ private void sendMessage(DisconnectMonitor monitor, String source, int sentMessa @Test public void retrieveState() throws Exception { File base = folder.newFolder(); - YamlPersistentStateStore stateStore = new YamlPersistentStateStore(base.toPath()); + YamlPersistentStateStore stateStore = new YamlPersistentStateStore(loader, base.toPath()); DisconnectMonitorState state = new DisconnectMonitorState(); ObservationKey key1 = new ObservationKey(PROJECT_ID, "a", "b"); ObservationKey key2 = new ObservationKey(PROJECT_ID, "b", "c"); @@ -166,7 +179,7 @@ public void retrieveState() throws Exception { (now -60L, now + 2L, 0)); stateStore.storeState("one", "two", state); - YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(base.toPath()); + YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(loader, base.toPath()); DisconnectMonitorState state2 = stateStore2.retrieveState("one", "two", new DisconnectMonitorState()); Map lastSeen = state2.getLastSeen(); assertThat(lastSeen.size(), is(2)); @@ -177,4 +190,4 @@ public void retrieveState() throws Exception { assertThat(reported, hasKey(stateStore.keyToString(key3))); } -} \ No newline at end of file +} diff --git a/src/test/java/org/radarcns/monitor/KafkaMonitorFactoryTest.java b/src/test/java/org/radarcns/monitor/KafkaMonitorFactoryTest.java index 10fc7c6d..be18dccf 100644 --- a/src/test/java/org/radarcns/monitor/KafkaMonitorFactoryTest.java +++ b/src/test/java/org/radarcns/monitor/KafkaMonitorFactoryTest.java @@ -16,17 +16,16 @@ package org.radarcns.monitor; +import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import org.apache.kafka.clients.consumer.ConsumerRecords; @@ -35,10 +34,11 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.radarbase.config.YamlConfigLoader; -import org.radarcns.config.BatteryMonitorConfig; +import org.radarcns.config.EmailServerConfig; +import org.radarcns.config.monitor.BatteryMonitorConfig; import org.radarcns.config.ConfigRadar; -import org.radarcns.config.DisconnectMonitorConfig; -import org.radarcns.config.NotifyConfig; +import org.radarcns.config.monitor.DisconnectMonitorConfig; +import org.radarcns.config.monitor.EmailNotifyConfig; import org.radarcns.config.RadarBackendOptions; import org.radarcns.config.RadarPropertyHandler; import org.radarcns.config.RadarPropertyHandlerImpl; @@ -96,9 +96,9 @@ public void createDisconnectMonitor() throws Exception { public void createAllMonitor() throws Exception { String[] args = {"monitor", "all"}; RadarBackendOptions options = RadarBackendOptions.parse(args); - ConfigRadar config = createBasicConfig(folder); - config.setBatteryMonitor(getBatteryMonitorConfig(emailServer.getPort())); - config.setDisconnectMonitor(getDisconnectMonitorConfig(emailServer.getPort())); + ConfigRadar config = createBasicConfig(folder, emailServer.getPort()); + config.setBatteryMonitor(getBatteryMonitorConfig()); + config.setDisconnectMonitor(getDisconnectMonitorConfig()); RadarPropertyHandler properties = getRadarPropertyHandler(config, folder); KafkaMonitor monitor = new KafkaMonitorFactory(options, properties).createMonitor(); @@ -120,59 +120,45 @@ public static RadarPropertyHandler getRadarPropertyHandler(ConfigRadar config, T return properties; } - public static ConfigRadar createBasicConfig(TemporaryFolder folder) throws IOException { + public static ConfigRadar createBasicConfig(TemporaryFolder folder, int port) throws IOException { ConfigRadar config = new ConfigRadar(); + EmailServerConfig serverConfig = new EmailServerConfig(); + serverConfig.setHost("localhost"); + serverConfig.setPort(port); + serverConfig.setUser("test@example"); + config.setEmailServerConfig(serverConfig); config.setPersistencePath(folder.newFolder().getAbsolutePath()); config.setSchemaRegistry(Collections.emptyList()); config.setBroker(Collections.emptyList()); return config; } - public static DisconnectMonitorConfig getDisconnectMonitorConfig(int port) { + public static DisconnectMonitorConfig getDisconnectMonitorConfig() { DisconnectMonitorConfig disconnectConfig = new DisconnectMonitorConfig(); - disconnectConfig.setNotifyConfig(List.of( - new NotifyConfig("test", List.of("test@localhost")))); - disconnectConfig.setEmailUser("test@localhost"); - disconnectConfig.setEmailHost("localhost"); - disconnectConfig.setEmailPort(port); + disconnectConfig.setEmailNotifyConfig(List.of( + new EmailNotifyConfig("test", List.of("test@localhost")))); disconnectConfig.setTimeout(1L); disconnectConfig.setAlertRepeatInterval(20L); return disconnectConfig; } public static ConfigRadar getDisconnectMonitorConfig(int port, TemporaryFolder folder) throws IOException { - ConfigRadar config = createBasicConfig(folder); - config.setDisconnectMonitor(getDisconnectMonitorConfig(port)); + ConfigRadar config = createBasicConfig(folder, port); + config.setDisconnectMonitor(getDisconnectMonitorConfig()); return config; } - public static BatteryMonitorConfig getBatteryMonitorConfig(int port) { + public static BatteryMonitorConfig getBatteryMonitorConfig() { BatteryMonitorConfig batteryConfig = new BatteryMonitorConfig(); - batteryConfig.setNotifyConfig(List.of( - new NotifyConfig("test", List.of("test@localhost")))); - batteryConfig.setEmailUser("test@localhost"); - batteryConfig.setEmailHost("localhost"); - batteryConfig.setEmailPort(port); + batteryConfig.setEmailNotifyConfig(List.of( + new EmailNotifyConfig("test", List.of("test@localhost")))); batteryConfig.setLevel("LOW"); - batteryConfig.setEmailUser("someuser"); return batteryConfig; } public static ConfigRadar getBatteryMonitorConfig(int port, TemporaryFolder folder) throws IOException { - ConfigRadar config = createBasicConfig(folder); - config.setBatteryMonitor(getBatteryMonitorConfig(port)); - return config; - } - - public static ConfigRadar getSourceStatisticsMonitorConfig(TemporaryFolder folder) throws IOException { - ConfigRadar config = createBasicConfig(folder); - SourceStatisticsStreamConfig sourceConfig = new SourceStatisticsStreamConfig(); - sourceConfig.setName("source_statistics_test"); - sourceConfig.setTopics(List.of("android_empatica_e4_battery_level", - "android_empatica_e4_battery_level_10sec")); - sourceConfig.setOutputTopic("statistics_android_empatica_e4"); - sourceConfig.setFlushTimeout(200L); - config.setStatisticsMonitors(Collections.singletonList(sourceConfig)); + ConfigRadar config = createBasicConfig(folder, port); + config.setBatteryMonitor(getBatteryMonitorConfig()); return config; } } diff --git a/src/test/java/org/radarcns/stream/DeviceTimestampExtractorTest.java b/src/test/java/org/radarcns/stream/DeviceTimestampExtractorTest.java index 34da01f6..d98f7e17 100644 --- a/src/test/java/org/radarcns/stream/DeviceTimestampExtractorTest.java +++ b/src/test/java/org/radarcns/stream/DeviceTimestampExtractorTest.java @@ -17,16 +17,14 @@ package org.radarcns.stream; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; /** * Created by nivethika on 20-12-16. @@ -36,9 +34,6 @@ public class DeviceTimestampExtractorTest { private DeviceTimestampExtractor timestampExtractor; private String topic; - @Rule - public ExpectedException exception = ExpectedException.none(); - @Before public void setUp() { this.timestampExtractor = new DeviceTimestampExtractor(); @@ -68,11 +63,7 @@ public void extractWithNotDoubleTimeReceived() { record.put("time", "timeValue"); ConsumerRecord consumerRecord = new ConsumerRecord<>(topic, 3, 30, null, record); - exception.expect(RuntimeException.class); - exception.expectMessage("Impossible to extract timeReceived from"); - long extracted = this.timestampExtractor.extract(consumerRecord, -1L); - assertNull(extracted); - + assertThrows(RuntimeException.class, () -> this.timestampExtractor.extract(consumerRecord, -1L)); } private static GenericRecord buildIndexedRecord(String userSchema) { diff --git a/src/test/java/org/radarcns/stream/phone/PlayStoreLookupTest.java b/src/test/java/org/radarcns/stream/phone/PlayStoreLookupTest.java index 25ed65ab..27f85103 100644 --- a/src/test/java/org/radarcns/stream/phone/PlayStoreLookupTest.java +++ b/src/test/java/org/radarcns/stream/phone/PlayStoreLookupTest.java @@ -33,7 +33,7 @@ public class PlayStoreLookupTest { public static Collection data() { return Arrays.asList(new Object[][] { { "nl.nos.app", "NEWS_AND_MAGAZINES" }, - { "com.twitter.android", "NEWS_AND_MAGAZINES" }, + { "com.twitter.android", "SOCIAL" }, { "com.facebook.katana", "SOCIAL" }, { "com.nintendo.zara", "GAME_ACTION" }, { "com.duolingo", "EDUCATION" }, diff --git a/src/test/java/org/radarcns/util/EmailSenderTest.java b/src/test/java/org/radarcns/util/EmailSenderTest.java index b2cba826..2caef87c 100644 --- a/src/test/java/org/radarcns/util/EmailSenderTest.java +++ b/src/test/java/org/radarcns/util/EmailSenderTest.java @@ -27,6 +27,7 @@ import javax.mail.internet.MimeMessage; import org.junit.Rule; import org.junit.Test; +import org.radarcns.config.EmailServerConfig; public class EmailSenderTest { @Rule @@ -34,7 +35,10 @@ public class EmailSenderTest { @Test public void testEmail() throws MessagingException, IOException { - EmailSender sender = new EmailSender("localhost", 2525, "no-reply@radar-cns.org", + EmailServerConfig emailServerConfig = new EmailServerConfig(); + emailServerConfig.setHost("localhost"); + emailServerConfig.setPort(emailServer.getPort()); + EmailSender sender = new EmailSender(emailServerConfig, "no-reply@radar-cns.org", List.of("test@radar-cns.org")); assertEquals(Collections.emptyList(), emailServer.getMessages()); @@ -58,7 +62,10 @@ public void testEmail() throws MessagingException, IOException { @Test(expected = IOException.class) public void testEmailNonExisting() throws MessagingException, IOException { - EmailSender sender = new EmailSender("non-existing-host", 2525, "no-reply@radar-cns.org", + EmailServerConfig emailServerConfig = new EmailServerConfig(); + emailServerConfig.setHost("non-existing-host"); + emailServerConfig.setPort(emailServer.getPort()); + EmailSender sender = new EmailSender(emailServerConfig, "no-reply@radar-cns.org", List.of("test@radar-cns.org")); } -} \ No newline at end of file +} diff --git a/src/test/java/org/radarcns/util/PersistentStateStoreTest.java b/src/test/java/org/radarcns/util/PersistentStateStoreTest.java index b24dd023..fa0e692d 100644 --- a/src/test/java/org/radarcns/util/PersistentStateStoreTest.java +++ b/src/test/java/org/radarcns/util/PersistentStateStoreTest.java @@ -27,6 +27,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.radarbase.config.YamlConfigLoader; import org.radarcns.kafka.ObservationKey; import org.radarcns.monitor.BatteryLevelMonitor.BatteryLevelState; @@ -34,10 +35,12 @@ public class PersistentStateStoreTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); + private final YamlConfigLoader loader = new YamlConfigLoader(); + @Test public void retrieveState() throws Exception { File base = folder.newFolder(); - YamlPersistentStateStore stateStore = new YamlPersistentStateStore(base.toPath()); + YamlPersistentStateStore stateStore = new YamlPersistentStateStore(loader, base.toPath()); BatteryLevelState state = new BatteryLevelState(); ObservationKey key1 = new ObservationKey("test", "a", "b"); state.updateLevel(stateStore.keyToString(key1), 0.1f); @@ -48,9 +51,9 @@ public void retrieveState() throws Exception { String rawFile = new String(Files.readAllBytes(outputFile.toPath())); assertThat(rawFile, equalTo("---\nlevels:\n test#a#b: 0.1\n")); - YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(base.toPath()); + YamlPersistentStateStore stateStore2 = new YamlPersistentStateStore(loader, base.toPath()); BatteryLevelState state2 = stateStore2.retrieveState("one", "two", new BatteryLevelState()); Map values = state2.getLevels(); assertThat(values, hasEntry(stateStore.keyToString(key1), 0.1f)); } -} \ No newline at end of file +} diff --git a/src/test/radar.yml b/src/test/radar.yml index 65f0eb22..b9f01325 100644 --- a/src/test/radar.yml +++ b/src/test/radar.yml @@ -1,12 +1,6 @@ version: 1.0 released: 2016-11-27 -#============================== Zookeeper ==============================# -#List of Zookeeper instances -zookeeper: - - host: localhost - port: 2181 - #================================ Kafka ================================# #List of Kafka brokers broker: @@ -30,4 +24,4 @@ stream: schema_registry: - host: localhost port: 8081 - protocol: http \ No newline at end of file + protocol: http diff --git a/src/test/resources/config/radar.yml b/src/test/resources/config/radar.yml index aabf6f46..94ede214 100644 --- a/src/test/resources/config/radar.yml +++ b/src/test/resources/config/radar.yml @@ -1,12 +1,6 @@ version: 1.0 released: 2016-11-27 -#============================== Zookeeper ==============================# -#List of Zookeeper instances -zookeeper: - - host: localhost - port: 2181 - #================================ Kafka ================================# #List of Kafka brokers broker: @@ -34,4 +28,4 @@ schema_registry: protocol: http extras: - somethingother: bla \ No newline at end of file + somethingother: bla