Merge pull request #213 from RADAR-base/release2.2.0

Release 2.2.0

README.md

@@ -1,4 +1,5 @@
-# RADAR-Docker 2.1.0
+# RADAR-Docker 2.2.0
+
 
 The dockerized RADAR stack for deploying the RADAR-base platform. Component repositories can be found at [RADAR-base DockerHub org](https://hub.docker.com/u/radarbase/dashboard/)
 

dcompose-stack/radar-cp-hadoop-stack/docker-compose.yml

@@ -808,3 +808,21 @@ services:
         interval: 1m30s
         timeout: 5s
         retries: 3
+
+  radar-output:
+    image: radarbase/radar-hdfs-restructure:0.6.0
+    restart: always
+    stop_signal: SIGINT
+    networks:
+      - hadoop
+    depends_on:
+      - hdfs-namenode-1
+      - hdfs-datanode-1
+      - hdfs-datanode-2
+      - hdfs-datanode-3
+    volumes:
+      - ${RESTRUCTURE_OUTPUT_DIR}:/output
+      - ./etc/hdfs-restructure/restructure.yml:/etc/restructure.yml
+    environment:
+      RADAR_HDFS_RESTRUCTURE_OPTS: -Xms250m -Xmx4g
+    command: -F /etc/restructure.yml -S

dcompose-stack/radar-cp-hadoop-stack/etc/env.template

@@ -30,9 +30,7 @@ KAFKA_MANAGER_PASSWORD=
 PORTAINER_PASSWORD_HASH=
 MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_0=radarbase-managementportal-ec
 MANAGEMENTPORTAL_OAUTH_CHECKING_KEY_ALIASES_1=selfsigned
-RESTRUCTURE_OUTPUT_DIR=output
-RESTRUCTURE_NUM_THREADS=3
-RESTRUCTURE_ENABLE_SNAPSHOTS=true
+RESTRUCTURE_OUTPUT_DIR=./output
 MANAGEMENTPORTAL_COMMON_ADMIN_PASSWORD=
 MANAGEMENTPORTAL_COMMON_PRIVACY_POLICY_URL=http://info.thehyve.nl/radar-cns-privacy-policy
 ENABLE_OPTIONAL_SERVICES=false

dcompose-stack/radar-cp-hadoop-stack/etc/hdfs-restructure/restructure.yml.template

@@ -0,0 +1,92 @@
+service:
+  # Whether to run the application as a polling service.
+  enable: true
+  # Polling interval in seconds.
+  interval: 300
+
+# Compression characteristics
+compression:
+  # Compression type: none, zip or gzip
+  type: gzip
+  # Compression Factory class
+  # factory: org.radarbase.hdfs.data.CompressionFactory
+  # Additional compression properties
+  # properties: {}
+
+# File format
+format:
+  # Format type: CSV or JSON
+  type: csv
+  # Whether to deduplicate the files in each topic by default
+  deduplication:
+    enable: true
+    # Deduplicate considering only distinct fields sourceId and time.
+    # This may incur data loss if multiple measurements are recorded
+    # at exactly the same time. By default, all values are considered
+    # when looking at distinct lines.
+    # distinctFields: [key.sourceId, value.time]
+  # Format factory class
+  # factory: org.radarbase.hdfs.data.FormatFactory
+  # Additional format properties
+  # properties: {}
+
+# Worker settings
+worker:
+  # Maximum number of files and converters to keep open while processing
+  cacheSize: 300
+  # Number of threads to do processing with
+  numThreads: 2
+  # Maximum number of files to process in any given topic.
+  # maxFilesPerTopic: null
+
+storage:
+  factory: org.radarbase.hdfs.storage.LocalStorageDriver
+  properties:
+    localUid: 0
+    localGid: 0
+
+# Path settings
+paths:
+  # Input directories in HDFS
+  inputs:
+    - /topicAndroidNew
+  # Root temporary directory for local file processing.
+  temp: /output/+tmp
+  # Output directory
+  output: /output
+  # Output path construction factory
+  #factory: org.radarbase.hdfs.MonthlyObservationKeyPathFactory
+  # Additional properties
+  # properties: {}
+
+# Individual topic configuration
+topics: {}
+#  # topic name
+#  connect_fitbit_source:
+#    # deduplicate this topic, regardless of the format settings
+#    deduplication:
+#      # deduplicate this topic only using given fields.
+#      distinctFields: [value.time]
+#  connect_fitbit_bad:
+#    # Do not process this topic
+#    exclude: true
+#  biovotion_acceleration:
+#    # Disable deduplication
+#    deduplication:
+#      enable: false
+
+# HDFS settings
+hdfs:
+  # HDFS name node in case of a single name node, or HDFS cluster ID in case of high availability.
+  name: hdfs-namenode-1
+  # High availability settings:
+  # nameNodes:
+  #   - name: hdfs1
+  #     hostname: hdfs-namenode-1
+  #   - name: hdfs2
+  #     hostname: hdfs-namenode-2
+  # Where files will be locked. This value should be the same for all restructure processes.
+  lockPath: /logs/org.radarbase.hdfs/lock
+  # Additional raw HDFS configuration properties
+  # properties: {}
+

dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.conf.template

@@ -77,7 +77,26 @@ http {
       deny all;
     }
     location /schema/ {
-      limit_except GET {
+      if ($request_method = 'OPTIONS') {
+        # Tell client that this pre-flight info is valid for 20 days
+        add_header 'Access-Control-Allow-Origin' "$http_origin" always;
+        add_header 'Access-Control-Allow-Credentials' 'true' always;
+        add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always;
+        add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always;
+        add_header 'Access-Control-Max-Age' 1728000;
+        add_header 'Content-Type' 'text/plain charset=UTF-8';
+        add_header 'Content-Length' 0;
+        return 204;
+      }
+
+      if ($request_method = 'GET') {
+        add_header 'Access-Control-Allow-Origin' "$http_origin" always;
+        add_header 'Access-Control-Allow-Credentials' 'true' always;
+        add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always;
+        add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always;
+      }
+
+      limit_except GET OPTIONS {
         deny all;
       }
       proxy_pass         http://schema-registry-1:8081/;

dcompose-stack/radar-cp-hadoop-stack/etc/webserver/nginx.nossl.conf.template

@@ -48,7 +48,26 @@ http {
       deny all;
     }
     location /schema/ {
-      limit_except GET {
+      if ($request_method = 'OPTIONS') {
+        # Tell client that this pre-flight info is valid for 20 days
+        add_header 'Access-Control-Allow-Origin' "$http_origin" always;
+        add_header 'Access-Control-Allow-Credentials' 'true' always;
+        add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always;
+        add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always;
+        add_header 'Access-Control-Max-Age' 1728000;
+        add_header 'Content-Type' 'text/plain charset=UTF-8';
+        add_header 'Content-Length' 0;
+        return 204;
+      }
+
+      if ($request_method = 'GET') {
+        add_header 'Access-Control-Allow-Origin' "$http_origin" always;
+        add_header 'Access-Control-Allow-Credentials' 'true' always;
+        add_header 'Access-Control-Allow-Methods' 'GET, OPTIONS' always;
+        add_header 'Access-Control-Allow-Headers' 'Accept,Authorization,Cache-Control,Content-Type,DNT,If-Modified-Since,Keep-Alive,Origin,User-Agent,X-Requested-With' always;
+      }
+
+      limit_except GET OPTIONS {
         deny all;
       }
       proxy_pass         http://schema-registry-1:8081/;

dcompose-stack/radar-cp-hadoop-stack/lib/install-systemd-wrappers.sh

@@ -16,8 +16,6 @@ fi
 
 echo "==> Copying templates"
 copy_template $BASE/radar-docker.service lib/systemd/radar-docker.service.template
-copy_template $BASE/radar-output.service lib/systemd/radar-output.service.template
-copy_template $BASE/radar-output.timer lib/systemd/radar-output.timer.template
 copy_template $BASE/radar-check-health.service lib/systemd/radar-check-health.service.template
 copy_template $BASE/radar-check-health.timer lib/systemd/radar-check-health.timer.template
 copy_template $BASE/radar-renew-certificate.service lib/systemd/radar-renew-certificate.service.template
@@ -27,9 +25,6 @@ echo "==> Inlining variables"
 inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-docker.service
 inline_variable 'ExecStart=' "$PWD/bin/radar-docker foreground" $BASE/radar-docker.service
 
-inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-output.service
-inline_variable 'ExecStart=' "$PWD/bin/hdfs-restructure /topicAndroidNew ${RESTRUCTURE_OUTPUT_DIR:-output}" $BASE/radar-output.service
-
 inline_variable 'WorkingDirectory=' "$PWD" $BASE/radar-check-health.service
 inline_variable 'ExecStart=' "$PWD/bin/radar-docker health" $BASE/radar-check-health.service
 
@@ -39,10 +34,8 @@ inline_variable 'ExecStart=' "$PWD/bin/radar-docker cert-renew" $BASE/radar-rene
 echo "==> Reloading systemd"
 systemctl "${SYSTEMCTL_OPTS[@]}" daemon-reload
 systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-docker
-systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-output.timer
 systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-check-health.timer
 systemctl "${SYSTEMCTL_OPTS[@]}" enable radar-renew-certificate.timer
 systemctl "${SYSTEMCTL_OPTS[@]}" start radar-docker
-systemctl "${SYSTEMCTL_OPTS[@]}" start radar-output.timer
 systemctl "${SYSTEMCTL_OPTS[@]}" start radar-check-health.timer
 systemctl "${SYSTEMCTL_OPTS[@]}" start radar-renew-certificate.timer

dcompose-stack/radar-cp-hadoop-stack/lib/perform-install.sh

@@ -20,6 +20,7 @@ copy_template_if_absent etc/webserver/ip-access-control.conf
 copy_template_if_absent etc/webserver/optional-services.conf
 copy_template_if_absent etc/fitbit/docker/source-fitbit.properties
 copy_template_if_absent etc/rest-source-authorizer/rest_source_clients_configs.yml
+copy_template_if_absent etc/hdfs-restructure/restructure.yml
 
 # Set permissions
 sudo-linux chmod og-rw ./.env