Preparing chart for radar-output 2.4.0

RADAR-base · Oct 25, 2023 · 89b4d3d · 89b4d3d
1 parent 4ae1cd7
commit 89b4d3d
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 113 deletions.
diff --git a/charts/radar-output/Chart.yaml b/charts/radar-output/Chart.yaml
@@ -1,8 +1,8 @@
 apiVersion: v2
-appVersion: "2.3.1"
+appVersion: "2.4.0"
 description: A Helm chart for RADAR-base output restructure service. This application reads data from intermediate storage and restructure the data into project-> subject-id-> data topic -> data split per hour. This service offers few options to choose the source and target of the pipeline.
 name: radar-output
-version: 0.3.4
+version: 0.4.0
 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png"
 sources:
 - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-output

diff --git a/charts/radar-output/README.md b/charts/radar-output/README.md
@@ -3,7 +3,7 @@
 # radar-output
 [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-output)](https://artifacthub.io/packages/helm/radar-base/radar-output)
 
-![Version: 0.3.4](https://img.shields.io/badge/Version-0.3.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.3.1](https://img.shields.io/badge/AppVersion-2.3.1-informational?style=flat-square)
+![Version: 0.4.0](https://img.shields.io/badge/Version-0.4.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.4.0](https://img.shields.io/badge/AppVersion-2.4.0-informational?style=flat-square)
 
 A Helm chart for RADAR-base output restructure service. This application reads data from intermediate storage and restructure the data into project-> subject-id-> data topic -> data split per hour. This service offers few options to choose the source and target of the pipeline.
 
@@ -33,7 +33,7 @@ A Helm chart for RADAR-base output restructure service. This application reads d
 |-----|------|---------|-------------|
 | replicaCount | int | `1` | Number of radar-output replicas to deploy |
 | image.repository | string | `"radarbase/radar-output-restructure"` | radar-output image repository |
-| image.tag | string | `"2.3.1"` | radar-output image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. |
+| image.tag | string | `"2.4.0"` | radar-output image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. |
 | image.pullPolicy | string | `"IfNotPresent"` | radar-output image pull policy |
 | imagePullSecrets | list | `[]` | Docker registry secret names as an array |
 | nameOverride | string | `""` | String to partially override radar-output.fullname template with a string (will prepend the release name) |
@@ -67,25 +67,26 @@ A Helm chart for RADAR-base output restructure service. This application reads d
 | source.azure.responseTimeout | string | `nil` | Azure HTTP response timeout in seconds |
 | source.azure.writeTimeout | string | `nil` | Azure HTTP write timeout in seconds |
 | source.azure.readTimeout | string | `nil` | Azure HTTP read timeout in seconds |
-| target.type | string | `"s3"` | Type of the output storage of the RADAR-base pipeline e.g. s3, local |
-| target.s3.endpoint | string | `"http://minio:9000"` | s3 endpoint of the output storage |
-| target.s3.accessToken | string | `"access_key"` | s3 access-key of the output storage |
-| target.s3.secretKey | string | `"secret"` | s3 secret-key of the output storage |
-| target.s3.bucket | string | `"radar-output-storage"` | s3 bucket name of the output storage |
-| target.s3.connectTimeout | string | `nil` | s3 HTTP connect timeout in seconds |
-| target.s3.writeTimeout | string | `nil` | s3 HTTP write timeout in seconds |
-| target.s3.readTimeout | string | `nil` | s3 HTTP read timeout in seconds |
-| target.azure.endpoint | string | `""` | Azure endpoint of the output storage |
-| target.azure.username | string | `""` | Azure username to access the s3 endpoint when using personal login |
-| target.azure.password | string | `""` | Azure password when using personal login |
-| target.azure.accountName | string | `""` | Azure account name when using shared access tokens |
-| target.azure.accountKey | string | `""` | Azure account key when using shared access tokens |
-| target.azure.sasToken | string | `""` | Azure SAS(shared access signature) token when using shared access tokens |
-| target.azure.container | string | `""` | Azure blob container name |
-| target.azure.connectTimeout | string | `nil` | Azure HTTP connect timeout in seconds |
-| target.azure.responseTimeout | string | `nil` | Azure HTTP response timeout in seconds |
-| target.azure.writeTimeout | string | `nil` | Azure HTTP write timeout in seconds |
-| target.azure.readTimeout | string | `nil` | Azure HTTP read timeout in seconds |
+| targets.radar-output-storage.type | string | `"s3"` | Type of the output storage of the RADAR-base pipeline e.g. s3, local |
+| targets.radar-output-storage.path | string | `"output"` | base path in the output storage to store data in |
+| targets.radar-output-storage.s3.endpoint | string | `"http://minio:9000"` | s3 endpoint of the output storage |
+| targets.radar-output-storage.s3.accessToken | string | `"access_key"` | s3 access-key of the output storage |
+| targets.radar-output-storage.s3.secretKey | string | `"secret"` | s3 secret-key of the output storage |
+| targets.radar-output-storage.s3.bucket | string | `"radar-output-storage"` | s3 bucket name of the output storage |
+| targets.radar-output-storage.s3.connectTimeout | string | `nil` | s3 HTTP connect timeout in seconds |
+| targets.radar-output-storage.s3.writeTimeout | string | `nil` | s3 HTTP write timeout in seconds |
+| targets.radar-output-storage.s3.readTimeout | string | `nil` | s3 HTTP read timeout in seconds |
+| targets.radar-output-storage.azure.endpoint | string | `""` | Azure endpoint of the output storage |
+| targets.radar-output-storage.azure.username | string | `""` | Azure username to access the s3 endpoint when using personal login |
+| targets.radar-output-storage.azure.password | string | `""` | Azure password when using personal login |
+| targets.radar-output-storage.azure.accountName | string | `""` | Azure account name when using shared access tokens |
+| targets.radar-output-storage.azure.accountKey | string | `""` | Azure account key when using shared access tokens |
+| targets.radar-output-storage.azure.sasToken | string | `""` | Azure SAS(shared access signature) token when using shared access tokens |
+| targets.radar-output-storage.azure.container | string | `""` | Azure blob container name |
+| targets.radar-output-storage.azure.connectTimeout | string | `nil` | Azure HTTP connect timeout in seconds |
+| targets.radar-output-storage.azure.responseTimeout | string | `nil` | Azure HTTP response timeout in seconds |
+| targets.radar-output-storage.azure.writeTimeout | string | `nil` | Azure HTTP write timeout in seconds |
+| targets.radar-output-storage.azure.readTimeout | string | `nil` | Azure HTTP read timeout in seconds |
 | redis.uri | string | `"redis://redis-master:6379"` | URL of the redis database |
 | worker.interval | int | `90` | Scanning interval (seconds) |
 | worker.cacheSize | int | `300` | Maximum number of files and converters to keep open while processing |
@@ -95,10 +96,9 @@ A Helm chart for RADAR-base output restructure service. This application reads d
 | worker.numThreads | int | `2` | Number of threads to do processing on |
 | cleaner.age | int | `7` | Number of days after which a source file is considered old |
 | cleaner.interval | int | `86400` | Interval in seconds between cleanups |
-| paths.input | string | `"topics"` | Relative path to intermediate storage root to browse for data |
-| paths.output | string | `"output"` | Relative path to output storage to write data |
 | paths.factory | string | `"org.radarbase.output.path.FormattedPathFactory"` | Output path construction factory |
 | paths.properties | object | `{}` | Additional properties. For details see https://github.com/RADAR-base/radar-output-restructure/blob/master/restructure.yml |
+| paths.target.default | string | `"radar-output-storage"` | Default target to write data to |
 | topics | object | `{"questionnaire_response":{"pathProperties":{"format":"${projectId}/${userId}/${topic}/${value:name}/${filename}","plugins":"fixed value"}}}` | Individual topic configuration |
 | topics.questionnaire_response.pathProperties.format | string | `"${projectId}/${userId}/${topic}/${value:name}/${filename}"` | Alternative path output of the questionnaire_response topic |
 | topics.questionnaire_response.pathProperties.plugins | string | `"fixed value"` | Alternative path plugins of the questionnaire_response topic |

diff --git a/charts/radar-output/templates/configmap.yaml b/charts/radar-output/templates/configmap.yaml
@@ -48,38 +48,8 @@ data:
         {{- end }}
     {{- end }}
 
-    {{- with .Values.target }}
-    target:
-      type: "{{ .type }}"
-      # Minio S3 settings
-      s3:
-        endpoint: "{{ .s3.endpoint }}"
-        bucket: "{{ .s3.bucket }}"
-        {{- if .s3.connectTimeout }}
-        connectTimeout: {{ .s3.connectTimeout | int }}
-        {{- end }}
-        {{- if .s3.writeTimeout }}
-        writeTimeout: {{ .s3.writeTimeout | int }}
-        {{- end }}
-        {{- if .s3.readTimeout }}
-        readTimeout: {{ .s3.readTimeout | int }}
-        {{- end }}
-      azure:
-        endpoint: "{{ .azure.endpoint }}"
-        container: "{{ .azure.container }}"
-        {{- if .azure.connectTimeout }}
-        connectTimeout: {{ .azure.connectTimeout | int }}
-        {{- end }}
-        {{- if .azure.responseTimeout }}
-        responseTimeout: {{ .azure.responseTimeout | int }}
-        {{- end }}
-        {{- if .azure.writeTimeout }}
-        writeTimeout: {{ .azure.writeTimeout | int }}
-        {{- end }}
-        {{- if .azure.readTimeout }}
-        readTimeout: {{ .azure.readTimeout | int }}
-        {{- end }}
-    {{- end }}
+    targets:
+      {{ .Values.targets | toYaml | indent 6 }}
 
     redis:
       uri: "{{ .Values.redis.uri }}"
@@ -128,18 +98,9 @@ data:
 
     # Path settings
     paths:
-      # Input directories in HDFS
-      inputs:
-        - "{{ .Values.paths.input }}"
       # Root temporary directory for local file processing.
       temp: /output/+tmp
-      # Output directory
-      output: "{{ .Values.paths.output }}"
-      # Output path construction factory
-      factory: {{ .Values.paths.factory }}
-      # Additional properties
-      properties:
-        {{ .Values.paths.properties | toYaml | indent 8 | trim }}
+      {{ .Values.paths | toYaml | indent 6 | trim }}
 
     # Individual topic configuration
     {{- if .Values.topics }}

diff --git a/charts/radar-output/values.yaml b/charts/radar-output/values.yaml
@@ -10,7 +10,7 @@ image:
   repository: radarbase/radar-output-restructure
   # -- radar-output image tag (immutable tags are recommended)
   # Overrides the image tag whose default is the chart appVersion.
-  tag: 2.3.1
+  tag: 2.4.0
   # -- radar-output image pull policy
   pullPolicy: IfNotPresent
 
@@ -110,47 +110,50 @@ source:
     # -- Azure HTTP read timeout in seconds
     readTimeout:
 
-target:
-  # -- Type of the output storage of the RADAR-base pipeline e.g. s3, local
-  type: s3
-  s3:
-    # -- s3 endpoint of the output storage
-    endpoint: "http://minio:9000"
-    # -- s3 access-key of the output storage
-    accessToken: access_key
-    # -- s3 secret-key of the output storage
-    secretKey: secret
-    # -- s3 bucket name of the output storage
-    bucket: radar-output-storage
-    # -- s3 HTTP connect timeout in seconds
-    connectTimeout:
-    # -- s3 HTTP write timeout in seconds
-    writeTimeout:
-    # -- s3 HTTP read timeout in seconds
-    readTimeout:
-  azure:
-    # -- Azure endpoint of the output storage
-    endpoint: ""
-    # -- Azure username to access the s3 endpoint when using personal login
-    username: ""
-    # -- Azure password when using personal login
-    password: ""
-    # -- Azure account name when using shared access tokens
-    accountName: ""
-    # -- Azure account key when using shared access tokens
-    accountKey: ""
-    # -- Azure SAS(shared access signature) token when using shared access tokens
-    sasToken: ""
-    # -- Azure blob container name
-    container: ""
-    # -- Azure HTTP connect timeout in seconds
-    connectTimeout:
-    # -- Azure HTTP response timeout in seconds
-    responseTimeout:
-    # -- Azure HTTP write timeout in seconds
-    writeTimeout:
-    # -- Azure HTTP read timeout in seconds
-    readTimeout:
+targets:
+  radar-output-storage:
+    # -- Type of the output storage of the RADAR-base pipeline e.g. s3, local
+    type: s3
+    # -- base path in the output storage to store data in
+    path: output
+    s3:
+      # -- s3 endpoint of the output storage
+      endpoint: "http://minio:9000"
+      # -- s3 access-key of the output storage
+      accessToken: access_key
+      # -- s3 secret-key of the output storage
+      secretKey: secret
+      # -- s3 bucket name of the output storage
+      bucket: radar-output-storage
+      # -- s3 HTTP connect timeout in seconds
+      connectTimeout:
+      # -- s3 HTTP write timeout in seconds
+      writeTimeout:
+      # -- s3 HTTP read timeout in seconds
+      readTimeout:
+    azure:
+      # -- Azure endpoint of the output storage
+      endpoint: ""
+      # -- Azure username to access the s3 endpoint when using personal login
+      username: ""
+      # -- Azure password when using personal login
+      password: ""
+      # -- Azure account name when using shared access tokens
+      accountName: ""
+      # -- Azure account key when using shared access tokens
+      accountKey: ""
+      # -- Azure SAS(shared access signature) token when using shared access tokens
+      sasToken: ""
+      # -- Azure blob container name
+      container: ""
+      # -- Azure HTTP connect timeout in seconds
+      connectTimeout:
+      # -- Azure HTTP response timeout in seconds
+      responseTimeout:
+      # -- Azure HTTP write timeout in seconds
+      writeTimeout:
+      # -- Azure HTTP read timeout in seconds
+      readTimeout:
 
 redis:
   # -- URL of the redis database
@@ -177,14 +180,13 @@ cleaner:
   interval: 86400
 
 paths:
-  # -- Relative path to intermediate storage root to browse for data
-  input: topics
-  # -- Relative path to output storage to write data
-  output: output
   # -- Output path construction factory
   factory: org.radarbase.output.path.FormattedPathFactory
   # -- Additional properties. For details see https://github.com/RADAR-base/radar-output-restructure/blob/master/restructure.yml
   properties: {}
+  target:
+    # -- Default target to write data to
+    default: radar-output-storage
 
 # -- Individual topic configuration
 topics: