Merge pull request #208 from RADAR-base/fix-replication-factor-s3-con…

…nector [radar-s3-connector] Fix replication factor and allow config of JVM parameters
RADAR-base · Jun 26, 2024 · 85f2038 · 85f2038
2 parents e9665cb + 3f88501
commit 85f2038
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 4 deletions.
diff --git a/charts/radar-s3-connector/Chart.yaml b/charts/radar-s3-connector/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 appVersion: "7.6.0-hotfix"
 description: A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options
 name: radar-s3-connector
-version: 0.3.3
+version: 0.3.4
 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png"
 sources:
 - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-s3-connector

diff --git a/charts/radar-s3-connector/README.md b/charts/radar-s3-connector/README.md
@@ -3,7 +3,7 @@
 # radar-s3-connector
 [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-s3-connector)](https://artifacthub.io/packages/helm/radar-base/radar-s3-connector)
 
-![Version: 0.3.3](https://img.shields.io/badge/Version-0.3.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 7.6.0-hotfix](https://img.shields.io/badge/AppVersion-7.6.0--hotfix-informational?style=flat-square)
+![Version: 0.3.4](https://img.shields.io/badge/Version-0.3.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 7.6.0-hotfix](https://img.shields.io/badge/AppVersion-7.6.0--hotfix-informational?style=flat-square)
 
 A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options
 
@@ -40,6 +40,7 @@ A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 conne
 | nameOverride | string | `""` | String to partially override radar-s3-connector.fullname template with a string (will prepend the release name) |
 | fullnameOverride | string | `""` | String to fully override radar-s3-connector.fullname template with a string |
 | podSecurityContext | object | `{}` | Configure radar-s3-connector pods' Security Context |
+| kafkaHeapOpts | string | `"-Xms3g -Xmx4g"` | JVM Options |
 | securityContext | object | `{}` | Configure radar-s3-connector containers' Security Context |
 | service.type | string | `"ClusterIP"` | Kubernetes Service type |
 | service.port | int | `8083` | radar-s3-connector port |

diff --git a/charts/radar-s3-connector/templates/configmap.yaml b/charts/radar-s3-connector/templates/configmap.yaml
@@ -27,11 +27,17 @@ data:
     transforms.combineKeyValue.type=org.radarbase.kafka.connect.transforms.CombineKeyValue
     rotate.schedule.interval.ms={{ .Values.rotateInterval | int }}
     timezone=UTC
+    # -- See https://stackoverflow.com/questions/62525026/kafka-connect-exception-replication-factor-3-larger-than-available-brokers-1
+    config.storage.replication.factor={{ .Values.kafka_num_brokers | int }}
+    offset.storage.replication.factor={{ .Values.kafka_num_brokers | int }}
+    status.storage.replication.factor={{ .Values.kafka_num_brokers | int }}
+    topic.creation.default.replication.factor={{ .Values.kafka_num_brokers | int }}
+    # --
     avro.codec=deflate
     errors.tolerance=all
     errors.log.enable=true
     errors.deadletterqueue.topic.name=dead_letter_queue_s3
-    errors.deadletterqueue.topic.replication.factor=3
+    errors.deadletterqueue.topic.replication.factor={{ .Values.kafka_num_brokers | int }}
     errors.deadletterqueue.context.headers.enable=true
     errors.retry.delay.max.ms=60000
     errors.retry.timeout=300000

diff --git a/charts/radar-s3-connector/templates/deployment.yaml b/charts/radar-s3-connector/templates/deployment.yaml
@@ -176,7 +176,7 @@ spec:
           - name: CONNECT_VALUE_CONVERTER_CONNECT_META_DATA
             value: "false"
           - name: KAFKA_HEAP_OPTS
-            value: "-Xms3g -Xmx4g"
+            value: "{{ .Values.kafkaHeapOpts }}"
           - name: KAFKA_BROKERS
             value: "{{ .Values.kafka_num_brokers }}"
           - name: CONNECT_LOG4J_LOGGERS

diff --git a/charts/radar-s3-connector/values.yaml b/charts/radar-s3-connector/values.yaml
@@ -26,6 +26,9 @@ fullnameOverride: ""
 podSecurityContext: {}
   # fsGroup: 2000
 
+# -- JVM Options
+kafkaHeapOpts: "-Xms3g -Xmx4g"
+
 # -- Configure radar-s3-connector containers' Security Context
 securityContext: {}
   # capabilities: