First data-plane version (knative#5)

* Transfer data-plane Signed-off-by: Pierangelo Di Pilato <pierangelodipilato@gmail.com> * Add integration tests Signed-off-by: Pierangelo Di Pilato <pierangelodipilato@gmail.com> * Add copyright header Signed-off-by: Pierangelo Di Pilato <pierangelodipilato@gmail.com>
odacremolbap · Jun 24, 2020 · 4980f4b · 4980f4b
1 parent 859685a
commit 4980f4b
Show file tree

Hide file tree

Showing 558 changed files with 83,068 additions and 12 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,9 @@
+**/.idea/
+
+**/target
+
+**/*.iml
+
+**/dependency-reduced-pom.xml
+
+**/?
diff --git a/data-plane/.dockerignore b/data-plane/.dockerignore
@@ -0,0 +1,8 @@
+.idea
+.git
+target
+*/target
+*.iml
+config
+build
+*.md
diff --git a/data-plane/.gitignore b/data-plane/.gitignore
@@ -0,0 +1 @@
+generated/src/main/java
diff --git a/data-plane/checkstyle/suppression.xml b/data-plane/checkstyle/suppression.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0"?>
+
+<!DOCTYPE suppressions PUBLIC
+  "-//Checkstyle//DTD SuppressionFilter Configuration 1.0//EN"
+  "https://checkstyle.org/dtds/suppressions_1_0.dtd">
+
+<suppressions>
+  <suppress checks="AbbreviationAsWordInName" files="\.java"/>
+  <suppress checks="MissingJavadocMethod" files="CoreObjects.java"/>
+  <suppress files="[/\\]generated[/\\]" checks=".*" />
+</suppressions>
diff --git a/data-plane/config/0-namespace.yaml b/data-plane/config/0-namespace.yaml
@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: knative-eventing
+  namespace: knative-eventing
diff --git a/data-plane/config/100-config-kafka-broker-dispatcher.yaml b/data-plane/config/100-config-kafka-broker-dispatcher.yaml
@@ -0,0 +1,107 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-kafka-broker-dispatcher
+  namespace: knative-eventing
+data:
+  config-kafka-broker-producer.properties: |
+    bootstrap.servers=my-cluster-kafka-bootstrap.kafka:9092
+    # key.serializer=org.apache.kafka.common.serialization.StringSerializer
+    # value.serializer=org.apache.kafka.common.serialization.StringSerializer
+    acks=1
+    buffer.memory=33554432
+    compression.type=snappy
+    retries=2147483647
+    batch.size=16384
+    client.dns.lookup=use_all_dns_ips
+    client.id=KKBD # Knative Kafka Broker Dispatcher
+    connections.max.idle.ms=600000
+    delivery.timeout.ms=120000
+    linger.ms=0
+    max.block.ms=60000
+    max.request.size=1048576
+    partitioner.class=org.apache.kafka.clients.producer.internals.DefaultPartitioner
+    receive.buffer.bytes=-1
+    request.timeout.ms=30000
+    enable.idempotence=false
+    # interceptor.classes=""
+    max.in.flight.requests.per.connection=5
+    metadata.max.age.ms=300000
+    # metric.reporters=""
+    metrics.num.samples=2
+    metrics.recording.level=INFO
+    metrics.sample.window.ms=30000
+    reconnect.backoff.max.ms=1000
+    reconnect.backoff.ms=50
+    retry.backoff.ms=100
+    # transaction.timeout.ms=60000
+    # transactional.id=null
+  config-kafka-broker-consumer.properties: |
+    bootstrap.servers=my-cluster-kafka-bootstrap.kafka:9092
+    # key.serializer=org.apache.kafka.common.serialization.StringSerializer
+    # value.serializer=org.apache.kafka.common.serialization.StringSerializer
+    fetch.min.bytes=1
+    # group.id= dynamically set
+    heartbeat.interval.ms=3000
+    max.partition.fetch.bytes=1048576
+    session.timeout.ms=10000
+    # ssl.key.password=
+    # ssl.keystore.location=
+    # ssl.keystore.password=
+    # ssl.truststore.location=
+    # ssl.truststore.password=
+    allow.auto.create.topics=true
+    auto.offsets.reset=latest
+    client.dns.lookup=use_all_dns_ips
+    connections.max.idle.ms=540000
+    default.api.timeout.ms=60000
+    enable.auto.commit=false
+    exclude.internals.topics=false
+    fetch.max.bytes=52428800
+    # group.instance.id=
+    isolation.level=read_uncommitted
+    max.poll.interval.ms=300000
+    max.poll.records=500
+    partition.assignement.strategy=org.apache.kafka.clients.consumer.CooperativeStickyAssignor # Kafka 2.3 required
+    receive.buffer.bytes=65536
+    request.timeout.ms=30000
+    # sasl.client.callback.handler.class=
+    # sasl.jaas.config=
+    # sasl.kerberos.service.name=
+    # sasl.login.callback.handler.class
+    # sasl.login.class
+    # sasl.mechanism
+    security.protocol=PLAINTEXT
+    send.buffer.bytes=131072
+    # ssl.enabled.protocols=
+    # ssl.keystore.type=
+    # ssl.protocol=
+    # ssl.provider=
+    auto.commit.interval.ms=5000
+    check.crcs=true
+    client.id=KKBD # Knative Kafka Broker Dispatcher
+    # client.rack=
+    fetch.max.wait.ms=500
+    # interceptor.classes=
+    metadata.max.age.ms=600000
+    # metrics.reporters=
+    # metrics.num.samples=
+    # metrics.recording.level=INFO
+    # metrics.sample.window.ms=
+    reconnect.backoff.max.ms=1000
+    retry.backoff.ms=100
+    # sasl.kerberos.kinit.cmd=
+    # sasl.kerberos.min.time.before.relogin=
+    # sasl.kerberos.ticket.renew.jitter=
+    # sasl.login.refresh.buffer.seconds=
+    # sasl.login.refresh.min.period.seconds=
+    # sasl.login.refresh.window.factor
+    # sasl.login.refresh.window.jitter
+    # security.providers
+    # ssl.cipher.suites
+    # ssl.endpoint.identification.algorithm
+    # ssl.keymanager.algorithm
+    # ssl.secure.random.implementation
+    # ssl.trustmanager.algorithm
+
+
diff --git a/data-plane/config/100-config-kafka-broker-receiver.yaml b/data-plane/config/100-config-kafka-broker-receiver.yaml
@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-kafka-broker-receiver
+  namespace: knative-eventing
+data:
+  config-kafka-broker-producer.properties: |
+    bootstrap.servers=my-cluster-kafka-bootstrap.kafka:9092
+    # key.serializer=org.apache.kafka.common.serialization.StringSerializer
+    # value.serializer=org.apache.kafka.common.serialization.StringSerializer
+    acks=1
+    buffer.memory=33554432
+    compression.type=snappy
+    retries=2147483647
+    batch.size=16384
+    client.dns.lookup=use_all_dns_ips
+    client.id=KKBR # Knative Kafka Broker Receiver
+    connections.max.idle.ms=600000
+    delivery.timeout.ms=120000
+    linger.ms=0
+    max.block.ms=60000
+    max.request.size=1048576
+    partitioner.class=org.apache.kafka.clients.producer.internals.DefaultPartitioner
+    receive.buffer.bytes=-1
+    request.timeout.ms=30000
+    enable.idempotence=false
+    # interceptor.classes=""
+    max.in.flight.requests.per.connection=5
+    metadata.max.age.ms=300000
+    # metric.reporters=""
+    metrics.num.samples=2
+    metrics.recording.level=INFO
+    metrics.sample.window.ms=30000
+    reconnect.backoff.max.ms=1000
+    reconnect.backoff.ms=50
+    retry.backoff.ms=100
+    # transaction.timeout.ms=60000
+    # transactional.id=null
diff --git a/data-plane/config/100-triggers-configmap.yaml b/data-plane/config/100-triggers-configmap.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kafka-broker-brokers-triggers
+  namespace: knative-eventing
+  labels:
+    eventing.knative.dev/release: devel
+
diff --git a/data-plane/config/template/500-dispatcher.yaml b/data-plane/config/template/500-dispatcher.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-broker-dispatcher
+  namespace: knative-eventing
+  labels:
+    app: kafka-broker-dispatcher
+    eventing.knative.dev/release: devel
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-broker-dispatcher
+  template:
+    metadata:
+      name: kafka-broker-dispatcher
+      labels:
+        app: kafka-broker-dispatcher
+        eventing.knative.dev/release: devel
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+      containers:
+        - name: kafka-broker-dispatcher
+          image: ${KNATIVE_KAFKA_BROKER_DISPATCHER_IMAGE}
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /etc/config
+              name: config-kafka-broker-dispatcher
+              readOnly: true
+            - mountPath: /etc/brokers-triggers
+              name: kafka-broker-brokers-triggers
+              readOnly: true
+            - mountPath: /tmp
+              name: cache
+          env:
+            - name: PRODUCER_CONFIG_FILE_PATH
+              value: /etc/config/config-kafka-broker-producer.properties
+            - name: CONSUMER_CONFIG_FILE_PATH
+              value: /etc/config/config-kafka-broker-consumer.properties
+            - name: BROKERS_TRIGGERS_PATH
+              value: /etc/brokers-triggers/data
+            - name: BROKERS_INITIAL_CAPACITY
+              value: "100"
+            - name: TRIGGERS_INITIAL_CAPACITY
+              value: "20"
+          command:
+            - "java"
+          # TODO add JVM arguments
+          args:
+            - "-jar"
+            - "/app/app.jar"
+          terminationMessagePolicy: FallbackToLogsOnError
+          terminationMessagePath: /dev/temination-log
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: true
+      volumes:
+        - name: config-kafka-broker-dispatcher
+          configMap:
+            name: config-kafka-broker-dispatcher
+        - name: kafka-broker-brokers-triggers
+          configMap:
+            name: kafka-broker-brokers-triggers
+        - name: cache
+          emptyDir: {}
+      restartPolicy: Always
diff --git a/data-plane/config/template/500-receiver.yaml b/data-plane/config/template/500-receiver.yaml
@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kafka-broker-receiver
+  namespace: knative-eventing
+  labels:
+    app: kafka-broker-receiver
+    eventing.knative.dev/release: devel
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: kafka-broker-receiver
+  template:
+    metadata:
+      name: kafka-broker-receiver
+      labels:
+        app: kafka-broker-receiver
+        eventing.knative.dev/release: devel
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 999
+      containers:
+        - name: kafka-broker-receiver
+          image: ${KNATIVE_KAFKA_BROKER_RECEIVER_IMAGE}
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /etc/config
+              name: config-kafka-broker-receiver
+              readOnly: true
+            - mountPath: /tmp
+              name: cache
+          env:
+            - name: INGRESS_PORT
+              value: "8080"
+            - name: PRODUCER_CONFIG_FILE_PATH
+              value: /etc/config/config-kafka-broker-producer.properties
+            - name: LIVENESS_PROBE_PATH
+              value: /healthz
+            - name: READINESS_PROBE_PATH
+              value: /readyz
+          command:
+            - "java"
+          # TODO add JVM arguments
+          args:
+            - "-jar"
+            - "/app/app.jar"
+          # TODO set resources (limits and requests)
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              port: 8080
+              path: /healthz
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 3
+            successThreshold: 1
+            timeoutSeconds: 1
+          # TODO check kafka reachability
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              port: 8080
+              path: /readyz
+              scheme: HTTP
+            initialDelaySeconds: 5
+            periodSeconds: 3
+            successThreshold: 1
+            timeoutSeconds: 1
+          terminationMessagePolicy: FallbackToLogsOnError
+          terminationMessagePath: /dev/temination-log
+          securityContext:
+            allowPrivilegeEscalation: false
+            privileged: false
+            readOnlyRootFilesystem: true
+      volumes:
+        - name: config-kafka-broker-receiver
+          configMap:
+            name: config-kafka-broker-receiver
+        - name: cache
+          emptyDir: {}
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kafka-broker-ingress
+  namespace: knative-eventing
+spec:
+  selector:
+    app: kafka-broker-receiver
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  type: NodePort