From 71fd9f5068786c7571c83e5a16dbf1629630919e Mon Sep 17 00:00:00 2001 From: Ljupco Vangelski Date: Thu, 13 Jul 2023 12:51:37 +0200 Subject: [PATCH] [#4108] Fix schemas and topics screens (#4109) * [#4108] Start Rest Karapace by default * Update docs for custom partitions * Add Kafka proxy to the airy-controller * Fix lint * Fix Helm test --- .../docs/getting-started/installation/helm.md | 10 ++++ infrastructure/controller/pkg/endpoints/BUILD | 1 + .../controller/pkg/endpoints/proxy.go | 58 +++++++++++++++++++ .../controller/pkg/endpoints/server.go | 6 ++ .../schema-registry/templates/service.yaml | 4 ++ .../kafka/charts/schema-registry/values.yaml | 2 +- .../templates/components/ingress.yaml | 2 +- 7 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 infrastructure/controller/pkg/endpoints/proxy.go diff --git a/docs/docs/getting-started/installation/helm.md b/docs/docs/getting-started/installation/helm.md index 03dcc7038f..e47ed9b447 100644 --- a/docs/docs/getting-started/installation/helm.md +++ b/docs/docs/getting-started/installation/helm.md @@ -290,6 +290,16 @@ Run the following command to create the `Airy` platform without the bundled inst helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --values ./airy.yaml ``` +### Kafka partitions per topic + +Currently all the default topics in the Airy instance are created with 10 partitions. To create these topics with a different number of partitions, add the following to your `airy.yaml` file before running `helm install` (before the initial creation of the topics): + +``` +provisioning: + kafka: + partitions: 2 +``` + ### Beanstalkd The default installation creates its own [Beanstalkd](https://beanstalkd.github.io/) deployment, as it is a prerequisite for using the `integration/webhook` component. diff --git a/infrastructure/controller/pkg/endpoints/BUILD b/infrastructure/controller/pkg/endpoints/BUILD index fdcbf72037..c8705df73c 100644 --- a/infrastructure/controller/pkg/endpoints/BUILD +++ b/infrastructure/controller/pkg/endpoints/BUILD @@ -12,6 +12,7 @@ go_library( "components_list.go", "components_update.go", "cors.go", + "proxy.go", "server.go", "services.go", ], diff --git a/infrastructure/controller/pkg/endpoints/proxy.go b/infrastructure/controller/pkg/endpoints/proxy.go new file mode 100644 index 0000000000..8dc0f90afc --- /dev/null +++ b/infrastructure/controller/pkg/endpoints/proxy.go @@ -0,0 +1,58 @@ +package endpoints + +import ( + "log" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + "k8s.io/client-go/kubernetes" + "k8s.io/helm/cmd/helm/search" +) + +type proxyTarget struct { + name string + url string + stripUri string +} + +type KafkaSubjects struct { + ClientSet *kubernetes.Clientset + Namespace string + Index *search.Index +} + +type KafkaTopics struct { + ClientSet *kubernetes.Clientset + Namespace string + Index *search.Index +} + +func (s *KafkaSubjects) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var proxyUpstream = proxyTarget{"subjects", "http://schema-registry:8081/subjects", "/kafka/subjects"} + proxyRequest(w, r, proxyUpstream) +} + +func (s *KafkaTopics) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var proxyUpstream = proxyTarget{"subjects", "http://schema-registry:8082/topics", "/kafka/topics"} + proxyRequest(w, r, proxyUpstream) +} + +func proxyRequest(w http.ResponseWriter, r *http.Request, proxyUpstream proxyTarget) { + target, err := url.Parse(proxyUpstream.url) + if err != nil { + log.Printf("Error parsing target URL: %v\n", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return + } + + proxy := httputil.NewSingleHostReverseProxy(target) + r.URL.Host = target.Host + r.URL.Scheme = target.Scheme + r.Header.Set("X-Forwarded-Host", r.Header.Get("Host")) + r.Host = target.Host + r.URL.Path = strings.TrimPrefix(r.URL.Path, proxyUpstream.stripUri) + + proxy.ServeHTTP(w, r) +} diff --git a/infrastructure/controller/pkg/endpoints/server.go b/infrastructure/controller/pkg/endpoints/server.go index 27e24c426c..b3c68443fe 100644 --- a/infrastructure/controller/pkg/endpoints/server.go +++ b/infrastructure/controller/pkg/endpoints/server.go @@ -70,6 +70,12 @@ func Serve(clientSet *kubernetes.Clientset, namespace string, kubeConfig *rest.C componentsList := ComponentsList{ClientSet: clientSet, Namespace: namespace, Index: helmIndex} r.Handle("/components.list", &componentsList) + kafkaSubjects := KafkaSubjects{ClientSet: clientSet, Namespace: namespace, Index: helmIndex} + r.Handle("/kafka/subjects", &kafkaSubjects) + + kafkaTopics := KafkaTopics{ClientSet: clientSet, Namespace: namespace, Index: helmIndex} + r.Handle("/kafka/topics", &kafkaTopics) + log.Fatal(http.ListenAndServe(":8080", r)) } diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml index 7d17a5bd92..52c14795e7 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/templates/service.yaml @@ -9,6 +9,10 @@ spec: ports: - name: schema-registry port: {{ .Values.servicePort }} +{{- if .Values.restEnabled }} + - name: rest-api + port: {{ .Values.restPort }} +{{- end}} selector: app: schema-registry release: {{ .Release.Name }} diff --git a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml index 3f89f02c9a..d95dfcad3f 100644 --- a/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml +++ b/infrastructure/helm-chart/charts/prerequisites/charts/kafka/charts/schema-registry/values.yaml @@ -9,4 +9,4 @@ kafka: bootstrapServers: kafka-headless:9092 minBrokers: 1 resources: {} -restEnabled: false \ No newline at end of file +restEnabled: true \ No newline at end of file diff --git a/infrastructure/helm-chart/templates/components/ingress.yaml b/infrastructure/helm-chart/templates/components/ingress.yaml index b4ddcdb3db..5a65593cd7 100644 --- a/infrastructure/helm-chart/templates/components/ingress.yaml +++ b/infrastructure/helm-chart/templates/components/ingress.yaml @@ -82,7 +82,7 @@ spec: pathType: Prefix backend: service: - name: api-admin + name: airy-controller port: number: 80 - path: /kafka