From 0c9b787cc6158473324ebf6b25ddcf629dd5b74e Mon Sep 17 00:00:00 2001 From: DylanGuedes Date: Thu, 4 Apr 2024 19:04:03 -0300 Subject: [PATCH] Add new pattern-ingester component --- .../_helpers-pattern-ingester.tpl | 58 ++++++ .../statefulset-pattern-ingester.yaml | 179 ++++++++++++++++++ production/helm/loki/values.yaml | 109 +++++++++++ 3 files changed, 346 insertions(+) create mode 100644 production/helm/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl create mode 100644 production/helm/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml diff --git a/production/helm/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl b/production/helm/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl new file mode 100644 index 0000000000000..5477214a0b5af --- /dev/null +++ b/production/helm/loki/templates/pattern-ingester/_helpers-pattern-ingester.tpl @@ -0,0 +1,58 @@ +{{/* +pattern ingester fullname +*/}} +{{- define "loki.patternIngesterFullname" -}} +{{ include "loki.fullname" . }}-pattern-ingester +{{- end }} + +{{/* +pattern ingester common labels +*/}} +{{- define "loki.patternIngesterLabels" -}} +{{ include "loki.labels" . }} +app.kubernetes.io/component: pattern-ingester +{{- end }} + +{{/* +pattern ingester selector labels +*/}} +{{- define "loki.patternIngesterSelectorLabels" -}} +{{ include "loki.selectorLabels" . }} +app.kubernetes.io/component: pattern-ingester +{{- end }} + +{{/* +pattern ingester readinessProbe +*/}} +{{- define "loki.patternIngester.readinessProbe" -}} +{{- with .Values.patternIngester.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- else }} +{{- with .Values.loki.readinessProbe }} +readinessProbe: + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end }} +{{- end -}} + +{{/* +pattern ingester priority class name +*/}} +{{- define "loki.patternIngesterPriorityClassName" }} +{{- $pcn := coalesce .Values.global.priorityClassName .Values.patternIngester.priorityClassName -}} +{{- if $pcn }} +priorityClassName: {{ $pcn }} +{{- end }} +{{- end }} + +{{/* +Create the name of the pattern ingester service account +*/}} +{{- define "loki.patternIngesterServiceAccountName" -}} +{{- if .Values.patternIngester.serviceAccount.create -}} + {{ default (print (include "loki.serviceAccountName" .) "-pattern-ingester") .Values.patternIngester.serviceAccount.name }} +{{- else -}} + {{ default (include "loki.serviceAccountName" .) .Values.patternIngester.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/production/helm/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml b/production/helm/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml new file mode 100644 index 0000000000000..5df7455b0761a --- /dev/null +++ b/production/helm/loki/templates/pattern-ingester/statefulset-pattern-ingester.yaml @@ -0,0 +1,179 @@ +{{- $isDistributed := eq (include "loki.deployment.isDistributed" .) "true" -}} +{{- if $isDistributed }} +{{- if (eq .Values.loki.usePatternIngester true) -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "loki.patternIngesterFullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "loki.patternIngesterLabels" . | nindent 4 }} + {{- with .Values.loki.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.patternIngester.replicas }} + podManagementPolicy: Parallel + updateStrategy: + rollingUpdate: + partition: 0 + serviceName: {{ include "loki.patternIngesterFullname" . }}-headless + revisionHistoryLimit: {{ .Values.loki.revisionHistoryLimit }} + {{- if and (semverCompare ">= 1.23-0" .Capabilities.KubeVersion.Version) (.Values.patternIngester.persistence.enableStatefulSetAutoDeletePVC) }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.patternIngester.persistence.whenDeleted }} + whenScaled: {{ .Values.patternIngester.persistence.whenScaled }} + {{- end }} + selector: + matchLabels: + {{- include "loki.patternIngesterSelectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + {{- include "loki.config.checksum" . | nindent 8 }} + {{- with .Values.loki.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "loki.patternIngesterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/part-of: memberlist + {{- with .Values.loki.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "loki.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.hostAliases }} + hostAliases: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "loki.patternIngesterPriorityClassName" . | nindent 6 }} + securityContext: + {{- toYaml .Values.loki.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.patternIngester.terminationGracePeriodSeconds }} + {{- with .Values.patternIngester.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: pattern-ingester + image: {{ include "loki.image" . }} + imagePullPolicy: {{ .Values.loki.image.pullPolicy }} + {{- if or .Values.loki.command .Values.patternIngester.command }} + command: + - {{ coalesce .Values.patternIngester.command .Values.loki.command | quote }} + {{- end }} + args: + - -config.file=/etc/loki/config/config.yaml + - -target=pattern-ingester + {{- with .Values.patternIngester.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 3100 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: http-memberlist + containerPort: 7946 + protocol: TCP + {{- with .Values.patternIngester.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.patternIngester.extraEnvFrom }} + envFrom: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- toYaml .Values.loki.containerSecurityContext | nindent 12 }} + {{- include "loki.patternIngester.readinessProbe" . | nindent 10 }} + volumeMounts: + - name: temp + mountPath: /tmp + - name: config + mountPath: /etc/loki/config + - name: runtime-config + mountPath: /etc/loki/runtime-config + - name: data + mountPath: /var/loki + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /etc/loki/license + {{- end }} + {{- with .Values.patternIngester.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.patternIngester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.patternIngester.extraContainers }} + {{- toYaml .Values.patternIngester.extraContainers | nindent 8}} + {{- end }} + {{- with .Values.patternIngester.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.patternIngester.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: temp + emptyDir: {} + - name: config + {{- include "loki.configVolume" . | nindent 10 }} + - name: runtime-config + configMap: + name: {{ template "loki.name" . }}-runtime + {{- if .Values.enterprise.enabled }} + - name: license + secret: + {{- if .Values.enterprise.useExternalLicense }} + secretName: {{ .Values.enterprise.externalLicenseName }} + {{- else }} + secretName: enterprise-logs-license + {{- end }} + {{- end }} + {{- if not .Values.patternIngester.persistence.enabled }} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.patternIngester.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.patternIngester.persistence.enabled }} + volumeClaimTemplates: + {{- range .Values.patternIngester.persistence.claims }} + - metadata: + name: {{ .name }} + {{- with .annotations }} + annotations: + {{- . | toYaml | nindent 10 }} + {{- end }} + spec: + accessModes: + - ReadWriteOnce + {{- with .storageClass }} + storageClassName: {{ if (eq "-" .) }}""{{ else }}{{ . }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ .size | quote }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 67afce4ab8221..aef488c9155d6 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -412,6 +412,7 @@ loki: tracing: enabled: false useBloomFilters: false + usePatternIngester: false ###################################################################################################################### # @@ -2551,6 +2552,114 @@ bloomCompactor: # -- Set this toggle to false to opt out of automounting API credentials for the service account automountServiceAccountToken: true +# -- Configuration for the pattern ingester +patternIngester: + # -- Number of replicas for the pattern ingester + replicas: 0 + # -- hostAliases to add + hostAliases: [] + # - ip: 1.2.3.4 + # hostnames: + # - domain.tld + image: + # -- The Docker registry for the pattern ingester image. Overrides `loki.image.registry` + registry: null + # -- Docker image repository for the pattern ingester image. Overrides `loki.image.repository` + repository: null + # -- Docker image tag for the pattern ingester image. Overrides `loki.image.tag` + tag: null + # -- Command to execute instead of defined in Docker image + command: null + # -- The name of the PriorityClass for pattern ingester pods + priorityClassName: null + # -- Labels for pattern ingester pods + podLabels: {} + # -- Annotations for pattern ingester pods + podAnnotations: {} + # -- Affinity for pattern ingester pods. + # @default -- Hard node anti-affinity + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/component: pattern-ingester + topologyKey: kubernetes.io/hostname + # -- Labels for pattern ingester service + serviceLabels: {} + # -- Additional CLI args for the pattern ingester + extraArgs: [] + # -- Environment variables to add to the pattern ingester pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the pattern ingester pods + extraEnvFrom: [] + # -- Volume mounts to add to the pattern ingester pods + extraVolumeMounts: [] + # -- Volumes to add to the pattern ingester pods + extraVolumes: [] + # -- readiness probe settings for ingester pods. If empty, use `loki.readinessProbe` + readinessProbe: {} + # -- liveness probe settings for ingester pods. If empty use `loki.livenessProbe` + livenessProbe: {} + # -- Resource requests and limits for the pattern ingester + resources: {} + # -- Containers to add to the pattern ingester pods + extraContainers: [] + # -- Init containers to add to the pattern ingester pods + initContainers: [] + # -- Grace period to allow the pattern ingester to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Node selector for pattern ingester pods + nodeSelector: {} + # -- Tolerations for pattern ingester pods + tolerations: [] + # -- Set the optional grpc service protocol. Ex: "grpc", "http2" or "https" + appProtocol: + grpc: "" + persistence: + # -- Enable creating PVCs for the pattern ingester + enabled: false + # -- Size of persistent disk + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # -- Annotations for pattern ingester PVCs + annotations: {} + # -- List of the pattern ingester PVCs + # @notationType -- list + claims: + - name: data + size: 10Gi + # -- Storage class to be used. + # If defined, storageClassName: . + # If set to "-", storageClassName: "", which disables dynamic provisioning. + # If empty or set to null, no storageClassName spec is + # set, choosing the default provisioner (gp2 on AWS, standard on GKE, AWS, and OpenStack). + storageClass: null + # - name: wal + # size: 150Gi + # -- Enable StatefulSetAutoDeletePVC feature + enableStatefulSetAutoDeletePVC: false + whenDeleted: Retain + whenScaled: Retain + + serviceAccount: + create: false + # -- The name of the ServiceAccount to use for the pattern ingester. + # If not set and create is true, a name is generated by appending + # "-pattern-ingester" to the common ServiceAccount. + name: null + # -- Image pull secrets for the pattern ingester service account + imagePullSecrets: [] + # -- Annotations for the pattern ingester service account + annotations: {} + # -- Set this toggle to false to opt out of automounting API credentials for the service account + automountServiceAccountToken: true + # -- Configuration for the ruler ruler: # -- The ruler component is optional and can be disabled if desired.