Add observability steps and configuration (#1006)

Signed-off-by: David Martin <davmarti@redhat.com>

config/install/README.md

@@ -94,4 +94,46 @@ Validate Kuadrant is in a ready state as before:
 ```
 kubectl get kuadrant kuadrant -n kuadrant-system -o=yaml
 
-```
+```
+
+## Set up observability
+
+Verify that user workload monitoring is enabled in your Openshift cluster.
+If it not enabled, check the [Openshift documentation](https://docs.openshift.com/container-platform/4.17/observability/monitoring/enabling-monitoring-for-user-defined-projects.html) for how to do this.
+
+```bash
+kubectl get configmap cluster-monitoring-config -n openshift-monitoring -o jsonpath='{.data.config\.yaml}'|grep enableUserWorkload
+
+(expected output)
+enableUserWorkload: true
+```
+
+Install the gateway & kuadrant metrics components and configuration, including Grafana.
+
+```bash
+kubectl apply -k config/install/configure/observability
+```
+
+Configure the openshift thanos-query instance as a data source in Grafana.
+
+```bash
+TOKEN="Bearer $(oc whoami -t)"
+HOST="$(kubectl -n openshift-monitoring get route thanos-querier -o jsonpath='https://{.status.ingress[].host}')"
+echo "TOKEN=$TOKEN" > config/observability/openshift/grafana/datasource.env
+echo "HOST=$HOST" >> config/observability/openshift/grafana/datasource.env
+kubectl apply -k config/observability/openshift/grafana
+```
+
+Create the example dashboards in Grafana
+
+```bash
+kubectl apply -k examples/dashboards
+```
+
+Access the Grafana UI, using the default user/pass of root/secret.
+You should see the example dashboards in the 'monitoring' folder.
+For more information on the example dashboards, check out the [documentation](https://docs.kuadrant.io/latest/kuadrant-operator/doc/observability/examples/).
+
+```bash
+kubectl -n monitoring get routes grafana-route -o jsonpath="https://{.status.ingress[].host}"
+```

config/install/configure/observability/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - github.com/Kuadrant/gateway-api-state-metrics/config/kuadrant?ref=0.5.0
+  - ../../../observability/openshift/
+  - ../../../observability/prometheus/monitors/

config/observability/openshift/grafana/.gitignore

@@ -0,0 +1 @@
+datasource.env

config/observability/openshift/grafana/dashboards.yaml

@@ -0,0 +1,59 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-platform-engineer
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  configMapRef:
+    name: grafana-platform-engineer
+    key: platform_engineer.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-business-user
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  configMapRef:
+    name: grafana-business-user
+    key: business_user.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-controller-resources-metrics
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  configMapRef:
+    name: grafana-controller-resources-metrics
+    key: controller-resources-metrics.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-controller-runtime-metrics
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  configMapRef:
+    name: grafana-controller-runtime-metrics
+    key: controller-runtime-metrics.json
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDashboard
+metadata:
+  name: grafana-app-developer
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: "grafana"
+  configMapRef:
+    name: grafana-app-developer
+    key: app_developer.json

config/observability/openshift/grafana/datasource.yaml

@@ -0,0 +1,20 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaDatasource
+metadata:
+  name: thanos-query-ds
+spec:
+  datasource:
+    access: proxy
+    isDefault: true
+    jsonData:
+      httpHeaderName1: 'Authorization'
+      timeInterval: 5s
+      tlsSkipVerify: true
+    secureJsonData:
+      httpHeaderValue1: 'REPLACED_BY_ENV'
+    name: thanos-query-ds
+    type: prometheus
+    url: 'REPLACED_BY_ENV'
+  instanceSelector:
+    matchLabels:
+      dashboards: grafana

config/observability/openshift/grafana/grafana.yaml

@@ -0,0 +1,19 @@
+apiVersion: grafana.integreatly.org/v1beta1
+kind: Grafana
+metadata:
+  labels:
+    dashboards: grafana
+  name: grafana
+spec:
+  config:
+    auth:
+      disable_login_form: 'false'
+    log:
+      mode: console
+    security:
+      admin_password: secret
+      admin_user: root
+  route:
+    metadata: {}
+    spec: {}
+  version: 10.4.3

config/observability/openshift/grafana/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: monitoring
+
+resources:
+  - grafana.yaml
+  - datasource.yaml
+  - dashboards.yaml
+
+# Generate a ConfigMap from the .env file
+configMapGenerator:
+  - name: datasource-env-config
+    envs:
+      - datasource.env
+
+replacements:
+  - source:
+      kind: ConfigMap
+      name: datasource-env-config
+      fieldPath: data.TOKEN
+    targets:
+      - select:
+          kind: GrafanaDatasource
+          name: thanos-query-ds
+        fieldPaths:
+          - spec.datasource.secureJsonData.httpHeaderValue1
+  - source:
+      kind: ConfigMap
+      name: datasource-env-config
+      fieldPath: data.HOST
+    targets:
+      - select:
+          kind: GrafanaDatasource
+          name: thanos-query-ds
+        fieldPaths:
+          - spec.datasource.url
+

config/observability/openshift/grafana/subscription.yaml

@@ -0,0 +1,13 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  labels:
+    operators.coreos.com/grafana-operator.openshift-operators: ""
+  name: grafana-operator
+  namespace: openshift-operators
+spec:
+  channel: v5
+  installPlanApproval: Automatic
+  name: grafana-operator
+  source: community-operators
+  sourceNamespace: openshift-marketplace

config/observability/openshift/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - telemetry.yaml
+  - kube-state-metrics.yaml
+  - grafana/subscription.yaml

config/observability/prometheus/monitors/istio/service-monitor-istiod.yaml

@@ -2,11 +2,11 @@ apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
   name: istiod
-  namespace: istio-system
+  namespace: gateway-system
 spec:
   namespaceSelector:
     matchNames:
-    - istio-system
+    - gateway-system
   selector:
     matchLabels:
       app: istiod

config/observability/prometheus/monitors/istio/telemetry.yaml

@@ -2,7 +2,7 @@ apiVersion: telemetry.istio.io/v1alpha1
 kind: Telemetry
 metadata:
   name: namespace-metrics
-  namespace: istio-system
+  namespace: gateway-system
 spec:
   metrics:
   - providers:

config/observability/prometheus/monitors/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - istio/service-monitor-istiod.yaml
+  - operators.yaml
+

examples/dashboards/kustomization.yaml

@@ -1,26 +1,23 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
+namespace: monitoring
+
 # grafana-dashboard dashboards as configmaps
 configMapGenerator:
 - name: grafana-app-developer
-  namespace: monitoring
   files:
   - ./app_developer.json
 - name: grafana-business-user
-  namespace: monitoring
   files:
   - ./business_user.json
 - name: grafana-platform-engineer
-  namespace: monitoring
   files:
   - ./platform_engineer.json
-- name: grafana-controller-runtime
-  namespace: monitoring
+- name: grafana-controller-runtime-metrics
   files:
   - ./controller-runtime-metrics.json
-- name: grafana-controller-resources
-  namespace: monitoring
+- name: grafana-controller-resources-metrics
   files:
   - ./controller-resources-metrics.json