Merge pull request #229 from go-faster/feat/metrics-chstorage

feat(chstorage): implement metrics storage

.github/workflows/e2e.yml

@@ -12,6 +12,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+        with:
+          submodules: true
 
       - name: Install Go
         uses: actions/setup-go@v4

.github/workflows/x.yml

@@ -13,7 +13,10 @@ jobs:
     uses: go-faster/x/.github/workflows/test.yml@main
     with:
       enable-386: false
+      submodules: true
   cover:
     uses: go-faster/x/.github/workflows/cover.yml@main
+    with:
+      submodules: true
   lint:
     uses: go-faster/x/.github/workflows/lint.yml@main

.gitignore

@@ -20,6 +20,7 @@ secret.yaml
 /oteldb
 /otelproxy
 /otelfaker
+/chotel
 
 # goreleaser
 dist

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "internal/otelschema/_testdata/semantic-conventions"]
+	path = internal/otelschema/_testdata/semantic-conventions
+	url = https://github.com/open-telemetry/semantic-conventions.git

.golangci.yml

@@ -135,8 +135,8 @@ issues:
       text: "dupSubExpr"
 
     # False positive.
-    - path: internal/metricsharding/prometheus\.go
-      linters: [govet]
+    - linters: [govet]
+      source: func.+Seek\(.+int64\).+chunkenc\.ValueType
       text: "method Seek"
 
     # Not gonna use const, does not make any sense.

README.md

@@ -8,30 +8,25 @@ The OpenTelemetry-compatible telemetry aggregation, storage and processing.
 
 Work in progress.
 
+> [!WARNING]  
+> Work in progress.
+
 ## Storage
 
 The oteldb is stateless and uses external storage systems for data persistence, processing and aggregation.
 
-### YTsaurus
-
-The [YTsaurus](https://ytsaurus.tech/) is a primary storage for telemetry data.
-An open source big data platform for distributed storage and processing.
-
-- Hierarchical multi-tenancy with secure resource isolation
-- OLAP and OLTP
-- MapReduce, ACID
-- ClickHouse protocol compatible
-- Exabyte scale, up to 1M CPU, 10K+ nodes
-
-### ClickHouse
+We focus on the following storage systems:
+- [ClickHouse](https://clickhouse.com/)
+- [YTsaurus](https://ytsaurus.tech/)
 
-The oteldb also supports [ClickHouse](https://clickhouse.com/) storage.
+Currently, ClickHouse looks more promising.
 
 ## Query
 
 Supported query languages:
 - LogQL (loki) for logs
 - TraceQL (Tempo) for traces
+- PromQL (Prometheus) for metrics
 
 ## Local development
 
@@ -46,5 +41,5 @@ docker compose -f dev/local/ytsaurus/docker-compose.yml up -d
 #### Clickhouse storage
 
 ```shell
-docker compose -f dev/local/clickhouse/docker-compose.yml up -d
+docker compose -f dev/local/ch/docker-compose.yml up -d
 ```

_oas/loki.yml

@@ -13,7 +13,6 @@ paths:
       operationId: query
       description: Query
       parameters:
-        - $ref: "#/components/parameters/GrafanaUser"
         - name: query
           description: The LogQL query to perform.
           in: query
@@ -59,7 +58,6 @@ paths:
       operationId: queryRange
       description: Query range
       parameters:
-        - $ref: "#/components/parameters/GrafanaUser"
         - name: start
           in: query
           schema:
@@ -126,7 +124,6 @@ paths:
         Get labels.
         Used by Grafana to test connection to Loki.
       parameters:
-        - $ref: "#/components/parameters/GrafanaUser"
         - name: start
           description: |
             The start time for the query as a nanosecond Unix epoch.
@@ -163,7 +160,6 @@ paths:
       operationId: labelValues
       description: Get values of label
       parameters:
-        - $ref: "#/components/parameters/GrafanaUser"
         - name: start
           description: |
             The start time for the query as a nanosecond Unix epoch.
@@ -262,16 +258,35 @@ paths:
           description: Successful push
         default:
           $ref: "#/components/responses/Error"
+  /loki/api/v1/index/stats:
+    get:
+      operationId: indexStats
+      description: Get index stats
+      parameters:
+        - name: start
+          in: query
+          schema:
+            $ref: "#/components/schemas/LokiTime"
+        - name: end
+          in: query
+          schema:
+            $ref: "#/components/schemas/LokiTime"
+        - name: query
+          description: The LogQL matchers to check.
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: Index stats
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/IndexStats"
+        default:
+          $ref: "#/components/responses/Error"
 components:
-  parameters:
-    # TODO(ernado): extract to separate file? Same for all datasources
-    GrafanaUser:
-      name: X-Grafana-User
-      in: header
-      required: false
-      description: "Grafana username that is passed to datasource when making requests from Grafana. Used for authentication and authorization."
-      schema:
-        type: string
   responses:
     Error:
       description: Error while processing request
@@ -402,6 +417,22 @@ components:
         - type: string
           description: value
           x-ogen-name: V
+    IndexStats:
+      type: object
+      required:
+        - streams
+        - chunks
+        - entries
+        - bytes
+      properties:
+        streams:
+          type: integer
+        chunks:
+          type: integer
+        entries:
+          type: integer
+        bytes:
+          type: integer
     Stats:
       type: object
     Direction:

chotel.Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:latest as builder
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . ./
+RUN CGO_ENABLED=0 GOOS=linux go build -o /app/chotel ./cmd/chotel
+
+FROM alpine:latest
+RUN apk --no-cache add ca-certificates
+
+WORKDIR /app
+COPY --from=builder /app/chotel /chotel
+
+ENTRYPOINT ["/chotel"]