From 9872717a7f4ec7b234612f2ede4f42039a2d9e9c Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 2 Oct 2024 02:35:39 -0700 Subject: [PATCH 1/2] feat: Add loki and promtail to observability --- docker-compose.yaml | 17 + grafana/dashboards/Throughput-testing.json | 2024 +++++++++++++------- loki/config.yaml | 39 + promtail/config.yaml | 21 + test-node.bash | 6 +- 5 files changed, 1452 insertions(+), 655 deletions(-) create mode 100644 loki/config.yaml create mode 100644 promtail/config.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml index 5f4dc52..0d755f5 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -412,6 +412,23 @@ services: depends_on: - prometheus + loki: + image: grafana/loki:3.1.1 + restart: unless-stopped + volumes: + - ./loki:/etc/loki + ports: + - 3200:3200 + command: -config.file=/etc/loki/config.yaml + + promtail: + image: grafana/promtail:3.1.1 + restart: unless-stopped + volumes: + - ./promtail:/etc/promtail + - /var/run/docker.sock:/var/run/docker.sock # Mount Docker socket to read container logs + command: -config.file=/etc/promtail/config.yaml + volumes: l1data: consensus: diff --git a/grafana/dashboards/Throughput-testing.json b/grafana/dashboards/Throughput-testing.json index 02830a8..11837e8 100644 --- a/grafana/dashboards/Throughput-testing.json +++ b/grafana/dashboards/Throughput-testing.json @@ -56,7 +56,7 @@ "h": 9, "w": 9, "x": 0, - "y": 1 + "y": 9 }, "id": 25, "options": { @@ -141,7 +141,7 @@ "h": 9, "w": 9, "x": 9, - "y": 1 + "y": 9 }, "id": 27, "options": { @@ -226,7 +226,7 @@ "h": 9, "w": 9, "x": 0, - "y": 10 + "y": 18 }, "id": 30, "options": { @@ -318,7 +318,7 @@ "h": 4, "w": 6, "x": 9, - "y": 10 + "y": 18 }, "id": 33, "options": { @@ -384,7 +384,7 @@ "h": 9, "w": 9, "x": 15, - "y": 10 + "y": 18 }, "id": 31, "options": { @@ -476,7 +476,7 @@ "h": 5, "w": 6, "x": 9, - "y": 14 + "y": 22 }, "id": 28, "options": { @@ -542,7 +542,7 @@ "h": 9, "w": 9, "x": 0, - "y": 19 + "y": 27 }, "id": 29, "options": { @@ -634,7 +634,7 @@ "h": 5, "w": 6, "x": 9, - "y": 19 + "y": 27 }, "id": 34, "options": { @@ -700,7 +700,7 @@ "h": 9, "w": 9, "x": 15, - "y": 19 + "y": 27 }, "id": 32, "options": { @@ -792,7 +792,7 @@ "h": 4, "w": 6, "x": 9, - "y": 24 + "y": 32 }, "id": 35, "options": { @@ -844,8 +844,46 @@ "x": 0, "y": 1 }, - "id": 21, + "id": 9, "panels": [ + { + "datasource": { + "default": false, + "type": "loki", + "uid": "cdzn4gd8wly4gf" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 39, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cdzn4gd8wly4gf" + }, + "editorMode": "builder", + "expr": "{service_name=\"nitro-testnode-validator-1\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs", + "type": "logs" + }, { "datasource": { "default": true, @@ -855,40 +893,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -911,21 +916,25 @@ "h": 8, "w": 12, "x": 0, - "y": 2 + "y": 10 }, - "id": 20, + "id": 7, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -934,18 +943,71 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_batchposter_estimated_batch_backlog{instance=\"poster:6070\"}", + "exemplar": false, + "expr": "arb_validator_msg_count_current_batch{instance=\"validator:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "msg count cur batch", "range": true, - "refId": "A", + "refId": "Validator Message Count", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_validator_validations_pending{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "validations pending", + "range": true, + "refId": "Validations Pending", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_validator_msg_count_validated{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "msg count validated", + "range": true, + "refId": "Validator Msg Count Validated", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "arb_validator_validations_valid{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "validations valid", + "range": true, + "refId": "Valid Validations", "useBackend": false } ], - "title": "Estimated Batch Backlog", - "type": "timeseries" + "title": "Validations", + "type": "gauge" }, { "datasource": { @@ -956,112 +1018,120 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "mappings": [ + { + "options": { + "{__name__\"": { + "index": 0 + } + }, + "type": "value" } - }, - "mappings": [], + ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "__name__*" + }, + "properties": [ + { + "id": "displayName" + } + ] + } + ] }, "gridPos": { "h": 8, - "w": 4, + "w": 3, "x": 12, - "y": 2 + "y": 10 }, - "id": 22, + "id": 17, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, + "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "ddshms3dlineoe" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_batchposter_wallet_eth{instance=\"poster:6070\"}", + "expr": "arb_sequencenumber_confirmed{instance=\"validator:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "Confirmed", "range": true, "refId": "A", "useBackend": false - } - ], - "title": "Wallet ETH", - "type": "timeseries" - } - ], - "title": "Batch Poster", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 2 - }, - "id": 16, - "panels": [ + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_sequencenumber_inblock{instance=\"sequencer:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Inblock", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_sequencenumber_latest{instance=\"sequencer:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Latest", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "nominal SeqNum", + "type": "gauge" + }, { "datasource": { "default": true, @@ -1125,11 +1195,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 3 + "w": 9, + "x": 15, + "y": 10 }, - "id": 15, + "id": 19, "options": { "legend": { "calcs": [], @@ -1150,7 +1220,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "eigenda_proxy_http_server_requests_total{job=\"eigenda_proxy\"}", + "expr": "arb_dataposter_queue_length{instance=\"validator:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1160,23 +1230,9 @@ "useBackend": false } ], - "title": "Proxy Requests Total", + "title": "Data Poster Queue", "type": "timeseries" - } - ], - "title": "EigenDA Proxy", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 9, - "panels": [ + }, { "datasource": { "default": true, @@ -1195,10 +1251,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1206,12 +1258,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 9, + "w": 6, "x": 0, - "y": 4 + "y": 18 }, - "id": 7, + "id": 23, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -1236,14 +1288,13 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "arb_validator_msg_count_current_batch{instance=\"validator:6070\"}", + "expr": "arb_staker_action_success{instance=\"validator:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "msg count cur batch", + "legendFormat": "{{__name__}}", "range": true, - "refId": "Validator Message Count", + "refId": "A", "useBackend": false }, { @@ -1253,14 +1304,14 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_validator_validations_pending{instance=\"validator:6070\"}", + "expr": "arb_staker_action_failure{instance=\"validator:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "validations pending", + "legendFormat": "{{__name__}}", "range": true, - "refId": "Validations Pending", + "refId": "B", "useBackend": false }, { @@ -1270,14 +1321,14 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_validator_msg_count_validated{instance=\"validator:6070\"}", + "expr": "arb_staker_confirmed_node{instance=\"validator:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "msg count validated", + "legendFormat": "{{__name__}}", "range": true, - "refId": "Validator Msg Count Validated", + "refId": "C", "useBackend": false }, { @@ -1287,142 +1338,18 @@ }, "disableTextWrap": false, "editorMode": "builder", - "exemplar": false, - "expr": "arb_validator_validations_valid{instance=\"validator:6070\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "validations valid", - "range": true, - "refId": "Valid Validations", - "useBackend": false - } - ], - "title": "Validations", - "type": "gauge" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "ddshms3dlineoe" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "{__name__\"": { - "index": 0 - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "__name__*" - }, - "properties": [ - { - "id": "displayName" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 12, - "y": 4 - }, - "id": 17, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "arb_sequencenumber_confirmed{instance=\"validator:6070\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Confirmed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "arb_sequencenumber_inblock{instance=\"sequencer:6070\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Inblock", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "arb_sequencenumber_latest{instance=\"sequencer:6070\"}", + "expr": "arb_staker_staked_node{instance=\"validator:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "Latest", + "legendFormat": "{{__name__}}", "range": true, - "refId": "C", + "refId": "D", "useBackend": false } ], - "title": "nominal SeqNum", + "title": "Assertion Chain Staking", "type": "gauge" }, { @@ -1431,6 +1358,7 @@ "type": "prometheus", "uid": "ddshms3dlineoe" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -1487,12 +1415,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 9, - "x": 15, - "y": 4 + "h": 9, + "w": 5, + "x": 6, + "y": 18 }, - "id": 19, + "id": 24, "options": { "legend": { "calcs": [], @@ -1513,7 +1441,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_dataposter_queue_length{instance=\"validator:6070\"}", + "expr": "arb_validator_recordingdb_size{instance=\"validator:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1523,7 +1451,7 @@ "useBackend": false } ], - "title": "Data Poster Queue", + "title": "Recording DB Size", "type": "timeseries" }, { @@ -1535,15 +1463,423 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 11, + "y": 18 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_batchposter_gasrefunder_eth{instance=\"poster:6070\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "ETH", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Gas Refunder ETH", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 5, + "x": 15, + "y": 18 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_dataposter_queue_length{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Length", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_dataposter_queue_weight{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Weight", + "range": true, + "refId": "B", + "useBackend": false + } + ], + "title": "Data Poster Queue", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 20, + "y": 18 + }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_dataposter_nonce_finalized{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Nonce Finalized", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_dataposter_nonce_softconfirmed{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Nonce Soft Confirmed", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_dataposter_nonce_unconfirmed{instance=\"validator:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Nonce Unconfirmed", + "range": true, + "refId": "C", + "useBackend": false + } + ], + "title": "Data Poster", + "type": "stat" + } + ], + "title": "Validator", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 21, + "panels": [ + { + "datasource": { + "default": false, + "type": "loki", + "uid": "cdzn4gd8wly4gf" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 37, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "cdzn4gd8wly4gf" + }, + "editorMode": "builder", + "expr": "{container=\"nitro-testnode-poster-1\"} |= ``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Logs", + "type": "logs" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } @@ -1551,12 +1887,177 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 6, + "h": 8, + "w": 7, "x": 0, - "y": 12 + "y": 7 }, - "id": 23, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_batchposter_estimated_batch_backlog{instance=\"poster:6070\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Estimated Batch Backlog", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 7, + "y": 7 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_batchposter_wallet_eth{instance=\"poster:6070\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Wallet ETH", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 7 + }, + "id": 40, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -1581,11 +2082,11 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_staker_action_success{instance=\"validator:6070\"}", + "expr": "arb_batchPoster_action_da_success{instance=\"poster:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{__name__}}", + "legendFormat": "DA Success", "range": true, "refId": "A", "useBackend": false @@ -1597,15 +2098,84 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_staker_action_failure{instance=\"validator:6070\"}", + "expr": "arb_batchPoster_action_da_failure{instance=\"poster:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{__name__}}", + "legendFormat": "DA Failures", "range": true, "refId": "B", "useBackend": false + } + ], + "title": "Data Availability Provider", + "type": "gauge" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 7 + }, + "id": 43, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_dataposter_nonce_finalized{instance=\"poster:6070\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Nonce Finalized", + "range": true, + "refId": "A", + "useBackend": false }, { "datasource": { @@ -1614,14 +2184,14 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_staker_confirmed_node{instance=\"validator:6070\"}", + "expr": "arb_dataposter_nonce_softconfirmed{instance=\"poster:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{__name__}}", + "legendFormat": "Nonce Soft Confirmed", "range": true, - "refId": "C", + "refId": "B", "useBackend": false }, { @@ -1631,18 +2201,18 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_staker_staked_node{instance=\"validator:6070\"}", + "expr": "arb_dataposter_nonce_unconfirmed{instance=\"poster:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "{{__name__}}", + "legendFormat": "Nonce Unconfirmed", "range": true, - "refId": "D", + "refId": "C", "useBackend": false } ], - "title": "Assertion Chain Staking", + "title": "Data Poster", "type": "gauge" }, { @@ -1651,7 +2221,6 @@ "type": "prometheus", "uid": "ddshms3dlineoe" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -1697,10 +2266,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1708,12 +2273,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 9, - "x": 6, - "y": 12 + "h": 8, + "w": 7, + "x": 0, + "y": 15 }, - "id": 24, + "id": 42, "options": { "legend": { "calcs": [], @@ -1726,6 +2291,7 @@ "sort": "none" } }, + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -1734,26 +2300,357 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_validator_recordingdb_size{instance=\"validator:6070\"}", + "expr": "arb_batchposter_l1gasprice{instance=\"poster:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "GWEI", "range": true, "refId": "A", "useBackend": false } ], - "title": "Recording DB Size", + "title": "L1 Gas Price", "type": "timeseries" + } + ], + "title": "Batch Poster", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 13, + "panels": [], + "title": "Chain", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 4 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_block_basefee", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "L2 Base Fee", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 9, + "y": 4 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_block_gasused", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Block Gas Used", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 16, + "y": 4 + }, + "id": 46, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_block_gasused", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Block Gas Used", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 16, + "panels": [ { "datasource": { "default": true, "type": "prometheus", "uid": "ddshms3dlineoe" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -1810,12 +2707,12 @@ "overrides": [] }, "gridPos": { - "h": 9, - "w": 9, - "x": 15, - "y": 12 + "h": 8, + "w": 12, + "x": 0, + "y": 11 }, - "id": 36, + "id": 15, "options": { "legend": { "calcs": [], @@ -1836,7 +2733,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_validator_recordingdb_size{instance=\"validator:6070\"}", + "expr": "eigenda_proxy_http_server_requests_total{job=\"eigenda_proxy\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1846,11 +2743,11 @@ "useBackend": false } ], - "title": "Recording DB Size", + "title": "Proxy Requests Total", "type": "timeseries" } ], - "title": "Validator", + "title": "EigenDA Proxy", "type": "row" }, { @@ -1859,110 +2756,47 @@ "h": 1, "w": 24, "x": 0, - "y": 4 + "y": 13 }, - "id": 13, + "id": 8, "panels": [ { "datasource": { - "default": true, - "type": "prometheus", - "uid": "ddshms3dlineoe" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] + "default": false, + "type": "loki", + "uid": "cdzn4gd8wly4gf" }, "gridPos": { "h": 8, - "w": 9, + "w": 24, "x": 0, - "y": 23 + "y": 6 }, - "id": 12, + "id": 38, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false }, "targets": [ { "datasource": { - "type": "prometheus", - "uid": "ddshms3dlineoe" + "type": "loki", + "uid": "cdzn4gd8wly4gf" }, - "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_block_basefee", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false + "expr": "{container=\"nitro-testnode-sequencer-1\"} |= ``", + "queryType": "range", + "refId": "A" } ], - "title": "L2 Base Fee", - "type": "timeseries" + "title": "Logs", + "type": "logs" }, { "datasource": { @@ -1972,41 +2806,8 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "color": { + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -2015,10 +2816,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -2026,23 +2823,26 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 7, - "x": 9, - "y": 23 + "h": 5, + "w": 4, + "x": 0, + "y": 14 }, - "id": 14, + "id": 11, "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "right", - "showLegend": false + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" }, "pluginVersion": "11.2.0", "targets": [ @@ -2053,33 +2853,36 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_block_gasused", + "expr": "arb_inbox_latest_batch{instance=\"sequencer:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "__auto", + "legendFormat": "latest batch", "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddshms3dlineoe" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_inbox_latest_batch_message{instance=\"sequencer:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "latest batch msg", + "range": true, + "refId": "B", + "useBackend": false } ], - "title": "Block Gas Used", - "type": "timeseries" - } - ], - "title": "Fees", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 8, - "panels": [ + "title": "Inbox", + "type": "gauge" + }, { "datasource": { "default": true, @@ -2091,7 +2894,16 @@ "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "options": { + "{__name__\"": { + "index": 0 + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -2102,15 +2914,27 @@ ] } }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "__name__*" + }, + "properties": [ + { + "id": "displayName" + } + ] + } + ] }, "gridPos": { "h": 5, - "w": 12, - "x": 0, - "y": 21 + "w": 8, + "x": 4, + "y": 14 }, - "id": 11, + "id": 2, "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -2131,15 +2955,15 @@ { "datasource": { "type": "prometheus", - "uid": "ddshms3dlineoe" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_inbox_latest_batch{instance=\"sequencer:6070\"}", + "expr": "arb_sequencenumber_confirmed{instance=\"sequencer:6070\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "latest batch", + "legendFormat": "Confirmed", "range": true, "refId": "A", "useBackend": false @@ -2147,22 +2971,39 @@ { "datasource": { "type": "prometheus", - "uid": "ddshms3dlineoe" + "uid": "${DS_PROMETHEUS}" }, "disableTextWrap": false, "editorMode": "builder", - "expr": "arb_inbox_latest_batch_message{instance=\"sequencer:6070\"}", + "expr": "arb_sequencenumber_inblock{instance=\"sequencer:6070\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, - "legendFormat": "latest batch msg", + "legendFormat": "Inblock", "range": true, "refId": "B", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "arb_sequencenumber_latest{instance=\"sequencer:6070\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Latest", + "range": true, + "refId": "C", + "useBackend": false } ], - "title": "Inbox", + "title": "nominal SeqNum", "type": "gauge" }, { @@ -2230,7 +3071,7 @@ "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 14 }, "id": 10, "options": { @@ -2331,7 +3172,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 19 }, "id": 6, "options": { @@ -2482,7 +3323,7 @@ "h": 8, "w": 12, "x": 12, - "y": 29 + "y": 22 }, "id": 5, "options": { @@ -2590,9 +3431,9 @@ }, "gridPos": { "h": 8, - "w": 9, + "w": 12, "x": 0, - "y": 34 + "y": 27 }, "id": 1, "options": { @@ -2662,129 +3503,6 @@ "title": "SeqNum rate", "type": "timeseries" }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "ddshms3dlineoe" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "{__name__\"": { - "index": 0 - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": "__name__*" - }, - "properties": [ - { - "id": "displayName" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 9, - "y": 34 - }, - "id": 2, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "11.2.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "arb_sequencenumber_confirmed{instance=\"sequencer:6070\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Confirmed", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "arb_sequencenumber_inblock{instance=\"sequencer:6070\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Inblock", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "arb_sequencenumber_latest{instance=\"sequencer:6070\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "Latest", - "range": true, - "refId": "C", - "useBackend": false - } - ], - "title": "nominal SeqNum", - "type": "gauge" - }, { "datasource": { "default": true, @@ -2850,7 +3568,7 @@ "h": 8, "w": 12, "x": 12, - "y": 37 + "y": 30 }, "id": 18, "options": { @@ -2951,7 +3669,7 @@ "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 35 }, "id": 4, "options": { @@ -3016,7 +3734,7 @@ "list": [] }, "time": { - "from": "now-15m", + "from": "now-30m", "to": "now" }, "timepicker": {}, diff --git a/loki/config.yaml b/loki/config.yaml new file mode 100644 index 0000000..493a908 --- /dev/null +++ b/loki/config.yaml @@ -0,0 +1,39 @@ +auth_enabled: false + +server: + http_listen_port: 3200 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration +# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/ +# +# Statistics help us better understand how Loki is used, and they show us performance +# levels for most users. This helps us prioritize features and documentation. +# For more information on what's sent, look at +# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go +# Refer to the buildReport method to see what goes into a report. +# +# If you would like to disable reporting, uncomment the following lines: +#analytics: +# reporting_enabled: false \ No newline at end of file diff --git a/promtail/config.yaml b/promtail/config.yaml new file mode 100644 index 0000000..7f2a6bf --- /dev/null +++ b/promtail/config.yaml @@ -0,0 +1,21 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3200/loki/api/v1/push + +scrape_configs: + - job_name: docker + docker_sd_configs: + - host: unix:///var/run/docker.sock + filters: + - name: name + values: ["sequencer", "poster", "validation*", "validator", "eigenda-proxy"] + relabel_configs: + - source_labels: ["__meta_docker_container_name"] + regex: "/(.*)" + target_label: "container" \ No newline at end of file diff --git a/test-node.bash b/test-node.bash index 8574401..5142530 100755 --- a/test-node.bash +++ b/test-node.bash @@ -176,6 +176,8 @@ while [[ $# -gt 0 ]]; do --monitor) prometheus=true grafana=true + loki=true + promtail=true shift ;; *) @@ -200,7 +202,7 @@ while [[ $# -gt 0 ]]; do echo --no-tokenbridge don\'t build or launch tokenbridge echo --no-run does not launch nodes \(useful with build or init\) echo --no-simple run a full configuration with separate sequencer/batch-poster/validator/relayer - echo --monitor start Prometheus and Grafana server + echo --monitor start Prometheus, Loki, Promtail and Grafana server echo echo script runs inside a separate docker. For SCRIPT-ARGS, run $0 script --help exit 0 @@ -264,7 +266,7 @@ if $blockscout; then NODES="$NODES blockscout" fi if $monitor; then - NODES="$NODES prometheus grafana" + NODES="$NODES prometheus grafana loki promtail" fi if $force_build; then echo == Building.. From 7c4adb1b1674cc3cd608cc1eb0de87ca555c68dc Mon Sep 17 00:00:00 2001 From: Ethen Pociask Date: Wed, 2 Oct 2024 02:39:42 -0700 Subject: [PATCH 2/2] feat: Add loki and promtail to observability - bump gh runner --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a78e451..2ea3c2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ on: jobs: build_and_run: - runs-on: ubuntu-8 + runs-on: ubuntu-latest steps: - name: Checkout