Upgrade Promethus lib to 1.x and adapt the code to the new version

Signed-off-by: Fabio Di Fabio <fabio.difabio@consensys.net> # Conflicts: # metrics/core/build.gradle # metrics/core/src/main/java/org/hyperledger/besu/metrics/noop/NoOpMetricsSystem.java # metrics/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java # metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusMetricsSystem.java # metrics/rocksdb/src/main/java/org/hyperledger/besu/metrics/rocksdb/RocksDBStats.java # plugin-api/build.gradle # plugin-api/src/main/java/org/hyperledger/besu/plugin/services/MetricsSystem.java
hyperledger · Nov 19, 2024 · 3f13acf · 3f13acf
1 parent 0c6ff35
commit 3f13acf
Show file tree

Hide file tree

Showing 18 changed files with 227 additions and 172 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,23 @@
 
 ### Breaking Changes
 - Removed Retesteth rpc service and commands [#7833](https://github.com/hyperledger/besu/pull/7783)
+- With the upgrade of the Prometheus Java Metrics library, gauge names are not allowed to end with `total`, therefore the following list of metrics is losing the `_total` suffix: 
+```
+1. besu_blockchain_difficulty_total
+2. besu_executors_ethscheduler_blockcreation_completed_tasks_total
+3. besu_executors_ethscheduler_blockcreation_submitted_tasks_total
+4. besu_executors_ethscheduler_computation_completed_tasks_total
+5. besu_executors_ethscheduler_computation_submitted_tasks_total
+6. besu_executors_ethscheduler_services_completed_tasks_total
+7. besu_executors_ethscheduler_services_submitted_tasks_total
+8. besu_executors_ethscheduler_timer_completed_tasks_total
+9. besu_executors_ethscheduler_timer_submitted_tasks_total
+10. besu_executors_ethscheduler_transactions_completed_tasks_total
+11. besu_executors_ethscheduler_transactions_submitted_tasks_total
+12. besu_executors_ethscheduler_workers_completed_tasks_total
+13. besu_executors_ethscheduler_workers_submitted_tasks_total
+```
+[Grafana Besu Full dashboard](https://grafana.com/grafana/dashboards/16455-besu-full/) will support both names for a smooth transition, but if you use these metrics outside of this dashboard, you have to update your custom solution, to support the new names. 
 
 ### Upcoming Breaking Changes
 
@@ -15,6 +32,7 @@
 - Update Java dependencies [#7786](https://github.com/hyperledger/besu/pull/7786)
 - Add a method to get all the transaction in the pool, to the `TransactionPoolService`, to easily access the transaction pool content from plugins [#7813](https://github.com/hyperledger/besu/pull/7813)
 - Add a method to check if a metric category is enabled to the plugin API [#7832](https://github.com/hyperledger/besu/pull/7832)
+- Prometheus Java Metrics library upgraded to version 1.3.3 [#7880](https://github.com/hyperledger/besu/pull/7880)
 
 ### Bug fixes
 - Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825)

diff --git a/...rc/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugMetrics.java b/...rc/main/java/org/hyperledger/besu/ethereum/api/jsonrpc/internal/methods/DebugMetrics.java
@@ -50,22 +50,22 @@ public JsonRpcResponse response(final JsonRpcRequestContext requestContext) {
   private void addObservation(
       final Map<String, Object> observations, final Observation observation) {
     final Map<String, Object> categoryObservations =
-        getNextMapLevel(observations, observation.getCategory().getName());
-    if (observation.getLabels().isEmpty()) {
-      categoryObservations.put(observation.getMetricName(), observation.getValue());
+        getNextMapLevel(observations, observation.category().getName());
+    if (observation.labels().isEmpty()) {
+      categoryObservations.put(observation.metricName(), observation.value());
     } else {
       addLabelledObservation(categoryObservations, observation);
     }
   }
 
   private void addLabelledObservation(
       final Map<String, Object> categoryObservations, final Observation observation) {
-    final List<String> labels = observation.getLabels();
-    Map<String, Object> values = getNextMapLevel(categoryObservations, observation.getMetricName());
+    final List<String> labels = observation.labels();
+    Map<String, Object> values = getNextMapLevel(categoryObservations, observation.metricName());
     for (int i = 0; i < labels.size() - 1; i++) {
       values = getNextMapLevel(values, labels.get(i));
     }
-    values.put(labels.get(labels.size() - 1), observation.getValue());
+    values.put(labels.get(labels.size() - 1), observation.value());
   }
 
   @SuppressWarnings("unchecked")

diff --git a/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java b/ethereum/core/src/main/java/org/hyperledger/besu/ethereum/chain/DefaultBlockchain.java
@@ -183,7 +183,7 @@ private void createGauges(final MetricsSystem metricsSystem) {
 
     metricsSystem.createGauge(
         BLOCKCHAIN,
-        "difficulty_total",
+        "difficulty",
         "Total difficulty of the chainhead",
         () -> this.getChainHead().getTotalDifficulty().toBigInteger().doubleValue());
 

diff --git a/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/MonitoredExecutors.java b/ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/manager/MonitoredExecutors.java
@@ -155,13 +155,13 @@ private static <T extends ThreadPoolExecutor> T newMonitoredExecutor(
 
     metricsSystem.createLongGauge(
         BesuMetricCategory.EXECUTORS,
-        metricName + "_completed_tasks_total",
+        metricName + "_completed_tasks",
         "Total number of tasks executed",
         executor::getCompletedTaskCount);
 
     metricsSystem.createLongGauge(
         BesuMetricCategory.EXECUTORS,
-        metricName + "_submitted_tasks_total",
+        metricName + "_submitted_tasks",
         "Total number of tasks executed",
         executor::getTaskCount);
 

diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/Observation.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/Observation.java
@@ -17,99 +17,14 @@
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
 
 import java.util.List;
-import java.util.Objects;
 
-import com.google.common.base.MoreObjects;
-
-/** The Observation. */
-public class Observation {
-  private final MetricCategory category;
-  private final String metricName;
-  private final List<String> labels;
-  private final Object value;
-
-  /**
-   * Instantiates a new Observation.
-   *
-   * @param category the category
-   * @param metricName the metric name
-   * @param value the value
-   * @param labels the labels
-   */
-  public Observation(
-      final MetricCategory category,
-      final String metricName,
-      final Object value,
-      final List<String> labels) {
-    this.category = category;
-    this.metricName = metricName;
-    this.value = value;
-    this.labels = labels;
-  }
-
-  /**
-   * Gets category.
-   *
-   * @return the category
-   */
-  public MetricCategory getCategory() {
-    return category;
-  }
-
-  /**
-   * Gets metric name.
-   *
-   * @return the metric name
-   */
-  public String getMetricName() {
-    return metricName;
-  }
-
-  /**
-   * Gets labels.
-   *
-   * @return the labels
-   */
-  public List<String> getLabels() {
-    return labels;
-  }
-
-  /**
-   * Gets value.
-   *
-   * @return the value
-   */
-  public Object getValue() {
-    return value;
-  }
-
-  @Override
-  public boolean equals(final Object o) {
-    if (this == o) {
-      return true;
-    }
-    if (o == null || getClass() != o.getClass()) {
-      return false;
-    }
-    final Observation that = (Observation) o;
-    return Objects.equals(category, that.category)
-        && Objects.equals(metricName, that.metricName)
-        && Objects.equals(labels, that.labels)
-        && Objects.equals(value, that.value);
-  }
-
-  @Override
-  public int hashCode() {
-    return Objects.hash(category, metricName, labels, value);
-  }
-
-  @Override
-  public String toString() {
-    return MoreObjects.toStringHelper(this)
-        .add("category", category)
-        .add("metricName", metricName)
-        .add("labels", labels)
-        .add("value", value)
-        .toString();
-  }
-}
+/**
+ * The Observation.
+ *
+ * @param category the category
+ * @param metricName the metric name
+ * @param value the value
+ * @param labels the labels
+ */
+public record Observation(
+    MetricCategory category, String metricName, Object value, List<String> labels) {}
diff --git a/...cs/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java b/...cs/core/src/main/java/org/hyperledger/besu/metrics/opentelemetry/OpenTelemetrySystem.java
@@ -134,7 +134,7 @@ public OpenTelemetrySystem(
 
   @Override
   public Stream<Observation> streamObservations(final MetricCategory category) {
-    return streamObservations().filter(metricData -> metricData.getCategory().equals(category));
+    return streamObservations().filter(metricData -> metricData.category().equals(category));
   }
 
   @Override

diff --git a/...core/src/main/java/org/hyperledger/besu/metrics/prometheus/AbstractPrometheusSummary.java b/...core/src/main/java/org/hyperledger/besu/metrics/prometheus/AbstractPrometheusSummary.java
@@ -14,6 +14,9 @@
  */
 package org.hyperledger.besu.metrics.prometheus;
 
+import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.addLabelValues;
+import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.getLabelValues;
+
 import org.hyperledger.besu.metrics.Observation;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
 
@@ -42,17 +45,14 @@ public Stream<Observation> streamObservations() {
     return collect().getDataPoints().stream()
         .flatMap(
             dataPoint -> {
-              final var labelValues = PrometheusCollector.getLabelValues(dataPoint.getLabels());
+              final var labelValues = getLabelValues(dataPoint.getLabels());
               final var quantiles = dataPoint.getQuantiles();
               final var observations = new ArrayList<Observation>(quantiles.size() + 2);
 
               if (dataPoint.hasSum()) {
                 observations.add(
                     new Observation(
-                        category,
-                        name,
-                        dataPoint.getSum(),
-                        PrometheusCollector.addLabelValues(labelValues, "sum")));
+                        category, name, dataPoint.getSum(), addLabelValues(labelValues, "sum")));
               }
 
               if (dataPoint.hasCount()) {
@@ -61,7 +61,7 @@ public Stream<Observation> streamObservations() {
                         category,
                         name,
                         dataPoint.getCount(),
-                        PrometheusCollector.addLabelValues(labelValues, "count")));
+                        addLabelValues(labelValues, "count")));
               }
 
               quantiles.forEach(
@@ -71,7 +71,7 @@ public Stream<Observation> streamObservations() {
                               category,
                               name,
                               quantile.getValue(),
-                              PrometheusCollector.addLabelValues(
+                              addLabelValues(
                                   labelValues,
                                   "quantile",
                                   Double.toString(quantile.getQuantile())))));

diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusCollector.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusCollector.java
@@ -29,15 +29,24 @@
 public interface PrometheusCollector {
 
   /**
-   * Get the name of the collector
+   * Get the identifier of the collector
    *
-   * @return the name of the collector
+   * @return the identifier of the collector
    */
-  String getName();
+  String getIdentifier();
 
-  /** Get the native Prometheus collector */
+  /**
+   * Register this collector to the specified registry
+   *
+   * @param registry the registry
+   */
   void register(final PrometheusRegistry registry);
 
+  /**
+   * Unregister this collector from the specified registry
+   *
+   * @param registry the registry
+   */
   void unregister(final PrometheusRegistry registry);
 
   /**
@@ -57,6 +66,13 @@ static List<String> getLabelValues(final Labels labels) {
     return labels.stream().map(Label::getValue).toList();
   }
 
+  /**
+   * Add new values to an existing list of label values
+   *
+   * @param labelValues existing list of label values
+   * @param values the values to add
+   * @return a new list with new values appended to the original list
+   */
   static List<String> addLabelValues(final List<String> labelValues, final String... values) {
     final var newList = new ArrayList<String>(labelValues.size() + values.length);
     newList.addAll(labelValues);

diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusCounter.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusCounter.java
@@ -14,6 +14,8 @@
  */
 package org.hyperledger.besu.metrics.prometheus;
 
+import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.getLabelValues;
+
 import org.hyperledger.besu.metrics.Observation;
 import org.hyperledger.besu.plugin.services.metrics.Counter;
 import org.hyperledger.besu.plugin.services.metrics.LabelledMetric;
@@ -47,7 +49,7 @@ public Counter labels(final String... labels) {
   }
 
   @Override
-  public String getName() {
+  public String getIdentifier() {
     return counter.getPrometheusName();
   }
 
@@ -67,10 +69,7 @@ public Stream<Observation> streamObservations() {
         .map(
             sample ->
                 new Observation(
-                    category,
-                    name,
-                    sample.getValue(),
-                    PrometheusCollector.getLabelValues(sample.getLabels())));
+                    category, name, sample.getValue(), getLabelValues(sample.getLabels())));
   }
 
   private record UnlabelledCounter(CounterDataPoint counter) implements Counter {

diff --git a/...core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusExternalSummary.java b/...core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusExternalSummary.java
@@ -52,7 +52,7 @@ public PrometheusExternalSummary(
   }
 
   @Override
-  public String getName() {
+  public String getIdentifier() {
     return summary.getPrometheusName();
   }
 

diff --git a/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusGauge.java b/metrics/core/src/main/java/org/hyperledger/besu/metrics/prometheus/PrometheusGauge.java
@@ -14,6 +14,8 @@
  */
 package org.hyperledger.besu.metrics.prometheus;
 
+import static org.hyperledger.besu.metrics.prometheus.PrometheusCollector.getLabelValues;
+
 import org.hyperledger.besu.metrics.Observation;
 import org.hyperledger.besu.plugin.services.metrics.LabelledGauge;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
@@ -93,7 +95,7 @@ public synchronized void labels(final DoubleSupplier valueSupplier, final String
   }
 
   @Override
-  public String getName() {
+  public String getIdentifier() {
     return gauge.getPrometheusName();
   }
 
@@ -107,17 +109,17 @@ public void unregister(final PrometheusRegistry registry) {
     registry.unregister(gauge);
   }
 
-  private Observation convertToObservation(final GaugeSnapshot.GaugeDataPointSnapshot sample) {
-    final List<String> labelValues = PrometheusCollector.getLabelValues(sample.getLabels());
-
-    return new Observation(category, name, sample.getValue(), labelValues);
-  }
-
   @Override
   public Stream<Observation> streamObservations() {
     final var snapshot = gauge.collect();
     return snapshot.getDataPoints().stream().map(this::convertToObservation);
   }
 
+  private Observation convertToObservation(final GaugeSnapshot.GaugeDataPointSnapshot sample) {
+    final List<String> labelValues = getLabelValues(sample.getLabels());
+
+    return new Observation(category, name, sample.getValue(), labelValues);
+  }
+
   private record CallbackData(DoubleSupplier valueSupplier, String[] labelValues) {}
 }