Merge pull request #32866 from vespa-engine/toregge/add-disk-io-metri…

…cs-for-document-type

Add disk io metrics for document type.

searchcore/src/vespa/searchcore/proton/metrics/CMakeLists.txt

@@ -5,6 +5,7 @@ vespa_add_library(searchcore_proton_metrics STATIC
     attribute_metrics_entry.cpp
     cache_metrics.cpp
     content_proton_metrics.cpp
+    disk_io_metrics.cpp
     documentdb_job_trackers.cpp
     documentdb_tagged_metrics.cpp
     document_db_commit_metrics.cpp

searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.cpp

@@ -0,0 +1,44 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "disk_io_metrics.h"
+#include <vespa/searchlib/util/cache_disk_io_stats.h>
+
+using search::CacheDiskIoStats;
+using search::DiskIoStats;
+
+namespace proton {
+
+namespace {
+
+void update_helper(metrics::LongValueMetric &metric, const DiskIoStats &stats) {
+    metric.addTotalValueBatch(stats.read_bytes_total(), stats.read_operations(),
+                              stats.read_bytes_min(), stats.read_bytes_max());
+}
+
+}
+
+DiskIoMetrics::SearchMetrics::SearchMetrics(metrics::MetricSet* parent)
+    : MetricSet("search", {}, "The search io for a given component", parent),
+      _read_bytes("read_bytes", {}, "Bytes read in posting list files as part of search", this),
+      _cached_read_bytes("cached_read_bytes", {}, "Bytes read from posting list files cache as part of search", this)
+{
+}
+
+DiskIoMetrics::SearchMetrics::~SearchMetrics() = default;
+
+void
+DiskIoMetrics::SearchMetrics::update(const CacheDiskIoStats& cache_disk_io_stats)
+{
+    update_helper(_read_bytes, cache_disk_io_stats.read());
+    update_helper(_cached_read_bytes, cache_disk_io_stats.cached_read());
+}
+
+DiskIoMetrics::DiskIoMetrics(metrics::MetricSet* parent)
+    : MetricSet("io", {}, "The disk usage for a given component", parent),
+      _search(this)
+{
+}
+
+DiskIoMetrics::~DiskIoMetrics() = default;
+
+}

searchcore/src/vespa/searchcore/proton/metrics/disk_io_metrics.h

@@ -0,0 +1,34 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/metrics/metricset.h>
+#include <vespa/metrics/valuemetric.h>
+
+namespace search { class CacheDiskIoStats; }
+
+namespace proton {
+
+/*
+ * Class containing disk io metrics, e.g. per index field or
+ * aggregated at document type level.
+ */
+class DiskIoMetrics : public metrics::MetricSet {
+    class SearchMetrics : public metrics::MetricSet {
+        metrics::LongValueMetric _read_bytes;
+        metrics::LongValueMetric _cached_read_bytes;
+    public:
+        explicit SearchMetrics(metrics::MetricSet* parent);
+        ~SearchMetrics() override;
+        void update(const search::CacheDiskIoStats& cache_disk_io_stats);
+    };
+
+    SearchMetrics _search;
+
+public:
+    explicit DiskIoMetrics(metrics::MetricSet* parent);
+    ~DiskIoMetrics() override;
+    void update(const search::CacheDiskIoStats& cache_disk_io_stats) { _search.update(cache_disk_io_stats); }
+};
+
+}

searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.cpp

@@ -93,7 +93,8 @@ DocumentDBTaggedMetrics::IndexMetrics::IndexMetrics(MetricSet *parent)
     : MetricSet("index", {}, "Index metrics (memory and disk) for this document db", parent),
       diskUsage("disk_usage", {}, "Disk space usage in bytes", this),
       memoryUsage(this),
-      docsInMemory("docs_in_memory", {}, "Number of documents in memory index", this)
+      docsInMemory("docs_in_memory", {}, "Number of documents in memory index", this),
+      disk_io(this)
 {
 }
 

searchcore/src/vespa/searchcore/proton/metrics/documentdb_tagged_metrics.h

@@ -97,6 +97,7 @@ struct DocumentDBTaggedMetrics : metrics::MetricSet
         metrics::LongValueMetric diskUsage;
         MemoryUsageMetrics memoryUsage;
         metrics::LongValueMetric docsInMemory;
+        DiskIoMetrics disk_io;
 
         IndexMetrics(metrics::MetricSet *parent);
         ~IndexMetrics() override;

searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.cpp

@@ -3,47 +3,15 @@
 #include "index_metrics_entry.h"
 #include <vespa/searchlib/util/cache_disk_io_stats.h>
 
-using search::CacheDiskIoStats;
-using search::DiskIoStats;
-
 namespace proton {
 
 namespace {
 
 const std::string entry_name("index");
 const std::string entry_description("Metrics for indexes for a given field");
 
-void update_helper(metrics::LongValueMetric &metric, const DiskIoStats &stats) {
-    metric.addTotalValueBatch(stats.read_bytes_total(), stats.read_operations(),
-                              stats.read_bytes_min(), stats.read_bytes_max());
-}
-
-}
-
-IndexMetricsEntry::DiskIoMetrics::SearchMetrics::SearchMetrics(metrics::MetricSet* parent)
-    : MetricSet("search", {}, "The search io for a given component", parent),
-      _read_bytes("read_bytes", {}, "Bytes read in posting list files as part of search", this),
-      _cached_read_bytes("cached_read_bytes", {}, "Bytes read from posting list files cache as part of search", this)
-{
-}
-
-IndexMetricsEntry::DiskIoMetrics::SearchMetrics::~SearchMetrics() = default;
-
-void
-IndexMetricsEntry::DiskIoMetrics::SearchMetrics::update(const CacheDiskIoStats& cache_disk_io_stats)
-{
-    update_helper(_read_bytes, cache_disk_io_stats.read());
-    update_helper(_cached_read_bytes, cache_disk_io_stats.cached_read());
 }
 
-IndexMetricsEntry::DiskIoMetrics::DiskIoMetrics(metrics::MetricSet* parent)
-    : MetricSet("io", {}, "The disk usage for a given component", parent),
-      _search(this)
-{
-}
-
-IndexMetricsEntry::DiskIoMetrics::~DiskIoMetrics() = default;
-
 IndexMetricsEntry::IndexMetricsEntry(const std::string& field_name)
     : FieldMetricsEntry(entry_name, field_name, entry_description),
       _disk_io(this)

searchcore/src/vespa/searchcore/proton/metrics/index_metrics_entry.h

@@ -2,35 +2,16 @@
 
 #pragma once
 
+#include "disk_io_metrics.h"
 #include "field_metrics_entry.h"
 
-namespace search { class CacheDiskIoStats; }
-
 namespace proton {
 
 /*
  * Class containing metrics for the index aspect of a field, i.e.
  * disk indexes and memory indexes.
  */
 class IndexMetricsEntry : public FieldMetricsEntry {
-    class DiskIoMetrics : public metrics::MetricSet {
-        class SearchMetrics : public metrics::MetricSet {
-            metrics::LongValueMetric _read_bytes;
-            metrics::LongValueMetric _cached_read_bytes;
-        public:
-            explicit SearchMetrics(metrics::MetricSet* parent);
-            ~SearchMetrics() override;
-            void update(const search::CacheDiskIoStats& cache_disk_io_stats);
-        };
-
-        SearchMetrics _search;
-
-    public:
-        explicit DiskIoMetrics(metrics::MetricSet* parent);
-        ~DiskIoMetrics() override;
-        void update(const search::CacheDiskIoStats& cache_disk_io_stats) { _search.update(cache_disk_io_stats); }
-    };
-
     DiskIoMetrics _disk_io;
 
 public:

searchcore/src/vespa/searchcore/proton/server/documentdb_metrics_updater.cpp

@@ -80,14 +80,17 @@ updateIndexMetrics(DocumentDBTaggedMetrics &metrics, const search::SearchableSta
     updateMemoryUsageMetrics(indexMetrics.memoryUsage, stats.memoryUsage(), totalStats);
     indexMetrics.docsInMemory.set(stats.docsInMemory());
     auto& field_metrics = metrics.ready.index;
+    search::CacheDiskIoStats disk_io;
     for (auto& field : stats.get_field_stats()) {
         auto entry = field_metrics.get_field_metrics_entry(field.first);
         if (entry) {
             entry->memoryUsage.update(field.second.memory_usage());
             entry->size_on_disk.set(field.second.size_on_disk());
             entry->update_disk_io(field.second.cache_disk_io_stats());
         }
+        disk_io.merge(field.second.cache_disk_io_stats());
     }
+    indexMetrics.disk_io.update(disk_io);
 }
 
 struct TempAttributeMetric