From c0006099abb320599b8727adefec43358f7a875b Mon Sep 17 00:00:00 2001
From: qicosmos
Date: Fri, 7 Jun 2024 23:15:51 +0800
Subject: [PATCH] [new feature]Add metrics (#674)
---
.github/workflows/windows.yml | 66 +--
include/ylt/coro_http/coro_http_server.hpp | 1 -
include/ylt/coro_io/io_context_pool.hpp | 17 +-
include/ylt/metric/counter.hpp | 258 +++++++++
include/ylt/metric/detail/ckms_quantiles.hpp | 175 ++++++
.../metric/detail/time_window_quantiles.hpp | 52 ++
include/ylt/metric/gauge.hpp | 52 ++
include/ylt/metric/histogram.hpp | 83 +++
include/ylt/metric/metric.hpp | 316 +++++++++++
include/ylt/metric/summary.hpp | 139 +++++
.../cinatra/coro_http_connection.hpp | 75 ++-
.../standalone/cinatra/coro_http_server.hpp | 37 +-
.../ylt/standalone/cinatra/metric_conf.hpp | 128 +++++
src/metric/tests/CMakeLists.txt | 5 +
src/metric/tests/test_metric.cpp | 280 ++++++++++
website/.vitepress/config/zh_data.ts | 4 +
website/.vitepress/config/zh_locale.ts | 1 +
.../docs/zh/guide/what_is_yalantinglibs.md | 8 +-
website/docs/zh/index.md | 4 +-
.../docs/zh/metric/metrict_introduction.md | 511 ++++++++++++++++++
20 files changed, 2169 insertions(+), 43 deletions(-)
create mode 100644 include/ylt/metric/counter.hpp
create mode 100644 include/ylt/metric/detail/ckms_quantiles.hpp
create mode 100644 include/ylt/metric/detail/time_window_quantiles.hpp
create mode 100644 include/ylt/metric/gauge.hpp
create mode 100644 include/ylt/metric/histogram.hpp
create mode 100644 include/ylt/metric/metric.hpp
create mode 100644 include/ylt/metric/summary.hpp
create mode 100644 include/ylt/standalone/cinatra/metric_conf.hpp
create mode 100644 src/metric/tests/CMakeLists.txt
create mode 100644 src/metric/tests/test_metric.cpp
create mode 100644 website/docs/zh/metric/metrict_introduction.md
diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index fe1c645a9..d79e6f74b 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -9,41 +9,41 @@ on:
workflow_dispatch:
jobs:
- windows_msvc:
- runs-on: windows-latest
+ # windows_msvc:
+ # runs-on: windows-latest
- strategy:
- matrix:
- mode: [ Release ] #[ Release, Debug ] #Debug not support ccache
- #https://github.com/ccache/ccache/wiki/MS-Visual-Studio
- #https://github.com/ccache/ccache/issues/1040
- arch: [ amd64, x86 ] #[ amd64,x86 ]
- ssl: [ OFF ] #[ ON, OFF ]
+ # strategy:
+ # matrix:
+ # mode: [ Release ] #[ Release, Debug ] #Debug not support ccache
+ # #https://github.com/ccache/ccache/wiki/MS-Visual-Studio
+ # #https://github.com/ccache/ccache/issues/1040
+ # arch: [ amd64, x86 ] #[ amd64,x86 ]
+ # ssl: [ OFF ] #[ ON, OFF ]
- steps:
- - name: Checkout
- uses: actions/checkout@v3
- - name: Enable Developer Command Prompt
- uses: ilammy/msvc-dev-cmd@v1.12.0
- with:
- arch: ${{ matrix.arch }}
- - name: Install ninja-build tool
- uses: seanmiddleditch/gha-setup-ninja@master
- with:
- version: 1.11.1
- - name: latest ccache
- run: choco install ccache
- - name: ccache
- uses: hendrikmuhs/ccache-action@v1.2
- with:
- key: ${{ github.job }}-${{ matrix.mode}}-ssl( ${{ matrix.ssl}} )-arch-${{ matrix.arch}}
- - name: Configure CMake
- run: cmake -B ${{github.workspace}}\build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.mode }} -DYLT_ENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON
- - name: Build
- run: cmake --build ${{github.workspace}}\build
- - name: Test
- working-directory: ${{github.workspace}}\build
- run: ctest -C ${{matrix.mode}} -j 1 -V
+ # steps:
+ # - name: Checkout
+ # uses: actions/checkout@v3
+ # - name: Enable Developer Command Prompt
+ # uses: ilammy/msvc-dev-cmd@v1.12.0
+ # with:
+ # arch: ${{ matrix.arch }}
+ # - name: Install ninja-build tool
+ # uses: seanmiddleditch/gha-setup-ninja@master
+ # with:
+ # version: 1.11.1
+ # - name: latest ccache
+ # run: choco install ccache
+ # - name: ccache
+ # uses: hendrikmuhs/ccache-action@v1.2
+ # with:
+ # key: ${{ github.job }}-${{ matrix.mode}}-ssl( ${{ matrix.ssl}} )-arch-${{ matrix.arch}}
+ # - name: Configure CMake
+ # run: cmake -B ${{github.workspace}}\build -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.mode }} -DYLT_ENABLE_SSL=${{matrix.ssl}} -DUSE_CCACHE=ON
+ # - name: Build
+ # run: cmake --build ${{github.workspace}}\build
+ # - name: Test
+ # working-directory: ${{github.workspace}}\build
+ # run: ctest -C ${{matrix.mode}} -j 1 -V
windows_msvc_2019:
runs-on: windows-2019
diff --git a/include/ylt/coro_http/coro_http_server.hpp b/include/ylt/coro_http/coro_http_server.hpp
index 46269f8dd..a4bb1ecf4 100644
--- a/include/ylt/coro_http/coro_http_server.hpp
+++ b/include/ylt/coro_http/coro_http_server.hpp
@@ -14,7 +14,6 @@
* limitations under the License.
*/
#pragma once
-#include "cinatra/uri.hpp"
#ifdef YLT_ENABLE_SSL
#define CINATRA_ENABLE_SSL
#endif
diff --git a/include/ylt/coro_io/io_context_pool.hpp b/include/ylt/coro_io/io_context_pool.hpp
index b45c63ff9..847fa9cfd 100644
--- a/include/ylt/coro_io/io_context_pool.hpp
+++ b/include/ylt/coro_io/io_context_pool.hpp
@@ -22,6 +22,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -117,6 +118,8 @@ class io_context_pool {
pool_size = 1; // set default value as 1
}
+ total_thread_num_ += pool_size;
+
for (std::size_t i = 0; i < pool_size; ++i) {
io_context_ptr io_context(new asio::io_context(1));
work_ptr work(new asio::io_context::work(*io_context));
@@ -150,8 +153,11 @@ class io_context_pool {
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
CPU_SET(i, &cpuset);
- pthread_setaffinity_np(threads.back()->native_handle(),
- sizeof(cpu_set_t), &cpuset);
+ int rc = pthread_setaffinity_np(threads.back()->native_handle(),
+ sizeof(cpu_set_t), &cpuset);
+ if (rc != 0) {
+ std::cerr << "Error calling pthread_setaffinity_np: " << rc << "\n";
+ }
}
#endif
}
@@ -201,6 +207,8 @@ class io_context_pool {
template
friend io_context_pool &g_io_context_pool();
+ static size_t get_total_thread_num() { return total_thread_num_; }
+
private:
using io_context_ptr = std::shared_ptr;
using work_ptr = std::shared_ptr;
@@ -213,8 +221,13 @@ class io_context_pool {
std::atomic has_run_or_stop_ = false;
std::once_flag flag_;
bool cpu_affinity_ = false;
+ inline static std::atomic total_thread_num_ = 0;
};
+inline size_t get_total_thread_num() {
+ return io_context_pool::get_total_thread_num();
+}
+
class multithread_context_pool {
public:
multithread_context_pool(size_t thd_num = std::thread::hardware_concurrency())
diff --git a/include/ylt/metric/counter.hpp b/include/ylt/metric/counter.hpp
new file mode 100644
index 000000000..a252106d1
--- /dev/null
+++ b/include/ylt/metric/counter.hpp
@@ -0,0 +1,258 @@
+#pragma once
+#include
+#include
+
+#include "metric.hpp"
+
+namespace ylt {
+enum class op_type_t { INC, DEC, SET };
+struct counter_sample {
+ op_type_t op_type;
+ std::vector labels_value;
+ double value;
+};
+
+class counter_t : public metric_t {
+ public:
+ // default, no labels, only contains an atomic value.
+ counter_t(std::string name, std::string help)
+ : metric_t(MetricType::Counter, std::move(name), std::move(help)) {
+ use_atomic_ = true;
+ }
+
+ // static labels value, contains a map with atomic value.
+ counter_t(std::string name, std::string help,
+ std::map labels)
+ : metric_t(MetricType::Counter, std::move(name), std::move(help)) {
+ for (auto &[k, v] : labels) {
+ labels_name_.push_back(k);
+ labels_value_.push_back(v);
+ }
+
+ atomic_value_map_.emplace(labels_value_, 0);
+ use_atomic_ = true;
+ }
+
+ // dynamic labels value
+ counter_t(std::string name, std::string help,
+ std::vector labels_name)
+ : metric_t(MetricType::Counter, std::move(name), std::move(help),
+ std::move(labels_name)) {}
+
+ virtual ~counter_t() {}
+
+ double value() { return default_lable_value_; }
+
+ double value(const std::vector &labels_value) {
+ if (use_atomic_) {
+ double val = atomic_value_map_[labels_value];
+ return val;
+ }
+ else {
+ std::lock_guard lock(mtx_);
+ return value_map_[labels_value];
+ }
+ }
+
+ std::map, double,
+ std::less>>
+ value_map() {
+ std::map, double,
+ std::less>>
+ map;
+ if (use_atomic_) {
+ map = {atomic_value_map_.begin(), atomic_value_map_.end()};
+ }
+ else {
+ std::lock_guard lock(mtx_);
+ map = value_map_;
+ }
+ return map;
+ }
+
+ void serialize(std::string &str) override {
+ if (labels_name_.empty()) {
+ if (default_lable_value_ == 0) {
+ return;
+ }
+ serialize_head(str);
+ serialize_default_label(str);
+ return;
+ }
+
+ serialize_head(str);
+ std::string s;
+ if (use_atomic_) {
+ serialize_map(atomic_value_map_, s);
+ }
+ else {
+ serialize_map(value_map_, s);
+ }
+
+ if (s.empty()) {
+ str.clear();
+ }
+ else {
+ str.append(s);
+ }
+ }
+
+ void inc(double val = 1) {
+ if (val < 0) {
+ throw std::invalid_argument("the value is less than zero");
+ }
+
+#ifdef __APPLE__
+ mac_os_atomic_fetch_add(&default_lable_value_, val);
+#else
+ default_lable_value_ += val;
+#endif
+ }
+
+ void inc(const std::vector &labels_value, double value = 1) {
+ if (value == 0) {
+ return;
+ }
+
+ validate(labels_value, value);
+ if (use_atomic_) {
+ if (labels_value != labels_value_) {
+ throw std::invalid_argument(
+ "the given labels_value is not match with origin labels_value");
+ }
+ set_value(atomic_value_map_[labels_value], value, op_type_t::INC);
+ }
+ else {
+ std::lock_guard lock(mtx_);
+ set_value(value_map_[labels_value], value, op_type_t::INC);
+ }
+ }
+
+ void update(double value) { default_lable_value_ = value; }
+
+ void update(const std::vector &labels_value, double value) {
+ if (labels_value.empty() || labels_name_.size() != labels_value.size()) {
+ throw std::invalid_argument(
+ "the number of labels_value name and labels_value is not match");
+ }
+ if (use_atomic_) {
+ if (labels_value != labels_value_) {
+ throw std::invalid_argument(
+ "the given labels_value is not match with origin labels_value");
+ }
+ set_value(atomic_value_map_[labels_value], value, op_type_t::SET);
+ }
+ else {
+ std::lock_guard lock(mtx_);
+ set_value(value_map_[labels_value], value, op_type_t::SET);
+ }
+ }
+
+ std::map, std::atomic,
+ std::less>>
+ &atomic_value_map() {
+ return atomic_value_map_;
+ }
+
+ protected:
+ void serialize_default_label(std::string &str) {
+ str.append(name_);
+ if (labels_name_.empty()) {
+ str.append(" ");
+ }
+
+ if (type_ == MetricType::Counter) {
+ str.append(std::to_string((int64_t)default_lable_value_));
+ }
+ else {
+ str.append(std::to_string(default_lable_value_));
+ }
+
+ str.append("\n");
+ }
+
+ template
+ void serialize_map(T &value_map, std::string &str) {
+ for (auto &[labels_value, value] : value_map) {
+ if (value == 0) {
+ continue;
+ }
+ str.append(name_);
+ str.append("{");
+ build_string(str, labels_name_, labels_value);
+ str.append("} ");
+
+ if (type_ == MetricType::Counter) {
+ str.append(std::to_string((int64_t)value));
+ }
+ else {
+ str.append(std::to_string(value));
+ }
+
+ str.append("\n");
+ }
+ }
+
+ void build_string(std::string &str, const std::vector &v1,
+ const std::vector &v2) {
+ for (size_t i = 0; i < v1.size(); i++) {
+ str.append(v1[i]).append("=\"").append(v2[i]).append("\"").append(",");
+ }
+ str.pop_back();
+ }
+
+ void validate(const std::vector &labels_value, double value) {
+ if (value < 0) {
+ throw std::invalid_argument("the value is less than zero");
+ }
+ if (labels_value.empty() || labels_name_.size() != labels_value.size()) {
+ throw std::invalid_argument(
+ "the number of labels_value name and labels_value is not match");
+ }
+ }
+
+ template
+ void set_value(T &label_val, double value, op_type_t type) {
+ switch (type) {
+ case op_type_t::INC: {
+#ifdef __APPLE__
+ if constexpr (is_atomic) {
+ mac_os_atomic_fetch_add(&label_val, value);
+ }
+ else {
+ label_val += value;
+ }
+#else
+ label_val += value;
+#endif
+ } break;
+ case op_type_t::DEC:
+#ifdef __APPLE__
+ if constexpr (is_atomic) {
+ mac_os_atomic_fetch_sub(&label_val, value);
+ }
+ else {
+ label_val -= value;
+ }
+
+#else
+ label_val -= value;
+#endif
+ break;
+ case op_type_t::SET:
+ label_val = value;
+ break;
+ }
+ }
+
+ std::map, std::atomic,
+ std::less>>
+ atomic_value_map_;
+ std::atomic default_lable_value_ = 0;
+
+ std::mutex mtx_;
+ std::map, double,
+ std::less>>
+ value_map_;
+};
+} // namespace ylt
\ No newline at end of file
diff --git a/include/ylt/metric/detail/ckms_quantiles.hpp b/include/ylt/metric/detail/ckms_quantiles.hpp
new file mode 100644
index 000000000..cdaf1db92
--- /dev/null
+++ b/include/ylt/metric/detail/ckms_quantiles.hpp
@@ -0,0 +1,175 @@
+#pragma once
+#include
+#include
+#include
+
+// https://github.com/jupp0r/prometheus-cpp/blob/master/core/include/prometheus/detail/ckms_quantiles.h
+
+namespace ylt {
+class CKMSQuantiles {
+ public:
+ struct Quantile {
+ Quantile(double quantile, double error)
+ : quantile(quantile),
+ error(error),
+ u(2.0 * error / (1.0 - quantile)),
+ v(2.0 * error / quantile) {}
+
+ double quantile;
+ double error;
+ double u;
+ double v;
+ };
+
+ private:
+ struct Item {
+ double value;
+ int g;
+ int delta;
+
+ Item(double value, int lower_delta, int delta)
+ : value(value), g(lower_delta), delta(delta) {}
+ };
+
+ public:
+ explicit CKMSQuantiles(const std::vector& quantiles)
+ : quantiles_(quantiles), count_(0), buffer_{}, buffer_count_(0) {}
+
+ void insert(double value) {
+ buffer_[buffer_count_] = value;
+ ++buffer_count_;
+
+ if (buffer_count_ == buffer_.size()) {
+ insertBatch();
+ compress();
+ }
+ }
+
+ double get(double q) {
+ insertBatch();
+ compress();
+
+ if (sample_.empty()) {
+ return std::numeric_limits::quiet_NaN();
+ }
+
+ int rankMin = 0;
+ const auto desired = static_cast(q * count_);
+ const auto bound = desired + (allowableError(desired) / 2);
+
+ auto it = sample_.begin();
+ decltype(it) prev;
+ auto cur = it++;
+
+ while (it != sample_.end()) {
+ prev = cur;
+ cur = it++;
+
+ rankMin += prev->g;
+
+ if (rankMin + cur->g + cur->delta > bound) {
+ return prev->value;
+ }
+ }
+
+ return sample_.back().value;
+ }
+ void reset() {
+ count_ = 0;
+ sample_.clear();
+ buffer_count_ = 0;
+ }
+
+ private:
+ double allowableError(int rank) {
+ auto size = sample_.size();
+ double minError = size + 1;
+
+ for (const auto& q : quantiles_.get()) {
+ double error;
+ if (rank <= q.quantile * size) {
+ error = q.u * (size - rank);
+ }
+ else {
+ error = q.v * rank;
+ }
+ if (error < minError) {
+ minError = error;
+ }
+ }
+ return minError;
+ }
+
+ bool insertBatch() {
+ if (buffer_count_ == 0) {
+ return false;
+ }
+
+ std::sort(buffer_.begin(), buffer_.begin() + buffer_count_);
+
+ std::size_t start = 0;
+ if (sample_.empty()) {
+ sample_.emplace_back(buffer_[0], 1, 0);
+ ++start;
+ ++count_;
+ }
+
+ std::size_t idx = 0;
+ std::size_t item = idx++;
+
+ for (std::size_t i = start; i < buffer_count_; ++i) {
+ double v = buffer_[i];
+ while (idx < sample_.size() && sample_[item].value < v) {
+ item = idx++;
+ }
+
+ if (sample_[item].value > v) {
+ --idx;
+ }
+
+ int delta;
+ if (idx - 1 == 0 || idx + 1 == sample_.size()) {
+ delta = 0;
+ }
+ else {
+ delta = static_cast(std::floor(allowableError(idx + 1))) + 1;
+ }
+
+ sample_.emplace(sample_.begin() + idx, v, 1, delta);
+ count_++;
+ item = idx++;
+ }
+
+ buffer_count_ = 0;
+ return true;
+ }
+ void compress() {
+ if (sample_.size() < 2) {
+ return;
+ }
+
+ std::size_t idx = 0;
+ std::size_t prev;
+ std::size_t next = idx++;
+
+ while (idx < sample_.size()) {
+ prev = next;
+ next = idx++;
+
+ if (sample_[prev].g + sample_[next].g + sample_[next].delta <=
+ allowableError(idx - 1)) {
+ sample_[next].g += sample_[prev].g;
+ sample_.erase(sample_.begin() + prev);
+ }
+ }
+ }
+
+ private:
+ const std::reference_wrapper> quantiles_;
+
+ std::size_t count_;
+ std::vector- sample_;
+ std::array buffer_;
+ std::size_t buffer_count_;
+};
+} // namespace ylt
\ No newline at end of file
diff --git a/include/ylt/metric/detail/time_window_quantiles.hpp b/include/ylt/metric/detail/time_window_quantiles.hpp
new file mode 100644
index 000000000..fd7df105f
--- /dev/null
+++ b/include/ylt/metric/detail/time_window_quantiles.hpp
@@ -0,0 +1,52 @@
+#pragma once
+#include "ckms_quantiles.hpp"
+// https://github.com/jupp0r/prometheus-cpp/blob/master/core/include/prometheus/detail/time_window_quantiles.h
+
+namespace ylt {
+class TimeWindowQuantiles {
+ using Clock = std::chrono::steady_clock;
+
+ public:
+ TimeWindowQuantiles(const std::vector& quantiles,
+ Clock::duration max_age_seconds, int age_buckets)
+ : quantiles_(quantiles),
+ ckms_quantiles_(age_buckets, CKMSQuantiles(quantiles_)),
+ current_bucket_(0),
+ last_rotation_(Clock::now()),
+ rotation_interval_(max_age_seconds / age_buckets) {}
+
+ double get(double q) const {
+ CKMSQuantiles& current_bucket = rotate();
+ return current_bucket.get(q);
+ }
+ void insert(double value) {
+ rotate();
+ for (auto& bucket : ckms_quantiles_) {
+ bucket.insert(value);
+ }
+ }
+
+ private:
+ CKMSQuantiles& rotate() const {
+ auto delta = Clock::now() - last_rotation_;
+ while (delta > rotation_interval_) {
+ ckms_quantiles_[current_bucket_].reset();
+
+ if (++current_bucket_ >= ckms_quantiles_.size()) {
+ current_bucket_ = 0;
+ }
+
+ delta -= rotation_interval_;
+ last_rotation_ += rotation_interval_;
+ }
+ return ckms_quantiles_[current_bucket_];
+ }
+
+ const std::vector& quantiles_;
+ mutable std::vector ckms_quantiles_;
+ mutable std::size_t current_bucket_;
+
+ mutable Clock::time_point last_rotation_;
+ const Clock::duration rotation_interval_;
+};
+} // namespace ylt
\ No newline at end of file
diff --git a/include/ylt/metric/gauge.hpp b/include/ylt/metric/gauge.hpp
new file mode 100644
index 000000000..19c4b65c2
--- /dev/null
+++ b/include/ylt/metric/gauge.hpp
@@ -0,0 +1,52 @@
+#pragma once
+#include
+
+#include "counter.hpp"
+
+namespace ylt {
+class gauge_t : public counter_t {
+ public:
+ gauge_t(std::string name, std::string help)
+ : counter_t(std::move(name), std::move(help)) {
+ set_metric_type(MetricType::Gauge);
+ }
+ gauge_t(std::string name, std::string help,
+ std::vector labels_name)
+ : counter_t(std::move(name), std::move(help), std::move(labels_name)) {
+ set_metric_type(MetricType::Gauge);
+ }
+
+ gauge_t(std::string name, std::string help,
+ std::map labels)
+ : counter_t(std::move(name), std::move(help), std::move(labels)) {
+ set_metric_type(MetricType::Gauge);
+ }
+
+ void dec(double value = 1) {
+#ifdef __APPLE__
+ mac_os_atomic_fetch_sub(&default_lable_value_, value);
+#else
+ default_lable_value_ -= value;
+#endif
+ }
+
+ void dec(const std::vector& labels_value, double value = 1) {
+ if (value == 0) {
+ return;
+ }
+
+ validate(labels_value, value);
+ if (use_atomic_) {
+ if (labels_value != labels_value_) {
+ throw std::invalid_argument(
+ "the given labels_value is not match with origin labels_value");
+ }
+ set_value(atomic_value_map_[labels_value], value, op_type_t::DEC);
+ }
+ else {
+ std::lock_guard lock(mtx_);
+ set_value(value_map_[labels_value], value, op_type_t::DEC);
+ }
+ }
+};
+} // namespace ylt
\ No newline at end of file
diff --git a/include/ylt/metric/histogram.hpp b/include/ylt/metric/histogram.hpp
new file mode 100644
index 000000000..dfd2ca131
--- /dev/null
+++ b/include/ylt/metric/histogram.hpp
@@ -0,0 +1,83 @@
+
+#pragma once
+#include
+#include
+#include
+#include
+
+#include "counter.hpp"
+#include "metric.hpp"
+
+namespace ylt {
+class histogram_t : public metric_t {
+ public:
+ histogram_t(std::string name, std::string help, std::vector buckets)
+ : bucket_boundaries_(buckets),
+ metric_t(MetricType::Histogram, std::move(name), std::move(help)),
+ sum_(std::make_shared("", "")) {
+ if (!is_strict_sorted(begin(bucket_boundaries_), end(bucket_boundaries_))) {
+ throw std::invalid_argument("Bucket Boundaries must be strictly sorted");
+ }
+
+ for (size_t i = 0; i < buckets.size() + 1; i++) {
+ bucket_counts_.push_back(std::make_shared("", ""));
+ }
+ use_atomic_ = true;
+ }
+
+ void observe(double value) {
+ const auto bucket_index = static_cast(
+ std::distance(bucket_boundaries_.begin(),
+ std::lower_bound(bucket_boundaries_.begin(),
+ bucket_boundaries_.end(), value)));
+ sum_->inc(value);
+ bucket_counts_[bucket_index]->inc();
+ }
+
+ auto get_bucket_counts() { return bucket_counts_; }
+
+ void serialize(std::string& str) override {
+ serialize_head(str);
+ double count = 0;
+ auto bucket_counts = get_bucket_counts();
+ for (size_t i = 0; i < bucket_counts.size(); i++) {
+ auto counter = bucket_counts[i];
+ str.append(name_).append("_bucket{");
+ if (i == bucket_boundaries_.size()) {
+ str.append("le=\"").append("+Inf").append("\"} ");
+ }
+ else {
+ str.append("le=\"")
+ .append(std::to_string(bucket_boundaries_[i]))
+ .append("\"} ");
+ }
+
+ count += counter->value();
+ str.append(std::to_string(count));
+ str.append("\n");
+ }
+
+ str.append(name_)
+ .append("_sum ")
+ .append(std::to_string(sum_->value()))
+ .append("\n");
+
+ str.append(name_)
+ .append("_count ")
+ .append(std::to_string(count))
+ .append("\n");
+ }
+
+ private:
+ template
+ bool is_strict_sorted(ForwardIterator first, ForwardIterator last) {
+ return std::adjacent_find(first, last,
+ std::greater_equal::value_type>()) == last;
+ }
+
+ std::vector bucket_boundaries_;
+ std::vector> bucket_counts_; // readonly
+ std::shared_ptr sum_;
+};
+} // namespace ylt
\ No newline at end of file
diff --git a/include/ylt/metric/metric.hpp b/include/ylt/metric/metric.hpp
new file mode 100644
index 000000000..58d0afc5b
--- /dev/null
+++ b/include/ylt/metric/metric.hpp
@@ -0,0 +1,316 @@
+#pragma once
+#include
+#include
+#include
@@ -10,7 +10,7 @@
[English Version](../../en/guide/what_is_yalantinglibs.md)
-yaLanTingLibs 是一个现代C++基础工具库的集合, 现在它包括 struct_pack, struct_json, struct_xml, struct_yaml, struct_pb, easylog, coro_rpc, coro_io, coro_http 和 async_simple, 目前我们正在开发并添加更多的新功能。
+yaLanTingLibs 是一个现代C++基础工具库的集合, 现在它包括 struct_pack, struct_json, struct_xml, struct_yaml, struct_pb, easylog, coro_rpc, coro_io, coro_http, metric 和 async_simple, 目前我们正在开发并添加更多的新功能。
yaLanTingLibs 的目标: 为C++开发者提供高性能,极度易用的现代C++基础工具库, 帮助用户构建高性能的现代C++应用。
@@ -430,6 +430,10 @@ yalantinglibs工程自身支持如下配置项,如果你使用cmake find_packa
无依赖。
+### metric
+
+无依赖。
+
## 独立子仓库
coro_http 由独立子仓库实现: [cinatra](https://github.com/qicosmos/cinatra)
diff --git a/website/docs/zh/index.md b/website/docs/zh/index.md
index 08db7b745..bf815f29b 100644
--- a/website/docs/zh/index.md
+++ b/website/docs/zh/index.md
@@ -27,5 +27,7 @@ features:
- title: easylog
details: C++17 实现的高性能易用的日志库, 支持cout 流式、sprintf 和 fmt::format/std::format 输出.
- title: struct_xml struct_json struct_yaml
- details: C++17 实现的高性能易用的序列化库, 支持xml, json和yaml 的序列化/反序列化.
+ details: C++17 实现的高性能易用的序列化库, 支持xml, json和yaml 的序列化/反序列化.
+ - title: metric
+ details: metric 介绍
---
diff --git a/website/docs/zh/metric/metrict_introduction.md b/website/docs/zh/metric/metrict_introduction.md
new file mode 100644
index 000000000..d61b627b2
--- /dev/null
+++ b/website/docs/zh/metric/metrict_introduction.md
@@ -0,0 +1,511 @@
+# metric 介绍
+metric 用于统计应用程序的各种指标,这些指标被用于系统见识和警报,常见的指标类型有四种:Counter、Gauge、Histogram和Summary,这些指标遵循[Prometheus](https://hulining.gitbook.io/prometheus/introduction)的数据格式。
+
+## Counter 计数器类型
+Counter是一个累计类型的数据指标,它代表单调递增的计数器,其值只能在重新启动时增加或重置为 0。例如,您可以使用计数器来表示已响应的请求数,已完成或出错的任务数。
+
+不要使用计数器来显示可以减小的值。例如,请不要使用计数器表示当前正在运行的进程数;使用 gauge 代替。
+
+## Gauge 数据轨迹类型
+Gauge 是可以任意上下波动数值的指标类型。
+
+Gauge 通常用于测量值,例如温度或当前的内存使用量,还可用于可能上下波动的"计数",例如请求并发数。
+
+如:
+```
+# HELP node_cpu Seconds the cpus spent in each mode.
+# TYPE node_cpu counter
+node_cpu{cpu="cpu0",mode="idle"} 362812.7890625
+# HELP node_load1 1m load average.
+# TYPE node_load1 gauge
+node_load1 3.0703125
+```
+
+## Histogram 直方图类型
+Histogram 对观测值(通常是请求持续时间或响应大小之类的数据)进行采样,并将其计数在可配置的数值区间中。它也提供了所有数据的总和。
+
+基本数据指标名称为的直方图类型数据指标,在数据采集期间会显示多个时间序列:
+
+数值区间的累计计数器,显示为_bucket{le="<数值区间的上边界>"}
+
+所有观测值的总和,显示为_sum
+
+统计到的事件计数,显示为_count(与上述_bucket{le="+Inf"}相同)
+
+如:
+```
+# A histogram, which has a pretty complex representation in the text format:
+# HELP http_request_duration_seconds A histogram of the request duration.
+# TYPE http_request_duration_seconds histogram
+http_request_duration_seconds_bucket{le="0.05"} 24054
+http_request_duration_seconds_bucket{le="0.1"} 33444
+http_request_duration_seconds_bucket{le="0.2"} 100392
+http_request_duration_seconds_bucket{le="+Inf"} 144320
+http_request_duration_seconds_sum 53423
+http_request_duration_seconds_count 144320
+```
+
+## Summary 汇总类型
+类似于 histogram,summary 会采样观察结果(通常是请求持续时间和响应大小之类的数据)。它不仅提供了观测值的总数和所有观测值的总和,还可以计算滑动时间窗口内的可配置分位数。
+
+基本数据指标名称为的 summary 类型数据指标,在数据采集期间会显示多个时间序列:
+
+流观察到的事件的 φ-quantiles(0≤φ≤1),显示为{quantile="<φ>"}
+
+所有观测值的总和,显示为_sum
+
+观察到的事件计数,显示为_count
+
+如:
+```
+# HELP prometheus_tsdb_wal_fsync_duration_seconds Duration of WAL fsync.
+# TYPE prometheus_tsdb_wal_fsync_duration_seconds summary
+prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.5"} 0.012352463
+prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.9"} 0.014458005
+prometheus_tsdb_wal_fsync_duration_seconds{quantile="0.99"} 0.017316173
+prometheus_tsdb_wal_fsync_duration_seconds_sum 2.888716127000002
+prometheus_tsdb_wal_fsync_duration_seconds_count 216
+```
+
+# 概述
+metric 包括4种指标类型:
+- couter:只会增加的指标;
+- gauge:可以增加或减少的指标,它派生于counter;
+- histogram:直方图,初始化的时候需要设置桶(bucket);
+- summary:分位数指标,初始化的时候需要设置桶和误差;
+
+# label
+
+label:标签,可选,指标可以没有标签。标签是指一个键值对,标签的键需要在创建指标的时候设置,是静态不可变的。
+
+标签的值可以在创建指标的时候设置,这样的label则被称为静态的label。
+
+标签的值在运行期动态创建,则label被称为动态的label。
+
+动态label的例子:
+
+```cpp
+{"method", "url"}
+```
+这个label只有键没有值,所以这个label是动态的label。后续动态生成label对应的值时,这样做:
+```cpp
+{"GET", "/"}
+{"POST", "/test"}
+```
+使用的时候填动态的方法名和url就行了:
+```cpp
+some_counter.inc({std::string(req.method()), req.url()}, 1);
+```
+如果传入的标签值数量和创建时的label 键的数量不匹配时则会抛异常。
+
+静态label的例子:
+```cpp
+{{"method", "GET"}, {"url", "/"}}
+```
+这个label的键值都确定了,是静态的,后面使用的时候需要显式调用静态的标签值使用:
+```cpp
+some_counter.inc({"GET", "/"}, 1);
+```
+
+无标签:创建指标的时候不设置标签,内部只有一个计数。
+
+# counter和gauge的使用
+
+## 创建没有标签的指标
+```cpp
+ gauge_t g{"test_gauge", "help"};
+ g.inc();
+ g.inc(1);
+
+ std::string str;
+ g.serialize(str);
+ CHECK(str.find("test_gauge 2") != std::string::npos);
+
+ g.dec(1);
+ CHECK(g.value() == 1);
+ g.update(1);
+
+ CHECK_THROWS_AS(g.dec({"test"}, 1), std::invalid_argument);
+ CHECK_THROWS_AS(g.inc({"test"}, 1), std::invalid_argument);
+ CHECK_THROWS_AS(g.update({"test"}, 1), std::invalid_argument);
+
+ counter_t c{"test_counter", "help"};
+ c.inc();
+ c.inc(1);
+ std::string str1;
+ c.serialize(str1);
+ CHECK(str1.find("test_counter 2") != std::string::npos);
+```
+## counter/gauge指标的api
+
+构造函数:
+```cpp
+// 无标签,调用inc时不带标签,如c.inc()
+// name: 指标对象的名称,注册到指标管理器时会使用这个名称
+// help: 指标对象的帮助信息
+counter_t(std::string name, std::string help);
+
+// labels: 静态标签,构造时需要将标签键值都填完整,如:{{"method", "GET"}, {"url", "/"}}
+// 调用inc时必须带静态标签的值,如:c.inc({"GET", "/"}, 1);
+counter_t(std::string name, std::string help,
+ std::map labels);
+
+// labels_name: 动态标签的键名称,因为标签的值是动态的,而键的名称是固定的,所以这里只需要填键名称,如: {"method", "url"}
+// 调用时inc时必须带动态标签的值,如:c.inc({method, url}, 1);
+counter_t(std::string name, std::string help,
+ std::vector labels_name);
+```
+
+基本函数:
+```cpp
+// 获取无标签指标的计数,
+double value();
+
+// 根据标签获取指标的计数,参数为动态或者静态标签的值
+double value(const std::vector &labels_value);
+
+// 无标签指标增加计数,counter的计数只能增加不能减少,如果填入的时负数时会抛异常;如果需要减少计数的指标则应该使用gauge;
+void inc(double val = 1);
+
+// 根据标签增加计数,如果创建的指标是静态标签值且和传入的标签值不匹配时会抛异常;如果标签的值的数量和构造指标时传入的标签数量不相等时会抛异常。
+void inc(const std::vector &labels_value, double value = 1);
+
+// 序列化,将指标序列化成prometheus 格式的字符串
+void serialize(std::string &str);
+
+// 返回带标签的指标内部的计数map,map的key是标签的值,值是对应计数,如:{{{"GET", "/"}, 100}, {{"POST", "/test"}, 20}}
+std::map, double,
+ std::less>>
+ value_map();
+```
+
+注意:如果使用动态标签的时候要注意这个动态的标签值是不是无限多的,如果是无限多的话,那么内部的map也会无限增长,应该避免这种情况,动态的标签也应该是有限的才对。
+
+gauge 派生于counter,相比counter多了一个减少计数的api
+```cpp
+// 无标签指标减少计数
+void dec(double value = 1);
+
+// 根据标签减少计数,如果创建的指标是静态标签值且和传入的标签值不匹配时会抛异常;如果标签的值的数量和构造指标时传入的标签数量不相等时会抛异常。
+void dec(const std::vector& labels_value, double value = 1);
+```
+
+# 基类公共函数
+所有指标都派生于metric_t 基类,提供了一些公共方法,如获取指标的名称,指标的类型,标签的键名称等等。
+
+```cpp
+class metric_t {
+ public:
+ // 获取指标对象的名称
+ std::string_view name();
+
+ // 获取指标对象的help 信息
+ std::string_view help();
+
+ // 指标类型
+ enum class MetricType {
+ Counter,
+ Gauge,
+ Histogram,
+ Summary,
+ Nil,
+ };
+
+ // 获取指标类型
+ MetricType metric_type();
+
+ // 获取指标类型的名称,比如counter, gauge, histogram和summary
+ std::string_view metric_name();
+
+ // 获取标签的键,如{"method", "url"}
+ const std::vector& labels_name();
+
+ // 序列化,调用派生类实现序列化
+ virtual void serialize(std::string& str);
+
+ // 给summary专用的api,序列化,调用派生类实现序列化
+ virtual async_simple::coro::Lazy serialize_async(std::string& out);
+
+ // 将基类指针向下转换到派生类指针,如:
+ // std::shared_ptr c = std::make_shared("test", "test");
+ // counter_t* t = c->as();
+ // t->value();
+ template
+ T* as();
+};
+```
+
+# 指标管理器
+如果希望集中管理多个指标时,则需要将指标注册到指标管理器,后面则可以多态调用管理器中的多个指标将各自的计数输出出来。
+
+**推荐在一开始就创建所有的指标并注册到管理器**,后面就可以无锁方式根据指标对象的名称来获取指标对象了。
+
+```cpp
+auto c = std::make_shared("qps_count", "qps help");
+auto g = std::make_shared("fd_count", "fd count help");
+default_metric_manager::register_metric_static(c);
+default_metric_manager::register_metric_static(g);
+
+c->inc();
+g->inc();
+
+auto m = default_metric_manager::get_metric_static("qps_count");
+CHECK(m->as()->value() == 1);
+
+auto m1 = default_metric_manager::get_metric_static("fd_count");
+CHECK(m1->as()->value() == 1);
+```
+
+如果希望动态注册的到管理器则应该调用register_metric_dynamic接口,后面根据名称获取指标对象时则调用get_metric_dynamic接口,dynamic接口内部会加锁。
+```cpp
+auto c = std::make_shared("qps_count", "qps help");
+auto g = std::make_shared("fd_count", "fd count help");
+default_metric_manager::register_metric_dynamic(c);
+default_metric_manager::register_metric_dynamic(g);
+
+c->inc();
+g->inc();
+
+auto m = default_metric_manager::get_metric_dynamic("qps_count");
+CHECK(m->as()->value() == 1);
+
+auto m1 = default_metric_manager::get_metric_dynamic("fd_count");
+CHECK(m1->as()->value() == 1);
+```
+注意:一旦注册时使用了static或者dynamic,那么后面调用default_metric_manager时则应该使用相同后缀的接口,比如注册时使用了get_metric_static,那么后面调用根据名称获取指标对象的方法必须是get_metric_static,否则会抛异常。同样,如果注册使用register_metric_dynamic,则后面应该get_metric_dynamic,否则会抛异常。
+
+指标管理器的api
+```cpp
+template
+struct metric_manager_t {
+ // 创建并注册指标,返回注册的指标对象
+ template
+ static std::shared_ptr create_metric_static(const std::string& name,
+ const std::string& help,
+ Args&&... args);
+ template
+ static std::shared_ptr create_metric_dynamic(const std::string& name,
+ const std::string& help,
+ Args&&... args)
+ // 注册metric
+ static bool register_metric_static(std::shared_ptr metric);
+ static bool register_metric_dynamic(std::shared_ptr metric);
+
+ // 获取注册的所有指标对象
+ static std::map> metric_map_static();
+ static std::map> metric_map_dynamic();
+
+ // 获取注册的指标对象的总数
+ static size_t metric_count_static();
+ static size_t metric_count_dynamic();
+
+ // 获取注册的指标对象的名称
+ static std::vector metric_keys_static();
+ static std::vector metric_keys_dynamic();
+
+ // 根据名称获取指标对象,T为具体指标的类型,如 get_metric_static();
+ // 如果找不到则返回nullptr
+ template
+ static T* get_metric_static(const std::string& name);
+ template
+ static T* get_metric_static(const std::string& name);
+
+ static std::shared_ptr get_metric_static(const std::string& name);
+ static std::shared_ptr get_metric_dynamic(const std::string& name);
+
+ // 序列化
+ static async_simple::coro::Lazy serialize_static();
+ static async_simple::coro::Lazy serialize_dynamic();
+};
+using default_metric_manager = metric_manager_t<0>;
+```
+metric_manager_t默认为default_metric_manager,如果希望有多个metric manager,用户可以自定义新的metric manager,如:
+
+```cpp
+constexpr size_t metric_id = 100;
+using my_metric_manager = metric_manager_t;
+```
+
+# histogram
+
+## api
+```cpp
+//
+// name: 对象名称,help:帮助信息
+// buckets:桶,如 {1, 3, 7, 11, 23},后面设置的值会自动落到某个桶中并增加该桶的计数;
+// 内部还有一个+Inf 默认的桶,当输入的数据不在前面设置这些桶中,则会落到+Inf 默认桶中。
+// 实际上桶的总数为 buckets.size() + 1
+// 每个bucket 实际上对应了一个counter指标
+histogram_t(std::string name, std::string help, std::vector buckets);
+
+// 往histogram_t 中插入数据,内部会自动增加对应桶的计数
+void observe(double value);
+
+// 获取所有桶对应的counter指标对象
+std::vector> get_bucket_counts();
+
+// 序列化
+void serialize(std::string& str);
+```
+## 例子
+```cpp
+ histogram_t h("test", "help", {5.0, 10.0, 20.0, 50.0, 100.0});
+ h.observe(23);
+ auto counts = h.get_bucket_counts();
+ CHECK(counts[3]->value() == 1);
+ h.observe(42);
+ CHECK(counts[3]->value() == 2);
+ h.observe(60);
+ CHECK(counts[4]->value() == 1);
+ h.observe(120);
+ CHECK(counts[5]->value() == 1);
+ h.observe(1);
+ CHECK(counts[0]->value() == 1);
+ std::string str;
+ h.serialize(str);
+ std::cout << str;
+ CHECK(str.find("test_count") != std::string::npos);
+ CHECK(str.find("test_sum") != std::string::npos);
+ CHECK(str.find("test_bucket{le=\"5") != std::string::npos);
+ CHECK(str.find("test_bucket{le=\"+Inf\"}") != std::string::npos);
+```
+
+创建Histogram时需要指定桶(bucket),采样点统计数据会落到不同的桶中,并且还需要统计采样点数据的累计总和(sum)以及次数的总和(count)。注意bucket 列表必须是有序的,否则构造时会抛异常。
+
+Histogram统计的特点是:数据是累积的,比如由10, 100,两个桶,第一个桶的数据是所有值 <= 10的样本数据存在桶中,第二个桶是所有 <=100 的样本数据存在桶中,其它数据则存放在`+Inf`的桶中。
+
+```cpp
+ auto h = std::make_shared(
+ std::string("test"), std::string("help"), std::vector{10.0, 100.0});
+ metric_t::regiter_metric(h);
+
+ h->observe(5);
+ h->observe(80);
+ h->observe(120);
+
+ std::string str;
+ h.serialize(str);
+ std::cout<> get_rates();
+
+// 获取总和
+async_simple::coro::Lazy get_sum();
+
+// 获取插入数据的个数
+async_simple::coro::Lazy get_count();
+
+// 序列化
+async_simple::coro::Lazy serialize_async(std::string &str);
+```
+
+## 例子
+```cpp
+ summary_t summary{"test_summary",
+ "summary help",
+ {{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}};
+ std::random_device rd;
+ std::mt19937 gen(rd());
+ std::uniform_int_distribution<> distr(1, 100);
+ for (int i = 0; i < 50; i++) {
+ summary.observe(distr(gen));
+ }
+
+ std::this_thread::sleep_for(std::chrono::milliseconds(10));
+ std::string str;
+ async_simple::coro::syncAwait(summary.serialize_async(str));
+ std::cout << str;
+ CHECK(async_simple::coro::syncAwait(summary.get_count()) == 50);
+ CHECK(async_simple::coro::syncAwait(summary.get_sum()) > 0);
+ CHECK(str.find("test_summary") != std::string::npos);
+ CHECK(str.find("test_summary_count") != std::string::npos);
+ CHECK(str.find("test_summary_sum") != std::string::npos);
+ CHECK(str.find("test_summary{quantile=\"") != std::string::npos);
+```
+summary 百分位的计算相比其它指标是最耗时的,应该避免在关键路径上使用它以免对性能造成影响。
+
+创建Summary时需要指定分位数和误差,分位数在0到1之间,左右都为闭区间,比如p50就是一个中位数,p99指中位数为0.99的分位数。
+```cpp
+ summary_t summary{"test_summary",
+ "summary help",
+ {{0.5, 0.05}, {0.9, 0.01}, {0.95, 0.005}, {0.99, 0.001}}};
+ std::random_device rd;
+ std::mt19937 gen(rd());
+ std::uniform_int_distribution<> distr(1, 100);
+ for (int i = 0; i < 50; i++) {
+ summary.observe(distr(gen));
+ }
+
+ std::string str;
+ summary.serialize(str);
+ std::cout << str;
+```
+输出:
+```
+# HELP test_summary summary help
+# TYPE test_summary summary
+test_summary{quantile="0.500000"} 45.000000
+test_summary{quantile="0.900000"} 83.000000
+test_summary{quantile="0.950000"} 88.000000
+test_summary{quantile="0.990000"} 93.000000
+test_summary_sum 2497.000000
+test_summary_count 50
+```
+
+## 配置prometheus 前端
+安装[prometheus](https://github.com/prometheus/prometheus)之后,打开其配置文件:prometheus.yml
+
+修改要连接的服务端地址:
+```
+- targets: ["127.0.0.1:9001"]
+```
+然后启动prometheus,prometheus会定时访问`http://127.0.0.1:9001/metrics` 拉取所有指标数据。
+
+在本地浏览器输入:127.0.0.1:9090, 打开prometheus前端,在前端页面的搜索框中输入指标的名称request_count之后就能看到table和graph 结果了。
+
+# cinatra http server中启用内置的metric指标
+
+http server 内置的指标:
+```cpp
+server_total_req: server总的请求数;
+server_failed_req:server总的失败请求数;
+server_total_fd:server使用的总的句柄数;
+server_total_recv_bytes:server总共收到的字节数;
+server_total_send_bytes:server总共发送的字节数;
+server_req_latency:http 请求的延迟,从收到请求到发送响应的时间间隔
+server_read_latency:http 读请求的延迟,读到完整的http数据的时间间隔
+```
+
+```cpp
+coro_http_server server(1, 9001);
+server.use_metrics("/metrics");//这个url默认就是/metrics,可以不填
+```
+在浏览器中输入`http://127.0.0.1:9001/metrics` 即可看到所有的指标。
+
+查看当前server的client pool中有多少client,调用`pool.free_client_count()`
+
+查看当前server内部线程池中有多少线程,调用`coro_io::get_total_thread_num()`
\ No newline at end of file