Add POC OpenTelemetry instrumentation for Solid Process

.gitignore

@@ -41,3 +41,5 @@
 
 /node_modules
 /.yarn
+
+/volumes/tempo/data/*

Gemfile

@@ -30,6 +30,11 @@ gem "tzinfo-data", platforms: %i[windows jruby]
 # Reduces boot times through caching; required in config/boot.rb
 gem "bootsnap", require: false
 
+# OpenTelemetry tracing
+gem "opentelemetry-sdk"
+gem "opentelemetry-instrumentation-all"
+gem "opentelemetry-exporter-otlp"
+
 group :development, :test do
   # See https://guides.rubyonrails.org/debugging_rails_applications.html#debugging-with-the-debug-gem
   gem "debug", platforms: %i[mri windows]

Gemfile.lock

@@ -176,6 +176,9 @@ GEM
     formtastic_i18n (0.7.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
+    google-protobuf (3.25.3-x86_64-linux)
+    googleapis-common-protos-types (1.14.0)
+      google-protobuf (~> 3.18)
     has_scope (0.8.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
@@ -239,18 +242,211 @@ GEM
     net-smtp (0.5.0)
       net-protocol
     nio4r (2.7.1)
-    nokogiri (1.16.4-aarch64-linux)
-      racc (~> 1.4)
-    nokogiri (1.16.4-arm-linux)
-      racc (~> 1.4)
-    nokogiri (1.16.4-arm64-darwin)
-      racc (~> 1.4)
-    nokogiri (1.16.4-x86-linux)
-      racc (~> 1.4)
-    nokogiri (1.16.4-x86_64-darwin)
-      racc (~> 1.4)
     nokogiri (1.16.4-x86_64-linux)
       racc (~> 1.4)
+    opentelemetry-api (1.2.5)
+    opentelemetry-common (0.20.1)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-otlp (0.26.3)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-helpers-mysql (0.1.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20)
+    opentelemetry-helpers-sql-obfuscation (0.1.0)
+      opentelemetry-common (~> 0.20)
+    opentelemetry-instrumentation-action_pack (0.9.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-instrumentation-rack (~> 0.21)
+    opentelemetry-instrumentation-action_view (0.7.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-active_support (~> 0.1)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-active_job (0.7.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-active_model_serializers (0.20.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-active_record (0.7.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-active_support (0.5.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-all (0.60.0)
+      opentelemetry-instrumentation-active_model_serializers (~> 0.20.1)
+      opentelemetry-instrumentation-aws_sdk (~> 0.5.0)
+      opentelemetry-instrumentation-bunny (~> 0.21.0)
+      opentelemetry-instrumentation-concurrent_ruby (~> 0.21.1)
+      opentelemetry-instrumentation-dalli (~> 0.25.0)
+      opentelemetry-instrumentation-delayed_job (~> 0.22.0)
+      opentelemetry-instrumentation-ethon (~> 0.21.1)
+      opentelemetry-instrumentation-excon (~> 0.22.0)
+      opentelemetry-instrumentation-faraday (~> 0.24.0)
+      opentelemetry-instrumentation-grape (~> 0.1.3)
+      opentelemetry-instrumentation-graphql (~> 0.28.0)
+      opentelemetry-instrumentation-gruf (~> 0.2.0)
+      opentelemetry-instrumentation-http (~> 0.23.1)
+      opentelemetry-instrumentation-http_client (~> 0.22.1)
+      opentelemetry-instrumentation-koala (~> 0.20.1)
+      opentelemetry-instrumentation-lmdb (~> 0.22.1)
+      opentelemetry-instrumentation-mongo (~> 0.22.1)
+      opentelemetry-instrumentation-mysql2 (~> 0.27.0)
+      opentelemetry-instrumentation-net_http (~> 0.22.1)
+      opentelemetry-instrumentation-pg (~> 0.27.0)
+      opentelemetry-instrumentation-que (~> 0.8.0)
+      opentelemetry-instrumentation-racecar (~> 0.3.0)
+      opentelemetry-instrumentation-rack (~> 0.24.0)
+      opentelemetry-instrumentation-rails (~> 0.30.0)
+      opentelemetry-instrumentation-rake (~> 0.2.1)
+      opentelemetry-instrumentation-rdkafka (~> 0.4.0)
+      opentelemetry-instrumentation-redis (~> 0.25.1)
+      opentelemetry-instrumentation-resque (~> 0.5.0)
+      opentelemetry-instrumentation-restclient (~> 0.22.1)
+      opentelemetry-instrumentation-ruby_kafka (~> 0.21.0)
+      opentelemetry-instrumentation-sidekiq (~> 0.25.0)
+      opentelemetry-instrumentation-sinatra (~> 0.23.1)
+      opentelemetry-instrumentation-trilogy (~> 0.59.0)
+    opentelemetry-instrumentation-aws_sdk (0.5.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-base (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-bunny (0.21.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-concurrent_ruby (0.21.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-dalli (0.25.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-delayed_job (0.22.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-ethon (0.21.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-excon (0.22.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-faraday (0.24.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-grape (0.1.6)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-instrumentation-rack (~> 0.21)
+    opentelemetry-instrumentation-graphql (0.28.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-gruf (0.2.0)
+      opentelemetry-api (>= 1.0.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-http (0.23.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-http_client (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-koala (0.20.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-lmdb (0.22.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-mongo (0.22.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-mysql2 (0.27.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-helpers-mysql
+      opentelemetry-helpers-sql-obfuscation
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-net_http (0.22.4)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-pg (0.27.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-helpers-sql-obfuscation
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-que (0.8.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-racecar (0.3.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rack (0.24.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rails (0.30.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-action_pack (~> 0.9.0)
+      opentelemetry-instrumentation-action_view (~> 0.7.0)
+      opentelemetry-instrumentation-active_job (~> 0.7.0)
+      opentelemetry-instrumentation-active_record (~> 0.7.0)
+      opentelemetry-instrumentation-active_support (~> 0.5.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rake (0.2.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-rdkafka (0.4.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-redis (0.25.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-resque (0.5.1)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-restclient (0.22.3)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-ruby_kafka (0.21.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-sidekiq (0.25.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+    opentelemetry-instrumentation-sinatra (0.23.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.20.0)
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-instrumentation-rack (~> 0.21)
+    opentelemetry-instrumentation-trilogy (0.59.2)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-helpers-mysql
+      opentelemetry-helpers-sql-obfuscation
+      opentelemetry-instrumentation-base (~> 0.22.1)
+      opentelemetry-semantic_conventions (>= 1.8.0)
+    opentelemetry-registry (0.3.1)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.4.1)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.10.0)
+      opentelemetry-api (~> 1.0)
     parallel (1.24.0)
     parser (3.3.0.5)
       ast (~> 2.4.1)
@@ -376,11 +572,6 @@ GEM
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     solid-result (2.0.0)
-    sqlite3 (1.7.3-aarch64-linux)
-    sqlite3 (1.7.3-arm-linux)
-    sqlite3 (1.7.3-arm64-darwin)
-    sqlite3 (1.7.3-x86-linux)
-    sqlite3 (1.7.3-x86_64-darwin)
     sqlite3 (1.7.3-x86_64-linux)
     standard (1.35.1)
       language_server-protocol (~> 3.17.0.2)
@@ -424,11 +615,6 @@ GEM
     zeitwerk (2.6.13)
 
 PLATFORMS
-  aarch64-linux
-  arm-linux
-  arm64-darwin
-  x86-linux
-  x86_64-darwin
   x86_64-linux
 
 DEPENDENCIES
@@ -440,6 +626,9 @@ DEPENDENCIES
   debug
   importmap-rails
   letter_opener (~> 1.9)
+  opentelemetry-exporter-otlp
+  opentelemetry-instrumentation-all
+  opentelemetry-sdk
   propshaft (~> 0.8.0)
   puma (>= 5.0)
   rails (~> 7.1.3, >= 7.1.3.2)

compose.dev.yml

@@ -0,0 +1,29 @@
+services:
+  grafana:
+    image: grafana/grafana:10.4.2
+    ports:
+      - "3001:3000"
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+    volumes:
+      - ./volumes/grafana/grafana.ini:/etc/grafana/grafana.ini
+      - ./volumes/grafana/provisioning:/etc/grafana/provisioning
+
+  tempo:
+    image: grafana/tempo:2.4.1
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    ports:
+      - "3200:3200"   # tempo metrics
+      - "4317:4317"  # otlp http
+    volumes:
+      - ./volumes/tempo/tempo.yml:/etc/tempo.yaml
+      - ./volumes/tempo/data:/tmp/tempo
+
+  otel-collector:
+    image: otel/opentelemetry-collector:0.98.0
+    command: [ "--config=/etc/otelcol-config.yml" ]
+    volumes:
+      - ./volumes/otelcol-config.yml:/etc/otelcol-config.yml
+    ports:
+      - "4318:4318"          # OTLP over HTTP receiver

config/initializers/opentelemetry.rb

@@ -0,0 +1,8 @@
+require "opentelemetry/sdk"
+require "opentelemetry/instrumentation/all"
+require "opentelemetry-exporter-otlp"
+
+OpenTelemetry::SDK.configure do |c|
+  c.service_name = "solid-rails-app"
+  c.use_all # enables all instrumentation!
+end

config/initializers/solid_process.rb

@@ -9,11 +9,17 @@
 require "solid/process/event_logs/json_logger_listener"
 # require "solid/process/event_logs/basic_logger_listener"
 
+require "solid/process/active_support_publisher"
+
+require "open_telemetry_tracer"
+OpenTelemetryTracer.subscribe
+
 # Solid::Process::EventLogs::BasicLoggerListener.logger = Rails.logger
 
 Solid::Result.configuration do |config|
   config.event_logs.listener = Solid::Result::EventLogs::Listeners[
     Solid::Process::EventLogs::JsonLoggerListener,
-    Solid::Process::EventLogs::Record::Listener
+    Solid::Process::EventLogs::Record::Listener,
+    Solid::Process::ActiveSupportPublisher
   ]
 end

lib/open_telemetry_tracer.rb

@@ -0,0 +1,53 @@
+module OpenTelemetryTracer
+  SolidResultTracer = ::OpenTelemetry.tracer_provider.tracer("solid-result")
+
+  module ProcessListener
+    def self.start(name, _id, payload)
+      scope = payload[:scope]
+
+      span = SolidResultTracer.start_span("#{scope[:name]}#call", attributes: {
+        "desc" => scope[:desc].inspect
+      })
+
+      token = OpenTelemetry::Context.attach(::OpenTelemetry::Trace.context_with_span(span))
+      payload.merge!(__otel: {span:, token:})
+    end
+
+    def self.finish(_name, _id, payload)
+      otel = payload.delete(:__otel)
+
+      otel => { span:, token: }
+
+      span.finish
+      OpenTelemetry::Context.detach(token)
+    end
+  end
+
+  module AndThenListener
+    def self.start(name, _id, payload)
+      payload => { scope:, and_then: }
+
+      span = SolidResultTracer.start_span("#{scope[:name]}##{and_then[:method_name] || "block"}", attributes: {
+        "type" => and_then[:type].to_s,
+        "arg" => and_then[:arg].inspect
+      })
+
+      token = OpenTelemetry::Context.attach(::OpenTelemetry::Trace.context_with_span(span))
+      payload.merge!(__otel: {span:, token:})
+    end
+
+    def self.finish(_name, _id, payload)
+      otel = payload.delete(:__otel)
+
+      otel => { span:, token: }
+
+      span.finish
+      OpenTelemetry::Context.detach(token)
+    end
+  end
+
+  def self.subscribe
+    ActiveSupport::Notifications.monotonic_subscribe("start_process.solid_process", ProcessListener)
+    ActiveSupport::Notifications.monotonic_subscribe("and_then.solid_process", AndThenListener)
+  end
+end