Add an optional metric exporter (#33734)

Gemfile

@@ -100,6 +100,8 @@ gem 'json-ld'
 gem 'json-ld-preloaded', '~> 3.2'
 gem 'rdf-normalize', '~> 0.5'
 
+gem 'prometheus_exporter', '~> 2.2', require: false
+
 gem 'opentelemetry-api', '~> 1.4.0'
 
 group :opentelemetry do

Gemfile.lock

@@ -580,6 +580,8 @@ GEM
       net-smtp
       premailer (~> 1.7, >= 1.7.9)
     prettyprint (0.2.0)
+    prometheus_exporter (2.2.0)
+      webrick
     propshaft (1.1.0)
       actionpack (>= 7.0.0)
       activesupport (>= 7.0.0)
@@ -984,6 +986,7 @@ DEPENDENCIES
   pg (~> 1.5)
   pghero
   premailer-rails
+  prometheus_exporter (~> 2.2)
   propshaft
   public_suffix (~> 6.0)
   puma (~> 6.3)

bin/prometheus_exporter

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'prometheus_exporter' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
+
+bundle_binstub = File.expand_path("bundle", __dir__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300).include?("This file was generated by Bundler")
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("prometheus_exporter", "prometheus_exporter")

config/initializers/prometheus_exporter.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+if ENV['MASTODON_PROMETHEUS_EXPORTER_ENABLED'] == 'true'
+  if ENV['MASTODON_PROMETHEUS_EXPORTER_LOCAL'] == 'true'
+    require 'prometheus_exporter/server'
+    require 'prometheus_exporter/client'
+
+    # bind is the address, on which the webserver will listen
+    # port is the port that will provide the /metrics route
+    server = PrometheusExporter::Server::WebServer.new bind: ENV.fetch('MASTODON_PROMETHEUS_EXPORTER_HOST', 'localhost'), port: ENV.fetch('MASTODON_PROMETHEUS_EXPORTER_PORT', '9394').to_i
+    server.start
+
+    # wire up a default local client
+    PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(collector: server.collector)
+  end
+
+  if ENV['MASTODON_PROMETHEUS_EXPORTER_WEB_DETAILED_METRICS'] == 'true'
+    # Optional, as those metrics might generate extra overhead and be redundant with what OTEL provides
+    require 'prometheus_exporter/middleware'
+
+    # Per-action/controller request stats like HTTP status and timings
+    Rails.application.middleware.unshift PrometheusExporter::Middleware
+  end
+end

config/initializers/sidekiq.rb

@@ -22,6 +22,48 @@ Sidekiq.configure_server do |config|
     end
   end
 
+  if ENV['MASTODON_PROMETHEUS_EXPORTER_ENABLED'] == 'true'
+    require 'prometheus_exporter'
+    require 'prometheus_exporter/instrumentation'
+
+    config.on :startup do
+      # Ruby process metrics (memory, GC, etc)
+      PrometheusExporter::Instrumentation::Process.start type: 'sidekiq'
+
+      # Sidekiq process metrics (concurrency, busy, etc)
+      PrometheusExporter::Instrumentation::SidekiqProcess.start
+
+      # ActiveRecord metrics (connection pool usage)
+      PrometheusExporter::Instrumentation::ActiveRecord.start(
+        custom_labels: { type: 'sidekiq' },
+        config_labels: [:database, :host]
+      )
+
+      if ENV['MASTODON_PROMETHEUS_EXPORTER_SIDEKIQ_DETAILED_METRICS'] == 'true'
+        # Optional, as those metrics might generate extra overhead and be redundant with what OTEL provides
+
+        # Per-job metrics
+        config.server_middleware do |chain|
+          chain.add PrometheusExporter::Instrumentation::Sidekiq
+        end
+        config.death_handlers << PrometheusExporter::Instrumentation::Sidekiq.death_handler
+
+        # Per-queue metrics for queues handled by this process (size, latency, etc)
+        # They will be reported by every process handling those queues, so do not sum them up
+        PrometheusExporter::Instrumentation::SidekiqQueue.start
+
+        # Global Sidekiq metrics (size of the global queues, number of jobs, etc)
+        # Will be the same for every Sidekiq process
+        PrometheusExporter::Instrumentation::SidekiqStats.start
+      end
+    end
+
+    at_exit do
+      # Wait for the latest metrics to be reported before shutting down
+      PrometheusExporter::Client.default.stop(wait_timeout_seconds: 10)
+    end
+  end
+
   config.server_middleware do |chain|
     chain.add Mastodon::SidekiqMiddleware
   end

config/puma.rb

@@ -17,6 +17,27 @@ workers     ENV.fetch('WEB_CONCURRENCY') { 2 }.to_i
 
 preload_app!
 
+if ENV['MASTODON_PROMETHEUS_EXPORTER_ENABLED'] == 'true'
+  require 'prometheus_exporter'
+  require 'prometheus_exporter/instrumentation'
+
+  on_worker_boot do
+    # Ruby process metrics (memory, GC, etc)
+    PrometheusExporter::Instrumentation::Process.start(type: 'puma')
+
+    # ActiveRecord metrics (connection pool usage)
+    PrometheusExporter::Instrumentation::ActiveRecord.start(
+      custom_labels: { type: 'puma' }, # optional params
+      config_labels: [:database, :host] # optional params
+    )
+  end
+
+  after_worker_boot do
+    # Puma metrics
+    PrometheusExporter::Instrumentation::Puma.start unless PrometheusExporter::Instrumentation::Puma.started?
+  end
+end
+
 on_worker_boot do
   ActiveSupport.on_load(:active_record) do
     ActiveRecord::Base.establish_connection