Skip to content

A platform for community discussion. Free, open, simple.

DEV: Add per-test SQL/Redis/network capture RSpec formatter (#40632)

Alan Guo Xiang Tan71a6dd9674ea
diff --git a/bin/rspec b/bin/rspec
index 6ecd833153a..054240e05d5 100755
--- a/bin/rspec
+++ b/bin/rspec
@@ -29,4 +29,8 @@ end
ENV["PLAYWRIGHT_HEADLESS"] = "0" if ARGV.delete("--headful")
ENV["CAPYBARA_PLAYWRIGHT_DEBUG_CLIENT_SETTLED"] = "1" if ARGV.delete("--debug-client-settled")
+if ARGV.any? { |arg| arg.include?("RspecPerformanceFormatter") }
+ ENV["DISCOURSE_RSPEC_PERFORMANCE_FORMATTER"] = "1"
+end
+
load Gem.bin_path("rspec-core", "rspec")
diff --git a/bin/turbo_rspec b/bin/turbo_rspec
index 2f4e6ad9a4d..94c721eac00 100755
--- a/bin/turbo_rspec
+++ b/bin/turbo_rspec
@@ -105,6 +105,11 @@ formatters << { name: "progress", outputs: [] } if formatters.empty?
formatters.each { |formatter| formatter[:outputs] << "-" if formatter[:outputs].empty? }
+if formatters.any? { |formatter| formatter[:name] == "RspecPerformanceFormatter" }
+ ENV["DISCOURSE_RSPEC_PERFORMANCE_FORMATTER"] = "1"
+ require "./spec/support/rspec_performance_formatter"
+end
+
puts "::group::Run turbo_rspec" if ENV["GITHUB_ACTIONS"]
if files.size > 5
puts "Running turbo_rspec on #{files.size} files"
diff --git a/lib/method_profiler.rb b/lib/method_profiler.rb
index 33acad54e7a..940afe95359 100644
--- a/lib/method_profiler.rb
+++ b/lib/method_profiler.rb
@@ -2,6 +2,14 @@
# see https://samsaffron.com/archive/2017/10/18/fastest-way-to-profile-a-method-in-ruby
class MethodProfiler
+ def self.register_instrumentation_callback(callback)
+ @instrumentation_callback = callback
+ end
+
+ def self.instrumentation_callback
+ @instrumentation_callback
+ end
+
def self.patch(klass, methods, name, no_recurse: false)
patches =
methods
@@ -24,9 +32,11 @@ class MethodProfiler
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
#{method_name}__mp_unpatched(*args, **kwargs, &blk)
ensure
+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
data = (prof[:#{name}] ||= {duration: 0.0, calls: 0})
- data[:duration] += Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+ data[:duration] += elapsed
data[:calls] += 1
+ MethodProfiler.instrumentation_callback&.call(:#{name}, self, args, data, elapsed)
#{"@mp_recurse_protect_#{method_name} = false" if no_recurse}
end
end
diff --git a/lib/turbo_tests/json_example.rb b/lib/turbo_tests/json_example.rb
index 13b9b7b45e5..a43e04f59f0 100644
--- a/lib/turbo_tests/json_example.rb
+++ b/lib/turbo_tests/json_example.rb
@@ -21,6 +21,7 @@ module TurboTests
js_deprecations: @rspec_example.metadata[:js_deprecations],
rerun_file_path: @rspec_example.metadata[:rerun_file_path],
active_record_debug_logs: @rspec_example.metadata[:active_record_debug_logs],
+ perf: @rspec_example.metadata[:perf],
},
location_rerun_argument: @rspec_example.location_rerun_argument,
}
diff --git a/spec/support/method_profiler_itemizer.rb b/spec/support/method_profiler_itemizer.rb
new file mode 100644
index 00000000000..b66289c73d6
--- /dev/null
+++ b/spec/support/method_profiler_itemizer.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+# Records each individual SQL/Redis/network call for the perf formatter. It is
+# registered via MethodProfiler.register_instrumentation_callback in the test
+# environment only, so production instrumentation carries no itemization logic.
+module MethodProfilerItemizer
+ MAX_ITEM_LENGTH = 2000
+ private_constant :MAX_ITEM_LENGTH
+
+ class << self
+ def call(name, receiver, args, data, elapsed)
+ item = build_item(name, receiver, args)
+ return if item.nil?
+ (data[:items] ||= []) << item.merge!(duration_ms: elapsed * 1000.0)
+ rescue => error
+ (data[:items] ||= []) << { error: truncate(utf8("#{error.class}: #{error.message}")) }
+ end
+
+ private
+
+ def build_item(name, receiver, args)
+ case name
+ when :sql
+ { sql: truncate(utf8(args[0])) }
+ when :redis
+ redis_item(args)
+ when :net
+ net_item(receiver, args)
+ end
+ end
+
+ def redis_item(args)
+ command = args[0]
+ return { command: truncate(utf8(command)) } unless command.is_a?(Array)
+ commands = command.first.is_a?(Array) ? command : [command]
+ { command: truncate(commands.map { |entry| redis_command(entry) }.join("; ")) }
+ end
+
+ def redis_command(command)
+ return utf8(command) unless command.is_a?(Array)
+ [utf8(command.first).upcase, *Array(command[1..]).map { |arg| utf8(arg) }].join(" ").strip
+ end
+
+ def net_item(receiver, args)
+ if defined?(Net::HTTP) && receiver.is_a?(Net::HTTP)
+ request = args[0]
+ url = http_url(receiver.use_ssl?, receiver.address, receiver.port, request.path)
+ { method: utf8(request.method), url: truncate(utf8(url)) }
+ elsif defined?(Excon::Connection) && receiver.is_a?(Excon::Connection)
+ params = args[0] || {}
+ data = receiver.respond_to?(:data) ? receiver.data.to_h : {}
+ url =
+ http_url(
+ data[:scheme].to_s == "https",
+ data[:host],
+ data[:port],
+ params[:path] || data[:path],
+ )
+ { method: utf8(params[:method] || data[:method]).upcase, url: truncate(utf8(url)) }
+ else
+ { method: "", url: "" }
+ end
+ end
+
+ def http_url(ssl, host, port, path)
+ scheme = ssl ? "https" : "http"
+ default_port = ssl ? 443 : 80
+ authority = port.nil? || port == default_port ? host : "#{host}:#{port}"
+ "#{scheme}://#{authority}#{path}"
+ end
+
+ def utf8(value)
+ value.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?")
+ end
+
+ def truncate(string)
+ return string if string.length <= MAX_ITEM_LENGTH
+ "#{string[0, MAX_ITEM_LENGTH]}…(truncated, #{string.bytesize} bytes)"
+ end
+ end
+end
+
+if ENV["DISCOURSE_RSPEC_PERFORMANCE_FORMATTER"] == "1"
+ MethodProfiler.register_instrumentation_callback(MethodProfilerItemizer)
+end
diff --git a/spec/support/perf_capture.rb b/spec/support/perf_capture.rb
new file mode 100644
index 00000000000..8ab756e33db
--- /dev/null
+++ b/spec/support/perf_capture.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+if ENV["DISCOURSE_RSPEC_PERFORMANCE_FORMATTER"] == "1"
+ require_relative "rspec_performance_formatter"
+
+ RSpec.configure do |config|
+ config.before(:suite) { MethodProfiler.ensure_discourse_instrumentation! }
+
+ config.around(:each) do |example|
+ test_summary = nil
+ request_groups =
+ RspecPerformanceFormatter.collect_requests do
+ test_summary = RspecPerformanceFormatter.measure { example.run }
+ end
+
+ example.metadata[:perf] = RspecPerformanceFormatter.assemble(test_summary, request_groups)
+ end
+ end
+end
diff --git a/spec/support/rspec_performance_formatter.rb b/spec/support/rspec_performance_formatter.rb
new file mode 100644
index 00000000000..0fb6b94dfd7
--- /dev/null
+++ b/spec/support/rspec_performance_formatter.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+class RspecPerformanceFormatter
+ RSpec::Core::Formatters.register(self, :example_passed, :example_failed, :example_pending)
+
+ class << self
+ def measure
+ MethodProfiler.ensure_discourse_instrumentation!
+ MethodProfiler.start
+ yield
+ summarize(MethodProfiler.stop)
+ ensure
+ MethodProfiler.clear
+ end
+
+ def collect_requests
+ groups = []
+ logger = ->(env, data) do
+ group = request_group(env, data)
+ groups << group if group
+ rescue => error
+ groups << summarize(nil).merge(error: "#{error.class}: #{error.message}")
+ end
+ Middleware::RequestTracker.register_detailed_request_logger(logger)
+ yield
+ groups
+ ensure
+ Middleware::RequestTracker.unregister_detailed_request_logger(logger)
+ end
+
+ def assemble(test_summary, request_groups)
+ all_totals = [test_summary[:totals], *request_groups.map { |group| group[:totals] }]
+ {
+ totals: combine_totals(all_totals),
+ sql: test_summary[:sql],
+ redis: test_summary[:redis],
+ net: test_summary[:net],
+ requests: request_groups,
+ }
+ end
+
+ def serialize(example, perf)
+ perf = normalize(perf)
+ execution = example.execution_result
+ result = {
+ example_id: example.location_rerun_argument,
+ description: example.full_description,
+ location: example.location,
+ status: execution.status.to_s,
+ totals: perf[:totals],
+ sql: perf[:sql],
+ redis: perf[:redis],
+ net: perf[:net],
+ requests: perf[:requests],
+ }
+ result[:error] = serialize_error(execution.exception) if execution.exception
+ result
+ end
+
+ private
+
+ def summarize(timing)
+ timing ||= {}
+ sql = items(timing, :sql)
+ redis = items(timing, :redis)
+ net = items(timing, :net)
+ { totals: { sql: totals(sql), redis: totals(redis), net: totals(net) }, sql:, redis:, net: }
+ end
+
+ def request_group(env, data)
+ return if data[:is_background]
+ { method: env["REQUEST_METHOD"], path: env["PATH_INFO"], status: data[:status] }.merge(
+ summarize(data[:timing]),
+ )
+ end
+
+ def normalize(perf)
+ return summarize(nil).merge(requests: []) if perf.blank?
+ perf.deep_symbolize_keys
+ end
+
+ def serialize_error(exception)
+ {
+ class: exception.class.name,
+ message: exception.message.to_s,
+ backtrace: Array(exception.backtrace).first(10),
+ }
+ end
+
+ def items(timing, key)
+ timing[key]&.dig(:items) || []
+ end
+
+ def totals(items)
+ { calls: items.size, duration_ms: items.sum { |item| item[:duration_ms] || 0.0 }.to_f }
+ end
+
+ def combine_totals(all_totals)
+ %i[sql redis net].each_with_object({}) do |key, result|
+ result[key] = {
+ calls: all_totals.sum { |category_totals| category_totals[key][:calls] },
+ duration_ms: all_totals.sum { |category_totals| category_totals[key][:duration_ms] }.to_f,
+ }
+ end
+ end
+ end
+
+ def initialize(output)
+ @output = output
+ end
+
+ def example_passed(notification)
+ write(notification.example)
+ end
+
+ def example_failed(notification)
+ write(notification.example)
+ end
+
+ def example_pending(notification)
+ write(notification.example)
+ end
+
+ private
+
+ def write(example)
+ @output.puts(JSON.generate(self.class.serialize(example, example.metadata[:perf])))
+ rescue => error
+ @output.puts(
+ JSON.generate(
+ example_id: example.location_rerun_argument,
+ status: example.execution_result.status.to_s,
+ error: {
+ class: error.class.name,
+ message: error.message.to_s,
+ },
+ ),
+ )
+ end
+end