A platform for community discussion. Free, open, simple.
FEATURE: Instrument image-processing duration (#42753)
Alan Guo Xiang Tan5baa0d5f2d36
diff --git a/app/models/optimized_image.rb b/app/models/optimized_image.rbindex 0dd42161095..35f563403e0 100644--- a/app/models/optimized_image.rb+++ b/app/models/optimized_image.rb@@ -314,30 +314,36 @@ class OptimizedImage < ActiveRecord::Baseenddef self.resize(from, to, width, height, opts = {})- optimize("resize", from, to, "#{width}x#{height}", opts)+ optimize(:optimized_image_resize, from, to, "#{width}x#{height}", opts)enddef self.crop(from, to, width, height, opts = {})- optimize("crop", from, to, "#{width}x#{height}", opts)+ optimize(:optimized_image_crop, from, to, "#{width}x#{height}", opts)enddef self.downsize(from, to, dimensions, opts = {})- optimize("downsize", from, to, dimensions, opts)+ optimize(:optimized_image_downsize, from, to, dimensions, opts)end- def self.optimize(operation, from, to, dimensions, opts = {})- method_name = "#{operation}_instructions"+ INSTRUCTION_METHODS = {+ optimized_image_resize: :resize_instructions,+ optimized_image_crop: :crop_instructions,+ optimized_image_downsize: :downsize_instructions,+ }.freeze+ private_constant :INSTRUCTION_METHODS- instructions = public_send(method_name.to_sym, from, to, dimensions, opts)- convert_with(instructions, from, to, opts)+ def self.optimize(operation, from, to, dimensions, opts = {})+ instructions = public_send(INSTRUCTION_METHODS.fetch(operation), from, to, dimensions, opts)+ convert_with(instructions, from, to, opts, operation:)endMAX_PNGQUANT_SIZE = 500_000MAX_CONVERT_SECONDS = 20- def self.convert_with(instructions, from, to, opts = {})+ def self.convert_with(instructions, from, to, opts = {}, operation:)ImageMagick.magick(*instructions,+ operation:,read: [from],write: [File.dirname(to)],nice: 10,diff --git a/app/models/upload.rb b/app/models/upload.rbindex 4b8334168c7..0ef087df7de 100644--- a/app/models/upload.rb+++ b/app/models/upload.rb@@ -311,6 +311,7 @@ class Upload < ActiveRecord::Base"-format","%w %h","MSVG:#{path}",+ operation: :upload_svg_dimensions,read: [path],timeout: MAX_IDENTIFY_SECONDS,).split(" ")@@ -407,6 +408,7 @@ class Upload < ActiveRecord::Base"-format","%c","histogram:info:",+ operation: :upload_dominant_color,read: [local_path],nice: 10,timeout: DOMINANT_COLOR_COMMAND_TIMEOUT_SECONDS,@@ -443,6 +445,7 @@ class Upload < ActiveRecord::Base"-format","%Q",local_path,+ operation: :upload_quality_probe,read: [local_path],timeout: MAX_IDENTIFY_SECONDS,).to_idiff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex f9950637fdb..12b1a94501d 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -2276,6 +2276,7 @@ en:top_topics_formula_least_likes_per_post_multiplier: "value of least likes per post multiplier (n) in top topics formula: `log(views_count) * 2 + op_likes_count * 0.5 + LEAST(likes_count / posts_count, (n)) + 10 + log(posts_count)`"enable_safe_mode: "Allow users to enter safe mode to debug plugins."+ instrument_image_processing: "Record wall-clock duration and success for image-processing operations. Disabled by default."rate_limit_create_topic: "After creating a topic, users must wait (n) seconds before creating another topic."rate_limit_create_post: "After posting, users must wait (n) seconds before creating another post."diff --git a/config/site_settings.yml b/config/site_settings.ymlindex 94a778972d9..6d546b905bf 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -3603,6 +3603,9 @@ developer:instrument_gc_stat_per_request:default: falsehidden: true+ instrument_image_processing:+ default: false+ hidden: truewarn_critical_js_deprecations:default: trueclient: truediff --git a/lib/image_magick.rb b/lib/image_magick.rbindex c0ffd22c702..1a4c33a9cee 100644--- a/lib/image_magick.rb+++ b/lib/image_magick.rb@@ -1,6 +1,7 @@# frozen_string_literal: truerequire "tmpdir"+require "image_processing/instrumentation"# Runs ImageMagick under the Landlock sandbox (via Discourse::SafeExec) so a# decoder bug parsing an untrusted upload is confined to an explicit filesystem@@ -28,39 +29,49 @@ module ImageMagick)end- def self.magick(*args, read: [], write: [], timeout: nil, nice: nil, failure_message: "")+ def self.magick(+ *args,+ operation:,+ read: [],+ write: [],+ timeout: nil,+ nice: nil,+ failure_message: ""+ )command = ["magick", *args]command = ["nice", "-n", nice.to_s, *command] if nice- run(*command, read:, write:, timeout:, failure_message:)+ run(*command, operation:, read:, write:, timeout:, failure_message:)end- def self.identify(*args, read: [], write: [], timeout: nil, failure_message: "")- run("identify", *args, read:, write:, timeout:, failure_message:)+ def self.identify(*args, operation:, read: [], write: [], timeout: nil, failure_message: "")+ run("identify", *args, operation:, read:, write:, timeout:, failure_message:)end- def self.run(*command, read:, write:, timeout:, failure_message:)+ def self.run(*command, operation:, read:, write:, timeout:, failure_message:)# A private scratch dir keeps ImageMagick's disk-backed pixel cache inside# the write allowlist, so large images that spill to disk still succeed.Dir.mktmpdir("discourse-imagemagick-") do |scratch|- Discourse::SafeExec.capture(- *command,- env: {- **ENV.slice("PATH", "MAGICK_CONFIGURE_PATH", "LANG", "LC_ALL"),- "MAGICK_TEMPORARY_PATH" => scratch,- "TMPDIR" => scratch,- "HOME" => scratch,- "XDG_CACHE_HOME" => scratch,- "MALLOC_ARENA_MAX" => "2",- },- unsetenv_others: true,- read: [*Discourse::SafeExec.default_read_paths, *asset_read_paths, *read],- write: [scratch, *write],- execute: Discourse::SafeExec.default_execute_paths,- timeout: timeout || DEFAULT_TIMEOUT,- rlimits: RLIMITS,- failure_message:,- seccomp_deny_network: true,- )+ ImageProcessing::Instrumentation.instrument(operation:) do+ Discourse::SafeExec.capture(+ *command,+ env: {+ **ENV.slice("PATH", "MAGICK_CONFIGURE_PATH", "LANG", "LC_ALL"),+ "MAGICK_TEMPORARY_PATH" => scratch,+ "TMPDIR" => scratch,+ "HOME" => scratch,+ "XDG_CACHE_HOME" => scratch,+ "MALLOC_ARENA_MAX" => "2",+ },+ unsetenv_others: true,+ read: [*Discourse::SafeExec.default_read_paths, *asset_read_paths, *read],+ write: [scratch, *write],+ execute: Discourse::SafeExec.default_execute_paths,+ timeout: timeout || DEFAULT_TIMEOUT,+ rlimits: RLIMITS,+ failure_message:,+ seccomp_deny_network: true,+ )+ endendendprivate_class_method :rundiff --git a/lib/image_processing/instrumentation.rb b/lib/image_processing/instrumentation.rbnew file mode 100644index 00000000000..0e820dbf1e5--- /dev/null+++ b/lib/image_processing/instrumentation.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++module ImageProcessing+ class Instrumentation+ def self.instrument(operation:)+ return yield if !SiteSetting.instrument_image_processing++ validate_operation!(operation)+ started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ success = false++ begin+ result = yield+ success = true+ result+ ensure+ duration_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at+ record(operation:, duration_seconds:, success:)+ end+ end++ def self.record(operation:, duration_seconds:, success:)+ DiscourseEvent.trigger(+ :image_processing_finished,+ { operation: operation.to_s, duration_seconds:, success: },+ continue_on_error: true,+ )+ end+ private_class_method :record++ def self.validate_operation!(operation)+ raise ArgumentError, "operation must be a Symbol" if !operation.is_a?(Symbol)+ end+ private_class_method :validate_operation!+ end+enddiff --git a/lib/letter_avatar.rb b/lib/letter_avatar.rbindex 4bbd22dc5e8..ad5f0e5c73c 100644--- a/lib/letter_avatar.rb+++ b/lib/letter_avatar.rb@@ -97,7 +97,11 @@ class LetterAvatar#{filename}]- ImageMagick.magick(*instructions, write: [File.dirname(filename)])+ ImageMagick.magick(+ *instructions,+ operation: :letter_avatar_render,+ write: [File.dirname(filename)],+ )## do not optimize image, it will end up larger than originalfilename@@ -111,7 +115,11 @@ class LetterAvatardef image_magick_version@image_magick_version ||=Digest::MD5.hexdigest(- ImageMagick.magick("--version") << ImageMagick.magick("-list", "font"),+ ImageMagick.magick("--version", operation: :letter_avatar_version) << ImageMagick.magick(+ "-list",+ "font",+ operation: :letter_avatar_font_list,+ ),)enddiff --git a/lib/topic_og_image_generator.rb b/lib/topic_og_image_generator.rbindex 77110c60a37..e446b8391b2 100644--- a/lib/topic_og_image_generator.rb+++ b/lib/topic_og_image_generator.rb@@ -327,6 +327,7 @@ class TopicOgImageGeneratorImageMagick.magick("MSVG:#{svg_path}",png_path,+ operation: :topic_og_asset_render,read: [svg_path],write: [directory],timeout: 10,@@ -383,6 +384,7 @@ class TopicOgImageGenerator"-define","png:compression-level=9",png_path,+ operation: :topic_og_render,read: [dir],write: [dir],nice: 10,diff --git a/lib/upload_creator.rb b/lib/upload_creator.rbindex 5f2f78a450b..36721c781a5 100644--- a/lib/upload_creator.rb+++ b/lib/upload_creator.rb@@ -202,6 +202,7 @@ class UploadCreator"-format","%w %h","MSVG:#{@file.path}",+ operation: :upload_svg_dimensions,read: [@file.path],timeout: Upload::MAX_IDENTIFY_SECONDS,)@@ -445,6 +446,7 @@ class UploadCreatorImageMagick.magick(*command,+ operation: :upload_format_conversion,read:,write:,failure_message: I18n.t("upload.png_to_jpg_conversion_failure_message"),@@ -557,6 +559,7 @@ class UploadCreatorpath,"-auto-orient",path,+ operation: :upload_auto_orient,read: [@file.path],write: [@file.path, File.dirname(@file.path)],timeout: MAX_FIX_ORIENTATION_TIME,@@ -700,6 +703,7 @@ class UploadCreator"-format","%n\\n",@file.path,+ operation: :upload_animation_probe,read: [@file.path],timeout: Upload::MAX_IDENTIFY_SECONDS,).to_idiff --git a/spec/lib/image_magick_spec.rb b/spec/lib/image_magick_spec.rbnew file mode 100644index 00000000000..b8dd73f4e1f--- /dev/null+++ b/spec/lib/image_magick_spec.rb@@ -0,0 +1,24 @@+# frozen_string_literal: true++require "image_magick"++RSpec.describe ImageMagick do+ describe ".magick" do+ it "emits one measurement for a real command" do+ SiteSetting.instrument_image_processing = true++ events =+ DiscourseEvent.track_events(:image_processing_finished) do+ described_class.magick("--version", operation: :letter_avatar_version)+ end++ expect(events.size).to eq(1)+ payload = events.first[:params].first+ expect(payload.except(:duration_seconds)).to eq(+ operation: "letter_avatar_version",+ success: true,+ )+ expect(payload[:duration_seconds]).to be >= 0+ end+ end+enddiff --git a/spec/support/image_orientation_helpers.rb b/spec/support/image_orientation_helpers.rbindex c5763695ba7..cd9090fe157 100644--- a/spec/support/image_orientation_helpers.rb+++ b/spec/support/image_orientation_helpers.rb@@ -26,7 +26,13 @@ module ImageOrientationHelpersDir.mktmpdir do |directory|png_path = File.join(directory, "stored.png")- ImageMagick.magick(stored_path, png_path, read: [stored_path], write: [directory])+ ImageMagick.magick(+ stored_path,+ png_path,+ operation: :upload_format_conversion,+ read: [stored_path],+ write: [directory],+ )stored_image = ChunkyPNG::Image.from_file(png_path)Array.new(rows) do |row_index|