Skip to content

A platform for community discussion. Free, open, simple.

FIX: Retry shared image worker connections within the timeout (#44625)

Alan Guo Xiang Tan8ae745b02c29
diff --git a/lib/discourse_vips/client.rb b/lib/discourse_vips/client.rb
index 821a850b9b4..0df1ae94dbe 100644
--- a/lib/discourse_vips/client.rb
+++ b/lib/discourse_vips/client.rb
@@ -23,6 +23,17 @@ module DiscourseVips
reset_worker_process
end
+ if Rails.env.test?
+ def self.shared_worker?
+ @shared_worker
+ end
+
+ def self.use_private_worker
+ reset_worker_process
+ @shared_worker = false
+ end
+ end
+
def self.call(command, operation:, timeout: DEFAULT_TIMEOUT_SECONDS, nice: nil, read:, write:)
payload = { command: command.map(&:to_s), timeout:, nice:, read:, write: }
@@ -69,7 +80,17 @@ module DiscourseVips
private_class_method :send_command
def self.connect_socket(deadline:)
- Addrinfo.unix(worker_socket_path).connect(timeout: remaining_time(deadline))
+ delay_seconds = 0.1
+
+ begin
+ Addrinfo.unix(worker_socket_path).connect(timeout: remaining_time(deadline))
+ rescue Errno::ENOENT, Errno::ECONNREFUSED
+ raise if !@shared_worker
+
+ sleep [delay_seconds, remaining_time(deadline)].min
+ delay_seconds = [delay_seconds * 2, 1].min
+ retry
+ end
rescue IO::TimeoutError
raise WorkerUnavailable, "libvips worker did not respond"
end
diff --git a/spec/lib/discourse_vips/client_spec.rb b/spec/lib/discourse_vips/client_spec.rb
new file mode 100644
index 00000000000..139eca86070
--- /dev/null
+++ b/spec/lib/discourse_vips/client_spec.rb
@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseVips::Client do
+ describe ".call" do
+ let(:directory) { Dir.mktmpdir }
+ let(:socket_path) { File.join(directory, "socket") }
+ let(:servers) { [] }
+ let(:server_threads) { [] }
+ let(:requests) { Queue.new }
+
+ around do |example|
+ shared_worker = described_class.shared_worker?
+ example.run
+ ensure
+ shared_worker ? described_class.use_shared_worker : described_class.use_private_worker
+ end
+
+ before do
+ DiscourseVips::WorkerProcess.stubs(:shared_socket_path).returns(socket_path)
+ described_class.use_shared_worker
+ end
+
+ after do
+ server_threads.each(&:kill)
+ server_threads.each(&:join)
+ servers.each(&:close)
+ FileUtils.remove_entry(directory)
+ end
+
+ def serve(response:, delay_seconds: 0)
+ server = UNIXServer.new(socket_path)
+ servers << server
+ server_threads << Thread.new do
+ loop do
+ connection = server.accept
+ begin
+ requests << MessagePack.unpack(connection.read)
+ sleep delay_seconds if delay_seconds.positive?
+ connection.write(response) if response
+ ensure
+ connection.close
+ end
+ end
+ end
+ end
+
+ def restore_socket_after_failure(
+ connection_error:,
+ outage_seconds: 0,
+ response_delay_seconds: 0
+ )
+ address = Addrinfo.unix(socket_path)
+ allow(Addrinfo).to receive(:unix).with(socket_path).and_return(address)
+ allow(address).to receive(:connect).and_wrap_original do |connect, **options|
+ connect.call(**options)
+ rescue connection_error
+ sleep outage_seconds if outage_seconds.positive?
+ File.unlink(socket_path) if File.exist?(socket_path)
+ serve(
+ response: MessagePack.pack({ "status" => "ok", "value" => "8.15.0" }),
+ delay_seconds: response_delay_seconds,
+ )
+ raise
+ end
+ end
+
+ it "recovers when the shared worker socket is missing" do
+ restore_socket_after_failure(connection_error: Errno::ENOENT)
+
+ result =
+ stub_const(described_class, :WORKER_GRACE_SECONDS, 0) do
+ Timeout.timeout(5) do
+ described_class.call(
+ ["version"],
+ operation: :vips_version,
+ timeout: 1,
+ read: [],
+ write: [],
+ )
+ end
+ end
+
+ expect(result).to eq("8.15.0")
+ expect(requests.size).to eq(1)
+ expect(requests.pop.fetch("timeout")).to eq(1)
+ end
+
+ it "recovers when the shared worker socket refuses connections" do
+ UNIXServer.new(socket_path).close
+ restore_socket_after_failure(connection_error: Errno::ECONNREFUSED)
+
+ result =
+ stub_const(described_class, :WORKER_GRACE_SECONDS, 0) do
+ Timeout.timeout(5) do
+ described_class.call(
+ ["version"],
+ operation: :vips_version,
+ timeout: 1,
+ read: [],
+ write: [],
+ )
+ end
+ end
+
+ expect(result).to eq("8.15.0")
+ expect(requests.size).to eq(1)
+ expect(requests.pop.fetch("timeout")).to eq(1)
+ end
+
+ it "charges shared connection recovery against the operation timeout" do
+ restore_socket_after_failure(
+ connection_error: Errno::ENOENT,
+ outage_seconds: 0.6,
+ response_delay_seconds: 0.6,
+ )
+
+ stub_const(described_class, :WORKER_GRACE_SECONDS, 0) do
+ Timeout.timeout(5) do
+ expect {
+ described_class.call(
+ ["version"],
+ operation: :vips_version,
+ timeout: 1,
+ read: [],
+ write: [],
+ )
+ }.to raise_error(DiscourseVips::WorkerUnavailable, "libvips worker did not respond")
+ end
+ end
+
+ expect(requests.size).to eq(1)
+ end
+
+ it "bounds the wait when the shared worker remains unavailable" do
+ started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+ stub_const(described_class, :WORKER_GRACE_SECONDS, 0) do
+ Timeout.timeout(5) do
+ expect {
+ described_class.call(
+ ["version"],
+ operation: :vips_version,
+ timeout: 0.2,
+ read: [],
+ write: [],
+ )
+ }.to raise_error(DiscourseVips::WorkerUnavailable)
+ end
+ end
+
+ elapsed_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+ expect(elapsed_seconds).to be_between(0.15, 2)
+ end
+
+ it "reports a private worker startup failure without retrying" do
+ described_class.use_private_worker
+ Process.expects(:spawn).once.raises(Errno::ENOENT)
+
+ expect {
+ described_class.call(["version"], operation: :vips_version, read: [], write: [])
+ }.to raise_error(DiscourseVips::WorkerUnavailable)
+ end
+
+ it "preserves an empty response without replaying the accepted request" do
+ serve(response: nil)
+
+ expect {
+ described_class.call(["version"], operation: :vips_version, read: [], write: [])
+ }.to raise_error(DiscourseVips::WorkerUnavailable)
+
+ expect(requests.size).to eq(1)
+ end
+
+ it "preserves an invalid response without replaying the accepted request" do
+ serve(response: "\xC1")
+
+ expect {
+ described_class.call(["version"], operation: :vips_version, read: [], write: [])
+ }.to raise_error(DiscourseVips::WorkerUnavailable)
+
+ expect(requests.size).to eq(1)
+ end
+
+ it "preserves an operation timeout without replaying the request" do
+ serve(response: MessagePack.pack({ "status" => "timeout" }))
+
+ expect {
+ described_class.call(["version"], operation: :vips_version, read: [], write: [])
+ }.to raise_error(DiscourseVips::OperationTimeout)
+
+ expect(requests.size).to eq(1)
+ end
+
+ it "preserves an invalid image error without replaying the request" do
+ serve(
+ response: MessagePack.pack({ "status" => "invalid_image", "message" => "invalid image" }),
+ )
+
+ expect {
+ described_class.call(["version"], operation: :vips_version, read: [], write: [])
+ }.to raise_error(DiscourseVips::InvalidImage)
+
+ expect(requests.size).to eq(1)
+ end
+
+ it "preserves an operation error without replaying the request" do
+ serve(response: MessagePack.pack({ "status" => "error", "message" => "operation failed" }))
+
+ expect {
+ described_class.call(["version"], operation: :vips_version, read: [], write: [])
+ }.to raise_error(DiscourseVips::Error)
+
+ expect(requests.size).to eq(1)
+ end
+ end
+end