Skip to content

A platform for community discussion. Free, open, simple.

FIX: Allow Sidekiq workers time to register during startup (#44219)

Kevin Hwangc5bab0b628cf
diff --git a/lib/demon/sidekiq.rb b/lib/demon/sidekiq.rb
index 8b26047c722..a08d95d5f30 100644
--- a/lib/demon/sidekiq.rb
+++ b/lib/demon/sidekiq.rb
@@ -33,6 +33,9 @@ class Demon::Sidekiq < ::Demon::Base
running_sidekiq_process = sidekiq_processes_for_current_hostname[daemon.pid]
+ # A newly forked process may not have registered its first heartbeat yet.
+ next if !running_sidekiq_process && daemon.starting?
+
if !running_sidekiq_process ||
(Time.now.to_i - running_sidekiq_process["beat"]) >
SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS
@@ -77,6 +80,18 @@ class Demon::Sidekiq < ::Demon::Base
[ENV["UNICORN_SIDEKIQ_MAX_RSS"].to_i, DEFAULT_MAX_ALLOWED_SIDEKIQ_RSS_MEGABYTES].max.megabytes
end
+ def run
+ super
+ @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ end
+
+ def starting?
+ return false unless @started_at
+
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at <
+ SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS
+ end
+
def stop_signal
"TERM"
end
diff --git a/spec/lib/demon/sidekiq_spec.rb b/spec/lib/demon/sidekiq_spec.rb
index ee9172ce25a..b4a5afd2a3c 100644
--- a/spec/lib/demon/sidekiq_spec.rb
+++ b/spec/lib/demon/sidekiq_spec.rb
@@ -2,6 +2,91 @@
RSpec.describe Demon::Sidekiq do
describe ".heartbeat_check" do
+ let(:daemon) { described_class.new(0, rails_root: @rails_root) }
+ let(:original_pid) { 123_456 }
+ let(:replacement_pid) { 123_457 }
+
+ around do |example|
+ Dir.mktmpdir do |directory|
+ @rails_root = Pathname.new(directory)
+ example.run
+ end
+ end
+
+ before do
+ Discourse.stubs(:before_fork)
+ Sidekiq::ProcessSet.stubs(:new).returns([])
+ Process.stubs(:clock_gettime).with(Process::CLOCK_MONOTONIC).returns(0)
+ Process.stubs(:kill).with(0, original_pid).returns(1)
+ daemon.stubs(:fork).returns(original_pid, replacement_pid)
+ described_class.set_demons({ "sidekiq_0" => daemon })
+ end
+
+ after { described_class.reset_demons }
+
+ it "allows an unregistered worker to start until the startup grace period expires" do
+ daemon.start
+ described_class.heartbeat_check
+ expect(daemon.pid).to eq(original_pid)
+
+ Process
+ .stubs(:clock_gettime)
+ .with(Process::CLOCK_MONOTONIC)
+ .returns(described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS - 1)
+ described_class.heartbeat_check
+ expect(daemon.pid).to eq(original_pid)
+
+ Process
+ .stubs(:clock_gettime)
+ .with(Process::CLOCK_MONOTONIC)
+ .returns(described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS)
+ Process.stubs(:kill).with(0, original_pid).returns(1).then.raises(Errno::ESRCH)
+ Process.expects(:kill).with("TERM", original_pid)
+ Process.stubs(:waitpid).with(original_pid, Process::WNOHANG).returns(original_pid)
+
+ described_class.heartbeat_check
+
+ expect(daemon.pid).to eq(replacement_pid)
+ end
+
+ it "renews the startup grace period when replacing a dead worker" do
+ daemon.start
+ Process
+ .stubs(:clock_gettime)
+ .with(Process::CLOCK_MONOTONIC)
+ .returns(described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS + 1)
+ Process.stubs(:waitpid).with(original_pid, Process::WNOHANG).returns(original_pid)
+ Process.stubs(:kill).with(0, original_pid).raises(Errno::ESRCH)
+ Process.stubs(:kill).with(0, replacement_pid).returns(1)
+ Process.expects(:kill).with("TERM", replacement_pid).never
+
+ described_class.ensure_running
+ described_class.heartbeat_check
+
+ expect(daemon.pid).to eq(replacement_pid)
+ end
+
+ it "restarts a registered worker with a stale heartbeat during the startup grace period" do
+ daemon.start
+ Sidekiq::ProcessSet.stubs(:new).returns(
+ [
+ {
+ "hostname" => described_class::HOSTNAME,
+ "pid" => original_pid,
+ "beat" =>
+ Time.now.to_i - described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS - 1,
+ },
+ ],
+ )
+ Process.stubs(:kill).with(0, original_pid).returns(1).then.raises(Errno::ESRCH)
+ Process.expects(:kill).with("TERM", original_pid)
+ Process.stubs(:waitpid).with(original_pid, Process::WNOHANG).returns(original_pid)
+
+ described_class.heartbeat_check
+
+ expect(daemon.pid).to eq(replacement_pid)
+ end
+
it "restarts Sidekiq daemons missing from Sidekiq::ProcessSet or with a missed heartbeat" do
running_sidekiq_daemon = described_class.new(1)
running_sidekiq_daemon.set_pid(1)