A platform for community discussion. Free, open, simple.
FIX: Allow Sidekiq workers time to register during startup (#44219)
Kevin Hwangc5bab0b628cf
diff --git a/lib/demon/sidekiq.rb b/lib/demon/sidekiq.rbindex 8b26047c722..a08d95d5f30 100644--- a/lib/demon/sidekiq.rb+++ b/lib/demon/sidekiq.rb@@ -33,6 +33,9 @@ class Demon::Sidekiq < ::Demon::Baserunning_sidekiq_process = sidekiq_processes_for_current_hostname[daemon.pid]+ # A newly forked process may not have registered its first heartbeat yet.+ next if !running_sidekiq_process && daemon.starting?+if !running_sidekiq_process ||(Time.now.to_i - running_sidekiq_process["beat"]) >SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS@@ -77,6 +80,18 @@ class Demon::Sidekiq < ::Demon::Base[ENV["UNICORN_SIDEKIQ_MAX_RSS"].to_i, DEFAULT_MAX_ALLOWED_SIDEKIQ_RSS_MEGABYTES].max.megabytesend+ def run+ super+ @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ end++ def starting?+ return false unless @started_at++ Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at <+ SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS+ end+def stop_signal"TERM"enddiff --git a/spec/lib/demon/sidekiq_spec.rb b/spec/lib/demon/sidekiq_spec.rbindex ee9172ce25a..b4a5afd2a3c 100644--- a/spec/lib/demon/sidekiq_spec.rb+++ b/spec/lib/demon/sidekiq_spec.rb@@ -2,6 +2,91 @@RSpec.describe Demon::Sidekiq dodescribe ".heartbeat_check" do+ let(:daemon) { described_class.new(0, rails_root: @rails_root) }+ let(:original_pid) { 123_456 }+ let(:replacement_pid) { 123_457 }++ around do |example|+ Dir.mktmpdir do |directory|+ @rails_root = Pathname.new(directory)+ example.run+ end+ end++ before do+ Discourse.stubs(:before_fork)+ Sidekiq::ProcessSet.stubs(:new).returns([])+ Process.stubs(:clock_gettime).with(Process::CLOCK_MONOTONIC).returns(0)+ Process.stubs(:kill).with(0, original_pid).returns(1)+ daemon.stubs(:fork).returns(original_pid, replacement_pid)+ described_class.set_demons({ "sidekiq_0" => daemon })+ end++ after { described_class.reset_demons }++ it "allows an unregistered worker to start until the startup grace period expires" do+ daemon.start+ described_class.heartbeat_check+ expect(daemon.pid).to eq(original_pid)++ Process+ .stubs(:clock_gettime)+ .with(Process::CLOCK_MONOTONIC)+ .returns(described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS - 1)+ described_class.heartbeat_check+ expect(daemon.pid).to eq(original_pid)++ Process+ .stubs(:clock_gettime)+ .with(Process::CLOCK_MONOTONIC)+ .returns(described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS)+ Process.stubs(:kill).with(0, original_pid).returns(1).then.raises(Errno::ESRCH)+ Process.expects(:kill).with("TERM", original_pid)+ Process.stubs(:waitpid).with(original_pid, Process::WNOHANG).returns(original_pid)++ described_class.heartbeat_check++ expect(daemon.pid).to eq(replacement_pid)+ end++ it "renews the startup grace period when replacing a dead worker" do+ daemon.start+ Process+ .stubs(:clock_gettime)+ .with(Process::CLOCK_MONOTONIC)+ .returns(described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS + 1)+ Process.stubs(:waitpid).with(original_pid, Process::WNOHANG).returns(original_pid)+ Process.stubs(:kill).with(0, original_pid).raises(Errno::ESRCH)+ Process.stubs(:kill).with(0, replacement_pid).returns(1)+ Process.expects(:kill).with("TERM", replacement_pid).never++ described_class.ensure_running+ described_class.heartbeat_check++ expect(daemon.pid).to eq(replacement_pid)+ end++ it "restarts a registered worker with a stale heartbeat during the startup grace period" do+ daemon.start+ Sidekiq::ProcessSet.stubs(:new).returns(+ [+ {+ "hostname" => described_class::HOSTNAME,+ "pid" => original_pid,+ "beat" =>+ Time.now.to_i - described_class::SIDEKIQ_HEARTBEAT_CHECK_MISS_THRESHOLD_SECONDS - 1,+ },+ ],+ )+ Process.stubs(:kill).with(0, original_pid).returns(1).then.raises(Errno::ESRCH)+ Process.expects(:kill).with("TERM", original_pid)+ Process.stubs(:waitpid).with(original_pid, Process::WNOHANG).returns(original_pid)++ described_class.heartbeat_check++ expect(daemon.pid).to eq(replacement_pid)+ end+it "restarts Sidekiq daemons missing from Sidekiq::ProcessSet or with a missed heartbeat" dorunning_sidekiq_daemon = described_class.new(1)running_sidekiq_daemon.set_pid(1)