A platform for community discussion. Free, open, simple.
MT: Keep forks away from an in-flight source DB connection (#44213)
Gerhard Schlager83f6337e7363
diff --git a/migrations/converters/lib/migrations/converters/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rbindex 30812ff6087..28ebc1c75b6 100644--- a/migrations/converters/lib/migrations/converters/adapter/postgres.rb+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb@@ -12,17 +12,20 @@ module Migrationsenddef initialize(settings)- @fork_hook = ForkManager.after_fork_child { discard! }-- begin- @connection = PG::Connection.new(settings)- @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)- @connection.field_name_type = :symbol- configure_connection- rescue StandardError- ForkManager.remove_after_fork_child(@fork_hook)- @fork_hook = nil- raise+ # Hook registration and connect are one unit; see ForkManager.synchronize.+ ForkManager.synchronize do+ @fork_hook = ForkManager.after_fork_child { discard! }++ begin+ @connection = PG::Connection.new(settings)+ @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)+ @connection.field_name_type = :symbol+ configure_connection+ rescue StandardError+ ForkManager.remove_after_fork_child(@fork_hook)+ @fork_hook = nil+ raise+ endendenddiff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rbindex 39f13205dd2..e13fa6e7aaf 100644--- a/migrations/core/lib/migrations/common/fork_manager.rb+++ b/migrations/core/lib/migrations/common/fork_manager.rb@@ -1,12 +1,14 @@# frozen_string_literal: true+require "monitor"+module Migrations# The fork hooks for a run. `before_fork` and `after_fork_parent` run around a# fork, so a connection can close before it and reopen after. `after_fork_child`# runs in the new child, e.g. to drop a connection it inherited.#- # Steps add and remove hooks from several threads at once, so a mutex guards the- # hook lists. A fork copies the child hooks under that mutex, so the child always+ # Steps add and remove hooks from several threads at once, so a monitor guards the+ # hook lists. A fork copies the child hooks under that monitor, so the child always# runs a consistent list.module ForkManager# The batching flag is per-thread: a batch belongs to the thread that opened@@ -18,91 +20,103 @@ module Migrations@before_fork_hooks = []@after_fork_parent_hooks = []@after_fork_child_hooks = []- @mutex = Mutex.new+ @monitor = Monitor.newclass << self+ # Excludes concurrent forks while the block runs. A connection must register its+ # after-fork hook and finish connecting as one unit: a fork in between+ # inherits the half-open socket without a usable hook, and the child+ # terminates the parent's session on exit. Reentrant, so nested use and+ # forking from inside a hook can't deadlock.+ def synchronize(&block)+ @monitor.synchronize(&block)+ end+def with_batched_forks- previous = Thread.current[BATCHED_FORKS_KEY]-- # Restore the flag no matter what. If a before-fork hook raises, the flag- # would otherwise stick as true on this thread, and every later plain- # `fork` on it would silently skip its parent-side hooks.- begin- Thread.current[BATCHED_FORKS_KEY] = true- @mutex.synchronize { run_before_fork_hooks }-- # Always run the after-fork hooks even if forking raises (e.g.- # `Errno::EAGAIN`/`ENOMEM` under fork pressure). Otherwise the- # before-fork hooks' effects — a locked writer mutex, a closed run- # connection — would never be undone and the run would hang instead of- # failing the step. If a before-fork hook raises, these after-fork- # hooks don't run — the batch never started.+ @monitor.synchronize do+ previous = Thread.current[BATCHED_FORKS_KEY]++ # Restore the flag no matter what. If a before-fork hook raises, the flag+ # would otherwise stick as true on this thread, and every later plain+ # `fork` on it would silently skip its parent-side hooks.begin- yield+ Thread.current[BATCHED_FORKS_KEY] = true+ run_before_fork_hooks++ # Always run the after-fork hooks even if forking raises (e.g.+ # `Errno::EAGAIN`/`ENOMEM` under fork pressure). Otherwise the+ # before-fork hooks' effects — a locked writer mutex, a closed run+ # connection — would never be undone and the run would hang instead of+ # failing the step. If a before-fork hook raises, these after-fork+ # hooks don't run — the batch never started.+ begin+ yield+ ensure+ run_after_fork_parent_hooks+ endensure- @mutex.synchronize { run_after_fork_parent_hooks }+ Thread.current[BATCHED_FORKS_KEY] = previousend- ensure- Thread.current[BATCHED_FORKS_KEY] = previousendenddef before_fork(&block)return unless block- @mutex.synchronize { @before_fork_hooks << block }+ @monitor.synchronize { @before_fork_hooks << block }blockenddef remove_before_fork(block)- @mutex.synchronize { @before_fork_hooks.delete(block) }+ @monitor.synchronize { @before_fork_hooks.delete(block) }enddef after_fork_parent(&block)return unless block- @mutex.synchronize { @after_fork_parent_hooks << block }+ @monitor.synchronize { @after_fork_parent_hooks << block }blockenddef remove_after_fork_parent(block)- @mutex.synchronize { @after_fork_parent_hooks.delete(block) }+ @monitor.synchronize { @after_fork_parent_hooks.delete(block) }enddef after_fork_child(&block)return unless block- @mutex.synchronize { @after_fork_child_hooks << block }+ @monitor.synchronize { @after_fork_child_hooks << block }blockenddef remove_after_fork_child(block)- @mutex.synchronize { @after_fork_child_hooks.delete(block) }+ @monitor.synchronize { @after_fork_child_hooks.delete(block) }enddef fork- # In a batch the parent-side hooks already ran around the whole batch.- execute_parent = !Thread.current[BATCHED_FORKS_KEY]+ @monitor.synchronize do+ # In a batch the parent-side hooks already ran around the whole batch.+ execute_parent = !Thread.current[BATCHED_FORKS_KEY]- @mutex.synchronize { run_before_fork_hooks } if execute_parent+ run_before_fork_hooks if execute_parent- pid =- Process.fork do- child_hooks = @mutex.synchronize { @after_fork_child_hooks.dup }- child_hooks.each(&:call)- yield- end+ pid =+ Process.fork do+ @after_fork_child_hooks.dup.each(&:call)+ yield+ end- @mutex.synchronize { run_after_fork_parent_hooks } if execute_parent+ run_after_fork_parent_hooks if execute_parent- pid+ pid+ endenddef hook_count- @mutex.synchronize do+ @monitor.synchronize do@before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.sizeendenddef clear!- @mutex.synchronize do+ @monitor.synchronize do@before_fork_hooks.clear@after_fork_parent_hooks.clear@after_fork_child_hooks.cleardiff --git a/migrations/core/spec/lib/common/fork_manager_spec.rb b/migrations/core/spec/lib/common/fork_manager_spec.rbindex 81acc815d50..0f40a871fd9 100644--- a/migrations/core/spec/lib/common/fork_manager_spec.rb+++ b/migrations/core/spec/lib/common/fork_manager_spec.rb@@ -100,11 +100,7 @@ RSpec.describe Migrations::ForkManager doexpect(parent_ran).to be(true)end- it "keeps the batched state thread-local so a concurrent batch doesn't leak" do- # Observe the parent-hook decision without a real fork: a batched fork skips- # the per-fork parent hooks (they run once in `with_batched_forks`), an- # unbatched one runs them itself. The hook tags itself with the running- # thread so we can tell whose fork ran it.+ it "delays a concurrent fork until the batch has finished" doallow(Process).to receive(:fork).and_return(4242)ran = Queue.newdescribed_class.after_fork_parent { ran << Thread.current[:tag] }@@ -122,21 +118,84 @@ RSpec.describe Migrations::ForkManager doa_batched.popb_done = Queue.new- Thread.new do- Thread.current[:tag] = :b- described_class.fork {} # not batched: must run the parent hook itself- b_done << true- end+ b =+ Thread.new do+ Thread.current[:tag] = :b+ described_class.fork {}+ b_done << true+ end++ sleep 0.05+ expect(b_done).to be_empty++ release_a << true+ a.joinb_done.pop+ b.join- # B's unbatched fork ran the parent hook despite A being mid-batch. With a- # process-global flag, B would have seen A's batched state and skipped it.collected = []collected << ran.pop until ran.empty?- expect(collected).to include(:b)+ expect(collected).to eq(%i[a b])+ end+ end- release_a << true- a.join+ describe ".synchronize" do+ it "delays a concurrent fork until the block has finished" do+ entered = Queue.new+ release = Queue.new+ holder =+ Thread.new do+ described_class.synchronize do+ entered << true+ release.pop+ end+ end+ entered.pop++ forked = Queue.new+ forker = Thread.new { forked << described_class.fork { exit!(0) } }++ sleep 0.05+ expect(forked).to be_empty++ release << true+ pid = forked.pop+ Process.waitpid(pid)+ holder.join+ forker.join+ end++ it "is reentrant, so forking inside the block works" do+ described_class.synchronize do+ described_class.synchronize do+ pid = described_class.fork { exit!(0) }+ Process.waitpid(pid)+ end+ end+ end++ it "waits until a fork's parent hooks have finished" do+ allow(Process).to receive(:fork).and_return(4242)+ hook_started = Queue.new+ release_hook = Queue.new+ described_class.before_fork do+ hook_started << true+ release_hook.pop+ end++ forker = Thread.new { described_class.fork {} }+ hook_started.pop++ synchronized = Queue.new+ waiter = Thread.new { described_class.synchronize { synchronized << true } }++ sleep 0.05+ expect(synchronized).to be_empty++ release_hook << true+ forker.join+ waiter.join+ expect(synchronized.pop).to be(true)endendend