Skip to content

A platform for community discussion. Free, open, simple.

MT: Keep forks away from an in-flight source DB connection (#44213)

Gerhard Schlager83f6337e7363
diff --git a/migrations/converters/lib/migrations/converters/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
index 30812ff6087..28ebc1c75b6 100644
--- a/migrations/converters/lib/migrations/converters/adapter/postgres.rb
+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
@@ -12,17 +12,20 @@ module Migrations
end
def initialize(settings)
- @fork_hook = ForkManager.after_fork_child { discard! }
-
- begin
- @connection = PG::Connection.new(settings)
- @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)
- @connection.field_name_type = :symbol
- configure_connection
- rescue StandardError
- ForkManager.remove_after_fork_child(@fork_hook)
- @fork_hook = nil
- raise
+ # Hook registration and connect are one unit; see ForkManager.synchronize.
+ ForkManager.synchronize do
+ @fork_hook = ForkManager.after_fork_child { discard! }
+
+ begin
+ @connection = PG::Connection.new(settings)
+ @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)
+ @connection.field_name_type = :symbol
+ configure_connection
+ rescue StandardError
+ ForkManager.remove_after_fork_child(@fork_hook)
+ @fork_hook = nil
+ raise
+ end
end
end
diff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rb
index 39f13205dd2..e13fa6e7aaf 100644
--- a/migrations/core/lib/migrations/common/fork_manager.rb
+++ b/migrations/core/lib/migrations/common/fork_manager.rb
@@ -1,12 +1,14 @@
# frozen_string_literal: true
+require "monitor"
+
module Migrations
# The fork hooks for a run. `before_fork` and `after_fork_parent` run around a
# fork, so a connection can close before it and reopen after. `after_fork_child`
# runs in the new child, e.g. to drop a connection it inherited.
#
- # Steps add and remove hooks from several threads at once, so a mutex guards the
- # hook lists. A fork copies the child hooks under that mutex, so the child always
+ # Steps add and remove hooks from several threads at once, so a monitor guards the
+ # hook lists. A fork copies the child hooks under that monitor, so the child always
# runs a consistent list.
module ForkManager
# The batching flag is per-thread: a batch belongs to the thread that opened
@@ -18,91 +20,103 @@ module Migrations
@before_fork_hooks = []
@after_fork_parent_hooks = []
@after_fork_child_hooks = []
- @mutex = Mutex.new
+ @monitor = Monitor.new
class << self
+ # Excludes concurrent forks while the block runs. A connection must register its
+ # after-fork hook and finish connecting as one unit: a fork in between
+ # inherits the half-open socket without a usable hook, and the child
+ # terminates the parent's session on exit. Reentrant, so nested use and
+ # forking from inside a hook can't deadlock.
+ def synchronize(&block)
+ @monitor.synchronize(&block)
+ end
+
def with_batched_forks
- previous = Thread.current[BATCHED_FORKS_KEY]
-
- # Restore the flag no matter what. If a before-fork hook raises, the flag
- # would otherwise stick as true on this thread, and every later plain
- # `fork` on it would silently skip its parent-side hooks.
- begin
- Thread.current[BATCHED_FORKS_KEY] = true
- @mutex.synchronize { run_before_fork_hooks }
-
- # Always run the after-fork hooks even if forking raises (e.g.
- # `Errno::EAGAIN`/`ENOMEM` under fork pressure). Otherwise the
- # before-fork hooks' effects — a locked writer mutex, a closed run
- # connection — would never be undone and the run would hang instead of
- # failing the step. If a before-fork hook raises, these after-fork
- # hooks don't run — the batch never started.
+ @monitor.synchronize do
+ previous = Thread.current[BATCHED_FORKS_KEY]
+
+ # Restore the flag no matter what. If a before-fork hook raises, the flag
+ # would otherwise stick as true on this thread, and every later plain
+ # `fork` on it would silently skip its parent-side hooks.
begin
- yield
+ Thread.current[BATCHED_FORKS_KEY] = true
+ run_before_fork_hooks
+
+ # Always run the after-fork hooks even if forking raises (e.g.
+ # `Errno::EAGAIN`/`ENOMEM` under fork pressure). Otherwise the
+ # before-fork hooks' effects — a locked writer mutex, a closed run
+ # connection — would never be undone and the run would hang instead of
+ # failing the step. If a before-fork hook raises, these after-fork
+ # hooks don't run — the batch never started.
+ begin
+ yield
+ ensure
+ run_after_fork_parent_hooks
+ end
ensure
- @mutex.synchronize { run_after_fork_parent_hooks }
+ Thread.current[BATCHED_FORKS_KEY] = previous
end
- ensure
- Thread.current[BATCHED_FORKS_KEY] = previous
end
end
def before_fork(&block)
return unless block
- @mutex.synchronize { @before_fork_hooks << block }
+ @monitor.synchronize { @before_fork_hooks << block }
block
end
def remove_before_fork(block)
- @mutex.synchronize { @before_fork_hooks.delete(block) }
+ @monitor.synchronize { @before_fork_hooks.delete(block) }
end
def after_fork_parent(&block)
return unless block
- @mutex.synchronize { @after_fork_parent_hooks << block }
+ @monitor.synchronize { @after_fork_parent_hooks << block }
block
end
def remove_after_fork_parent(block)
- @mutex.synchronize { @after_fork_parent_hooks.delete(block) }
+ @monitor.synchronize { @after_fork_parent_hooks.delete(block) }
end
def after_fork_child(&block)
return unless block
- @mutex.synchronize { @after_fork_child_hooks << block }
+ @monitor.synchronize { @after_fork_child_hooks << block }
block
end
def remove_after_fork_child(block)
- @mutex.synchronize { @after_fork_child_hooks.delete(block) }
+ @monitor.synchronize { @after_fork_child_hooks.delete(block) }
end
def fork
- # In a batch the parent-side hooks already ran around the whole batch.
- execute_parent = !Thread.current[BATCHED_FORKS_KEY]
+ @monitor.synchronize do
+ # In a batch the parent-side hooks already ran around the whole batch.
+ execute_parent = !Thread.current[BATCHED_FORKS_KEY]
- @mutex.synchronize { run_before_fork_hooks } if execute_parent
+ run_before_fork_hooks if execute_parent
- pid =
- Process.fork do
- child_hooks = @mutex.synchronize { @after_fork_child_hooks.dup }
- child_hooks.each(&:call)
- yield
- end
+ pid =
+ Process.fork do
+ @after_fork_child_hooks.dup.each(&:call)
+ yield
+ end
- @mutex.synchronize { run_after_fork_parent_hooks } if execute_parent
+ run_after_fork_parent_hooks if execute_parent
- pid
+ pid
+ end
end
def hook_count
- @mutex.synchronize do
+ @monitor.synchronize do
@before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.size
end
end
def clear!
- @mutex.synchronize do
+ @monitor.synchronize do
@before_fork_hooks.clear
@after_fork_parent_hooks.clear
@after_fork_child_hooks.clear
diff --git a/migrations/core/spec/lib/common/fork_manager_spec.rb b/migrations/core/spec/lib/common/fork_manager_spec.rb
index 81acc815d50..0f40a871fd9 100644
--- a/migrations/core/spec/lib/common/fork_manager_spec.rb
+++ b/migrations/core/spec/lib/common/fork_manager_spec.rb
@@ -100,11 +100,7 @@ RSpec.describe Migrations::ForkManager do
expect(parent_ran).to be(true)
end
- it "keeps the batched state thread-local so a concurrent batch doesn't leak" do
- # Observe the parent-hook decision without a real fork: a batched fork skips
- # the per-fork parent hooks (they run once in `with_batched_forks`), an
- # unbatched one runs them itself. The hook tags itself with the running
- # thread so we can tell whose fork ran it.
+ it "delays a concurrent fork until the batch has finished" do
allow(Process).to receive(:fork).and_return(4242)
ran = Queue.new
described_class.after_fork_parent { ran << Thread.current[:tag] }
@@ -122,21 +118,84 @@ RSpec.describe Migrations::ForkManager do
a_batched.pop
b_done = Queue.new
- Thread.new do
- Thread.current[:tag] = :b
- described_class.fork {} # not batched: must run the parent hook itself
- b_done << true
- end
+ b =
+ Thread.new do
+ Thread.current[:tag] = :b
+ described_class.fork {}
+ b_done << true
+ end
+
+ sleep 0.05
+ expect(b_done).to be_empty
+
+ release_a << true
+ a.join
b_done.pop
+ b.join
- # B's unbatched fork ran the parent hook despite A being mid-batch. With a
- # process-global flag, B would have seen A's batched state and skipped it.
collected = []
collected << ran.pop until ran.empty?
- expect(collected).to include(:b)
+ expect(collected).to eq(%i[a b])
+ end
+ end
- release_a << true
- a.join
+ describe ".synchronize" do
+ it "delays a concurrent fork until the block has finished" do
+ entered = Queue.new
+ release = Queue.new
+ holder =
+ Thread.new do
+ described_class.synchronize do
+ entered << true
+ release.pop
+ end
+ end
+ entered.pop
+
+ forked = Queue.new
+ forker = Thread.new { forked << described_class.fork { exit!(0) } }
+
+ sleep 0.05
+ expect(forked).to be_empty
+
+ release << true
+ pid = forked.pop
+ Process.waitpid(pid)
+ holder.join
+ forker.join
+ end
+
+ it "is reentrant, so forking inside the block works" do
+ described_class.synchronize do
+ described_class.synchronize do
+ pid = described_class.fork { exit!(0) }
+ Process.waitpid(pid)
+ end
+ end
+ end
+
+ it "waits until a fork's parent hooks have finished" do
+ allow(Process).to receive(:fork).and_return(4242)
+ hook_started = Queue.new
+ release_hook = Queue.new
+ described_class.before_fork do
+ hook_started << true
+ release_hook.pop
+ end
+
+ forker = Thread.new { described_class.fork {} }
+ hook_started.pop
+
+ synchronized = Queue.new
+ waiter = Thread.new { described_class.synchronize { synchronized << true } }
+
+ sleep 0.05
+ expect(synchronized).to be_empty
+
+ release_hook << true
+ forker.join
+ waiter.join
+ expect(synchronized.pop).to be(true)
end
end
end