Skip to content

A platform for community discussion. Free, open, simple.

MT: Discard inherited source DB connections in worker processes

Gerhard Schlagercadc068cf448
diff --git a/migrations/README.md b/migrations/README.md
index 8a9b6aec01c..18730b6751d 100644
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -29,6 +29,22 @@ Public converters live in `converters/lib/migrations/converters/`. To run a priv
(closed-source) converter, put its code in a subdirectory of `private/converters/`
(or point `MIGRATIONS_PRIVATE_CONVERTERS_PATH` at it).
+### Source DB adapters and fork safety
+
+Worker processes inherit the source DB connection's socket from the main process. Whether
+that's dangerous depends on the client library: a destructor that only closes the file
+descriptor is harmless (the parent still holds it, so the kernel sends nothing over the
+wire), but a destructor that writes a protocol goodbye kills the parent's session as soon
+as a worker exits — libpq sends a Terminate message, MySQL clients send `COM_QUIT`.
+
+`Adapter::Postgres` handles this by registering a `ForkManager.after_fork_child` hook that
+calls `discard!` in each worker: the inherited socket is redirected to `/dev/null`, and any
+later use of the adapter in the worker raises `DiscardedError`. New adapters should follow
+the same pattern. The discard mechanism itself is library-specific — mysql2 has
+`automatic_close = false`, trilogy has a native `discard!`. To check whether a library
+needs one at all: connect, fork an empty child that exits normally, wait for it, and query
+again from the parent (see the fork-safety specs in `postgres_spec.rb`).
+
## Schema DSL
The schema DSL lives in `migrations/tooling/lib/migrations/tooling/schema/dsl/`. Config sources
diff --git a/migrations/converters/lib/migrations/converters/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
index b7cd222e3a4..9b11d5015f2 100644
--- a/migrations/converters/lib/migrations/converters/adapter/postgres.rb
+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
@@ -6,23 +6,28 @@ module Migrations
module Converters
module Adapter
class Postgres
+ class DiscardedError < StandardError
+ end
+
def initialize(settings)
@connection = PG::Connection.new(settings)
@connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)
@connection.field_name_type = :symbol
configure_connection
+
+ @fork_hook = ForkManager.after_fork_child { discard! }
end
def exec(sql)
- @connection.exec(sql)
+ connection.exec(sql)
end
def query(sql, *params)
- @connection.send_query_params(sql, params)
- @connection.set_single_row_mode
+ connection.send_query_params(sql, params)
+ connection.set_single_row_mode
Enumerator.new do |y|
- while (result = @connection.get_result)
+ while (result = connection.get_result)
result.stream_each { |row| y.yield(row) }
result.clear
end
@@ -30,7 +35,7 @@ module Migrations
end
def query_first_row(sql)
- @connection.exec(sql).first
+ connection.exec(sql).first
end
def query_value(sql, column = nil)
@@ -46,19 +51,41 @@ module Migrations
end
def close
- unless @connection&.finished?
- @connection.finish
- @connection = nil
+ @connection.finish if @connection && [email protected]?
+ @connection = nil
+
+ if @fork_hook
+ ForkManager.remove_after_fork_child_hook(@fork_hook)
+ @fork_hook = nil
+ end
+ end
+
+ # Forked worker processes inherit the connection's socket. When such a
+ # process exits, the pg gem's cleanup sends a libpq Terminate message
+ # over that socket and the server closes the session — including for
+ # the main process, which still uses it. Redirecting the inherited
+ # file descriptor to /dev/null makes the cleanup harmless; calling
+ # `#finish` or `#close` instead would send the very message this
+ # needs to suppress.
+ def discard!
+ begin
+ @connection&.socket_io&.reopen(IO::NULL)
+ rescue StandardError
+ # When there's no usable socket (already closed, failed connection,
+ # older pg gem), there's also nothing the cleanup could corrupt.
end
+
+ @connection = nil
+ @discarded = true
end
def reset
- @connection.reset
+ connection.reset
configure_connection
end
def escape_string(str)
- @connection.escape_string(str)
+ connection.escape_string(str)
end
def encode_array(array)
@@ -69,6 +96,17 @@ module Migrations
private
+ def connection
+ if @discarded
+ raise DiscardedError,
+ "The source DB connection was discarded in this worker process. " \
+ "Only sources may query the source DB; processors must work " \
+ "with the items they receive."
+ end
+
+ @connection
+ end
+
def configure_connection
@connection.exec("SET client_min_messages TO WARNING")
end
diff --git a/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb
new file mode 100644
index 00000000000..fa2a7b099d1
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb
@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Adapter::Postgres do
+ def create_adapter(settings = {})
+ adapter = described_class.new(settings)
+ yield adapter
+ ensure
+ adapter&.close
+ end
+
+ context "with a stubbed connection" do
+ let(:connection) { instance_double(PG::Connection) }
+ let(:socket_io) { instance_double(IO) }
+
+ before do
+ allow(PG::Connection).to receive(:new).and_return(connection)
+ allow(PG::BasicTypeMapForResults).to receive(:new).with(connection).and_return(
+ instance_double(PG::BasicTypeMapForResults),
+ )
+
+ allow(connection).to receive(:type_map_for_results=)
+ allow(connection).to receive(:field_name_type=)
+ allow(connection).to receive(:exec)
+ allow(connection).to receive(:socket_io).and_return(socket_io)
+ allow(connection).to receive(:finished?).and_return(false)
+ allow(connection).to receive(:finish)
+ allow(socket_io).to receive(:reopen)
+ end
+
+ it "registers a fork hook on creation and removes it on `close`" do
+ expect(Migrations::ForkManager.size).to eq(0)
+
+ create_adapter do |adapter|
+ expect(Migrations::ForkManager.size).to eq(1)
+ adapter.close
+ expect(Migrations::ForkManager.size).to eq(0)
+ end
+ end
+
+ describe "#discard!" do
+ it "redirects the connection's socket to the null device without closing it" do
+ create_adapter do |adapter|
+ adapter.discard!
+
+ expect(socket_io).to have_received(:reopen).with(IO::NULL)
+ expect(connection).to_not have_received(:finish)
+ end
+ end
+
+ it "raises a `DiscardedError` when the adapter is used afterwards" do
+ create_adapter do |adapter|
+ adapter.discard!
+
+ expect { adapter.exec("SELECT 1") }.to raise_error(described_class::DiscardedError)
+ expect { adapter.query("SELECT 1") }.to raise_error(described_class::DiscardedError)
+ expect { adapter.query_value("SELECT 1") }.to raise_error(described_class::DiscardedError)
+ expect { adapter.reset }.to raise_error(described_class::DiscardedError)
+ end
+ end
+
+ it "ignores connections that no longer expose a usable socket" do
+ allow(connection).to receive(:socket_io).and_raise(PG::ConnectionBad)
+
+ create_adapter { |adapter| expect { adapter.discard! }.to_not raise_error }
+ end
+
+ it "can be followed by `close` without errors" do
+ create_adapter do |adapter|
+ adapter.discard!
+
+ expect { adapter.close }.to_not raise_error
+ expect(connection).to_not have_received(:finish)
+ expect(Migrations::ForkManager.size).to eq(0)
+ end
+ end
+ end
+
+ context "when `Migrations::ForkManager.fork` is used" do
+ it "discards the adapter in the child process, but not in the parent" do
+ create_adapter do |adapter|
+ pid =
+ Migrations::ForkManager.fork do
+ adapter.exec("SELECT 1")
+ exit!(1)
+ rescue described_class::DiscardedError
+ exit!(0)
+ end
+
+ _, status = Process.waitpid2(pid)
+ expect(status.exitstatus).to eq(0)
+
+ expect { adapter.exec("SELECT 1") }.to_not raise_error
+ end
+ end
+
+ it "no longer discards the adapter in child processes after `close`" do
+ create_adapter do |adapter|
+ allow(adapter).to receive(:discard!).and_call_original
+ adapter.close
+
+ pid =
+ Migrations::ForkManager.fork do
+ expect(adapter).to_not have_received(:discard!)
+ exit!(0)
+ rescue RSpec::Expectations::ExpectationNotMetError
+ exit!(1)
+ end
+
+ _, status = Process.waitpid2(pid)
+ expect(status.exitstatus).to eq(0)
+ end
+ end
+ end
+ end
+
+ context "with a real database connection", :rails do
+ def create_adapter(&block)
+ config = ActiveRecord::Base.connection_db_config.configuration_hash
+
+ super(
+ {
+ host: config[:host],
+ port: config[:port],
+ user: config[:username],
+ password: config[:password],
+ dbname: config[:database],
+ }.compact,
+ &block
+ )
+ end
+
+ it "keeps the connection of the main process usable after a forked process exits" do
+ create_adapter do |adapter|
+ expect(adapter.query_value("SELECT 1")).to eq(1)
+
+ # An empty fork is all it takes: without `discard!` the pg gem's
+ # cleanup at child exit sends a libpq Terminate message over the
+ # shared socket and the server closes the parent's session.
+ _, status = Process.waitpid2(Migrations::ForkManager.fork {})
+ expect(status).to be_success
+
+ expect(adapter.query_value("SELECT 1")).to eq(1)
+ end
+ end
+
+ it "doesn't disturb an in-flight single-row-mode stream in the main process" do
+ create_adapter do |adapter|
+ rows = adapter.query("SELECT i FROM generate_series(1, 100) AS s(i)")
+ expect(rows.next).to eq({ i: 1 })
+
+ # With concurrently running steps, workers fork while another step's
+ # source is still streaming results. The child-side discard must
+ # leave that stream alone; this is why there's no parent-side
+ # `before_fork` close/reopen for this connection.
+ _, status = Process.waitpid2(Migrations::ForkManager.fork {})
+ expect(status).to be_success
+
+ remaining = []
+ loop { remaining << rows.next }
+ expect(remaining.size).to eq(99)
+
+ expect(adapter.query_value("SELECT 1")).to eq(1)
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rb
index 7d87a37f6cf..7f905d53b74 100644
--- a/migrations/core/lib/migrations/common/fork_manager.rb
+++ b/migrations/core/lib/migrations/common/fork_manager.rb
@@ -4,6 +4,7 @@ module Migrations
module ForkManager
@before_fork_hooks = []
@after_fork_parent_hooks = []
+ @after_fork_child_hooks = []
@execute_parent_forks = true
class << self
@@ -39,10 +40,25 @@ module Migrations
@after_fork_parent_hooks.delete(block)
end
+ def after_fork_child(&block)
+ if block
+ @after_fork_child_hooks << block
+ block
+ end
+ end
+
+ def remove_after_fork_child_hook(block)
+ @after_fork_child_hooks.delete(block)
+ end
+
def fork
run_before_fork_hooks if @execute_parent_forks
- pid = Process.fork { yield }
+ pid =
+ Process.fork do
+ run_after_fork_child_hooks
+ yield
+ end
run_after_fork_parent_hooks if @execute_parent_forks
@@ -50,12 +66,13 @@ module Migrations
end
def size
- @before_fork_hooks.size + @after_fork_parent_hooks.size
+ @before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.size
end
def clear!
@before_fork_hooks.clear
@after_fork_parent_hooks.clear
+ @after_fork_child_hooks.clear
end
private
@@ -67,6 +84,10 @@ module Migrations
def run_after_fork_parent_hooks
@after_fork_parent_hooks.each(&:call)
end
+
+ def run_after_fork_child_hooks
+ @after_fork_child_hooks.each(&:call)
+ end
end
end
end
diff --git a/migrations/core/spec/lib/common/fork_manager_spec.rb b/migrations/core/spec/lib/common/fork_manager_spec.rb
new file mode 100644
index 00000000000..c02cfb37810
--- /dev/null
+++ b/migrations/core/spec/lib/common/fork_manager_spec.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::ForkManager do
+ after { described_class.clear! }
+
+ describe ".after_fork_child" do
+ it "runs hooks in the forked process before the fork block" do
+ read_io, write_io = IO.pipe
+ described_class.after_fork_child { write_io.write("hook:#{Process.pid}") }
+
+ pid = described_class.fork { write_io.write(" block:#{Process.pid}") }
+ Process.waitpid(pid)
+ write_io.close
+
+ expect(read_io.read).to eq("hook:#{pid} block:#{pid}")
+ read_io.close
+ end
+
+ it "runs hooks in every fork created by `batch_forks`" do
+ read_io, write_io = IO.pipe
+ described_class.after_fork_child { write_io.write("x") }
+
+ described_class.batch_forks { 2.times { Process.waitpid(described_class.fork {}) } }
+ write_io.close
+
+ expect(read_io.read).to eq("xx")
+ read_io.close
+ end
+
+ it "returns the hook so it can be removed again" do
+ hook = described_class.after_fork_child { raise "this hook should not run" }
+ expect(described_class.size).to eq(1)
+
+ described_class.remove_after_fork_child_hook(hook)
+ expect(described_class.size).to eq(0)
+
+ _, status = Process.waitpid2(described_class.fork {})
+ expect(status).to be_success
+ end
+ end
+
+ describe ".remove_after_fork_child_hook" do
+ it "stops the removed hook from running while other hooks keep running" do
+ read_io, write_io = IO.pipe
+ hook = described_class.after_fork_child { write_io.write("removed") }
+ described_class.after_fork_child { write_io.write("kept") }
+
+ described_class.remove_after_fork_child_hook(hook)
+ expect(described_class.size).to eq(1)
+
+ Process.waitpid(described_class.fork {})
+ write_io.close
+
+ expect(read_io.read).to eq("kept")
+ read_io.close
+ end
+ end
+end