A platform for community discussion. Free, open, simple.
MT: Discard inherited source DB connections in worker processes
Gerhard Schlagercadc068cf448
diff --git a/migrations/README.md b/migrations/README.mdindex 8a9b6aec01c..18730b6751d 100644--- a/migrations/README.md+++ b/migrations/README.md@@ -29,6 +29,22 @@ Public converters live in `converters/lib/migrations/converters/`. To run a priv(closed-source) converter, put its code in a subdirectory of `private/converters/`(or point `MIGRATIONS_PRIVATE_CONVERTERS_PATH` at it).+### Source DB adapters and fork safety++Worker processes inherit the source DB connection's socket from the main process. Whether+that's dangerous depends on the client library: a destructor that only closes the file+descriptor is harmless (the parent still holds it, so the kernel sends nothing over the+wire), but a destructor that writes a protocol goodbye kills the parent's session as soon+as a worker exits — libpq sends a Terminate message, MySQL clients send `COM_QUIT`.++`Adapter::Postgres` handles this by registering a `ForkManager.after_fork_child` hook that+calls `discard!` in each worker: the inherited socket is redirected to `/dev/null`, and any+later use of the adapter in the worker raises `DiscardedError`. New adapters should follow+the same pattern. The discard mechanism itself is library-specific — mysql2 has+`automatic_close = false`, trilogy has a native `discard!`. To check whether a library+needs one at all: connect, fork an empty child that exits normally, wait for it, and query+again from the parent (see the fork-safety specs in `postgres_spec.rb`).+## Schema DSLThe schema DSL lives in `migrations/tooling/lib/migrations/tooling/schema/dsl/`. Config sourcesdiff --git a/migrations/converters/lib/migrations/converters/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rbindex b7cd222e3a4..9b11d5015f2 100644--- a/migrations/converters/lib/migrations/converters/adapter/postgres.rb+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb@@ -6,23 +6,28 @@ module Migrationsmodule Convertersmodule Adapterclass Postgres+ class DiscardedError < StandardError+ end+def initialize(settings)@connection = PG::Connection.new(settings)@connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)@connection.field_name_type = :symbolconfigure_connection++ @fork_hook = ForkManager.after_fork_child { discard! }enddef exec(sql)- @connection.exec(sql)+ connection.exec(sql)enddef query(sql, *params)- @connection.send_query_params(sql, params)- @connection.set_single_row_mode+ connection.send_query_params(sql, params)+ connection.set_single_row_modeEnumerator.new do |y|- while (result = @connection.get_result)+ while (result = connection.get_result)result.stream_each { |row| y.yield(row) }result.clearend@@ -30,7 +35,7 @@ module Migrationsenddef query_first_row(sql)- @connection.exec(sql).first+ connection.exec(sql).firstenddef query_value(sql, column = nil)@@ -46,19 +51,41 @@ module Migrationsenddef close- unless @connection&.finished?- @connection.finish- @connection = nil+ @connection.finish if @connection && [email protected]?+ @connection = nil++ if @fork_hook+ ForkManager.remove_after_fork_child_hook(@fork_hook)+ @fork_hook = nil+ end+ end++ # Forked worker processes inherit the connection's socket. When such a+ # process exits, the pg gem's cleanup sends a libpq Terminate message+ # over that socket and the server closes the session — including for+ # the main process, which still uses it. Redirecting the inherited+ # file descriptor to /dev/null makes the cleanup harmless; calling+ # `#finish` or `#close` instead would send the very message this+ # needs to suppress.+ def discard!+ begin+ @connection&.socket_io&.reopen(IO::NULL)+ rescue StandardError+ # When there's no usable socket (already closed, failed connection,+ # older pg gem), there's also nothing the cleanup could corrupt.end++ @connection = nil+ @discarded = trueenddef reset- @connection.reset+ connection.resetconfigure_connectionenddef escape_string(str)- @connection.escape_string(str)+ connection.escape_string(str)enddef encode_array(array)@@ -69,6 +96,17 @@ module Migrationsprivate+ def connection+ if @discarded+ raise DiscardedError,+ "The source DB connection was discarded in this worker process. " \+ "Only sources may query the source DB; processors must work " \+ "with the items they receive."+ end++ @connection+ end+def configure_connection@connection.exec("SET client_min_messages TO WARNING")enddiff --git a/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rbnew file mode 100644index 00000000000..fa2a7b099d1--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb@@ -0,0 +1,166 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Adapter::Postgres do+ def create_adapter(settings = {})+ adapter = described_class.new(settings)+ yield adapter+ ensure+ adapter&.close+ end++ context "with a stubbed connection" do+ let(:connection) { instance_double(PG::Connection) }+ let(:socket_io) { instance_double(IO) }++ before do+ allow(PG::Connection).to receive(:new).and_return(connection)+ allow(PG::BasicTypeMapForResults).to receive(:new).with(connection).and_return(+ instance_double(PG::BasicTypeMapForResults),+ )++ allow(connection).to receive(:type_map_for_results=)+ allow(connection).to receive(:field_name_type=)+ allow(connection).to receive(:exec)+ allow(connection).to receive(:socket_io).and_return(socket_io)+ allow(connection).to receive(:finished?).and_return(false)+ allow(connection).to receive(:finish)+ allow(socket_io).to receive(:reopen)+ end++ it "registers a fork hook on creation and removes it on `close`" do+ expect(Migrations::ForkManager.size).to eq(0)++ create_adapter do |adapter|+ expect(Migrations::ForkManager.size).to eq(1)+ adapter.close+ expect(Migrations::ForkManager.size).to eq(0)+ end+ end++ describe "#discard!" do+ it "redirects the connection's socket to the null device without closing it" do+ create_adapter do |adapter|+ adapter.discard!++ expect(socket_io).to have_received(:reopen).with(IO::NULL)+ expect(connection).to_not have_received(:finish)+ end+ end++ it "raises a `DiscardedError` when the adapter is used afterwards" do+ create_adapter do |adapter|+ adapter.discard!++ expect { adapter.exec("SELECT 1") }.to raise_error(described_class::DiscardedError)+ expect { adapter.query("SELECT 1") }.to raise_error(described_class::DiscardedError)+ expect { adapter.query_value("SELECT 1") }.to raise_error(described_class::DiscardedError)+ expect { adapter.reset }.to raise_error(described_class::DiscardedError)+ end+ end++ it "ignores connections that no longer expose a usable socket" do+ allow(connection).to receive(:socket_io).and_raise(PG::ConnectionBad)++ create_adapter { |adapter| expect { adapter.discard! }.to_not raise_error }+ end++ it "can be followed by `close` without errors" do+ create_adapter do |adapter|+ adapter.discard!++ expect { adapter.close }.to_not raise_error+ expect(connection).to_not have_received(:finish)+ expect(Migrations::ForkManager.size).to eq(0)+ end+ end+ end++ context "when `Migrations::ForkManager.fork` is used" do+ it "discards the adapter in the child process, but not in the parent" do+ create_adapter do |adapter|+ pid =+ Migrations::ForkManager.fork do+ adapter.exec("SELECT 1")+ exit!(1)+ rescue described_class::DiscardedError+ exit!(0)+ end++ _, status = Process.waitpid2(pid)+ expect(status.exitstatus).to eq(0)++ expect { adapter.exec("SELECT 1") }.to_not raise_error+ end+ end++ it "no longer discards the adapter in child processes after `close`" do+ create_adapter do |adapter|+ allow(adapter).to receive(:discard!).and_call_original+ adapter.close++ pid =+ Migrations::ForkManager.fork do+ expect(adapter).to_not have_received(:discard!)+ exit!(0)+ rescue RSpec::Expectations::ExpectationNotMetError+ exit!(1)+ end++ _, status = Process.waitpid2(pid)+ expect(status.exitstatus).to eq(0)+ end+ end+ end+ end++ context "with a real database connection", :rails do+ def create_adapter(&block)+ config = ActiveRecord::Base.connection_db_config.configuration_hash++ super(+ {+ host: config[:host],+ port: config[:port],+ user: config[:username],+ password: config[:password],+ dbname: config[:database],+ }.compact,+ &block+ )+ end++ it "keeps the connection of the main process usable after a forked process exits" do+ create_adapter do |adapter|+ expect(adapter.query_value("SELECT 1")).to eq(1)++ # An empty fork is all it takes: without `discard!` the pg gem's+ # cleanup at child exit sends a libpq Terminate message over the+ # shared socket and the server closes the parent's session.+ _, status = Process.waitpid2(Migrations::ForkManager.fork {})+ expect(status).to be_success++ expect(adapter.query_value("SELECT 1")).to eq(1)+ end+ end++ it "doesn't disturb an in-flight single-row-mode stream in the main process" do+ create_adapter do |adapter|+ rows = adapter.query("SELECT i FROM generate_series(1, 100) AS s(i)")+ expect(rows.next).to eq({ i: 1 })++ # With concurrently running steps, workers fork while another step's+ # source is still streaming results. The child-side discard must+ # leave that stream alone; this is why there's no parent-side+ # `before_fork` close/reopen for this connection.+ _, status = Process.waitpid2(Migrations::ForkManager.fork {})+ expect(status).to be_success++ remaining = []+ loop { remaining << rows.next }+ expect(remaining.size).to eq(99)++ expect(adapter.query_value("SELECT 1")).to eq(1)+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rbindex 7d87a37f6cf..7f905d53b74 100644--- a/migrations/core/lib/migrations/common/fork_manager.rb+++ b/migrations/core/lib/migrations/common/fork_manager.rb@@ -4,6 +4,7 @@ module Migrationsmodule ForkManager@before_fork_hooks = []@after_fork_parent_hooks = []+ @after_fork_child_hooks = []@execute_parent_forks = trueclass << self@@ -39,10 +40,25 @@ module Migrations@after_fork_parent_hooks.delete(block)end+ def after_fork_child(&block)+ if block+ @after_fork_child_hooks << block+ block+ end+ end++ def remove_after_fork_child_hook(block)+ @after_fork_child_hooks.delete(block)+ end+def forkrun_before_fork_hooks if @execute_parent_forks- pid = Process.fork { yield }+ pid =+ Process.fork do+ run_after_fork_child_hooks+ yield+ endrun_after_fork_parent_hooks if @execute_parent_forks@@ -50,12 +66,13 @@ module Migrationsenddef size- @before_fork_hooks.size + @after_fork_parent_hooks.size+ @before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.sizeenddef clear!@before_fork_hooks.clear@after_fork_parent_hooks.clear+ @after_fork_child_hooks.clearendprivate@@ -67,6 +84,10 @@ module Migrationsdef run_after_fork_parent_hooks@after_fork_parent_hooks.each(&:call)end++ def run_after_fork_child_hooks+ @after_fork_child_hooks.each(&:call)+ endendendenddiff --git a/migrations/core/spec/lib/common/fork_manager_spec.rb b/migrations/core/spec/lib/common/fork_manager_spec.rbnew file mode 100644index 00000000000..c02cfb37810--- /dev/null+++ b/migrations/core/spec/lib/common/fork_manager_spec.rb@@ -0,0 +1,58 @@+# frozen_string_literal: true++RSpec.describe Migrations::ForkManager do+ after { described_class.clear! }++ describe ".after_fork_child" do+ it "runs hooks in the forked process before the fork block" do+ read_io, write_io = IO.pipe+ described_class.after_fork_child { write_io.write("hook:#{Process.pid}") }++ pid = described_class.fork { write_io.write(" block:#{Process.pid}") }+ Process.waitpid(pid)+ write_io.close++ expect(read_io.read).to eq("hook:#{pid} block:#{pid}")+ read_io.close+ end++ it "runs hooks in every fork created by `batch_forks`" do+ read_io, write_io = IO.pipe+ described_class.after_fork_child { write_io.write("x") }++ described_class.batch_forks { 2.times { Process.waitpid(described_class.fork {}) } }+ write_io.close++ expect(read_io.read).to eq("xx")+ read_io.close+ end++ it "returns the hook so it can be removed again" do+ hook = described_class.after_fork_child { raise "this hook should not run" }+ expect(described_class.size).to eq(1)++ described_class.remove_after_fork_child_hook(hook)+ expect(described_class.size).to eq(0)++ _, status = Process.waitpid2(described_class.fork {})+ expect(status).to be_success+ end+ end++ describe ".remove_after_fork_child_hook" do+ it "stops the removed hook from running while other hooks keep running" do+ read_io, write_io = IO.pipe+ hook = described_class.after_fork_child { write_io.write("removed") }+ described_class.after_fork_child { write_io.write("kept") }++ described_class.remove_after_fork_child_hook(hook)+ expect(described_class.size).to eq(1)++ Process.waitpid(described_class.fork {})+ write_io.close++ expect(read_io.read).to eq("kept")+ read_io.close+ end+ end+end