A platform for community discussion. Free, open, simple.
MT: Only use INSERT OR IGNORE for tables that want it in the shard merge
Gerhard Schlagerb8d389cb1b19
diff --git a/migrations/core/lib/migrations/conversion/consolidator.rb b/migrations/core/lib/migrations/conversion/consolidator.rbindex d0a82cf9bcd..4a11093c531 100644--- a/migrations/core/lib/migrations/conversion/consolidator.rb+++ b/migrations/core/lib/migrations/conversion/consolidator.rb@@ -53,7 +53,9 @@ module Migrationsenddef merge(shard_path)- @fork_mutex.synchronize { @connection.merge_database(shard_path, tables: mergeable_tables) }+ @fork_mutex.synchronize do+ @connection.merge_database(shard_path, tables: mergeable_tables, dedupe_tables:)+ endrescue StandardError => e@errors << eensure@@ -65,6 +67,16 @@ module Migrationsdef mergeable_tables@mergeable_tables ||= @connection.tables - %w[config schema_migrations]end++ # The mergeable tables whose model opts into `INSERT OR IGNORE`; the rest+ # raise on a duplicate. Derived from the models, so adding a table needs no+ # change here (see `IntermediateDB.conflict_strategy_for`).+ def dedupe_tables+ @dedupe_tables ||=+ mergeable_tables.select do |table|+ Database::IntermediateDB.conflict_strategy_for(table) == :ignore+ end+ endendendenddiff --git a/migrations/core/lib/migrations/conversion/shard_manager.rb b/migrations/core/lib/migrations/conversion/shard_manager.rbindex 785390de076..04cc4355dbb 100644--- a/migrations/core/lib/migrations/conversion/shard_manager.rb+++ b/migrations/core/lib/migrations/conversion/shard_manager.rb@@ -13,10 +13,9 @@ module Migrations# them.## The template is a fresh, empty database migrated from the same schema as the- # run DB — not a copy of the run DB. That keeps a shard cheap to copy, and —- # importantly — means a run against an existing IntermediateDB doesn't drag the- # existing rows into every shard; the merge just adds each shard's new rows- # with `INSERT OR IGNORE`.+ # run DB — not a copy of the run DB. That keeps a shard cheap to copy, and+ # means a run against an existing IntermediateDB doesn't drag the existing rows+ # into every shard; the merge just adds each shard's new rows.class ShardManager# @param canonical_path [String] the run's IntermediateDB; the shards and# their template are placed next to itdiff --git a/migrations/core/lib/migrations/database/connection.rb b/migrations/core/lib/migrations/database/connection.rbindex b8c2af15ad8..b852c9299ed 100644--- a/migrations/core/lib/migrations/database/connection.rb+++ b/migrations/core/lib/migrations/database/connection.rb@@ -87,13 +87,20 @@ module Migrationsend# `ATTACH` can't run inside a transaction, so commit any open batch first.- def merge_database(other_path, tables:)+ # `dedupe_tables` merge with `INSERT OR IGNORE`; the rest raise on a+ # duplicate row (see `Consolidator`).+ def merge_database(other_path, tables:, dedupe_tables: [])commit_transaction@db.execute("ATTACH DATABASE ? AS merge_source", other_path)begintables.each do |table|quoted = quote_identifier(table)- @db.execute("INSERT OR IGNORE INTO main.#{quoted} SELECT * FROM merge_source.#{quoted}")+ conflict_clause = dedupe_tables.include?(table) ? "OR IGNORE " : ""+ @db.execute(+ "INSERT #{conflict_clause}INTO main.#{quoted} SELECT * FROM merge_source.#{quoted}",+ )+ rescue Extralite::Error => e+ raise "Failed to merge table #{table.inspect}: #{e.message}"endensure@db.execute("DETACH DATABASE merge_source")diff --git a/migrations/core/lib/migrations/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rbindex f81be38c122..71c4c8f761a 100644--- a/migrations/core/lib/migrations/database/intermediate_db.rb+++ b/migrations/core/lib/migrations/database/intermediate_db.rb@@ -23,6 +23,18 @@ module Migrations@db.insert(sql, parameters)end+ # How the shard merge should handle a duplicate row in `table`: `:ignore`+ # for a model that inserts with `INSERT OR IGNORE`, else `:raise` — so a+ # genuine duplicate is an error, not a silently dropped row. Read from the+ # model, so a new table needs no change here.+ def self.conflict_strategy_for(table)+ module_name = table.to_s.singularize.camelize+ return :raise unless const_defined?(module_name, false)++ model = const_get(module_name, false)+ model.respond_to?(:conflict_strategy) ? model.conflict_strategy : :raise+ end+def self.close@db.close if @dbenddiff --git a/migrations/core/lib/migrations/database/intermediate_db/upload.rb b/migrations/core/lib/migrations/database/intermediate_db/upload.rbindex 8ad12e9d748..ade71e89a22 100644--- a/migrations/core/lib/migrations/database/intermediate_db/upload.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/upload.rb@@ -20,6 +20,13 @@ module MigrationsSQLprivate_constant :SQL+ # Several steps can reference the same file, and rows are identical for a+ # given `id` (a content hash), so a duplicate `id` is expected: keep the+ # first, ignore the rest — both here (`SQL` above) and in the shard merge.+ def self.conflict_strategy+ :ignore+ end+def self.create_for_file(path:,filename: nil,diff --git a/migrations/core/spec/lib/conversion/consolidator_spec.rb b/migrations/core/spec/lib/conversion/consolidator_spec.rbindex 548e539c8f6..4e4a28018cf 100644--- a/migrations/core/spec/lib/conversion/consolidator_spec.rb+++ b/migrations/core/spec/lib/conversion/consolidator_spec.rb@@ -18,9 +18,21 @@ RSpec.describe Migrations::Conversion::Consolidator doerrors = consolidator.drain# `config` and `schema_migrations` are excluded from the merge- expect(connection).to have_received(:merge_database).with("a.db", tables: %w[topics])- expect(connection).to have_received(:merge_database).with("b.db", tables: %w[topics])- expect(connection).to have_received(:merge_database).with("c.db", tables: %w[topics])+ expect(connection).to have_received(:merge_database).with(+ "a.db",+ tables: %w[topics],+ dedupe_tables: [],+ )+ expect(connection).to have_received(:merge_database).with(+ "b.db",+ tables: %w[topics],+ dedupe_tables: [],+ )+ expect(connection).to have_received(:merge_database).with(+ "c.db",+ tables: %w[topics],+ dedupe_tables: [],+ )expect(shard_manager).to have_received(:discard).with("a.db")expect(shard_manager).to have_received(:discard).with("c.db")expect(errors).to be_empty@@ -28,7 +40,11 @@ RSpec.describe Migrations::Conversion::Consolidator doit "collects a merge error and still discards the shard" doboom = StandardError.new("merge boom")- allow(connection).to receive(:merge_database).with("bad.db", tables: anything).and_raise(boom)+ allow(connection).to receive(:merge_database).with(+ "bad.db",+ tables: anything,+ dedupe_tables: anything,+ ).and_raise(boom)consolidator = described_class.new(shard_manager, connection, fork_mutex)consolidator.enqueue(%w[bad.db])@@ -37,4 +53,30 @@ RSpec.describe Migrations::Conversion::Consolidator doexpect(errors).to eq([boom])expect(shard_manager).to have_received(:discard).with("bad.db")end++ it "merges `OR IGNORE` for the tables whose model declares it, and plain for the rest" do+ # A fake model that opts into `OR IGNORE`, resolved by table name exactly like+ # a real one — so the merge clause is derived from the models, not hard-coded.+ fake_model =+ Module.new do+ def self.conflict_strategy+ :ignore+ end+ end+ stub_const("Migrations::Database::IntermediateDB::FakeThing", fake_model)++ allow(connection).to receive(:tables).and_return(+ %w[uploads topics fake_things config schema_migrations],+ )++ consolidator = described_class.new(shard_manager, connection, fork_mutex)+ consolidator.enqueue(%w[a.db])+ consolidator.drain++ expect(connection).to have_received(:merge_database).with(+ "a.db",+ tables: %w[uploads topics fake_things],+ dedupe_tables: %w[uploads fake_things],+ )+ endenddiff --git a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rbindex 29b70bb198e..7018c76c2b9 100644--- a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb+++ b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb@@ -535,7 +535,25 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doObject.send(:remove_const, "TempIntegrationSteps")end- it "adds new rows to an existing IntermediateDB without dropping or overwriting" do+ # Runs `TempIntegrationSteps::Keyed`, whose processor writes each item into the+ # `keyed` table. `keyed` has no `INSERT OR IGNORE` model, so its shard merges+ # with a plain, raising `INSERT`.+ def run_keyed_step+ step_classes = [TempIntegrationSteps::Keyed]+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))+ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 2,+ reporter:,+ step_factory: ->(step_class) { step_class.new },+ shard_manager: @shard_manager,+ writer: @writer,+ ).run+ ensure+ reporter&.close+ end++ it "adds new, non-colliding rows to an existing IntermediateDB" do# A row from a previous run already lives in the DB.Migrations::Database::IntermediateDB.insert("INSERT INTO keyed VALUES (?, ?)", 1, "existing")@@ -547,8 +565,8 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doClass.new(Migrations::Conversion::Step) dosource dodef items- # id 1 collides with the existing row; id 2 is new- [{ id: 1, label: "new" }, { id: 2, label: "added" }]+ # both ids are new; neither collides with the existing row+ [{ id: 2, label: "added" }, { id: 3, label: "more" }]endend@@ -566,25 +584,52 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doend,)- step_classes = [TempIntegrationSteps::Keyed]- reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))- Migrations::Conversion::StepScheduler.new(- step_classes:,- budget: 2,- reporter:,- step_factory: ->(step_class) { step_class.new },- shard_manager: @shard_manager,- writer: @writer,- ).run- reporter.close+ run_keyed_stepMigrations::Database::IntermediateDB.closedb = Extralite::Database.new(@db_path)keyed = db.query_array("SELECT id, label FROM keyed ORDER BY id")db.close- # the existing row is kept as-is (not overwritten), and the new row is added- expect(keyed).to eq([[1, "existing"], [2, "added"]])+ # the existing row is kept and the new, non-colliding rows are appended+ expect(keyed).to eq([[1, "existing"], [2, "added"], [3, "more"]])+ ensure+ Object.send(:remove_const, "TempIntegrationSteps")+ end++ it "fails the run, naming the table, when a shard row collides with an existing one" do+ # `keyed` merges with a plain `INSERT`, so a shard row that duplicates a row+ # already in the run DB is a genuine double-write: it fails the run loudly+ # instead of being silently dropped, matching the single-writer contract.+ Migrations::Database::IntermediateDB.insert("INSERT INTO keyed VALUES (?, ?)", 1, "existing")++ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ const_set(+ "Keyed",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 1, label: "collides" }] # id 1 already lives in the run DB+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO keyed VALUES (?, ?)",+ item[:id],+ item[:label],+ )+ end+ end+ end,+ )+ end,+ )++ expect { run_keyed_step }.to raise_error(Migrations::Conversion::ConvertError, /keyed/)ensureObject.send(:remove_const, "TempIntegrationSteps")enddiff --git a/migrations/core/spec/lib/database/connection_spec.rb b/migrations/core/spec/lib/database/connection_spec.rbindex dc4dac629bc..3696e50500a 100644--- a/migrations/core/spec/lib/database/connection_spec.rb+++ b/migrations/core/spec/lib/database/connection_spec.rb@@ -125,6 +125,90 @@ RSpec.describe Migrations::Database::Connection doendend+ describe "#merge_database" do+ def create_schema(db)+ db.execute(<<~SQL)+ CREATE TABLE topic_tags (topic_id INTEGER, tag_id INTEGER, PRIMARY KEY (topic_id, tag_id))+ SQL+ db.execute("CREATE TABLE uploads (id TEXT PRIMARY KEY, filename TEXT)")+ end++ # Yields a run-DB `connection` and a closed shard database at `source_path`,+ # both carrying the same schema. The block seeds each and runs the merge.+ def with_merge_dbs+ Dir.mktmpdir do |dir|+ source_path = File.join(dir, "source.db")+ connection = described_class.new(path: File.join(dir, "main.db"))+ create_schema(connection.db)++ source = described_class.open_database(path: source_path)+ create_schema(source)++ yield connection, source, source_path+ ensure+ connection&.close+ end+ end++ it "merges non-colliding rows from every listed table" do+ with_merge_dbs do |connection, source, source_path|+ connection.db.execute("INSERT INTO topic_tags VALUES (1, 10)")+ source.execute("INSERT INTO topic_tags VALUES (2, 20)")+ source.execute("INSERT INTO uploads VALUES ('a', 'a.png')")+ source.close++ connection.merge_database(source_path, tables: %w[topic_tags uploads])++ expect(connection.db.query("SELECT topic_id, tag_id FROM topic_tags")).to contain_exactly(+ { topic_id: 1, tag_id: 10 },+ { topic_id: 2, tag_id: 20 },+ )+ expect(connection.db.query_splat("SELECT id FROM uploads")).to contain_exactly("a")+ end+ end++ it "dedups an `OR IGNORE` table across shards, keeping the first writer's row" do+ with_merge_dbs do |connection, source, source_path|+ connection.db.execute("INSERT INTO uploads VALUES ('x', 'first.png')")+ source.execute("INSERT INTO uploads VALUES ('x', 'second.png')")+ source.close++ connection.merge_database(source_path, tables: %w[uploads], dedupe_tables: %w[uploads])++ expect(connection.db.query("SELECT id, filename FROM uploads")).to contain_exactly(+ { id: "x", filename: "first.png" },+ )+ end+ end++ it "raises, naming the table, when a plain table has a cross-shard duplicate" do+ with_merge_dbs do |connection, source, source_path|+ connection.db.execute("INSERT INTO topic_tags VALUES (1, 10)")+ source.execute("INSERT INTO topic_tags VALUES (1, 10)")+ source.close++ expect { connection.merge_database(source_path, tables: %w[topic_tags]) }.to raise_error(+ /topic_tags/,+ )+ end+ end++ it "detaches the merge source even when a merge raises, so the next merge works" do+ with_merge_dbs do |connection, source, source_path|+ connection.db.execute("INSERT INTO topic_tags VALUES (1, 10)")+ source.execute("INSERT INTO topic_tags VALUES (1, 10)")+ source.close++ expect { connection.merge_database(source_path, tables: %w[topic_tags]) }.to raise_error(+ /topic_tags/,+ )++ # A leaked `merge_source` attachment would make this second merge raise.+ expect { connection.merge_database(source_path, tables: %w[uploads]) }.not_to raise_error+ end+ end+ end+context "when `Migrations::ForkManager.fork` is used" doit "temporarily closes the connection while a process fork is created" docreate_connection do |connection|diff --git a/migrations/core/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rbindex 8ac93a4efc3..a23f12e3411 100644--- a/migrations/core/spec/lib/database/intermediate_db_spec.rb+++ b/migrations/core/spec/lib/database/intermediate_db_spec.rb@@ -57,6 +57,32 @@ RSpec.describe Migrations::Database::IntermediateDB doendend+ describe ".conflict_strategy_for" do+ it "returns `:ignore` for a table whose model declares it" do+ expect(described_class.conflict_strategy_for("uploads")).to eq(:ignore)+ end++ it "returns `:raise` for a table whose model does not declare a strategy" do+ expect(described_class.conflict_strategy_for("topic_tags")).to eq(:raise)+ end++ it "returns `:raise` for a table without a model" do+ expect(described_class.conflict_strategy_for("schema_migrations")).to eq(:raise)+ end++ it "reads the strategy from the model, so a new `OR IGNORE` model flips it" do+ model =+ Module.new do+ def self.conflict_strategy+ :ignore+ end+ end+ stub_const("#{described_class}::Widget", model)++ expect(described_class.conflict_strategy_for("widgets")).to eq(:ignore)+ end+ end+describe ".with_connection" dolet(:previous_connection) { create_connection_double }let(:temporary_connection) { create_connection_double }