Skip to content

A platform for community discussion. Free, open, simple.

MT: Only use INSERT OR IGNORE for tables that want it in the shard merge

Gerhard Schlagerb8d389cb1b19
diff --git a/migrations/core/lib/migrations/conversion/consolidator.rb b/migrations/core/lib/migrations/conversion/consolidator.rb
index d0a82cf9bcd..4a11093c531 100644
--- a/migrations/core/lib/migrations/conversion/consolidator.rb
+++ b/migrations/core/lib/migrations/conversion/consolidator.rb
@@ -53,7 +53,9 @@ module Migrations
end
def merge(shard_path)
- @fork_mutex.synchronize { @connection.merge_database(shard_path, tables: mergeable_tables) }
+ @fork_mutex.synchronize do
+ @connection.merge_database(shard_path, tables: mergeable_tables, dedupe_tables:)
+ end
rescue StandardError => e
@errors << e
ensure
@@ -65,6 +67,16 @@ module Migrations
def mergeable_tables
@mergeable_tables ||= @connection.tables - %w[config schema_migrations]
end
+
+ # The mergeable tables whose model opts into `INSERT OR IGNORE`; the rest
+ # raise on a duplicate. Derived from the models, so adding a table needs no
+ # change here (see `IntermediateDB.conflict_strategy_for`).
+ def dedupe_tables
+ @dedupe_tables ||=
+ mergeable_tables.select do |table|
+ Database::IntermediateDB.conflict_strategy_for(table) == :ignore
+ end
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/shard_manager.rb b/migrations/core/lib/migrations/conversion/shard_manager.rb
index 785390de076..04cc4355dbb 100644
--- a/migrations/core/lib/migrations/conversion/shard_manager.rb
+++ b/migrations/core/lib/migrations/conversion/shard_manager.rb
@@ -13,10 +13,9 @@ module Migrations
# them.
#
# The template is a fresh, empty database migrated from the same schema as the
- # run DB — not a copy of the run DB. That keeps a shard cheap to copy, and —
- # importantly — means a run against an existing IntermediateDB doesn't drag the
- # existing rows into every shard; the merge just adds each shard's new rows
- # with `INSERT OR IGNORE`.
+ # run DB — not a copy of the run DB. That keeps a shard cheap to copy, and
+ # means a run against an existing IntermediateDB doesn't drag the existing rows
+ # into every shard; the merge just adds each shard's new rows.
class ShardManager
# @param canonical_path [String] the run's IntermediateDB; the shards and
# their template are placed next to it
diff --git a/migrations/core/lib/migrations/database/connection.rb b/migrations/core/lib/migrations/database/connection.rb
index b8c2af15ad8..b852c9299ed 100644
--- a/migrations/core/lib/migrations/database/connection.rb
+++ b/migrations/core/lib/migrations/database/connection.rb
@@ -87,13 +87,20 @@ module Migrations
end
# `ATTACH` can't run inside a transaction, so commit any open batch first.
- def merge_database(other_path, tables:)
+ # `dedupe_tables` merge with `INSERT OR IGNORE`; the rest raise on a
+ # duplicate row (see `Consolidator`).
+ def merge_database(other_path, tables:, dedupe_tables: [])
commit_transaction
@db.execute("ATTACH DATABASE ? AS merge_source", other_path)
begin
tables.each do |table|
quoted = quote_identifier(table)
- @db.execute("INSERT OR IGNORE INTO main.#{quoted} SELECT * FROM merge_source.#{quoted}")
+ conflict_clause = dedupe_tables.include?(table) ? "OR IGNORE " : ""
+ @db.execute(
+ "INSERT #{conflict_clause}INTO main.#{quoted} SELECT * FROM merge_source.#{quoted}",
+ )
+ rescue Extralite::Error => e
+ raise "Failed to merge table #{table.inspect}: #{e.message}"
end
ensure
@db.execute("DETACH DATABASE merge_source")
diff --git a/migrations/core/lib/migrations/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rb
index f81be38c122..71c4c8f761a 100644
--- a/migrations/core/lib/migrations/database/intermediate_db.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db.rb
@@ -23,6 +23,18 @@ module Migrations
@db.insert(sql, parameters)
end
+ # How the shard merge should handle a duplicate row in `table`: `:ignore`
+ # for a model that inserts with `INSERT OR IGNORE`, else `:raise` — so a
+ # genuine duplicate is an error, not a silently dropped row. Read from the
+ # model, so a new table needs no change here.
+ def self.conflict_strategy_for(table)
+ module_name = table.to_s.singularize.camelize
+ return :raise unless const_defined?(module_name, false)
+
+ model = const_get(module_name, false)
+ model.respond_to?(:conflict_strategy) ? model.conflict_strategy : :raise
+ end
+
def self.close
@db.close if @db
end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/upload.rb b/migrations/core/lib/migrations/database/intermediate_db/upload.rb
index 8ad12e9d748..ade71e89a22 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/upload.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/upload.rb
@@ -20,6 +20,13 @@ module Migrations
SQL
private_constant :SQL
+ # Several steps can reference the same file, and rows are identical for a
+ # given `id` (a content hash), so a duplicate `id` is expected: keep the
+ # first, ignore the rest — both here (`SQL` above) and in the shard merge.
+ def self.conflict_strategy
+ :ignore
+ end
+
def self.create_for_file(
path:,
filename: nil,
diff --git a/migrations/core/spec/lib/conversion/consolidator_spec.rb b/migrations/core/spec/lib/conversion/consolidator_spec.rb
index 548e539c8f6..4e4a28018cf 100644
--- a/migrations/core/spec/lib/conversion/consolidator_spec.rb
+++ b/migrations/core/spec/lib/conversion/consolidator_spec.rb
@@ -18,9 +18,21 @@ RSpec.describe Migrations::Conversion::Consolidator do
errors = consolidator.drain
# `config` and `schema_migrations` are excluded from the merge
- expect(connection).to have_received(:merge_database).with("a.db", tables: %w[topics])
- expect(connection).to have_received(:merge_database).with("b.db", tables: %w[topics])
- expect(connection).to have_received(:merge_database).with("c.db", tables: %w[topics])
+ expect(connection).to have_received(:merge_database).with(
+ "a.db",
+ tables: %w[topics],
+ dedupe_tables: [],
+ )
+ expect(connection).to have_received(:merge_database).with(
+ "b.db",
+ tables: %w[topics],
+ dedupe_tables: [],
+ )
+ expect(connection).to have_received(:merge_database).with(
+ "c.db",
+ tables: %w[topics],
+ dedupe_tables: [],
+ )
expect(shard_manager).to have_received(:discard).with("a.db")
expect(shard_manager).to have_received(:discard).with("c.db")
expect(errors).to be_empty
@@ -28,7 +40,11 @@ RSpec.describe Migrations::Conversion::Consolidator do
it "collects a merge error and still discards the shard" do
boom = StandardError.new("merge boom")
- allow(connection).to receive(:merge_database).with("bad.db", tables: anything).and_raise(boom)
+ allow(connection).to receive(:merge_database).with(
+ "bad.db",
+ tables: anything,
+ dedupe_tables: anything,
+ ).and_raise(boom)
consolidator = described_class.new(shard_manager, connection, fork_mutex)
consolidator.enqueue(%w[bad.db])
@@ -37,4 +53,30 @@ RSpec.describe Migrations::Conversion::Consolidator do
expect(errors).to eq([boom])
expect(shard_manager).to have_received(:discard).with("bad.db")
end
+
+ it "merges `OR IGNORE` for the tables whose model declares it, and plain for the rest" do
+ # A fake model that opts into `OR IGNORE`, resolved by table name exactly like
+ # a real one — so the merge clause is derived from the models, not hard-coded.
+ fake_model =
+ Module.new do
+ def self.conflict_strategy
+ :ignore
+ end
+ end
+ stub_const("Migrations::Database::IntermediateDB::FakeThing", fake_model)
+
+ allow(connection).to receive(:tables).and_return(
+ %w[uploads topics fake_things config schema_migrations],
+ )
+
+ consolidator = described_class.new(shard_manager, connection, fork_mutex)
+ consolidator.enqueue(%w[a.db])
+ consolidator.drain
+
+ expect(connection).to have_received(:merge_database).with(
+ "a.db",
+ tables: %w[uploads topics fake_things],
+ dedupe_tables: %w[uploads fake_things],
+ )
+ end
end
diff --git a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
index 29b70bb198e..7018c76c2b9 100644
--- a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
+++ b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
@@ -535,7 +535,25 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
Object.send(:remove_const, "TempIntegrationSteps")
end
- it "adds new rows to an existing IntermediateDB without dropping or overwriting" do
+ # Runs `TempIntegrationSteps::Keyed`, whose processor writes each item into the
+ # `keyed` table. `keyed` has no `INSERT OR IGNORE` model, so its shard merges
+ # with a plain, raising `INSERT`.
+ def run_keyed_step
+ step_classes = [TempIntegrationSteps::Keyed]
+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 2,
+ reporter:,
+ step_factory: ->(step_class) { step_class.new },
+ shard_manager: @shard_manager,
+ writer: @writer,
+ ).run
+ ensure
+ reporter&.close
+ end
+
+ it "adds new, non-colliding rows to an existing IntermediateDB" do
# A row from a previous run already lives in the DB.
Migrations::Database::IntermediateDB.insert("INSERT INTO keyed VALUES (?, ?)", 1, "existing")
@@ -547,8 +565,8 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
Class.new(Migrations::Conversion::Step) do
source do
def items
- # id 1 collides with the existing row; id 2 is new
- [{ id: 1, label: "new" }, { id: 2, label: "added" }]
+ # both ids are new; neither collides with the existing row
+ [{ id: 2, label: "added" }, { id: 3, label: "more" }]
end
end
@@ -566,25 +584,52 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
end,
)
- step_classes = [TempIntegrationSteps::Keyed]
- reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
- Migrations::Conversion::StepScheduler.new(
- step_classes:,
- budget: 2,
- reporter:,
- step_factory: ->(step_class) { step_class.new },
- shard_manager: @shard_manager,
- writer: @writer,
- ).run
- reporter.close
+ run_keyed_step
Migrations::Database::IntermediateDB.close
db = Extralite::Database.new(@db_path)
keyed = db.query_array("SELECT id, label FROM keyed ORDER BY id")
db.close
- # the existing row is kept as-is (not overwritten), and the new row is added
- expect(keyed).to eq([[1, "existing"], [2, "added"]])
+ # the existing row is kept and the new, non-colliding rows are appended
+ expect(keyed).to eq([[1, "existing"], [2, "added"], [3, "more"]])
+ ensure
+ Object.send(:remove_const, "TempIntegrationSteps")
+ end
+
+ it "fails the run, naming the table, when a shard row collides with an existing one" do
+ # `keyed` merges with a plain `INSERT`, so a shard row that duplicates a row
+ # already in the run DB is a genuine double-write: it fails the run loudly
+ # instead of being silently dropped, matching the single-writer contract.
+ Migrations::Database::IntermediateDB.insert("INSERT INTO keyed VALUES (?, ?)", 1, "existing")
+
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ const_set(
+ "Keyed",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 1, label: "collides" }] # id 1 already lives in the run DB
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO keyed VALUES (?, ?)",
+ item[:id],
+ item[:label],
+ )
+ end
+ end
+ end,
+ )
+ end,
+ )
+
+ expect { run_keyed_step }.to raise_error(Migrations::Conversion::ConvertError, /keyed/)
ensure
Object.send(:remove_const, "TempIntegrationSteps")
end
diff --git a/migrations/core/spec/lib/database/connection_spec.rb b/migrations/core/spec/lib/database/connection_spec.rb
index dc4dac629bc..3696e50500a 100644
--- a/migrations/core/spec/lib/database/connection_spec.rb
+++ b/migrations/core/spec/lib/database/connection_spec.rb
@@ -125,6 +125,90 @@ RSpec.describe Migrations::Database::Connection do
end
end
+ describe "#merge_database" do
+ def create_schema(db)
+ db.execute(<<~SQL)
+ CREATE TABLE topic_tags (topic_id INTEGER, tag_id INTEGER, PRIMARY KEY (topic_id, tag_id))
+ SQL
+ db.execute("CREATE TABLE uploads (id TEXT PRIMARY KEY, filename TEXT)")
+ end
+
+ # Yields a run-DB `connection` and a closed shard database at `source_path`,
+ # both carrying the same schema. The block seeds each and runs the merge.
+ def with_merge_dbs
+ Dir.mktmpdir do |dir|
+ source_path = File.join(dir, "source.db")
+ connection = described_class.new(path: File.join(dir, "main.db"))
+ create_schema(connection.db)
+
+ source = described_class.open_database(path: source_path)
+ create_schema(source)
+
+ yield connection, source, source_path
+ ensure
+ connection&.close
+ end
+ end
+
+ it "merges non-colliding rows from every listed table" do
+ with_merge_dbs do |connection, source, source_path|
+ connection.db.execute("INSERT INTO topic_tags VALUES (1, 10)")
+ source.execute("INSERT INTO topic_tags VALUES (2, 20)")
+ source.execute("INSERT INTO uploads VALUES ('a', 'a.png')")
+ source.close
+
+ connection.merge_database(source_path, tables: %w[topic_tags uploads])
+
+ expect(connection.db.query("SELECT topic_id, tag_id FROM topic_tags")).to contain_exactly(
+ { topic_id: 1, tag_id: 10 },
+ { topic_id: 2, tag_id: 20 },
+ )
+ expect(connection.db.query_splat("SELECT id FROM uploads")).to contain_exactly("a")
+ end
+ end
+
+ it "dedups an `OR IGNORE` table across shards, keeping the first writer's row" do
+ with_merge_dbs do |connection, source, source_path|
+ connection.db.execute("INSERT INTO uploads VALUES ('x', 'first.png')")
+ source.execute("INSERT INTO uploads VALUES ('x', 'second.png')")
+ source.close
+
+ connection.merge_database(source_path, tables: %w[uploads], dedupe_tables: %w[uploads])
+
+ expect(connection.db.query("SELECT id, filename FROM uploads")).to contain_exactly(
+ { id: "x", filename: "first.png" },
+ )
+ end
+ end
+
+ it "raises, naming the table, when a plain table has a cross-shard duplicate" do
+ with_merge_dbs do |connection, source, source_path|
+ connection.db.execute("INSERT INTO topic_tags VALUES (1, 10)")
+ source.execute("INSERT INTO topic_tags VALUES (1, 10)")
+ source.close
+
+ expect { connection.merge_database(source_path, tables: %w[topic_tags]) }.to raise_error(
+ /topic_tags/,
+ )
+ end
+ end
+
+ it "detaches the merge source even when a merge raises, so the next merge works" do
+ with_merge_dbs do |connection, source, source_path|
+ connection.db.execute("INSERT INTO topic_tags VALUES (1, 10)")
+ source.execute("INSERT INTO topic_tags VALUES (1, 10)")
+ source.close
+
+ expect { connection.merge_database(source_path, tables: %w[topic_tags]) }.to raise_error(
+ /topic_tags/,
+ )
+
+ # A leaked `merge_source` attachment would make this second merge raise.
+ expect { connection.merge_database(source_path, tables: %w[uploads]) }.not_to raise_error
+ end
+ end
+ end
+
context "when `Migrations::ForkManager.fork` is used" do
it "temporarily closes the connection while a process fork is created" do
create_connection do |connection|
diff --git a/migrations/core/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rb
index 8ac93a4efc3..a23f12e3411 100644
--- a/migrations/core/spec/lib/database/intermediate_db_spec.rb
+++ b/migrations/core/spec/lib/database/intermediate_db_spec.rb
@@ -57,6 +57,32 @@ RSpec.describe Migrations::Database::IntermediateDB do
end
end
+ describe ".conflict_strategy_for" do
+ it "returns `:ignore` for a table whose model declares it" do
+ expect(described_class.conflict_strategy_for("uploads")).to eq(:ignore)
+ end
+
+ it "returns `:raise` for a table whose model does not declare a strategy" do
+ expect(described_class.conflict_strategy_for("topic_tags")).to eq(:raise)
+ end
+
+ it "returns `:raise` for a table without a model" do
+ expect(described_class.conflict_strategy_for("schema_migrations")).to eq(:raise)
+ end
+
+ it "reads the strategy from the model, so a new `OR IGNORE` model flips it" do
+ model =
+ Module.new do
+ def self.conflict_strategy
+ :ignore
+ end
+ end
+ stub_const("#{described_class}::Widget", model)
+
+ expect(described_class.conflict_strategy_for("widgets")).to eq(:ignore)
+ end
+ end
+
describe ".with_connection" do
let(:previous_connection) { create_connection_double }
let(:temporary_connection) { create_connection_double }