A platform for community discussion. Free, open, simple.
MT: Close already-imported topics marked closed in the intermediate DB (#42936)
Ruben Oussoren0d4394e1daeb
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 1f83c1adaf1..c5e14d8895e 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -1839,7 +1839,11 @@ class BulkImport::Generic < BulkImport::Baseputs "", "Importing topics..."start_delta_entity(:topics, "topics", :topics, @last_topic_id)- update_delta_topics if delta_import?+ if delta_import?+ update_delta_topics+ else+ close_existing_topics+ endtopics = query(<<~SQL)SELECT *@@ -1881,6 +1885,23 @@ class BulkImport::Generic < BulkImport::Basefinish_delta_entity(:topics, :topics)end+ # Under MERGE_IMPORT the imported-id maps are cleared to avoid cross-source+ # collisions, so this pass maps nothing and is a deliberate no-op there.+ def close_existing_topics+ closed_topics = query(<<~SQL)+ SELECT id+ FROM topics+ WHERE closed = 1+ ORDER BY id+ SQL+ mapped_topic_ids = closed_topics.filter_map { |row| topic_id_from_imported_id(row["id"]) }.uniq+ closed_topics.close++ mapped_topic_ids.each_slice(1_000) do |topic_ids|+ Topic.where(id: topic_ids, closed: false).update_all(closed: true)+ end+ end+def update_delta_topicssource_columns = table_column_names("topics")columns = %i[@@ -1924,11 +1945,12 @@ class BulkImport::Generic < BulkImport::Baseif row["category_id"].present?update[:category_id] = category_id_from_imported_id(row["category_id"])end- %i[closed archived pinned_globally].each do |column|+ %i[archived pinned_globally].each do |column|if source_columns.include?(column.to_s)update[column] = to_nullable_boolean(row[column.to_s])endend+ update[:closed] = true if source_columns.include?("closed") && to_boolean(row["closed"])update[:subtype] = row["subtype"] if source_columns.include?("subtype")%i[pinned_at pinned_until].each do |column|if source_columns.include?(column.to_s) && row[column.to_s].present?diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rbindex 14ebdb0942b..2efd97dd2b5 100644--- a/spec/script/bulk_import/generic_bulk_spec.rb+++ b/spec/script/bulk_import/generic_bulk_spec.rb@@ -62,6 +62,125 @@ if generic_import_dependencies_availableendend+ describe "#close_existing_topics" do+ before { allow(Topic).to receive(:fancy_title) { |title| title } }++ def create_topic(closed: false)+ Fabricate.build(:topic, closed:).tap { |topic| topic.save!(validate: false) }+ end++ let!(:open_topic) { create_topic }+ let!(:closed_topic) { create_topic(closed: true) }+ let!(:unchanged_topic) { create_topic }++ it "closes only mapped existing topics requested as closed" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute("CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER)")+ source_db.execute(+ "INSERT INTO topics (id, closed) VALUES " \+ "(10, 1), (20, 1), (30, 0), (40, NULL), (50, 1)",+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(+ :@topics,+ {+ 10 => open_topic.id,+ 20 => closed_topic.id,+ 30 => unchanged_topic.id,+ 40 => unchanged_topic.id,+ 50 => Topic.maximum(:id) + 10_000,+ },+ )++ expect { importer.close_existing_topics }.not_to change { Post.count }++ expect(+ [open_topic.reload.closed, closed_topic.reload.closed, unchanged_topic.reload.closed],+ ).to(eq([true, true, false]))+ ensure+ source_db&.close+ end++ it "updates every mapped topic across batches" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute("CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER)")+ topics = Array.new(1_001) { create_topic }+ source_db.transaction do+ topics.each_with_index do |_topic, index|+ source_db.execute("INSERT INTO topics (id, closed) VALUES (?, 1)", index + 1)+ end+ end+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(+ :@topics,+ topics.each_with_index.to_h { |topic, index| [index + 1, topic.id] },+ )++ importer.close_existing_topics++ expect(Topic.where(id: topics.map(&:id), closed: true).count).to eq(1_001)+ ensure+ source_db&.close+ end+ end++ describe "#import_topics" do+ it "runs the close pass for full imports and the delta updater for deltas" do+ importer = described_class.allocate+ result =+ Class+ .new do+ def close+ end+ end+ .new+ allow(importer).to receive_messages(+ puts: nil,+ start_delta_entity: nil,+ finish_delta_entity: nil,+ query: result,+ create_topics: nil,+ close_existing_topics: nil,+ update_delta_topics: nil,+ )++ allow(importer).to receive(:delta_import?).and_return(false)+ importer.import_topics+ expect(importer).to have_received(:close_existing_topics)+ expect(importer).not_to have_received(:update_delta_topics)++ allow(importer).to receive(:delta_import?).and_return(true)+ importer.import_topics+ expect(importer).to have_received(:update_delta_topics)+ expect(importer).to have_received(:close_existing_topics).once+ end+ end++ describe "#update_delta_topics" do+ it "treats closed as close-only while retaining other delta updates" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute("CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER)")+ source_db.execute("INSERT INTO topics (id, closed) VALUES (10, 0), (20, 1)")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(:@delta_update_mappings, topics: { 10 => 100, 20 => 200 })++ allow(importer).to receive(:update_records).and_return(updated_keys: [])++ importer.update_delta_topics++ expect(importer).to have_received(:update_records) do |updates, name, columns|+ expect(name).to eq("topic")+ expect(columns).to include(:closed)+ expect(updates).to contain_exactly({ id: 100 }, { id: 200, closed: true })+ end+ ensure+ source_db&.close+ end+ end+describe "delta preflight" dofab!(:mapped_user) doFabricate(