A platform for community discussion. Free, open, simple.
# frozen_string_literal: true
generic_import_dependencies_available =
begin
require "sqlite3"
require "redcarpet"
true
rescue LoadError
false
end
if generic_import_dependencies_available
require_relative "../../../script/bulk_import/generic_bulk"
RSpec.describe BulkImport::Generic do
describe ".validate_modes!" do
it "rejects merge and delta mode together without changing ordinary mode" do
expect {
described_class.validate_modes!(merge_import: true, delta_import: true)
}.to raise_error("MERGE_IMPORT and DELTA_IMPORT cannot be enabled together")
expect {
described_class.validate_modes!(merge_import: false, delta_import: false)
}.not_to raise_error
end
end
describe "#initialize" do
it "refuses a missing intermediate database instead of creating one" do
expect { described_class.new("/nonexistent/intermediate.db") }.to raise_error(
RuntimeError,
/Intermediate database not found/,
)
end
it "refuses a missing uploads database instead of creating one" do
Tempfile.create(%w[intermediate .db]) do |intermediate|
expect {
described_class.new(intermediate.path, "/nonexistent/uploads.db")
}.to raise_error(RuntimeError, /Uploads database not found/)
end
end
end
describe "permalink topic URL placeholders" do
fab!(:topic)
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(:@topics, { 291_850_680 => topic.id })
instance.instance_variable_set(
:@raw_connection,
ActiveRecord::Base.connection.raw_connection,
)
instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
end
end
before do
source_db.execute(<<~SQL)
CREATE TABLE permalinks (
url TEXT, topic_id INTEGER, post_id INTEGER, category_id INTEGER,
tag_id INTEGER, user_id INTEGER, external_url TEXT, external_url_placeholders TEXT
)
SQL
source_db.execute(
"INSERT INTO permalinks (url, external_url, external_url_placeholders) VALUES (?, ?, ?)",
[
"old-feed",
"/[notice-topic].rss",
[{ type: "topic_url", id: 291_850_680, placeholder: "[notice-topic]" }].to_json,
],
)
end
after { source_db.close }
describe "#calculate_external_url" do
it "resolves a topic URL to the destination ID and final slug" do
topic.update!(title: "Final notice slug")
row = source_db.get_first_row("SELECT * FROM permalinks")
expect(importer.calculate_external_url(row)).to eq("/t/final-notice-slug/#{topic.id}.rss")
expect(topic.id).not_to eq(291_850_680)
end
it "skips the URL when a later topic placeholder has no mapping" do
row = source_db.get_first_row("SELECT * FROM permalinks")
row["external_url"] += "?next=[missing-topic]"
placeholders = JSON.parse(row["external_url_placeholders"])
placeholders << { type: "topic_url", id: 999, placeholder: "[missing-topic]" }
row["external_url_placeholders"] = placeholders.to_json
result = nil
expect { result = importer.calculate_external_url(row) }.to output(
/WARNING: Skipping permalink old-feed: missing topic target for 999/,
).to_stdout
expect(result).to eq(nil)
end
it "warns and returns no URL when the mapped topic no longer exists" do
importer.instance_variable_set(:@topics, { 291_850_680 => -999 })
row = source_db.get_first_row("SELECT * FROM permalinks")
result = nil
expect { result = importer.calculate_external_url(row) }.to output(
/WARNING: Skipping permalink old-feed: missing topic target/,
).to_stdout
expect(result).to eq(nil)
end
end
describe "#import_permalinks" do
it "warns and skips a permalink whose topic mapping is missing" do
importer.instance_variable_set(:@topics, {})
expect { importer.import_permalinks }.to output(
/WARNING: Skipping permalink old-feed: missing topic target for 291850680/,
).to_stdout
expect(Permalink.exists?(url: "old-feed")).to eq(false)
end
it "preserves an existing destination permalink" do
Permalink.create!(url: "old-feed", external_url: "/reviewed-feed.rss")
importer.import_permalinks
expect(Permalink.find_by!(url: "old-feed").external_url).to eq("/reviewed-feed.rss")
end
end
end
describe "importing notification choices" do
fab!(:subscriber, :user)
fab!(:category)
fab!(:topic) { Fabricate(:topic, user: subscriber, category: category) }
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(:@users, { 10 => subscriber.id, 11 => subscriber.id })
instance.instance_variable_set(:@topics, { 20 => topic.id })
instance.instance_variable_set(:@categories, { 30 => category.id, 31 => category.id })
instance.instance_variable_set(
:@raw_connection,
ActiveRecord::Base.connection.raw_connection,
)
instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
end
end
before do
source_db.execute(
"CREATE TABLE topic_users (user_id INTEGER, topic_id INTEGER, notification_level INTEGER)",
)
source_db.execute(
"CREATE TABLE category_users (user_id INTEGER, category_id INTEGER, notification_level INTEGER)",
)
end
after { source_db.close }
it "keeps an explicitly Tracking author Tracking while updating posting history" do
Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
TopicUser.where(topic: topic).delete_all
source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2)")
importer.import_topic_users
importer.update_topic_users
choice = TopicUser.find_by!(user: subscriber, topic: topic)
expect(choice.notification_level).to eq(NotificationLevels.topic_levels[:tracking])
expect(choice.notifications_reason_id).to eq(TopicUser.notification_reasons[:user_changed])
expect(choice.posted).to eq(true)
expect(choice.last_read_post_number).to eq(1)
end
it "uses normal posting defaults when there is no explicit choice" do
Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
replier = Fabricate(:user)
Fabricate(:post, topic: topic, user: replier, post_number: 2)
TopicUser.where(topic: topic).delete_all
importer.import_topic_users
importer.update_topic_users
expect(TopicUser.find_by!(user: subscriber, topic: topic).notification_level).to eq(
NotificationLevels.topic_levels[:watching],
)
expect(TopicUser.find_by!(user: replier, topic: topic).notification_level).to eq(
NotificationLevels.topic_levels[:tracking],
)
end
it "collapses categories and user aliases after destination IDs resolve using notification precedence" do
CategoryUser.where(category: category).delete_all
TopicUser.where(topic: topic).delete_all
source_db.execute(
"INSERT INTO category_users VALUES (10, 30, 4), (11, 31, 3), (10, 31, 0), (999, 30, 3), (10, 999, 3)",
)
source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2), (11, 20, 3), (10, 20, 0)")
importer.import_category_users
importer.import_topic_users
expect(CategoryUser.where(category: category).pluck(:user_id, :notification_level)).to eq(
[[subscriber.id, NotificationLevels.all[:watching]]],
)
expect(TopicUser.where(topic: topic).pluck(:user_id, :notification_level)).to eq(
[[subscriber.id, NotificationLevels.topic_levels[:watching]]],
)
end
it "still applies posting defaults to rows created without a notification reason" do
Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
TopicUser.where(topic: topic).delete_all
TopicUser.create!(
user: subscriber,
topic: topic,
notification_level: NotificationLevels.topic_levels[:regular],
bookmarked: true,
)
importer.update_topic_users
choice = TopicUser.find_by!(user: subscriber, topic: topic)
expect(choice.notification_level).to eq(NotificationLevels.topic_levels[:watching])
expect(choice.notifications_reason_id).to eq(TopicUser.notification_reasons[:created_topic])
expect(choice.bookmarked).to eq(true)
end
it "preserves destination choices through repeat imports and an API delta with no subscription rows" do
Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
TopicUser.where(topic: topic).delete_all
CategoryUser.where(category: category).delete_all
source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2)")
source_db.execute("INSERT INTO category_users VALUES (10, 30, 4)")
importer.import_topic_users
importer.import_category_users
TopicUser.find_by!(user: subscriber, topic: topic).update!(notification_level: 0)
CategoryUser.find_by!(user: subscriber, category: category).update!(notification_level: 1)
importer.import_topic_users
importer.import_category_users
importer.update_topic_users
source_db.execute("DELETE FROM topic_users")
source_db.execute("DELETE FROM category_users")
allow(importer).to receive(:delta_import?).and_return(true)
importer.import_topic_users
importer.import_category_users
importer.update_topic_users
expect(TopicUser.find_by!(user: subscriber, topic: topic).notification_level).to eq(0)
expect(
CategoryUser.find_by!(user: subscriber, category: category).notification_level,
).to eq(1)
end
end
describe "#import_user_notes" do
fab!(:note_owner, :user)
fab!(:note_author, :user)
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(:@users, { 10 => note_owner.id, 20 => note_author.id })
instance.instance_variable_set(
:@raw_connection,
ActiveRecord::Base.connection.raw_connection,
)
instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
end
end
before do
allow(importer).to receive(:delta_import?).and_return(true)
source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")
source_db.execute("INSERT INTO users (id) VALUES (10), (20)")
source_db.execute(<<~SQL)
CREATE TABLE user_notes (
id INTEGER PRIMARY KEY, user_id INTEGER, raw TEXT,
created_by_user_id INTEGER, created_at TEXT
)
SQL
end
around do |example|
plugin_defined = Object.const_defined?(:DiscourseUserNotes)
Object.const_set(:DiscourseUserNotes, Module.new) unless plugin_defined
example.run
ensure
Object.send(:remove_const, :DiscourseUserNotes) unless plugin_defined
end
after { source_db.close }
def add_source_note(id, raw, owner: 10, author: 20, timestamp: "2024-01-02T00:00:00Z")
source_db.execute(
"INSERT INTO user_notes VALUES (?, ?, ?, ?, ?)",
[id, owner, raw, author, timestamp],
)
end
def stored_note(raw, **extra)
{
"id" => SecureRandom.hex(16),
"user_id" => note_owner.id,
"raw" => raw,
"created_by" => note_author.id,
"created_at" => "2024-01-02T00:00:00Z",
}.merge(extra.stringify_keys)
end
it "adds missing notes while preserving existing IDs, content, metadata, and ordering" do
existing = [stored_note("Imported note"), stored_note("Staff note", post_id: 123)]
PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "2")
add_source_note(1, "Imported note")
add_source_note(2, "New note")
importer.import_user_notes
importer.import_user_note_counts
notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
expect(notes.first(2)).to eq(existing)
expect(notes.last).to include("raw" => "New note", "created_by" => note_author.id)
expect(notes.last["id"]).to match(/\A[0-9a-f]{32}\z/)
expect(
UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
).to eq("3")
stored_value =
PluginStoreRow.find_by(plugin_name: "user_notes", key: "notes:#{note_owner.id}").value
importer.import_user_notes
expect(
PluginStoreRow.find_by(plugin_name: "user_notes", key: "notes:#{note_owner.id}").value,
).to eq(stored_value)
end
it "counts identical occurrences across aliases and ignores reordered source IDs on reruns" do
PluginStore.set("user_notes", "notes:#{note_owner.id}", [stored_note("Repeated")])
source_db.execute("INSERT INTO users (id) VALUES (11)")
importer.instance_variable_get(:@users)[11] = note_owner.id
add_source_note(3, "Repeated")
add_source_note(1, "Repeated", owner: 11)
add_source_note(2, "Other")
importer.import_user_notes
notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
expect(notes.map { |note| note["raw"] }).to eq(%w[Repeated Other Repeated])
expect(notes.map { |note| note["id"] }.uniq.size).to eq(3)
source_db.execute("UPDATE user_notes SET id = 100 - id")
importer.import_user_notes
expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(notes)
expect(
UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
).to eq("3")
end
it "matches equivalent timestamps and resolved author aliases without rewriting existing JSON" do
existing = stored_note("Same", created_at: "2024-01-01T19:00:00.123-05:00")
row =
PluginStoreRow.create!(
plugin_name: "user_notes",
key: "notes:#{note_owner.id}",
type_name: "JSON",
value: JSON.pretty_generate([existing]),
)
importer.instance_variable_get(:@users)[21] = note_author.id
add_source_note(1, "Same", author: 21, timestamp: "2024-01-02T00:00:00.123Z")
UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "99")
expect { importer.import_user_notes }.not_to change { row.reload.value }
expect(
UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
).to eq("1")
end
it "retains changed source text as another note and leaves absent notes in place" do
existing = [stored_note("Original"), stored_note("Absent from delta")]
PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
add_source_note(1, "Edited")
importer.import_user_notes
notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
expect(notes.first(2)).to eq(existing)
expect(notes.last["raw"]).to eq("Edited")
end
it "creates notes for newly mapped users and uses the system user for unresolved authors" do
add_source_note(1, "Missing author", author: 99, timestamp: nil)
add_source_note(2, "No author", author: nil, timestamp: nil)
importer.import_user_notes
importer.import_user_notes
notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
expect(notes.map { |note| note.values_at("raw", "created_by", "created_at") }).to eq(
[
["Missing author", Discourse::SYSTEM_USER_ID, nil],
["No author", Discourse::SYSTEM_USER_ID, nil],
],
)
expect(
UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
).to eq("2")
end
it "matches legacy unset authors to the system fallback without modifying existing metadata" do
existing = stored_note("Legacy", created_by: nil, created_at: nil)
PluginStore.set("user_notes", "notes:#{note_owner.id}", [existing])
add_source_note(1, "Legacy", author: nil, timestamp: nil)
importer.import_user_notes
expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq([existing])
end
it "skips anonymized, unmapped, missing-destination, and unverified owners" do
source_db.execute(
"INSERT INTO users (id, anonymized) VALUES (30, 1), (40, 0), (50, 0), (60, 0)",
)
importer.instance_variable_get(:@users).merge!(
30 => note_owner.id,
50 => note_owner.id,
60 => User.maximum(:id) + 1000,
)
importer.instance_variable_set(:@delta_unverified_user_source_ids, Set[50])
[30, 40, 50, 60].each { |owner| add_source_note(owner, "Excluded", owner: owner) }
importer.import_user_notes
importer.import_user_note_counts
expect(PluginStoreRow.where(plugin_name: "user_notes")).to be_empty
expect(UserCustomField.where(name: "user_notes_count")).to be_empty
end
it "leaves notes and counts untouched when the source has no notes" do
existing = [stored_note("Existing")]
PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
count_field =
UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")
importer.import_user_notes
importer.import_user_note_counts
expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)
expect(count_field.reload.value).to eq("1")
end
it "rolls back appended notes if saving their count fails" do
existing = [stored_note("Existing")]
PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
count_field =
UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")
add_source_note(1, "New")
DB.exec(
"ALTER TABLE user_custom_fields ADD CONSTRAINT delta_note_count_test CHECK (name <> 'user_notes_count' OR value <> '2')",
)
expect { importer.import_user_notes }.to raise_error(ActiveRecord::StatementInvalid)
expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)
expect(count_field.reload.value).to eq("1")
end
it "rejects malformed stored notes without leaking private content or replacing the row" do
private_text = "Private staff note"
row =
PluginStoreRow.create!(
plugin_name: "user_notes",
key: "notes:#{note_owner.id}",
type_name: "JSON",
value: private_text,
)
add_source_note(1, "New")
[
private_text,
{ raw: private_text }.to_json,
[{ id: "old", raw: private_text, created_at: private_text }].to_json,
].each do |value|
row.update!(value: value)
expect { importer.import_user_notes }.to raise_error(RuntimeError) do |error|
expect(error.message).to include("user #{note_owner.id}")
expect(error.full_message).not_to include(private_text)
expect(error.cause).to be_nil
end
expect(row.reload.value).to eq(value)
end
expect(UserCustomField.where(name: "user_notes_count")).to be_empty
end
it "skips notes and counts when the plugin is missing" do
hide_const("DiscourseUserNotes")
add_source_note(1, "New")
importer.import_user_notes
importer.import_user_note_counts
expect(PluginStoreRow.where(plugin_name: "user_notes")).to be_empty
expect(UserCustomField.where(name: "user_notes_count")).to be_empty
end
[false, true].each do |merge_import|
it "preserves existing notes and counts in #{merge_import ? "merge" : "ordinary"} imports" do
allow(importer).to receive(:delta_import?).and_return(false)
existing = [stored_note("Existing")]
PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")
add_source_note(1, "New")
stub_const(described_class, :MERGE_IMPORT, merge_import) do
importer.import_user_notes
importer.import_user_note_counts
end
expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)
expect(
UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
).to eq("1")
end
end
end
describe "#import_category_topic_voting" do
fab!(:voting_category, :category)
fab!(:other_category, :category)
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(
:@categories,
{ 1 => voting_category.id, 2 => other_category.id },
)
end
end
after { source_db.close }
context "when the topic voting plugin is installed" do
before do
skip "requires the topic voting plugin" unless defined?(DiscourseTopicVoting)
SiteSetting.topic_voting_enabled = true
end
it "enables voting only on flagged categories, also when run again" do
source_db.execute("CREATE TABLE categories (id INTEGER, topic_voting BOOLEAN)")
source_db.execute("INSERT INTO categories VALUES (1, 1), (2, NULL), (3, 1)")
importer.import_category_topic_voting
importer.import_category_topic_voting
expect(DiscourseTopicVoting::CategorySetting.pluck(:category_id)).to contain_exactly(
voting_category.id,
)
expect(Category.can_vote?(voting_category.id)).to eq(true)
expect(Category.can_vote?(other_category.id)).to eq(false)
end
it "skips intermediate databases without a topic_voting column" do
source_db.execute("CREATE TABLE categories (id INTEGER)")
source_db.execute("INSERT INTO categories VALUES (1)")
expect { importer.import_category_topic_voting }.not_to raise_error
expect(DiscourseTopicVoting::CategorySetting.count).to eq(0)
end
end
it "skips categories when the plugin is missing" do
hide_const("DiscourseTopicVoting")
source_db.execute("CREATE TABLE categories (id INTEGER, topic_voting BOOLEAN)")
source_db.execute("INSERT INTO categories VALUES (1, 1)")
expect { importer.import_category_topic_voting }.to output(
/plugin is not installed/,
).to_stdout
end
end
describe "#import_tags" do
fab!(:tag_group)
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(:@tag_group_mapping, { 7 => tag_group.id })
end
end
after { source_db.close }
it "adds tags to the group from the single tag_group_id column" do
source_db.execute(<<~SQL)
CREATE TABLE tags (
id INTEGER, name TEXT, description TEXT, tag_group_ids TEXT, tag_group_id INTEGER
)
SQL
source_db.execute("INSERT INTO tags (id, name, tag_group_id) VALUES (1, 'grouped', 7)")
importer.import_tags
expect(tag_group.reload.tags.pluck(:name)).to contain_exactly("grouped")
end
end
describe "badges without a group" do
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(:@badge_mapping, {})
end
end
before do
source_db.execute("CREATE TABLE badges (id INTEGER, name TEXT, badge_group TEXT)")
source_db.execute(
"INSERT INTO badges (id, name, badge_group) VALUES (1, 'Grouped', 'Imported'), (2, 'Ungrouped', NULL)",
)
end
after { source_db.close }
it "puts them into the Other grouping" do
badges = []
allow(importer).to receive(:create_badges) do |rows, &block|
badges = rows.filter_map(&block)
end
importer.import_badge_groupings
importer.import_badges
imported_grouping = BadgeGrouping.find_by!(name: "Imported")
expect(badges.map { |badge| badge.slice(:name, :badge_grouping_id) }).to contain_exactly(
{ name: "Grouped", badge_grouping_id: imported_grouping.id },
{ name: "Ungrouped", badge_grouping_id: BadgeGrouping::Other },
)
end
end
describe "events" do
fab!(:vienna_post, :post)
fab!(:untimed_post, :post)
fab!(:invalid_post, :post)
fab!(:attendee, :user)
fab!(:other_attendee, :user)
fab!(:members) { Fabricate(:group, name: "members", users: [attendee]) }
fab!(:staff_group) { Fabricate(:group, name: "staff_only") }
fab!(:image, :upload)
let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
let(:import_issue_log_path) { File.join(Dir.mktmpdir, "issues.log") }
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(:@source_db, source_db)
instance.instance_variable_set(
:@posts,
{ 1 => vienna_post.id, 2 => untimed_post.id, 3 => invalid_post.id },
)
instance.instance_variable_set(
:@topic_id_by_post_id,
[vienna_post, untimed_post, invalid_post].to_h { |post| [post.id, post.topic_id] },
)
instance.instance_variable_set(:@users, { 10 => attendee.id, 11 => other_attendee.id })
instance.instance_variable_set(:@groups, { 50 => members.id, 51 => staff_group.id })
instance.instance_variable_set(:@uploads_mapping, { "70" => image.id.to_s })
instance.instance_variable_set(:@import_issue_log_path, import_issue_log_path)
instance.instance_variable_set(
:@raw_connection,
ActiveRecord::Base.connection.raw_connection,
)
instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
end
end
let(:public_status) { DiscourseEvents::Events::Event.statuses[:public] }
let(:standalone_status) { DiscourseEvents::Events::Event.statuses[:standalone] }
let(:private_status) { DiscourseEvents::Events::Event.statuses[:private] }
before { skip "requires the events plugin" unless defined?(DiscourseEvents::Events) }
after { source_db.close }
def create_events_table(new_columns: true)
source_db.execute(<<~SQL)
CREATE TABLE events (
id INTEGER, post_id INTEGER, status INTEGER, starts_at DATETIME, ends_at DATETIME,
name TEXT, url TEXT, custom_fields JSON_TEXT
#{", location TEXT, timezone TEXT, closed BOOLEAN, image_upload_id TEXT, allowed_group_ids JSON_TEXT" if new_columns}
)
SQL
end
def insert_event(id, **attributes)
row = {
id:,
post_id: id,
status: 0,
starts_at: "2026-07-01T08:00:00Z",
ends_at: "2026-07-01T09:30:00Z",
}.merge(attributes)
source_db.execute(
"INSERT INTO events (#{row.keys.join(", ")}) VALUES (#{(["?"] * row.size).join(", ")})",
row.values,
)
end
def event_row(id)
source_db.get_first_row("SELECT * FROM events WHERE id = ?", id)
end
def event_bbcode(id)
importer.load_event_lookups
importer.event_bbcode(event_row(id))
end
def import_issues
File.exist?(import_issue_log_path) ? File.read(import_issue_log_path) : ""
end
describe "#import_posts" do
it "loads the event lookups before updating delta posts" do
create_events_table
insert_event(1, image_upload_id: "70")
posts =
Class
.new do
def close
end
end
.new
allow(importer).to receive_messages(
puts: nil,
start_delta_entity: nil,
finish_delta_entity: nil,
create_posts: nil,
delta_import?: true,
)
allow(importer).to receive(:query).and_call_original
allow(importer).to receive(:query).with(/FROM posts/).and_return(posts)
bbcode = nil
allow(importer).to receive(:update_delta_posts) {
bbcode = importer.event_bbcode(event_row(1))
}
importer.import_posts
expect(bbcode).to include(%{ image="#{image.short_url}"})
end
end
describe "#event_bbcode" do
it "writes start and end as local times in the event's timezone" do
create_events_table
insert_event(1, timezone: "Europe/Vienna")
bbcode = event_bbcode(1)
expect(bbcode).to include(
'start="2026-07-01 10:00" end="2026-07-01 11:30" timezone="Europe/Vienna"',
)
expect(ActiveSupport::TimeZone["Europe/Vienna"].parse("2026-07-01 10:00")).to eq(
Time.utc(2026, 7, 1, 8),
)
end
it "falls back to UTC for missing and invalid timezones" do
create_events_table
insert_event(2)
insert_event(3, timezone: "+01:00")
[2, 3].each do |id|
expect(event_bbcode(id)).to include(
'start="2026-07-01 08:00" end="2026-07-01 09:30" timezone="UTC"',
)
end
end
it "writes the location as one line of at most 1000 characters" do
create_events_table
insert_event(1, location: "Town Hall\nRoom 1")
insert_event(2, location: "x" * 1200)
expect(event_bbcode(1)).to include(%{ location="Town Hall Room 1"})
expect(event_bbcode(2)).to include(%{ location="#{"x" * 1000}"})
end
it "writes allowed groups of private events, the image and the closed flag" do
SiteSetting.discourse_post_event_enabled = true
create_events_table
insert_event(
1,
status: private_status,
allowed_group_ids: [50, 51, 52].to_json,
image_upload_id: "70",
closed: 1,
)
insert_event(2, status: public_status, allowed_group_ids: [50].to_json, closed: 0)
private_bbcode = event_bbcode(1)
public_bbcode = event_bbcode(2)
expect(
DiscourseEvents::Events::Parser.extract_events(
Fabricate.build(:post, raw: private_bbcode),
).first,
).to include(
"allowed-groups": "members,staff_only",
image: image.short_url,
closed: "true",
)
expect(public_bbcode).not_to include("allowed-groups", "image", "closed")
end
it "quotes attributes so the plugin reads them back unchanged" do
SiteSetting.discourse_post_event_enabled = true
SiteSetting.discourse_post_event_allowed_custom_fields = "room"
create_events_table
insert_event(
1,
status: public_status,
name: %{The "Old" Town Hall},
location: %{The "Old" Town Hall, 'Room' «1»},
custom_fields: { room: %{"A" 'B'} }.to_json,
)
bbcode = event_bbcode(1)
post = Fabricate.build(:post, raw: bbcode)
expect(bbcode).to include(%{ location=“The "Old" Town Hall, 'Room' «1»”})
expect(DiscourseEvents::Events::Parser.extract_events(post).first).to include(
name: %{The "Old" Town Hall},
location: %{The "Old" Town Hall, 'Room' «1»},
room: CGI.escapeHTML(%{"A" 'B'}),
)
end
it "uses UTC and no location when the events table has neither column" do
create_events_table(new_columns: false)
insert_event(1)
bbcode = event_bbcode(1)
expect(bbcode).to include('timezone="UTC"')
expect(bbcode).not_to include("location")
end
end
describe "#import_post_events" do
it "imports valid timezones and logs invalid ones" do
create_events_table
insert_event(1, timezone: "Europe/Vienna")
insert_event(2)
insert_event(3, timezone: "+01:00")
importer.import_post_events
expect(
DiscourseEvents::Events::Event.where(
id: [vienna_post.id, untimed_post.id, invalid_post.id],
).pluck(:id, :timezone),
).to contain_exactly(
[vienna_post.id, "Europe/Vienna"],
[untimed_post.id, "UTC"],
[invalid_post.id, "UTC"],
)
expect(import_issues).to eq("[invalid event timezone, using UTC] +01:00 (event 3)\n")
end
it "imports the location" do
create_events_table
insert_event(1, location: "Town Hall")
insert_event(2)
insert_event(3, location: "x" * 1200)
importer.import_post_events
expect(
DiscourseEvents::Events::Event.where(
id: [vienna_post.id, untimed_post.id, invalid_post.id],
).pluck(:id, :location),
).to contain_exactly(
[vienna_post.id, "Town Hall"],
[untimed_post.id, nil],
[invalid_post.id, "x" * 1000],
)
end
it "imports events from intermediate databases without the new columns" do
create_events_table(new_columns: false)
insert_event(1)
importer.import_post_events
event = DiscourseEvents::Events::Event.find(vienna_post.id)
expect(event.timezone).to eq("UTC")
expect(event.location).to be_nil
end
end
describe "#import_post_events with groups, image and closed flag" do
it "imports them and logs groups that can't be used" do
create_events_table
insert_event(
1,
status: private_status,
allowed_group_ids: [50, 52].to_json,
image_upload_id: "70",
closed: 1,
)
insert_event(2, status: public_status, allowed_group_ids: [51].to_json)
insert_event(3, status: standalone_status)
importer.import_post_events
importer.import_event_upload_references
expect(
DiscourseEvents::Events::Event.where(
id: [vienna_post.id, untimed_post.id, invalid_post.id],
).pluck(:id, :raw_invitees, :closed, :image_upload_id),
).to contain_exactly(
[vienna_post.id, ["members"], true, image.id],
[untimed_post.id, [DiscourseEvents::Events::Event::PUBLIC_GROUP], false, nil],
[invalid_post.id, [], false, nil],
)
expect(vienna_post.reload.image_upload_id).to eq(image.id)
expect(vienna_post.topic.reload.image_upload_id).to eq(image.id)
expect(
UploadReference.where(
upload: image,
target_type: "DiscourseEvents::Events::Event",
target_id: vienna_post.id,
),
).to exist
expect(import_issues.lines).to contain_exactly(
"[event allowed groups not found] 52 (event 1)\n",
"[event allowed groups ignored (event isn't private)] event 2\n",
)
end
end
describe "#import_event_invitees" do
def create_invitees_table
source_db.execute(<<~SQL)
CREATE TABLE event_invitees (
event_id INTEGER, user_id INTEGER, status INTEGER, created_at DATETIME
)
SQL
end
def insert_invitee(event_id, user_id, status: nil, created_at: "2017-03-04T05:06:07Z")
source_db.execute(
"INSERT INTO event_invitees VALUES (?, ?, ?, ?)",
[event_id, user_id, status, created_at],
)
end
def imported_invitees
DiscourseEvents::Events::Invitee
.unscoped
.order(:post_id, :user_id)
.pluck(:post_id, :user_id, :status, :notified, :created_at)
end
it "imports sign-ups of public events as notified attendees" do
going = DiscourseEvents::Events::Invitee.statuses[:going]
interested = DiscourseEvents::Events::Invitee.statuses[:interested]
create_events_table
create_invitees_table
# The intermediate event id differs from the post id on purpose.
insert_event(101, post_id: 1, status: public_status)
insert_invitee(101, 10)
insert_invitee(101, 11, status: interested)
importer.import_post_events
importer.import_event_invitees
importer.import_event_invitees
signed_up_at = Time.utc(2017, 3, 4, 5, 6, 7)
expect(imported_invitees).to eq(
[
[vienna_post.id, attendee.id, going, true, signed_up_at],
[vienna_post.id, other_attendee.id, interested, true, signed_up_at],
],
)
end
it "skips invitees the plugin wouldn't keep" do
create_events_table
create_invitees_table
insert_event(1, status: public_status)
insert_event(2, status: standalone_status)
insert_event(3, status: private_status, allowed_group_ids: [50].to_json)
insert_event(99, status: public_status)
insert_invitee(1, 99)
insert_invitee(99, 10)
insert_invitee(2, 10)
insert_invitee(1, 11, status: 42)
insert_invitee(3, 10)
insert_invitee(3, 11)
importer.import_post_events
importer.import_event_invitees
expect(imported_invitees.map { |invitee| invitee.first(2) }).to eq(
[[invalid_post.id, attendee.id]],
)
expect(import_issues.lines).to contain_exactly(
"[event invitee skipped (standalone event)] user 10 of event 2\n",
"[event invitee skipped (unknown status)] status 42 of user 11 in event 1\n",
"[event invitee skipped (not in the event's allowed groups)] user 11 of event 3\n",
)
end
it "skips intermediate databases without an event_invitees table" do
create_events_table(new_columns: false)
expect { importer.import_event_invitees }.not_to raise_error
expect(imported_invitees).to be_empty
end
end
describe "#import_event_invitee_topic_tracking" do
fab!(:not_going_attendee, :user)
fab!(:muting_attendee, :user)
it "sets the topic tracking like an RSVP, keeping imported subscriptions" do
statuses = DiscourseEvents::Events::Invitee.statuses
levels = TopicUser.notification_levels
signed_up_at = Time.utc(2017, 3, 4, 5, 6, 7)
DiscourseEvents::Events::Invitee.insert_all!(
[
[attendee, :going],
[other_attendee, :interested],
[not_going_attendee, :not_going],
[muting_attendee, :going],
].map do |user, status|
{
post_id: vienna_post.id,
user_id: user.id,
status: statuses[status],
created_at: signed_up_at,
updated_at: signed_up_at,
}
end,
)
# Posting history alone would make this attendee track the topic.
Fabricate(:post, topic: vienna_post.topic, user: attendee)
TopicUser.create!(
user: muting_attendee,
topic: vienna_post.topic,
notification_level: levels[:muted],
notifications_reason_id: TopicUser.notification_reasons[:user_changed],
)
importer.import_event_invitee_topic_tracking
importer.update_topic_users
expect(
TopicUser
.where(topic: vienna_post.topic)
.where.not(user: vienna_post.user)
.pluck(:user_id, :notification_level, :notifications_reason_id),
).to contain_exactly(
[attendee.id, levels[:watching], TopicUser.notification_reasons[:user_changed]],
[other_attendee.id, levels[:tracking], TopicUser.notification_reasons[:user_changed]],
[
not_going_attendee.id,
levels[:regular],
TopicUser.notification_reasons[:user_changed],
],
[muting_attendee.id, levels[:muted], TopicUser.notification_reasons[:user_changed]],
)
expect(TopicUser.find_by(user: attendee).notifications_changed_at).to eq_time(
signed_up_at,
)
end
end
end
describe "mapping selection" do
fab!(:canonical_user, :user)
fab!(:other_user, :user)
it "loads plain 64-bit IDs without consuming merge namespaces" do
UserCustomField.create!(
user: canonical_user,
name: "import_id",
value: "9223372036854775000",
)
UserCustomField.create!(user: other_user, name: "import_id", value: "forum:42")
mappings = described_class.allocate.plain_import_mappings("user")
expect(mappings).to eq(9_223_372_036_854_775_000 => canonical_user.id)
end
it "keeps the oldest source mapping canonical unless the delta only exports a newer one" do
UserCustomField.create!(
user: canonical_user,
name: "import_id",
value: "10",
created_at: 2.days.ago,
)
UserCustomField.create!(
user: canonical_user,
name: "import_id",
value: "20",
created_at: 1.day.ago,
)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
expect(importer.canonical_user_import_mappings).to eq(10 => canonical_user.id)
source_db.execute("INSERT INTO users (id) VALUES (20)")
expect(importer.canonical_user_import_mappings).to eq(20 => canonical_user.id)
ensure
source_db&.close
end
end
describe "#close_existing_topics" do
before { allow(Topic).to receive(:fancy_title) { |title| title } }
def create_topic(closed: false)
Fabricate.build(:topic, closed:).tap { |topic| topic.save!(validate: false) }
end
let!(:open_topic) { create_topic }
let!(:closed_topic) { create_topic(closed: true) }
let!(:unchanged_topic) { create_topic }
it "closes only mapped existing topics requested as closed" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute("CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER)")
source_db.execute(
"INSERT INTO topics (id, closed) VALUES " \
"(10, 1), (20, 1), (30, 0), (40, NULL), (50, 1)",
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@topics,
{
10 => open_topic.id,
20 => closed_topic.id,
30 => unchanged_topic.id,
40 => unchanged_topic.id,
50 => Topic.maximum(:id) + 10_000,
},
)
expect { importer.close_existing_topics }.not_to change { Post.count }
expect(
[open_topic.reload.closed, closed_topic.reload.closed, unchanged_topic.reload.closed],
).to(eq([true, true, false]))
ensure
source_db&.close
end
it "updates every mapped topic across batches" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute("CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER)")
topics = Array.new(1_001) { create_topic }
source_db.transaction do
topics.each_with_index do |_topic, index|
source_db.execute("INSERT INTO topics (id, closed) VALUES (?, 1)", index + 1)
end
end
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@topics,
topics.each_with_index.to_h { |topic, index| [index + 1, topic.id] },
)
importer.close_existing_topics
expect(Topic.where(id: topics.map(&:id), closed: true).count).to eq(1_001)
ensure
source_db&.close
end
end
describe "#import_topics" do
it "runs the close pass for full imports and the delta updater for deltas" do
importer = described_class.allocate
result =
Class
.new do
def close
end
end
.new
allow(importer).to receive_messages(
puts: nil,
start_delta_entity: nil,
finish_delta_entity: nil,
query: result,
create_topics: nil,
close_existing_topics: nil,
update_delta_topics: nil,
)
allow(importer).to receive(:delta_import?).and_return(false)
importer.import_topics
expect(importer).to have_received(:close_existing_topics)
expect(importer).not_to have_received(:update_delta_topics)
allow(importer).to receive(:delta_import?).and_return(true)
importer.import_topics
expect(importer).to have_received(:update_delta_topics)
expect(importer).to have_received(:close_existing_topics).once
end
end
describe "#update_delta_topics" do
it "replaces non-NULL views, including zero, while treating closed as close-only" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(
"CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER, views INTEGER)",
)
source_db.execute(
"INSERT INTO topics (id, closed, views) VALUES " \
"(10, 0, 1223), (20, 1, 0), (30, NULL, NULL)",
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@delta_update_mappings,
topics: {
10 => 100,
20 => 200,
30 => 300,
},
)
allow(importer).to receive(:update_records).and_return(updated_keys: [])
importer.update_delta_topics
expect(importer).to have_received(:update_records) do |updates, name, columns|
expect(name).to eq("topic")
expect(columns).to include(:closed)
expect(columns).to include(:views)
expect(updates).to contain_exactly(
{ id: 100, views: 1_223 },
{ id: 200, closed: true, views: 0 },
{ id: 300 },
)
end
ensure
source_db&.close
end
it "skips views when the intermediate database predates the column" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute("CREATE TABLE topics (id INTEGER PRIMARY KEY, closed INTEGER)")
source_db.execute("INSERT INTO topics (id, closed) VALUES (10, 1)")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(:@delta_update_mappings, topics: { 10 => 100 })
allow(importer).to receive(:update_records).and_return(updated_keys: [])
importer.update_delta_topics
expect(importer).to have_received(:update_records) do |updates, name, columns|
expect(name).to eq("topic")
expect(columns).to include(:views)
expect(updates).to contain_exactly({ id: 100, closed: true })
end
ensure
source_db&.close
end
it "reindexes only topics whose title or category changed" do
title_changed = Fabricate(:topic, title: "Original topic title", views: 10)
category_changed = Fabricate(:topic, category: Fabricate(:category), views: 20)
views_changed = Fabricate(:topic, title: "Views only topic", views: 30)
replacement_category = Fabricate(:category)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(
"CREATE TABLE topics (id INTEGER PRIMARY KEY, title TEXT, category_id INTEGER, views INTEGER)",
)
source_db.execute(
"INSERT INTO topics (id, title, category_id, views) VALUES " \
"(10, 'Replacement title', NULL, NULL), " \
"(20, NULL, 2000, NULL), " \
"(30, NULL, NULL, 31)",
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@delta_update_mappings,
topics: {
10 => title_changed.id,
20 => category_changed.id,
30 => views_changed.id,
},
)
importer.instance_variable_set(:@categories, { 2000 => replacement_category.id })
allow(importer).to receive(:update_records) do |updates, _name, columns|
updated_keys =
updates.filter_map do |update|
topic = Topic.find(update[:id])
attributes = update.slice(*columns).compact
next if attributes.all? { |column, value| topic.public_send(column) == value }
topic.update_columns(attributes)
topic.id
end
{ updated_keys: updated_keys }
end
indexed_topic_ids = []
allow(SearchIndexer).to receive(:index) do |topic, force:|
indexed_topic_ids << topic.id
expect(force).to eq(true)
end
importer.update_delta_topics
expect(indexed_topic_ids).to contain_exactly(title_changed.id, category_changed.id)
expect(title_changed.reload.title).to eq("Replacement title")
expect(category_changed.reload.category_id).to eq(replacement_category.id)
expect(views_changed.reload.views).to eq(31)
ensure
source_db&.close
end
end
describe "delta preflight" do
fab!(:mapped_user) do
Fabricate(
:user,
username: "mapped-user",
email: "[email protected]",
created_at: 1.year.ago,
)
end
fab!(:conflicting_user) do
Fabricate(:user, username: "taken-user", email: "[email protected]")
end
fab!(:other_mapped_user, :user)
fab!(:topic)
fab!(:other_topic, :topic)
fab!(:post) { Fabricate(:post, topic: topic) }
it "reports immutable identity and ownership conflicts" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
[1, conflicting_user.username, conflicting_user.email, Time.zone.now.iso8601],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => mapped_user.id }, errors)
expect(errors).to contain_exactly(
"user 1 changes immutable created_at",
"user 1 email belongs to Discourse user #{conflicting_user.id}",
)
expect(
importer.instance_variable_get(:@delta_username_conflict_source_ids),
).to contain_exactly(1)
ensure
source_db&.close
end
it "accepts an email owned by the base-imported account of a declared alias" do
UserCustomField.create!(user: conflicting_user, name: "import_id", value: "2")
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"CREATE TABLE user_aliases (alias_user_id INTEGER PRIMARY KEY, canonical_user_id INTEGER NOT NULL)",
)
source_db.execute(
"INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
[1, mapped_user.username, conflicting_user.email, mapped_user.created_at.iso8601],
)
source_db.execute("INSERT INTO user_aliases VALUES (2, 1)")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => mapped_user.id }, errors)
expect(errors).to be_empty
source_db.execute("DELETE FROM user_aliases")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.preflight_users({ 1 => mapped_user.id }, errors)
expect(errors).to contain_exactly(
"user 1 email belongs to Discourse user #{conflicting_user.id}",
)
ensure
source_db&.close
end
it "rejects duplicate new emails and usernames within the delta" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
[1, "dupe_user", "[email protected]", mapped_user.created_at.iso8601],
)
source_db.execute(
"INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
[2, "dupe_user", "[email protected]", other_mapped_user.created_at.iso8601],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => mapped_user.id, 2 => other_mapped_user.id }, errors)
expect(errors).to contain_exactly(
"user 2 duplicates email of source user 1",
"user 2 duplicates username of source user 1",
)
ensure
source_db&.close
end
it "accepts an unchanged username whose base import result was suffix-deduplicated" do
Fabricate(:user, username: "Andrew_S")
suffixed_user = Fabricate(:user, username: "Andrew_S_1", created_at: 1.year.ago)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",
[1, "Andrew_S_", suffixed_user.created_at.iso8601],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => suffixed_user.id }, errors)
expect(errors).to be_empty
expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty
ensure
source_db&.close
end
it "skips renames to taken usernames unless the import recorded them as unchanged" do
Fabricate(:user, username: "Andrew_S")
renamed_user = Fabricate(:user, username: "SomethingElse", created_at: 1.year.ago)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",
[1, "Andrew_S_", renamed_user.created_at.iso8601],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => renamed_user.id }, errors)
expect(errors).to be_empty
expect(
importer.instance_variable_get(:@delta_username_conflict_source_ids),
).to contain_exactly(1)
UserCustomField.create!(user: renamed_user, name: "import_username", value: "Andrew_S_")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => renamed_user.id }, errors)
expect(errors).to be_empty
expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty
ensure
source_db&.close
end
it "keeps base-import fallback usernames untouched when unicode support is enabled later" do
SiteSetting.unicode_usernames = true
fallback_user = Fabricate(:user, username: "Anonymous_339dfc", created_at: 1.year.ago)
UserCustomField.create!(user: fallback_user, name: "import_username", value: "千风")
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",
[1, "千风", fallback_user.created_at.iso8601],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => fallback_user.id }, errors)
expect(errors).to be_empty
expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty
expect(
importer.delta_username_unchanged?(
{ "username" => "千风" },
fallback_user.id,
fallback_user.username_lower,
),
).to eq(true)
ensure
source_db&.close
end
it "keeps a destination username the source still carries verbatim under a stricter sanitizer" do
kept_user = Fabricate(:user, username: "userjs")
kept_user.update_columns(username: "user.js", username_lower: "user.js")
importer = described_class.allocate
expect(importer.fix_name("user.js")).to eq("user")
expect(
importer.delta_username_unchanged?({ "username" => "user.js" }, kept_user.id, "user.js"),
).to eq(true)
end
it "keeps users merged into pre-existing accounts updatable instead of failing" do
preexisting_user = Fabricate(:user, email: "[email protected]")
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT
)
SQL
source_db.execute(
"INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
[1, "PreUser", "[email protected]", 1.year.ago.iso8601],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => preexisting_user.id }, errors)
expect(errors).to be_empty
expect(importer.instance_variable_get(:@delta_preexisting_user_source_ids)).to include(1)
source_db.execute("UPDATE users SET email = '[email protected]' WHERE id = 1")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => preexisting_user.id }, errors)
expect(errors).to contain_exactly("user 1 changes immutable created_at")
source_db.execute("UPDATE users SET email = NULL WHERE id = 1")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_users({ 1 => preexisting_user.id }, errors)
expect(errors).to be_empty
expect(
importer.instance_variable_get(:@delta_unverified_user_source_ids),
).to contain_exactly(1)
ensure
source_db&.close
end
it "rejects duplicate group names within the delta" do
group_a = Fabricate(:group)
group_b = Fabricate(:group)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE groups (
id INTEGER PRIMARY KEY,
name TEXT,
existing_id INTEGER
)
SQL
source_db.execute("INSERT INTO groups (id, name) VALUES (1, 'Team Alpha')")
source_db.execute("INSERT INTO groups (id, name) VALUES (2, 'Team Alpha')")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_groups({ 1 => group_a.id, 2 => group_b.id }, errors)
expect(errors).to contain_exactly("group 2 duplicates name of source group 1")
ensure
source_db&.close
end
it "rejects topic archetype changes" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE topics (
id INTEGER PRIMARY KEY,
created_at TEXT,
private_message TEXT
)
SQL
source_db.execute(
"INSERT INTO topics (id, created_at, private_message) VALUES (?, ?, ?)",
[3_000_000_000, topic.created_at.iso8601, '{"user_ids":[]}'],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_topics({ topics: { 3_000_000_000 => topic.id } }, errors)
expect(errors).to contain_exactly("topic 3000000000 changes immutable archetype")
ensure
source_db&.close
end
it "rejects duplicate group names differing only in unicode case within the delta" do
SiteSetting.unicode_usernames = true
group_a = Fabricate(:group)
group_b = Fabricate(:group)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE groups (
id INTEGER PRIMARY KEY,
name TEXT,
existing_id INTEGER
)
SQL
source_db.execute("INSERT INTO groups (id, name) VALUES (1, 'Équipe')")
source_db.execute("INSERT INTO groups (id, name) VALUES (2, 'équipe')")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
importer.preflight_groups({ 1 => group_a.id, 2 => group_b.id }, errors)
expect(errors).to contain_exactly("group 2 duplicates name of source group 1")
ensure
source_db&.close
end
it "rejects mapped post moves" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE posts (
id INTEGER PRIMARY KEY,
topic_id INTEGER,
created_at TEXT,
post_number INTEGER,
reply_to_post_id INTEGER
)
SQL
source_db.execute(
"INSERT INTO posts (id, topic_id, created_at, post_number) VALUES (?, ?, ?, ?)",
[4_000_000_000, 3_000_000_001, post.created_at.iso8601, post.post_number],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
mappings = {
posts: {
4_000_000_000 => post.id,
},
topics: {
3_000_000_001 => other_topic.id,
},
}
importer.preflight_posts(mappings, errors)
expect(errors).to contain_exactly("post 4000000000 changes immutable topic")
ensure
source_db&.close
end
it "accepts a delta-local post number while still checking the reply parent" do
parent = topic.first_post
reply = Fabricate(:post, topic: topic, reply_to_post_number: nil)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE posts (
id INTEGER PRIMARY KEY,
topic_id INTEGER,
created_at TEXT,
post_number INTEGER,
reply_to_post_id INTEGER
)
SQL
source_db.execute(
"INSERT INTO posts (id, topic_id, created_at, post_number, reply_to_post_id) " \
"VALUES (?, ?, ?, ?, ?)",
[4_000_000_000, 3_000_000_000, reply.created_at.iso8601, 2, 4_000_000_001],
)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
errors = []
mappings = {
posts: {
4_000_000_000 => reply.id,
4_000_000_001 => parent.id,
},
topics: {
3_000_000_000 => topic.id,
},
}
importer.preflight_posts(mappings, errors)
expect(errors).to be_empty
mappings[:posts][4_000_000_001] = Fabricate(:post, topic: topic).id
importer.preflight_posts(mappings, errors)
expect(errors).to contain_exactly("post 4000000000 changes immutable reply parent")
ensure
source_db&.close
end
end
describe "delta user aliases" do
COUNTERS = %i[
badge
bookmark
category_group
category
chat_channel
chat_direct_message_channel
chat_message
chat_thread
discourse_reaction
group
poll
poll_option
post_action
post
sso_record
topic
upload
user_avatar
user
]
def build_delta_user_importer(source_db, users:)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@raw_connection,
ActiveRecord::Base.connection.raw_connection,
)
COUNTERS.each { |name| importer.instance_variable_set(:"@last_#{name}_id", 0) }
importer.instance_variable_set(:@last_user_email_id, UserEmail.maximum(:id))
importer.instance_variable_set(:@users, users)
importer.instance_variable_set(:@delta_update_mappings, users: users)
importer.instance_variable_set(:@delta_stats, users: { updated_ids: Set.new })
importer.instance_variable_set(:@delta_username_conflict_source_ids, Set.new)
importer.instance_variable_set(:@mapped_usernames, {})
importer.instance_variable_set(:@usernames_lower, Set.new)
importer.instance_variable_set(:@user_ids_by_username_lower, {})
importer.instance_variable_set(:@usernames_by_id, {})
importer.instance_variable_set(:@user_full_names_by_id, {})
importer.instance_variable_set(
:@import_issue_log_path,
File.join(Dir.mktmpdir, "issues.log"),
)
allow(importer).to receive(:update_records) do |rows, name, columns, keys: [:id]|
model = name.classify.constantize
rows.each do |row|
attributes = row.slice(*columns).compact
model.find_by!(row.slice(*keys)).update_columns(attributes) if attributes.any?
end
{ updated_keys: rows.map { |row| row[:id] } }
end
importer
end
it "merges a base-imported alias into its winner so the winner can take over the email" do
winner = Fabricate(:user, username: "winner", email: "[email protected]")
alias_user = Fabricate(:user, username: "alias", email: "[email protected]")
alias_username = alias_user.username
UserCustomField.create!(user: winner, name: "import_id", value: "1")
UserCustomField.create!(user: alias_user, name: "import_id", value: "2")
moved_post = Fabricate(:post, user: alias_user)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
CREATE TABLE users (
id INTEGER PRIMARY KEY,
username TEXT,
email TEXT,
created_at TEXT,
last_seen_at TEXT,
anonymized INTEGER
)
SQL
source_db.execute(
"INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
[1, winner.username, "[email protected]", winner.created_at.iso8601],
)
source_db.execute(
"CREATE TABLE user_aliases (alias_user_id INTEGER PRIMARY KEY, canonical_user_id INTEGER NOT NULL)",
)
source_db.execute("INSERT INTO user_aliases VALUES (2, 1)")
importer =
build_delta_user_importer(source_db, users: { 1 => winner.id, 2 => alias_user.id })
# rows COPYed earlier in a run outrun the sequence the merger inserts with
UserEmail.connection.execute(
"SELECT setval('#{UserEmail.sequence_name}', #{alias_user.primary_email.id - 1})",
)
expect { importer.merge_delta_user_aliases }.to change { User.exists?(alias_user.id) }.to(
false,
)
expect(importer.instance_variable_get(:@last_user_email_id)).to eq(UserEmail.maximum(:id))
expect(importer.user_id_from_original_username(alias_username)).to be_nil
expect(importer.user_id_from_original_username(winner.username)).to eq(winner.id)
expect(importer.instance_variable_get(:@emails)).to include(
"[email protected]" => winner.id,
)
importer.update_delta_users
expect(moved_post.reload.user_id).to eq(winner.id)
expect(importer.user_id_from_imported_id(2)).to eq(winner.id)
expect(
UserCustomField.where(user: winner, name: "import_id").pluck(:value),
).to contain_exactly("1", "2")
expect(winner.reload.email).to eq("[email protected]")
expect(UserEmail.where(user: winner).count).to eq(1)
expect { importer.merge_delta_user_aliases }.not_to change { User.count }
importer.update_delta_users
expect(importer).to have_received(:update_records).with([], "user_email", [:email])
ensure
source_db&.close
end
it "clears both rows when two users exchange identities so the create pass rebuilds them" do
first = Fabricate(:user)
second = Fabricate(:user)
Fabricate(:user_associated_account, user: first, provider_name: "khoros", provider_uid: "a")
Fabricate(
:user_associated_account,
user: second,
provider_name: "khoros",
provider_uid: "b",
)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")
source_db.execute(
"CREATE TABLE user_associated_accounts (user_id INTEGER, provider_name TEXT, provider_uid TEXT)",
)
source_db.execute("INSERT INTO users (id) VALUES (1), (2)")
source_db.execute("INSERT INTO user_associated_accounts VALUES (1, 'khoros', 'b')")
source_db.execute("INSERT INTO user_associated_accounts VALUES (2, 'khoros', 'a')")
importer = build_delta_user_importer(source_db, users: { 1 => first.id, 2 => second.id })
importer.update_delta_user_associated_accounts
expect(UserAssociatedAccount.where(provider_name: "khoros")).to be_empty
expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to include(
"khoros/a moved from user #{first.id} to user #{second.id}",
"khoros/b moved from user #{second.id} to user #{first.id}",
)
ensure
source_db&.close
end
it "moves a provider identity to the delta's owner and drops userless rows" do
owner = Fabricate(:user)
previous_owner = Fabricate(:user)
Fabricate(
:user_associated_account,
user: owner,
provider_name: "khoros",
provider_uid: "stale",
)
Fabricate(
:user_associated_account,
user: previous_owner,
provider_name: "khoros",
provider_uid: "current",
)
UserAssociatedAccount.create!(user_id: nil, provider_name: "khoros", provider_uid: "orphan")
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")
source_db.execute(
"CREATE TABLE user_associated_accounts (user_id INTEGER, provider_name TEXT, provider_uid TEXT)",
)
source_db.execute("INSERT INTO users (id) VALUES (1), (3)")
source_db.execute("INSERT INTO user_associated_accounts VALUES (1, 'khoros', 'current')")
source_db.execute("INSERT INTO user_associated_accounts VALUES (3, 'khoros', 'orphan')")
newcomer = Fabricate(:user)
importer = build_delta_user_importer(source_db, users: { 1 => owner.id, 3 => newcomer.id })
importer.update_delta_user_associated_accounts
expect(
UserAssociatedAccount.where(provider_name: "khoros").pluck(:user_id, :provider_uid),
).to contain_exactly([owner.id, "current"])
expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to include(
"khoros/current moved from user #{previous_owner.id} to user #{owner.id}",
"khoros/orphan moved from user nil to user #{newcomer.id}",
)
ensure
source_db&.close
end
end
describe "#update_delta_user_avatars" do
it "persists an upload reference only for an avatar that changed" do
changed_user = Fabricate(:user)
unchanged_user = Fabricate(:user)
old_upload = Fabricate(:upload)
new_upload = Fabricate(:upload)
unchanged_upload = Fabricate(:upload)
changed_avatar = Fabricate(:user_avatar, user: changed_user)
unchanged_avatar = Fabricate(:user_avatar, user: unchanged_user)
changed_avatar.update_column(:custom_upload_id, old_upload.id)
unchanged_avatar.update_column(:custom_upload_id, unchanged_upload.id)
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(
"CREATE TABLE users (id INTEGER PRIMARY KEY, avatar_upload_id INTEGER, anonymized INTEGER)",
)
source_db.execute("INSERT INTO users VALUES (1, 101, NULL), (2, 102, NULL)")
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@delta_update_mappings,
users: {
1 => changed_user.id,
2 => unchanged_user.id,
},
)
importer.instance_variable_set(
:@uploads_mapping,
{ "101" => new_upload.id, "102" => unchanged_upload.id },
)
importer.instance_variable_set(:@delta_stats, users: { updated_ids: Set.new })
allow(importer).to receive(:update_records) do |updates, _name, _columns|
updated_keys =
updates.filter_map do |update|
avatar = UserAvatar.find(update[:id])
next if avatar.custom_upload_id == update[:custom_upload_id]
avatar.update_column(:custom_upload_id, update[:custom_upload_id])
avatar.id
end
{ updated_keys: updated_keys }
end
importer.update_delta_user_avatars
expect(changed_avatar.reload.custom_upload_id).to eq(new_upload.id)
expect(
UploadReference.where(target_type: "UserAvatar", target_id: changed_avatar.id).pluck(
:upload_id,
),
).to contain_exactly(new_upload.id)
expect(
UploadReference.where(target_type: "UserAvatar", target_id: unchanged_avatar.id),
).to be_empty
ensure
source_db&.close
end
end
describe "#configure_unicode_usernames!" do
def build_source_db(usernames: [], group_names: [], site_settings: {})
db = SQLite3::Database.new(":memory:", results_as_hash: true)
db.execute(
"CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT, original_username TEXT)",
)
db.execute("CREATE TABLE groups (id INTEGER PRIMARY KEY, name TEXT)")
db.execute("CREATE TABLE site_settings (name TEXT, value TEXT, action TEXT)")
usernames.each_with_index do |username, index|
db.execute("INSERT INTO users (id, username) VALUES (?, ?)", [index + 1, username])
end
group_names.each_with_index do |name, index|
db.execute("INSERT INTO groups (id, name) VALUES (?, ?)", [index + 1, name])
end
site_settings.each do |name, value|
db.execute(
"INSERT INTO site_settings (name, value, action) VALUES (?, ?, 'update')",
[name.to_s, value],
)
end
db
end
def build_importer(source_db)
importer = described_class.allocate
importer.instance_variable_set(:@source_db, source_db)
importer.instance_variable_set(
:@import_issue_log_path,
File.join(Dir.mktmpdir, "issues.log"),
)
importer
end
it "enables unicode_usernames when the source contains non-ASCII usernames" do
source_db = build_source_db(usernames: %w[alice 千风])
importer = build_importer(source_db)
importer.configure_unicode_usernames!
expect(SiteSetting.unicode_usernames).to eq(true)
expect(importer.instance_variable_get(:@import_issue_counts)).to eq(
"unicode_usernames auto-enabled" => 1,
)
ensure
source_db&.close
end
it "enables unicode_usernames when only group names contain non-ASCII characters" do
source_db = build_source_db(usernames: %w[alice], group_names: ["中文组"])
importer = build_importer(source_db)
importer.configure_unicode_usernames!
expect(SiteSetting.unicode_usernames).to eq(true)
ensure
source_db&.close
end
it "keeps unicode_usernames off for ASCII-only sources" do
source_db = build_source_db(usernames: %w[alice bob], group_names: %w[team])
importer = build_importer(source_db)
importer.configure_unicode_usernames!
expect(SiteSetting.unicode_usernames).to eq(false)
expect(importer.instance_variable_get(:@import_issue_counts)).to be_nil
ensure
source_db&.close
end
it "lets an explicit source setting disable the automatic detection" do
source_db = build_source_db(usernames: %w[千风], site_settings: { unicode_usernames: "f" })
importer = build_importer(source_db)
importer.configure_unicode_usernames!
expect(SiteSetting.unicode_usernames).to eq(false)
ensure
source_db&.close
end
it "applies an explicit source setting and allowlist for ASCII-only sources" do
source_db =
build_source_db(
usernames: %w[alice],
site_settings: {
unicode_usernames: "t",
allowed_unicode_username_characters: '\p{Han}',
},
)
importer = build_importer(source_db)
importer.configure_unicode_usernames!
expect(SiteSetting.unicode_usernames).to eq(true)
expect(SiteSetting.allowed_unicode_username_characters).to eq('\p{Han}')
ensure
source_db&.close
end
it "raises a clear error when external system avatars are unavailable" do
SiteSetting.external_system_avatars_url = ""
source_db = build_source_db(usernames: %w[千风])
importer = build_importer(source_db)
expect { importer.configure_unicode_usernames! }.to raise_error(
/external_system_avatars_url/,
)
expect(SiteSetting.unicode_usernames).to eq(false)
ensure
source_db&.close
end
end
describe "#update_category_read_restricted" do
it "reconciles uploads when imported permissions restrict an existing category" do
category = Fabricate(:category)
CategoryGroup.create!(
category: category,
group: Group[:admins],
permission_type: CategoryGroup.permission_types[:full],
)
expect_enqueued_with(
job: :update_category_upload_security,
args: {
category_id: category.id,
},
) { described_class.allocate.update_category_read_restricted }
expect(category.reload.read_restricted).to eq(true)
end
end
describe "#import_user_associated_groups" do
fab!(:employee, :user)
fab!(:customer, :user)
fab!(:claimed_customer, :user)
fab!(:manual_customer, :user)
fab!(:employee_group, :group)
fab!(:customer_group, :group)
fab!(:unlinked_group, :group)
fab!(:internal_claim) do
AssociatedGroup.create!(
name: "SailPoint Internal",
provider_name: "oidc",
provider_id: "SailPoint Internal",
last_used: 2.weeks.ago,
)
end
fab!(:employee_claim) do
AssociatedGroup.create!(
name: "SailPoint Employee",
provider_name: "oidc",
provider_id: "SailPoint Employee",
last_used: 2.weeks.ago,
)
end
fab!(:customer_claim) do
AssociatedGroup.create!(
name: "Customer",
provider_name: "oidc",
provider_id: "Customer",
last_used: 2.weeks.ago,
)
end
fab!(:saml_claim) do
AssociatedGroup.create!(
name: "Customer",
provider_name: "saml",
provider_id: "Customer",
last_used: 2.weeks.ago,
)
end
let(:importer) do
described_class.allocate.tap do |instance|
instance.instance_variable_set(
:@users,
{ 10 => employee.id, 20 => customer.id, 30 => claimed_customer.id },
)
instance.instance_variable_set(
:@raw_connection,
ActiveRecord::Base.connection.raw_connection,
)
instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
end
end
def claims_for(user)
user.associated_groups.reload.order(:provider_id).pluck(:provider_name, :provider_id)
end
before do
GroupAssociatedGroup.create!(group: employee_group, associated_group: internal_claim)
GroupAssociatedGroup.create!(group: employee_group, associated_group: employee_claim)
GroupAssociatedGroup.create!(group: customer_group, associated_group: customer_claim)
GroupAssociatedGroup.create!(group: customer_group, associated_group: saml_claim)
[employee_group, customer_group, unlinked_group].each do |group|
group.group_users.delete_all
end
GroupUser.create!(group: employee_group, user: employee)
GroupUser.create!(group: customer_group, user: customer)
GroupUser.create!(group: unlinked_group, user: customer)
GroupUser.create!(group: customer_group, user: claimed_customer)
GroupUser.create!(group: customer_group, user: manual_customer)
UserAssociatedGroup.create!(user: claimed_customer, associated_group: customer_claim)
allow(Discourse).to receive(:enabled_authenticators).and_return(
[instance_double(Auth::Authenticator, name: "oidc", provides_groups?: true)],
)
end
it "seeds every claim linked to an imported member's groups, once" do
expect { importer.import_user_associated_groups }.to change {
UserAssociatedGroup.count
}.by(3)
expect(claims_for(employee)).to eq(
[%w[oidc SailPoint\ Employee], %w[oidc SailPoint\ Internal]],
)
expect(claims_for(customer)).to eq([%w[oidc Customer]])
expect(claims_for(claimed_customer)).to eq([%w[oidc Customer]])
expect(claims_for(manual_customer)).to be_empty
expect(saml_claim.reload.users).to be_empty
expect { importer.import_user_associated_groups }.not_to change {
UserAssociatedGroup.count
}
end
it "marks the seeded associated groups as used and leaves the others alone" do
importer.import_user_associated_groups
expect(internal_claim.reload.last_used).to be_within(1.minute).of(Time.zone.now)
expect(employee_claim.reload.last_used).to be_within(1.minute).of(Time.zone.now)
expect(customer_claim.reload.last_used).to be_within(1.minute).of(Time.zone.now)
expect(saml_claim.reload.last_used).to be_within(1.minute).of(2.weeks.ago)
end
it "reconciles seeded memberships with the claims returned by subsequent logins" do
GroupUser.create!(group: unlinked_group, user: employee)
importer.import_user_associated_groups
result = Auth::Result.new
result.user = employee
result.authenticator_name = "oidc"
result.extra_data = { provider: "oidc" }
result.associated_groups = [{ id: employee_claim.provider_id, name: employee_claim.name }]
result.apply_associated_attributes!
expect(employee.associated_groups.reload).to contain_exactly(employee_claim)
expect(employee.groups.reload).to include(employee_group, unlinked_group)
result.associated_groups = []
result.apply_associated_attributes!
expect(employee.associated_groups.reload).to be_empty
expect(employee.groups.reload).not_to include(employee_group)
expect(employee.groups).to include(unlinked_group)
end
it "does nothing when no enabled authenticator provides group claims" do
allow(Discourse).to receive(:enabled_authenticators).and_return(
[instance_double(Auth::Authenticator, name: "oidc", provides_groups?: false)],
)
expect { importer.import_user_associated_groups }.not_to change {
UserAssociatedGroup.count
}
end
it "does nothing when the provider's associated groups grant no Discourse group" do
GroupAssociatedGroup.where.not(associated_group: saml_claim).delete_all
expect { importer.import_user_associated_groups }.not_to change {
UserAssociatedGroup.count
}
end
end
end
RSpec.describe BulkImport::Base do
describe "#update_records" do
it "updates only supplied, changed values in one transactional batch" do
db = ActiveRecord::Base.connection_db_config.configuration_hash
connection = PG.connect(dbname: db[:database], port: db[:port])
connection.exec(<<~SQL)
CREATE TEMP TABLE delta_test_records (
id BIGINT PRIMARY KEY,
value TEXT,
preserved TEXT
)
SQL
connection.exec(
"INSERT INTO delta_test_records VALUES (1, 'old', 'keep'), (2, 'same', 'stay')",
)
importer = described_class.allocate
importer.instance_variable_set(:@raw_connection, connection)
importer.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
result =
importer.update_records(
[{ id: 1, value: "new", preserved: nil }, { id: 2, value: "same", preserved: nil }],
"delta_test_record",
%i[value preserved],
)
records = connection.exec("SELECT id, value, preserved FROM delta_test_records ORDER BY id")
expect(records.map(&:values)).to eq([%w[1 new keep], %w[2 same stay]])
expect(result).to include(
total: 2,
updated: 1,
unchanged: 1,
updated_keys: [1],
changed_counts: {
value: 1,
preserved: 0,
},
)
repeated_result =
importer.update_records(
[{ id: 1, value: "new", preserved: nil }, { id: 2, value: "same", preserved: nil }],
"delta_test_record",
%i[value preserved],
)
expect(repeated_result).to include(
total: 2,
updated: 0,
unchanged: 2,
updated_keys: [],
changed_counts: {
value: 0,
preserved: 0,
},
)
ensure
connection&.close
end
it "accepts a lazy enumeration of updates" do
db = ActiveRecord::Base.connection_db_config.configuration_hash
connection = PG.connect(dbname: db[:database], port: db[:port])
connection.exec(<<~SQL)
CREATE TEMP TABLE delta_test_records (
id BIGINT PRIMARY KEY,
value TEXT,
preserved TEXT
)
SQL
connection.exec("INSERT INTO delta_test_records VALUES (1, 'old', 'keep')")
importer = described_class.allocate
importer.instance_variable_set(:@raw_connection, connection)
importer.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
result =
importer.update_records(
[{ id: 1, value: "new", preserved: nil }].lazy,
"delta_test_record",
%i[value preserved],
)
records = connection.exec("SELECT id, value, preserved FROM delta_test_records ORDER BY id")
expect(records.map(&:values)).to eq([%w[1 new keep]])
expect(result).to include(
total: 1,
updated: 1,
unchanged: 0,
updated_keys: [1],
changed_counts: {
value: 1,
preserved: 0,
},
)
ensure
connection&.close
end
end
describe "#fix_name" do
it "transliterates names to ASCII when unicode usernames are disabled" do
importer = described_class.allocate
expect(importer.fix_name("José")).to eq("Jose")
expect(importer.fix_name("千风")).to be_nil
end
it "keeps and NFC-normalizes unicode names when unicode usernames are enabled" do
SiteSetting.unicode_usernames = true
importer = described_class.allocate
expect(importer.fix_name("千风")).to eq("千风")
expect(importer.fix_name("José")).to eq("José")
expect(importer.fix_name("风" * 61)).to eq("风" * 60)
expect(importer.fix_name("😀😀")).to be_nil
end
it "respects the unicode username character allowlist" do
SiteSetting.unicode_usernames = true
SiteSetting.allowed_unicode_username_characters = '\p{Han}'
importer = described_class.allocate
expect(importer.fix_name("千风")).to eq("千风")
expect(importer.fix_name("Кир")).to be_nil
end
end
describe "#process_user" do
def build_user_importer(importer_class = described_class)
importer = importer_class.allocate
importer.instance_variable_set(:@usernames_lower, Set.new)
importer.instance_variable_set(:@last_user_id, 0)
importer.instance_variable_set(
:@import_issue_log_path,
File.join(Dir.mktmpdir, "issues.log"),
)
%i[
users
emails
external_ids
imported_usernames
mapped_usernames
user_ids_by_username_lower
usernames_by_id
user_full_names_by_id
].each { |name| importer.instance_variable_set(:"@#{name}", {}) }
importer
end
it "imports unicode usernames with a normalized username_lower" do
SiteSetting.unicode_usernames = true
importer = build_user_importer
user = importer.process_user(imported_id: 1, username: "千风")
expect(user[:username]).to eq("千风")
expect(user[:username_lower]).to eq(User.normalize_username("千风"))
end
it "falls back to a random username and logs an issue when sanitization strips everything" do
importer = build_user_importer
user = importer.process_user(imported_id: 1, username: "千风")
expect(user[:username]).to start_with("Anonymous_")
expect(importer.instance_variable_get(:@import_issue_counts)).to eq(
"username sanitized to blank" => 1,
)
expect(importer.instance_variable_get(:@mapped_usernames)).to eq("千风" => user[:username])
end
it "keeps deduplicated unicode usernames within the 60 character limit" do
SiteSetting.unicode_usernames = true
importer = build_user_importer
long_name = "风" * 60
importer.instance_variable_get(:@usernames_lower) << User.normalize_username(long_name)
user = importer.process_user(imported_id: 1, username: long_name)
expect(user[:username]).to eq("#{"风" * 58}_1")
end
describe "reserving valid usernames" do
def import_usernames(importer, rows)
importer.reserve_valid_usernames(rows)
rows.map do |row|
external_id = JSON.parse(row["sso_record"])["external_id"] if row["sso_record"].present?
importer.process_user(
imported_id: row["id"],
username: row["username"],
email: row["email"],
external_id: external_id,
)
end
end
it "suffixes a sanitized username instead of a later valid one" do
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[{ "id" => 1, "username" => "a_a_" }, { "id" => 2, "username" => "A_A" }],
)
expect(users.map { |user| user[:username] }).to eq(%w[a_a_1 A_A])
end
it "skips suffixes that later valid usernames reserve" do
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[
{ "id" => 1, "username" => "Alex_R_" },
{ "id" => 2, "username" => "Alex_R" },
{ "id" => 3, "username" => "Alex_R_1" },
],
)
expect(users.map { |user| user[:username] }).to eq(%w[Alex_R_2 Alex_R Alex_R_1])
end
it "reserves unicode usernames that are valid under the unicode setting" do
SiteSetting.unicode_usernames = true
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[{ "id" => 1, "username" => "Michał_" }, { "id" => 2, "username" => "Michał" }],
)
expect(users.map { |user| user[:username] }).to eq(%w[Michał_1 Michał])
end
it "does not reserve unicode usernames that the ASCII setting transliterates" do
SiteSetting.unicode_usernames = false
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[{ "id" => 1, "username" => "Michał_" }, { "id" => 2, "username" => "Michał" }],
)
expect(users.map { |user| user[:username] }).to eq(%w[Michal Michal_1])
end
it "keeps the lowest source ID for valid usernames that differ only in case" do
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[{ "id" => 1, "username" => "manuel" }, { "id" => 2, "username" => "Manuel" }],
)
expect(users.map { |user| user[:username] }).to eq(%w[manuel Manuel_1])
end
it "reserves nothing for rows that map onto an existing user" do
importer = build_user_importer(BulkImport::Generic)
importer.instance_variable_set(:@emails, { "[email protected]" => 99 })
users =
import_usernames(
importer,
[
{ "id" => 1, "username" => "Bob_" },
{ "id" => 2, "username" => "Bob", "email" => "[email protected]" },
{ "id" => 3, "username" => "Carol_", "email" => "[email protected]" },
{ "id" => 4, "username" => "Carol", "email" => "[email protected]" },
],
)
expect(users.map { |user| user[:username] }).to eq(%w[Bob Bob Carol Carol])
expect(users.map { |user| user[:skip] }).to eq([nil, true, nil, true])
end
it "keeps an email available when an external ID maps an earlier row to an existing user" do
importer = build_user_importer(BulkImport::Generic)
importer.instance_variable_set(:@external_ids, { "existing-id" => 99 })
users =
import_usernames(
importer,
[
{
"id" => 1,
"username" => "Existing",
"email" => "[email protected]",
"sso_record" => { external_id: "existing-id" }.to_json,
},
{ "id" => 2, "username" => "Bob_" },
{ "id" => 3, "username" => "Bob", "email" => "[email protected]" },
],
)
expect(users.map { |user| user[:username] }).to eq(%w[Existing Bob_1 Bob])
expect(users.map { |user| user[:skip] }).to eq([true, nil, nil])
end
it "keeps an email available when an external ID duplicates an earlier source row" do
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[
{
"id" => 1,
"username" => "First",
"sso_record" => { external_id: "shared-id" }.to_json,
},
{
"id" => 2,
"username" => "Duplicate",
"email" => "[email protected]",
"sso_record" => { external_id: "shared-id" }.to_json,
},
{ "id" => 3, "username" => "Bob_" },
{ "id" => 4, "username" => "Bob", "email" => "[email protected]" },
],
)
expect(users.map { |user| user[:username] }).to eq(%w[First Duplicate Bob_1 Bob])
expect(users.map { |user| user[:skip] }).to eq([nil, true, nil, nil])
end
it "reserves usernames for users without SSO records after a user with one" do
importer = build_user_importer(BulkImport::Generic)
users =
import_usernames(
importer,
[
{
"id" => 1,
"username" => "Alice",
"sso_record" => { external_id: "alice-id" }.to_json,
},
{ "id" => 2, "username" => "Bob_" },
{ "id" => 3, "username" => "Bob" },
],
)
expect(users.map { |user| user[:username] }).to eq(%w[Alice Bob_1 Bob])
expect(users.map { |user| user[:skip] }).to eq([nil, nil, nil])
end
it "does not let a reservation displace an existing site user" do
importer = build_user_importer(BulkImport::Generic)
importer.instance_variable_get(:@usernames_lower) << "dana"
users = import_usernames(importer, [{ "id" => 1, "username" => "Dana" }])
expect(users.first[:username]).to eq("Dana_1")
end
end
end
describe "#create_posts" do
it "does not resolve or persist an import ID for a post skipped due to NUL content" do
topic = Fabricate(:topic)
importer = described_class.new
importer.instance_variable_set(:@last_post_id, Post.maximum(:id) || 0)
importer.instance_variable_set(:@posts, {})
importer.instance_variable_set(:@highest_post_number_by_topic_id, {})
importer.instance_variable_set(:@post_number_by_post_id, {})
importer.instance_variable_set(:@topic_id_by_post_id, {})
importer.instance_variable_set(
:@import_issue_log_path,
File.join(Dir.mktmpdir, "issues.log"),
)
allow(importer).to receive(:pre_cook).and_return("cooked")
expect {
importer.create_posts(
[{ imported_id: 123, topic_id: topic.id, raw: "bad\0raw" }],
) { |row| row }
}.not_to change { Post.count }
expect(importer.post_id_from_imported_id(123)).to be_nil
expect(PostCustomField.where(name: "import_id", value: "123")).to be_empty
ensure
importer&.instance_variable_get(:@raw_connection)&.close
end
end
describe "#pre_cook" do
it "links unicode mentions to imported users and groups" do
importer = described_class.allocate
importer.instance_variable_set(
:@markdown,
Redcarpet::Markdown.new(
Redcarpet::Render::HTML.new(hard_wrap: true),
no_intra_emphasis: true,
fenced_code_blocks: true,
autolink: true,
),
)
importer.instance_variable_set(:@mapped_usernames, { "千风" => "Anonymous_abc" })
importer.instance_variable_set(
:@usernames_lower,
Set.new(["anonymous_abc", User.normalize_username("张伟")]),
)
importer.instance_variable_set(
:@group_names_lower,
Set.new([User.normalize_username("中文组")]),
)
cooked = importer.pre_cook("Hello @千风 and @张伟 and @中文组")
expect(cooked).to include(
%{<a class="mention" href="/u/anonymous_abc">@Anonymous_abc</a>},
%{<a class="mention" href="/u/张伟">@张伟</a>},
%{<a class="mention-group" href="/groups/中文组">@中文组</a>},
)
end
end
describe "#log_import_issue" do
it "aggregates counts per category and streams details to the log file" do
importer = described_class.allocate
log_path = File.join(Dir.mktmpdir, "issues.log")
importer.instance_variable_set(:@import_issue_log_path, log_path)
importer.log_import_issue("unresolved user mention", "[mention|abc] (content 1)")
importer.log_import_issue("unresolved user mention", "[mention|def] (content 2)")
importer.log_import_issue("post skipped (raw contains null bytes)", "post 3")
expect(File.readlines(log_path, chomp: true)).to eq(
[
"[unresolved user mention] [mention|abc] (content 1)",
"[unresolved user mention] [mention|def] (content 2)",
"[post skipped (raw contains null bytes)] post 3",
],
)
expect { importer.report_import_issues }.to output(
[
"",
"Import issues (details in #{log_path}):",
" unresolved user mention: 2",
" post skipped (raw contains null bytes): 1",
"",
].join("\n"),
).to_stdout
end
it "reports nothing when no issues were logged" do
expect { described_class.allocate.report_import_issues }.not_to output.to_stdout
end
end
end
else
RSpec.describe "generic bulk import specs" do
it "requires the generic_import bundle group" do
skip "Run with BUNDLE_WITH=generic_import"
end
end
end