Skip to content

A platform for community discussion. Free, open, simple.

MT: Let valid usernames keep their names in the generic bulk importer (#44131)

Ruben Oussorencd6ca3bf233d
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb
index c28358425d3..38c79a37b15 100644
--- a/script/bulk_import/base.rb
+++ b/script/bulk_import/base.rb
@@ -1451,7 +1451,7 @@ class BulkImport::Base
end
# unique username_lower
- if user_exist?(user[:username])
+ if username_taken?(user[:username], user[:imported_id])
i = 0
candidate = nil
begin
@@ -1459,7 +1459,7 @@ class BulkImport::Base
suffix = "_#{i}"
candidate =
truncate_name(user[:username], UsernameValidator::MAX_CHARS - suffix.length) + suffix
- end while user_exist?(candidate)
+ end while username_taken?(candidate, user[:imported_id])
user[:username] = candidate
end
@@ -1496,6 +1496,14 @@ class BulkImport::Base
@usernames_lower.add?(username_lowercase).nil?
end
+ # Like user_exist?, but also treats a username reserved for another source
+ # user as taken.
+ def username_taken?(username, imported_id)
+ reserved_by = @reserved_usernames&.dig(User.normalize_username(username))
+ return true if reserved_by && reserved_by != imported_id.to_i
+ user_exist?(username)
+ end
+
def process_user_email(user_email)
user_email[:id] = @last_user_email_id += 1
user_email[:primary] = true
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rb
index 6b286f035b7..dc318b416ee 100644
--- a/script/bulk_import/generic_bulk.rb
+++ b/script/bulk_import/generic_bulk.rb
@@ -1474,6 +1474,8 @@ class BulkImport::Generic < BulkImport::Base
update_delta_users
end
+ query("SELECT * FROM users ORDER BY id") { |rows| reserve_valid_usernames(rows) }
+
users = query(<<~SQL)
SELECT *
FROM users
@@ -1535,6 +1537,34 @@ class BulkImport::Generic < BulkImport::Base
finish_delta_entity(:users, :users)
end
+ # Lets each source username that is already valid claim its name before an
+ # earlier row's sanitized username can take it, so only the sanitized one
+ # receives a dedup suffix. Rows that process_user will map onto an existing
+ # user by email or external ID reserve nothing.
+ def reserve_valid_usernames(rows)
+ @reserved_usernames = {}
+ emails = Set.new
+ external_ids = Set.new
+
+ rows.each do |row|
+ next if user_id_from_imported_id(row["id"]).present? || row["anonymized"] == 1
+
+ if (email = row["email"].presence&.downcase)
+ next if @emails.key?(email) || !emails.add?(email)
+ end
+
+ external_id = JSON.parse(row["sso_record"])["external_id"] if row["sso_record"].present?
+ if external_id.present?
+ next if @external_ids.key?(external_id) || !external_ids.add?(external_id)
+ end
+
+ username = row["username"]
+ next if username.blank? || fix_name(username) != username
+
+ @reserved_usernames[User.normalize_username(username)] ||= row["id"].to_i
+ end
+ end
+
def update_delta_users
destination_users = User.unscoped.where(id: delta_update_mapping(:users).values).index_by(&:id)
columns = %i[
diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rb
index 226f0da3cd1..b875ad3aaa1 100644
--- a/spec/script/bulk_import/generic_bulk_spec.rb
+++ b/spec/script/bulk_import/generic_bulk_spec.rb
@@ -1884,8 +1884,8 @@ if generic_import_dependencies_available
end
describe "#process_user" do
- def build_user_importer
- importer = described_class.allocate
+ def build_user_importer(importer_class = described_class)
+ importer = importer_class.allocate
importer.instance_variable_set(:@usernames_lower, Set.new)
importer.instance_variable_set(:@last_user_id, 0)
importer.instance_variable_set(
@@ -1937,6 +1937,113 @@ if generic_import_dependencies_available
expect(user[:username]).to eq("#{"风" * 58}_1")
end
+
+ describe "reserving valid usernames" do
+ def import_usernames(importer, rows)
+ importer.reserve_valid_usernames(rows)
+ rows.map do |row|
+ importer.process_user(
+ imported_id: row["id"],
+ username: row["username"],
+ email: row["email"],
+ )
+ end
+ end
+
+ it "suffixes a sanitized username instead of a later valid one" do
+ importer = build_user_importer(BulkImport::Generic)
+
+ users =
+ import_usernames(
+ importer,
+ [{ "id" => 1, "username" => "a_a_" }, { "id" => 2, "username" => "A_A" }],
+ )
+
+ expect(users.map { |user| user[:username] }).to eq(%w[a_a_1 A_A])
+ end
+
+ it "skips suffixes that later valid usernames reserve" do
+ importer = build_user_importer(BulkImport::Generic)
+
+ users =
+ import_usernames(
+ importer,
+ [
+ { "id" => 1, "username" => "Alex_R_" },
+ { "id" => 2, "username" => "Alex_R" },
+ { "id" => 3, "username" => "Alex_R_1" },
+ ],
+ )
+
+ expect(users.map { |user| user[:username] }).to eq(%w[Alex_R_2 Alex_R Alex_R_1])
+ end
+
+ it "reserves unicode usernames that are valid under the unicode setting" do
+ SiteSetting.unicode_usernames = true
+ importer = build_user_importer(BulkImport::Generic)
+
+ users =
+ import_usernames(
+ importer,
+ [{ "id" => 1, "username" => "Michał_" }, { "id" => 2, "username" => "Michał" }],
+ )
+
+ expect(users.map { |user| user[:username] }).to eq(%w[Michał_1 Michał])
+ end
+
+ it "does not reserve unicode usernames that the ASCII setting transliterates" do
+ SiteSetting.unicode_usernames = false
+ importer = build_user_importer(BulkImport::Generic)
+
+ users =
+ import_usernames(
+ importer,
+ [{ "id" => 1, "username" => "Michał_" }, { "id" => 2, "username" => "Michał" }],
+ )
+
+ expect(users.map { |user| user[:username] }).to eq(%w[Michal Michal_1])
+ end
+
+ it "keeps the lowest source ID for valid usernames that differ only in case" do
+ importer = build_user_importer(BulkImport::Generic)
+
+ users =
+ import_usernames(
+ importer,
+ [{ "id" => 1, "username" => "manuel" }, { "id" => 2, "username" => "Manuel" }],
+ )
+
+ expect(users.map { |user| user[:username] }).to eq(%w[manuel Manuel_1])
+ end
+
+ it "reserves nothing for rows that map onto an existing user" do
+ importer = build_user_importer(BulkImport::Generic)
+ importer.instance_variable_set(:@emails, { "[email protected]" => 99 })
+
+ users =
+ import_usernames(
+ importer,
+ [
+ { "id" => 1, "username" => "Bob_" },
+ { "id" => 2, "username" => "Bob", "email" => "[email protected]" },
+ { "id" => 3, "username" => "Carol_", "email" => "[email protected]" },
+ { "id" => 4, "username" => "Carol", "email" => "[email protected]" },
+ ],
+ )
+
+ expect(users.map { |user| user[:username] }).to eq(%w[Bob Bob Carol Carol])
+ expect(users.map { |user| user[:skip] }).to eq([nil, true, nil, true])
+ end
+
+ it "does not let a reservation displace an existing site user" do
+ importer = build_user_importer(BulkImport::Generic)
+ importer.instance_variable_get(:@usernames_lower) << "dana"
+
+ users = import_usernames(importer, [{ "id" => 1, "username" => "Dana" }])
+
+ expect(users.first[:username]).to eq("Dana_1")
+ end
+ end
end
describe "#create_posts" do