A platform for community discussion. Free, open, simple.
MT: Let valid usernames keep their names in the generic bulk importer (#44131)
Ruben Oussorencd6ca3bf233d
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex c28358425d3..38c79a37b15 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -1451,7 +1451,7 @@ class BulkImport::Baseend# unique username_lower- if user_exist?(user[:username])+ if username_taken?(user[:username], user[:imported_id])i = 0candidate = nilbegin@@ -1459,7 +1459,7 @@ class BulkImport::Basesuffix = "_#{i}"candidate =truncate_name(user[:username], UsernameValidator::MAX_CHARS - suffix.length) + suffix- end while user_exist?(candidate)+ end while username_taken?(candidate, user[:imported_id])user[:username] = candidateend@@ -1496,6 +1496,14 @@ class BulkImport::Base@usernames_lower.add?(username_lowercase).nil?end+ # Like user_exist?, but also treats a username reserved for another source+ # user as taken.+ def username_taken?(username, imported_id)+ reserved_by = @reserved_usernames&.dig(User.normalize_username(username))+ return true if reserved_by && reserved_by != imported_id.to_i+ user_exist?(username)+ end+def process_user_email(user_email)user_email[:id] = @last_user_email_id += 1user_email[:primary] = truediff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 6b286f035b7..dc318b416ee 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -1474,6 +1474,8 @@ class BulkImport::Generic < BulkImport::Baseupdate_delta_usersend+ query("SELECT * FROM users ORDER BY id") { |rows| reserve_valid_usernames(rows) }+users = query(<<~SQL)SELECT *FROM users@@ -1535,6 +1537,34 @@ class BulkImport::Generic < BulkImport::Basefinish_delta_entity(:users, :users)end+ # Lets each source username that is already valid claim its name before an+ # earlier row's sanitized username can take it, so only the sanitized one+ # receives a dedup suffix. Rows that process_user will map onto an existing+ # user by email or external ID reserve nothing.+ def reserve_valid_usernames(rows)+ @reserved_usernames = {}+ emails = Set.new+ external_ids = Set.new++ rows.each do |row|+ next if user_id_from_imported_id(row["id"]).present? || row["anonymized"] == 1++ if (email = row["email"].presence&.downcase)+ next if @emails.key?(email) || !emails.add?(email)+ end++ external_id = JSON.parse(row["sso_record"])["external_id"] if row["sso_record"].present?+ if external_id.present?+ next if @external_ids.key?(external_id) || !external_ids.add?(external_id)+ end++ username = row["username"]+ next if username.blank? || fix_name(username) != username++ @reserved_usernames[User.normalize_username(username)] ||= row["id"].to_i+ end+ end+def update_delta_usersdestination_users = User.unscoped.where(id: delta_update_mapping(:users).values).index_by(&:id)columns = %i[diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rbindex 226f0da3cd1..b875ad3aaa1 100644--- a/spec/script/bulk_import/generic_bulk_spec.rb+++ b/spec/script/bulk_import/generic_bulk_spec.rb@@ -1884,8 +1884,8 @@ if generic_import_dependencies_availableenddescribe "#process_user" do- def build_user_importer- importer = described_class.allocate+ def build_user_importer(importer_class = described_class)+ importer = importer_class.allocateimporter.instance_variable_set(:@usernames_lower, Set.new)importer.instance_variable_set(:@last_user_id, 0)importer.instance_variable_set(@@ -1937,6 +1937,113 @@ if generic_import_dependencies_availableexpect(user[:username]).to eq("#{"风" * 58}_1")end++ describe "reserving valid usernames" do+ def import_usernames(importer, rows)+ importer.reserve_valid_usernames(rows)+ rows.map do |row|+ importer.process_user(+ imported_id: row["id"],+ username: row["username"],+ email: row["email"],+ )+ end+ end++ it "suffixes a sanitized username instead of a later valid one" do+ importer = build_user_importer(BulkImport::Generic)++ users =+ import_usernames(+ importer,+ [{ "id" => 1, "username" => "a_a_" }, { "id" => 2, "username" => "A_A" }],+ )++ expect(users.map { |user| user[:username] }).to eq(%w[a_a_1 A_A])+ end++ it "skips suffixes that later valid usernames reserve" do+ importer = build_user_importer(BulkImport::Generic)++ users =+ import_usernames(+ importer,+ [+ { "id" => 1, "username" => "Alex_R_" },+ { "id" => 2, "username" => "Alex_R" },+ { "id" => 3, "username" => "Alex_R_1" },+ ],+ )++ expect(users.map { |user| user[:username] }).to eq(%w[Alex_R_2 Alex_R Alex_R_1])+ end++ it "reserves unicode usernames that are valid under the unicode setting" do+ SiteSetting.unicode_usernames = true+ importer = build_user_importer(BulkImport::Generic)++ users =+ import_usernames(+ importer,+ [{ "id" => 1, "username" => "Michał_" }, { "id" => 2, "username" => "Michał" }],+ )++ expect(users.map { |user| user[:username] }).to eq(%w[Michał_1 Michał])+ end++ it "does not reserve unicode usernames that the ASCII setting transliterates" do+ SiteSetting.unicode_usernames = false+ importer = build_user_importer(BulkImport::Generic)++ users =+ import_usernames(+ importer,+ [{ "id" => 1, "username" => "Michał_" }, { "id" => 2, "username" => "Michał" }],+ )++ expect(users.map { |user| user[:username] }).to eq(%w[Michal Michal_1])+ end++ it "keeps the lowest source ID for valid usernames that differ only in case" do+ importer = build_user_importer(BulkImport::Generic)++ users =+ import_usernames(+ importer,+ [{ "id" => 1, "username" => "manuel" }, { "id" => 2, "username" => "Manuel" }],+ )++ expect(users.map { |user| user[:username] }).to eq(%w[manuel Manuel_1])+ end++ it "reserves nothing for rows that map onto an existing user" do+ importer = build_user_importer(BulkImport::Generic)+ importer.instance_variable_set(:@emails, { "[email protected]" => 99 })++ users =+ import_usernames(+ importer,+ [+ { "id" => 1, "username" => "Bob_" },+ { "id" => 2, "username" => "Bob", "email" => "[email protected]" },+ { "id" => 3, "username" => "Carol_", "email" => "[email protected]" },+ { "id" => 4, "username" => "Carol", "email" => "[email protected]" },+ ],+ )++ expect(users.map { |user| user[:username] }).to eq(%w[Bob Bob Carol Carol])+ expect(users.map { |user| user[:skip] }).to eq([nil, true, nil, true])+ end++ it "does not let a reservation displace an existing site user" do+ importer = build_user_importer(BulkImport::Generic)+ importer.instance_variable_get(:@usernames_lower) << "dana"++ users = import_usernames(importer, [{ "id" => 1, "username" => "Dana" }])++ expect(users.first[:username]).to eq("Dana_1")+ end+ endenddescribe "#create_posts" do