A platform for community discussion. Free, open, simple.
MT: Support unicode usernames in the generic bulk importer (#42923)
Ruben Oussoren47806a3ac26d
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex 43c9a7a35b7..aef5d31d8d8 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -306,7 +306,8 @@ class BulkImport::Basedef load_indexesputs "Loading groups indexes..."@last_group_id = last_id(Group)- @group_names_lower = Group.unscoped.pluck(:name).map(&:downcase).to_set+ @group_names_lower =+ Group.unscoped.pluck(:name).map { |name| User.normalize_username(name) }.to_setputs "Loading users indexes..."@last_user_id = last_id(User)@@ -1360,7 +1361,16 @@ class BulkImport::Base@groups[group[:imported_id].to_i] = group[:id] = @last_group_id += 1- group[:name] = fix_name(group[:name])+ fixed_name = fix_name(group[:name])+ if fixed_name.blank?+ fallback = "group_#{SecureRandom.hex(8)}"+ log_import_issue(+ "group name sanitized to blank",+ "group #{group[:imported_id]} #{group[:name].inspect} -> #{fallback}",+ )+ fixed_name = fallback+ end+ group[:name] = fixed_nameif group_or_user_exist?(group[:name])group_name = group[:name] + "_1"@@ -1387,7 +1397,7 @@ class BulkImport::Baseenddef group_or_user_exist?(name)- name_lowercase = name.downcase+ name_lowercase = User.normalize_username(name)return true if @usernames_lower.include?(name_lowercase)@group_names_lower.add?(name_lowercase).nil?end@@ -1419,7 +1429,15 @@ class BulkImport::Baseimported_username = user[:original_username].presence || user[:username].dup- user[:username] = fix_name(user[:username]).presence || random_username+ user[:username] = fix_name(user[:username]).presence++ if user[:username].blank?+ user[:username] = random_username+ log_import_issue(+ "username sanitized to blank",+ "user #{user[:imported_id]} #{imported_username.inspect} -> #{user[:username]}",+ )+ endif user[:username] != imported_username@imported_usernames[imported_username] = user[:id] if persist_imported_username@@ -1428,12 +1446,18 @@ class BulkImport::Base# unique username_lowerif user_exist?(user[:username])- username = user[:username] + "_1"- username.next! while user_exist?(username)- user[:username] = username+ i = 0+ candidate = nil+ begin+ i += 1+ suffix = "_#{i}"+ candidate =+ truncate_name(user[:username], UsernameValidator::MAX_CHARS - suffix.length) + suffix+ end while user_exist?(candidate)+ user[:username] = candidateend- user[:username_lower] = user[:username].downcase+ user[:username_lower] = User.normalize_username(user[:username])user[:trust_level] ||= TrustLevel[1]user[:active] = true unless user.has_key?(:active)user[:staged] = false if user[:staged].nil?@@ -1464,7 +1488,7 @@ class BulkImport::Baseenddef user_exist?(username)- username_lowercase = username.downcase+ username_lowercase = User.normalize_username(username)@usernames_lower.add?(username_lowercase).nil?end@@ -2443,16 +2467,20 @@ class BulkImport::Baseenddef fix_name(name)- name.scrub! if name && !name.valid_encoding?+ name = name.scrub if name && !name.valid_encoding?+ return if name.blank?+ name = UserNameSuggester.sanitize_username(name)return if name.blank?- # TODO Support Unicode if allowed in site settings and try to reuse logic from UserNameSuggester if possible- name = ActiveSupport::Inflector.transliterate(name)- name.gsub!(/[^\w.-]+/, "_")- name.gsub!(/^\W+/, "")- name.gsub!(/[^A-Za-z0-9]+$/, "")- name.gsub!(/([-_.]{2,})/) { $1.first }- name.strip!- name.truncate(60, omission: "")+ UserNameSuggester.truncate(name, UsernameValidator::MAX_CHARS)+ end++ # unlike UserNameSuggester.truncate, caps the codepoint length at max_chars+ # so an ASCII dedup suffix can be appended without overflowing varchar(60)+ def truncate_name(name, max_chars)+ clusters = name.grapheme_clusters+ clusters = clusters[0...max_chars] if clusters.size > max_chars+ clusters.pop while clusters.join.length > max_chars+ clusters.joinenddef random_username@@ -2536,7 +2564,9 @@ class BulkImport::Base%{<img src="#{cdn_url}" data-base62-sha1="#{upload_base62} #{attributes}>}end- cooked.gsub!(/@([-_.\w]+)/) do+ cooked.gsub!(+ /@([\p{Alpha}\p{M}\p{Nd}_](?:[\p{Alpha}\p{M}\p{Nd}._-]{0,58}[\p{Alpha}\p{M}\p{Nd}])?)/,+ ) doname = @mapped_usernames[$1] || $1normalized_name = User.normalize_username(name)diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 213ec97b6f9..1f83c1adaf1 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -55,6 +55,8 @@ class BulkImport::Generic < BulkImport::Baseenddef preflight+ configure_unicode_usernames!+return unless delta_import?puts "Running delta import preflight..."@@ -113,6 +115,59 @@ class BulkImport::Generic < BulkImport::Baseraise "Delta import preflight failed:\n #{shown_errors.join("\n ")}#{suffix}"end+ # non-ASCII source names require unicode_usernames before any name is+ # sanitized; the final value must be live before both the fresh import+ # and the delta preflight run+ def configure_unicode_usernames!+ if (allowlist = source_site_setting_value("allowed_unicode_username_characters"))+ SiteSetting.set_and_log(:allowed_unicode_username_characters, allowlist)+ end++ if (explicit = source_site_setting_value("unicode_usernames"))+ desired = %w[t true 1].include?(explicit.to_s.downcase)+ enable_unicode_usernames! if desired && !SiteSetting.unicode_usernames+ return+ end++ return if SiteSetting.unicode_usernames+ return unless source_has_non_ascii_names?++ enable_unicode_usernames!+ log_import_issue(+ "unicode_usernames auto-enabled",+ "source contains non-ASCII usernames or group names",+ )+ end++ def enable_unicode_usernames!+ raise <<~MSG if SiteSetting.external_system_avatars_url.blank?+ The source contains non-ASCII usernames, which requires enabling the+ 'unicode_usernames' site setting, but its validator requires+ 'external_system_avatars_url' to be set. Configure+ external_system_avatars_url on the destination site and re-run the import.+ MSG++ SiteSetting.set_and_log(:unicode_usernames, true)+ puts "Enabled unicode_usernames (non-ASCII names detected in source)"+ end++ def source_site_setting_value(name)+ query(+ "SELECT value FROM site_settings WHERE name = ? AND action = 'update' ORDER BY ROWID DESC LIMIT 1",+ name,+ ) { |rows| rows.first&.fetch("value", nil) }+ end++ def source_has_non_ascii_names?+ non_ascii = ->(value) { value.present? && !value.ascii_only? }+ user_columns = %w[username original_username] & table_column_names("users").to_a++ query("SELECT #{user_columns.join(", ")} FROM users") do |rows|+ rows.any? { |row| user_columns.any? { |column| non_ascii.call(row[column]) } }+ end ||+ query("SELECT name FROM groups") { |rows| rows.any? { |row| non_ascii.call(row["name"]) } }+ end+def load_imported_idssuperreturn unless delta_import?@@ -267,9 +322,16 @@ class BulkImport::Generic < BulkImport::Base# destination may hold a deduplicated variant of it that must be keptdef delta_username_unchanged?(row, discourse_id, destination_username_lower)source_username = row["original_username"].presence || row["username"]- return true if imported_usernames_by_user_id[discourse_id] == source_username+ if User.normalize_username(imported_usernames_by_user_id[discourse_id]) ==+ User.normalize_username(source_username)+ return true+ endreturn false if destination_username_lower.blank?+ # the destination still carries the source username verbatim: the base+ # import kept it, so a stricter sanitizer must not rename it now+ return true if User.normalize_username(source_username) == destination_username_lower+# suffix-deduplicated names are recorded nowhere when fix_name left the# name itself unchanged, so recognize the suffix pattern directlyfixed = fix_name(row["username"])@@ -289,10 +351,11 @@ class BulkImport::Generic < BulkImport::Basenext if row["existing_id"].present?next if (name = fix_name(row["name"])).blank?- if (previous_source_id = seen_names[name.downcase])+ name_lower = User.normalize_username(name)+ if (previous_source_id = seen_names[name_lower])errors << "group #{row["id"]} duplicates name of source group #{previous_source_id}"else- seen_names[name.downcase] = row["id"]+ seen_names[name_lower] = row["id"]endendensure@@ -1212,7 +1275,18 @@ class BulkImport::Generic < BulkImport::Baseupdate = { id: discourse_id }columns.each { |column| update[column] = row[column.to_s] }- update[:name] = fix_name(update[:name]) if update[:name].present?+ if update[:name].present?+ fixed_name = fix_name(update[:name])+ if fixed_name.present?+ update[:name] = fixed_name+ else+ update.delete(:name)+ log_import_issue(+ "group name sanitized to blank, name kept",+ "group #{row["id"]} #{row["name"].inspect}",+ )+ end+ endupdate[:bio_raw] = row["description"] unless row["description"].nil?update[:bio_cooked] = pre_cook(update[:bio_raw]) unless update[:bio_raw].nil?update@@ -1222,7 +1296,7 @@ class BulkImport::Generic < BulkImport::BaseGroup.where(id: updates.map { |update| update[:id] }).pluck(:name)- .each { |name| @group_names_lower << name.downcase }+ .each { |name| @group_names_lower << User.normalize_username(name) }ensurerows&.closeenddiff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rbindex ddd404b483b..14ebdb0942b 100644--- a/spec/script/bulk_import/generic_bulk_spec.rb+++ b/spec/script/bulk_import/generic_bulk_spec.rb@@ -210,6 +210,53 @@ if generic_import_dependencies_availablesource_db&.closeend+ it "keeps base-import fallback usernames untouched when unicode support is enabled later" do+ SiteSetting.unicode_usernames = true+ fallback_user = Fabricate(:user, username: "Anonymous_339dfc", created_at: 1.year.ago)+ UserCustomField.create!(user: fallback_user, name: "import_username", value: "千风")+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",+ [1, "千风", fallback_user.created_at.iso8601],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => fallback_user.id }, errors)++ expect(errors).to be_empty+ expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty+ expect(+ importer.delta_username_unchanged?(+ { "username" => "千风" },+ fallback_user.id,+ fallback_user.username_lower,+ ),+ ).to eq(true)+ ensure+ source_db&.close+ end++ it "keeps a destination username the source still carries verbatim under a stricter sanitizer" do+ kept_user = Fabricate(:user, username: "userjs")+ kept_user.update_columns(username: "user.js", username_lower: "user.js")+ importer = described_class.allocate++ expect(importer.fix_name("user.js")).to eq("user")+ expect(+ importer.delta_username_unchanged?({ "username" => "user.js" }, kept_user.id, "user.js"),+ ).to eq(true)+ end+it "keeps users merged into pre-existing accounts updatable instead of failing" dopreexisting_user = Fabricate(:user, email: "[email protected]")source_db = SQLite3::Database.new(":memory:", results_as_hash: true)@@ -306,6 +353,31 @@ if generic_import_dependencies_availablesource_db&.closeend+ it "rejects duplicate group names differing only in unicode case within the delta" do+ SiteSetting.unicode_usernames = true+ group_a = Fabricate(:group)+ group_b = Fabricate(:group)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE groups (+ id INTEGER PRIMARY KEY,+ name TEXT,+ existing_id INTEGER+ )+ SQL+ source_db.execute("INSERT INTO groups (id, name) VALUES (1, 'Équipe')")+ source_db.execute("INSERT INTO groups (id, name) VALUES (2, 'équipe')")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_groups({ 1 => group_a.id, 2 => group_b.id }, errors)++ expect(errors).to contain_exactly("group 2 duplicates name of source group 1")+ ensure+ source_db&.close+ end+it "rejects mapped post moves" dosource_db = SQLite3::Database.new(":memory:", results_as_hash: true)source_db.execute(<<~SQL)@@ -340,6 +412,120 @@ if generic_import_dependencies_availablesource_db&.closeendend++ describe "#configure_unicode_usernames!" do+ def build_source_db(usernames: [], group_names: [], site_settings: {})+ db = SQLite3::Database.new(":memory:", results_as_hash: true)+ db.execute(+ "CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT, original_username TEXT)",+ )+ db.execute("CREATE TABLE groups (id INTEGER PRIMARY KEY, name TEXT)")+ db.execute("CREATE TABLE site_settings (name TEXT, value TEXT, action TEXT)")+ usernames.each_with_index do |username, index|+ db.execute("INSERT INTO users (id, username) VALUES (?, ?)", [index + 1, username])+ end+ group_names.each_with_index do |name, index|+ db.execute("INSERT INTO groups (id, name) VALUES (?, ?)", [index + 1, name])+ end+ site_settings.each do |name, value|+ db.execute(+ "INSERT INTO site_settings (name, value, action) VALUES (?, ?, 'update')",+ [name.to_s, value],+ )+ end+ db+ end++ def build_importer(source_db)+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(+ :@import_issue_log_path,+ File.join(Dir.mktmpdir, "issues.log"),+ )+ importer+ end++ it "enables unicode_usernames when the source contains non-ASCII usernames" do+ source_db = build_source_db(usernames: %w[alice 千风])+ importer = build_importer(source_db)++ importer.configure_unicode_usernames!++ expect(SiteSetting.unicode_usernames).to eq(true)+ expect(importer.instance_variable_get(:@import_issue_counts)).to eq(+ "unicode_usernames auto-enabled" => 1,+ )+ ensure+ source_db&.close+ end++ it "enables unicode_usernames when only group names contain non-ASCII characters" do+ source_db = build_source_db(usernames: %w[alice], group_names: ["中文组"])+ importer = build_importer(source_db)++ importer.configure_unicode_usernames!++ expect(SiteSetting.unicode_usernames).to eq(true)+ ensure+ source_db&.close+ end++ it "keeps unicode_usernames off for ASCII-only sources" do+ source_db = build_source_db(usernames: %w[alice bob], group_names: %w[team])+ importer = build_importer(source_db)++ importer.configure_unicode_usernames!++ expect(SiteSetting.unicode_usernames).to eq(false)+ expect(importer.instance_variable_get(:@import_issue_counts)).to be_nil+ ensure+ source_db&.close+ end++ it "lets an explicit source setting disable the automatic detection" do+ source_db = build_source_db(usernames: %w[千风], site_settings: { unicode_usernames: "f" })+ importer = build_importer(source_db)++ importer.configure_unicode_usernames!++ expect(SiteSetting.unicode_usernames).to eq(false)+ ensure+ source_db&.close+ end++ it "applies an explicit source setting and allowlist for ASCII-only sources" do+ source_db =+ build_source_db(+ usernames: %w[alice],+ site_settings: {+ unicode_usernames: "t",+ allowed_unicode_username_characters: '\p{Han}',+ },+ )+ importer = build_importer(source_db)++ importer.configure_unicode_usernames!++ expect(SiteSetting.unicode_usernames).to eq(true)+ expect(SiteSetting.allowed_unicode_username_characters).to eq('\p{Han}')+ ensure+ source_db&.close+ end++ it "raises a clear error when external system avatars are unavailable" do+ SiteSetting.external_system_avatars_url = ""+ source_db = build_source_db(usernames: %w[千风])+ importer = build_importer(source_db)++ expect { importer.configure_unicode_usernames! }.to raise_error(+ /external_system_avatars_url/,+ )+ expect(SiteSetting.unicode_usernames).to eq(false)+ ensure+ source_db&.close+ end+ endendRSpec.describe BulkImport::Base do@@ -440,6 +626,122 @@ if generic_import_dependencies_availableendend+ describe "#fix_name" do+ it "transliterates names to ASCII when unicode usernames are disabled" do+ importer = described_class.allocate++ expect(importer.fix_name("José")).to eq("Jose")+ expect(importer.fix_name("千风")).to be_nil+ end++ it "keeps and NFC-normalizes unicode names when unicode usernames are enabled" do+ SiteSetting.unicode_usernames = true+ importer = described_class.allocate++ expect(importer.fix_name("千风")).to eq("千风")+ expect(importer.fix_name("José")).to eq("José")+ expect(importer.fix_name("风" * 61)).to eq("风" * 60)+ expect(importer.fix_name("😀😀")).to be_nil+ end++ it "respects the unicode username character allowlist" do+ SiteSetting.unicode_usernames = true+ SiteSetting.allowed_unicode_username_characters = '\p{Han}'+ importer = described_class.allocate++ expect(importer.fix_name("千风")).to eq("千风")+ expect(importer.fix_name("Кир")).to be_nil+ end+ end++ describe "#process_user" do+ def build_user_importer+ importer = described_class.allocate+ importer.instance_variable_set(:@usernames_lower, Set.new)+ importer.instance_variable_set(:@last_user_id, 0)+ importer.instance_variable_set(+ :@import_issue_log_path,+ File.join(Dir.mktmpdir, "issues.log"),+ )+ %i[+ users+ emails+ external_ids+ imported_usernames+ mapped_usernames+ user_ids_by_username_lower+ usernames_by_id+ user_full_names_by_id+ ].each { |name| importer.instance_variable_set(:"@#{name}", {}) }+ importer+ end++ it "imports unicode usernames with a normalized username_lower" do+ SiteSetting.unicode_usernames = true+ importer = build_user_importer++ user = importer.process_user(imported_id: 1, username: "千风")++ expect(user[:username]).to eq("千风")+ expect(user[:username_lower]).to eq(User.normalize_username("千风"))+ end++ it "falls back to a random username and logs an issue when sanitization strips everything" do+ importer = build_user_importer++ user = importer.process_user(imported_id: 1, username: "千风")++ expect(user[:username]).to start_with("Anonymous_")+ expect(importer.instance_variable_get(:@import_issue_counts)).to eq(+ "username sanitized to blank" => 1,+ )+ expect(importer.instance_variable_get(:@mapped_usernames)).to eq("千风" => user[:username])+ end++ it "keeps deduplicated unicode usernames within the 60 character limit" do+ SiteSetting.unicode_usernames = true+ importer = build_user_importer+ long_name = "风" * 60+ importer.instance_variable_get(:@usernames_lower) << User.normalize_username(long_name)++ user = importer.process_user(imported_id: 1, username: long_name)++ expect(user[:username]).to eq("#{"风" * 58}_1")+ end+ end++ describe "#pre_cook" do+ it "links unicode mentions to imported users and groups" do+ importer = described_class.allocate+ importer.instance_variable_set(+ :@markdown,+ Redcarpet::Markdown.new(+ Redcarpet::Render::HTML.new(hard_wrap: true),+ no_intra_emphasis: true,+ fenced_code_blocks: true,+ autolink: true,+ ),+ )+ importer.instance_variable_set(:@mapped_usernames, { "千风" => "Anonymous_abc" })+ importer.instance_variable_set(+ :@usernames_lower,+ Set.new(["anonymous_abc", User.normalize_username("张伟")]),+ )+ importer.instance_variable_set(+ :@group_names_lower,+ Set.new([User.normalize_username("中文组")]),+ )++ cooked = importer.pre_cook("Hello @千风 and @张伟 and @中文组")++ expect(cooked).to include(+ %{<a class="mention" href="/u/anonymous_abc">@Anonymous_abc</a>},+ %{<a class="mention" href="/u/张伟">@张伟</a>},+ %{<a class="mention-group" href="/groups/中文组">@中文组</a>},+ )+ end+ end+describe "#log_import_issue" doit "aggregates counts per category and streams details to the log file" doimporter = described_class.allocate