Skip to content

A platform for community discussion. Free, open, simple.

MT: Support unicode usernames in the generic bulk importer (#42923)

Ruben Oussoren47806a3ac26d
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb
index 43c9a7a35b7..aef5d31d8d8 100644
--- a/script/bulk_import/base.rb
+++ b/script/bulk_import/base.rb
@@ -306,7 +306,8 @@ class BulkImport::Base
def load_indexes
puts "Loading groups indexes..."
@last_group_id = last_id(Group)
- @group_names_lower = Group.unscoped.pluck(:name).map(&:downcase).to_set
+ @group_names_lower =
+ Group.unscoped.pluck(:name).map { |name| User.normalize_username(name) }.to_set
puts "Loading users indexes..."
@last_user_id = last_id(User)
@@ -1360,7 +1361,16 @@ class BulkImport::Base
@groups[group[:imported_id].to_i] = group[:id] = @last_group_id += 1
- group[:name] = fix_name(group[:name])
+ fixed_name = fix_name(group[:name])
+ if fixed_name.blank?
+ fallback = "group_#{SecureRandom.hex(8)}"
+ log_import_issue(
+ "group name sanitized to blank",
+ "group #{group[:imported_id]} #{group[:name].inspect} -> #{fallback}",
+ )
+ fixed_name = fallback
+ end
+ group[:name] = fixed_name
if group_or_user_exist?(group[:name])
group_name = group[:name] + "_1"
@@ -1387,7 +1397,7 @@ class BulkImport::Base
end
def group_or_user_exist?(name)
- name_lowercase = name.downcase
+ name_lowercase = User.normalize_username(name)
return true if @usernames_lower.include?(name_lowercase)
@group_names_lower.add?(name_lowercase).nil?
end
@@ -1419,7 +1429,15 @@ class BulkImport::Base
imported_username = user[:original_username].presence || user[:username].dup
- user[:username] = fix_name(user[:username]).presence || random_username
+ user[:username] = fix_name(user[:username]).presence
+
+ if user[:username].blank?
+ user[:username] = random_username
+ log_import_issue(
+ "username sanitized to blank",
+ "user #{user[:imported_id]} #{imported_username.inspect} -> #{user[:username]}",
+ )
+ end
if user[:username] != imported_username
@imported_usernames[imported_username] = user[:id] if persist_imported_username
@@ -1428,12 +1446,18 @@ class BulkImport::Base
# unique username_lower
if user_exist?(user[:username])
- username = user[:username] + "_1"
- username.next! while user_exist?(username)
- user[:username] = username
+ i = 0
+ candidate = nil
+ begin
+ i += 1
+ suffix = "_#{i}"
+ candidate =
+ truncate_name(user[:username], UsernameValidator::MAX_CHARS - suffix.length) + suffix
+ end while user_exist?(candidate)
+ user[:username] = candidate
end
- user[:username_lower] = user[:username].downcase
+ user[:username_lower] = User.normalize_username(user[:username])
user[:trust_level] ||= TrustLevel[1]
user[:active] = true unless user.has_key?(:active)
user[:staged] = false if user[:staged].nil?
@@ -1464,7 +1488,7 @@ class BulkImport::Base
end
def user_exist?(username)
- username_lowercase = username.downcase
+ username_lowercase = User.normalize_username(username)
@usernames_lower.add?(username_lowercase).nil?
end
@@ -2443,16 +2467,20 @@ class BulkImport::Base
end
def fix_name(name)
- name.scrub! if name && !name.valid_encoding?
+ name = name.scrub if name && !name.valid_encoding?
+ return if name.blank?
+ name = UserNameSuggester.sanitize_username(name)
return if name.blank?
- # TODO Support Unicode if allowed in site settings and try to reuse logic from UserNameSuggester if possible
- name = ActiveSupport::Inflector.transliterate(name)
- name.gsub!(/[^\w.-]+/, "_")
- name.gsub!(/^\W+/, "")
- name.gsub!(/[^A-Za-z0-9]+$/, "")
- name.gsub!(/([-_.]{2,})/) { $1.first }
- name.strip!
- name.truncate(60, omission: "")
+ UserNameSuggester.truncate(name, UsernameValidator::MAX_CHARS)
+ end
+
+ # unlike UserNameSuggester.truncate, caps the codepoint length at max_chars
+ # so an ASCII dedup suffix can be appended without overflowing varchar(60)
+ def truncate_name(name, max_chars)
+ clusters = name.grapheme_clusters
+ clusters = clusters[0...max_chars] if clusters.size > max_chars
+ clusters.pop while clusters.join.length > max_chars
+ clusters.join
end
def random_username
@@ -2536,7 +2564,9 @@ class BulkImport::Base
%{<img src="#{cdn_url}" data-base62-sha1="#{upload_base62} #{attributes}>}
end
- cooked.gsub!(/@([-_.\w]+)/) do
+ cooked.gsub!(
+ /@([\p{Alpha}\p{M}\p{Nd}_](?:[\p{Alpha}\p{M}\p{Nd}._-]{0,58}[\p{Alpha}\p{M}\p{Nd}])?)/,
+ ) do
name = @mapped_usernames[$1] || $1
normalized_name = User.normalize_username(name)
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rb
index 213ec97b6f9..1f83c1adaf1 100644
--- a/script/bulk_import/generic_bulk.rb
+++ b/script/bulk_import/generic_bulk.rb
@@ -55,6 +55,8 @@ class BulkImport::Generic < BulkImport::Base
end
def preflight
+ configure_unicode_usernames!
+
return unless delta_import?
puts "Running delta import preflight..."
@@ -113,6 +115,59 @@ class BulkImport::Generic < BulkImport::Base
raise "Delta import preflight failed:\n #{shown_errors.join("\n ")}#{suffix}"
end
+ # non-ASCII source names require unicode_usernames before any name is
+ # sanitized; the final value must be live before both the fresh import
+ # and the delta preflight run
+ def configure_unicode_usernames!
+ if (allowlist = source_site_setting_value("allowed_unicode_username_characters"))
+ SiteSetting.set_and_log(:allowed_unicode_username_characters, allowlist)
+ end
+
+ if (explicit = source_site_setting_value("unicode_usernames"))
+ desired = %w[t true 1].include?(explicit.to_s.downcase)
+ enable_unicode_usernames! if desired && !SiteSetting.unicode_usernames
+ return
+ end
+
+ return if SiteSetting.unicode_usernames
+ return unless source_has_non_ascii_names?
+
+ enable_unicode_usernames!
+ log_import_issue(
+ "unicode_usernames auto-enabled",
+ "source contains non-ASCII usernames or group names",
+ )
+ end
+
+ def enable_unicode_usernames!
+ raise <<~MSG if SiteSetting.external_system_avatars_url.blank?
+ The source contains non-ASCII usernames, which requires enabling the
+ 'unicode_usernames' site setting, but its validator requires
+ 'external_system_avatars_url' to be set. Configure
+ external_system_avatars_url on the destination site and re-run the import.
+ MSG
+
+ SiteSetting.set_and_log(:unicode_usernames, true)
+ puts "Enabled unicode_usernames (non-ASCII names detected in source)"
+ end
+
+ def source_site_setting_value(name)
+ query(
+ "SELECT value FROM site_settings WHERE name = ? AND action = 'update' ORDER BY ROWID DESC LIMIT 1",
+ name,
+ ) { |rows| rows.first&.fetch("value", nil) }
+ end
+
+ def source_has_non_ascii_names?
+ non_ascii = ->(value) { value.present? && !value.ascii_only? }
+ user_columns = %w[username original_username] & table_column_names("users").to_a
+
+ query("SELECT #{user_columns.join(", ")} FROM users") do |rows|
+ rows.any? { |row| user_columns.any? { |column| non_ascii.call(row[column]) } }
+ end ||
+ query("SELECT name FROM groups") { |rows| rows.any? { |row| non_ascii.call(row["name"]) } }
+ end
+
def load_imported_ids
super
return unless delta_import?
@@ -267,9 +322,16 @@ class BulkImport::Generic < BulkImport::Base
# destination may hold a deduplicated variant of it that must be kept
def delta_username_unchanged?(row, discourse_id, destination_username_lower)
source_username = row["original_username"].presence || row["username"]
- return true if imported_usernames_by_user_id[discourse_id] == source_username
+ if User.normalize_username(imported_usernames_by_user_id[discourse_id]) ==
+ User.normalize_username(source_username)
+ return true
+ end
return false if destination_username_lower.blank?
+ # the destination still carries the source username verbatim: the base
+ # import kept it, so a stricter sanitizer must not rename it now
+ return true if User.normalize_username(source_username) == destination_username_lower
+
# suffix-deduplicated names are recorded nowhere when fix_name left the
# name itself unchanged, so recognize the suffix pattern directly
fixed = fix_name(row["username"])
@@ -289,10 +351,11 @@ class BulkImport::Generic < BulkImport::Base
next if row["existing_id"].present?
next if (name = fix_name(row["name"])).blank?
- if (previous_source_id = seen_names[name.downcase])
+ name_lower = User.normalize_username(name)
+ if (previous_source_id = seen_names[name_lower])
errors << "group #{row["id"]} duplicates name of source group #{previous_source_id}"
else
- seen_names[name.downcase] = row["id"]
+ seen_names[name_lower] = row["id"]
end
end
ensure
@@ -1212,7 +1275,18 @@ class BulkImport::Generic < BulkImport::Base
update = { id: discourse_id }
columns.each { |column| update[column] = row[column.to_s] }
- update[:name] = fix_name(update[:name]) if update[:name].present?
+ if update[:name].present?
+ fixed_name = fix_name(update[:name])
+ if fixed_name.present?
+ update[:name] = fixed_name
+ else
+ update.delete(:name)
+ log_import_issue(
+ "group name sanitized to blank, name kept",
+ "group #{row["id"]} #{row["name"].inspect}",
+ )
+ end
+ end
update[:bio_raw] = row["description"] unless row["description"].nil?
update[:bio_cooked] = pre_cook(update[:bio_raw]) unless update[:bio_raw].nil?
update
@@ -1222,7 +1296,7 @@ class BulkImport::Generic < BulkImport::Base
Group
.where(id: updates.map { |update| update[:id] })
.pluck(:name)
- .each { |name| @group_names_lower << name.downcase }
+ .each { |name| @group_names_lower << User.normalize_username(name) }
ensure
rows&.close
end
diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rb
index ddd404b483b..14ebdb0942b 100644
--- a/spec/script/bulk_import/generic_bulk_spec.rb
+++ b/spec/script/bulk_import/generic_bulk_spec.rb
@@ -210,6 +210,53 @@ if generic_import_dependencies_available
source_db&.close
end
+ it "keeps base-import fallback usernames untouched when unicode support is enabled later" do
+ SiteSetting.unicode_usernames = true
+ fallback_user = Fabricate(:user, username: "Anonymous_339dfc", created_at: 1.year.ago)
+ UserCustomField.create!(user: fallback_user, name: "import_username", value: "千风")
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",
+ [1, "千风", fallback_user.created_at.iso8601],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => fallback_user.id }, errors)
+
+ expect(errors).to be_empty
+ expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty
+ expect(
+ importer.delta_username_unchanged?(
+ { "username" => "千风" },
+ fallback_user.id,
+ fallback_user.username_lower,
+ ),
+ ).to eq(true)
+ ensure
+ source_db&.close
+ end
+
+ it "keeps a destination username the source still carries verbatim under a stricter sanitizer" do
+ kept_user = Fabricate(:user, username: "userjs")
+ kept_user.update_columns(username: "user.js", username_lower: "user.js")
+ importer = described_class.allocate
+
+ expect(importer.fix_name("user.js")).to eq("user")
+ expect(
+ importer.delta_username_unchanged?({ "username" => "user.js" }, kept_user.id, "user.js"),
+ ).to eq(true)
+ end
+
it "keeps users merged into pre-existing accounts updatable instead of failing" do
preexisting_user = Fabricate(:user, email: "[email protected]")
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
@@ -306,6 +353,31 @@ if generic_import_dependencies_available
source_db&.close
end
+ it "rejects duplicate group names differing only in unicode case within the delta" do
+ SiteSetting.unicode_usernames = true
+ group_a = Fabricate(:group)
+ group_b = Fabricate(:group)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE groups (
+ id INTEGER PRIMARY KEY,
+ name TEXT,
+ existing_id INTEGER
+ )
+ SQL
+ source_db.execute("INSERT INTO groups (id, name) VALUES (1, 'Équipe')")
+ source_db.execute("INSERT INTO groups (id, name) VALUES (2, 'équipe')")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_groups({ 1 => group_a.id, 2 => group_b.id }, errors)
+
+ expect(errors).to contain_exactly("group 2 duplicates name of source group 1")
+ ensure
+ source_db&.close
+ end
+
it "rejects mapped post moves" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
@@ -340,6 +412,120 @@ if generic_import_dependencies_available
source_db&.close
end
end
+
+ describe "#configure_unicode_usernames!" do
+ def build_source_db(usernames: [], group_names: [], site_settings: {})
+ db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ db.execute(
+ "CREATE TABLE users (id INTEGER PRIMARY KEY, username TEXT, original_username TEXT)",
+ )
+ db.execute("CREATE TABLE groups (id INTEGER PRIMARY KEY, name TEXT)")
+ db.execute("CREATE TABLE site_settings (name TEXT, value TEXT, action TEXT)")
+ usernames.each_with_index do |username, index|
+ db.execute("INSERT INTO users (id, username) VALUES (?, ?)", [index + 1, username])
+ end
+ group_names.each_with_index do |name, index|
+ db.execute("INSERT INTO groups (id, name) VALUES (?, ?)", [index + 1, name])
+ end
+ site_settings.each do |name, value|
+ db.execute(
+ "INSERT INTO site_settings (name, value, action) VALUES (?, ?, 'update')",
+ [name.to_s, value],
+ )
+ end
+ db
+ end
+
+ def build_importer(source_db)
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ importer.instance_variable_set(
+ :@import_issue_log_path,
+ File.join(Dir.mktmpdir, "issues.log"),
+ )
+ importer
+ end
+
+ it "enables unicode_usernames when the source contains non-ASCII usernames" do
+ source_db = build_source_db(usernames: %w[alice 千风])
+ importer = build_importer(source_db)
+
+ importer.configure_unicode_usernames!
+
+ expect(SiteSetting.unicode_usernames).to eq(true)
+ expect(importer.instance_variable_get(:@import_issue_counts)).to eq(
+ "unicode_usernames auto-enabled" => 1,
+ )
+ ensure
+ source_db&.close
+ end
+
+ it "enables unicode_usernames when only group names contain non-ASCII characters" do
+ source_db = build_source_db(usernames: %w[alice], group_names: ["中文组"])
+ importer = build_importer(source_db)
+
+ importer.configure_unicode_usernames!
+
+ expect(SiteSetting.unicode_usernames).to eq(true)
+ ensure
+ source_db&.close
+ end
+
+ it "keeps unicode_usernames off for ASCII-only sources" do
+ source_db = build_source_db(usernames: %w[alice bob], group_names: %w[team])
+ importer = build_importer(source_db)
+
+ importer.configure_unicode_usernames!
+
+ expect(SiteSetting.unicode_usernames).to eq(false)
+ expect(importer.instance_variable_get(:@import_issue_counts)).to be_nil
+ ensure
+ source_db&.close
+ end
+
+ it "lets an explicit source setting disable the automatic detection" do
+ source_db = build_source_db(usernames: %w[千风], site_settings: { unicode_usernames: "f" })
+ importer = build_importer(source_db)
+
+ importer.configure_unicode_usernames!
+
+ expect(SiteSetting.unicode_usernames).to eq(false)
+ ensure
+ source_db&.close
+ end
+
+ it "applies an explicit source setting and allowlist for ASCII-only sources" do
+ source_db =
+ build_source_db(
+ usernames: %w[alice],
+ site_settings: {
+ unicode_usernames: "t",
+ allowed_unicode_username_characters: '\p{Han}',
+ },
+ )
+ importer = build_importer(source_db)
+
+ importer.configure_unicode_usernames!
+
+ expect(SiteSetting.unicode_usernames).to eq(true)
+ expect(SiteSetting.allowed_unicode_username_characters).to eq('\p{Han}')
+ ensure
+ source_db&.close
+ end
+
+ it "raises a clear error when external system avatars are unavailable" do
+ SiteSetting.external_system_avatars_url = ""
+ source_db = build_source_db(usernames: %w[千风])
+ importer = build_importer(source_db)
+
+ expect { importer.configure_unicode_usernames! }.to raise_error(
+ /external_system_avatars_url/,
+ )
+ expect(SiteSetting.unicode_usernames).to eq(false)
+ ensure
+ source_db&.close
+ end
+ end
end
RSpec.describe BulkImport::Base do
@@ -440,6 +626,122 @@ if generic_import_dependencies_available
end
end
+ describe "#fix_name" do
+ it "transliterates names to ASCII when unicode usernames are disabled" do
+ importer = described_class.allocate
+
+ expect(importer.fix_name("José")).to eq("Jose")
+ expect(importer.fix_name("千风")).to be_nil
+ end
+
+ it "keeps and NFC-normalizes unicode names when unicode usernames are enabled" do
+ SiteSetting.unicode_usernames = true
+ importer = described_class.allocate
+
+ expect(importer.fix_name("千风")).to eq("千风")
+ expect(importer.fix_name("José")).to eq("José")
+ expect(importer.fix_name("风" * 61)).to eq("风" * 60)
+ expect(importer.fix_name("😀😀")).to be_nil
+ end
+
+ it "respects the unicode username character allowlist" do
+ SiteSetting.unicode_usernames = true
+ SiteSetting.allowed_unicode_username_characters = '\p{Han}'
+ importer = described_class.allocate
+
+ expect(importer.fix_name("千风")).to eq("千风")
+ expect(importer.fix_name("Кир")).to be_nil
+ end
+ end
+
+ describe "#process_user" do
+ def build_user_importer
+ importer = described_class.allocate
+ importer.instance_variable_set(:@usernames_lower, Set.new)
+ importer.instance_variable_set(:@last_user_id, 0)
+ importer.instance_variable_set(
+ :@import_issue_log_path,
+ File.join(Dir.mktmpdir, "issues.log"),
+ )
+ %i[
+ users
+ emails
+ external_ids
+ imported_usernames
+ mapped_usernames
+ user_ids_by_username_lower
+ usernames_by_id
+ user_full_names_by_id
+ ].each { |name| importer.instance_variable_set(:"@#{name}", {}) }
+ importer
+ end
+
+ it "imports unicode usernames with a normalized username_lower" do
+ SiteSetting.unicode_usernames = true
+ importer = build_user_importer
+
+ user = importer.process_user(imported_id: 1, username: "千风")
+
+ expect(user[:username]).to eq("千风")
+ expect(user[:username_lower]).to eq(User.normalize_username("千风"))
+ end
+
+ it "falls back to a random username and logs an issue when sanitization strips everything" do
+ importer = build_user_importer
+
+ user = importer.process_user(imported_id: 1, username: "千风")
+
+ expect(user[:username]).to start_with("Anonymous_")
+ expect(importer.instance_variable_get(:@import_issue_counts)).to eq(
+ "username sanitized to blank" => 1,
+ )
+ expect(importer.instance_variable_get(:@mapped_usernames)).to eq("千风" => user[:username])
+ end
+
+ it "keeps deduplicated unicode usernames within the 60 character limit" do
+ SiteSetting.unicode_usernames = true
+ importer = build_user_importer
+ long_name = "风" * 60
+ importer.instance_variable_get(:@usernames_lower) << User.normalize_username(long_name)
+
+ user = importer.process_user(imported_id: 1, username: long_name)
+
+ expect(user[:username]).to eq("#{"风" * 58}_1")
+ end
+ end
+
+ describe "#pre_cook" do
+ it "links unicode mentions to imported users and groups" do
+ importer = described_class.allocate
+ importer.instance_variable_set(
+ :@markdown,
+ Redcarpet::Markdown.new(
+ Redcarpet::Render::HTML.new(hard_wrap: true),
+ no_intra_emphasis: true,
+ fenced_code_blocks: true,
+ autolink: true,
+ ),
+ )
+ importer.instance_variable_set(:@mapped_usernames, { "千风" => "Anonymous_abc" })
+ importer.instance_variable_set(
+ :@usernames_lower,
+ Set.new(["anonymous_abc", User.normalize_username("张伟")]),
+ )
+ importer.instance_variable_set(
+ :@group_names_lower,
+ Set.new([User.normalize_username("中文组")]),
+ )
+
+ cooked = importer.pre_cook("Hello @千风 and @张伟 and @中文组")
+
+ expect(cooked).to include(
+ %{<a class="mention" href="/u/anonymous_abc">@Anonymous_abc</a>},
+ %{<a class="mention" href="/u/张伟">@张伟</a>},
+ %{<a class="mention-group" href="/groups/中文组">@中文组</a>},
+ )
+ end
+ end
+
describe "#log_import_issue" do
it "aggregates counts per category and streams details to the log file" do
importer = described_class.allocate