Skip to content

A platform for community discussion. Free, open, simple.

FIX: Prevent anonymized user data leaks in bulk imports (#42113)

Ruben Oussoren19849e5ba4d4
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb
index a906cfbcc76..56d6ad2d72a 100644
--- a/script/bulk_import/base.rb
+++ b/script/bulk_import/base.rb
@@ -1349,6 +1349,8 @@ class BulkImport::Base
end
def process_user(user)
+ persist_imported_username = user.delete(:persist_imported_username) != false
+
if user[:email].present?
user[:email] = user[:email].downcase
@@ -1376,7 +1378,7 @@ class BulkImport::Base
user[:username] = fix_name(user[:username]).presence || random_username
if user[:username] != imported_username
- @imported_usernames[imported_username] = user[:id]
+ @imported_usernames[imported_username] = user[:id] if persist_imported_username
@mapped_usernames[imported_username] = user[:username]
end
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rb
index 9c36acef8bb..10b7753c557 100644
--- a/script/bulk_import/generic_bulk.rb
+++ b/script/bulk_import/generic_bulk.rb
@@ -575,8 +575,6 @@ class BulkImport::Generic < BulkImport::Base
create_users(users) do |row|
next if user_id_from_imported_id(row["id"]).present?
- sso_record = JSON.parse(row["sso_record"]) if row["sso_record"].present?
-
if row["suspension"].present?
suspension = JSON.parse(row["suspension"])
suspended_at = suspension["suspended_at"]
@@ -590,8 +588,11 @@ class BulkImport::Generic < BulkImport::Base
row["registration_ip_address"] = nil
row["date_of_birth"] = nil
row["title"] = nil
+ row["sso_record"] = nil
end
+ sso_record = JSON.parse(row["sso_record"]) if row["sso_record"].present?
+
{
imported_id: row["id"],
username: row["username"],
@@ -600,6 +601,7 @@ class BulkImport::Generic < BulkImport::Base
email: row["email"],
locale: row["locale"],
external_id: sso_record&.fetch("external_id", nil),
+ persist_imported_username: row["anonymized"] != 1,
created_at: to_datetime(row["created_at"]),
staged: row["staged"],
last_seen_at: to_datetime(row["last_seen_at"]),
@@ -687,10 +689,11 @@ class BulkImport::Generic < BulkImport::Base
end
users = query(<<~SQL)
- SELECT id, timezone, email_level, email_messages_level, email_digests,
+ SELECT id, timezone, email_level, email_messages_level, email_digests, anonymized,
#{select_hide_profile_columns.join(", ")}
FROM users
- WHERE timezone IS NOT NULL
+ WHERE anonymized IS TRUE
+ OR timezone IS NOT NULL
OR email_level IS NOT NULL
OR email_messages_level IS NOT NULL
OR email_digests IS NOT NULL
@@ -704,6 +707,13 @@ class BulkImport::Generic < BulkImport::Base
user_id = user_id_from_imported_id(row["id"])
next unless user_id && existing_user_ids.add?(user_id)
+ if row["anonymized"] == 1
+ row["mailing_list_mode"] = false
+ row["email_digests"] = false
+ row["email_level"] = UserOption.email_level_types[:never]
+ row["email_messages_level"] = UserOption.email_level_types[:never]
+ end
+
options = {
user_id: user_id,
timezone: row["timezone"],
@@ -713,9 +723,11 @@ class BulkImport::Generic < BulkImport::Base
hide_profile: row["hide_profile"],
hide_presence: row["hide_presence"],
}
+ options[:mailing_list_mode] = row["mailing_list_mode"] if !row["mailing_list_mode"].nil?
options[:hide_profile_and_presence] = row["hide_profile_and_presence"] if !row[
"hide_profile_and_presence"
].nil?
+
options
end
@@ -767,12 +779,11 @@ class BulkImport::Generic < BulkImport::Base
user_fields.close
- # TODO make restriction to non-anonymized users configurable
values = query(<<~SQL)
SELECT v.*
FROM user_field_values v
JOIN users u ON v.user_id = u.id
- WHERE u.anonymized = FALSE
+ WHERE u.anonymized IS NOT TRUE
SQL
existing_user_fields =
@@ -796,6 +807,7 @@ class BulkImport::Generic < BulkImport::Base
SELECT id, sso_record
FROM users
WHERE sso_record IS NOT NULL
+ AND anonymized IS NOT TRUE
ORDER BY id
SQL
@@ -820,6 +832,7 @@ class BulkImport::Generic < BulkImport::Base
SELECT a.*, COALESCE(u.last_seen_at, u.created_at) AS last_used_at, u.email, u.username
FROM user_associated_accounts a
JOIN users u ON u.id = a.user_id
+ WHERE u.anonymized IS NOT TRUE
ORDER BY a.user_id, a.provider_name
SQL
@@ -1850,12 +1863,14 @@ class BulkImport::Generic < BulkImport::Base
end
user_notes = query(<<~SQL)
- SELECT user_id,
- JSON_GROUP_ARRAY(JSON_OBJECT('raw', raw, 'created_by', created_by_user_id, 'created_at',
- created_at)) AS note_json_text
- FROM user_notes
- GROUP BY user_id
- ORDER BY user_id, id
+ SELECT un.user_id,
+ JSON_GROUP_ARRAY(JSON_OBJECT('raw', un.raw, 'created_by', un.created_by_user_id,
+ 'created_at', un.created_at)) AS note_json_text
+ FROM user_notes un
+ JOIN users u ON u.id = un.user_id
+ WHERE u.anonymized IS NOT TRUE
+ GROUP BY un.user_id
+ ORDER BY un.user_id, un.id
SQL
existing_user_ids =
@@ -1903,10 +1918,12 @@ class BulkImport::Generic < BulkImport::Base
end
user_note_counts = query(<<~SQL)
- SELECT user_id, COUNT(*) AS count
- FROM user_notes
- GROUP BY user_id
- ORDER BY user_id
+ SELECT un.user_id, COUNT(*) AS count
+ FROM user_notes un
+ JOIN users u ON u.id = un.user_id
+ WHERE u.anonymized IS NOT TRUE
+ GROUP BY un.user_id
+ ORDER BY un.user_id
SQL
existing_user_ids = UserCustomField.where(name: "user_notes_count").pluck(:user_id).to_set
@@ -1925,12 +1942,17 @@ class BulkImport::Generic < BulkImport::Base
puts "", "Importing user custom fields..."
rows = query(<<~SQL)
- SELECT user_id, name, value, created_at
- FROM user_custom_fields
+ SELECT ucf.user_id, ucf.name, ucf.value, ucf.created_at
+ FROM user_custom_fields ucf
+ JOIN users u ON u.id = ucf.user_id
+ WHERE u.anonymized IS NOT TRUE
SQL
existing_names = query(<<~SQL) { |rs| rs.map { |r| r["name"] } }
- SELECT DISTINCT name FROM user_custom_fields
+ SELECT DISTINCT ucf.name
+ FROM user_custom_fields ucf
+ JOIN users u ON u.id = ucf.user_id
+ WHERE u.anonymized IS NOT TRUE
SQL
return if existing_names.empty?
@@ -1955,9 +1977,11 @@ class BulkImport::Generic < BulkImport::Base
puts "", "Cooking signature_cooked from signature_raw..."
sig_rows = query(<<~SQL)
- SELECT user_id, value
- FROM user_custom_fields
- WHERE name = 'signature_raw'
+ SELECT ucf.user_id, ucf.value
+ FROM user_custom_fields ucf
+ JOIN users u ON u.id = ucf.user_id
+ WHERE ucf.name = 'signature_raw'
+ AND u.anonymized IS NOT TRUE
SQL
cooked_existing = UserCustomField.where(name: "signature_cooked").pluck(:user_id).to_set
@@ -2083,6 +2107,7 @@ class BulkImport::Generic < BulkImport::Base
SELECT id, avatar_upload_id
FROM users
WHERE avatar_upload_id IS NOT NULL
+ AND anonymized IS NOT TRUE
ORDER BY id
SQL