A platform for community discussion. Free, open, simple.
FIX: Prevent anonymized user data leaks in bulk imports (#42113)
Ruben Oussoren19849e5ba4d4
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex a906cfbcc76..56d6ad2d72a 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -1349,6 +1349,8 @@ class BulkImport::Baseenddef process_user(user)+ persist_imported_username = user.delete(:persist_imported_username) != false+if user[:email].present?user[:email] = user[:email].downcase@@ -1376,7 +1378,7 @@ class BulkImport::Baseuser[:username] = fix_name(user[:username]).presence || random_usernameif user[:username] != imported_username- @imported_usernames[imported_username] = user[:id]+ @imported_usernames[imported_username] = user[:id] if persist_imported_username@mapped_usernames[imported_username] = user[:username]enddiff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 9c36acef8bb..10b7753c557 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -575,8 +575,6 @@ class BulkImport::Generic < BulkImport::Basecreate_users(users) do |row|next if user_id_from_imported_id(row["id"]).present?- sso_record = JSON.parse(row["sso_record"]) if row["sso_record"].present?-if row["suspension"].present?suspension = JSON.parse(row["suspension"])suspended_at = suspension["suspended_at"]@@ -590,8 +588,11 @@ class BulkImport::Generic < BulkImport::Baserow["registration_ip_address"] = nilrow["date_of_birth"] = nilrow["title"] = nil+ row["sso_record"] = nilend+ sso_record = JSON.parse(row["sso_record"]) if row["sso_record"].present?+{imported_id: row["id"],username: row["username"],@@ -600,6 +601,7 @@ class BulkImport::Generic < BulkImport::Baseemail: row["email"],locale: row["locale"],external_id: sso_record&.fetch("external_id", nil),+ persist_imported_username: row["anonymized"] != 1,created_at: to_datetime(row["created_at"]),staged: row["staged"],last_seen_at: to_datetime(row["last_seen_at"]),@@ -687,10 +689,11 @@ class BulkImport::Generic < BulkImport::Baseendusers = query(<<~SQL)- SELECT id, timezone, email_level, email_messages_level, email_digests,+ SELECT id, timezone, email_level, email_messages_level, email_digests, anonymized,#{select_hide_profile_columns.join(", ")}FROM users- WHERE timezone IS NOT NULL+ WHERE anonymized IS TRUE+ OR timezone IS NOT NULLOR email_level IS NOT NULLOR email_messages_level IS NOT NULLOR email_digests IS NOT NULL@@ -704,6 +707,13 @@ class BulkImport::Generic < BulkImport::Baseuser_id = user_id_from_imported_id(row["id"])next unless user_id && existing_user_ids.add?(user_id)+ if row["anonymized"] == 1+ row["mailing_list_mode"] = false+ row["email_digests"] = false+ row["email_level"] = UserOption.email_level_types[:never]+ row["email_messages_level"] = UserOption.email_level_types[:never]+ end+options = {user_id: user_id,timezone: row["timezone"],@@ -713,9 +723,11 @@ class BulkImport::Generic < BulkImport::Basehide_profile: row["hide_profile"],hide_presence: row["hide_presence"],}+ options[:mailing_list_mode] = row["mailing_list_mode"] if !row["mailing_list_mode"].nil?options[:hide_profile_and_presence] = row["hide_profile_and_presence"] if !row["hide_profile_and_presence"].nil?+optionsend@@ -767,12 +779,11 @@ class BulkImport::Generic < BulkImport::Baseuser_fields.close- # TODO make restriction to non-anonymized users configurablevalues = query(<<~SQL)SELECT v.*FROM user_field_values vJOIN users u ON v.user_id = u.id- WHERE u.anonymized = FALSE+ WHERE u.anonymized IS NOT TRUESQLexisting_user_fields =@@ -796,6 +807,7 @@ class BulkImport::Generic < BulkImport::BaseSELECT id, sso_recordFROM usersWHERE sso_record IS NOT NULL+ AND anonymized IS NOT TRUEORDER BY idSQL@@ -820,6 +832,7 @@ class BulkImport::Generic < BulkImport::BaseSELECT a.*, COALESCE(u.last_seen_at, u.created_at) AS last_used_at, u.email, u.usernameFROM user_associated_accounts aJOIN users u ON u.id = a.user_id+ WHERE u.anonymized IS NOT TRUEORDER BY a.user_id, a.provider_nameSQL@@ -1850,12 +1863,14 @@ class BulkImport::Generic < BulkImport::Baseenduser_notes = query(<<~SQL)- SELECT user_id,- JSON_GROUP_ARRAY(JSON_OBJECT('raw', raw, 'created_by', created_by_user_id, 'created_at',- created_at)) AS note_json_text- FROM user_notes- GROUP BY user_id- ORDER BY user_id, id+ SELECT un.user_id,+ JSON_GROUP_ARRAY(JSON_OBJECT('raw', un.raw, 'created_by', un.created_by_user_id,+ 'created_at', un.created_at)) AS note_json_text+ FROM user_notes un+ JOIN users u ON u.id = un.user_id+ WHERE u.anonymized IS NOT TRUE+ GROUP BY un.user_id+ ORDER BY un.user_id, un.idSQLexisting_user_ids =@@ -1903,10 +1918,12 @@ class BulkImport::Generic < BulkImport::Baseenduser_note_counts = query(<<~SQL)- SELECT user_id, COUNT(*) AS count- FROM user_notes- GROUP BY user_id- ORDER BY user_id+ SELECT un.user_id, COUNT(*) AS count+ FROM user_notes un+ JOIN users u ON u.id = un.user_id+ WHERE u.anonymized IS NOT TRUE+ GROUP BY un.user_id+ ORDER BY un.user_idSQLexisting_user_ids = UserCustomField.where(name: "user_notes_count").pluck(:user_id).to_set@@ -1925,12 +1942,17 @@ class BulkImport::Generic < BulkImport::Baseputs "", "Importing user custom fields..."rows = query(<<~SQL)- SELECT user_id, name, value, created_at- FROM user_custom_fields+ SELECT ucf.user_id, ucf.name, ucf.value, ucf.created_at+ FROM user_custom_fields ucf+ JOIN users u ON u.id = ucf.user_id+ WHERE u.anonymized IS NOT TRUESQLexisting_names = query(<<~SQL) { |rs| rs.map { |r| r["name"] } }- SELECT DISTINCT name FROM user_custom_fields+ SELECT DISTINCT ucf.name+ FROM user_custom_fields ucf+ JOIN users u ON u.id = ucf.user_id+ WHERE u.anonymized IS NOT TRUESQLreturn if existing_names.empty?@@ -1955,9 +1977,11 @@ class BulkImport::Generic < BulkImport::Baseputs "", "Cooking signature_cooked from signature_raw..."sig_rows = query(<<~SQL)- SELECT user_id, value- FROM user_custom_fields- WHERE name = 'signature_raw'+ SELECT ucf.user_id, ucf.value+ FROM user_custom_fields ucf+ JOIN users u ON u.id = ucf.user_id+ WHERE ucf.name = 'signature_raw'+ AND u.anonymized IS NOT TRUESQLcooked_existing = UserCustomField.where(name: "signature_cooked").pluck(:user_id).to_set@@ -2083,6 +2107,7 @@ class BulkImport::Generic < BulkImport::BaseSELECT id, avatar_upload_idFROM usersWHERE avatar_upload_id IS NOT NULL+ AND anonymized IS NOT TRUEORDER BY idSQL