Skip to content

A platform for community discussion. Free, open, simple.

FIX: Stop mangling CSV exports and uploads

Régis Hanolc23eef1b6fa7
diff --git a/app/controllers/admin/badges_controller.rb b/app/controllers/admin/badges_controller.rb
index ba186a7479e..6ed9f01cf50 100644
--- a/app/controllers/admin/badges_controller.rb
+++ b/app/controllers/admin/badges_controller.rb
@@ -52,53 +52,33 @@ class Admin::BadgesController < Admin::AdminController
raise Discourse::InvalidParameters if csv_file.try(:tempfile).nil? || badge.nil?
if !badge.enabled?
- render_json_error(
- I18n.t("badges.mass_award.errors.badge_disabled", badge_name: badge.display_name),
- status: 422,
- )
- return
+ error = I18n.t("badges.mass_award.errors.badge_disabled", badge_name: badge.display_name)
+ return render_json_error(error, status: 422)
end
replace_badge_owners = params[:replace_badge_owners] == "true"
ensure_users_have_badge_once = params[:grant_existing_holders] != "true"
if !ensure_users_have_badge_once && !badge.multiple_grant?
- render_json_error(
- I18n.t(
- "badges.mass_award.errors.cant_grant_multiple_times",
- badge_name: badge.display_name,
- ),
- status: 422,
- )
- return
+ error =
+ I18n.t("badges.mass_award.errors.cant_grant_multiple_times", badge_name: badge.display_name)
+ return render_json_error(error, status: 422)
end
- line_number = 1
- usernames = []
emails = []
- File.open(csv_file) do |csv|
- csv.each_line do |line|
- line = CSV.parse_line(line).first&.strip
- line_number += 1
-
- if line.present?
- if line.include?("@")
- emails << line
- else
- usernames << line
- end
- end
+ usernames = []
- if emails.size + usernames.size > MAX_CSV_LINES
- return(
- render_json_error I18n.t(
- "badges.mass_award.errors.too_many_csv_entries",
- count: MAX_CSV_LINES,
- ),
- status: 400
- )
- end
+ CSV.foreach(csv_file) do |row|
+ entry = row.first&.strip
+ next if entry.blank?
+
+ entry.include?("@") ? emails << entry : usernames << entry
+
+ if emails.size + usernames.size > MAX_CSV_LINES
+ error = I18n.t("badges.mass_award.errors.too_many_csv_entries", count: MAX_CSV_LINES)
+ return render_json_error(error, status: 400)
end
end
+
BadgeGranter.revoke_all(badge) if replace_badge_owners
results =
@@ -115,9 +95,9 @@ class Admin::BadgesController < Admin::AdminController
unmatched_entries_count: results[:unmatched_entries_count],
},
status: :ok
- rescue CSV::MalformedCSVError
- render_json_error I18n.t("badges.mass_award.errors.invalid_csv", line_number: line_number),
- status: 400
+ rescue CSV::MalformedCSVError => e
+ error = I18n.t("badges.mass_award.errors.invalid_csv", line_number: e.line_number)
+ render_json_error(error, status: 400)
end
def badge_types
diff --git a/app/controllers/admin/watched_words_controller.rb b/app/controllers/admin/watched_words_controller.rb
index 6a93e4fcdfa..49ded3fc9a2 100644
--- a/app/controllers/admin/watched_words_controller.rb
+++ b/app/controllers/admin/watched_words_controller.rb
@@ -58,7 +58,7 @@ class Admin::WatchedWordsController < Admin::StaffController
action_key = params[:action_key].to_sym
has_replacement = WatchedWord.has_replacement?(action_key)
- content = Encodings.to_utf8(File.read(file.tempfile, mode: "rb"))
+ content = File.read(file.tempfile, encoding: "bom|utf-8")
Scheduler::Defer.later("Upload watched words") do
begin
@@ -96,14 +96,16 @@ class Admin::WatchedWordsController < Admin::StaffController
action = WatchedWord.actions[name]
raise Discourse::NotFound if !action
- content = WatchedWord.where(action: action)
- if WatchedWord.has_replacement?(name)
- content = content.pluck(:word, :replacement).map(&:to_csv).join
- else
- content = content.pluck(:word).join("\n")
- end
+ content =
+ CSV.generate(+Encodings::BOM) do |csv|
+ WatchedWord
+ .where(action:)
+ .pluck(:word, :replacement, :case_sensitive)
+ .each { |row| csv << row }
+ end
headers["Content-Length"] = content.bytesize.to_s
+
send_data content,
filename: "#{Discourse.current_hostname}-watched-words-#{name}.csv",
content_type: "text/csv"
diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rb
index 95fd7a8fc05..1b9e08aa4c3 100644
--- a/app/jobs/regular/export_csv_file.rb
+++ b/app/jobs/regular/export_csv_file.rb
@@ -89,30 +89,7 @@ module Jobs
zip_filename = write_to_csv_and_zip(filename, entity)
- # create upload
- upload = nil
-
- if File.exist?(zip_filename)
- File.open(zip_filename) do |file|
- upload =
- UploadCreator.new(
- file,
- File.basename(zip_filename),
- type: "csv_export",
- for_export: "true",
- ).create_for(@current_user.id)
-
- if upload.persisted?
- user_export.update_columns(upload_id: upload.id)
- else
- Rails.logger.warn(
- "Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}",
- )
- end
- end
-
- File.delete(zip_filename)
- end
+ upload = user_export.attach_upload(zip_filename, @current_user.id)
ensure
post = notify_user(upload, export_title)
@@ -270,10 +247,6 @@ module Jobs
private
- def escape_comma(string)
- string&.include?(",") ? %Q|"#{string}"| : string
- end
-
def get_base_user_array(user)
# preloading scopes is hard, do this by hand
secondary_emails = []
@@ -289,10 +262,10 @@ module Jobs
[
user.id,
- escape_comma(user.name),
+ user.name,
user.username,
primary_email,
- escape_comma(user.title),
+ user.title,
user.created_at,
user.last_seen_at,
user.last_posted_at,
@@ -315,7 +288,7 @@ module Jobs
user.user_stat.post_count,
user.user_stat.likes_given,
user.user_stat.likes_received,
- escape_comma(user.user_profile.location),
+ user.user_profile.location,
user.user_profile.website,
user.user_profile.views,
]
@@ -327,7 +300,7 @@ module Jobs
user.single_sign_on_record.external_id,
user.single_sign_on_record.external_email,
user.single_sign_on_record.external_username,
- escape_comma(user.single_sign_on_record.external_name),
+ user.single_sign_on_record.external_name,
user.single_sign_on_record.external_avatar_url,
)
else
@@ -338,20 +311,14 @@ module Jobs
def add_custom_fields(user, user_info_array, user_field_ids)
if user_field_ids.present?
- user.user_fields.each do |custom_field|
- user_info_array << escape_comma(custom_field[1].to_s)
- end
+ user.user_fields.each { |custom_field| user_info_array << custom_field[1].to_s }
end
user_info_array
end
def add_group_names(user, user_info_array)
- group_names = user.groups.map { |g| g.name }.join(";")
- if group_names.present?
- user_info_array << escape_comma(group_names)
- else
- user_info_array << nil
- end
+ group_names = user.groups.map(&:name).join(";")
+ user_info_array << group_names.presence
user_info_array
end
diff --git a/app/jobs/regular/export_user_archive.rb b/app/jobs/regular/export_user_archive.rb
index 72054ba0648..62caeabcd65 100644
--- a/app/jobs/regular/export_user_archive.rb
+++ b/app/jobs/regular/export_user_archive.rb
@@ -188,7 +188,7 @@ module Jobs
end
def provide_results(user_export, zip_filename, export_title, args)
- create_upload_for_user(user_export, zip_filename)
+ user_export.attach_upload(zip_filename, @requesting_user.id)
ensure
post = notify_user(user_export, export_title)
@@ -494,54 +494,10 @@ module Jobs
.each { |uv| yield [uv.visited_at, uv.posts_read, uv.mobile, uv.time_read] }
end
- def get_header(entity)
- if entity == "user_list"
- header_array =
- HEADER_ATTRS_FOR["user_list"] + HEADER_ATTRS_FOR["user_stats"] +
- HEADER_ATTRS_FOR["user_profile"]
- header_array.concat(HEADER_ATTRS_FOR["user_sso"]) if SiteSetting.enable_discourse_connect
- user_custom_fields = UserField.all
- if user_custom_fields.present?
- user_custom_fields.each do |custom_field|
- header_array.push("#{custom_field.name} (custom user field)")
- end
- end
- header_array.push("group_names")
- else
- header_array = HEADER_ATTRS_FOR[entity]
- end
-
- header_array
- end
+ def get_header(entity) = HEADER_ATTRS_FOR[entity]
private
- def create_upload_for_user(user_export, zip_filename)
- upload = nil
- if File.exist?(zip_filename)
- File.open(zip_filename) do |file|
- upload =
- UploadCreator.new(
- file,
- File.basename(zip_filename),
- type: "csv_export",
- for_export: "true",
- ).create_for(@requesting_user.id)
-
- if upload.persisted?
- user_export.update_columns(upload_id: upload.id)
- else
- Rails.logger.warn(
- "Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}",
- )
- end
- end
-
- File.delete(zip_filename)
- end
- upload
- end
-
def guardian
@guardian ||= Guardian.new(@archive_for_user)
end
diff --git a/app/models/user_export.rb b/app/models/user_export.rb
index 1d14b03f038..09e715face0 100644
--- a/app/models/user_export.rb
+++ b/app/models/user_export.rb
@@ -30,6 +30,29 @@ class UserExport < ActiveRecord::Base
end
end
+ def attach_upload(zip_filename, uploader_id)
+ return if !File.exist?(zip_filename)
+
+ upload =
+ File.open(zip_filename) do |file|
+ UploadCreator.new(
+ file,
+ File.basename(zip_filename),
+ type: "csv_export",
+ for_export: "true",
+ ).create_for(uploader_id)
+ end
+
+ if upload.persisted?
+ update_columns(upload_id: upload.id)
+ else
+ Rails.logger.warn("Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}")
+ end
+
+ File.delete(zip_filename)
+ upload
+ end
+
def retain_hours
(created_at + DESTROY_CREATED_BEFORE - Time.zone.now).to_i / 1.hour
end
diff --git a/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb b/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb
index 199f7dc4392..152891a3d5c 100644
--- a/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb
+++ b/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb
@@ -17,9 +17,7 @@ module DiscourseEvents
invitees << { identifier: identifier, attendance: attendance || DEFAULT_ATTENDANCE }
end
invitees
- rescue StandardError
- # A malformed or unreadable file yields no invitees; the empty result
- # is surfaced as an upload error downstream.
+ rescue CSV::MalformedCSVError
[]
end
end
diff --git a/spec/fixtures/csv/usernames_with_bom.csv b/spec/fixtures/csv/usernames_with_bom.csv
new file mode 100644
index 00000000000..4c62a5b25cd
--- /dev/null
+++ b/spec/fixtures/csv/usernames_with_bom.csv
@@ -0,0 +1,3 @@
+username1
+username2
+username3
\ No newline at end of file
diff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rb
index 0091a87afec..21af47e3f64 100644
--- a/spec/jobs/export_csv_file_spec.rb
+++ b/spec/jobs/export_csv_file_spec.rb
@@ -564,7 +564,7 @@ RSpec.describe Jobs::ExportCsvFile do
user = to_hash(user_list_export.find { |u| u[0].to_i == user.id })
- expect(user["location"]).to eq('"La,La Land"')
+ expect(user["location"]).to eq("La,La Land")
expect(user["external_id"]).to eq("123")
expect(user["external_email"]).to eq("[email protected]")
end
diff --git a/spec/requests/admin/badges_controller_spec.rb b/spec/requests/admin/badges_controller_spec.rb
index 33422290c5e..de69e036172 100644
--- a/spec/requests/admin/badges_controller_spec.rb
+++ b/spec/requests/admin/badges_controller_spec.rb
@@ -560,6 +560,31 @@ RSpec.describe Admin::BadgesController do
expect(UserBadge.where(user: user, badge: badge).count).to eq(1)
end
+ it "awards the badge using a list of usernames saved with a UTF-8 BOM" do
+ Jobs.run_immediately!
+
+ file = file_from_fixtures("usernames_with_bom.csv", "csv")
+
+ post "/admin/badges/award/#{badge.id}.json", params: { file: fixture_file_upload(file) }
+
+ expect(response.status).to eq(200)
+ expect(UserBadge.where(user: user, badge: badge).count).to eq(1)
+ end
+
+ it "awards the badge when an entry spans a quoted newline" do
+ Jobs.run_immediately!
+
+ file = file_from_contents(%Q{"multi\nline"\n#{user.username}\n}, "usernames.csv")
+
+ post "/admin/badges/award/#{badge.id}.json",
+ params: {
+ file: Rack::Test::UploadedFile.new(file),
+ }
+
+ expect(response.status).to eq(200)
+ expect(UserBadge.where(user: user, badge: badge).count).to eq(1)
+ end
+
it "works with a CSV containing nil values" do
Jobs.run_immediately!
diff --git a/spec/requests/admin/watched_words_controller_spec.rb b/spec/requests/admin/watched_words_controller_spec.rb
index 1ffd29c5dd2..e271d41064f 100644
--- a/spec/requests/admin/watched_words_controller_spec.rb
+++ b/spec/requests/admin/watched_words_controller_spec.rb
@@ -320,8 +320,8 @@ RSpec.describe Admin::WatchedWordsController do
expect(WatchedWord.count).to eq(6)
end
- it "handles files with invalid UTF-8 sequences" do
- content = String.new("h\xE9llo\nworld\x99").force_encoding("Windows-1250")
+ it "preserves non-ASCII words in a UTF-8 file without a BOM" do
+ content = (1..30).map { |i| "badword#{i}" }.join("\n") + "\nCafé"
post "/admin/customize/watched_words/upload.json",
params: {
@@ -330,7 +330,7 @@ RSpec.describe Admin::WatchedWordsController do
}
expect(response.status).to eq(200)
- expect(WatchedWord.pluck(:word)).to contain_exactly("héllo", "world™")
+ expect(WatchedWord.pluck(:word)).to include("Café")
end
end
end
@@ -362,25 +362,57 @@ RSpec.describe Admin::WatchedWordsController do
get "/admin/customize/watched_words/action/block/download"
expect(response.status).to eq(200)
- block_words = response.body.split("\n")
- expect(block_words).to contain_exactly(block_word_1.word, block_word_2.word)
+ expect(response.body).to start_with(Encodings::BOM)
+ block_words = response.body.delete_prefix(Encodings::BOM).split("\n").map(&:parse_csv)
+ expect(block_words).to contain_exactly(
+ [block_word_1.word, nil, "false"],
+ [block_word_2.word, nil, "false"],
+ )
get "/admin/customize/watched_words/action/censor/download"
expect(response.status).to eq(200)
- censor_words = response.body.split("\n")
- expect(censor_words).to contain_exactly(censor_word_1.word)
+ censor_words = response.body.delete_prefix(Encodings::BOM).split("\n").map(&:parse_csv)
+ expect(censor_words).to contain_exactly([censor_word_1.word, nil, "false"])
get "/admin/customize/watched_words/action/tag/download"
expect(response.status).to eq(200)
- tag_words = response.body.split("\n").map(&:parse_csv)
+ tag_words = response.body.delete_prefix(Encodings::BOM).split("\n").map(&:parse_csv)
expect(tag_words).to contain_exactly(
- [autotag_1.word, autotag_1.replacement],
- [autotag_2.word, autotag_2.replacement],
+ [autotag_1.word, autotag_1.replacement, "false"],
+ [autotag_2.word, autotag_2.replacement, "false"],
)
end
end
end
+ describe "#download then #upload" do
+ before { sign_in(admin) }
+
+ it "round-trips delimiters and case sensitivity" do
+ Fabricate(
+ :watched_word,
+ action: WatchedWord.actions[:block],
+ word: 'MiXeD,"word"',
+ case_sensitive: true,
+ )
+
+ get "/admin/customize/watched_words/action/block/download"
+ expect(response.status).to eq(200)
+ exported = response.body
+
+ WatchedWord.delete_all
+
+ post "/admin/customize/watched_words/upload.json",
+ params: {
+ action_key: "block",
+ file: Rack::Test::UploadedFile.new(file_from_contents(exported, "words.csv")),
+ }
+
+ expect(response.status).to eq(200)
+ expect(WatchedWord.pluck(:word, :case_sensitive)).to eq([['MiXeD,"word"', true]])
+ end
+ end
+
describe "#clear_all" do
context "with non admins" do
it "doesn't allow them to perform #clear_all" do