A platform for community discussion. Free, open, simple.
FIX: Stop mangling CSV exports and uploads
Régis Hanolc23eef1b6fa7
diff --git a/app/controllers/admin/badges_controller.rb b/app/controllers/admin/badges_controller.rbindex ba186a7479e..6ed9f01cf50 100644--- a/app/controllers/admin/badges_controller.rb+++ b/app/controllers/admin/badges_controller.rb@@ -52,53 +52,33 @@ class Admin::BadgesController < Admin::AdminControllerraise Discourse::InvalidParameters if csv_file.try(:tempfile).nil? || badge.nil?if !badge.enabled?- render_json_error(- I18n.t("badges.mass_award.errors.badge_disabled", badge_name: badge.display_name),- status: 422,- )- return+ error = I18n.t("badges.mass_award.errors.badge_disabled", badge_name: badge.display_name)+ return render_json_error(error, status: 422)endreplace_badge_owners = params[:replace_badge_owners] == "true"ensure_users_have_badge_once = params[:grant_existing_holders] != "true"if !ensure_users_have_badge_once && !badge.multiple_grant?- render_json_error(- I18n.t(- "badges.mass_award.errors.cant_grant_multiple_times",- badge_name: badge.display_name,- ),- status: 422,- )- return+ error =+ I18n.t("badges.mass_award.errors.cant_grant_multiple_times", badge_name: badge.display_name)+ return render_json_error(error, status: 422)end- line_number = 1- usernames = []emails = []- File.open(csv_file) do |csv|- csv.each_line do |line|- line = CSV.parse_line(line).first&.strip- line_number += 1-- if line.present?- if line.include?("@")- emails << line- else- usernames << line- end- end+ usernames = []- if emails.size + usernames.size > MAX_CSV_LINES- return(- render_json_error I18n.t(- "badges.mass_award.errors.too_many_csv_entries",- count: MAX_CSV_LINES,- ),- status: 400- )- end+ CSV.foreach(csv_file) do |row|+ entry = row.first&.strip+ next if entry.blank?++ entry.include?("@") ? emails << entry : usernames << entry++ if emails.size + usernames.size > MAX_CSV_LINES+ error = I18n.t("badges.mass_award.errors.too_many_csv_entries", count: MAX_CSV_LINES)+ return render_json_error(error, status: 400)endend+BadgeGranter.revoke_all(badge) if replace_badge_ownersresults =@@ -115,9 +95,9 @@ class Admin::BadgesController < Admin::AdminControllerunmatched_entries_count: results[:unmatched_entries_count],},status: :ok- rescue CSV::MalformedCSVError- render_json_error I18n.t("badges.mass_award.errors.invalid_csv", line_number: line_number),- status: 400+ rescue CSV::MalformedCSVError => e+ error = I18n.t("badges.mass_award.errors.invalid_csv", line_number: e.line_number)+ render_json_error(error, status: 400)enddef badge_typesdiff --git a/app/controllers/admin/watched_words_controller.rb b/app/controllers/admin/watched_words_controller.rbindex 6a93e4fcdfa..49ded3fc9a2 100644--- a/app/controllers/admin/watched_words_controller.rb+++ b/app/controllers/admin/watched_words_controller.rb@@ -58,7 +58,7 @@ class Admin::WatchedWordsController < Admin::StaffControlleraction_key = params[:action_key].to_symhas_replacement = WatchedWord.has_replacement?(action_key)- content = Encodings.to_utf8(File.read(file.tempfile, mode: "rb"))+ content = File.read(file.tempfile, encoding: "bom|utf-8")Scheduler::Defer.later("Upload watched words") dobegin@@ -96,14 +96,16 @@ class Admin::WatchedWordsController < Admin::StaffControlleraction = WatchedWord.actions[name]raise Discourse::NotFound if !action- content = WatchedWord.where(action: action)- if WatchedWord.has_replacement?(name)- content = content.pluck(:word, :replacement).map(&:to_csv).join- else- content = content.pluck(:word).join("\n")- end+ content =+ CSV.generate(+Encodings::BOM) do |csv|+ WatchedWord+ .where(action:)+ .pluck(:word, :replacement, :case_sensitive)+ .each { |row| csv << row }+ endheaders["Content-Length"] = content.bytesize.to_s+send_data content,filename: "#{Discourse.current_hostname}-watched-words-#{name}.csv",content_type: "text/csv"diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rbindex 95fd7a8fc05..1b9e08aa4c3 100644--- a/app/jobs/regular/export_csv_file.rb+++ b/app/jobs/regular/export_csv_file.rb@@ -89,30 +89,7 @@ module Jobszip_filename = write_to_csv_and_zip(filename, entity)- # create upload- upload = nil-- if File.exist?(zip_filename)- File.open(zip_filename) do |file|- upload =- UploadCreator.new(- file,- File.basename(zip_filename),- type: "csv_export",- for_export: "true",- ).create_for(@current_user.id)-- if upload.persisted?- user_export.update_columns(upload_id: upload.id)- else- Rails.logger.warn(- "Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}",- )- end- end-- File.delete(zip_filename)- end+ upload = user_export.attach_upload(zip_filename, @current_user.id)ensurepost = notify_user(upload, export_title)@@ -270,10 +247,6 @@ module Jobsprivate- def escape_comma(string)- string&.include?(",") ? %Q|"#{string}"| : string- end-def get_base_user_array(user)# preloading scopes is hard, do this by handsecondary_emails = []@@ -289,10 +262,10 @@ module Jobs[user.id,- escape_comma(user.name),+ user.name,user.username,primary_email,- escape_comma(user.title),+ user.title,user.created_at,user.last_seen_at,user.last_posted_at,@@ -315,7 +288,7 @@ module Jobsuser.user_stat.post_count,user.user_stat.likes_given,user.user_stat.likes_received,- escape_comma(user.user_profile.location),+ user.user_profile.location,user.user_profile.website,user.user_profile.views,]@@ -327,7 +300,7 @@ module Jobsuser.single_sign_on_record.external_id,user.single_sign_on_record.external_email,user.single_sign_on_record.external_username,- escape_comma(user.single_sign_on_record.external_name),+ user.single_sign_on_record.external_name,user.single_sign_on_record.external_avatar_url,)else@@ -338,20 +311,14 @@ module Jobsdef add_custom_fields(user, user_info_array, user_field_ids)if user_field_ids.present?- user.user_fields.each do |custom_field|- user_info_array << escape_comma(custom_field[1].to_s)- end+ user.user_fields.each { |custom_field| user_info_array << custom_field[1].to_s }enduser_info_arrayenddef add_group_names(user, user_info_array)- group_names = user.groups.map { |g| g.name }.join(";")- if group_names.present?- user_info_array << escape_comma(group_names)- else- user_info_array << nil- end+ group_names = user.groups.map(&:name).join(";")+ user_info_array << group_names.presenceuser_info_arrayenddiff --git a/app/jobs/regular/export_user_archive.rb b/app/jobs/regular/export_user_archive.rbindex 72054ba0648..62caeabcd65 100644--- a/app/jobs/regular/export_user_archive.rb+++ b/app/jobs/regular/export_user_archive.rb@@ -188,7 +188,7 @@ module Jobsenddef provide_results(user_export, zip_filename, export_title, args)- create_upload_for_user(user_export, zip_filename)+ user_export.attach_upload(zip_filename, @requesting_user.id)ensurepost = notify_user(user_export, export_title)@@ -494,54 +494,10 @@ module Jobs.each { |uv| yield [uv.visited_at, uv.posts_read, uv.mobile, uv.time_read] }end- def get_header(entity)- if entity == "user_list"- header_array =- HEADER_ATTRS_FOR["user_list"] + HEADER_ATTRS_FOR["user_stats"] +- HEADER_ATTRS_FOR["user_profile"]- header_array.concat(HEADER_ATTRS_FOR["user_sso"]) if SiteSetting.enable_discourse_connect- user_custom_fields = UserField.all- if user_custom_fields.present?- user_custom_fields.each do |custom_field|- header_array.push("#{custom_field.name} (custom user field)")- end- end- header_array.push("group_names")- else- header_array = HEADER_ATTRS_FOR[entity]- end-- header_array- end+ def get_header(entity) = HEADER_ATTRS_FOR[entity]private- def create_upload_for_user(user_export, zip_filename)- upload = nil- if File.exist?(zip_filename)- File.open(zip_filename) do |file|- upload =- UploadCreator.new(- file,- File.basename(zip_filename),- type: "csv_export",- for_export: "true",- ).create_for(@requesting_user.id)-- if upload.persisted?- user_export.update_columns(upload_id: upload.id)- else- Rails.logger.warn(- "Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}",- )- end- end-- File.delete(zip_filename)- end- upload- end-def guardian@guardian ||= Guardian.new(@archive_for_user)enddiff --git a/app/models/user_export.rb b/app/models/user_export.rbindex 1d14b03f038..09e715face0 100644--- a/app/models/user_export.rb+++ b/app/models/user_export.rb@@ -30,6 +30,29 @@ class UserExport < ActiveRecord::Baseendend+ def attach_upload(zip_filename, uploader_id)+ return if !File.exist?(zip_filename)++ upload =+ File.open(zip_filename) do |file|+ UploadCreator.new(+ file,+ File.basename(zip_filename),+ type: "csv_export",+ for_export: "true",+ ).create_for(uploader_id)+ end++ if upload.persisted?+ update_columns(upload_id: upload.id)+ else+ Rails.logger.warn("Failed to upload the file #{zip_filename}: #{upload.errors.full_messages}")+ end++ File.delete(zip_filename)+ upload+ end+def retain_hours(created_at + DESTROY_CREATED_BEFORE - Time.zone.now).to_i / 1.hourenddiff --git a/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb b/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rbindex 199f7dc4392..152891a3d5c 100644--- a/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb+++ b/plugins/discourse-events/app/services/discourse_events/events/action/parse_invitees_csv.rb@@ -17,9 +17,7 @@ module DiscourseEventsinvitees << { identifier: identifier, attendance: attendance || DEFAULT_ATTENDANCE }endinvitees- rescue StandardError- # A malformed or unreadable file yields no invitees; the empty result- # is surfaced as an upload error downstream.+ rescue CSV::MalformedCSVError[]endenddiff --git a/spec/fixtures/csv/usernames_with_bom.csv b/spec/fixtures/csv/usernames_with_bom.csvnew file mode 100644index 00000000000..4c62a5b25cd--- /dev/null+++ b/spec/fixtures/csv/usernames_with_bom.csv@@ -0,0 +1,3 @@+username1+username2+username3\ No newline at end of filediff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rbindex 0091a87afec..21af47e3f64 100644--- a/spec/jobs/export_csv_file_spec.rb+++ b/spec/jobs/export_csv_file_spec.rb@@ -564,7 +564,7 @@ RSpec.describe Jobs::ExportCsvFile douser = to_hash(user_list_export.find { |u| u[0].to_i == user.id })- expect(user["location"]).to eq('"La,La Land"')+ expect(user["location"]).to eq("La,La Land")expect(user["external_id"]).to eq("123")expect(user["external_email"]).to eq("[email protected]")enddiff --git a/spec/requests/admin/badges_controller_spec.rb b/spec/requests/admin/badges_controller_spec.rbindex 33422290c5e..de69e036172 100644--- a/spec/requests/admin/badges_controller_spec.rb+++ b/spec/requests/admin/badges_controller_spec.rb@@ -560,6 +560,31 @@ RSpec.describe Admin::BadgesController doexpect(UserBadge.where(user: user, badge: badge).count).to eq(1)end+ it "awards the badge using a list of usernames saved with a UTF-8 BOM" do+ Jobs.run_immediately!++ file = file_from_fixtures("usernames_with_bom.csv", "csv")++ post "/admin/badges/award/#{badge.id}.json", params: { file: fixture_file_upload(file) }++ expect(response.status).to eq(200)+ expect(UserBadge.where(user: user, badge: badge).count).to eq(1)+ end++ it "awards the badge when an entry spans a quoted newline" do+ Jobs.run_immediately!++ file = file_from_contents(%Q{"multi\nline"\n#{user.username}\n}, "usernames.csv")++ post "/admin/badges/award/#{badge.id}.json",+ params: {+ file: Rack::Test::UploadedFile.new(file),+ }++ expect(response.status).to eq(200)+ expect(UserBadge.where(user: user, badge: badge).count).to eq(1)+ end+it "works with a CSV containing nil values" doJobs.run_immediately!diff --git a/spec/requests/admin/watched_words_controller_spec.rb b/spec/requests/admin/watched_words_controller_spec.rbindex 1ffd29c5dd2..e271d41064f 100644--- a/spec/requests/admin/watched_words_controller_spec.rb+++ b/spec/requests/admin/watched_words_controller_spec.rb@@ -320,8 +320,8 @@ RSpec.describe Admin::WatchedWordsController doexpect(WatchedWord.count).to eq(6)end- it "handles files with invalid UTF-8 sequences" do- content = String.new("h\xE9llo\nworld\x99").force_encoding("Windows-1250")+ it "preserves non-ASCII words in a UTF-8 file without a BOM" do+ content = (1..30).map { |i| "badword#{i}" }.join("\n") + "\nCafé"post "/admin/customize/watched_words/upload.json",params: {@@ -330,7 +330,7 @@ RSpec.describe Admin::WatchedWordsController do}expect(response.status).to eq(200)- expect(WatchedWord.pluck(:word)).to contain_exactly("héllo", "world™")+ expect(WatchedWord.pluck(:word)).to include("Café")endendend@@ -362,25 +362,57 @@ RSpec.describe Admin::WatchedWordsController doget "/admin/customize/watched_words/action/block/download"expect(response.status).to eq(200)- block_words = response.body.split("\n")- expect(block_words).to contain_exactly(block_word_1.word, block_word_2.word)+ expect(response.body).to start_with(Encodings::BOM)+ block_words = response.body.delete_prefix(Encodings::BOM).split("\n").map(&:parse_csv)+ expect(block_words).to contain_exactly(+ [block_word_1.word, nil, "false"],+ [block_word_2.word, nil, "false"],+ )get "/admin/customize/watched_words/action/censor/download"expect(response.status).to eq(200)- censor_words = response.body.split("\n")- expect(censor_words).to contain_exactly(censor_word_1.word)+ censor_words = response.body.delete_prefix(Encodings::BOM).split("\n").map(&:parse_csv)+ expect(censor_words).to contain_exactly([censor_word_1.word, nil, "false"])get "/admin/customize/watched_words/action/tag/download"expect(response.status).to eq(200)- tag_words = response.body.split("\n").map(&:parse_csv)+ tag_words = response.body.delete_prefix(Encodings::BOM).split("\n").map(&:parse_csv)expect(tag_words).to contain_exactly(- [autotag_1.word, autotag_1.replacement],- [autotag_2.word, autotag_2.replacement],+ [autotag_1.word, autotag_1.replacement, "false"],+ [autotag_2.word, autotag_2.replacement, "false"],)endendend+ describe "#download then #upload" do+ before { sign_in(admin) }++ it "round-trips delimiters and case sensitivity" do+ Fabricate(+ :watched_word,+ action: WatchedWord.actions[:block],+ word: 'MiXeD,"word"',+ case_sensitive: true,+ )++ get "/admin/customize/watched_words/action/block/download"+ expect(response.status).to eq(200)+ exported = response.body++ WatchedWord.delete_all++ post "/admin/customize/watched_words/upload.json",+ params: {+ action_key: "block",+ file: Rack::Test::UploadedFile.new(file_from_contents(exported, "words.csv")),+ }++ expect(response.status).to eq(200)+ expect(WatchedWord.pluck(:word, :case_sensitive)).to eq([['MiXeD,"word"', true]])+ end+ end+describe "#clear_all" docontext "with non admins" doit "doesn't allow them to perform #clear_all" do