A platform for community discussion. Free, open, simple.
FIX: Add a UTF-8 BOM to CSV exports so spreadsheets read them correctly
Régis Hanolfe8486fe6e34
diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rbindex 8774b66a85b..95fd7a8fc05 100644--- a/app/jobs/regular/export_csv_file.rb+++ b/app/jobs/regular/export_csv_file.rb@@ -465,6 +465,7 @@ module JobsFileUtils.mkdir_p(dirname) unless Dir.exist?(dirname)beginCSV.open("#{dirname}/#{entity[:filename]}.csv", "w") do |csv|+ csv.to_io.write(Encodings::BOM)csv << get_header(entity[:name]) if entity[:name] != "report"public_send(entity[:method]) { |d| csv << d }enddiff --git a/app/jobs/regular/export_user_archive.rb b/app/jobs/regular/export_user_archive.rbindex ab7f96f7686..72054ba0648 100644--- a/app/jobs/regular/export_user_archive.rb+++ b/app/jobs/regular/export_user_archive.rb@@ -166,6 +166,7 @@ module Jobscase component[:filetype]when :csvCSV.open("#{dirname}/#{component[:filename]}.csv", "w") do |csv|+ csv.to_io.write(Encodings::BOM)csv << get_header(component[:name])public_send(component[:method]) { |d| csv << d }enddiff --git a/app/services/custom_emoji/action/build_export_archive.rb b/app/services/custom_emoji/action/build_export_archive.rbindex 9105dabb4a1..231799da038 100644--- a/app/services/custom_emoji/action/build_export_archive.rb+++ b/app/services/custom_emoji/action/build_export_archive.rb@@ -33,6 +33,7 @@ class CustomEmoji::Action::BuildExportArchive < Service::ActionBasedef write_manifest(temp_dir)CSV.open(File.join(temp_dir, MANIFEST_FILENAME), "w") do |csv|+ csv.to_io.write(Encodings::BOM)csv << MANIFEST_HEADERSemojis.each do |emoji|csv << [emoji.name, CustomEmoji.normalize_group(emoji.group), image_filename(emoji)]diff --git a/app/services/custom_emoji/action/parse_import_manifest.rb b/app/services/custom_emoji/action/parse_import_manifest.rbindex 90ade5a51af..7cf208ebc66 100644--- a/app/services/custom_emoji/action/parse_import_manifest.rb+++ b/app/services/custom_emoji/action/parse_import_manifest.rb@@ -32,7 +32,10 @@ class CustomEmoji::Action::ParseImportManifest < Service::ActionBasedef parse_manifestCSV.parse(- reader.read_entry(MANIFEST_FILENAME, max_bytes: MAX_MANIFEST_BYTES, required: true),+ reader+ .read_entry(MANIFEST_FILENAME, max_bytes: MAX_MANIFEST_BYTES, required: true)+ .force_encoding(Encoding::UTF_8)+ .delete_prefix(Encodings::BOM),headers: true,)enddiff --git a/lib/encodings.rb b/lib/encodings.rbindex 8e33f214d8b..f78de14a602 100644--- a/lib/encodings.rb+++ b/lib/encodings.rb@@ -3,6 +3,8 @@require "rchardet"module Encodings+ BOM = "\u{FEFF}"+def self.to_utf8(string)result = CharDet.detect(string)@@ -26,7 +28,7 @@ module Encodingsenddef self.delete_bom!(string)- string.presence&.sub!(/\A\xEF\xBB\xBF/, "")+ string.presence&.delete_prefix!(BOM)stringendenddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rbindex 60ea39613ef..10bb98f3995 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb@@ -25,7 +25,7 @@ module DiscourseDataExplorerdef to_csvrequire "csv"- CSV.generate do |csv|+ CSV.generate(+Encodings::BOM) do |csv|csv << colspg_result.values.each { |row| csv << row }enddiff --git a/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rbindex 431b1a19aba..4897f0fb854 100644--- a/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb+++ b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb@@ -27,7 +27,9 @@ describe DiscourseDataExplorer::QueryResultDownloader doexpect(result[:error]).to be_nilexpect(result[:format]).to eq(:csv)- rows = result[:data].split("\n")+ expect(result[:data]).to start_with(Encodings::BOM)++ rows = result[:data].delete_prefix(Encodings::BOM).split("\n")expect(rows[0]).to eq("id,name")expect(rows[1]).to eq("1,tomtom")expect(rows[2]).to eq("2,steak")diff --git a/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb b/plugins/discourse-data-explorer/spec/result_format_converter_spec.rbindex 5ea2f24d2fd..04f2a5a3728 100644--- a/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb+++ b/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb@@ -12,14 +12,10 @@ describe DiscourseDataExplorer::ResultFormatConverter dodescribe ".convert" docontext "for csv files" do- it "format results as a csv table with headers and columns" do+ it "format results as a csv table with a BOM and a header row" doresult = described_class.convert(:csv, query_result)- table = <<~CSV- liker_user_id,liked_user_id,count- CSV-- expect(result).to include(table)+ expect(result).to start_with("#{Encodings::BOM}liker_user_id,liked_user_id,count\n")endenddiff --git a/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb b/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rbindex a80ba53e1b5..638343d938f 100644--- a/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb+++ b/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb@@ -64,7 +64,8 @@ describe Jobs::ExportCsvFile dofiles << entry.nameinput_stream = entry.get_input_stream- parsed_csv = CSV.parse(input_stream.read)+ content = input_stream.read.force_encoding(Encoding::UTF_8)+ parsed_csv = CSV.parse(content.delete_prefix(Encodings::BOM))expect(parsed_csv[0]).to eq(%w[username status first_answered_at last_updated_at])invitee_1 = post_event.invitees.find_by(user_id: user_1.id)diff --git a/plugins/poll/assets/javascripts/discourse/components/poll.gjs b/plugins/poll/assets/javascripts/discourse/components/poll.gjsindex 83bac186884..fa05cfa986c 100644--- a/plugins/poll/assets/javascripts/discourse/components/poll.gjs+++ b/plugins/poll/assets/javascripts/discourse/components/poll.gjs@@ -13,6 +13,7 @@ import { ajax } from "discourse/lib/ajax";import { popupAjaxError } from "discourse/lib/ajax-error";import { deferAnonymousAction } from "discourse/lib/anonymous-action";import { afterRender } from "discourse/lib/decorators";+import { triggerBlobDownload } from "discourse/lib/download-blob";import round from "discourse/lib/round";import dIcon from "discourse/ui-kit/helpers/d-icon";import { i18n } from "discourse-i18n";@@ -891,17 +892,13 @@ export default class PollComponent extends Component {},}).then((csvContent) => {- const downloadLink = document.createElement("a");- const blob = new Blob([csvContent], {+ const blob = new Blob(["\uFEFF" + csvContent], {type: "text/csv;charset=utf-8;",});- downloadLink.href = URL.createObjectURL(blob);- downloadLink.setAttribute(- "download",- `poll-export-${this.poll.name}-${this.post.id}.csv`- );- downloadLink.click();- downloadLink.remove();++ triggerBlobDownload(blob, {+ fallbackFilename: `poll-export-${this.poll.name}-${this.post.id}.csv`,+ });}).catch((error) => {if (error) {diff --git a/plugins/poll/test/javascripts/component/poll-export-test.gjs b/plugins/poll/test/javascripts/component/poll-export-test.gjsnew file mode 100644index 00000000000..0695bfdff0e--- /dev/null+++ b/plugins/poll/test/javascripts/component/poll-export-test.gjs@@ -0,0 +1,69 @@+import EmberObject from "@ember/object";+import { click, render } from "@ember/test-helpers";+import { module, test } from "qunit";+import { setupRenderingTest } from "discourse/tests/helpers/component-test";+import pretender, { OK } from "discourse/tests/helpers/create-pretender";+import Poll from "discourse/plugins/poll/discourse/components/poll";++const BOM = "";+const CSV_ROWS = "option,votes\n晚凪,3\n";++module("Component | Poll | export", function (hooks) {+ setupRenderingTest(hooks);++ let downloadedBlob;+ let originalCreateObjectURL;++ hooks.beforeEach(function () {+ downloadedBlob = null;+ originalCreateObjectURL = URL.createObjectURL;+ URL.createObjectURL = (blob) => {+ downloadedBlob = blob;+ return "blob:stubbed";+ };++ this.siteSettings.data_explorer_enabled = true;+ this.siteSettings.poll_export_data_explorer_query_id = 18;+ this.currentUser.admin = true;++ pretender.post(+ "/admin/plugins/discourse-data-explorer/queries/18/run.csv",+ () => OK(CSV_ROWS, { "Content-Type": "text/csv" })+ );+ });++ hooks.afterEach(function () {+ URL.createObjectURL = originalCreateObjectURL;+ });++ test("the downloaded CSV carries exactly one UTF-8 BOM", async function (assert) {+ this.setProperties({+ post: EmberObject.create({+ id: 42,+ topic: { archived: false },+ user_id: 29,+ }),+ poll: EmberObject.create({+ name: "poll",+ type: "regular",+ status: "open",+ results: "always",+ options: [{ id: "abc", html: "晚凪", votes: 3 }],+ voters: 3,+ chart_type: "bar",+ }),+ });++ await render(+ <template><Poll @poll={{this.poll}} @post={{this.post}} /></template>+ );+ await click(".widget-dropdown-header");+ await click(".export-results");++ const text = new TextDecoder("utf-8", { ignoreBOM: true }).decode(+ await downloadedBlob.arrayBuffer()+ );++ assert.strictEqual(text, BOM + CSV_ROWS, "exactly one BOM is prepended");+ });+});diff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rbindex d36465a87fd..0091a87afec 100644--- a/spec/jobs/export_csv_file_spec.rb+++ b/spec/jobs/export_csv_file_spec.rb@@ -21,6 +21,16 @@ RSpec.describe Jobs::ExportCsvFile doadmin.uploads.each(&:destroy!)end+ it "prefixes every exported CSV with a UTF-8 BOM" do+ Jobs::ExportCsvFile.new.execute(user_id: admin.id, entity: "user_list")++ Zip::File.open(Discourse.store.path_for(Upload.last)) do |zip_file|+ zip_file.each { |entry| expect(zip_file.read(entry)).to start_with(Encodings::BOM.b) }+ end+ ensure+ admin.uploads.each(&:destroy!)+ end+it "raises an error when the admin was demoted after enqueueing" doadmin.revoke_admin!@@ -514,7 +524,8 @@ RSpec.describe Jobs::ExportCsvFile dorows = []Zip::File.open(Discourse.store.path_for(upload)) do |zip_file|zip_file.each do |entry|- csv_rows = CSV.parse(zip_file.read(entry), headers: true)+ content = zip_file.read(entry).force_encoding(Encoding::UTF_8)+ csv_rows = CSV.parse(content.delete_prefix(Encodings::BOM), headers: true)rows.concat(csv_rows.map(&:to_h))endenddiff --git a/spec/jobs/export_user_archive_spec.rb b/spec/jobs/export_user_archive_spec.rbindex 944afd8e661..f4882ea0d23 100644--- a/spec/jobs/export_user_archive_spec.rb+++ b/spec/jobs/export_user_archive_spec.rb@@ -90,7 +90,12 @@ RSpec.describe Jobs::ExportUserArchive dofiles = []Zip::File.open(Discourse.store.path_for(upload)) do |zip_file|- zip_file.each { |entry| files << entry.name }+ zip_file.each do |entry|+ files << entry.name++ bom = entry.name.end_with?(".csv") ? be_truthy : be_falsey+ expect(zip_file.read(entry).start_with?(Encodings::BOM.b)).to bom+ endendexpect(files.size).to eq(Jobs::ExportUserArchive::COMPONENTS.length)diff --git a/spec/requests/admin/emojis_controller_spec.rb b/spec/requests/admin/emojis_controller_spec.rbindex 8398352ff22..bc6c6e52172 100644--- a/spec/requests/admin/emojis_controller_spec.rb+++ b/spec/requests/admin/emojis_controller_spec.rb@@ -281,7 +281,9 @@ RSpec.describe Admin::EmojiController doexpect(entries).to include("emoji-a.png")expect(entries).to include("emoji-b.png")- csv = zip.read("emojis.csv")+ csv = zip.read("emojis.csv").force_encoding(Encoding::UTF_8)+ expect(csv).to start_with(Encodings::BOM)+ csv = csv.delete_prefix(Encodings::BOM)rows = CSV.parse(csv, headers: true)expect(rows.map { |r| r["name"] }).to contain_exactly("emoji-a", "emoji-b")emoji_b_row = rows.find { |r| r["name"] == "emoji-b" }diff --git a/spec/services/custom_emoji/action/parse_import_manifest_spec.rb b/spec/services/custom_emoji/action/parse_import_manifest_spec.rbindex 7f0fe72bc79..f97a3412e6c 100644--- a/spec/services/custom_emoji/action/parse_import_manifest_spec.rb+++ b/spec/services/custom_emoji/action/parse_import_manifest_spec.rb@@ -29,6 +29,14 @@ RSpec.describe CustomEmoji::Action::ParseImportManifest doendend+ context "with a manifest saved with a UTF-8 BOM and non-ASCII values" do+ let(:csv_content) { "#{Encodings::BOM}name,group,filename\nparty,晚凪,party.png\n" }++ it "strips the BOM and preserves the encoding" do+ expect(rows.map { [it.name, it.group, it.filename] }).to eq([%w[party 晚凪 party.png]])+ end+ end+context "with names needing sanitization" dolet(:csv_content) { "name,group,filename\nMy Emoji!,,my-emoji.png\n" }diff --git a/spec/services/custom_emoji/export_spec.rb b/spec/services/custom_emoji/export_spec.rbindex 522a73ae982..4be0d5ee99e 100644--- a/spec/services/custom_emoji/export_spec.rb+++ b/spec/services/custom_emoji/export_spec.rb@@ -55,7 +55,8 @@ RSpec.describe CustomEmoji::Export doit "writes a manifest row per emoji with a blank group for the default group" dowith_archive do |zip|- manifest = CSV.parse(zip.read("emojis.csv"), headers: true)+ csv = zip.read("emojis.csv").force_encoding(Encoding::UTF_8)+ manifest = CSV.parse(csv.delete_prefix(Encodings::BOM), headers: true)expect(manifest.map { |row| row.to_h.slice("name", "group", "filename") }).to eq([