Skip to content

A platform for community discussion. Free, open, simple.

FIX: Add a UTF-8 BOM to CSV exports so spreadsheets read them correctly

Régis Hanolfe8486fe6e34
diff --git a/app/jobs/regular/export_csv_file.rb b/app/jobs/regular/export_csv_file.rb
index 8774b66a85b..95fd7a8fc05 100644
--- a/app/jobs/regular/export_csv_file.rb
+++ b/app/jobs/regular/export_csv_file.rb
@@ -465,6 +465,7 @@ module Jobs
FileUtils.mkdir_p(dirname) unless Dir.exist?(dirname)
begin
CSV.open("#{dirname}/#{entity[:filename]}.csv", "w") do |csv|
+ csv.to_io.write(Encodings::BOM)
csv << get_header(entity[:name]) if entity[:name] != "report"
public_send(entity[:method]) { |d| csv << d }
end
diff --git a/app/jobs/regular/export_user_archive.rb b/app/jobs/regular/export_user_archive.rb
index ab7f96f7686..72054ba0648 100644
--- a/app/jobs/regular/export_user_archive.rb
+++ b/app/jobs/regular/export_user_archive.rb
@@ -166,6 +166,7 @@ module Jobs
case component[:filetype]
when :csv
CSV.open("#{dirname}/#{component[:filename]}.csv", "w") do |csv|
+ csv.to_io.write(Encodings::BOM)
csv << get_header(component[:name])
public_send(component[:method]) { |d| csv << d }
end
diff --git a/app/services/custom_emoji/action/build_export_archive.rb b/app/services/custom_emoji/action/build_export_archive.rb
index 9105dabb4a1..231799da038 100644
--- a/app/services/custom_emoji/action/build_export_archive.rb
+++ b/app/services/custom_emoji/action/build_export_archive.rb
@@ -33,6 +33,7 @@ class CustomEmoji::Action::BuildExportArchive < Service::ActionBase
def write_manifest(temp_dir)
CSV.open(File.join(temp_dir, MANIFEST_FILENAME), "w") do |csv|
+ csv.to_io.write(Encodings::BOM)
csv << MANIFEST_HEADERS
emojis.each do |emoji|
csv << [emoji.name, CustomEmoji.normalize_group(emoji.group), image_filename(emoji)]
diff --git a/app/services/custom_emoji/action/parse_import_manifest.rb b/app/services/custom_emoji/action/parse_import_manifest.rb
index 90ade5a51af..7cf208ebc66 100644
--- a/app/services/custom_emoji/action/parse_import_manifest.rb
+++ b/app/services/custom_emoji/action/parse_import_manifest.rb
@@ -32,7 +32,10 @@ class CustomEmoji::Action::ParseImportManifest < Service::ActionBase
def parse_manifest
CSV.parse(
- reader.read_entry(MANIFEST_FILENAME, max_bytes: MAX_MANIFEST_BYTES, required: true),
+ reader
+ .read_entry(MANIFEST_FILENAME, max_bytes: MAX_MANIFEST_BYTES, required: true)
+ .force_encoding(Encoding::UTF_8)
+ .delete_prefix(Encodings::BOM),
headers: true,
)
end
diff --git a/lib/encodings.rb b/lib/encodings.rb
index 8e33f214d8b..f78de14a602 100644
--- a/lib/encodings.rb
+++ b/lib/encodings.rb
@@ -3,6 +3,8 @@
require "rchardet"
module Encodings
+ BOM = "\u{FEFF}"
+
def self.to_utf8(string)
result = CharDet.detect(string)
@@ -26,7 +28,7 @@ module Encodings
end
def self.delete_bom!(string)
- string.presence&.sub!(/\A\xEF\xBB\xBF/, "")
+ string.presence&.delete_prefix!(BOM)
string
end
end
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb
index 60ea39613ef..10bb98f3995 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_format_converter.rb
@@ -25,7 +25,7 @@ module DiscourseDataExplorer
def to_csv
require "csv"
- CSV.generate do |csv|
+ CSV.generate(+Encodings::BOM) do |csv|
csv << cols
pg_result.values.each { |row| csv << row }
end
diff --git a/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb
index 431b1a19aba..4897f0fb854 100644
--- a/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb
+++ b/plugins/discourse-data-explorer/spec/lib/query_result_downloader_spec.rb
@@ -27,7 +27,9 @@ describe DiscourseDataExplorer::QueryResultDownloader do
expect(result[:error]).to be_nil
expect(result[:format]).to eq(:csv)
- rows = result[:data].split("\n")
+ expect(result[:data]).to start_with(Encodings::BOM)
+
+ rows = result[:data].delete_prefix(Encodings::BOM).split("\n")
expect(rows[0]).to eq("id,name")
expect(rows[1]).to eq("1,tomtom")
expect(rows[2]).to eq("2,steak")
diff --git a/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb b/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb
index 5ea2f24d2fd..04f2a5a3728 100644
--- a/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb
+++ b/plugins/discourse-data-explorer/spec/result_format_converter_spec.rb
@@ -12,14 +12,10 @@ describe DiscourseDataExplorer::ResultFormatConverter do
describe ".convert" do
context "for csv files" do
- it "format results as a csv table with headers and columns" do
+ it "format results as a csv table with a BOM and a header row" do
result = described_class.convert(:csv, query_result)
- table = <<~CSV
- liker_user_id,liked_user_id,count
- CSV
-
- expect(result).to include(table)
+ expect(result).to start_with("#{Encodings::BOM}liker_user_id,liked_user_id,count\n")
end
end
diff --git a/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb b/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb
index a80ba53e1b5..638343d938f 100644
--- a/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb
+++ b/plugins/discourse-events/spec/jobs/export_post_event_report_csv_spec.rb
@@ -64,7 +64,8 @@ describe Jobs::ExportCsvFile do
files << entry.name
input_stream = entry.get_input_stream
- parsed_csv = CSV.parse(input_stream.read)
+ content = input_stream.read.force_encoding(Encoding::UTF_8)
+ parsed_csv = CSV.parse(content.delete_prefix(Encodings::BOM))
expect(parsed_csv[0]).to eq(%w[username status first_answered_at last_updated_at])
invitee_1 = post_event.invitees.find_by(user_id: user_1.id)
diff --git a/plugins/poll/assets/javascripts/discourse/components/poll.gjs b/plugins/poll/assets/javascripts/discourse/components/poll.gjs
index 83bac186884..fa05cfa986c 100644
--- a/plugins/poll/assets/javascripts/discourse/components/poll.gjs
+++ b/plugins/poll/assets/javascripts/discourse/components/poll.gjs
@@ -13,6 +13,7 @@ import { ajax } from "discourse/lib/ajax";
import { popupAjaxError } from "discourse/lib/ajax-error";
import { deferAnonymousAction } from "discourse/lib/anonymous-action";
import { afterRender } from "discourse/lib/decorators";
+import { triggerBlobDownload } from "discourse/lib/download-blob";
import round from "discourse/lib/round";
import dIcon from "discourse/ui-kit/helpers/d-icon";
import { i18n } from "discourse-i18n";
@@ -891,17 +892,13 @@ export default class PollComponent extends Component {
},
})
.then((csvContent) => {
- const downloadLink = document.createElement("a");
- const blob = new Blob([csvContent], {
+ const blob = new Blob(["\uFEFF" + csvContent], {
type: "text/csv;charset=utf-8;",
});
- downloadLink.href = URL.createObjectURL(blob);
- downloadLink.setAttribute(
- "download",
- `poll-export-${this.poll.name}-${this.post.id}.csv`
- );
- downloadLink.click();
- downloadLink.remove();
+
+ triggerBlobDownload(blob, {
+ fallbackFilename: `poll-export-${this.poll.name}-${this.post.id}.csv`,
+ });
})
.catch((error) => {
if (error) {
diff --git a/plugins/poll/test/javascripts/component/poll-export-test.gjs b/plugins/poll/test/javascripts/component/poll-export-test.gjs
new file mode 100644
index 00000000000..0695bfdff0e
--- /dev/null
+++ b/plugins/poll/test/javascripts/component/poll-export-test.gjs
@@ -0,0 +1,69 @@
+import EmberObject from "@ember/object";
+import { click, render } from "@ember/test-helpers";
+import { module, test } from "qunit";
+import { setupRenderingTest } from "discourse/tests/helpers/component-test";
+import pretender, { OK } from "discourse/tests/helpers/create-pretender";
+import Poll from "discourse/plugins/poll/discourse/components/poll";
+
+const BOM = "";
+const CSV_ROWS = "option,votes\n晚凪,3\n";
+
+module("Component | Poll | export", function (hooks) {
+ setupRenderingTest(hooks);
+
+ let downloadedBlob;
+ let originalCreateObjectURL;
+
+ hooks.beforeEach(function () {
+ downloadedBlob = null;
+ originalCreateObjectURL = URL.createObjectURL;
+ URL.createObjectURL = (blob) => {
+ downloadedBlob = blob;
+ return "blob:stubbed";
+ };
+
+ this.siteSettings.data_explorer_enabled = true;
+ this.siteSettings.poll_export_data_explorer_query_id = 18;
+ this.currentUser.admin = true;
+
+ pretender.post(
+ "/admin/plugins/discourse-data-explorer/queries/18/run.csv",
+ () => OK(CSV_ROWS, { "Content-Type": "text/csv" })
+ );
+ });
+
+ hooks.afterEach(function () {
+ URL.createObjectURL = originalCreateObjectURL;
+ });
+
+ test("the downloaded CSV carries exactly one UTF-8 BOM", async function (assert) {
+ this.setProperties({
+ post: EmberObject.create({
+ id: 42,
+ topic: { archived: false },
+ user_id: 29,
+ }),
+ poll: EmberObject.create({
+ name: "poll",
+ type: "regular",
+ status: "open",
+ results: "always",
+ options: [{ id: "abc", html: "晚凪", votes: 3 }],
+ voters: 3,
+ chart_type: "bar",
+ }),
+ });
+
+ await render(
+ <template><Poll @poll={{this.poll}} @post={{this.post}} /></template>
+ );
+ await click(".widget-dropdown-header");
+ await click(".export-results");
+
+ const text = new TextDecoder("utf-8", { ignoreBOM: true }).decode(
+ await downloadedBlob.arrayBuffer()
+ );
+
+ assert.strictEqual(text, BOM + CSV_ROWS, "exactly one BOM is prepended");
+ });
+});
diff --git a/spec/jobs/export_csv_file_spec.rb b/spec/jobs/export_csv_file_spec.rb
index d36465a87fd..0091a87afec 100644
--- a/spec/jobs/export_csv_file_spec.rb
+++ b/spec/jobs/export_csv_file_spec.rb
@@ -21,6 +21,16 @@ RSpec.describe Jobs::ExportCsvFile do
admin.uploads.each(&:destroy!)
end
+ it "prefixes every exported CSV with a UTF-8 BOM" do
+ Jobs::ExportCsvFile.new.execute(user_id: admin.id, entity: "user_list")
+
+ Zip::File.open(Discourse.store.path_for(Upload.last)) do |zip_file|
+ zip_file.each { |entry| expect(zip_file.read(entry)).to start_with(Encodings::BOM.b) }
+ end
+ ensure
+ admin.uploads.each(&:destroy!)
+ end
+
it "raises an error when the admin was demoted after enqueueing" do
admin.revoke_admin!
@@ -514,7 +524,8 @@ RSpec.describe Jobs::ExportCsvFile do
rows = []
Zip::File.open(Discourse.store.path_for(upload)) do |zip_file|
zip_file.each do |entry|
- csv_rows = CSV.parse(zip_file.read(entry), headers: true)
+ content = zip_file.read(entry).force_encoding(Encoding::UTF_8)
+ csv_rows = CSV.parse(content.delete_prefix(Encodings::BOM), headers: true)
rows.concat(csv_rows.map(&:to_h))
end
end
diff --git a/spec/jobs/export_user_archive_spec.rb b/spec/jobs/export_user_archive_spec.rb
index 944afd8e661..f4882ea0d23 100644
--- a/spec/jobs/export_user_archive_spec.rb
+++ b/spec/jobs/export_user_archive_spec.rb
@@ -90,7 +90,12 @@ RSpec.describe Jobs::ExportUserArchive do
files = []
Zip::File.open(Discourse.store.path_for(upload)) do |zip_file|
- zip_file.each { |entry| files << entry.name }
+ zip_file.each do |entry|
+ files << entry.name
+
+ bom = entry.name.end_with?(".csv") ? be_truthy : be_falsey
+ expect(zip_file.read(entry).start_with?(Encodings::BOM.b)).to bom
+ end
end
expect(files.size).to eq(Jobs::ExportUserArchive::COMPONENTS.length)
diff --git a/spec/requests/admin/emojis_controller_spec.rb b/spec/requests/admin/emojis_controller_spec.rb
index 8398352ff22..bc6c6e52172 100644
--- a/spec/requests/admin/emojis_controller_spec.rb
+++ b/spec/requests/admin/emojis_controller_spec.rb
@@ -281,7 +281,9 @@ RSpec.describe Admin::EmojiController do
expect(entries).to include("emoji-a.png")
expect(entries).to include("emoji-b.png")
- csv = zip.read("emojis.csv")
+ csv = zip.read("emojis.csv").force_encoding(Encoding::UTF_8)
+ expect(csv).to start_with(Encodings::BOM)
+ csv = csv.delete_prefix(Encodings::BOM)
rows = CSV.parse(csv, headers: true)
expect(rows.map { |r| r["name"] }).to contain_exactly("emoji-a", "emoji-b")
emoji_b_row = rows.find { |r| r["name"] == "emoji-b" }
diff --git a/spec/services/custom_emoji/action/parse_import_manifest_spec.rb b/spec/services/custom_emoji/action/parse_import_manifest_spec.rb
index 7f0fe72bc79..f97a3412e6c 100644
--- a/spec/services/custom_emoji/action/parse_import_manifest_spec.rb
+++ b/spec/services/custom_emoji/action/parse_import_manifest_spec.rb
@@ -29,6 +29,14 @@ RSpec.describe CustomEmoji::Action::ParseImportManifest do
end
end
+ context "with a manifest saved with a UTF-8 BOM and non-ASCII values" do
+ let(:csv_content) { "#{Encodings::BOM}name,group,filename\nparty,晚凪,party.png\n" }
+
+ it "strips the BOM and preserves the encoding" do
+ expect(rows.map { [it.name, it.group, it.filename] }).to eq([%w[party 晚凪 party.png]])
+ end
+ end
+
context "with names needing sanitization" do
let(:csv_content) { "name,group,filename\nMy Emoji!,,my-emoji.png\n" }
diff --git a/spec/services/custom_emoji/export_spec.rb b/spec/services/custom_emoji/export_spec.rb
index 522a73ae982..4be0d5ee99e 100644
--- a/spec/services/custom_emoji/export_spec.rb
+++ b/spec/services/custom_emoji/export_spec.rb
@@ -55,7 +55,8 @@ RSpec.describe CustomEmoji::Export do
it "writes a manifest row per emoji with a blank group for the default group" do
with_archive do |zip|
- manifest = CSV.parse(zip.read("emojis.csv"), headers: true)
+ csv = zip.read("emojis.csv").force_encoding(Encoding::UTF_8)
+ manifest = CSV.parse(csv.delete_prefix(Encodings::BOM), headers: true)
expect(manifest.map { |row| row.to_h.slice("name", "group", "filename") }).to eq(
[