Skip to content

A platform for community discussion. Free, open, simple.

FIX: Generate Data Explorer reports without crashing on ordinary results

Régis Hanol2179c9c25655
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb
index a2fde64b1f0..e1c7b3f3782 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb
@@ -4,7 +4,7 @@ module DiscourseDataExplorer
class ReportGenerator
def self.generate(query_id, query_params, recipients, opts = {})
query = DiscourseDataExplorer::Query.find(query_id)
- return [] if !query || recipients.empty?
+ return [] if recipients.empty?
recipients =
filter_recipients_by_query_access(
@@ -13,11 +13,9 @@ module DiscourseDataExplorer
users_from_group: opts[:users_from_group],
)
params = params_to_hash(query_params)
+ result = run_query(query, params)
- result = DataExplorer.run_query(query, params)
- query.record_run!
-
- return [] if opts[:skip_empty] && result[:pg_result].values.empty?
+ return [] if opts[:skip_empty] && result[:pg_result].ntuples.zero?
table =
ResultToMarkdown.convert(result[:pg_result], render_url_columns: opts[:render_url_columns])
@@ -26,88 +24,80 @@ module DiscourseDataExplorer
def self.generate_post(query_id, query_params, opts = {})
query = DiscourseDataExplorer::Query.find(query_id)
- return {} if !query
-
params = params_to_hash(query_params)
+ result = run_query(query, params)
- result = DataExplorer.run_query(query, params)
- query.record_run!
-
- return {} if opts[:skip_empty] && result[:pg_result].values.empty?
+ return {} if opts[:skip_empty] && result[:pg_result].ntuples.zero?
table =
ResultToMarkdown.convert(result[:pg_result], render_url_columns: opts[:render_url_columns])
build_report_post(query, table, attach_csv: opts[:attach_csv], result:)
end
+ def self.run_query(query, params)
+ result = DataExplorer.run_query(query, params)
+ query.record_run!
+ raise result[:error] if result[:error]
+
+ result
+ end
+
def self.params_to_hash(query_params)
params = JSON.parse(query_params)
- params.map { |p| p.is_a?(Hash) ? [p["key"], p["value"]] : p }.to_h
+ params.to_h { |p| p.is_a?(Hash) ? [p["key"], p["value"]] : p }
end
def self.build_report_pms(query, table = "", targets = [], attach_csv: false, result: nil)
- pms = []
- upload = create_csv_upload(query, result) if attach_csv
-
- targets.each do |target|
- name = target[0]
- pm_type = "target_#{target[1]}s"
-
- pm = {}
- pm["title"] = I18n.t(
- "data_explorer.report_generator.private_message.title",
- query_name: query.name,
- )
- pm[pm_type] = Array(name)
- pm["raw"] = I18n.t(
- "data_explorer.report_generator.private_message.body",
- recipient_name: name,
- query_name: query.name,
- table: table,
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- )
- if upload
- pm["raw"] << "\n\n" +
+ appendix = upload_appendix(query, result, attach_csv:)
+
+ targets.map do |name, type|
+ {
+ "title" =>
+ I18n.t("data_explorer.report_generator.private_message.title", query_name: query.name),
+ "target_#{type}s" => Array(name),
+ "raw" =>
I18n.t(
- "data_explorer.report_generator.upload_appendix",
- filename: upload.original_filename,
- short_url: upload.short_url,
- )
- end
- pms << pm
+ "data_explorer.report_generator.private_message.body",
+ recipient_name: name,
+ query_name: query.name,
+ table: table,
+ base_url: Discourse.base_url,
+ query_id: query.id,
+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
+ timezone: Time.zone.name,
+ ) + appendix,
+ }
end
- pms
end
def self.build_report_post(query, table = "", attach_csv: false, result: nil)
- upload = create_csv_upload(query, result) if attach_csv
-
- post = {}
- post["raw"] = I18n.t(
- "data_explorer.report_generator.post.body",
- recipient_name: name,
- query_name: query.name,
- table: table,
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- )
-
- if upload
- post["raw"] << "\n\n" +
+ {
+ "raw" =>
I18n.t(
- "data_explorer.report_generator.upload_appendix",
- filename: upload.original_filename,
- short_url: upload.short_url,
- )
- end
+ "data_explorer.report_generator.post.body",
+ query_name: query.name,
+ table: table,
+ base_url: Discourse.base_url,
+ query_id: query.id,
+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
+ timezone: Time.zone.name,
+ ) + upload_appendix(query, result, attach_csv:),
+ }
+ end
+
+ def self.upload_appendix(query, result, attach_csv:)
+ return "" unless attach_csv
+
+ upload = create_csv_upload(query, result)
+ return "" unless upload.persisted?
- post
+ "\n\n" +
+ I18n.t(
+ "data_explorer.report_generator.upload_appendix",
+ filename: upload.original_filename,
+ short_url: upload.short_url,
+ )
end
def self.create_csv_upload(query, result)
@@ -120,18 +110,17 @@ module DiscourseDataExplorer
end
def self.filter_recipients_by_query_access(recipients, query, users_from_group: false)
- users = User.where(username: recipients)
- groups = Group.where(name: recipients)
- emails = recipients - users.pluck(:username) - groups.pluck(:name)
- result = []
+ users = User.where(username: recipients).includes(:groups).to_a
+ groups = Group.where(name: recipients).to_a
+ emails = recipients - users.map(&:username) - groups.map(&:name)
query_group_ids = [Group::AUTO_GROUPS[:admins]].concat(query.groups.pluck(:group_id)).uniq
- if users_from_group
- result.concat(
+ group_targets =
+ if users_from_group
User
.joins(:group_users)
- .where(group_users: { group_id: groups.ids })
+ .where(group_users: { group_id: groups.map(&:id) })
.where(
"users.admin OR EXISTS (
SELECT 1 FROM group_users gu
@@ -142,21 +131,19 @@ module DiscourseDataExplorer
)
.distinct
.pluck(:username)
- .map { |username| [username, "username"] },
- )
- else
- groups.each do |group|
- result << [group.name, "group_name"] if query_group_ids.include?(group.id)
+ .map { |username| [username, "username"] }
+ else
+ groups.filter_map do |group|
+ [group.name, "group_name"] if query_group_ids.include?(group.id)
+ end
end
- end
- users.each do |user|
- result << [user.username, "username"] if Guardian.new(user).user_can_access_query?(query)
- end
-
- emails.each { |email| result << [email, "email"] if Email.is_valid?(email) }
+ user_targets =
+ users.filter_map do |user|
+ [user.username, "username"] if user.guardian.user_can_access_query?(query)
+ end
- result
+ group_targets + user_targets + emails.filter_map { |e| [e, "email"] if Email.is_valid?(e) }
end
end
end
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb
index 44e00179576..93f78ee33b4 100644
--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb
+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb
@@ -1,56 +1,58 @@
# frozen_string_literal: true
-include HasSanitizableFields
-
module DiscourseDataExplorer
class ResultToMarkdown
- def self.convert(pg_result, render_url_columns = false)
- relations, colrender = DataExplorer.add_extra_data(pg_result)
- result_data = []
+ extend HasSanitizableFields
- # column names to search in place of id columns (topic_id, user_id etc)
- cols = %w[name title username]
+ LABEL_COLUMNS = %w[username title name]
+ SANITIZABLE_REGEX = /[<>&%]/
- # find values from extra data, based on result id
- pg_result.values.each do |row|
- row_data = []
+ def self.convert(pg_result, render_url_columns: false)
+ relations, colrender = DataExplorer.add_extra_data(pg_result)
+ relations.transform_values! { |related| related.object.index_by(&:id) }
- row.each_with_index do |col, col_index|
- col_name = pg_result.fields[col_index]
+ column_renders =
+ pg_result.fields.each_index.map do |col_index|
col_render = colrender[col_index]
- related = relations.dig(col_render.to_sym) if col_render.present?
-
- if related.is_a?(ActiveModel::ArraySerializer)
- related_row = related.object.find_by(id: col)
- if col_name.include?("_id")
- column = cols.find { |c| related_row.try c }
- else
- column = related_row.try(col_name)
- end
+ [col_render, relations[col_render&.to_sym]]
+ end
- if column.nil?
- row_data[col_index] = col
- else
- row_data[col_index] = "#{related_row[column]} (#{col})"
+ result_data =
+ pg_result.each_row.map do |row|
+ cells =
+ row.each_with_index.map do |col, col_index|
+ col_render, table = column_renders[col_index]
+ related_row = table[col.to_i] if table && col
+
+ if related_row
+ label = LABEL_COLUMNS.filter_map { |c| related_row.try(c) }.first
+ label ? "#{label} (#{col})" : col
+ elsif col_render == "url" && render_url_columns && col.present?
+ url, text = guess_url(col)
+ "[#{text}](#{url})"
+ else
+ col
+ end
end
- elsif col_render == "url" && render_url_columns
- url, text = guess_url(col)
- row_data[col_index] = "[#{text}](#{url})"
- else
- row_data[col_index] = col
- end
- end
- result_data << row_data.map { |c| "| #{sanitize_field(c.to_s)} " }.join + "|\n"
- end
+ cells.map { |c| "| #{escape_cell(c)} " }.join + "|\n"
+ end
- table_headers = pg_result.fields.map { |c| " #{c.gsub("_id", "")} |" }.join
+ table_headers = pg_result.fields.map { |c| " #{c.delete_suffix("_id")} |" }.join
table_body = pg_result.fields.size.times.map { " :----- |" }.join
"|#{table_headers}\n|#{table_body}\n#{result_data.join}"
end
+ def self.escape_cell(value)
+ value = value.to_s
+ value = sanitize_field(value) if value.match?(SANITIZABLE_REGEX)
+
+ value.gsub("|", "\\|").gsub(/\R+/, " ")
+ end
+
def self.guess_url(column_value)
+ column_value = column_value.to_s
text, url = column_value.split(/,(.+)/)
[url || column_value, text || column_value]
diff --git a/plugins/discourse-data-explorer/spec/report_generator_spec.rb b/plugins/discourse-data-explorer/spec/report_generator_spec.rb
index 3afde8ff43e..860da33154f 100644
--- a/plugins/discourse-data-explorer/spec/report_generator_spec.rb
+++ b/plugins/discourse-data-explorer/spec/report_generator_spec.rb
@@ -15,7 +15,37 @@ describe DiscourseDataExplorer::ReportGenerator do
SiteSetting.authorized_extensions = "csv"
end
+ def pm_body(recipient_name)
+ I18n.t(
+ "data_explorer.report_generator.private_message.body",
+ recipient_name:,
+ query_name: query.name,
+ table: "le table",
+ base_url: Discourse.base_url,
+ query_id: query.id,
+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
+ timezone: Time.zone.name,
+ )
+ end
+
+ def post_body
+ I18n.t(
+ "data_explorer.report_generator.post.body",
+ query_name: query.name,
+ table: "le table",
+ base_url: Discourse.base_url,
+ query_id: query.id,
+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
+ timezone: Time.zone.name,
+ )
+ end
+
describe ".generate" do
+ before do
+ DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
+ freeze_time
+ end
+
it "returns [] if the recipient is not in query group" do
Fabricate(:query_group, query: query, group: group)
result =
@@ -31,8 +61,6 @@ describe DiscourseDataExplorer::ReportGenerator do
it "returns a list of pms for authorised users" do
Fabricate(:query_group, query: query, group: group)
SiteSetting.personal_message_enabled_groups = group.id
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
result = described_class.generate(query.id, query_params, [user.username])
@@ -45,17 +73,7 @@ describe DiscourseDataExplorer::ReportGenerator do
query_name: query.name,
),
"target_usernames" => [user.username],
- "raw" =>
- I18n.t(
- "data_explorer.report_generator.private_message.body",
- recipient_name: user.username,
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ),
+ "raw" => pm_body(user.username),
},
],
)
@@ -63,10 +81,7 @@ describe DiscourseDataExplorer::ReportGenerator do
it "still returns a list of pms if a group or user does not exist" do
Fabricate(:query_group, query: query, group: group)
-
SiteSetting.personal_message_enabled_groups = group.id
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
result = described_class.generate(query.id, query_params, [group.name, "non-existent-group"])
expect(result).to eq(
@@ -78,26 +93,13 @@ describe DiscourseDataExplorer::ReportGenerator do
query_name: query.name,
),
"target_group_names" => [group.name],
- "raw" =>
- I18n.t(
- "data_explorer.report_generator.private_message.body",
- recipient_name: group.name,
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ),
+ "raw" => pm_body(group.name),
},
],
)
end
it "works with email recipients" do
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
-
result = described_class.generate(query.id, query_params, [email])
@@ -110,17 +112,7 @@ describe DiscourseDataExplorer::ReportGenerator do
query_name: query.name,
),
"target_emails" => [email],
- "raw" =>
- I18n.t(
- "data_explorer.report_generator.private_message.body",
- recipient_name: email,
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ),
+ "raw" => pm_body(email),
},
],
)
@@ -128,8 +120,6 @@ describe DiscourseDataExplorer::ReportGenerator do
it "works with duplicate recipients" do
Fabricate(:query_group, query: query, group: group)
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
result = described_class.generate(query.id, query_params, [user.username, user.username])
@@ -142,17 +132,7 @@ describe DiscourseDataExplorer::ReportGenerator do
query_name: query.name,
),
"target_usernames" => [user.username],
- "raw" =>
- I18n.t(
- "data_explorer.report_generator.private_message.body",
- recipient_name: user.username,
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ),
+ "raw" => pm_body(user.username),
},
],
)
@@ -160,7 +140,6 @@ describe DiscourseDataExplorer::ReportGenerator do
it "works with multiple recipient types" do
Fabricate(:query_group, query: query, group: group)
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
result =
described_class.generate(
@@ -235,8 +214,6 @@ describe DiscourseDataExplorer::ReportGenerator do
it "works with attached csv file" do
Fabricate(:query_group, query: query, group: group)
SiteSetting.personal_message_enabled_groups = group.id
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
result =
described_class.generate(query.id, query_params, [user.username], { attach_csv: true })
@@ -245,16 +222,7 @@ describe DiscourseDataExplorer::ReportGenerator do
"#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
expect(result[0]["raw"]).to eq(
- I18n.t(
- "data_explorer.report_generator.private_message.body",
- recipient_name: user.username,
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ) + "\n\n" +
+ pm_body(user.username) + "\n\n" +
I18n.t(
"data_explorer.report_generator.upload_appendix",
filename: filename,
@@ -265,50 +233,49 @@ describe DiscourseDataExplorer::ReportGenerator do
end
describe ".generate_post" do
- it "works without attached csv file" do
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
+ it "raises the query error" do
+ broken_query = Fabricate(:query, sql: "SELECT 1;")
- result = described_class.generate_post(query.id, query_params)
-
- expect(result["raw"]).to eq(
- I18n.t(
- "data_explorer.report_generator.post.body",
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ),
+ expect { described_class.generate_post(broken_query.id, query_params) }.to raise_error(
+ DiscourseDataExplorer::ValidationError,
)
end
- it "works with attached csv file" do
- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
- freeze_time
+ context "with a runnable query" do
+ before do
+ DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")
+ freeze_time
+ end
- result = described_class.generate_post(query.id, query_params, { attach_csv: true })
+ it "works without attached csv file" do
+ result = described_class.generate_post(query.id, query_params)
- filename =
- "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
+ expect(result["raw"]).to eq(post_body)
+ end
- expect(result["raw"]).to eq(
- I18n.t(
- "data_explorer.report_generator.post.body",
- query_name: query.name,
- table: "le table",
- base_url: Discourse.base_url,
- query_id: query.id,
- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),
- timezone: Time.zone.name,
- ) + "\n\n" +
- I18n.t(
- "data_explorer.report_generator.upload_appendix",
- filename: filename,
- short_url: Upload.find_by(original_filename: filename).short_url,
- ),
- )
+ it "works with attached csv file" do
+ result = described_class.generate_post(query.id, query_params, { attach_csv: true })
+
+ filename =
+ "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"
+
+ expect(result["raw"]).to eq(
+ post_body + "\n\n" +
+ I18n.t(
+ "data_explorer.report_generator.upload_appendix",
+ filename: filename,
+ short_url: Upload.find_by(original_filename: filename).short_url,
+ ),
+ )
+ end
+
+ it "omits the upload appendix when the upload is rejected" do
+ SiteSetting.authorized_extensions = "pdf"
+
+ result = described_class.generate_post(query.id, query_params, { attach_csv: true })
+
+ expect(result["raw"]).not_to include("Appendix")
+ end
end
end
diff --git a/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb b/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb
index 3e581404459..0ac6f2ac862 100644
--- a/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb
+++ b/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb
@@ -10,6 +10,11 @@ describe DiscourseDataExplorer::ResultToMarkdown do
before { SiteSetting.data_explorer_enabled = true }
+ def run(sql)
+ query = DiscourseDataExplorer::Query.new(sql:)
+ DiscourseDataExplorer::DataExplorer.run_query(query, {})[:pg_result]
+ end
+
describe ".convert" do
it "format results as a markdown table with headers and columns" do
result = described_class.convert(query_result[:pg_result])
@@ -30,5 +35,48 @@ describe DiscourseDataExplorer::ResultToMarkdown do
"| #{user.username} (#{user.id}) | #{post.user.username} (#{post.user.id}) | 1 |\n",
)
end
+
+ it "renders url columns as links only when render_url_columns is set" do
+ pg_result =
+ run(
+ "SELECT '3,https://test.com' AS some_url, NULL AS null_url, 3 AS int_url, true AS bool_url",
+ )
+
+ expect(described_class.convert(pg_result, render_url_columns: false)).to include(
+ "| 3,https://test.com | | 3 | true |\n",
+ )
+ expect(described_class.convert(pg_result, render_url_columns: true)).to include(
+ "| [3](https://test.com) | | [3](3) | [true](true) |\n",
+ )
+ end
+
+ it "labels post columns with the author's username and strips only the _id suffix" do
+ pg_result = run("SELECT #{post.id} AS post_id, 1 AS user_id_count")
+
+ result = described_class.convert(pg_result)
+
+ expect(result).to include("| post | user_id_count |\n")
+ expect(result).to include("| #{post.user.username} (#{post.id}) | 1 |\n")
+ end
+
+ it "loads each relation once" do
+ pg_result = run("SELECT id AS user_id FROM users")
+
+ queries = track_sql_queries { described_class.convert(pg_result) }
+
+ expect(queries.count { |sql| sql.include?('FROM "users"') }).to eq(1)
+ end
+
+ it "escapes pipes and collapses newlines in cells" do
+ pg_result = run("SELECT E'a|b\\nc' AS data")
+
+ expect(described_class.convert(pg_result)).to include("| a\\|b c |\n")
+ end
+
+ it "sanitizes cells that contain html" do
+ pg_result = run("SELECT '<script>x</script>a & b %7Bc%7D' AS data")
+
+ expect(described_class.convert(pg_result)).to include("| xa & b {c} |\n")
+ end
end
end