A platform for community discussion. Free, open, simple.
FIX: Generate Data Explorer reports without crashing on ordinary results
Régis Hanol2179c9c25655
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rbindex a2fde64b1f0..e1c7b3f3782 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/report_generator.rb@@ -4,7 +4,7 @@ module DiscourseDataExplorerclass ReportGeneratordef self.generate(query_id, query_params, recipients, opts = {})query = DiscourseDataExplorer::Query.find(query_id)- return [] if !query || recipients.empty?+ return [] if recipients.empty?recipients =filter_recipients_by_query_access(@@ -13,11 +13,9 @@ module DiscourseDataExplorerusers_from_group: opts[:users_from_group],)params = params_to_hash(query_params)+ result = run_query(query, params)- result = DataExplorer.run_query(query, params)- query.record_run!-- return [] if opts[:skip_empty] && result[:pg_result].values.empty?+ return [] if opts[:skip_empty] && result[:pg_result].ntuples.zero?table =ResultToMarkdown.convert(result[:pg_result], render_url_columns: opts[:render_url_columns])@@ -26,88 +24,80 @@ module DiscourseDataExplorerdef self.generate_post(query_id, query_params, opts = {})query = DiscourseDataExplorer::Query.find(query_id)- return {} if !query-params = params_to_hash(query_params)+ result = run_query(query, params)- result = DataExplorer.run_query(query, params)- query.record_run!-- return {} if opts[:skip_empty] && result[:pg_result].values.empty?+ return {} if opts[:skip_empty] && result[:pg_result].ntuples.zero?table =ResultToMarkdown.convert(result[:pg_result], render_url_columns: opts[:render_url_columns])build_report_post(query, table, attach_csv: opts[:attach_csv], result:)end+ def self.run_query(query, params)+ result = DataExplorer.run_query(query, params)+ query.record_run!+ raise result[:error] if result[:error]++ result+ end+def self.params_to_hash(query_params)params = JSON.parse(query_params)- params.map { |p| p.is_a?(Hash) ? [p["key"], p["value"]] : p }.to_h+ params.to_h { |p| p.is_a?(Hash) ? [p["key"], p["value"]] : p }enddef self.build_report_pms(query, table = "", targets = [], attach_csv: false, result: nil)- pms = []- upload = create_csv_upload(query, result) if attach_csv-- targets.each do |target|- name = target[0]- pm_type = "target_#{target[1]}s"-- pm = {}- pm["title"] = I18n.t(- "data_explorer.report_generator.private_message.title",- query_name: query.name,- )- pm[pm_type] = Array(name)- pm["raw"] = I18n.t(- "data_explorer.report_generator.private_message.body",- recipient_name: name,- query_name: query.name,- table: table,- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- )- if upload- pm["raw"] << "\n\n" ++ appendix = upload_appendix(query, result, attach_csv:)++ targets.map do |name, type|+ {+ "title" =>+ I18n.t("data_explorer.report_generator.private_message.title", query_name: query.name),+ "target_#{type}s" => Array(name),+ "raw" =>I18n.t(- "data_explorer.report_generator.upload_appendix",- filename: upload.original_filename,- short_url: upload.short_url,- )- end- pms << pm+ "data_explorer.report_generator.private_message.body",+ recipient_name: name,+ query_name: query.name,+ table: table,+ base_url: Discourse.base_url,+ query_id: query.id,+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),+ timezone: Time.zone.name,+ ) + appendix,+ }end- pmsenddef self.build_report_post(query, table = "", attach_csv: false, result: nil)- upload = create_csv_upload(query, result) if attach_csv-- post = {}- post["raw"] = I18n.t(- "data_explorer.report_generator.post.body",- recipient_name: name,- query_name: query.name,- table: table,- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- )-- if upload- post["raw"] << "\n\n" ++ {+ "raw" =>I18n.t(- "data_explorer.report_generator.upload_appendix",- filename: upload.original_filename,- short_url: upload.short_url,- )- end+ "data_explorer.report_generator.post.body",+ query_name: query.name,+ table: table,+ base_url: Discourse.base_url,+ query_id: query.id,+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),+ timezone: Time.zone.name,+ ) + upload_appendix(query, result, attach_csv:),+ }+ end++ def self.upload_appendix(query, result, attach_csv:)+ return "" unless attach_csv++ upload = create_csv_upload(query, result)+ return "" unless upload.persisted?- post+ "\n\n" ++ I18n.t(+ "data_explorer.report_generator.upload_appendix",+ filename: upload.original_filename,+ short_url: upload.short_url,+ )enddef self.create_csv_upload(query, result)@@ -120,18 +110,17 @@ module DiscourseDataExplorerenddef self.filter_recipients_by_query_access(recipients, query, users_from_group: false)- users = User.where(username: recipients)- groups = Group.where(name: recipients)- emails = recipients - users.pluck(:username) - groups.pluck(:name)- result = []+ users = User.where(username: recipients).includes(:groups).to_a+ groups = Group.where(name: recipients).to_a+ emails = recipients - users.map(&:username) - groups.map(&:name)query_group_ids = [Group::AUTO_GROUPS[:admins]].concat(query.groups.pluck(:group_id)).uniq- if users_from_group- result.concat(+ group_targets =+ if users_from_groupUser.joins(:group_users)- .where(group_users: { group_id: groups.ids })+ .where(group_users: { group_id: groups.map(&:id) }).where("users.admin OR EXISTS (SELECT 1 FROM group_users gu@@ -142,21 +131,19 @@ module DiscourseDataExplorer).distinct.pluck(:username)- .map { |username| [username, "username"] },- )- else- groups.each do |group|- result << [group.name, "group_name"] if query_group_ids.include?(group.id)+ .map { |username| [username, "username"] }+ else+ groups.filter_map do |group|+ [group.name, "group_name"] if query_group_ids.include?(group.id)+ endend- end- users.each do |user|- result << [user.username, "username"] if Guardian.new(user).user_can_access_query?(query)- end-- emails.each { |email| result << [email, "email"] if Email.is_valid?(email) }+ user_targets =+ users.filter_map do |user|+ [user.username, "username"] if user.guardian.user_can_access_query?(query)+ end- result+ group_targets + user_targets + emails.filter_map { |e| [e, "email"] if Email.is_valid?(e) }endendenddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rbindex 44e00179576..93f78ee33b4 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/result_to_markdown.rb@@ -1,56 +1,58 @@# frozen_string_literal: true-include HasSanitizableFields-module DiscourseDataExplorerclass ResultToMarkdown- def self.convert(pg_result, render_url_columns = false)- relations, colrender = DataExplorer.add_extra_data(pg_result)- result_data = []+ extend HasSanitizableFields- # column names to search in place of id columns (topic_id, user_id etc)- cols = %w[name title username]+ LABEL_COLUMNS = %w[username title name]+ SANITIZABLE_REGEX = /[<>&%]/- # find values from extra data, based on result id- pg_result.values.each do |row|- row_data = []+ def self.convert(pg_result, render_url_columns: false)+ relations, colrender = DataExplorer.add_extra_data(pg_result)+ relations.transform_values! { |related| related.object.index_by(&:id) }- row.each_with_index do |col, col_index|- col_name = pg_result.fields[col_index]+ column_renders =+ pg_result.fields.each_index.map do |col_index|col_render = colrender[col_index]- related = relations.dig(col_render.to_sym) if col_render.present?-- if related.is_a?(ActiveModel::ArraySerializer)- related_row = related.object.find_by(id: col)- if col_name.include?("_id")- column = cols.find { |c| related_row.try c }- else- column = related_row.try(col_name)- end+ [col_render, relations[col_render&.to_sym]]+ end- if column.nil?- row_data[col_index] = col- else- row_data[col_index] = "#{related_row[column]} (#{col})"+ result_data =+ pg_result.each_row.map do |row|+ cells =+ row.each_with_index.map do |col, col_index|+ col_render, table = column_renders[col_index]+ related_row = table[col.to_i] if table && col++ if related_row+ label = LABEL_COLUMNS.filter_map { |c| related_row.try(c) }.first+ label ? "#{label} (#{col})" : col+ elsif col_render == "url" && render_url_columns && col.present?+ url, text = guess_url(col)+ "[#{text}](#{url})"+ else+ col+ endend- elsif col_render == "url" && render_url_columns- url, text = guess_url(col)- row_data[col_index] = "[#{text}](#{url})"- else- row_data[col_index] = col- end- end- result_data << row_data.map { |c| "| #{sanitize_field(c.to_s)} " }.join + "|\n"- end+ cells.map { |c| "| #{escape_cell(c)} " }.join + "|\n"+ end- table_headers = pg_result.fields.map { |c| " #{c.gsub("_id", "")} |" }.join+ table_headers = pg_result.fields.map { |c| " #{c.delete_suffix("_id")} |" }.jointable_body = pg_result.fields.size.times.map { " :----- |" }.join"|#{table_headers}\n|#{table_body}\n#{result_data.join}"end+ def self.escape_cell(value)+ value = value.to_s+ value = sanitize_field(value) if value.match?(SANITIZABLE_REGEX)++ value.gsub("|", "\\|").gsub(/\R+/, " ")+ end+def self.guess_url(column_value)+ column_value = column_value.to_stext, url = column_value.split(/,(.+)/)[url || column_value, text || column_value]diff --git a/plugins/discourse-data-explorer/spec/report_generator_spec.rb b/plugins/discourse-data-explorer/spec/report_generator_spec.rbindex 3afde8ff43e..860da33154f 100644--- a/plugins/discourse-data-explorer/spec/report_generator_spec.rb+++ b/plugins/discourse-data-explorer/spec/report_generator_spec.rb@@ -15,7 +15,37 @@ describe DiscourseDataExplorer::ReportGenerator doSiteSetting.authorized_extensions = "csv"end+ def pm_body(recipient_name)+ I18n.t(+ "data_explorer.report_generator.private_message.body",+ recipient_name:,+ query_name: query.name,+ table: "le table",+ base_url: Discourse.base_url,+ query_id: query.id,+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),+ timezone: Time.zone.name,+ )+ end++ def post_body+ I18n.t(+ "data_explorer.report_generator.post.body",+ query_name: query.name,+ table: "le table",+ base_url: Discourse.base_url,+ query_id: query.id,+ created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),+ timezone: Time.zone.name,+ )+ end+describe ".generate" do+ before do+ DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")+ freeze_time+ end+it "returns [] if the recipient is not in query group" doFabricate(:query_group, query: query, group: group)result =@@ -31,8 +61,6 @@ describe DiscourseDataExplorer::ReportGenerator doit "returns a list of pms for authorised users" doFabricate(:query_group, query: query, group: group)SiteSetting.personal_message_enabled_groups = group.id- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_timeresult = described_class.generate(query.id, query_params, [user.username])@@ -45,17 +73,7 @@ describe DiscourseDataExplorer::ReportGenerator doquery_name: query.name,),"target_usernames" => [user.username],- "raw" =>- I18n.t(- "data_explorer.report_generator.private_message.body",- recipient_name: user.username,- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ),+ "raw" => pm_body(user.username),},],)@@ -63,10 +81,7 @@ describe DiscourseDataExplorer::ReportGenerator doit "still returns a list of pms if a group or user does not exist" doFabricate(:query_group, query: query, group: group)-SiteSetting.personal_message_enabled_groups = group.id- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_timeresult = described_class.generate(query.id, query_params, [group.name, "non-existent-group"])expect(result).to eq(@@ -78,26 +93,13 @@ describe DiscourseDataExplorer::ReportGenerator doquery_name: query.name,),"target_group_names" => [group.name],- "raw" =>- I18n.t(- "data_explorer.report_generator.private_message.body",- recipient_name: group.name,- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ),+ "raw" => pm_body(group.name),},],)endit "works with email recipients" do- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_time-email = "[email protected]"result = described_class.generate(query.id, query_params, [email])@@ -110,17 +112,7 @@ describe DiscourseDataExplorer::ReportGenerator doquery_name: query.name,),"target_emails" => [email],- "raw" =>- I18n.t(- "data_explorer.report_generator.private_message.body",- recipient_name: email,- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ),+ "raw" => pm_body(email),},],)@@ -128,8 +120,6 @@ describe DiscourseDataExplorer::ReportGenerator doit "works with duplicate recipients" doFabricate(:query_group, query: query, group: group)- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_timeresult = described_class.generate(query.id, query_params, [user.username, user.username])@@ -142,17 +132,7 @@ describe DiscourseDataExplorer::ReportGenerator doquery_name: query.name,),"target_usernames" => [user.username],- "raw" =>- I18n.t(- "data_explorer.report_generator.private_message.body",- recipient_name: user.username,- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ),+ "raw" => pm_body(user.username),},],)@@ -160,7 +140,6 @@ describe DiscourseDataExplorer::ReportGenerator doit "works with multiple recipient types" doFabricate(:query_group, query: query, group: group)- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")result =described_class.generate(@@ -235,8 +214,6 @@ describe DiscourseDataExplorer::ReportGenerator doit "works with attached csv file" doFabricate(:query_group, query: query, group: group)SiteSetting.personal_message_enabled_groups = group.id- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_timeresult =described_class.generate(query.id, query_params, [user.username], { attach_csv: true })@@ -245,16 +222,7 @@ describe DiscourseDataExplorer::ReportGenerator do"#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"expect(result[0]["raw"]).to eq(- I18n.t(- "data_explorer.report_generator.private_message.body",- recipient_name: user.username,- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ) + "\n\n" ++ pm_body(user.username) + "\n\n" +I18n.t("data_explorer.report_generator.upload_appendix",filename: filename,@@ -265,50 +233,49 @@ describe DiscourseDataExplorer::ReportGenerator doenddescribe ".generate_post" do- it "works without attached csv file" do- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_time+ it "raises the query error" do+ broken_query = Fabricate(:query, sql: "SELECT 1;")- result = described_class.generate_post(query.id, query_params)-- expect(result["raw"]).to eq(- I18n.t(- "data_explorer.report_generator.post.body",- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ),+ expect { described_class.generate_post(broken_query.id, query_params) }.to raise_error(+ DiscourseDataExplorer::ValidationError,)end- it "works with attached csv file" do- DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")- freeze_time+ context "with a runnable query" do+ before do+ DiscourseDataExplorer::ResultToMarkdown.expects(:convert).returns("le table")+ freeze_time+ end- result = described_class.generate_post(query.id, query_params, { attach_csv: true })+ it "works without attached csv file" do+ result = described_class.generate_post(query.id, query_params)- filename =- "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"+ expect(result["raw"]).to eq(post_body)+ end- expect(result["raw"]).to eq(- I18n.t(- "data_explorer.report_generator.post.body",- query_name: query.name,- table: "le table",- base_url: Discourse.base_url,- query_id: query.id,- created_at: Time.zone.now.strftime("%Y-%m-%d at %H:%M:%S"),- timezone: Time.zone.name,- ) + "\n\n" +- I18n.t(- "data_explorer.report_generator.upload_appendix",- filename: filename,- short_url: Upload.find_by(original_filename: filename).short_url,- ),- )+ it "works with attached csv file" do+ result = described_class.generate_post(query.id, query_params, { attach_csv: true })++ filename =+ "#{query.slug}@#{Slug.for(Discourse.current_hostname, "discourse")}-#{Date.today}.dcqresult.csv"++ expect(result["raw"]).to eq(+ post_body + "\n\n" ++ I18n.t(+ "data_explorer.report_generator.upload_appendix",+ filename: filename,+ short_url: Upload.find_by(original_filename: filename).short_url,+ ),+ )+ end++ it "omits the upload appendix when the upload is rejected" do+ SiteSetting.authorized_extensions = "pdf"++ result = described_class.generate_post(query.id, query_params, { attach_csv: true })++ expect(result["raw"]).not_to include("Appendix")+ endendenddiff --git a/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb b/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rbindex 3e581404459..0ac6f2ac862 100644--- a/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb+++ b/plugins/discourse-data-explorer/spec/result_to_markdown_spec.rb@@ -10,6 +10,11 @@ describe DiscourseDataExplorer::ResultToMarkdown dobefore { SiteSetting.data_explorer_enabled = true }+ def run(sql)+ query = DiscourseDataExplorer::Query.new(sql:)+ DiscourseDataExplorer::DataExplorer.run_query(query, {})[:pg_result]+ end+describe ".convert" doit "format results as a markdown table with headers and columns" doresult = described_class.convert(query_result[:pg_result])@@ -30,5 +35,48 @@ describe DiscourseDataExplorer::ResultToMarkdown do"| #{user.username} (#{user.id}) | #{post.user.username} (#{post.user.id}) | 1 |\n",)end++ it "renders url columns as links only when render_url_columns is set" do+ pg_result =+ run(+ "SELECT '3,https://test.com' AS some_url, NULL AS null_url, 3 AS int_url, true AS bool_url",+ )++ expect(described_class.convert(pg_result, render_url_columns: false)).to include(+ "| 3,https://test.com | | 3 | true |\n",+ )+ expect(described_class.convert(pg_result, render_url_columns: true)).to include(+ "| [3](https://test.com) | | [3](3) | [true](true) |\n",+ )+ end++ it "labels post columns with the author's username and strips only the _id suffix" do+ pg_result = run("SELECT #{post.id} AS post_id, 1 AS user_id_count")++ result = described_class.convert(pg_result)++ expect(result).to include("| post | user_id_count |\n")+ expect(result).to include("| #{post.user.username} (#{post.id}) | 1 |\n")+ end++ it "loads each relation once" do+ pg_result = run("SELECT id AS user_id FROM users")++ queries = track_sql_queries { described_class.convert(pg_result) }++ expect(queries.count { |sql| sql.include?('FROM "users"') }).to eq(1)+ end++ it "escapes pipes and collapses newlines in cells" do+ pg_result = run("SELECT E'a|b\\nc' AS data")++ expect(described_class.convert(pg_result)).to include("| a\\|b c |\n")+ end++ it "sanitizes cells that contain html" do+ pg_result = run("SELECT '<script>x</script>a & b %7Bc%7D' AS data")++ expect(described_class.convert(pg_result)).to include("| xa & b {c} |\n")+ endendend