A platform for community discussion. Free, open, simple.
FIX: Resolve Data Explorer relations from the right table
Régis Hanol3d8a848a8715
diff --git a/plugins/discourse-data-explorer/app/serializers/discourse_data_explorer/small_badge_serializer.rb b/plugins/discourse-data-explorer/app/serializers/discourse_data_explorer/small_badge_serializer.rbindex 27679b0aceb..496434466e2 100644--- a/plugins/discourse-data-explorer/app/serializers/discourse_data_explorer/small_badge_serializer.rb+++ b/plugins/discourse-data-explorer/app/serializers/discourse_data_explorer/small_badge_serializer.rb@@ -1,5 +1,6 @@# frozen_string_literal: trueclass DiscourseDataExplorer::SmallBadgeSerializer < ApplicationSerializer- attributes :id, :name, :display_name, :badge_type, :description, :icon+ attributes :id, :name, :display_name, :description, :icon, :image_url+ has_one :badge_type, serializer: BadgeTypeSerializer, embed: :objectsenddiff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/data_explorer.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/data_explorer.rbindex 8dc0a7bd154..1832a039ed0 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/data_explorer.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/data_explorer.rb@@ -23,6 +23,17 @@ module DiscourseDataExplorerPG_TYPE_OID_DATE = 1082PG_TYPE_OID_TIMESTAMP = 1114+ JOIN_TABLE_ID_COLUMNS = %w[+ user_badge_id+ group_user_id+ topic_user_id+ category_user_id+ topic_allowed_user_id+ topic_allowed_group_id+ associated_group_id+ watched_word_group_id+ ]+# Run a data explorer query on the currently connected database.## @param [Query] query the Query object to run@@ -281,8 +292,8 @@ module DiscourseDataExplorer},badge: {class: Badge,- fields: %i[id name badge_type_id description icon],- include: [:badge_type],+ fields: %i[id name badge_type_id description icon image_upload_id],+ include: %i[badge_type image_upload],serializer: SmallBadgeSerializer,},post: {@@ -333,9 +344,17 @@ module DiscourseDataExplorerdef self.column_regexes@column_regexes ||=- extra_data_pluck_fields- .map { |key, val| /(#{val[:class].to_s.underscore})_id$/ if val[:class] }- .compact+ extra_data_pluck_fields.filter_map do |key, val|+ [/#{val[:class].to_s.underscore}_id$/, key] if val[:class]+ end + [[/_(by|editor)_id$/, :user]]+ end++ def self.relation_for(col)+ prefix = col[/^(\w+)\$/, 1]&.to_sym+ return prefix if extra_data_pluck_fields.key?(prefix)+ return if JOIN_TABLE_ID_COLUMNS.include?(col)++ column_regexes.find { |rgx, _| rgx.match?(col) }&.lastenddef self.add_extra_data(pg_result, guardian: nil)@@ -343,15 +362,8 @@ module DiscourseDataExplorerret = {}col_map = {}pg_result.fields.each_with_index do |col, idx|- rgx = column_regexes.find { |r| r.match col }- if rgx- cls = (rgx.match col)[1].to_sym- needed_classes[cls] ||= []- needed_classes[cls] << idx- elsif col =~ /^(\w+)\$/- cls = $1.to_sym- needed_classes[cls] ||= []- needed_classes[cls] << idx+ if (cls = relation_for(col))+ (needed_classes[cls] ||= []) << idxelsif col =~ /^\w+_url$/col_map[idx] = "url"elsif col =~ /^\w+_payload$/ || col == "payload" ||@@ -361,16 +373,11 @@ module DiscourseDataExplorerendneeded_classes.each do |cls, column_nums|- next if column_nums.blank?support_info = extra_data_pluck_fields[cls]next unless support_infocolumn_nums.each { |col_n| col_map[col_n] = cls }-- if support_info[:ignore]- ret[cls] = []- next- end+ next if support_info[:ignore]ids = Set.newcolumn_nums.each { |col_n| ids.merge(pg_result.column_values(col_n)) }@@ -411,41 +418,42 @@ module DiscourseDataExplorerdef self.sensitive_column_names%w[- #_IP_Addressestopic_views.ip_addressusers.ip_addressusers.registration_ip_address+ user_ip_address_histories.ip_address+ user_auth_tokens.client_ip+ user_auth_token_logs.client_ipincoming_links.ip_addresstopic_link_clicks.ip_addressuser_histories.ip_address- #_Emailsemail_tokens.email- users.email+ user_emails.email+ user_emails.normalized_emailinvites.emailuser_histories.emailemail_logs.to_addressposts.raw_emailbadge_posts.raw_email- #_Secret_Tokens- email_tokens.token- email_logs.reply_key- api_keys.key+ email_tokens.token_hash+ post_reply_keys.reply_key+ api_keys.key_hash+ user_api_keys.key_hashsite_settings.value- users.auth_token- users.password_hash- users.salt- #_Authentication_Info+ user_auth_tokens.auth_token+ user_auth_tokens.prev_auth_token+ user_auth_token_logs.auth_token+ user_passwords.password_hash+ user_passwords.password_saltuser_open_ids.emailoauth2_user_infos.uidoauth2_user_infos.email- facebook_user_infos.facebook_user_id- facebook_user_infos.email- twitter_user_infos.twitter_user_id- github_user_infos.github_user_id+ user_associated_accounts.provider_uid+ user_associated_accounts.info+ user_associated_accounts.credentials+ user_associated_accounts.extrasingle_sign_on_records.external_emailsingle_sign_on_records.external_id- google_user_infos.google_user_id- google_user_infos.email]end@@ -638,7 +646,6 @@ module DiscourseDataExplorer"topics.featured_user2_id": :users,"topics.featured_user3_id": :users,"topics.featured_user4_id": :users,- "topics.featured_user5_id": :users,"users.seen_notification_id": :notifications,"users.uploaded_avatar_id": :uploads,"users.primary_group_id": :groups,@@ -648,20 +655,18 @@ module DiscourseDataExplorer"badges.badge_grouping_id": :badge_groupings,"post_actions.related_post_id": :posts,"color_scheme_colors.color_scheme_id": :color_schemes,- "color_schemes.versioned_id": :color_schemes,"incoming_links.incoming_referer_id": :incoming_referers,"incoming_referers.incoming_domain_id": :incoming_domains,- "post_replies.reply_id": :posts,+ "post_replies.reply_post_id": :posts,"quoted_posts.quoted_post_id": :posts,"topic_link_clicks.topic_link_id": :topic_links,- "topic_link_clicks.link_topic_id": :topics,- "topic_link_clicks.link_post_id": :posts,+ "topic_links.link_topic_id": :topics,+ "topic_links.link_post_id": :posts,"user_actions.target_topic_id": :topics,"user_actions.target_post_id": :posts,"user_avatars.custom_upload_id": :uploads,"user_avatars.gravatar_upload_id": :uploads,"user_badges.notification_id": :notifications,- "user_profiles.card_image_badge_id": :badges,}.with_indifferent_accessenddiff --git a/plugins/discourse-data-explorer/spec/data_explorer_spec.rb b/plugins/discourse-data-explorer/spec/data_explorer_spec.rbindex 6ea489e9868..edf120ea693 100644--- a/plugins/discourse-data-explorer/spec/data_explorer_spec.rb+++ b/plugins/discourse-data-explorer/spec/data_explorer_spec.rb@@ -343,7 +343,8 @@ describe DiscourseDataExplorer::DataExplorer doSQLquery = DiscourseDataExplorer::Query.create!(name: "some query", sql: sql)result = described_class.run_query(query)- _, colrender = DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result])+ _, colrender =+ DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result], guardian: nil)expect(colrender).to eq({ 1 => "json" })end@@ -361,7 +362,8 @@ describe DiscourseDataExplorer::DataExplorer doSQLquery = DiscourseDataExplorer::Query.create!(name: "some query", sql: sql)result = described_class.run_query(query)- _, colrender = DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result])+ _, colrender =+ DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result], guardian: nil)expect(colrender).to eq({ 1 => "json" })end@@ -373,7 +375,8 @@ describe DiscourseDataExplorer::DataExplorer do)result = described_class.run_query(query)- _, colrender = DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result])+ _, colrender =+ DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result], guardian: nil)expect(colrender).to eq({ 0 => "json" })end@@ -386,7 +389,8 @@ describe DiscourseDataExplorer::DataExplorer doSQLresult = described_class.run_query(query)- _, colrender = DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result])+ _, colrender =+ DiscourseDataExplorer::DataExplorer.add_extra_data(result[:pg_result], guardian: nil)expect(colrender).to eq({ 0 => "json", 1 => "json" })end@@ -401,18 +405,69 @@ describe DiscourseDataExplorer::DataExplorer doSQLpg_result = described_class.run_query(query)[:pg_result]- relations, _ = DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result)+ relations, _ = DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result, guardian: nil)expect(relations[:topic].as_json.size).to eq(2)end+ it "classifies id columns" do+ query = Fabricate(:query, sql: <<~SQL)+ SELECT+ 1 AS "html$post_id",+ 1 AS user_badge_id,+ 1 AS group_user_id,+ 1 AS topic_user_id,+ 1 AS category_user_id,+ 1 AS topic_allowed_user_id,+ 1 AS topic_allowed_group_id,+ 1 AS associated_group_id,+ 1 AS watched_word_group_id,+ 1 AS badge_id,+ 1 AS card_image_badge_id,+ 1 AS acting_user_id,+ 1 AS topic_id,+ 1 AS tag_group_id,+ 1 AS quoted_post_id,+ 1 AS deleted_by_id,+ 1 AS last_editor_id,+ 1 AS group_id,+ 1 AS category_id,+ 1 AS "author$user_id"+ SQL++ relations, colrender =+ described_class.add_extra_data(+ described_class.run_query(query)[:pg_result],+ guardian: nil,+ )++ expect(colrender).to eq(+ {+ 0 => :html,+ 9 => :badge,+ 10 => :badge,+ 11 => :user,+ 12 => :topic,+ 13 => :tag_group,+ 14 => :post,+ 15 => :user,+ 16 => :user,+ 17 => :group,+ 18 => :category,+ 19 => :user,+ },+ )+ expect(relations.keys).not_to include(:group, :category)+ end+describe "serializing models to serializer" doit "serializes correctly to BasicTopicSerializer for topic relations" dotopic = Fabricate(:topic, locale: "ja")query = Fabricate(:query, sql: "SELECT id AS topic_id FROM topics WHERE id = #{topic.id}")pg_result = described_class.run_query(query)[:pg_result]- relations, _ = DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result)+ relations, _ =+ DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result, guardian: nil)expect {records = relations[:topic].object@@ -423,6 +478,22 @@ describe DiscourseDataExplorer::DataExplorer doexpect(json).to include(BasicTopicSerializer.new(topic, root: false).as_json)end+ it "serializes badge relations with their type and image" do+ badge = Fabricate(:badge, image_upload: Fabricate(:upload))+ query = Fabricate(:query, sql: "SELECT #{badge.id} AS badge_id")++ pg_result = described_class.run_query(query)[:pg_result]+ relations, _ =+ DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result, guardian: nil)++ expect(MultiJson.load(relations[:badge].to_json)).to include(+ include(+ "image_url" => badge.image_url,+ "badge_type" => include("id" => badge.badge_type_id, "name" => badge.badge_type.name),+ ),+ )+ end+it "chooses the correct serializer for tag_group" dotag_group = Fabricate(:tag_group)tag1 = Fabricate(:tag)@@ -432,7 +503,8 @@ describe DiscourseDataExplorer::DataExplorer doquery = Fabricate(:query, sql: "SELECT tag_id, tag_group_id FROM tag_group_memberships")pg_result = described_class.run_query(query)[:pg_result]- relations, colrender = DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result)+ relations, colrender =+ DiscourseDataExplorer::DataExplorer.add_extra_data(pg_result, guardian: nil)expect(colrender).to eq({ 1 => :tag_group })expect(relations[:tag_group].as_json).to include(@@ -442,4 +514,17 @@ describe DiscourseDataExplorer::DataExplorer doendendend++ describe "schema hints" do+ it "only names existing columns" do+ schema_columns = DB.query_single(<<~SQL)+ SELECT table_name || '.' || column_name+ FROM INFORMATION_SCHEMA.COLUMNS+ WHERE table_schema = 'public'+ SQL++ expect(described_class.foreign_keys.keys - schema_columns).to be_empty+ expect(described_class.sensitive_column_names - schema_columns).to be_empty+ end+ endend