A platform for community discussion. Free, open, simple.
DEV: Fix data explorer prompt and seeded queries (#43438)
Natalie Tayb7c3f7c52f1b
diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rbindex 757dda19650..41e37d42119 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/ai_query_generator.rb@@ -55,12 +55,8 @@ module DiscourseDataExplorer## Data Explorer SQL rules- NEVER end SQL with a semicolon (;)- Format SQL with line breaks, indentation, and spaces around operators- - Params are substituted as TEXT strings. You MUST wrap every param in CAST() when using it in date/time expressions or comparisons:- WRONG: WHERE created_at >= :start_date- WRONG: WHERE created_at < :end_date + INTERVAL '1 day'- RIGHT: WHERE created_at >= CAST(:start_date AS date)- RIGHT: WHERE created_at < CAST(:end_date AS date) + INTERVAL '1 day'- - NEVER use :: to cast params (:: conflicts with :param syntax)+ - Params are PostgreSQL bind values, not SQL text substitutions. Declare their types in the `-- [params]` block. Date and datetime params support direct comparisons and date arithmetic. Use CAST() when an expression needs an explicit type; PostgreSQL's :: syntax is also supported.+ - Keep params outside quoted strings, quoted identifiers, and dollar-quoted literals. They represent values, not table names, column names, or SQL fragments. Use `title ILIKE '%' || :search || '%'`, not `title ILIKE '%:search%'`. Use `:months_ago * INTERVAL '1 month'`, not `INTERVAL ':months_ago months'`. Apply these rules even when an existing query example uses the old syntax.## Column renderingColumns named user_id, group_id, topic_id, post_id, badge_id, category_id render as clickable links.@@ -94,7 +90,7 @@ module DiscourseDataExplorer- Optional params: prefix the type with "null" and OMIT the default. The "null" prefix is the optional marker; do NOT use `= #null` as a default value (e.g. write `-- null category_id :category`, NOT `-- category_id :category = #null`).- Date param defaults MUST be real ISO dates in YYYY-MM-DD form (today is #{Date.today.strftime("%Y-%m-%d")}). NEVER use natural-language defaults like "today", "yesterday", "3 months ago", or "14 jul 2015". For a date range, name params `:start_date` and `:end_date`.- Plural nouns → list-style param TYPES. Apply this rule independently to EACH plural noun in the prompt; don't skip one. Only `int_list`, `string_list`, `user_list`, `group_list` accept multiple values — single-value types like `category_id`, `user_id`, `topic_id` do NOT. Plural "categories" MUST use `int_list :category_ids`, not `category_id :category`. Example: "Get topics for selected categories and tags." → `-- null int_list :category_ids` AND `-- null string_list :tag_names` (BOTH list types).- - For list-style params, use `column IN (:param)`. Do NOT use `ANY(:param)`; Data Explorer substitutes list params as comma-separated values, so `ANY(:param)` becomes invalid SQL. With single-value types (`category_id`, `user_id`, etc.), use `column = :param` instead.+ - For list-style params, use `column IN (:param)`. Data Explorer expands each list into separate bind values, not a PostgreSQL array. Do NOT use `ANY(:param)` or cast the list param to an array such as `text[]`. With single-value types (`category_id`, `user_id`, etc.), use `column = :param` instead.- For optional list-style params, wrap only the param in parentheses for the null check: `((:param) IS NULL OR column IN (:param))`. The first `)` must come immediately after the param name. Examples: `((:category_ids) IS NULL OR t.category_id IN (:category_ids))` and `((:tag_names) IS NULL OR tags.name IN (:tag_names))`. Do NOT write `(:param IS NULL OR column IN (:param))` because the list expands to multiple SQL values.- First-person prompts ("my posts", "queries about me", "topics I've replied to") MUST use `current_user_id` — this auto-injects the requester's user id at run time. Do NOT use `user_id :user = system` for "my" queries.diff --git a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rbindex c62705849b4..0a11a5884ca 100644--- a/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb+++ b/plugins/discourse-data-explorer/lib/discourse_data_explorer/queries.rb@@ -315,8 +315,8 @@ module DiscourseDataExplorerWITH query_period as (SELECT- date_trunc('month', CURRENT_DATE) - INTERVAL ':months_ago months' as period_start,- date_trunc('month', CURRENT_DATE) - INTERVAL ':months_ago months' + INTERVAL '1 month' - INTERVAL '1 second' as period_end+ date_trunc('month', CURRENT_DATE) - :months_ago * INTERVAL '1 month' as period_start,+ date_trunc('month', CURRENT_DATE) - :months_ago * INTERVAL '1 month' + INTERVAL '1 month' - INTERVAL '1 second' as period_end)SELECT@@ -336,8 +336,8 @@ module DiscourseDataExplorer-- int :months_ago = 1WITH query_period AS- (SELECT date_trunc('month', CURRENT_DATE) - INTERVAL ':months_ago months' AS period_start,- date_trunc('month', CURRENT_DATE) - INTERVAL ':months_ago months' + INTERVAL '1 month' - INTERVAL '1 second' AS period_end)+ (SELECT date_trunc('month', CURRENT_DATE) - :months_ago * INTERVAL '1 month' AS period_start,+ date_trunc('month', CURRENT_DATE) - :months_ago * INTERVAL '1 month' + INTERVAL '1 month' - INTERVAL '1 second' AS period_end)SELECT t.id AS topic_id,t.category_id,COUNT(p.id) AS reply_count@@ -358,8 +358,8 @@ module DiscourseDataExplorerWITH query_period AS (SELECT- date_trunc('month', CURRENT_DATE) - INTERVAL ':months_ago months' as period_start,- date_trunc('month', CURRENT_DATE) - INTERVAL ':months_ago months' + INTERVAL '1 month' - INTERVAL '1 second' as period_end+ date_trunc('month', CURRENT_DATE) - :months_ago * INTERVAL '1 month' as period_start,+ date_trunc('month', CURRENT_DATE) - :months_ago * INTERVAL '1 month' + INTERVAL '1 month' - INTERVAL '1 second' as period_end)SELECT@@ -754,7 +754,7 @@ module DiscourseDataExplorerCOALESCE(SUM(tvs.anonymous_views + tvs.logged_in_views), 0) AS total_viewsFROM topics tJOIN topic_view_stats tvs ON tvs.topic_id = t.id- AND tvs.viewed_at >= CURRENT_DATE - :days_ago+ AND tvs.viewed_at >= CURRENT_DATE - CAST(:days_ago AS integer)WHERE t.deleted_at IS NULLAND t.archetype = 'regular'GROUP BY t.id, t.category_id@@ -771,7 +771,7 @@ module DiscourseDataExplorerCOUNT(*) AS searches,COUNT(DISTINCT user_id) AS distinct_usersFROM search_logs- WHERE created_at >= CURRENT_DATE - :days_ago+ WHERE created_at >= CURRENT_DATE - CAST(:days_ago AS integer)GROUP BY termORDER BY distinct_users DESC, searches DESCLIMIT 200@@ -785,7 +785,7 @@ module DiscourseDataExplorer-- gapless day series, so days with zero activity still show up as rowsWITH day_series AS (SELECT generate_series(- CURRENT_DATE - :days_ago,+ CURRENT_DATE - CAST(:days_ago AS integer),CURRENT_DATE,'1 day'::interval)::date AS day@@ -797,7 +797,7 @@ module DiscourseDataExplorerFROM topic_link_clicks tlcJOIN topic_links tl ON tl.id = tlc.topic_link_idWHERE tl.topic_id = :topic_id- AND tlc.created_at >= CURRENT_DATE - :days_ago+ AND tlc.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)GROUP BY tlc.created_at::date),daily_views AS (@@ -807,7 +807,7 @@ module DiscourseDataExplorerCOALESCE(SUM(logged_in_views), 0) AS logged_in_viewsFROM topic_view_statsWHERE topic_id = :topic_id- AND viewed_at >= CURRENT_DATE - :days_ago+ AND viewed_at >= CURRENT_DATE - CAST(:days_ago AS integer)GROUP BY viewed_at)SELECT@@ -947,10 +947,10 @@ module DiscourseDataExplorerLEFT JOIN (SELECT DISTINCT target_user_idFROM user_histories- WHERE created_at >= CURRENT_DATE - :days_ago+ WHERE created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND action IN (2, 15) -- change_trust_level, auto_trust_level_change) tlc ON u.id = tlc.target_user_id- WHERE u.created_at >= CURRENT_DATE - :days_ago+ WHERE u.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND tlc.target_user_id IS NULL),trust_level_changes AS (@@ -959,7 +959,7 @@ module DiscourseDataExplorerCOUNT(DISTINCT uh.target_user_id) AS users_gainedFROM user_histories uhJOIN users u ON uh.target_user_id = u.id- WHERE uh.created_at >= CURRENT_DATE - :days_ago+ WHERE uh.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND uh.action IN (2, 15) -- change_trust_level, auto_trust_level_changeAND uh.new_value IN ('1', '2', '3', '4')GROUP BY uh.new_value@@ -999,9 +999,9 @@ module DiscourseDataExplorerAND p.post_type = 1WHERE t.archetype = 'regular'AND t.deleted_at IS NULL- AND t.created_at >= CURRENT_DATE - :days_ago+ AND t.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND (- :category_id IS NULL+ CAST(:category_id AS integer) IS NULLOR t.category_id = :category_idOR (:include_subcategories AND t.category_id IN (SELECT id FROM categories WHERE parent_category_id = :category_id@@ -1317,7 +1317,7 @@ module DiscourseDataExplorerw.topic_id,w.created_atFROM user_warnings w- WHERE w.created_at >= CURRENT_DATE - :days_ago+ WHERE w.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)ORDER BY w.created_at DESCSQL@@ -1413,7 +1413,7 @@ module DiscourseDataExplorerCOUNT(*) AS totalFROM reviewable_scores rsLEFT JOIN flags f ON f.id = rs.reviewable_score_type- WHERE rs.created_at >= CURRENT_DATE - :days_ago+ WHERE rs.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)GROUP BY 1ORDER BY total DESCSQL@@ -1428,7 +1428,7 @@ module DiscourseDataExplorerFROM reviewable_scores rsJOIN users u ON u.id = rs.reviewed_by_idWHERE (u.admin OR u.moderator)- AND rs.reviewed_at >= CURRENT_DATE - :days_ago+ AND rs.reviewed_at >= CURRENT_DATE - CAST(:days_ago AS integer)GROUP BY rs.reviewed_by_idORDER BY flags_handled DESCLIMIT 100@@ -1483,7 +1483,7 @@ module DiscourseDataExplorerWHERE t.locale IS NOT NULLAND t.locale <> ''AND split_part(t.locale, '_', 1) <> split_part(:primary_locale, '_', 1)- AND t.created_at >= CURRENT_DATE - :days_ago+ AND t.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND t.deleted_at IS NULLORDER BY t.created_at DESCSQL@@ -1503,7 +1503,7 @@ module DiscourseDataExplorerWHERE p.locale IS NOT NULLAND p.locale <> ''AND split_part(p.locale, '_', 1) <> split_part(:primary_locale, '_', 1)- AND p.created_at >= CURRENT_DATE - :days_ago+ AND p.created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND p.deleted_at IS NULLORDER BY p.created_at DESCSQL@@ -1600,7 +1600,7 @@ module DiscourseDataExplorerMAX(created_at) AS last_seen,(array_agg(user_agent ORDER BY created_at DESC))[1] AS sample_user_agentFROM browser_pageview_events- WHERE created_at >= CURRENT_DATE - :days_ago+ WHERE created_at >= CURRENT_DATE - CAST(:days_ago AS integer)AND source = #{BrowserPageviewEvent::SOURCE_BEACON}GROUP BY ip_address, asn, country_codeORDER BY asn_total_pageviews DESC, pageviews DESCdiff --git a/plugins/discourse-data-explorer/spec/lib/queries_spec.rb b/plugins/discourse-data-explorer/spec/lib/queries_spec.rbnew file mode 100644index 00000000000..528003b5656--- /dev/null+++ b/plugins/discourse-data-explorer/spec/lib/queries_spec.rb@@ -0,0 +1,50 @@+# frozen_string_literal: true++describe DiscourseDataExplorer::Queries do+ fab!(:user)+ fab!(:post)+ fab!(:group)++ before { SiteSetting.data_explorer_enabled = true }++ describe ".default" do+ described_class.default.each do |id, attributes|+ it "runs #{attributes[:name]} with its declared parameters" do+ query =+ DiscourseDataExplorer::Query.new(id: id, name: attributes[:name], sql: attributes["sql"])+ values = {+ "start_date" => 30.days.ago.to_date.iso8601,+ "end_date" => Date.current.iso8601,+ "group_name" => group.name,+ "poll_name" => "poll",+ "post_id" => post.id.to_s,+ "topic_id" => post.topic_id.to_s,+ "category_id" => post.topic.category_id.to_s,+ "user" => user.id.to_s,+ "notification_level" => "3",+ "silenced_by" => user.username,+ "suspended_by" => user.username,+ }+ params =+ values.slice(*query.params.reject { |param| param.default.present? }.map(&:identifier))+ variants = [params]+ optional_params = query.params.select(&:nullable).map(&:identifier)+ variants << params.except(*optional_params) if optional_params.any?++ variants.each do |query_params|+ result =+ DiscourseDataExplorer::DataExplorer.run_query(+ query,+ query_params,+ current_user: user,+ limit: 10,+ )++ expect(result[:error]).to be_nil,+ "#{attributes[:name]} with #{query_params.inspect}: #{result[:error]}"+ expect(result[:pg_result]).to be_a(PG::Result)+ end+ end+ end+ end+enddiff --git a/plugins/discourse-workflows/spec/lib/discourse_workflows/ai/tools/workflow_ai_agent_catalog_spec.rb b/plugins/discourse-workflows/spec/lib/discourse_workflows/ai/tools/workflow_ai_agent_catalog_spec.rbindex d5edd4d0dd7..0c3b2b699bb 100644--- a/plugins/discourse-workflows/spec/lib/discourse_workflows/ai/tools/workflow_ai_agent_catalog_spec.rb+++ b/plugins/discourse-workflows/spec/lib/discourse_workflows/ai/tools/workflow_ai_agent_catalog_spec.rb@@ -49,7 +49,7 @@ RSpec.describe DiscourseWorkflows::Ai::Tools::WorkflowAiAgentCatalog doendcontext "when disabled agents are included" do- let(:query) { "old" }+ let(:query) { disabled_agent.name }let(:include_disabled) { true }it "returns disabled matches for awareness", :aggregate_failures do