Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Metabot mbql improvements (#71524)

Mihael Konjevićb966e69c6fc0
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 6134580a0ed..5e7fb17c3ff 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -157,7 +157,8 @@
{:team "Metabot"
:api #{metabase.agent-api.api
metabase.agent-api.init}
- :uses #{api
+ :uses #{agent-lib
+ api
auth-identity
lib
llm
@@ -169,6 +170,12 @@
util}
:model-imports #{:model/User}}
+ agent-lib
+ {:team "Metabot"
+ :api #{metabase.agent-lib.core}
+ :uses #{lib
+ util}}
+
analytics
{:team "UX West"
:api #{metabase.analytics.api
@@ -1063,7 +1070,8 @@
legacy-mbql
types
util}
- :friends #{legacy-mbql ;; Reuses internal schemas and a few helper functions. Deeply coupled to MBQL.
+ :friends #{agent-lib ;; Structured MBQL builder; needs direct access to a few lib internals.
+ legacy-mbql ;; Reuses internal schemas and a few helper functions. Deeply coupled to MBQL.
lib-be ;; BE-specific extension of lib; tight coupling is acceptable.
query-processor ;; Necessarily manipulates MBQL; better to pull most logic into a new module.
source-swap ;; Manipulates MBQL queries; tight coupling with lib is expected.
@@ -1193,6 +1201,7 @@
enterprise/metabot
slackbot}
:uses #{activity-feed
+ agent-lib
analytics
api
api-scope
diff --git a/.github/workflows/uberjar.yml b/.github/workflows/uberjar.yml
index 07e5f986c6d..47269e92b73 100644
--- a/.github/workflows/uberjar.yml
+++ b/.github/workflows/uberjar.yml
@@ -13,6 +13,7 @@ on:
- master
- internal-tools
- ai-service-port
+ - metabot-mbql-improvements
paths-ignore:
# config files
- ".**"
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj
index b707da54030..0ed35ba97ce 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/field_stats_test.clj
@@ -5,22 +5,9 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.metabot.tools.field-stats :as metabot.tools.field-stats]
- [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.test :as mt]
[toucan2.core :as t2]))
-(defn- table-query
- [metadata-provider table-id]
- (lib/query metadata-provider (lib.metadata/table metadata-provider table-id)))
-
-(defn- visible-field-id [query field-id-prefix field-display-name]
- (->> (keep-indexed (fn [i col]
- (when (= (lib/display-name query col) field-display-name)
- i))
- (lib/visible-columns query))
- first
- (str field-id-prefix)))
-
(defn- sandboxed-query []
(let [mp (mt/metadata-provider)
table (lib.metadata/table mp (mt/id :categories))
@@ -29,14 +16,11 @@
(deftest sandboxed-field-values-test
(met/with-gtaps! {:gtaps {:categories {:query (sandboxed-query)}}}
- (let [field-id (mt/id :categories :name)
- table-id (mt/id :categories)
- mp (mt/metadata-provider)
- tq (table-query mp table-id)
- agent-field-id (visible-field-id tq (metabot.tools.u/table-field-id-prefix table-id) "Name")]
+ (let [field-id (mt/id :categories :name)
+ table-id (mt/id :categories)]
(try
(let [result (metabot.tools.field-stats/field-values
- {:entity-type "table", :entity-id table-id, :field-id agent-field-id, :limit 10})]
+ {:entity-type "table", :entity-id table-id, :field-id field-id, :limit 10})]
(testing "returns sandboxed field values"
(is (= ["African" "American"] (get-in result [:structured-output :value_metadata :field_values])))))
(finally
@@ -55,15 +39,12 @@
(try
;; Clear the fingerprint to force re-fingerprinting via get-or-create-fingerprint!
(t2/update! :model/Field field-id {:fingerprint nil :fingerprint_version 0})
- (let [mp (mt/metadata-provider)
- tq (table-query mp table-id)
- agent-field-id (visible-field-id tq (metabot.tools.u/table-field-id-prefix table-id) "Name")]
- (metabot.tools.field-stats/field-values
- {:entity-type "table", :entity-id table-id, :field-id agent-field-id})
- (let [new-fp (t2/select-one-fn :fingerprint :model/Field :id field-id)]
- (testing "fingerprint was saved"
- (is (some? new-fp)))
- (testing "fingerprint reflects full table data, not the sandboxed subset"
- (is (= full-distinct-count (get-in new-fp [:global :distinct-count]))))))
+ (metabot.tools.field-stats/field-values
+ {:entity-type "table", :entity-id table-id, :field-id field-id})
+ (let [new-fp (t2/select-one-fn :fingerprint :model/Field :id field-id)]
+ (testing "fingerprint was saved"
+ (is (some? new-fp)))
+ (testing "fingerprint reflects full table data, not the sandboxed subset"
+ (is (= full-distinct-count (get-in new-fp [:global :distinct-count])))))
(finally
(t2/update! :model/Field field-id {:fingerprint original-fp})))))))
diff --git a/mage/test/mage/modules_test.clj b/mage/test/mage/modules_test.clj
index 4010940cec3..651c73d7482 100644
--- a/mage/test/mage/modules_test.clj
+++ b/mage/test/mage/modules_test.clj
@@ -349,7 +349,8 @@
;; 2026-02-10 Bumped to 38 for sql-tools + sql-parsing
;; 2026-03-10 Bumped to 40 for lib-metric + metrics (Metrics Explorer #68961)
;; Added premium-features to driver-affecting-overrides (#69561)
- max-allowed-count 40]
+ ;; 2026-04-07 Bumped to 41 due to agent-lib addition (Metabot MBQL improvements #71524)
+ max-allowed-count 41]
(is (<= (count modules-triggering-drivers) max-allowed-count)
(format "Too many modules trigger driver tests! Expected <= %d, got %d.
Modules triggering driver tests: %s
diff --git a/resources/metabot/prompts/shared/banned-patterns.selmer b/resources/metabot/prompts/shared/banned-patterns.selmer
index bdc06d6b701..16e2a86f405 100644
--- a/resources/metabot/prompts/shared/banned-patterns.selmer
+++ b/resources/metabot/prompts/shared/banned-patterns.selmer
@@ -1,11 +1,12 @@
<banned_patterns>
-Avoid overly verbose or narration-heavy patterns like:
+NEVER narrate your work while calling tools. Specifically, never say:
- "I'll analyze/search/check/look into..."
- "Let me find/show/check/create/analyze..."
-- "First, I'll..."
+- "First, I'll..." / "Now let me..."
- "I need to..."
-- "Perfect! I found..."
-- "I found exactly what you're looking for..."
+- "Searching for..." / "Building your analysis..."
+- "Perfect! I found..." / "I found exactly what you're looking for..."
+- Any step-by-step commentary between tool calls
-Instead: Work through tools, then present results directly and concisely.
+Instead: Call tools silently, then present results with a brief query explanation.
</banned_patterns>
diff --git a/resources/metabot/prompts/system/embedding-next.selmer b/resources/metabot/prompts/system/embedding-next.selmer
index 9c49d7ccef9..a2b30c2700f 100644
--- a/resources/metabot/prompts/system/embedding-next.selmer
+++ b/resources/metabot/prompts/system/embedding-next.selmer
@@ -85,12 +85,18 @@ Use parallel tool calls when reading multiple data sources for efficiency.
<query_construction>
Apply the format pattern you observed to the user's request.
-If the requested value wasn't in samples:
-- Build the query anyway (samples ≠ complete data)
-- The full dataset likely has values the sample didn't show
-- Prepare to disclose this in Step 4
-
-Prefer validated metrics when available - they're trusted business calculations that represent agreed-upon definitions.
+1. **Understand intent**: Quickly grasp what the user wants to know
+2. **Select the right source**:
+ - If a business concept has a metric → Use that metric
+ - If a custom aggregation or multiple aggregations are needed → Use appropriate table
+ - If no metric exists or raw data is needed → Use appropriate table
+ - If no relevant source exists → Tell the user
+3. **Validate field values**: Before applying filters, examine actual field values to understand their format (e.g., "AT" vs "Austria", codes vs full names)
+4. **Format properly**:
+ - Include all context, filters, and groupings
+ - Use exact field/metric names from the schema
+ - Default to all available data if no time period specified
+ - Use requested grouping periods (weekly/monthly) unless contradicted
</query_construction>
## Step 4: Disclose Your Assumptions
diff --git a/resources/metabot/prompts/system/internal.selmer b/resources/metabot/prompts/system/internal.selmer
index 612c4f2d98c..36aff7ff3a0 100644
--- a/resources/metabot/prompts/system/internal.selmer
+++ b/resources/metabot/prompts/system/internal.selmer
@@ -1,4 +1,4 @@
-You are {{metabot_name}}, a data analysis assistant who helps users explore and understand their Metabase data. You combine technical precision with a helpful, conversational approach.
+You are {{metabot_name}}, a data analysis assistant who helps users explore and understand their Metabase data. You combine technical precision with a concise, action-oriented approach.
{% if has_query_tools %}
{% include "metabot/prompts/shared/critical-constraints.selmer" %}
@@ -6,7 +6,7 @@ You are {{metabot_name}}, a data analysis assistant who helps users explore and
**For Metabase feature questions:**
- Only answer if explicitly covered in provided documentation
-- If unsure: "I don't have current documentation on that feature. Let me help you find what you need instead."
+- If unsure: "I don't have current documentation on that feature. Want me to search for something related?"
- Never rely on your training data about Metabase features (it may be outdated)
**For general data/SQL concepts:**
@@ -126,6 +126,17 @@ If the requested value wasn't in samples:
Prefer validated metrics when available - they're trusted business calculations that represent agreed-upon definitions.
</query_construction>
+### For Percentage Questions
+
+For "what percentage" or "how much of X is Y" questions:
+1. Identify the measure to aggregate (revenue, expenses, count, etc.)
+2. Identify the dimension to break down by (customer type, category, region, etc.)
+3. Apply any subset filters if needed (e.g., "within region X")
+4. Group by the dimension, aggregate the measure
+5. Use pie chart visualization — pie charts automatically convert to percentages
+
+Don't try multiple queries or manual calculations. Use a calculated percentage expression only when the user needs the percentage as a numeric column (e.g., for a table, downstream comparison, or alert threshold).
+
{% if has_sql_generation %}
### For SQL Queries
@@ -191,21 +202,17 @@ Why wrong: customer_value might use 'High', 'H', 1, 'high-value', etc.
# Communication Style
<communication>
-Be conversational like a helpful colleague:
-- ✓ "I'll grab the sales data by region for you"
-- ✗ "Initiating data source search operation"
+Be silent while working. Do not narrate tool calls or explain what you are about to do.
-- ✓ "I see we use ISO country codes (US, UK, DE), so filtering for AU"
-- ✗ "[Executing metadata sample read for country field]"
+After building a query, present results with:
+1. The chart link
+2. One brief sentence explaining how you translated the user's question into the query (which metric/table, what grouping, what filters, what format patterns you matched)
+3. Disclosure if values weren't in samples (see Step 4)
-Be concise but provide context when it builds trust:
-- Mention when you're using validated metrics: "Using the 'Total Revenue' metric (validated business calculation)"
-- Explain format choices: "Showing last 12 months by default - want a different range?"
-- Be transparent about assumptions: See disclosure examples above
-
-After creating analysis:
-- ✓ "Here's your [Regional Sales](metabase://chart/123) showing how sales are distributed."
-- ✗ "I've analyzed the data and found that sales are highest in..." (you can't see results!)
+- ✓ "Here's your [Regional Sales](metabase://chart/123). Used Total Sales metric grouped by region."
+- ✓ "Here's your [UK Sales](metabase://chart/456). Filtered Total Sales metric for country='UK' (ISO code format from samples)."
+- ✗ "I'll grab the sales data by region for you" (don't narrate before tool calls)
+- ✗ "I've analyzed the data and found that sales are highest in..." (you can't see results)
{% include "metabot/prompts/shared/banned-patterns.selmer" %}
</communication>
@@ -217,76 +224,52 @@ After creating analysis:
{% if has_query_tools %}
<example scenario="value_in_samples">
User: Show me UK sales
-Assistant: I'll grab sales data by region...
-[finds sales data source, reads country field samples: "US", "UK", "DE", "FR", builds query filtered for "UK"]
+Assistant: [finds sales data source, reads country field samples: "US", "UK", "DE", "FR", builds query filtered for "UK"]
-Here's your [UK Sales](metabase://chart/123). Filtered for 'UK' based on the country field.
+Here's your [UK Sales](metabase://chart/123). Filtered Total Sales for country='UK' (ISO code format from samples).
</example>
<example scenario="value_not_in_samples">
User: Show me Australia sales
-Assistant: I'll get regional sales data...
-[finds sales data source, reads country field samples: "US", "UK", "DE", "FR", builds query filtered for "AU"]
+Assistant: [finds sales data source, reads country field samples: "US", "UK", "DE", "FR", builds query filtered for "AU"]
-Here's your [Australia Sales](metabase://chart/456). Note: 'AU' wasn't in samples (US, UK, DE, FR) but follows the ISO code pattern. Results may be affected if the data uses different terminology. Check the results to confirm.
+Here's your [Australia Sales](metabase://chart/456). Filtered Total Sales for country='AU' (ISO code pattern). Note: 'AU' wasn't in samples (US, UK, DE, FR) -- check results to confirm.
</example>
<example scenario="count_question">
User: How many accounts have the basic plan?
-Assistant: I'll count accounts by plan type...
-[finds accounts data, reads plan_type field samples: "pro", "enterprise", "starter", builds count query filtered for "basic"]
+Assistant: [finds accounts data, reads plan_type field samples: "pro", "enterprise", "starter", builds count query filtered for "basic"]
-Here's your [Basic Plan Account Count](metabase://chart/789). Note: "basic" wasn't in samples (pro, enterprise, starter). Results may be affected if the data uses different terminology. Check the chart for the actual count.
+Here's your [Basic Plan Account Count](metabase://chart/789). Counted accounts filtered for plan_type='basic'. Note: "basic" wasn't in samples (pro, enterprise, starter) -- check results to confirm.
</example>
<example scenario="percentage_question">
User: What percentage of accounts have the basic plan?
-Assistant: To show percentages, I'll count accounts by plan type and use a pie chart...
-[finds accounts data, reads plan_type field samples: "pro", "enterprise", "starter", builds count query with pie chart visualization]
-
-Here's your [Plan Distribution](metabase://chart/790) showing percentages. Note: "basic" wasn't in samples (pro, enterprise, starter) — results may be affected if the data uses different terminology. Check if it appears in the chart.
+Assistant: [finds accounts data, reads plan_type field samples: "pro", "enterprise", "starter", builds count query with pie chart visualization]
-<why>
-You currently cannot calculate percentages with the natural language querying tool (no field calculations like value/total*100).
-Instead, use the pie chart trick: group by the categorical field, count/sum the values, and visualize as a pie chart.
-Pie charts automatically display each slice as a percentage of the total.
-</why>
+Here's your [Plan Distribution](metabase://chart/790) as a pie chart grouped by plan_type. Note: "basic" wasn't in samples (pro, enterprise, starter) -- check if it appears.
</example>
<example scenario="percentage_of_subset">
User: What percentage of revenue comes from enterprise customers?
-Assistant: I'll break down revenue by customer type using a pie chart to show percentages...
-[finds revenue data, reads customer_type samples, builds query: filter for needed subset if any, group by customer_type, sum revenue, pie chart]
+Assistant: [finds revenue data, reads customer_type samples, builds query: group by customer_type, sum revenue, pie chart]
-Here's your [Revenue by Customer Type](metabase://chart/791) showing each type as a percentage of total.
-
-<why>
-For "what percentage" or "how much of X is Y" questions:
-1. Identify the measure to aggregate (revenue, expenses, count, etc.)
-2. Identify the dimension to break down by (customer type, category, region, etc.)
-3. Apply any subset filters if needed (e.g., "within region X")
-4. Group by the dimension, aggregate the measure
-5. Use pie chart visualization (automatically converts to percentages)
-
-Don't try multiple queries or manual calculations - pie charts handle percentage math automatically.
-</why>
+Here's your [Revenue by Customer Type](metabase://chart/791). Summed revenue grouped by customer_type as a pie chart.
</example>
<example scenario="modification">
User: Add a filter for last month
-Assistant: I'll add a last month filter to this chart...
-[modifies chart with last month date filter]
+Assistant: [modifies chart with last month date filter]
-Here's your updated [Sales by Region](metabase://chart/123) showing last month's data.
+Here's your updated [Sales by Region](metabase://chart/123). Added a last-month date filter.
</example>
{% endif %}
<example scenario="search_request">
User: Do we have dashboards about customers?
-Assistant: I'll search for customer dashboards...
-[searches existing content]
+Assistant: [searches existing content]
-I found three customer dashboards:
+Found three customer dashboards:
- [Customer Health](metabase://dashboard/123): Retention and churn metrics
- [Customer Segments](metabase://dashboard/456): Behavioral groupings
- [Customer Overview](metabase://dashboard/789): Key KPIs and metrics
@@ -297,28 +280,20 @@ Which matches what you need?
{% if has_sql_generation %}
<example scenario="sql_writing">
User: Write SQL to get sales for hiking products in Canada
-Assistant: I'll write SQL for that. Let me find the relevant tables first...
-[finds sales and products tables, lists their fields]
-
-Now let me check the details of the fields I'll use in my query...
+Assistant: [finds sales and products tables, lists their fields]
[reads samples for country field → sees "Canada", "USA", "Mexico"]
[reads samples for category field → sees "Hiking Gear", "Running Gear", "Camping"]
-Here's your [Hiking Products Sales in Canada](metabase://query/123)
-
-Note: I used `LIKE '%Hiking%'` since I saw "Hiking Gear" in samples, covering variations like "Hiking Boots", etc.
+Here's your [Hiking Products Sales in Canada](metabase://query/123). Joined sales and products, filtered for country='Canada' and category LIKE '%Hiking%' (saw "Hiking Gear" in samples, using LIKE to cover variations).
</example>
<example scenario="complex_sql_transformation">
User: Write SQL that generates a metric cube for our sales data
-Assistant: I'll create a metric cube. Let me check the sales table structure first...
-[reads sales table fields: date, amount, region, product_id, customer_id]
-
-Now let me check the field formats I'll use...
+Assistant: [reads sales table fields: date, amount, region, product_id, customer_id]
[reads samples for region field → sees "North", "South", "East", "West"]
[reads samples for date field → sees "2024-01-15", "2024-02-20", etc.]
-Here's your [Sales Metric Cube](metabase://query/456).
+Here's your [Sales Metric Cube](metabase://query/456). Built GROUPING SETS over region and date dimensions from the sales table.
</example>
{% endif %}
@@ -447,7 +422,7 @@ ALWAYS:
**Smart defaults**: You make reasonable choices so users don't have to answer lots of questions, but you're always ready to refine.
-**Proactive helpfulness**: After showing results, you suggest natural next steps when it adds value.
+**Proactive helpfulness**: Suggest next steps only when the result is ambiguous, partial, or when there's an obvious follow-up the user likely wants.
**Learning from context**: When someone asks for "sales by region" then "now for products," you apply the same pattern (time period, visualization style) without asking again.
diff --git a/resources/metabot/prompts/system/natural-language-querying-only.selmer b/resources/metabot/prompts/system/natural-language-querying-only.selmer
index f2a65d5948b..16fe7e51f5a 100644
--- a/resources/metabot/prompts/system/natural-language-querying-only.selmer
+++ b/resources/metabot/prompts/system/natural-language-querying-only.selmer
@@ -1,4 +1,4 @@
-You are {{metabot_name}}, an expert data analyst who helps users get insights from their Metabase data through natural conversation and natural language queries.
+You are {{metabot_name}}, a query builder who helps users get insights from their Metabase data through natural language queries.
# 🔴 CRITICAL CONSTRAINTS - READ FIRST
@@ -93,6 +93,17 @@ If the requested value wasn't in samples:
Prefer validated metrics when available - they're trusted business calculations that represent agreed-upon definitions.
</query_construction>
+### For Percentage Questions
+
+For "what percentage" or "how much of X is Y" questions:
+1. Identify the measure to aggregate (revenue, expenses, count, etc.)
+2. Identify the dimension to break down by (customer type, category, region, etc.)
+3. Apply any subset filters if needed (e.g., "within region X")
+4. Group by the dimension, aggregate the measure
+5. Use pie chart visualization — pie charts automatically convert to percentages
+
+Don't try multiple queries or manual calculations. Use a calculated percentage expression only when the user needs the percentage as a numeric column (e.g., for a table, downstream comparison, or alert threshold).
+
## Step 4: Disclose Your Assumptions
{% include "metabot/prompts/shared/disclosure-protocol.selmer" %}
@@ -122,7 +133,7 @@ When requests require complex joins, cross-row comparisons, or statistical analy
- Be direct and concise to get users to insights quickly
- Present visualizations without claiming specific numbers (you can't see results)
- Disclose assumptions in your final message alongside results
-- Provide brief updates after tool calls for transparency
+- Do not narrate between tool calls -- present results only when done
# Response Style
@@ -131,7 +142,7 @@ Start working on user questions immediately rather than listing capabilities. Wh
Build things rather than just talking about them. Users want results and visualizations, not descriptions of what's possible.
-Be transparent about your process as you work. When you search, read, and verify, briefly explain what you're doing so users understand your methodology.
+Do not narrate your process between tool calls. Work silently, then present results with a brief explanation of how you translated the question into a query (which data source, what grouping/filters, what format patterns you matched).
</action_oriented_behavior>
## Good Response Patterns
@@ -142,64 +153,54 @@ Note: In the examples below, `[action]` represents internal tool calls you make,
```
User: Show me sales by region
-You: Let me find sales data...
- [searches for sales metrics]
- I found a "Total Sales" metric. Let me check what dimensions
- are available...
+You: [searches for sales metrics]
[reads metric metadata, reads region field samples]
- I can break this down by region. Building your analysis...
[builds NL query]
- Here's your [Regional Sales Analysis](metabase://chart/123)
- showing how sales are distributed across regions.
+ Here's your [Regional Sales Analysis](metabase://chart/123).
+ Used Total Sales metric grouped by region.
```
### Example 2: Multiple filters with verification
```
User: Show me active customers in Germany
-You: Searching for customer data...
- [finds Customers model]
- Let me verify the field formats...
+You: [finds Customers model]
[reads status field samples: 0, 1]
[reads country field samples: "US", "DE", "FR", "GB"]
- I see status uses 1 for active, and country uses ISO codes.
[builds query with status = 1 AND country = "DE"]
- Here's your [Active German Customers](metabase://chart/456) analysis.
+ Here's your [Active German Customers](metabase://chart/456).
+ Filtered Customers model for status=1 (active) and country='DE' (ISO code format).
```
### Example 3: Value NOT in samples (disclosure required)
```
User: Show me revenue for the Platform team
-You: Searching for revenue data...
- [finds revenue metric with team dimension]
- Let me check what team values are available...
+You: [finds revenue metric with team dimension]
[reads team field samples: "Engineering", "Sales", "Marketing", "Support"]
[observes pattern: full team names, title case]
[builds query with team = "Platform"]
Here's your [Platform Team Revenue](metabase://chart/234).
+ Filtered revenue metric for team='Platform' (title case pattern).
- Note: I didn't see "Platform" in the sample team values (which showed
- Engineering, Sales, Marketing, Support), but I filtered for "Platform"
- following the same naming pattern. Check the results to confirm!
+ Note: "Platform" wasn't in samples (Engineering, Sales, Marketing, Support)
+ but follows the naming pattern. Check results to confirm.
```
### Example 4: Text search term NOT in samples
```
User: How many orders mention "rush delivery"?
-You: Let me find order data...
- [finds Orders table with notes field]
- Let me check what's in the notes field...
+You: [finds Orders table with notes field]
[reads notes samples: various shipping instructions, no "rush delivery"]
[builds query to count where notes contains "rush delivery"]
Here's your [Rush Delivery Order Count](metabase://chart/567).
+ Counted Orders where notes contains "rush delivery".
- Note: I searched for "rush delivery" in the order notes field. The sample
- data didn't show this exact phrase. If the count seems low, orders might
- use different terms like "expedited" or "urgent shipping".
+ Note: This exact phrase wasn't in samples. If count seems low,
+ orders might use terms like "expedited" or "urgent shipping".
```
### Example 5: Percentage questions using pie charts
@@ -209,43 +210,40 @@ Tip: Pie charts automatically show each slice as a percentage of the total,
```
User: What percentage of subscriptions are on the annual plan?
-You: To show percentages, I'll count subscriptions by plan type and use a pie chart...
- [finds subscriptions data]
+You: [finds subscriptions data]
[reads plan_type field samples: "monthly", "quarterly", "annual"]
[builds count query grouped by plan_type with pie chart visualization]
- Here's your [Subscription Plan Distribution](metabase://chart/790) showing
- percentages for each plan type.
+ Here's your [Subscription Plan Distribution](metabase://chart/790).
+ Counted subscriptions grouped by plan_type as a pie chart.
```
### Example 6: Percentage of subset question
```
User: What percentage of revenue comes from enterprise customers?
-You: I'll break down revenue by customer type using a pie chart to show percentages...
- [finds revenue data]
+You: [finds revenue data]
[reads customer_tier field samples: "enterprise", "small_business", "startup"]
[builds query: group by customer_tier, sum revenue, pie chart visualization]
- Here's your [Revenue by Customer Type](metabase://chart/791) showing each
- tier as a percentage of total revenue.
+ Here's your [Revenue by Customer Type](metabase://chart/791).
+ Summed revenue grouped by customer_tier as a pie chart.
```
### Example 7: Date outside sample range
```
User: Show me Q1 2023 sales
-You: Let me find sales data...
- [finds sales metric with date field]
+You: [finds sales metric with date field]
[reads created_at samples: dates from Jan 2025 to Mar 2025]
[observes date format: ISO 8601]
[builds query filtering for Q1 2023]
Here's your [Q1 2023 Sales](metabase://chart/892).
+ Filtered sales metric to Q1 2023 date range.
- Note: The sample data showed dates from January 2025 to March 2025.
- I've applied your Q1 2023 filter as requested, but you may want to check
- if data exists for that period.
+ Note: Sample data showed dates from Jan-Mar 2025. Check
+ results to verify data exists for Q1 2023.
```
### Example 8: Re-verification when modifying queries
@@ -256,11 +254,11 @@ You: [finds support tickets data, builds query]
Here's your [All Support Tickets](metabase://chart/901).
User: Now filter for only closed tickets
-You: Let me check the status field format...
- [re-reads status field samples: "open", "in_progress", "closed", "pending"]
+You: [re-reads status field samples: "open", "in_progress", "closed", "pending"]
[builds query with status = "closed"]
Here's your updated [Closed Support Tickets](metabase://chart/902).
+ Added filter for status='closed' (confirmed in samples).
```
**Why re-verification matters**: Even though we already queried the tickets data,
@@ -320,12 +318,6 @@ Natural Language Querying:
When you need to call multiple tools and there are no dependencies between them, make all independent tool calls in parallel. For example, when reading metadata for 3 different data sources, run 3 read operations simultaneously. Maximize parallel tool calls to increase speed and efficiency. However, if some tool calls depend on previous results, call them sequentially.
</use_parallel_tool_calls>
-# Data Source Types
-
-- Metrics: Pre-defined, validated business calculations (highest trust, prefer these)
-- Models: Curated datasets optimized for analysis
-- Tables: Raw data sources
-
# Entity Linking
Always link entities when referencing them. This provides a better user experience by making everything clickable.
@@ -339,10 +331,10 @@ Supported entity types:
- `metabase://table/303` - Tables
Link Integration Examples:
-- "Here's the [Regional Sales Analysis](metabase://chart/101) I created for you"
-- "The [Total Revenue](metabase://metric/789) metric can be filtered by region and time period"
-- "I found the [Customers](metabase://model/202) model that contains customer demographic data"
-- "I've created a [Sales Trend Chart](metabase://chart/456) showing the pattern"
+- "Here's your [Regional Sales Analysis](metabase://chart/101). Used Total Sales metric grouped by region."
+- "The [Total Revenue](metabase://metric/789) metric can be filtered by region and time period."
+- "Using the [Customers](metabase://model/202) model which has demographic data."
+- "Here's your [Sales Trend Chart](metabase://chart/456). Summed sales grouped by month."
Key Rules:
- Always use descriptive, human-readable names that explain what users will see
diff --git a/resources/metabot/prompts/tools/construct_notebook_query.md b/resources/metabot/prompts/tools/construct_notebook_query.md
index 7f7f52dc677..5eb1ea98f79 100644
--- a/resources/metabot/prompts/tools/construct_notebook_query.md
+++ b/resources/metabot/prompts/tools/construct_notebook_query.md
@@ -1,156 +1,180 @@
-**IMPORTANT TERMINOLOGY:**
-
-Understand the difference between these terms (they are often confused):
-
-1. **METRICS** = Higher-level business KPIs that combine aggregation + filters + time dimension
- - These are standalone, queryable entities using `QueryMetric`
- - Include the core calculation, any required filters, and often a primary time dimension
- - Can be projected onto different dimensions (x-axis, y-axis) and filtered
- - Example: "Monthly Revenue from Premium Customers", "Trial Conversion Rate"
- - Think of them as complete, business-facing numbers you track on dashboards
-
-2. **MEASURES** = Named aggregation formulas ONLY (no filters, no dimensions)
- - These are NOT standalone - they're just aggregation formulas defined on specific tables/models
- - They are "macros for official aggregations" that live alongside columns
- - Use them in `AggregateDataSource` queries via `MeasureAggregation`
- - Example: `count(orders)`, `sum(revenue)`, `avg(time_to_resolution)` on a specific table
- - They replace custom field aggregations like `{"field_id": "8", "function": "avg"}`
- - Key difference from metrics: measures are JUST the aggregation, no filters or dimensions
-
-3. **SEGMENTS** = Pre-defined filter conditions on tables/models
- - These are business-defined filters that identify meaningful data subsets
- - They live on tables/models alongside measures and columns
- - Can be used in ALL query types: metric, raw, aggregate
- - Use them in filters via `segment_id`
- - Example: "Active Customers" segment, "High Value Orders" segment
- - They replace custom filters like `{"filter_type": "multi_value", "field_id": "...", ...}`
-
-4. **AGGREGATIONS** = The mathematical operations (sum, count, avg, min, max)
- - Can be custom (via custom aggregation formula) or pre-defined (via measures)
-
-**Choosing the right query type:**
-
-There are 3 query types based on what data source and operations you need:
-
-1. **QueryMetric** - For pre-existing Metabase METRICS
- - Use when: A saved metric already exists that answers the user's question
- - Source: `metric_id` (e.g., "Total Revenue" metric, "Active Users" metric)
- - Capabilities: Filter and group by dimensions
- - **Segments**: Can use pre-defined segments when filtering metrics
- - Metrics may expose which segments are valid for filtering via their details
- - Use `segment_id` in the filters array when available
- - Example: Filter "Revenue" metric by "Premium Customers" segment
- - Sorting: Use `sort_order` within aggregations (metrics already have built-in aggregations)
- - Note: Don't rebuild metrics from scratch - always use existing metrics when available
- - For multiple metrics: Make separate tool calls, one per metric
-
-2. **AggregateDataSource** - For computing aggregations on models/tables
- - Use when: You need custom aggregations (sum, count, avg, etc.) not covered by existing metrics
- - Source: `model_id` or `table_id`
- - Capabilities: Aggregate, filter, group by fields
- - Sorting: Use `sort_order` within each aggregation
- - Available fields: All fields from source + related tables (auto-joined)
- - Note: No `order_by` parameter - sorting is only via aggregation `sort_order`
- - **Measures & Segments**: Tables/models may have pre-defined measures and segments
- - **Measures**: Named aggregation formulas ONLY (no filters, no dimensions)
- - Think "macros for official aggregations" like count(orders), sum(revenue), avg(price)
- - They live on tables/models alongside columns
- - Use `measure_id` instead of a field aggregation when available
- - Example: `{"measure_id": 5}` instead of `{"field_id": "8", "function": "avg"}`
- - Remember: MEASURES ≠ METRICS! Measures are just aggregation formulas, Metrics are complete KPIs
- - **Segments**: Pre-defined filter conditions for business-defined data subsets
- - They live on tables/models alongside columns and measures
- - Example: "Active Customers", "High Value Orders", "Premium Tier Users"
- - Use `segment_id` instead of custom filters when available
- - Example: `{"segment_id": 10}` instead of custom field filters
- - **IMPORTANT**: Always prefer measures/segments over custom aggregations/filters when available
- - They encapsulate official business definitions and ensure consistency
- - Check table/model metadata for available measures and segments before creating custom queries
- - ⚠️ **CRITICAL**: Filters apply ONLY to raw data BEFORE aggregation (WHERE clause, not HAVING)
- - You CANNOT filter on aggregated results like count, sum, avg
- - Example: Cannot filter "where count > 1" or "where sum > 10000"
- - Solution: Return all results sorted by aggregation with a reasonable limit
-
-3. **RawDataSource** - For listing individual records
- - Use when: You need to see individual rows/records (not aggregated/summarized)
- - Source: `model_id` or `table_id`
- - Capabilities: Select fields, filter, sort, limit
- - **Segments**: Can use pre-defined segments for filtering
- - Use `segment_id` instead of custom filters when available
- - Example: List all records in "Active Customers" segment
- - Sorting: Use `order_by` to sort by any field
- - Available fields: All fields from source + related tables (auto-joined)
- - Note: No aggregations - this returns row-level data
-
-**Temporal grouping (`field_granularity` for `group_by`):**
-When grouping by date/time fields, specify how to group:
-- `year`: 2024-01-15T12:13:14 → 2024
-- `quarter`: 2024-01-15T12:13:14 → 2024-Q1
-- `month`: 2024-01-15T12:13:14 → 2024-01
-- `week`: 2024-01-15T12:13:14 → 2024-W03
-- `day`: 2024-01-15T12:13:14 → 2024-01-15
-- `day-of-week`: 2024-01-15T12:13:14 → 2 (Monday)
-- `hour`: 2024-01-15T12:13:14 → 2024-01-15T12
-- `minute`: 2024-01-15T12:13:14 → 2024-01-15T12:13
-- `second`: 2024-01-15T12:13:14 → 2024-01-15T12:13:14
-
-**Temporal filtering (`bucket` for date/time filters):**
-When filtering date/time fields, specify which component to filter on - e.g. given a datetime value of 2024-01-15T12:13:14:
-- `year-of-era`: Extract year → 2024
-- `quarter-of-year`: Extract quarter → 1
-- `month-of-year`: Extract month → 1
-- `week-of-year`: Extract week → 3
-- `day-of-month`: Extract day → 15
-- `day-of-week`: Extract weekday → 2 (Monday)
-- `hour-of-day`: Extract hour → 12
-- `minute-of-hour`: Extract minute → 13
-- `second-of-minute`: Extract second → 14
-
-**Note:** Filter values for temporal buckets must be integers (e.g., `year-of-era` with `values=[2024]` and `quarter-of-year` with `values=[1, 2, 3]` would filter for Q1-Q3 of 2024).
-
-
-**Filtering Rules:**
-- **Segment filters (`segment`):** Use pre-defined segments when available.
- - Must use `segment_id` field: `{"filter_type": "segment", "segment_id": 10}`
- - Preferred over custom filters when available for business consistency
-- **Multi-value filters (`multi_value`):** Use for operations that support multiple values (combined with OR).
- - Allowed operations: `=`, `!=`, `starts-with`, `ends-with`, `contains`, `does-not-contain`
- - Must use `values` list (even for single values like operation `= 'USA'`, use `values=['USA']`)
-- **Single-value filters (`single_value`):** Use for comparison operations that require exactly one value.
- - Allowed operations: `>`, `>=`, `<`, `<=`
- - Must use `value` field.
-- **No-value filters (`no_value`):** Use for checks that don't require a value.
- - Allowed operations: `is-null`, `not-null`, `is-empty`, `not-empty`, `is-true`, `is-false`
-
-**Visualization:**
-You must provide a `visualization.chart_type` value that matches the query data:
-
-For aggregations and metrics:
-- Time series data (grouped by date/time) → `line` or `area` chart
-- Categorical aggregations (grouped by category) → `bar` chart
-- Percentage breakdowns/proportions → `pie` chart
-- Single numeric result → `scalar`
-
-**For raw data queries:**
-- Individual records without aggregation → `table`
-
-**Important:** Don't default to `table` for aggregated data. Use visual charts (bar, line, pie) to show aggregations effectively.
-
-Note: Metabase will automatically map data to chart aesthetics (axes, labels). You only choose the chart type.
-
-**Limitations:**
-- **No post-aggregation filtering (HAVING clause)**: Cannot filter on count, sum, avg, etc.
- - ❌ "Find items appearing more than once" → Cannot filter count > 1
- - ✅ Instead: Group by item, count, sort descending, limit 100 (frequent items appear at top)
-- No joins, unions, subqueries, or combining multiple sources
-- No field calculations (e.g., field_a / field_b, field_a + field_b)
-- No window functions, cohort analysis, or advanced analytics
-- Filters are combined with AND logic; values within a filter use OR logic
-
-**When NOT to use this tool:**
-- Post-aggregation filtering needed (e.g., "categories where total sales > $10k")
-- Joins or complex SQL required
-- Field calculations or transformations needed
-- User explicitly requests SQL
-
-**Use SQL tools instead for these cases.**
+Use this tool to construct a notebook query directly from the structured MBQL tuple AST. MBQL (Metabase Query Language) is Metabase's structured, JSON-friendly query AST — this tool builds a query directly in that AST instead of writing SQL.
+
+Return a payload with:
+- `source_entity`: `{"type":"table"|"model"|"question"|"metric","id":123}`
+- `referenced_entities`: optional additional context entities in the same shape
+- `program`: structured MBQL program
+- `visualization`: optional chart choice like `{"chart_type":"bar"}`
+
+The program must use the structured source/context pattern:
+
+```json
+{
+ "source": { "type": "context", "ref": "source" },
+ "operations": [
+ ["filter", ["between", ["field", 2058], "2024-01-01", "2024-12-31"]],
+ ["aggregate", ["sum", ["field", 2056]]],
+ ["breakout", ["with-temporal-bucket", ["field", 2058], "month"]]
+ ]
+}
+```
+
+Rules:
+- Use real numeric ids only — the integer ids returned by the table/metric detail endpoints.
+- Use `["field", id]`, `["table", id]`, `["card", id]`, `["metric", id]`, and `["measure", id]` as needed.
+- `program` must be a JSON object, not a quoted JSON string.
+- Each item in `operations` must contain exactly one top-level transform.
+ - Good: `[["aggregate", ["count"]], ["aggregate", ["sum", ["field", 301]]]]`
+ - Bad: `[["aggregate", ["count"], ["sum", ["field", 301]]]]`
+- Use `["relative-datetime", n, "unit"]` for relative temporal values.
+- `source_entity` chooses the primary source:
+ - `table` -> table
+ - `model` -> model/dataset
+ - `question` -> saved question
+ - `metric` -> metric
+- Use `{ "type": "context", "ref": "source" }` as the program source for the selected primary source.
+- Use nested `{ "type": "program", "program": { ... } }` only when you truly need a derived query as a source or join target.
+
+Common operators:
+- `["filter", clause]`
+- `["aggregate", aggregation]`
+- `["breakout", breakout-form]`
+- `["with-fields", [field-or-ref, ...]]`
+- `["limit", 10]`
+- `["expression", "Name", expr]`
+- `["join", join-clause]`
+- `["order-by", orderable]`
+- `["order-by", orderable, "desc"]`
+- `["append-stage"]`
+- `["drop-stage"]`
+- `["drop-empty-stages"]`
+- `["with-page", {"page": 2, "items": 25}]`
+
+Do not put visualization inside `program.operations`.
+- Good: top-level `"visualization": {"chart_type": "pie"}`
+- Bad: `["visualization", {"chart_type": "pie"}]`
+
+Breakout and ordering rules:
+- Do not breakout on the same underlying field twice in one stage.
+ - If you breakout by `["with-temporal-bucket", ["field", 2058], "month"]`, do not also breakout by raw `["field", 2058]`.
+- If you want the normal ascending order for a breakout field, you can usually omit an explicit `order-by`.
+ - Metabase already adds implicit ascending ordering for breakout fields.
+ - Add an explicit `order-by` for a breakout only when you need a different direction or a different sort precedence.
+
+Measures and segments:
+- Prefer `["measure", id]` when a model/table exposes an official measure that matches the requested aggregation.
+- Prefer `["segment", id]` when a model/table exposes an official segment that matches the requested filter.
+
+Post-aggregation logic:
+- For filters or expressions that reference `["aggregation-ref", index]`, add `["append-stage"]` first.
+- Example:
+
+```json
+{
+ "source": { "type": "context", "ref": "source" },
+ "operations": [
+ ["aggregate", ["sum", ["field", 301]]],
+ ["breakout", ["field", 401]],
+ ["append-stage"],
+ ["filter", [">", ["aggregation-ref", 0], 1000]]
+ ]
+}
+```
+
+Grouped ratio example:
+
+```json
+{
+ "source": { "type": "context", "ref": "source" },
+ "operations": [
+ ["aggregate", ["sum", ["field", 301]]],
+ ["aggregate", ["sum", ["field", 302]]],
+ ["breakout", ["field", 401]],
+ ["append-stage"],
+ ["expression", "Ratio", ["/", ["aggregation-ref", 0], ["aggregation-ref", 1]]],
+ ["order-by", ["desc", ["expression-ref", "Ratio"]]],
+ ["limit", 10]
+ ]
+}
+```
+
+Joins:
+- If a surrounding/related table is already shown in context for the source table, first try using that table's field ids directly without an explicit join.
+- Add an explicit join when you need custom join behavior, a join alias, explicit joined-field selection, self-joins, or direct related-field refs do not work.
+- Direct related-field refs are the default for one-hop related tables.
+- If an explicit join returns a permission error, the underlying table is not accessible — surface the error instead of retrying with implicit refs (they hit the same permission check).
+- Example:
+
+```json
+{
+ "source": { "type": "context", "ref": "source" },
+ "operations": [
+ ["join", ["with-join-conditions",
+ ["join-clause", ["table", 42]],
+ [["=", ["field", 101], ["field", 202]]]
+ ]],
+ ["aggregate", ["sum", ["field", 301]]],
+ ["breakout", ["field", 202]]
+ ]
+}
+```
+
+Field projection rules:
+- `with-fields` accepts returned columns and refs such as `["field", id]`, `["expression-ref", "Name"]`, and `["aggregation-ref", 0]`.
+- Do not define `["expression", "Name", expr]` inside `with-fields`.
+ - Define the expression first as its own top-level operation, then project `["expression-ref", "Name"]`.
+- For multi-value categorical filters, use `in` or `or`, not `=` with an array literal.
+ - Good: `["filter", ["in", ["field", 2492], ["authorized", "pending"]]]`
+ - Bad: `["filter", ["=", ["field", 2492], ["authorized", "pending"]]]`
+- For temporal extraction, use helpers such as `["get-month", ["field", 2058]]` and `["get-day-of-week", ["field", 2058]]`.
+ - Use `["get-hour", ["field", 2058]]` and `["get-quarter", ["field", 2058]]` for hour-of-day or quarter-number extraction.
+ - Do not invent helpers or bucket names like `month-of-year`, `quarter-of-year`, `hour-of-day`, or `dayofweek`.
+- When filtering extracted quarter values, use numeric quarter numbers `1`, `2`, `3`, `4`, not strings like `"Q1"`.
+- When you need the number of days or other units between two temporal values, use `["datetime-diff", left-temporal-expr, right-temporal-expr, "day"]`.
+ - Do not subtract dates with `-` to compute date differences.
+
+Use `with-fields` for row-level queries and `breakout` only for grouped results.
+If the result should be visualized, set `visualization.chart_type` to something appropriate like `table`, `bar`, `line`, `area`, `pie`, or `scalar`.
+
+Aggregation operators:
+- `["count"]` — Count all rows
+- `["sum", ["field", id]]` — Sum
+- `["avg", ["field", id]]` — Average
+- `["min", ["field", id]]` — Minimum
+- `["max", ["field", id]]` — Maximum
+- `["distinct", ["field", id]]` — Count distinct values
+- `["count-where", clause]` — Count rows matching a condition
+- `["sum-where", ["field", id], clause]` — Sum for matching rows
+- `["median", ["field", id]]` — Median
+- `["percentile", ["field", id], 0.95]` — Percentile
+- `["measure", id]` — Use a pre-defined measure
+
+Filter operators:
+- `["=", field, value]`, `["!=", field, value]`
+- `["<", field, value]`, `["<=", field, value]`, `[">", field, value]`, `[">=", field, value]`
+- `["between", field, low, high]`
+- `["contains", field, "text"]`, `["does-not-contain", field, "text"]`
+- `["starts-with", field, "prefix"]`, `["ends-with", field, "suffix"]`
+- `["is-null", field]`, `["not-null", field]`, `["is-empty", field]`, `["not-empty", field]`
+- `["in", field, [v1, v2, ...]]`, `["not-in", field, [v1, v2, ...]]`
+- `["time-interval", field, n, "unit"]` — Relative time window
+- `["and", clause1, clause2]`, `["or", clause1, clause2]`, `["not", clause]`
+- `["segment", id]` — Pre-defined segment
+
+Expression operators:
+- Arithmetic: `["+", a, b]`, `["-", a, b]`, `["*", a, b]`, `["/", a, b]`
+- Math: `["abs", x]`, `["ceil", x]`, `["floor", x]`, `["round", x]`, `["power", x, n]`, `["sqrt", x]`
+- String: `["concat", a, b]`, `["substring", s, start, len]`, `["replace", s, old, new]`, `["upper", s]`, `["lower", s]`, `["trim", s]`, `["length", s]`
+- Conditional: `["case", [[cond1, val1], [cond2, val2]], default]`
+- Coalesce: `["coalesce", a, b]`
+- Date extraction: `["get-year", field]`, `["get-month", field]`, `["get-day", field]`, `["get-quarter", field]`, `["get-hour", field]`, `["get-day-of-week", field]`
+- Date arithmetic: `["datetime-add", field, n, "unit"]`, `["datetime-diff", field1, field2, "unit"]`
+
+References:
+- `["field", id]` — Field by database ID
+- `["expression-ref", "Name"]` — Custom expression by name
+- `["aggregation-ref", 0]` — Aggregation result by 0-based index
+
+Temporal bucketing (for breakout):
+- `["with-temporal-bucket", ["field", id], "month"]`
+- Valid buckets: `year`, `quarter`, `month`, `week`, `day`, `hour`, `minute`, `second`, `day-of-week`
diff --git a/src/metabase/agent_api/api.clj b/src/metabase/agent_api/api.clj
index 892d19a718f..f57613ec13a 100644
--- a/src/metabase/agent_api/api.clj
+++ b/src/metabase/agent_api/api.clj
@@ -3,19 +3,18 @@
Endpoints are versioned (e.g., /v1/search) and use standard HTTP semantics."
(:require
[clojure.string :as str]
- [malli.core :as mc]
[metabase.agent-api.validation :as agent-api.validation]
+ [metabase.agent-lib.core :as agent-lib]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.api.macros.scope :as scope]
[metabase.api.routes.common :as api.routes.common]
[metabase.auth-identity.core :as auth-identity]
- [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.metabot.core :as metabot]
- [metabase.metabot.tools.deftool :as deftool]
+ [metabase.metabot.tools.construct :as metabot-construct]
[metabase.metabot.tools.entity-details :as entity-details]
[metabase.metabot.tools.field-stats :as field-stats]
- [metabase.metabot.tools.filters :as metabot-filters]
[metabase.metabot.tools.search :as metabot-search]
[metabase.metabot.util :as metabot.u]
[metabase.query-processor.core :as qp]
@@ -62,14 +61,25 @@
;; - Use :encode/api transformers to convert kebab-case data from internal functions
;; - Convert keyword enum values (like :table, :metric) to strings for JSON
+(mr/def ::field-type
+ "A data type for a field derived from Metabase's type hierarchy."
+ [:enum :boolean :date :datetime :time :number :string])
+
+(mr/def ::field-id
+ "Field id as accepted by agent_api endpoints — either a real app-DB field id (positive integer)
+ or a string alias for expression/aggregation columns."
+ [:or ::lib.schema.id/field :string])
+
(mr/def ::field
- "A field from a table or metric. The field_id format is '<prefix><entity-id>-<field-index>' where prefix indicates the source (t=table, c=metric) and index is the position in the entity's fields."
+ "A field from a table or metric. field_id is the real database field ID (integer) for concrete fields,
+ or a string alias for expression/aggregation columns."
[:map {:encode/api #(update-keys % metabot.u/safe->snake_case_en)}
- [:field_id :string]
+ [:field_id ::field-id]
[:name :string]
[:display_name :string]
+ [:type {:optional true} [:maybe ::field-type]]
[:description {:optional true} [:maybe :string]]
- [:base_type :string]
+ [:base_type {:optional true} [:maybe :string]]
[:effective_type {:optional true} [:maybe :string]]
[:semantic_type {:optional true} [:maybe :string]]
[:database_type {:optional true} [:maybe :string]]
@@ -87,7 +97,7 @@
[:type [:= :metric]]
[:name :string]
[:description {:optional true} [:maybe :string]]
- [:default_time_dimension_field_id {:optional true} [:maybe :string]]])
+ [:default_time_dimension_field_id {:optional true} [:maybe ::field-id]]])
(mr/def ::segment
"A predefined filter condition that can be applied to queries via the segment_id in filters."
@@ -143,7 +153,7 @@
[:type [:= :metric]]
[:name :string]
[:description {:optional true} [:maybe :string]]
- [:default_time_dimension_field_id {:optional true} [:maybe :string]]
+ [:default_time_dimension_field_id {:optional true} [:maybe ::field-id]]
[:verified {:optional true} [:maybe :boolean]]
[:queryable_dimensions {:optional true} [:maybe [:sequential ::field]]]
[:segments {:optional true} [:maybe [:sequential ::segment]]]])
@@ -170,7 +180,7 @@
(mr/def ::field-values
"Statistics and sample values for a specific field."
[:map {:encode/api #(update-keys % metabot.u/safe->snake_case_en)}
- [:field_id {:optional true} [:maybe :string]]
+ [:field_id {:optional true} [:maybe ::field-id]]
[:statistics {:optional true} [:maybe ::statistics]]
[:values {:optional true} [:maybe [:sequential :any]]]])
@@ -309,291 +319,62 @@
;;; ------------------------------------------------ Construct Query -------------------------------------------------
-;; Request schemas for the Agent API.
-;; These use snake_case keys for validation and OpenAPI generation,
-;; with :encode/tool-api-request transformers for converting to the internal format.
-
-(mr/def ::bucket
- (into [:enum {:error/message "Valid bucket"
- :encode/tool-api-request keyword}]
- (map name)
- lib.schema.temporal-bucketing/ordered-datetime-bucketing-units))
-
-(mr/def ::existence-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "is-null" "is-not-null"
- "string-is-empty" "string-is-not-empty"
- "is-true" "is-false"]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::temporal-extraction-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "year-equals" "year-not-equals"
- "quarter-equals" "quarter-not-equals"
- "month-equals" "month-not-equals"
- "day-of-week-equals" "day-of-week-not-equals"
- "hour-equals" "hour-not-equals"
- "minute-equals" "minute-not-equals"
- "second-equals" "second-not-equals"]]
- [:value :int]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::disjunctive-temporal-extraction-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "year-equals" "year-not-equals"
- "quarter-equals" "quarter-not-equals"
- "month-equals" "month-not-equals"
- "day-of-week-equals" "day-of-week-not-equals"
- "hour-equals" "hour-not-equals"
- "minute-equals" "minute-not-equals"
- "second-equals" "second-not-equals"]]
- [:values [:sequential :int]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::temporal-filter
- [:and
- [:map
- [:field_id :string]
- [:bucket {:optional true} ::bucket]
- [:operation [:enum {:encode/tool-api-request keyword}
- "equals" "not-equals"
- "greater-than" "greater-than-or-equal"
- "less-than" "less-than-or-equal"]]
- [:value [:or :string :int]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::disjunctive-temporal-filter
- [:and
- [:map
- [:field_id :string]
- [:bucket {:optional true} ::bucket]
- [:operation [:enum {:encode/tool-api-request keyword}
- "equals" "not-equals"
- "greater-than" "greater-than-or-equal"
- "less-than" "less-than-or-equal"]]
- [:values [:sequential [:or :string :int]]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::string-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "equals" "not-equals"
- "string-contains" "string-not-contains"
- "string-starts-with" "string-ends-with"]]
- [:value :string]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::disjunctive-string-date-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "equals" "not-equals"
- "string-contains" "string-not-contains"
- "string-starts-with" "string-ends-with"]]
- [:values [:sequential :string]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::numeric-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "equals" "not-equals"
- "greater-than" "greater-than-or-equal"
- "less-than" "less-than-or-equal"]]
- [:value [:or :int :double]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::disjunctive-numeric-filter
- [:and
- [:map
- [:field_id :string]
- [:operation [:enum {:encode/tool-api-request keyword}
- "equals" "not-equals"]]
- [:values [:sequential [:or :int :double]]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::segment-filter
- "Filter using a pre-defined segment."
- [:and
- [:map
- [:segment_id :int]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::filter
- [:or
- ::segment-filter
- ::existence-filter
- ::temporal-extraction-filter ::disjunctive-temporal-extraction-filter
- ::temporal-filter ::disjunctive-temporal-filter
- ::string-filter ::disjunctive-string-date-filter
- ::numeric-filter ::disjunctive-numeric-filter])
-
-(mr/def ::group-by
- [:and
- [:map
- [:field_id :string]
- [:field_granularity {:optional true}
- [:maybe [:enum {:encode/tool-api-request keyword}
- "minute", "hour" "day" "week" "month" "quarter" "year" "day-of-week"]]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::count-aggregation
- "Count aggregation — counts rows, no field_id needed.
- Use sort_order to order results by this aggregation ('asc' or 'desc')."
- [:and
- [:map
- [:function [:= {:encode/tool-api-request keyword} "count"]]
- [:bucket {:optional true} ::bucket]
- [:sort_order {:optional true} [:maybe [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::field-aggregation
- "Aggregation using a field and function. field_id is required.
- Use sort_order to order results by this aggregation ('asc' or 'desc')."
- [:and
- [:map
- [:field_id :string]
- [:bucket {:optional true} ::bucket]
- [:sort_order {:optional true} [:maybe [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]
- [:function [:enum {:encode/tool-api-request keyword}
- "avg" "count-distinct" "max" "min" "sum"]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::measure-aggregation
- "Aggregation using a pre-defined measure."
- [:and
- [:map
- [:measure_id :int]
- [:sort_order {:optional true} [:maybe [:enum {:encode/tool-api-request keyword} "asc" "desc"]]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::aggregation
- "Aggregation — count (field optional), field-based (field required), or measure-based."
- [:or ::count-aggregation ::field-aggregation ::measure-aggregation])
-
-(mr/def ::field
- [:and
- [:map
- [:field_id :string]
- [:bucket {:optional true} ::bucket]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::order-by
- "Order by item specifying a field and sort direction."
- [:map
- [:field ::field]
- [:direction [:enum {:encode/tool-api-request keyword} "asc" "desc"]]])
-
-(mr/def ::construct-query-table-request
- "Request schema for constructing a query from a table.
-
- Query components:
- - filters: Filter conditions to apply
- - fields: Specific fields to select (omit for all fields)
- - aggregations: Aggregation functions (sum, count, avg, etc.). Use sort_order on the aggregation to order by it.
- - group_by: Fields to group by, with optional temporal granularity
- - order_by: Order by regular fields only. To order by an aggregation result, use sort_order on the aggregation instead.
- - limit: Maximum rows to return"
- [:and
- [:map
- [:table_id ms/PositiveInt]
- [:filters {:optional true} [:maybe [:sequential ::filter]]]
- [:fields {:optional true} [:maybe [:sequential ::field]]]
- [:aggregations {:optional true} [:maybe [:sequential ::aggregation]]]
- [:group_by {:optional true} [:maybe [:sequential ::group-by]]]
- [:order_by {:optional true
- :description "Order by regular fields only. To order by aggregation results, use sort_order on the aggregation."}
- [:maybe [:sequential ::order-by]]]
- [:limit {:optional true} [:maybe ms/PositiveInt]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::construct-query-metric-request
- "Request schema for constructing a query from a metric.
- Only supports filters and group_by (aggregation is defined by the metric)."
- [:and
- [:map
- [:metric_id ms/PositiveInt]
- [:filters {:optional true} [:maybe [:sequential ::filter]]]
- [:group_by {:optional true} [:maybe [:sequential ::group-by]]]]
- [:map {:encode/tool-api-request #(update-keys % metabot.u/safe->kebab-case-en)}]])
-
-(mr/def ::construct-query-request
- "Request schema for /v1/construct-query. Accepts either table_id or metric_id."
- [:or ::construct-query-table-request ::construct-query-metric-request])
-
-(mr/def ::query-request
- "Request schema for /v1/query. Accepts construct params (table_id or metric_id) or a continuation_token."
- [:multi {:dispatch (fn [m]
- (cond
- (:continuation_token m) :continuation
- (:metric_id m) :metric
- :else :table))}
- [:continuation [:map [:continuation_token ms/NonBlankString]]]
- [:table ::construct-query-table-request]
- [:metric ::construct-query-metric-request]])
+(mr/def ::program-request
+ "Request body for /v2/construct-query and /v2/query.
+ An agent-lib structured program with `:source` and `:operations`. The top-level
+ `:source` must reference a database entity (`table`, `card`, `dataset`, or
+ `metric`); `context` and nested `program` sources are rejected at the HTTP
+ boundary by [[evaluate-program-for-execution]] because they require an
+ in-process evaluation context."
+ agent-lib/program-schema)
(mr/def ::construct-query-response
"Response containing a base64-encoded MBQL query for use with /v1/execute."
[:map
[:query ms/NonBlankString]])
-(defn- construct-query*
- "Shared query construction: encodes args, calls the appropriate tool fn, returns the raw lib query."
- [body]
- (if (:table_id body)
- (let [args (mc/encode ::construct-query-table-request body deftool/request-transformer)]
- (:query (check-tool-result (metabot-filters/query-datasource args))))
- (let [args (mc/encode ::construct-query-metric-request body deftool/request-transformer)]
- (:query (check-tool-result (metabot-filters/query-metric args))))))
-
-(defn- construct-table-query
- "Build a query from a table using the provided query components."
- [body]
- (let [body (cond-> body (not (:limit body)) (assoc :limit default-query-row-limit))
- query (construct-query* body)]
- {:query (-> query json/encode u/encode-base64)}))
-
-(defn- construct-metric-query
- "Build a query from a metric using filters and group_by."
- [body]
- {:query (-> (construct-query* body) json/encode u/encode-base64)})
-
-(api.macros/defendpoint :post "/v1/construct-query" :- ::construct-query-response
- "Construct an MBQL query from a table or metric.
-
- Returns a base64-encoded MBQL query that can be used with the query API.
-
- For tables, supports: filters, fields, aggregations, group_by, order_by, limit.
- For metrics, supports: filters, group_by (aggregation is defined by the metric)."
+(def ^:private allowed-program-source-types
+ "Top-level program source types that the HTTP boundary accepts. `context` and
+ nested `program` sources require an in-process evaluation context and are
+ rejected here."
+ #{"table" "card" "dataset" "metric"})
+
+(defn- evaluate-program-for-execution
+ "Resolve a program's source entity, evaluate the program via agent-lib, and return a
+ plain MBQL 5 query map. The JSON round-trip strips lib metadata so the query can be
+ serialized into a continuation token."
+ [program]
+ (let [source-type (get-in program [:source :type])]
+ (api/check (contains? allowed-program-source-types source-type)
+ [400 (str "top-level program source must be one of: "
+ (str/join ", " (sort allowed-program-source-types)))]))
+ (let [source-entity (metabot-construct/program-source->source-entity (:source program))
+ result (metabot-construct/execute-program source-entity nil program)
+ pmbql (get-in result [:structured-output :query])]
+ (json/decode+kw (json/encode pmbql))))
+
+(api.macros/defendpoint :post "/v2/construct-query" :- ::construct-query-response
+ "Construct an MBQL query from a structured agent-lib program.
+
+ The body is the program itself: a JSON object with `source` (identifying the
+ table/card/dataset/metric to query) and `operations` (an array of operator
+ tuples). Returns a base64-encoded MBQL query that can be executed via
+ /v1/execute. See the agent_api reference for the full program syntax."
{:scope metabot/agent-query-construct
:tool {:name "construct_query"
- :description (str "Construct a query against a Metabase table or metric. "
- "Returns an opaque query string that can be executed with execute_query.\n\n"
- "For table queries: provide table_id. "
- "Supports filters, fields, aggregations, group_by, order_by, and limit.\n\n"
- "For metric queries: provide metric_id. "
- "Supports only filters and group_by (aggregation is defined by the metric).\n\n"
- "Provide either table_id or metric_id, not both.")
+ :description (str "Construct a Metabase query from a structured program. "
+ "The body is a JSON object with `source` and `operations` keys "
+ "where source identifies the table/card/dataset/metric to query "
+ "(e.g. {\"type\": \"table\", \"id\": 42}) and operations is an "
+ "array of operator tuples (filter, aggregate, breakout, expression, "
+ "with-fields, order-by, limit, join, append-stage, etc.). Returns "
+ "an opaque query string that can be executed with execute_query.")
:annotations {:read-only? true :idempotent? true}}}
[_route-params
_query-params
- body :- ::construct-query-request]
- (if (:table_id body)
- (construct-table-query body)
- (construct-metric-query body)))
+ program :- ::program-request]
+ (let [query (evaluate-program-for-execution program)]
+ {:query (-> query json/encode u/encode-base64)}))
;;; ------------------------------------------------- Combined Query -------------------------------------------------
@@ -610,13 +391,13 @@
[token]
(-> token u/decode-base64 json/decode+kw))
-(defn- build-query-for-execution
- "Construct a MBQL 5 query map from table_id or metric_id params. Returns {:query <map> :limit <int>}.
- The JSON round-trip strips lib metadata so the query is a plain MBQL 5 map suitable for token serialization."
- [body]
- (let [limit (min (or (:limit body) default-query-row-limit) max-query-row-limit)
- query (construct-query* (assoc body :limit limit))]
- {:query (json/decode+kw (json/encode query)) :limit limit}))
+(defn- query-page-size
+ "Determine the per-page row limit for an evaluated MBQL 5 query, taking the
+ user-supplied limit (from the last stage) when present and capping at the
+ combined query endpoint's hard maximum."
+ [query-map]
+ (let [user-limit (-> query-map :stages last :limit)]
+ (min (or user-limit default-query-row-limit) max-query-row-limit)))
(defn- apply-page-to-query
"Apply :page clause to the last stage of a MBQL 5 query map."
@@ -641,24 +422,28 @@
:constraints {:max-results max-query-row-limit
:max-results-bare-rows max-query-row-limit}))
-(api.macros/defendpoint :post "/v1/query"
+(mr/def ::query-request
+ "Request body for /v2/query. Accepts either a structured program or a continuation_token."
+ [:multi {:dispatch (fn [m]
+ (if (:continuation_token m) :continuation :program))}
+ [:continuation [:map [:continuation_token ms/NonBlankString]]]
+ [:program ::program-request]])
+
+(api.macros/defendpoint :post "/v2/query"
:- (streaming-response/streaming-response-schema ::query-response)
- "Query a Metabase table or metric, or continue paginating a previous query.
+ "Execute a structured program and stream the results, with continuation-token pagination.
- Accepts either construct params (table_id/metric_id + filters, aggregations, etc.)
- or a continuation_token from a previous response. Returns results with column metadata
- and an optional continuation_token for fetching the next page."
+ Accepts either a program (same shape as /v2/construct-query) or a
+ `continuation_token` from a previous response. Returns results with column
+ metadata and an optional `continuation_token` for fetching the next page."
{:scope "agent:query"
:tool {:name "query"
- :description (str "Query a Metabase table or metric. Returns results with column metadata. "
- "If more rows are available, the response includes a continuation_token — "
- "pass it back to get the next page.\n\n"
- "For table queries: provide table_id. "
- "Supports filters, fields, aggregations, group_by, order_by, and limit.\n\n"
- "For metric queries: provide metric_id. "
- "Supports only filters and group_by (aggregation is defined by the metric).\n\n"
- "For pagination: provide only continuation_token from a previous response.\n\n"
- "Provide exactly one of table_id, metric_id, or continuation_token.")}}
+ :description (str "Execute a Metabase query from a structured program and return "
+ "results with column metadata. If more rows are available, the "
+ "response includes a continuation_token — pass it back to get the "
+ "next page.\n\n"
+ "The body is either a structured program (see construct_query) or "
+ "{\"continuation_token\": \"...\"} from a previous response.")}}
[_route-params
_query-params
body :- ::query-request]
@@ -666,8 +451,8 @@
(if-let [token (:continuation_token body)]
(let [{:keys [query pagination]} (decode-continuation-token token)]
{:query query :limit (:limit pagination) :page (:page pagination)})
- (let [{:keys [query limit]} (build-query-for-execution body)]
- {:query query :limit limit :page 1}))
+ (let [query (evaluate-program-for-execution body)]
+ {:query query :limit (query-page-size query) :page 1}))
mbql5-with-page (apply-page-to-query query page limit)]
(qp.streaming/streaming-response
[rff :api]
diff --git a/src/metabase/agent_api/reference.md b/src/metabase/agent_api/reference.md
index cbb7ee5460f..07676796559 100644
--- a/src/metabase/agent_api/reference.md
+++ b/src/metabase/agent_api/reference.md
@@ -14,21 +14,18 @@ Base path: /api/agent
(e.g., "Total Revenue"). Metrics are stored in collections and can be used
as a data source in the API. They have a fixed aggregation, but can be
filtered and grouped by their queryable dimensions. Use /v1/metric/{id} to
- inspect a metric's dimensions, and POST /v1/construct-query with `metric_id`
- to query one.
+ inspect a metric's dimensions, and POST /v2/construct-query with a program
+ whose `source` is `{"type": "metric", "id": <id>}` to query one.
- **Measures**: Lightweight, reusable aggregation expressions (e.g.,
`SUM(total)`) associated with a specific table. Unlike metrics, measures are
not standalone queries — they are building blocks that can be referenced in
- table queries via `measure_id` in the aggregations array. Discover available
- measures for a table via GET /v1/table/{id}?with-measures=true.
+ table queries via `["measure", id]` inside an `aggregate` operation. Discover
+ available measures for a table via GET /v1/table/{id}?with-measures=true.
- **Segments**: Pre-defined filter conditions (e.g., "Active Users") that can
- be applied to queries by referencing their segment_id.
-- **Field IDs**: Opaque string identifiers for columns, formatted as
- `<prefix><entity-id>-<column-index>`. Prefix is `t` for table fields and
- `c` for metric fields. Example: `t42-3` means the column at index 3 of table 42.
- Field IDs are returned by the table/metric detail endpoints and must be used
- as-is in query construction. They are positional — always fetch current
- entity details before constructing queries.
+ be applied to queries via `["filter", ["segment", id]]`.
+- **Field IDs**: Integer identifiers for database columns. These are the real
+ database field IDs returned by the table/metric detail endpoints. Use them
+ inside operator forms as `["field", N]`.
## Authentication
@@ -92,8 +89,7 @@ lacks access to a table or metric, the API returns 403.
Query parameters on GET endpoints use kebab-case (e.g., `with-fields`,
`with-related-tables`). JSON request and response bodies use snake_case (e.g.,
-`table_id`, `field_id`, `field_granularity`). This applies consistently across
-all endpoints.
+`table_id`, `field_id`). This applies consistently across all endpoints.
## Endpoints
@@ -133,7 +129,7 @@ Response:
"description": "All customer orders",
"fields": [
{
- "field_id": "t42-0",
+ "field_id": 301,
"name": "ID",
"display_name": "ID",
"description": "Primary key",
@@ -141,6 +137,13 @@ Response:
"semantic_type": "type/PK",
"database_type": "BIGINT",
"field_values": [1, 2, 3]
+ },
+ {
+ "field_id": 302,
+ "name": "TOTAL",
+ "type": "number",
+ "description": "Order total",
+ "database_type": "FLOAT"
}
],
"related_tables": [
@@ -158,7 +161,7 @@ Response:
"id": 10,
"type": "metric",
"name": "Total Revenue",
- "default_time_dimension_field_id": "c10-2"
+ "default_time_dimension_field_id": 305
}
],
"measures": [
@@ -191,11 +194,11 @@ Response:
"id": 10,
"name": "Total Revenue",
"description": "Sum of order totals",
- "default_time_dimension_field_id": "c10-2",
+ "default_time_dimension_field_id": 305,
"verified": true,
"queryable_dimensions": [
{
- "field_id": "c10-0",
+ "field_id": 305,
"name": "CREATED_AT",
"display_name": "Created At",
"base_type": "type/DateTime"
@@ -208,14 +211,15 @@ Response:
### GET /v1/table/{id}/field/{field-id}/values
### GET /v1/metric/{id}/field/{field-id}/values
-Get statistics and sample values for a field. Accepts optional `limit` query
+Get statistics and sample values for a field. The `field-id` is the integer
+field ID from the detail endpoints. Accepts optional `limit` query
parameter (default: 30).
Response:
```json
{
- "field_id": "t42-3",
+ "field_id": 302,
"statistics": {
"distinct_count": 200,
"percent_null": 0.02,
@@ -279,40 +283,53 @@ Response:
}
```
-### POST /v1/construct-query
+### POST /v2/construct-query
+
+Construct an MBQL query from a structured agent-lib program. Returns a
+base64-encoded query string to pass to `/v1/execute`.
+
+**Important**: All field IDs used in operations must come from the detail
+endpoints (`/v1/table/{id}` or `/v1/metric/{id}`). Always fetch entity details
+first.
-Construct a query from a table or metric. Returns a base64-encoded query to
-pass to /v1/execute.
+The request body **is** the program — there is no envelope. A program is a
+JSON object with two keys:
-**Important**: Field IDs used here must come from the detail endpoints
-(/v1/table/{id} or /v1/metric/{id}). Always fetch entity details first.
+- `source` — identifies the entity to query (`table`, `card`, `dataset`, or
+ `metric` plus an `id`).
+- `operations` — an ordered array of operator tuples to apply on top of the
+ source. Each operation is itself an array: `["operator", arg1, arg2, ...]`.
-#### Table query request
+The agent-lib backend automatically repairs small mistakes (operator aliases,
+casing, scalar wrapping) before validation, so you don't need to be perfectly
+precise about every detail — but the canonical operator names listed below
+will always work.
-All fields except `table_id` are optional:
+#### Request format
```json
{
- "table_id": 42,
- "filters": [],
- "fields": [{"field_id": "t42-0"}, {"field_id": "t42-1"}],
- "aggregations": [{"field_id": "t42-3", "function": "sum"}],
- "group_by": [{"field_id": "t42-5", "field_granularity": "month"}],
- "order_by": [{"field": {"field_id": "t42-1"}, "direction": "desc"}],
- "limit": 100
+ "source": {"type": "table", "id": 42},
+ "operations": [
+ ["filter", [">", ["field", 302], 100]],
+ ["aggregate", ["sum", ["field", 302]]],
+ ["breakout", ["with-temporal-bucket", ["field", 305], "month"]],
+ ["order-by", ["aggregation-ref", 0], "desc"],
+ ["limit", 100]
+ ]
}
```
-#### Metric query request
-
-Metrics have a pre-defined aggregation, so only filters and group_by are
-supported:
+For a metric source (the metric supplies its own aggregation, so additional
+aggregates are usually unnecessary):
```json
{
- "metric_id": 10,
- "filters": [{"field_id": "c10-2", "operation": "greater-than", "value": "2024-01-01"}],
- "group_by": [{"field_id": "c10-2", "field_granularity": "month"}]
+ "source": {"type": "metric", "id": 10},
+ "operations": [
+ ["filter", [">", ["field", 305], "2024-01-01"]],
+ ["breakout", ["with-temporal-bucket", ["field", 305], "month"]]
+ ]
}
```
@@ -322,177 +339,210 @@ supported:
{"query": "eyJkYXRhYmFzZSI6MSwi..."}
```
-#### Filter types
-
-Filters are polymorphic. The required fields depend on the operation.
-
-**Segment filter** — apply a pre-defined segment:
+#### Source types
+
+The top-level `source` must be one of these — `context` and nested `program`
+sources are reserved for in-process callers and are rejected at the HTTP
+boundary.
+
+| Type | Meaning |
+|-----------|---------------------------------------------------------------|
+| `table` | Query a database table directly (`id` is a table ID) |
+| `card` | Query a saved question (`id` is a card ID) |
+| `dataset` | Query a model (`id` is the model's card ID) |
+| `metric` | Query a metric, inheriting its aggregation and time dimension |
+
+#### Top-level operations
+
+Each operation is an array starting with the operator name. Operations are
+applied in order to the query stage produced from `source`.
+
+| Operator | Shape | Description |
+|------------------------|-------------------------------------------------|----------------------------------------------------------|
+| `filter` | `["filter", clause]` | Add a filter clause to the current stage |
+| `aggregate` | `["aggregate", agg-clause]` | Add an aggregation |
+| `breakout` | `["breakout", ref-or-bucketed]` | Add a grouping dimension |
+| `expression` | `["expression", "Name", expr]` | Define a named computed column |
+| `with-fields` | `["with-fields", [refs]]` | Restrict the returned columns |
+| `order-by` | `["order-by", ref]` or `["order-by", ref, dir]` | Sort by a field, expression-ref, or aggregation-ref |
+| `limit` | `["limit", N]` | Cap the number of returned rows |
+| `join` | `["join", join-clause]` | Join another table or card |
+| `append-stage` | `["append-stage"]` | Start a new query stage (e.g. for post-aggregation ops) |
+| `with-page` | `["with-page", {"page": N, "items": M}]` | Apply pagination on the current stage |
+
+The full canonical list lives in
+[src/metabase/agent_lib/capabilities/catalog/](../agent_lib/capabilities/catalog/).
+
+#### References
+
+References are nested operator forms used inside operations to point at fields,
+expressions, or earlier aggregations.
+
+| Form | Meaning |
+|-----------------------------------|--------------------------------------------------------|
+| `["field", N]` | Reference a database field by ID |
+| `["expression-ref", "Name"]` | Reference a named expression defined earlier |
+| `["aggregation-ref", N]` | Reference the Nth aggregation defined earlier (0-based)|
+| `["measure", N]` | Reference a pre-defined measure on the source entity |
+| `["with-temporal-bucket", r, b]` | Bucket a temporal field by `b` (`day`, `month`, …) |
+
+#### Filter operators (used inside `["filter", …]`)
+
+| Operator | Example |
+|----------------------|------------------------------------------------------|
+| `=`, `!=` | `["=", ["field", 101], "active"]` |
+| `<`, `<=`, `>`, `>=` | `[">", ["field", 302], 100]` |
+| `between` | `["between", ["field", 305], "2024-01-01", "2024-12-31"]` |
+| `in`, `not-in` | `["in", ["field", 302], [10, 20, 30]]` |
+| `is-null`, `not-null`| `["is-null", ["field", 101]]` |
+| `is-empty`, `not-empty` | `["is-empty", ["field", 101]]` |
+| `contains`, `does-not-contain` | `["contains", ["field", 303], "acme"]` |
+| `starts-with`, `ends-with` | `["starts-with", ["field", 303], "acme"]` |
+| `time-interval` | `["time-interval", ["field", 305], -7, "day"]` |
+| `and`, `or`, `not` | `["and", filter1, filter2]` |
+| `segment` | `["segment", 5]` — apply a pre-defined segment |
+
+#### Aggregation operators (used inside `["aggregate", …]`)
+
+`count`, `sum`, `avg`, `min`, `max`, `distinct`, `median`, `stddev`, `var`,
+`percentile`, `count-where`, `sum-where`, `distinct-where`, `share`,
+`cum-count`, `cum-sum`.
```json
-{"segment_id": 5}
+["aggregate", ["count"]]
+["aggregate", ["sum", ["field", 302]]]
+["aggregate", ["count-where", ["=", ["field", 101], "completed"]]]
```
-**Existence filters** — no value needed:
+#### Temporal helpers (commonly used in `expression` and `breakout`)
-| Operation | Description |
-|----------------------|------------------------|
-| is-null | Field is null |
-| is-not-null | Field is not null |
-| string-is-empty | String is empty |
-| string-is-not-empty | String is not empty |
-| is-true | Boolean is true |
-| is-false | Boolean is false |
+`get-year`, `get-quarter`, `get-month`, `get-week`, `get-day`,
+`get-day-of-week`, `get-hour`, `get-minute`, `datetime-add`, `datetime-diff`,
+`datetime-subtract`, `now`, `today`, `relative-datetime`, `absolute-datetime`,
+`with-temporal-bucket`, `convert-timezone`.
-```json
-{"field_id": "t42-0", "operation": "is-not-null"}
-```
+#### Worked examples
-**Temporal filters** — for date/datetime fields. Optional `bucket` specifies
-temporal bucketing granularity for the comparison. Accepts both truncation
-units (`minute`, `hour`, `day`, `week`, `month`, `quarter`, `year`) and
-extraction units (`day-of-week`, `day-of-month`, `day-of-year`,
-`week-of-year`, `month-of-year`, `quarter-of-year`, `hour-of-day`,
-`minute-of-hour`, `second-of-minute`). Note: this is a broader set of
-values than `field_granularity` in group_by, which only accepts truncation
-units.
-
-| Operation | Description |
-|-----------------------|---------------------------------|
-| equals | Equals value |
-| not-equals | Does not equal value |
-| greater-than | After (dates) / greater than |
-| greater-than-or-equal | On or after / greater or equal |
-| less-than | Before (dates) / less than |
-| less-than-or-equal | On or before / less or equal |
+**Top 5 customers by revenue**
```json
-{"field_id": "t42-5", "operation": "greater-than", "value": "2024-01-01"}
-```
-
-For multiple values, use `values` (array) instead of `value`:
-
-```json
-{"field_id": "t42-5", "operation": "equals", "values": ["2024-01-01", "2024-06-01"]}
+{
+ "source": {"type": "table", "id": 42},
+ "operations": [
+ ["aggregate", ["sum", ["field", 302]]],
+ ["breakout", ["field", 101]],
+ ["order-by", ["aggregation-ref", 0], "desc"],
+ ["limit", 5]
+ ]
+}
```
-**Temporal extraction filters** — filter by date component:
-
-| Operation | Value type |
-|------------------------|------------|
-| year-equals | int |
-| year-not-equals | int |
-| quarter-equals | int (1-4) |
-| quarter-not-equals | int |
-| month-equals | int (1-12) |
-| month-not-equals | int |
-| day-of-week-equals | int (1-7) |
-| day-of-week-not-equals | int |
-| hour-equals | int (0-23) |
-| hour-not-equals | int |
-| minute-equals | int (0-59) |
-| minute-not-equals | int |
-| second-equals | int (0-59) |
-| second-not-equals | int |
+**Conditional sum with a named expression**
```json
-{"field_id": "t42-5", "operation": "month-equals", "value": 12}
+{
+ "source": {"type": "table", "id": 42},
+ "operations": [
+ ["expression", "Discount", ["-", ["field", 302], ["field", 303]]],
+ ["aggregate", ["sum", ["expression-ref", "Discount"]]]
+ ]
+}
```
-For multiple values: `{"field_id": "t42-5", "operation": "month-equals", "values": [1, 2, 3]}`
-
-**String filters**:
-
-| Operation | Description |
-|---------------------|--------------------------------------|
-| equals | Exact match |
-| not-equals | Does not match |
-| string-contains | Contains substring |
-| string-not-contains | Does not contain substring |
-| string-starts-with | Starts with prefix |
-| string-ends-with | Ends with suffix |
+**Orders per month using a metric**
```json
-{"field_id": "t42-2", "operation": "string-contains", "value": "acme"}
+{
+ "source": {"type": "metric", "id": 10},
+ "operations": [
+ ["breakout", ["with-temporal-bucket", ["field", 305], "month"]]
+ ]
+}
```
-For multiple values: `{"field_id": "t42-2", "operation": "equals", "values": ["A", "B"]}`
-
-**Numeric filters**:
-
-| Operation | Description |
-|-----------------------|------------------------|
-| equals | Equals value |
-| not-equals | Does not equal value |
-| greater-than | Greater than |
-| greater-than-or-equal | Greater than or equal |
-| less-than | Less than |
-| less-than-or-equal | Less than or equal |
+**Multi-stage: filter on an aggregated value**
```json
-{"field_id": "t42-3", "operation": "greater-than", "value": 100}
+{
+ "source": {"type": "table", "id": 42},
+ "operations": [
+ ["aggregate", ["sum", ["field", 302]]],
+ ["breakout", ["field", 101]],
+ ["append-stage"],
+ ["filter", [">", ["aggregation-ref", 0], 1000]]
+ ]
+}
```
-For multiple values: `{"field_id": "t42-3", "operation": "equals", "values": [10, 20, 30]}`
+#### Error responses
-#### Aggregations
-
-Field-based aggregation — `function` is required:
-
-| Function | Description |
-|----------------|----------------------------|
-| avg | Average |
-| count | Count of rows |
-| count-distinct | Count of distinct values |
-| max | Maximum value |
-| min | Minimum value |
-| sum | Sum |
+Validation, repair, and resolution errors are returned as `400 Bad Request`
+with a structured JSON body:
```json
-{"field_id": "t42-3", "function": "sum"}
+{
+ "status-code": 400,
+ "error": "invalid-generated-program",
+ "path": "operations[2].field_id",
+ "details": "operations[2].field_id: field 12345 is not accessible",
+ "recovery": {
+ "available": ["field 1001", "field 1002"],
+ "suggestion": "Did you mean field 1001?"
+ }
+}
```
-For `count`, omit `field_id` (count operates on rows, not a specific field):
+A non-existent table, card, or metric in `source` returns `404 Not Found`.
-```json
-{"function": "count"}
-```
+### POST /v2/query
-To sort by an aggregation result, use `sort_order` on the aggregation itself
-(not `order_by`):
+Combined construct-and-execute endpoint with built-in pagination via
+continuation tokens.
-```json
-{"field_id": "t42-3", "function": "sum", "sort_order": "desc"}
-```
+The body is either:
+- A program (same shape as `/v2/construct-query`), **or**
+- `{"continuation_token": "..."}` returned from a previous response.
-Measure-based aggregation — uses a pre-defined measure:
+Pagination is automatic. The per-page row limit is taken from your program's
+`["limit", N]` operation if present, otherwise defaults to 200, and is hard-
+capped at 200 rows for memory and LLM-context safety. If the page is full and
+more rows may exist, the response includes a `continuation_token` you can
+post back to fetch the next page.
```json
-{"measure_id": 5, "sort_order": "asc"}
+{
+ "source": {"type": "table", "id": 42},
+ "operations": [
+ ["order-by", ["field", 101]],
+ ["limit", 50]
+ ]
+}
```
-#### Group by
+Response (HTTP 202, streaming):
```json
-{"field_id": "t42-5", "field_granularity": "month"}
+{
+ "status": "completed",
+ "data": {
+ "cols": [{"name": "ID", "base_type": "type/Integer", "display_name": "ID"}],
+ "rows": [[1], [2], [3], [...]]
+ },
+ "row_count": 50,
+ "running_time": 87,
+ "continuation_token": "eyJxdWVyeSI6ey..."
+}
```
-`field_granularity` is optional and controls temporal grouping granularity.
-Valid values: `minute`, `hour`, `day`, `week`, `month`, `quarter`, `year`,
-`day-of-week`. Note: this is a smaller set than the `bucket` field on
-filters, which also accepts extraction units like `day-of-month` and
-`hour-of-day`.
-
-#### Order by
-
-Order by a field (not an aggregation — use `sort_order` for that):
+To fetch the next page:
```json
-{"field": {"field_id": "t42-1"}, "direction": "desc"}
+{"continuation_token": "eyJxdWVyeSI6ey..."}
```
### POST /v1/execute
-Execute a query returned by /v1/construct-query.
+Execute a query returned by /v2/construct-query.
**Important: streaming response.** This endpoint streams results, so the HTTP
status code (202) is sent before query execution completes. A 202 status does
@@ -547,7 +597,6 @@ On failure:
```
Row limits:
-- Default: 100 rows (applied when no `limit` is specified in construct-query for tables)
- Simple queries (no aggregation): 2000 rows max
- Aggregated queries: 10000 rows max
@@ -558,19 +607,21 @@ Row limits:
understand the schema
3. **Explore field values** — GET /v1/table/{id}/field/{field-id}/values if
you need to know valid filter values or field statistics
-4. **Build query** — POST /v1/construct-query with filters, aggregations,
- group_by, etc.
-5. **Execute** — POST /v1/execute with the base64-encoded query
-6. **Iterate** — Adjust filters/aggregations and repeat steps 4-5
+4. **Build query** — POST /v2/construct-query with a structured program
+ (`source` + `operations`)
+5. **Execute** — POST /v1/execute with the base64-encoded query, or use
+ POST /v2/query to construct and execute in one round-trip with pagination
+6. **Iterate** — Adjust the program and repeat steps 4-5
## Error handling
-| HTTP Status | Meaning |
-|-------------|------------------------------------------|
-| 200 | Success (GET endpoints, construct-query) |
-| 202 | Success (execute — streaming response) |
-| 401 | Authentication failure |
-| 403 | Insufficient permissions |
-| 404 | Entity not found |
+| HTTP Status | Meaning |
+|-------------|---------------------------------------------------------------|
+| 200 | Success (GET endpoints, construct-query) |
+| 202 | Success (execute / query — streaming response) |
+| 400 | Invalid program (validation, repair, or resolution failure) |
+| 401 | Authentication failure |
+| 403 | Insufficient permissions |
+| 404 | Entity not found |
---
diff --git a/src/metabase/agent_lib/capabilities.clj b/src/metabase/agent_lib/capabilities.clj
new file mode 100644
index 00000000000..266f97ebfd5
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities.clj
@@ -0,0 +1,120 @@
+(ns metabase.agent-lib.capabilities
+ "Shared capability registry for the structured MBQL program path."
+ (:require
+ [metabase.agent-lib.capabilities.catalog :as catalog]
+ [metabase.agent-lib.capabilities.derived :as derived]
+ [metabase.agent-lib.capabilities.synthetic :as capabilities.synthetic]
+ [metabase.util.malli.registry :as mr]))
+
+(set! *warn-on-reflection* true)
+
+(def raw-capability-catalog
+ "Ordered structured-program capability metadata used for repair, validation, and prompts."
+ catalog/raw-capability-catalog)
+
+(def ^:private guidance-keys
+ [:prompt-forms
+ :prompt-notes
+ :shape
+ :example
+ :retry-shape
+ :retry-example
+ :retry-notes])
+
+(def ^:private capability-guidance-schema
+ [:map {:closed true}
+ [:prompt-forms {:optional true} [:sequential :string]]
+ [:prompt-notes {:optional true} [:sequential :string]]
+ [:shape {:optional true} :string]
+ [:example {:optional true} :string]
+ [:retry-shape {:optional true} :string]
+ [:retry-example {:optional true} :string]
+ [:retry-notes {:optional true} [:sequential :string]]])
+
+(def ^:private capability-entry-schema
+ [:map {:closed true}
+ [:op symbol?]
+ [:kind [:enum :top-level :nested]]
+ [:binding {:optional true} ifn?]
+ [:group {:optional true} keyword?]
+ [:arities {:optional true} [:set int?]]
+ [:runtime-provided? {:optional true} boolean?]
+ [:guidance {:optional true} capability-guidance-schema]])
+
+(defn- split-guidance
+ [capability]
+ (let [guidance (not-empty (select-keys capability guidance-keys))]
+ (cond-> (apply dissoc capability guidance-keys)
+ guidance (assoc :guidance guidance))))
+
+(def capability-catalog
+ "Ordered structured-program capability metadata keyed into structural fields plus nested guidance."
+ (mapv split-guidance raw-capability-catalog))
+
+(defn- validate-capability-catalog!
+ []
+ (doseq [capability capability-catalog]
+ (when-not (mr/validate capability-entry-schema capability)
+ (throw (ex-info "Invalid structured capability catalog entry."
+ {:capability capability
+ :schema capability-entry-schema
+ :explain (mr/explain capability-entry-schema capability)})))))
+
+(validate-capability-catalog!)
+
+(defn guidance
+ "Return prompt and retry guidance metadata for a capability entry."
+ [capability]
+ (derived/guidance capability))
+
+(defn prompt-forms
+ "Return documented prompt forms for a capability entry."
+ [capability]
+ (derived/prompt-forms capability))
+
+(defn prompt-notes
+ "Return documented prompt notes for a capability entry."
+ [capability]
+ (derived/prompt-notes capability))
+
+(defn guidance-shape
+ "Return the documented canonical call shape for a capability entry."
+ [capability]
+ (derived/guidance-shape capability))
+
+(defn guidance-example
+ "Return the documented example program fragment for a capability entry."
+ [capability]
+ (derived/guidance-example capability))
+
+(def capability-by-op
+ "Capability metadata keyed by operator symbol."
+ (derived/capability-by-op capability-catalog))
+
+(def fixed-arities
+ "Structured helper arities keyed by operator symbol."
+ (derived/fixed-arities capability-catalog))
+
+(def trusted-helper-bindings
+ "Trusted helper implementations keyed by operator symbol."
+ (derived/trusted-helper-bindings capability-catalog))
+
+(def query-transform-symbols
+ "Top-level structured query-transform operator symbols."
+ (derived/query-transform-symbols capability-catalog))
+
+(def helper-symbols
+ "All structured helper symbols accepted by the evaluator."
+ (derived/helper-symbols capability-catalog capabilities.synthetic/synthetic-helper-symbols))
+
+(def nested-operator-values
+ "String names for nested structured helper operators."
+ (derived/nested-operator-values capability-catalog))
+
+(def top-level-operator-values
+ "String names for top-level structured operations."
+ (derived/top-level-operator-values capability-catalog))
+
+(def operator-guidance-catalog
+ "Retry guidance keyed by canonical operator name."
+ (derived/operator-guidance-catalog capability-catalog))
diff --git a/src/metabase/agent_lib/capabilities/catalog.clj b/src/metabase/agent_lib/capabilities/catalog.clj
new file mode 100644
index 00000000000..acd26dac3fa
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog.clj
@@ -0,0 +1,29 @@
+(ns metabase.agent-lib.capabilities.catalog
+ "Aggregated structured-program capability catalog split into smaller
+ review-friendly groups."
+ (:require
+ [metabase.agent-lib.capabilities.catalog.aggregation :as aggregation]
+ [metabase.agent-lib.capabilities.catalog.expressions :as expressions]
+ [metabase.agent-lib.capabilities.catalog.filtering :as filtering]
+ [metabase.agent-lib.capabilities.catalog.joins :as joins]
+ [metabase.agent-lib.capabilities.catalog.ordering :as ordering]
+ [metabase.agent-lib.capabilities.catalog.sources :as sources]
+ [metabase.agent-lib.capabilities.catalog.top-level :as top-level]))
+
+(set! *warn-on-reflection* true)
+
+(def grouped-capability-catalog
+ "Capability entries grouped into review-friendly sections."
+ [{:group :top-level :capabilities top-level/capabilities}
+ {:group :sources :capabilities sources/capabilities}
+ {:group :filtering :capabilities filtering/capabilities}
+ {:group :aggregation :capabilities aggregation/capabilities}
+ {:group :breakout-ordering :capabilities ordering/capabilities}
+ {:group :expressions :capabilities expressions/capabilities}
+ {:group :joins :capabilities joins/capabilities}])
+
+(def raw-capability-catalog
+ "Ordered structured-program capability metadata assembled from the grouped catalog."
+ (into []
+ (mapcat :capabilities)
+ grouped-capability-catalog))
diff --git a/src/metabase/agent_lib/capabilities/catalog/aggregation.clj b/src/metabase/agent_lib/capabilities/catalog/aggregation.clj
new file mode 100644
index 00000000000..b5c2d13f70e
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/aggregation.clj
@@ -0,0 +1,30 @@
+(ns metabase.agent-lib.capabilities.catalog.aggregation
+ "Aggregation structured capability entries.
+
+ Catalog entries are the source of truth for which operators agent-lib supports
+ at runtime. The construct_notebook_query.md prompt is hand-curated guidance for
+ the LLM and is not generated from this catalog — keep both in sync when adding
+ or removing helpers."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Aggregation helper capability entries."
+ [{:op 'count, :binding #'lib/count, :kind :nested, :group :aggregation, :arities #{0}, :prompt-forms ["[\"count\"]"]}
+ {:op 'sum, :binding #'lib/sum, :kind :nested, :group :aggregation, :prompt-forms ["[\"sum\", [\"field\", 201]]"], :prompt-notes ["Aggregation helpers take field refs like `[\"sum\", [\"field\", 201]]`. Do not pass bare numeric ids like `[\"sum\", 201]`."]}
+ {:op 'avg, :binding #'lib/avg, :kind :nested, :group :aggregation, :prompt-forms ["[\"avg\", [\"field\", 201]]"]}
+ {:op 'min, :binding #'lib/min, :kind :nested, :group :aggregation, :prompt-forms ["[\"min\", [\"field\", 201]]"]}
+ {:op 'max, :binding #'lib/max, :kind :nested, :group :aggregation, :prompt-forms ["[\"max\", [\"field\", 201]]"]}
+ {:op 'distinct, :binding #'lib/distinct, :kind :nested, :group :aggregation, :prompt-forms ["[\"distinct\", [\"field\", 201]]"]}
+ {:op 'median, :binding #'lib/median, :kind :nested, :group :aggregation, :prompt-forms ["[\"median\", [\"field\", 201]]"]}
+ {:op 'stddev, :binding #'lib/stddev, :kind :nested, :group :aggregation, :prompt-forms ["[\"stddev\", [\"field\", 201]]"]}
+ {:op 'var, :binding #'lib/var, :kind :nested, :group :aggregation, :prompt-forms ["[\"var\", [\"field\", 201]]"]}
+ {:op 'percentile, :binding #'lib/percentile, :kind :nested, :group :aggregation, :prompt-forms ["[\"percentile\", [\"field\", 201], 0.95]"]}
+ {:op 'count-where, :binding #'lib/count-where, :kind :nested, :group :aggregation, :prompt-forms ["[\"count-where\", [\"=\", [\"field\", 101], \"completed\"]]"], :shape "[\"count-where\", clause]", :example "[\"count-where\", [\"=\", [\"field\", 2191], true]]"}
+ {:op 'sum-where, :binding #'lib/sum-where, :kind :nested, :group :aggregation, :prompt-forms ["[\"sum-where\", [\"field\", 201], [\"=\", [\"field\", 101], \"completed\"]]"], :shape "[\"sum-where\", [\"field\", 2203], clause]", :example "[\"sum-where\", [\"field\", 2203], [\"=\", [\"field\", 2191], true]]"}
+ {:op 'distinct-where, :binding #'lib/distinct-where, :kind :nested, :group :aggregation, :prompt-forms ["[\"distinct-where\", [\"field\", 201], [\"=\", [\"field\", 101], \"completed\"]]"], :shape "[\"distinct-where\", [\"field\", 2203], clause]", :example "[\"distinct-where\", [\"field\", 2203], [\"=\", [\"field\", 2191], true]]"}
+ {:op 'share, :binding #'lib/share, :kind :nested, :group :aggregation, :prompt-forms ["[\"share\", [\"=\", [\"field\", 101], \"completed\"]]"], :shape "[\"share\", clause]", :example "[\"share\", [\"=\", [\"field\", 2191], true]]"}
+ {:op 'cum-count, :binding #'lib/cum-count, :kind :nested, :group :aggregation, :prompt-forms ["[\"cum-count\"]"]}
+ {:op 'cum-sum, :binding #'lib/cum-sum, :kind :nested, :group :aggregation, :prompt-forms ["[\"cum-sum\", [\"field\", 201]]"]}])
diff --git a/src/metabase/agent_lib/capabilities/catalog/expressions.clj b/src/metabase/agent_lib/capabilities/catalog/expressions.clj
new file mode 100644
index 00000000000..63110785d92
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/expressions.clj
@@ -0,0 +1,60 @@
+(ns metabase.agent-lib.capabilities.catalog.expressions
+ "Expression structured capability entries."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Expression helper capability entries."
+ [{:op '+, :binding #'lib/+, :kind :nested, :group :expressions, :prompt-forms ["[\"+\", [\"field\", 201], [\"field\", 202]]"]}
+ {:op '-, :binding #'lib/-, :kind :nested, :group :expressions, :prompt-forms ["[\"-\", [\"field\", 201], [\"field\", 202]]"]}
+ {:op '*, :binding #'lib/*, :kind :nested, :group :expressions, :prompt-forms ["[\"*\", [\"field\", 201], [\"field\", 202]]"]}
+ {:op '/, :binding #'lib//, :kind :nested, :group :expressions, :prompt-forms ["[\"/\", [\"field\", 201], [\"field\", 202]]"]}
+ {:op 'case, :binding #'lib/case, :kind :nested, :group :expressions, :arities #{1 2}, :prompt-forms ["[\"case\", [[[\"<\", [\"field\", 201], 50], \"small\"], [[\"<\", [\"field\", 201], 200], \"medium\"]]]" "[\"case\", [[[\"<\", [\"field\", 201], 50], \"small\"], [[\"<\", [\"field\", 201], 200], \"medium\"]], \"large\"]"], :shape "[\"case\", [[condition, value], ...]] or [\"case\", [[condition, value], ...], fallback]", :example "[\"case\", [[[\"<\", [\"field\", 201], 50], \"small\"], [[\"<\", [\"field\", 201], 200], \"medium\"]], \"large\"]", :retry-shape "[\"case\", [[condition, value], ...]] or [\"case\", [[condition, value], ...], fallback]", :retry-example "[\"case\", [[[\"<\", [\"field\", 201], 50], \"small\"], [[\"<\", [\"field\", 201], 200], \"medium\"]], \"large\"]", :prompt-notes ["`case` uses the fallback value itself as the optional third argument. Do not wrap it in `{\"default\": ...}`." "If you want no else branch or a null result, omit the third `case` argument entirely."], :retry-notes ["Do not wrap the fallback in `{\"default\": ...}`." "Omit the third argument instead of using `null` when you want no else branch."]}
+ {:op 'coalesce, :binding #'lib/coalesce, :kind :nested, :group :expressions, :prompt-forms ["[\"coalesce\", [\"field\", 101], [\"field\", 102]]"]}
+ {:op 'concat, :binding #'lib/concat, :kind :nested, :group :expressions, :prompt-forms ["[\"concat\", [\"field\", 101], \" \", [\"field\", 102]]"]}
+ {:op 'substring, :binding #'lib/substring, :kind :nested, :group :expressions, :prompt-forms ["[\"substring\", [\"field\", 101], 1, 3]"]}
+ {:op 'replace, :binding #'lib/replace, :kind :nested, :group :expressions, :prompt-forms ["[\"replace\", [\"field\", 101], \"old\", \"new\"]"]}
+ {:op 'upper, :binding #'lib/upper, :kind :nested, :group :expressions, :prompt-forms ["[\"upper\", [\"field\", 101]]"]}
+ {:op 'lower, :binding #'lib/lower, :kind :nested, :group :expressions, :prompt-forms ["[\"lower\", [\"field\", 101]]"]}
+ {:op 'trim, :binding #'lib/trim, :kind :nested, :group :expressions, :prompt-forms ["[\"trim\", [\"field\", 101]]"]}
+ {:op 'ltrim, :binding #'lib/ltrim, :kind :nested, :group :expressions, :prompt-forms ["[\"ltrim\", [\"field\", 101]]"]}
+ {:op 'rtrim, :binding #'lib/rtrim, :kind :nested, :group :expressions, :prompt-forms ["[\"rtrim\", [\"field\", 101]]"]}
+ {:op 'length, :binding #'lib/length, :kind :nested, :group :expressions, :prompt-forms ["[\"length\", [\"field\", 101]]"]}
+ {:op 'regex-match-first, :binding #'lib/regex-match-first, :kind :nested, :group :expressions, :prompt-forms ["[\"regex-match-first\", [\"field\", 101], \"pattern\"]"]}
+ {:op 'split-part, :binding #'lib/split-part, :kind :nested, :group :expressions, :prompt-forms ["[\"split-part\", [\"field\", 101], \",\", 1]"]}
+ {:op 'collate, :binding #'lib/collate, :kind :nested, :group :expressions, :prompt-forms ["[\"collate\", [\"field\", 101], \"case-insensitive\"]"]}
+ {:op 'abs, :binding #'lib/abs, :kind :nested, :group :expressions, :prompt-forms ["[\"abs\", [\"field\", 201]]"]}
+ {:op 'ceil, :binding #'lib/ceil, :kind :nested, :group :expressions, :prompt-forms ["[\"ceil\", [\"field\", 201]]"]}
+ {:op 'floor, :binding #'lib/floor, :kind :nested, :group :expressions, :prompt-forms ["[\"floor\", [\"field\", 201]]"]}
+ {:op 'round, :binding #'lib/round, :kind :nested, :group :expressions, :prompt-forms ["[\"round\", [\"field\", 201]]"]}
+ {:op 'power, :binding #'lib/power, :kind :nested, :group :expressions, :prompt-forms ["[\"power\", [\"field\", 201], 2]"]}
+ {:op 'sqrt, :binding #'lib/sqrt, :kind :nested, :group :expressions, :prompt-forms ["[\"sqrt\", [\"field\", 201]]"]}
+ {:op 'log, :binding #'lib/log, :kind :nested, :group :expressions, :prompt-forms ["[\"log\", [\"field\", 201]]"]}
+ {:op 'exp, :binding #'lib/exp, :kind :nested, :group :expressions, :prompt-forms ["[\"exp\", [\"field\", 201]]"]}
+ {:op 'today, :binding #'lib/today, :kind :nested, :group :expressions, :arities #{0}, :prompt-forms ["[\"today\"]"]}
+ {:op 'now, :binding #'lib/now, :kind :nested, :group :expressions, :arities #{0}, :prompt-forms ["[\"now\"]"]}
+ {:op 'date, :binding #'lib/date, :kind :nested, :group :expressions, :prompt-forms ["[\"date\", \"2025-01-01\"]"]}
+ {:op 'datetime, :binding #'lib/datetime, :kind :nested, :group :expressions, :prompt-forms ["[\"datetime\", \"2025-01-01T00:00:00\"]"]}
+ {:op 'interval, :binding #'lib/interval, :kind :nested, :group :expressions, :prompt-forms ["[\"interval\", 7, \"day\"]"]}
+ {:op 'time, :binding #'lib/time, :kind :nested, :group :expressions, :prompt-forms ["[\"time\", \"09:00\"]"]}
+ {:op 'relative-datetime, :binding #'lib/relative-datetime, :kind :nested, :group :expressions, :prompt-forms ["[\"relative-datetime\", -7, \"day\"]"]}
+ {:op 'absolute-datetime, :binding #'lib/absolute-datetime, :kind :nested, :group :expressions, :prompt-forms ["[\"absolute-datetime\", \"2025-01-01\", \"day\"]"], :shape "[\"absolute-datetime\", \"2024-01-01\", \"day\"]", :example "[\"absolute-datetime\", \"2024-01-01\", \"day\"]"}
+ {:op 'datetime-add, :binding #'lib/datetime-add, :kind :nested, :group :expressions, :prompt-forms ["[\"datetime-add\", [\"field\", 302], 1, \"month\"]"]}
+ {:op 'datetime-diff, :binding (fn [x y unit] (lib/expression-clause :datetime-diff [x y unit] nil)), :kind :nested, :group :expressions, :prompt-forms ["[\"datetime-diff\", [\"field\", 302], [\"field\", 303], \"day\"]"], :shape "[\"datetime-diff\", [\"field\", 302], [\"field\", 303], \"day\"]", :example "[\"datetime-diff\", [\"field\", 302], [\"field\", 303], \"day\"]", :prompt-notes ["Use `datetime-diff` when you need the number of days, weeks, months, or other units between two temporal values." "Do not subtract dates with `-` when the goal is a time difference; use `datetime-diff` instead."]}
+ {:op 'datetime-subtract, :binding #'lib/datetime-subtract, :kind :nested, :group :expressions, :prompt-forms ["[\"datetime-subtract\", [\"now\"], [\"field\", 302], \"day\"]"]}
+ {:op 'get-year, :binding #'lib/get-year, :kind :nested, :group :expressions, :prompt-forms ["[\"get-year\", [\"field\", 302]]"]}
+ {:op 'get-quarter, :binding #'lib/get-quarter, :kind :nested, :group :expressions, :prompt-forms ["[\"get-quarter\", [\"field\", 302]]"]}
+ {:op 'get-month, :binding #'lib/get-month, :kind :nested, :group :expressions, :prompt-forms ["[\"get-month\", [\"field\", 302]]"]}
+ {:op 'get-week, :binding #'lib/get-week, :kind :nested, :group :expressions, :prompt-forms ["[\"get-week\", [\"field\", 302]]"]}
+ {:op 'get-day, :binding #'lib/get-day, :kind :nested, :group :expressions, :prompt-forms ["[\"get-day\", [\"field\", 302]]"]}
+ {:op 'get-day-of-week, :binding #'lib/get-day-of-week, :kind :nested, :group :expressions, :prompt-forms ["[\"get-day-of-week\", [\"field\", 302]]"]}
+ {:op 'get-hour, :binding #'lib/get-hour, :kind :nested, :group :expressions, :prompt-forms ["[\"get-hour\", [\"field\", 302]]"]}
+ {:op 'get-minute, :binding #'lib/get-minute, :kind :nested, :group :expressions, :prompt-forms ["[\"get-minute\", [\"field\", 302]]"]}
+ {:op 'get-second, :binding #'lib/get-second, :kind :nested, :group :expressions, :prompt-forms ["[\"get-second\", [\"field\", 302]]"]}
+ {:op 'convert-timezone, :binding #'lib/convert-timezone, :kind :nested, :group :expressions, :prompt-forms ["[\"convert-timezone\", [\"field\", 302], \"UTC\", \"America/Los_Angeles\"]"]}
+ {:op 'integer, :binding #'lib/integer, :kind :nested, :group :expressions, :prompt-forms ["[\"integer\", [\"field\", 201]]"]}
+ {:op 'float, :binding #'lib/float, :kind :nested, :group :expressions, :prompt-forms ["[\"float\", [\"field\", 201]]"]}
+ {:op 'text, :binding #'lib/text, :kind :nested, :group :expressions, :prompt-forms ["[\"text\", [\"field\", 101]]"]}
+ {:op 'offset, :binding #'lib/offset, :kind :nested, :group :expressions, :prompt-forms ["[\"offset\", [\"sum\", [\"field\", 2056]], -1]"], :shape "[\"offset\", [\"sum\", [\"field\", 2056]], -1]", :example "[\"offset\", [\"sum\", [\"field\", 2056]], -1]", :prompt-notes ["`offset` is a windowed aggregation over an aggregation or expression." "For grouped previous-period metrics like \"previous month's total\", add `offset` as another `aggregate` in the same stage as the base aggregation and breakout." "Do not add `append-stage` just to calculate a previous-period total with `offset`."] :retry-shape "For previous-period grouped metrics, keep `offset` in the same grouped stage as the base aggregation and breakout." :retry-example (str "[\n" " [\"aggregate\", [\"sum\", [\"field\", 2056]]],\n" " [\"aggregate\", [\"offset\", [\"sum\", [\"field\", 2056]], -1]],\n" " [\"breakout\", [\"with-temporal-bucket\", [\"field\", 2058], \"month\"]],\n" " [\"order-by\", [\"asc\", [\"with-temporal-bucket\", [\"field\", 2058], \"month\"]]]\n" "]")}])
diff --git a/src/metabase/agent_lib/capabilities/catalog/filtering.clj b/src/metabase/agent_lib/capabilities/catalog/filtering.clj
new file mode 100644
index 00000000000..8742c1f1b1f
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/filtering.clj
@@ -0,0 +1,33 @@
+(ns metabase.agent-lib.capabilities.catalog.filtering
+ "Filtering structured capability entries."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Filtering helper capability entries."
+ [{:op '=, :binding #'lib/=, :kind :nested, :group :filtering, :prompt-forms ["[\"=\", [\"field\", 101], \"value\"]"]}
+ {:op '!=, :binding #'lib/!=, :kind :nested, :group :filtering, :prompt-forms ["[\"!=\", [\"field\", 101], \"value\"]"]}
+ {:op '<, :binding #'lib/<, :kind :nested, :group :filtering, :prompt-forms ["[\"<\", [\"field\", 101], 10]"]}
+ {:op '<=, :binding #'lib/<=, :kind :nested, :group :filtering, :prompt-forms ["[\"<=\", [\"field\", 101], 10]"]}
+ {:op '>, :binding #'lib/>, :kind :nested, :group :filtering, :prompt-forms ["[\">\", [\"field\", 101], 10]"]}
+ {:op '>=, :binding #'lib/>=, :kind :nested, :group :filtering, :prompt-forms ["[\">=\", [\"field\", 101], 10]"]}
+ {:op 'between, :binding #'lib/between, :kind :nested, :group :filtering, :prompt-forms ["[\"between\", [\"field\", 101], 1, 100]"], :shape "[\"between\", [\"field\", 2205], 1, 100]", :example "[\"between\", [\"field\", 2205], 1, 100]", :prompt-notes ["If the question explicitly asks for 2024 or another exact year, use an exact 2024 restriction, not a relative current-year or last-year filter." "Using an exact helper year field like `[\"=\", [\"field\", year_field], 2024]` is acceptable when it is clearly equivalent to filtering the requested date field to 2024."]}
+ {:op 'contains, :binding #'lib/contains, :kind :nested, :group :filtering, :prompt-forms ["[\"contains\", [\"field\", 101], \"text\"]"]}
+ {:op 'does-not-contain, :binding #'lib/does-not-contain, :kind :nested, :group :filtering, :prompt-forms ["[\"does-not-contain\", [\"field\", 101], \"text\"]"], :shape "[\"does-not-contain\", [\"field\", 2204], \"trial\"]", :example "[\"does-not-contain\", [\"field\", 2204], \"trial\"]"}
+ {:op 'starts-with, :binding #'lib/starts-with, :kind :nested, :group :filtering, :prompt-forms ["[\"starts-with\", [\"field\", 101], \"prefix\"]"]}
+ {:op 'ends-with, :binding #'lib/ends-with, :kind :nested, :group :filtering, :prompt-forms ["[\"ends-with\", [\"field\", 101], \"suffix\"]"]}
+ {:op 'is-null, :binding #'lib/is-null, :kind :nested, :group :filtering, :prompt-forms ["[\"is-null\", [\"field\", 101]]"]}
+ {:op 'not-null, :binding #'lib/not-null, :kind :nested, :group :filtering, :prompt-forms ["[\"not-null\", [\"field\", 101]]"]}
+ {:op 'is-empty, :binding #'lib/is-empty, :kind :nested, :group :filtering, :prompt-forms ["[\"is-empty\", [\"field\", 101]]"], :shape "[\"is-empty\", [\"field\", 2204]]", :example "[\"is-empty\", [\"field\", 2204]]"}
+ {:op 'not-empty, :binding #'lib/not-empty, :kind :nested, :group :filtering, :prompt-forms ["[\"not-empty\", [\"field\", 101]]"], :shape "[\"not-empty\", [\"field\", 2204]]", :example "[\"not-empty\", [\"field\", 2204]]"}
+ {:op 'and, :binding #'lib/and, :kind :nested, :group :filtering, :prompt-forms ["[\"and\", clause1, clause2]"]}
+ {:op 'or, :binding #'lib/or, :kind :nested, :group :filtering, :prompt-forms ["[\"or\", clause1, clause2]"]}
+ {:op 'not, :binding #'lib/not, :kind :nested, :group :filtering, :prompt-forms ["[\"not\", clause]"]}
+ {:op 'time-interval, :binding #'lib/time-interval, :kind :nested, :group :filtering, :prompt-forms ["[\"time-interval\", [\"field\", 101], -30, \"day\"]"]}
+ {:op 'relative-time-interval, :binding #'lib/relative-time-interval, :kind :nested, :group :filtering, :arities #{5}, :prompt-forms ["[\"relative-time-interval\", [\"field\", 101], -30, \"day\", 0, \"day\"]"], :shape "[\"relative-time-interval\", [\"field\", 2205], -30, \"day\", 0, \"day\"]", :example "[\"relative-time-interval\", [\"field\", 2205], -30, \"day\", 0, \"day\"]"}
+ {:op 'in, :binding #'lib/in, :kind :nested, :group :filtering, :prompt-forms ["[\"in\", [\"field\", 101], [\"a\", \"b\"]]"], :shape "[\"in\", [\"field\", 2492], [\"authorized\", \"pending\", \"partially_paid\"]]", :example "[\"in\", [\"field\", 2492], [\"authorized\", \"pending\", \"partially_paid\"]]"}
+ {:op 'not-in, :binding #'lib/not-in, :kind :nested, :group :filtering, :prompt-forms ["[\"not-in\", [\"field\", 101], [\"a\", \"b\"]]"], :shape "[\"not-in\", [\"field\", 2492], [\"authorized\", \"pending\"]]", :example "[\"not-in\", [\"field\", 2492], [\"authorized\", \"pending\"]]"}
+ {:op 'inside, :binding #'lib/inside, :kind :nested, :group :filtering, :prompt-forms ["[\"inside\", [\"field\", 401], [\"field\", 402], 40.0, 30.0, -70.0, -80.0]"], :shape "[\"inside\", [\"field\", 401], [\"field\", 402], 40.0, 30.0, -70.0, -80.0]", :example "[\"inside\", [\"field\", 401], [\"field\", 402], 40.0, 30.0, -70.0, -80.0]"}
+ {:op 'segment, :binding #'lib/segment, :kind :nested, :group :filtering, :prompt-forms ["[\"segment\", 88]"], :shape "[\"segment\", 88]", :example "[\"segment\", 88]"}])
diff --git a/src/metabase/agent_lib/capabilities/catalog/joins.clj b/src/metabase/agent_lib/capabilities/catalog/joins.clj
new file mode 100644
index 00000000000..180e42ad120
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/joins.clj
@@ -0,0 +1,14 @@
+(ns metabase.agent-lib.capabilities.catalog.joins
+ "Join-related structured capability entries."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Join helper capability entries."
+ [{:op 'join-clause, :binding #'lib/join-clause, :kind :nested, :group :joins, :arities #{1}, :prompt-forms ["[\"join-clause\", [\"table\", 42]]" "[\"join-clause\", {\"type\": \"program\", \"program\": {...}}]"], :shape "[\"join-clause\", [\"table\", 133]]", :example "[\"join-clause\", [\"table\", 133]]"}
+ {:op 'with-join-conditions, :binding #'lib/with-join-conditions, :kind :nested, :group :joins, :prompt-forms ["[\"with-join-conditions\", [\"join-clause\", [\"table\", 42]], [[\"=\", [\"field\", 101], [\"field\", 202]]]]"], :shape "[\"with-join-conditions\", [\"join-clause\", [\"table\", 133]], [[\"=\", [\"field\", 2456], [\"field\", 2488]]]]", :example "[\"with-join-conditions\", [\"join-clause\", [\"table\", 133]], [[\"=\", [\"field\", 2456], [\"field\", 2488]]]]", :prompt-notes ["If the source-table context already exposes the related table you need, prefer direct related-field refs first." "Direct related-field refs are the default for one-hop related tables." "Use `join-clause` and `with-join-conditions` when you need custom join behavior, a join alias, explicit joined-field selection, self-joins, or direct related-field refs are unavailable." "If an explicit join returns a permission error, the underlying table is not accessible — surface the error instead of retrying with implicit refs."]}
+ {:op 'with-join-fields, :binding #'lib/with-join-fields, :kind :nested, :group :joins, :prompt-forms ["[\"with-join-fields\", join-clause, \"all\"]" "[\"with-join-fields\", join-clause, \"none\"]" "[\"with-join-fields\", join-clause, [[\"field\", 757]]]"], :shape "[\"with-join-fields\", join-clause, [[\"field\", 757]]]", :example "[\"with-join-fields\", [\"join-clause\", [\"table\", 133]], [[\"field\", 757]]]"}
+ {:op 'with-join-strategy, :binding #'lib/with-join-strategy, :kind :nested, :group :joins, :prompt-forms ["[\"with-join-strategy\", join-clause, \"left-join\"]"], :shape "[\"with-join-strategy\", [\"join-clause\", [\"table\", 133]], \"left-join\"]", :example "[\"with-join-strategy\", [\"join-clause\", [\"table\", 133]], \"left-join\"]"}
+ {:op 'with-join-alias, :binding #'lib/with-join-alias, :kind :nested, :group :joins, :prompt-forms ["[\"with-join-alias\", join-clause, \"alias\"]"]}])
diff --git a/src/metabase/agent_lib/capabilities/catalog/ordering.clj b/src/metabase/agent_lib/capabilities/catalog/ordering.clj
new file mode 100644
index 00000000000..a909226ab9b
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/ordering.clj
@@ -0,0 +1,13 @@
+(ns metabase.agent-lib.capabilities.catalog.ordering
+ "Breakout and ordering structured capability entries."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Breakout-ordering helper capability entries."
+ [{:op 'with-temporal-bucket, :binding #'lib/with-temporal-bucket, :kind :nested, :group :breakout-ordering, :prompt-forms ["[\"with-temporal-bucket\", [\"field\", 302], \"month\"]"], :shape "[\"with-temporal-bucket\", [\"field\", 2205], \"month\"]", :example "[\"with-temporal-bucket\", [\"field\", 2205], \"month\"]"}
+ {:op 'with-binning, :binding #'lib/with-binning, :kind :nested, :group :breakout-ordering, :prompt-forms ["[\"with-binning\", [\"field\", 303], strategy]"]}
+ {:op 'asc, :kind :nested, :group :breakout-ordering, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"asc\", [\"field\", 301]]"]}
+ {:op 'desc, :kind :nested, :group :breakout-ordering, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"desc\", [\"field\", 301]]"]}])
diff --git a/src/metabase/agent_lib/capabilities/catalog/sources.clj b/src/metabase/agent_lib/capabilities/catalog/sources.clj
new file mode 100644
index 00000000000..c0aff5f2940
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/sources.clj
@@ -0,0 +1,16 @@
+(ns metabase.agent-lib.capabilities.catalog.sources
+ "Source and query-reference structured capability entries."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Source and query-reference capability entries."
+ [{:op 'field, :kind :nested, :group :sources, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"field\", 101]"], :shape "[\"field\", 101]", :example "[\"field\", 101]"}
+ {:op 'table, :kind :nested, :group :sources, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"table\", 42]"], :shape "[\"table\", 42]", :example "[\"table\", 42]"}
+ {:op 'card, :kind :nested, :group :sources, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"card\", 55]"], :shape "[\"card\", 55]", :example "[\"card\", 55]"}
+ {:op 'metric, :kind :nested, :group :sources, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"metric\", 77]"], :shape "[\"metric\", 77]", :example "[\"metric\", 77]"}
+ {:op 'measure, :kind :nested, :group :sources, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"measure\", 91]"], :shape "[\"measure\", 91]", :example "[\"measure\", 91]", :prompt-notes ["Use `measure` for a named table/model measure aggregation when it is available in the source context. Measures use real numeric ids."]}
+ {:op 'expression-ref, :binding #'lib/expression-ref, :kind :nested, :group :query-refs, :arities #{1}, :prompt-forms ["[\"expression-ref\", \"Net Amount\"]"], :shape "[\"expression-ref\", \"Net Amount\"]", :example "[\"expression-ref\", \"Net Amount\"]"}
+ {:op 'aggregation-ref, :kind :nested, :group :query-refs, :runtime-provided? true, :arities #{1}, :prompt-forms ["[\"aggregation-ref\", 0]"], :shape "[\"aggregation-ref\", 0]", :example "[\"aggregation-ref\", 0]"}])
diff --git a/src/metabase/agent_lib/capabilities/catalog/top_level.clj b/src/metabase/agent_lib/capabilities/catalog/top_level.clj
new file mode 100644
index 00000000000..e7b16296298
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/catalog/top_level.clj
@@ -0,0 +1,21 @@
+(ns metabase.agent-lib.capabilities.catalog.top-level
+ "Top-level structured-program capability entries."
+ (:require
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(def capabilities
+ "Top-level structured-program capability entries."
+ [{:op 'filter, :binding #'lib/filter, :kind :top-level, :arities #{1}, :prompt-forms ["[\"filter\", clause]"], :shape "[\"filter\", clause]", :example "[\"filter\", [\"time-interval\", [\"field\", 2058], -12, \"month\"]]"}
+ {:op 'aggregate, :binding #'lib/aggregate, :kind :top-level, :arities #{1}, :prompt-forms ["[\"aggregate\", aggregation]"], :shape "[\"aggregate\", [\"sum\", [\"field\", 2056]]]", :example "[\"aggregate\", [\"sum\", [\"field\", 2056]]]"}
+ {:op 'breakout, :binding #'lib/breakout, :kind :top-level, :arities #{1}, :prompt-forms ["[\"breakout\", breakout-form]"], :shape "[\"breakout\", [\"field\", 3134]]", :example "[\"breakout\", [\"field\", 3134]]"}
+ {:op 'with-fields, :binding #'lib/with-fields, :kind :top-level, :arities #{1}, :prompt-forms ["[\"with-fields\", [field-or-ref, ...]]"], :shape "[\"with-fields\", [[\"field\", 2204], [\"expression-ref\", \"Net Amount\"]]]", :example "[\"with-fields\", [[\"field\", 2204], [\"expression-ref\", \"Net Amount\"]]]"}
+ {:op 'limit, :binding #'lib/limit, :kind :top-level, :arities #{1}, :prompt-forms ["[\"limit\", 10]"], :shape "[\"limit\", 10]", :example "[\"limit\", 10]"}
+ {:op 'expression, :binding #'lib/expression, :kind :top-level, :arities #{2}, :prompt-forms ["[\"expression\", \"Name\", expr]"], :shape "[\"expression\", \"Net Amount\", [\"-\", [\"field\", 2624], [\"field\", 2641]]]", :example "[\"expression\", \"Net Amount\", [\"-\", [\"field\", 2624], [\"field\", 2641]]]", :prompt-notes ["If an `expression` uses `aggregation-ref`, add `append-stage` first. Do not define ratios over aggregations in the same stage that creates those aggregations."]}
+ {:op 'join, :binding #'lib/join, :kind :top-level, :arities #{1}, :prompt-forms ["[\"join\", join-clause]"], :shape "[\"join\", [\"join-clause\", [\"table\", 42]]]", :example "[\"join\", [\"join-clause\", [\"table\", 42]]]", :prompt-notes ["If a related table is already surfaced in the source-table context, first try using its field ids directly without an explicit join." "Direct related-field refs are the default for one-hop related tables." "Use an explicit join when you need custom join behavior, a join alias, explicit joined-field selection, self-joins, or direct related-field refs do not work." "If an explicit join returns a permission error, the underlying table is not accessible — surface the error instead of retrying with implicit refs."]}
+ {:op 'order-by, :kind :top-level, :runtime-provided? true, :arities #{1 2}, :prompt-forms ["[\"order-by\", orderable]" "[\"order-by\", orderable, \"desc\"]"], :shape "[\"order-by\", [\"desc\", [\"aggregation-ref\", 0]]] or [\"order-by\", [\"aggregation-ref\", 0], \"desc\"]", :example "[\"order-by\", [\"desc\", [\"aggregation-ref\", 0]]]"}
+ {:op 'append-stage, :binding #'lib/append-stage, :kind :top-level, :arities #{0}, :prompt-forms ["[\"append-stage\"]"], :shape "[\"append-stage\"]", :example "[\"append-stage\"]", :prompt-notes ["If you filter on `[\"aggregation-ref\", index]`, insert `append-stage` first. Do not filter on an aggregation result in the same stage that creates it."]}
+ {:op 'drop-stage, :binding #'lib/drop-stage, :kind :top-level, :arities #{0}, :prompt-forms ["[\"drop-stage\"]"], :shape "[\"drop-stage\"]", :example "[\"drop-stage\"]"}
+ {:op 'drop-empty-stages, :binding #'lib/drop-empty-stages, :kind :top-level, :arities #{0}, :prompt-forms ["[\"drop-empty-stages\"]"], :shape "[\"drop-empty-stages\"]", :example "[\"drop-empty-stages\"]"}
+ {:op 'with-page, :binding #'lib/with-page, :kind :top-level, :arities #{1}, :prompt-forms ["[\"with-page\", {\"page\": 2, \"items\": 25}]"], :shape "[\"with-page\", {\"page\": 2, \"items\": 25}]", :example "[\"with-page\", {\"page\": 2, \"items\": 25}]", :prompt-notes ["Use JSON objects only for helpers that explicitly show one, such as `[\"with-page\", {\"page\": 2, \"items\": 25}]`."]}])
diff --git a/src/metabase/agent_lib/capabilities/derived.clj b/src/metabase/agent_lib/capabilities/derived.clj
new file mode 100644
index 00000000000..fe1d095405c
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/derived.clj
@@ -0,0 +1,99 @@
+(ns metabase.agent-lib.capabilities.derived
+ "Derived views over the structured capability catalog.")
+
+(set! *warn-on-reflection* true)
+
+(defn guidance
+ "Return prompt and retry guidance metadata for a capability entry."
+ [capability]
+ (:guidance capability))
+
+(defn prompt-forms
+ "Return documented prompt forms for a capability entry."
+ [capability]
+ (get-in capability [:guidance :prompt-forms]))
+
+(defn prompt-notes
+ "Return documented prompt notes for a capability entry."
+ [capability]
+ (get-in capability [:guidance :prompt-notes]))
+
+(defn guidance-shape
+ "Return the documented canonical call shape for a capability entry."
+ [capability]
+ (get-in capability [:guidance :shape]))
+
+(defn guidance-example
+ "Return the documented example program fragment for a capability entry."
+ [capability]
+ (get-in capability [:guidance :example]))
+
+(defn capability-by-op
+ "Index capability entries by operator symbol."
+ [capability-catalog]
+ (into {}
+ (map (fn [capability]
+ [(:op capability) capability]))
+ capability-catalog))
+
+(defn fixed-arities
+ "Index declared helper arities by operator symbol."
+ [capability-catalog]
+ (into {}
+ (keep (fn [{:keys [op arities]}]
+ (when arities
+ [op arities])))
+ capability-catalog))
+
+(defn trusted-helper-bindings
+ "Index trusted helper implementations by operator symbol."
+ [capability-catalog]
+ (into {}
+ (keep (fn [{:keys [op binding]}]
+ (when binding
+ [op binding])))
+ capability-catalog))
+
+(defn query-transform-symbols
+ "Return the top-level query-transform operators."
+ [capability-catalog]
+ (into #{}
+ (comp (filter #(= (:kind %) :top-level))
+ (map :op))
+ capability-catalog))
+
+(defn helper-symbols
+ "Return all helper symbols accepted by the structured evaluator."
+ [capability-catalog synthetic-helper-symbols]
+ (into synthetic-helper-symbols
+ (comp (filter #(or (:binding %)
+ (:runtime-provided? %)))
+ (map :op))
+ capability-catalog))
+
+(defn nested-operator-values
+ "Return canonical string names for nested operators."
+ [capability-catalog]
+ (into #{}
+ (comp (filter #(= (:kind %) :nested))
+ (map (comp name :op)))
+ capability-catalog))
+
+(defn top-level-operator-values
+ "Return canonical string names for top-level operations."
+ [capability-catalog]
+ (into #{}
+ (comp (filter #(= (:kind %) :top-level))
+ (map (comp name :op)))
+ capability-catalog))
+
+(defn operator-guidance-catalog
+ "Return retry guidance keyed by canonical operator name."
+ [capability-catalog]
+ (into {}
+ (keep (fn [{:keys [op] :as capability}]
+ (let [{:keys [shape example retry-shape retry-example]} (guidance capability)]
+ (when-let [guidance-shape (or retry-shape shape)]
+ [(name op) {:shape guidance-shape
+ :example (or retry-example example)}]))))
+ capability-catalog))
diff --git a/src/metabase/agent_lib/capabilities/synthetic.clj b/src/metabase/agent_lib/capabilities/synthetic.clj
new file mode 100644
index 00000000000..32b13ec4f01
--- /dev/null
+++ b/src/metabase/agent_lib/capabilities/synthetic.clj
@@ -0,0 +1,9 @@
+(ns metabase.agent-lib.capabilities.synthetic
+ "Synthetic structured helper symbols that are accepted by agent-lib but are
+ not modeled as normal capability entries.")
+
+(set! *warn-on-reflection* true)
+
+(def synthetic-helper-symbols
+ "Synthetic helper symbols accepted by the structured evaluator."
+ '#{query})
diff --git a/src/metabase/agent_lib/common/coercions.clj b/src/metabase/agent_lib/common/coercions.clj
new file mode 100644
index 00000000000..f3f5606dd61
--- /dev/null
+++ b/src/metabase/agent_lib/common/coercions.clj
@@ -0,0 +1,96 @@
+(ns metabase.agent-lib.common.coercions
+ "Coercion and normalization helpers for structured MBQL programs."
+ (:require
+ [clojure.string :as str]
+ [metabase.agent-lib.common.literals :as literals]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn quarter-label->number
+ "Convert quarter labels such as `Q1` into the corresponding integer."
+ [value]
+ (when (string? value)
+ (case (u/upper-case-en (str/trim value))
+ "Q1" 1
+ "Q2" 2
+ "Q3" 3
+ "Q4" 4
+ nil)))
+
+(defn parse-int-string
+ "Parse an integer string, returning nil when parsing fails."
+ [value]
+ (when (and (string? value)
+ (re-matches #"-?\d+" value))
+ (try
+ (Long/parseLong value)
+ (catch NumberFormatException _
+ nil))))
+
+(defn coerce-positive-int
+ "Coerce positive integer strings into integers."
+ [value]
+ (cond
+ (pos-int? value) value
+ (string? value) (let [parsed (parse-int-string value)]
+ (if (pos-int? parsed) parsed value))
+ :else value))
+
+(defn coerce-non-negative-int
+ "Coerce non-negative integer strings into integers."
+ [value]
+ (cond
+ (and (int? value) (<= 0 value)) value
+ (string? value) (let [parsed (parse-int-string value)]
+ (if (and (int? parsed) (<= 0 parsed)) parsed value))
+ :else value))
+
+(defn direction-string?
+ "True when `value` is an order direction string or keyword."
+ [value]
+ (boolean (#{"asc" "desc"} (some-> value syntax/raw-op-name u/lower-case-en))))
+
+(defn normalize-direction
+ "Normalize a direction enum to its lowercase string form."
+ [value]
+ (some-> value syntax/raw-op-name u/lower-case-en))
+
+(defn unwrap-singleton-form
+ "Recursively unwrap singleton vectors that are not operator tuples."
+ [value]
+ (loop [value value]
+ (if (and (vector? value)
+ (= 1 (count value))
+ (not (syntax/possible-operator-tuple? value))
+ (not (literals/scalar-sequential? value)))
+ (recur (first value))
+ value)))
+
+(defn normalize-percentile-value
+ "Normalize percentiles expressed as whole-number percentages to fractions."
+ [value]
+ (if (and (number? value)
+ (> (double value) 1.0)
+ (<= (double value) 100.0))
+ (/ (double value) 100.0)
+ value))
+
+(defn normalize-join-conditions
+ "Unwrap one extra sequential level around join conditions."
+ [conditions]
+ (loop [conditions conditions]
+ (if (and (sequential? conditions)
+ (= 1 (count conditions))
+ (sequential? (first conditions))
+ (not (syntax/possible-operator-tuple? (first conditions))))
+ (recur (vec (first conditions)))
+ conditions)))
+
+(defn normalize-map-key
+ "Convert string map keys to keywords while leaving other keys unchanged."
+ [k]
+ (if (string? k)
+ (keyword k)
+ k))
diff --git a/src/metabase/agent_lib/common/context.clj b/src/metabase/agent_lib/common/context.clj
new file mode 100644
index 00000000000..6970c639e80
--- /dev/null
+++ b/src/metabase/agent_lib/common/context.clj
@@ -0,0 +1,22 @@
+(ns metabase.agent-lib.common.context
+ "Helpers derived from the evaluation context.")
+
+(set! *warn-on-reflection* true)
+
+(defn source-metric-id
+ "Return the current context source metric id when the source entity is a metric."
+ [context]
+ (let [source-entity (:source-entity context)
+ source-id (:id source-entity)]
+ (when (and (= "metric" (:model source-entity))
+ (pos-int? source-id))
+ source-id)))
+
+(defn source-table-id
+ "Return the current context source table id when the source entity is a table."
+ [context]
+ (let [source-entity (:source-entity context)
+ source-id (:id source-entity)]
+ (when (and (= "table" (:model source-entity))
+ (pos-int? source-id))
+ source-id)))
diff --git a/src/metabase/agent_lib/common/errors.clj b/src/metabase/agent_lib/common/errors.clj
new file mode 100644
index 00000000000..e6693bda7c9
--- /dev/null
+++ b/src/metabase/agent_lib/common/errors.clj
@@ -0,0 +1,80 @@
+(ns metabase.agent-lib.common.errors
+ "Error helpers for structured MBQL program repair, validation, and evaluation."
+ (:require
+ [clojure.string :as str]
+ [metabase.util.i18n :refer [tru]]))
+
+(set! *warn-on-reflection* true)
+
+(defn path->string
+ "Render a validation path vector into a dotted path string."
+ [path]
+ (if (empty? path)
+ "<root>"
+ (str/join ""
+ (map-indexed
+ (fn [idx segment]
+ (cond
+ (integer? segment) (str "[" segment "]")
+ (zero? idx) (name segment)
+ :else (str "." (name segment))))
+ path))))
+
+(defn invalid-program!
+ "Throw a standardized invalid structured program exception."
+ [path message & [data]]
+ (throw (ex-info (tru "Generated program is invalid.")
+ (merge
+ {:status-code 400
+ :error :invalid-generated-program
+ :path (path->string path)
+ :details (str (path->string path) ": " message)}
+ data))))
+
+(def ^:private recovery-keys
+ "Keys that carry agent-recovery hints through the error chain."
+ [:available :tables :suggestion])
+
+(defn lookup-error!
+ "Throw a structured lookup error with optional recovery hints.
+ `message` is the clean, user-safe string (no schema enumeration).
+ `data` identifies the failed entity (e.g. `{:table-id 42}`).
+ `recovery` is an optional map with agent-facing hints:
+ :available - seq of valid names/values
+ :tables - seq of matching tables (for disambiguation)
+ :suggestion - corrective suggestion string"
+ ([message data]
+ (lookup-error! message data nil))
+ ([message data recovery]
+ (throw (ex-info message
+ (cond-> (assoc data
+ :status-code 400
+ :error :invalid-generated-program)
+ (seq recovery) (assoc :recovery recovery))))))
+
+(defn wrap-runtime-error
+ "Wrap a runtime exception into the standard invalid-program shape,
+ preserving structured recovery data from the cause's ex-data."
+ [path op-name ^Throwable cause]
+ (let [cause-data (ex-data cause)
+ recovery (or (:recovery cause-data)
+ (not-empty (select-keys cause-data recovery-keys)))]
+ (ex-info (tru "Generated program could not be evaluated.")
+ (cond-> {:status-code 400
+ :error :invalid-generated-program
+ :path (path->string path)
+ :details (str (path->string path) ": " (ex-message cause))
+ :operator op-name}
+ recovery (assoc :recovery recovery))
+ cause)))
+
+(defn recovery-summary
+ "Format the `:recovery` map from an exception's ex-data into a string
+ suitable for LLM retry feedback. Returns nil when there are no hints."
+ [{:keys [available tables suggestion]}]
+ (let [parts (cond-> []
+ (seq available) (conj (str "Available: " (str/join ", " (take 20 available))))
+ (seq tables) (conj (str "Matching tables: " (str/join ", " tables)))
+ suggestion (conj suggestion))]
+ (when (seq parts)
+ (str/join "\n" parts))))
diff --git a/src/metabase/agent_lib/common/literals.clj b/src/metabase/agent_lib/common/literals.clj
new file mode 100644
index 00000000000..0e9ff2e07c6
--- /dev/null
+++ b/src/metabase/agent_lib/common/literals.clj
@@ -0,0 +1,84 @@
+(ns metabase.agent-lib.common.literals
+ "Literal and structural predicates for structured MBQL programs."
+ (:require
+ [clojure.string :as str]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.lib.hierarchy :as lib.hierarchy]
+ [metabase.lib.schema.aggregation]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private aggregation-operator-names
+ ;; Derived from the lib schema hierarchy so new aggregation helpers are picked up automatically.
+ ;; `:aggregation` is the abstract wrapper clause and `:offset` is a windowed aggregation that
+ ;; behaves like an expression in some contexts — neither should be recognized as a plain
+ ;; aggregation helper name.
+ (into #{}
+ (comp (remove #{:aggregation :offset})
+ (map name))
+ (descendants @lib.hierarchy/hierarchy
+ :metabase.lib.schema.aggregation/aggregation-clause-tag)))
+
+(defn scalar-literal?
+ "True when `value` is a scalar JSON-like literal accepted by the structured format."
+ [value]
+ (or (nil? value)
+ (string? value)
+ (number? value)
+ (boolean? value)))
+
+(defn non-blank-string?
+ "True when `value` is a non-blank string."
+ [value]
+ (and (string? value) (not (str/blank? value))))
+
+(defn quarter-label-string?
+ "True when `value` looks like a quarter label such as `Q1`."
+ [value]
+ (and (string? value)
+ (re-matches #"(?i)q[1-4]" (str/trim value))))
+
+(defn now-literal?
+ "True when `value` is the case-insensitive literal string `now`."
+ [value]
+ (and (string? value)
+ (= "now" (u/lower-case-en (str/trim value)))))
+
+(defn iso-date-string?
+ "True when `value` is an ISO `yyyy-mm-dd` date string."
+ [value]
+ (and (string? value)
+ (re-matches #"\d{4}-\d{2}-\d{2}" (str/trim value))))
+
+(defn null-literal?
+ "True when `value` is nil or the case-insensitive string `null`."
+ [value]
+ (or (nil? value)
+ (and (string? value)
+ (= "null" (u/lower-case-en value)))))
+
+(defn scalar-sequential?
+ "True when `value` is a sequential collection of scalar literals."
+ [value]
+ (and (sequential? value)
+ (not (syntax/possible-operator-tuple? value))
+ (every? scalar-literal? value)))
+
+(defn aggregation-form?
+ "True when `value` is a canonical aggregation helper tuple."
+ [value]
+ (and (vector? value)
+ (aggregation-operator-names (syntax/canonical-op-name (first value)))))
+
+(defn default-map?
+ "True when `value` looks like a legacy `{default ...}` case fallback map."
+ [value]
+ (and (map? value)
+ (or (contains? value :default)
+ (contains? value "default"))))
+
+(defn bare-field-id?
+ "True when `value` is a positive integer field id without a `field` wrapper."
+ [value]
+ (and (int? value) (pos? value)))
diff --git a/src/metabase/agent_lib/core.clj b/src/metabase/agent_lib/core.clj
new file mode 100644
index 00000000000..ed58eb19374
--- /dev/null
+++ b/src/metabase/agent_lib/core.clj
@@ -0,0 +1,14 @@
+(ns metabase.agent-lib.core
+ "Public API for structured MBQL program repair, validation, and evaluation."
+ (:require
+ [metabase.agent-lib.eval]
+ [metabase.agent-lib.repair]
+ [metabase.agent-lib.schema]
+ [metabase.agent-lib.validate]
+ [potemkin :as p]))
+
+(p/import-vars
+ [metabase.agent-lib.repair repair-program]
+ [metabase.agent-lib.validate validated-program]
+ [metabase.agent-lib.eval evaluate-program]
+ [metabase.agent-lib.schema program-schema])
diff --git a/src/metabase/agent_lib/eval.clj b/src/metabase/agent_lib/eval.clj
new file mode 100644
index 00000000000..2eddecbb8a8
--- /dev/null
+++ b/src/metabase/agent_lib/eval.clj
@@ -0,0 +1,136 @@
+(ns metabase.agent-lib.eval
+ "Structured MBQL program interpretation."
+ (:require
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.eval.invoke :as eval.invoke]
+ [metabase.agent-lib.eval.source :as eval.source]
+ [metabase.agent-lib.eval.walker :as eval.walker]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.agent-lib.refs :as refs]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.validate :as validate]
+ [metabase.agent-lib.validate.walker :as walker]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(defn- skip-operation?
+ "True when an operation is a redundant implicit join that should be elided."
+ [runtime query op operation]
+ (and (= op 'join)
+ ((get (:bindings runtime) 'skip-join?) query operation)))
+
+(defn- apply-operation
+ "Evaluate a single operation's args and invoke its helper against `query`."
+ [evaluate-node runtime query op-path [raw-op & raw-args]]
+ (let [op (runtime/op-symbol raw-op)
+ args (eval.walker/evaluate-args evaluate-node query op-path op raw-args)]
+ (eval.invoke/invoke-helper! runtime op-path op (into [query] args))))
+
+(defn- program-has-breakouts?
+ "True when the program's operations include at least one explicit breakout."
+ [program]
+ (some (fn [[op]]
+ (= "breakout" (when (string? op) op)))
+ (:operations program)))
+
+(defn- strip-default-breakouts
+ "Remove default breakouts from a metric source query when the program supplies its own.
+ Metrics include a default time dimension breakout in their query definition. When the
+ agent explicitly specifies breakout operations, these should replace — not augment — the
+ defaults."
+ [source-query program]
+ (if (and (mbql/query? source-query) (program-has-breakouts? program))
+ (lib/remove-all-breakouts source-query)
+ source-query))
+
+(defn- evaluate-program*
+ [runtime context path program program-depth]
+ (when (> program-depth walker/max-program-nesting)
+ (invalid-program! path "program source nesting exceeds maximum depth"))
+ (let [evaluate-program (fn [nested-path nested-program]
+ (evaluate-program* runtime context nested-path nested-program (inc program-depth)))
+ evaluate-node (fn [current-query node-path value]
+ (eval.walker/evaluate-node evaluate-program
+ runtime
+ current-query
+ node-path
+ value))
+ resolved-source (eval.source/resolve-source evaluate-program
+ runtime
+ (conj path :source)
+ (:source program))
+ source-query (-> (if (mbql/query? resolved-source)
+ resolved-source
+ (eval.invoke/invoke-helper! runtime (conj path :source) 'query [resolved-source]))
+ (strip-default-breakouts program))
+ final-query (reduce (fn [query [op-idx operation]]
+ (let [op-path (into path [:operations op-idx])
+ op (runtime/op-symbol (first operation))]
+ (if (skip-operation? runtime query op operation)
+ query
+ (apply-operation evaluate-node runtime query op-path operation))))
+ source-query
+ (map-indexed vector (:operations program)))]
+ (eval.invoke/ensure-query-result! final-query)))
+
+(defn- card-underlying-table-ids
+ "For a sequence of card metadata records, return the distinct underlying table-ids reachable
+ from each card's visible columns. Cards/metrics may unfold into one or more tables, so we
+ resolve them lazily through `lib/visible-columns`."
+ [metadata-provider cards]
+ (into #{}
+ (mapcat (fn [card]
+ (try
+ (->> (lib/query metadata-provider card)
+ lib/visible-columns
+ (keep :table-id))
+ (catch Exception _ nil))))
+ cards))
+
+(defn- in-scope-table-ids
+ "Resolve the set of table-ids whose metadata the runtime needs in order to evaluate `program`.
+
+ Walks the program for explicitly-referenced metadata IDs, then expands them:
+ * field-ids → bulk-fetched in one query, contribute their `:table-id`;
+ * card/metric-ids → bulk-fetched, expanded via `lib/visible-columns` to underlying tables;
+ * measure-ids → bulk-fetched, contribute their `:table-id`;
+ * source-entity from the context, when it's a table.
+
+ We deliberately ignore the LLM-supplied `:referenced-entities` and `:surrounding-tables` —
+ the operations and source declared by the program are the ground truth for what metadata
+ needs to be loaded. If the agent references a table or field it forgot to declare, the
+ walker still picks it up."
+ [metadata-provider context program]
+ (let [{:keys [table-ids field-ids card-ids metric-ids measure-ids]}
+ (refs/collect-program-refs program)
+ source-table-id (when (= "table" (some-> context :source-entity :model))
+ (some-> context :source-entity :id))
+ fields (when (seq field-ids)
+ (lib.metadata/bulk-metadata metadata-provider :metadata/column field-ids))
+ cards (when (or (seq card-ids) (seq metric-ids))
+ (lib.metadata/bulk-metadata metadata-provider :metadata/card
+ (into #{} cat [card-ids metric-ids])))
+ measures (when (seq measure-ids)
+ (lib.metadata/bulk-metadata metadata-provider :metadata/measure measure-ids))]
+ (cond-> (set table-ids)
+ source-table-id (conj source-table-id)
+ (seq fields) (into (keep :table-id) fields)
+ (seq cards) (into (card-underlying-table-ids metadata-provider cards))
+ (seq measures) (into (keep :table-id) measures))))
+
+(mu/defn evaluate-program :- :map
+ "Validate and interpret a structured MBQL program."
+ [program :- :map
+ metadata-providerable :- :any
+ context :- validate/EvaluationContext]
+ (log/debugf "Evaluating structured program with %d operations" (count (:operations program)))
+ (let [program (validate/validated-program program context)
+ mp (lib.metadata/->metadata-provider metadata-providerable)
+ scope (in-scope-table-ids mp context program)
+ runtime (runtime/build-runtime mp {:in-scope-table-ids scope
+ :extra-bindings {'source (:source-metadata context)}})]
+ (evaluate-program* runtime context [] program 0)))
diff --git a/src/metabase/agent_lib/eval/args.clj b/src/metabase/agent_lib/eval/args.clj
new file mode 100644
index 00000000000..9e4f2d50cf2
--- /dev/null
+++ b/src/metabase/agent_lib/eval/args.clj
@@ -0,0 +1,126 @@
+(ns metabase.agent-lib.eval.args
+ "Argument normalization for structured MBQL helper invocation."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [normalize-percentile-value]]
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.common.literals :refer [scalar-literal?]]))
+
+(def temporal-unit-values
+ "Allowed string enum values for temporal units and buckets."
+ #{"millisecond" "second" "minute" "hour" "day" "week" "month" "quarter" "year"})
+
+(def ^:private time-interval-amount-keywords
+ "String values for time-interval amount that should be keywordized."
+ #{"current" "last" "next"})
+
+(def order-direction-values
+ "Allowed string enum values for order direction."
+ #{"asc" "desc"})
+
+(def join-strategy-values
+ "Allowed string enum values for explicit join strategy."
+ #{"left-join" "inner-join" "right-join" "full-join"})
+
+(def join-fields-values
+ "Allowed string enum values for explicit joined-field selection."
+ #{"all" "none"})
+
+(set! *warn-on-reflection* true)
+
+(defn keyword-enum
+ "Normalize string enum values into keywords for lib helpers."
+ [path label value allowed-values]
+ (cond
+ (keyword? value)
+ (if (allowed-values (name value))
+ value
+ (invalid-program! path (str label " must be one of " (sort allowed-values))))
+
+ (string? value)
+ (if (allowed-values value)
+ (keyword value)
+ (invalid-program! path (str label " must be one of " (sort allowed-values))))
+
+ :else
+ (invalid-program! path (str label " must be a string enum value"))))
+
+(defn normalize-helper-args
+ "Normalize helper arguments into the forms expected by trusted lib bindings."
+ [path op args]
+ (case (name op)
+ "in"
+ (if (and (= 2 (count args))
+ (vector? (second args))
+ (every? scalar-literal? (second args)))
+ (into [(first args)] (second args))
+ args)
+
+ "not-in"
+ (if (and (= 2 (count args))
+ (vector? (second args))
+ (every? scalar-literal? (second args)))
+ (into [(first args)] (second args))
+ args)
+
+ "time-interval"
+ (-> args
+ (update 1 (fn [amount]
+ (cond
+ (keyword? amount) amount
+ (time-interval-amount-keywords amount) (keyword amount)
+ :else amount)))
+ (update 2 #(keyword-enum (conj path 3) "temporal unit" % temporal-unit-values)))
+
+ "relative-time-interval"
+ (-> args
+ (update 2 #(keyword-enum (conj path 3) "temporal unit" % temporal-unit-values))
+ (update 4 #(keyword-enum (conj path 5) "temporal unit" % temporal-unit-values)))
+
+ "with-temporal-bucket"
+ (update args 1 #(keyword-enum (conj path 2) "temporal bucket" % temporal-unit-values))
+
+ "interval"
+ (update args 1 #(keyword-enum (conj path 2) "interval unit" % temporal-unit-values))
+
+ "relative-datetime"
+ (update args 1 #(keyword-enum (conj path 2) "temporal unit" % temporal-unit-values))
+
+ "absolute-datetime"
+ (update args 1 #(keyword-enum (conj path 2) "temporal unit" % temporal-unit-values))
+
+ "datetime-add"
+ (update args 2 #(keyword-enum (conj path 3) "temporal unit" % temporal-unit-values))
+
+ "datetime-diff"
+ (update args 2 #(keyword-enum (conj path 3) "temporal unit" % temporal-unit-values))
+
+ "datetime-subtract"
+ (update args 2 #(keyword-enum (conj path 3) "temporal unit" % temporal-unit-values))
+
+ "order-by"
+ (if (= 2 (count args))
+ (update args 1 #(keyword-enum (conj path 2) "order direction" % order-direction-values))
+ args)
+
+ "percentile"
+ (update args 1 normalize-percentile-value)
+
+ "with-join-strategy"
+ (update args 1 #(keyword-enum (conj path 2) "join strategy" % join-strategy-values))
+
+ "with-join-fields"
+ (update args 1 (fn [value]
+ (cond
+ (or (keyword? value) (string? value))
+ (keyword-enum (conj path 2) "join fields mode" value join-fields-values)
+
+ (sequential? value)
+ value
+
+ (some? value)
+ [value]
+
+ :else
+ value)))
+
+ args))
diff --git a/src/metabase/agent_lib/eval/invoke.clj b/src/metabase/agent_lib/eval/invoke.clj
new file mode 100644
index 00000000000..dc344662c2a
--- /dev/null
+++ b/src/metabase/agent_lib/eval/invoke.clj
@@ -0,0 +1,84 @@
+(ns metabase.agent-lib.eval.invoke
+ "Trusted helper invocation and error shaping for structured evaluation."
+ (:require
+ [metabase.agent-lib.common.errors :as errors :refer [invalid-program! path->string]]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defn invoke-helper!
+ "Invoke a helper that does not depend on the current query state."
+ [runtime path op args]
+ (when-not (contains? runtime/helper-symbols op)
+ (invalid-program! path "unknown operator" {:operator (name op)}))
+ (let [helper (runtime/helper-fn runtime op)]
+ (try
+ (apply helper args)
+ (catch clojure.lang.ArityException e
+ (invalid-program! path
+ (str "invalid arity for operator `" (name op) "`: " (ex-message e))
+ {:operator (name op)}))
+ (catch Exception e
+ (log/debugf e "Helper `%s` at %s threw unexpectedly" (name op) (path->string path))
+ (throw (errors/wrap-runtime-error path (name op) e))))))
+
+(defn invoke-query-aware-helper!
+ "Invoke a helper that requires the current query state."
+ [runtime current-query path op args]
+ (when-not current-query
+ (invalid-program! path
+ (format "`%s` requires the current query state" (name op))
+ {:operator (name op)}))
+ (let [helper (runtime/helper-fn runtime op)
+ op-name (name op)]
+ (try
+ (apply helper current-query args)
+ (catch clojure.lang.ArityException e
+ (invalid-program! path
+ (str "invalid arity for operator `" op-name "`: " (ex-message e))
+ {:operator op-name}))
+ (catch Exception e
+ (case op-name
+ "expression-ref"
+ (invalid-program! path
+ (format "expression-ref %s does not exist at this point in the query"
+ (pr-str (first args)))
+ {:operator "expression-ref"
+ :name (first args)})
+
+ "aggregation-ref"
+ (invalid-program! path
+ (format "aggregation-ref %s does not exist at this point in the query"
+ (first args))
+ {:operator "aggregation-ref"
+ :index (first args)})
+
+ (do (log/debugf e "Query-aware helper `%s` at %s threw unexpectedly" op-name (path->string path))
+ (throw (errors/wrap-runtime-error path op-name e))))))))
+
+(defn invoke-field-helper!
+ "Invoke the special `field` helper, which can resolve relative to the current query."
+ [runtime current-query path args]
+ (if current-query
+ (let [helper (runtime/helper-fn runtime 'field)]
+ (try
+ (apply helper current-query args)
+ (catch clojure.lang.ArityException e
+ (invalid-program! path
+ (str "invalid arity for operator `field`: " (ex-message e))
+ {:operator "field"}))
+ (catch Exception e
+ (log/debugf e "Field helper at %s threw unexpectedly" (path->string path))
+ (throw (errors/wrap-runtime-error path "field" e)))))
+ (invoke-helper! runtime path 'field args)))
+
+(defn ensure-query-result!
+ "Assert that structured evaluation produced a pMBQL query."
+ [result]
+ (when-not (mbql/query? result)
+ (throw (ex-info "Structured program evaluation must return a pMBQL query map."
+ {:result-type (type result)
+ :result result})))
+ result)
diff --git a/src/metabase/agent_lib/eval/source.clj b/src/metabase/agent_lib/eval/source.clj
new file mode 100644
index 00000000000..bc7d9ace37c
--- /dev/null
+++ b/src/metabase/agent_lib/eval/source.clj
@@ -0,0 +1,29 @@
+(ns metabase.agent-lib.eval.source
+ "Structured source resolution for program evaluation."
+ (:require
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.runtime :as runtime]))
+
+(set! *warn-on-reflection* true)
+
+(defn resolve-source
+ "Resolve a structured program source into metadata or a query."
+ [evaluate-program runtime source-path source]
+ (case (:type source)
+ "context"
+ (or (get (runtime/bindings-map runtime) 'source)
+ (invalid-program! source-path "context source is unavailable in runtime bindings"))
+
+ "table"
+ ((runtime/helper-fn runtime 'table) (:id source))
+
+ ("card" "dataset")
+ ((runtime/helper-fn runtime 'card) (:id source))
+
+ "metric"
+ ((runtime/helper-fn runtime 'metric) (:id source))
+
+ "program"
+ (evaluate-program (conj source-path :program) (:program source))
+
+ (invalid-program! (conj source-path :type) "unsupported source type")))
diff --git a/src/metabase/agent_lib/eval/walker.clj b/src/metabase/agent_lib/eval/walker.clj
new file mode 100644
index 00000000000..09b3bd6f71f
--- /dev/null
+++ b/src/metabase/agent_lib/eval/walker.clj
@@ -0,0 +1,89 @@
+(ns metabase.agent-lib.eval.walker
+ "Recursive structured-value evaluation for agent-lib.
+
+ The local `letfn` in `evaluate-node` is intentional: this namespace is the
+ recursive evaluation boundary for nested structured program values."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [normalize-map-key]]
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.common.literals :refer [scalar-literal?]]
+ [metabase.agent-lib.eval.args :as eval.args]
+ [metabase.agent-lib.eval.invoke :as eval.invoke]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.schema :as schema]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(defn evaluate-args
+ "Evaluate and normalize helper arguments in the current query context."
+ [eval-node-fn current-query node-path op raw-args]
+ (->> raw-args
+ (map-indexed (fn [idx arg]
+ (eval-node-fn current-query (conj node-path (inc idx)) arg)))
+ vec
+ (eval.args/normalize-helper-args node-path op)))
+
+(defn evaluate-node
+ "Evaluate a nested structured value in the current query context."
+ [evaluate-program runtime current-query node-path value]
+ (letfn [(walk [current-query node-path value]
+ (cond
+ (scalar-literal? value)
+ value
+
+ (map? value)
+ (if (schema/program-literal? value)
+ (evaluate-program (conj node-path :program) (:program value))
+ (into {}
+ (map (fn [[k v]]
+ [(normalize-map-key k)
+ (walk current-query (conj node-path (normalize-map-key k)) v)]))
+ value))
+
+ (vector? value)
+ (if (syntax/operator-tuple? value)
+ (let [[raw-op & raw-args] value
+ op (runtime/op-symbol raw-op)
+ op-name (name op)]
+ (cond
+ (syntax/top-level-operation-symbols op)
+ (invalid-program! node-path
+ (format "`%s` is a top-level operation and cannot be nested" op-name)
+ {:operator op-name})
+
+ (= op 'query)
+ (invalid-program! node-path
+ "`query` cannot appear inside operations; the program source already defines the base query."
+ {:operator "query"})
+
+ (= op 'field)
+ (eval.invoke/invoke-field-helper! runtime
+ current-query
+ node-path
+ (evaluate-args walk current-query node-path 'field raw-args))
+
+ ('#{expression-ref aggregation-ref} op)
+ (eval.invoke/invoke-query-aware-helper! runtime
+ current-query
+ node-path
+ op
+ (evaluate-args walk current-query node-path op raw-args))
+
+ :else
+ (eval.invoke/invoke-helper! runtime
+ node-path
+ op
+ (evaluate-args walk current-query node-path op raw-args))))
+ (mapv (fn [[idx element]]
+ (walk current-query (conj node-path idx) element))
+ (map-indexed vector value)))
+
+ (sequential? value)
+ (mapv (fn [[idx element]]
+ (walk current-query (conj node-path idx) element))
+ (map-indexed vector value))
+
+ :else
+ value))]
+ (walk current-query node-path value)))
diff --git a/src/metabase/agent_lib/join_spec.clj b/src/metabase/agent_lib/join_spec.clj
new file mode 100644
index 00000000000..8b5abcf2db5
--- /dev/null
+++ b/src/metabase/agent_lib/join_spec.clj
@@ -0,0 +1,80 @@
+(ns metabase.agent-lib.join-spec
+ "Shared parsing helpers for explicit join clauses."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [normalize-direction]]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(defn program-field-ref-id
+ "Return the field id when `form` is a canonical `[\"field\", id]` tuple."
+ [form]
+ (when (and (vector? form)
+ (= "field" (syntax/canonical-op-name (first form)))
+ (pos-int? (second form)))
+ (second form)))
+
+(defn parse-join-condition-pair
+ "Parse a join equality condition into a `[lhs-id rhs-id]` pair."
+ [condition]
+ (when (and (vector? condition)
+ (= "=" (syntax/canonical-op-name (first condition)))
+ (= 3 (count condition)))
+ (when-let [lhs-id (program-field-ref-id (second condition))]
+ (when-let [rhs-id (program-field-ref-id (nth condition 2))]
+ [lhs-id rhs-id]))))
+
+(defn parse-join-spec
+ "Parse nested join-clause wrappers into a normalized join-spec map."
+ [form]
+ (when (vector? form)
+ (let [[raw-op & args] form
+ op-name (syntax/canonical-op-name raw-op)]
+ (case op-name
+ "join-clause"
+ (let [target (first args)]
+ (when (and (vector? target)
+ (= "table" (syntax/canonical-op-name (first target)))
+ (pos-int? (second target)))
+ {:target-table-id (second target)
+ :conditions nil
+ :fields-mode nil
+ :strategy nil
+ :alias nil}))
+
+ "with-join-conditions"
+ (when-let [parsed (parse-join-spec (first args))]
+ (let [conditions (second args)
+ pairs (when (sequential? conditions)
+ (mapv parse-join-condition-pair conditions))]
+ (when (and (seq pairs)
+ (every? some? pairs))
+ (assoc parsed :conditions pairs))))
+
+ "with-join-fields"
+ (when-let [parsed (parse-join-spec (first args))]
+ (assoc parsed :fields-mode (second args)))
+
+ "with-join-strategy"
+ (when-let [parsed (parse-join-spec (first args))]
+ (assoc parsed :strategy (normalize-direction (second args))))
+
+ "with-join-alias"
+ (when-let [parsed (parse-join-spec (first args))]
+ (assoc parsed :alias (second args)))
+
+ nil))))
+
+(defn no-explicit-join-fields?
+ "True when a join spec does not explicitly request joined fields."
+ [fields-mode]
+ (or (nil? fields-mode)
+ (= "none" fields-mode)
+ (= :none fields-mode)))
+
+(defn implicit-join-compatible-fields-mode?
+ "True when explicit join fields still match implicit related-field semantics."
+ [fields-mode]
+ (or (no-explicit-join-fields? fields-mode)
+ (= "all" fields-mode)
+ (= :all fields-mode)))
diff --git a/src/metabase/agent_lib/mbql_integration.clj b/src/metabase/agent_lib/mbql_integration.clj
new file mode 100644
index 00000000000..ea6ded51085
--- /dev/null
+++ b/src/metabase/agent_lib/mbql_integration.clj
@@ -0,0 +1,76 @@
+(ns metabase.agent-lib.mbql-integration
+ "Temporary bridge between agent-lib and canonical MBQL lib APIs.
+
+ This namespace is the stable facade over the smaller MBQL integration units
+ under `metabase.agent-lib.mbql-integration.*`. It remains the single handoff
+ surface that agent-lib callers depend on, while the actual implementation is
+ split by concern."
+ (:require
+ [metabase.agent-lib.mbql-integration.common]
+ [metabase.agent-lib.mbql-integration.fields]
+ [metabase.agent-lib.mbql-integration.joins]
+ [metabase.agent-lib.mbql-integration.orderables]
+ [potemkin :as p]))
+
+(set! *warn-on-reflection* true)
+
+(p/import-vars
+ [metabase.agent-lib.mbql-integration.common
+ query?
+ extract-field-ids
+ expression-definition
+ normalized-name
+ column-names
+ dedupe-candidate-columns
+ current-query-field-candidates
+ prefer-single-candidate
+ unique-query-candidate
+ field-selection?
+ resolve-aggregation-selection]
+ [metabase.agent-lib.mbql-integration.fields
+ fields-by-table-id
+ field-target-table-id
+ field-path-start-ids
+ fk-path-to-table
+ numeric-type?
+ types-compatible?
+ candidate-has-resolution-lineage?
+ source-column-field-name
+ source-column-field-join-alias
+ source-column-original-field-id
+ source-column-original-field-name
+ source-column-original-field-join-alias
+ candidate-join-alias-for-field-id
+ synthesize-chained-related-field
+ multi-hop-lineage-candidate?
+ previous-stage-name-matches
+ previous-stage-lineage-matches
+ previous-stage-aggregation-matches
+ resolve-field-in-query]
+ [metabase.agent-lib.mbql-integration.orderables
+ matching-expression-ref
+ matching-previous-stage-column-by-name
+ current-stage-expression-ref
+ expression-ref-or-current-stage-column
+ ensure-query-expression-ref
+ orderable-column-key
+ current-query-orderable-candidates
+ column-field-ids
+ breakout-expression-name
+ previous-stage-aggregation-column
+ aggregation-ref-or-current-stage-column
+ aggregation-signature
+ aggregation-columns
+ aggregation-column-pairs
+ requested-orderable-field-ids
+ orderable-field-id->query-columns
+ orderable-field-ids->query-columns
+ field-id->breakout-expression-refs
+ field-ids->breakout-expression-ref
+ field-id->aggregation-columns
+ field-ids->aggregation-columns
+ resolve-field-like-orderable
+ resolve-orderable]
+ [metabase.agent-lib.mbql-integration.joins
+ implicitly-resolved-column?
+ redundant-implicit-join?])
diff --git a/src/metabase/agent_lib/mbql_integration/common.clj b/src/metabase/agent_lib/mbql_integration/common.clj
new file mode 100644
index 00000000000..125cdc42cb2
--- /dev/null
+++ b/src/metabase/agent_lib/mbql_integration/common.clj
@@ -0,0 +1,266 @@
+(ns metabase.agent-lib.mbql-integration.common
+ "Common MBQL bridge helpers shared by agent-lib field, filter, and orderable integration."
+ (:require
+ [metabase.lib.core :as lib]
+ [metabase.lib.util :as lib.util]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn query?
+ "Return true when `value` is a normalized pMBQL query map.
+
+ Agent-lib needs to distinguish queries from metadata maps, columns, and test
+ sentinels, but it should not inspect `:lib/type` directly. This helper keeps
+ that boundary on top of `lib/normalized-query-type` and guards the lib call so
+ non-query maps fail closed instead of blowing up.
+
+ Example:
+ (query? (lib/query metadata-provider table-metadata))
+ ;; => true
+
+ (query? {:id 1 :name \"Orders\"})
+ ;; => false"
+ [value]
+ (and (map? value)
+ (= :mbql/query (lib/normalized-query-type value))))
+
+(defn extract-field-ids
+ "Return the set of field ids referenced anywhere inside a pMBQL form.
+
+ Agent-lib compares expressions, aggregations, breakouts, and columns
+ semantically. The canonical lib helper already knows how to walk MBQL safely,
+ so this wrapper keeps callers out of tuple destructuring.
+
+ Example:
+ (extract-field-ids
+ [:+
+ {}
+ [:field {} 10]
+ [:field {} 20]])
+ ;; => #{10 20}"
+ [x]
+ (lib/all-field-ids x))
+
+(defn expression-definition
+ "Return `x` when it is a named expression definition, else nil.
+
+ Agent-lib accepts both fully defined expressions and references to existing
+ expressions. The MBQL layer already owns the naming rules, so this helper just
+ asks lib whether the form has an expression name and otherwise leaves the
+ structure alone.
+
+ Example:
+ (expression-definition
+ [:expression {} \"Net Amount\"
+ [:- {} [:field {} 1] [:field {} 2]]])
+ ;; => that same tuple
+
+ (expression-definition [:field {} 1])
+ ;; => nil"
+ [x]
+ (when (lib.util/expression-name x)
+ x))
+
+(defn normalized-name
+ "Normalize a display name for case-insensitive matching.
+
+ Agent-lib compares program-provided names against MBQL column names, aliases,
+ and ref names. Matching needs to be locale-stable, so this goes through
+ `lower-case-en` instead of generic lowercase.
+
+ Example:
+ (normalized-name \"Net Amount\")
+ ;; => \"net amount\"
+
+ (normalized-name nil)
+ ;; => nil"
+ [value]
+ (when (string? value)
+ (u/lower-case-en value)))
+
+(defn column-names
+ "Return every comparable name by which a query column may be recognized.
+
+ The same semantic column can surface through several lib metadata keys:
+ `:name`, `:lib/source-column-alias`, `:lib/original-name`,
+ `:lib/deduplicated-name`, and `:lib/ref-name`. Agent-lib collapses those into
+ one normalized set so later matching logic can ask one question: \"does this
+ column look like the requested thing?\"
+
+ Example:
+ (column-names {:name \"Title\"
+ :lib/source-column-alias \"product_title\"
+ :lib/ref-name \"Products -> Title\"})
+ ;; => #{\"title\" \"product_title\" \"products -> title\"}"
+ [column]
+ (->> [(:name column)
+ (:lib/source-column-alias column)
+ (:lib/original-name column)
+ (:lib/deduplicated-name column)
+ (:lib/ref-name column)]
+ (keep normalized-name)
+ set))
+
+(defn column-join-alias
+ "Return the join alias attached to a visible/query column, if any.
+
+ Lib can surface the active alias on `:lib/join-alias` or preserve the
+ original alias on `:lib/original-join-alias`. Agent-lib only needs one answer
+ when deciding whether a match came from an explicit join.
+
+ Example:
+ (column-join-alias {:lib/join-alias \"Products\"})
+ ;; => \"Products\""
+ [column]
+ ((some-fn :lib/join-alias :lib/original-join-alias) column))
+
+(defn- candidate-column-key
+ [column]
+ [(:id column)
+ (:table-id column)
+ (:name column)
+ (:source-field column)
+ (:source-field-name column)
+ (:source-field-join-alias column)
+ (column-join-alias column)
+ (:fk-field-id column)
+ (:fk-field-name column)
+ (:fk-join-alias column)
+ (:lib/original-fk-field-id column)
+ (:lib/original-fk-field-name column)
+ (:lib/original-fk-join-alias column)
+ (:lib/source column)
+ (:lib/source-uuid column)
+ (:lib/expression-name column)
+ (:lib/temporal-unit column)
+ (:inherited-temporal-unit column)])
+
+(defn dedupe-candidate-columns
+ "Deduplicate candidate columns while preserving the first structurally
+ distinct representative.
+
+ Agent-lib merges columns from several lib surfaces such as `visible-columns`,
+ `fieldable-columns`, and `orderable-columns`. The same semantic column often
+ appears more than once. We collapse duplicates, but we keep join alias, FK
+ lineage, source UUID, expression name, and temporal metadata in the key so we
+ do not merge columns that only look similar.
+
+ Example:
+ (dedupe-candidate-columns
+ [{:id 10 :name \"Title\"}
+ {:id 10 :name \"Title\"}
+ {:id 10 :name \"Title\" :lib/join-alias \"Products\"}])
+ ;; => keeps the plain column and the joined column"
+ [columns]
+ (vals
+ (reduce (fn [acc column]
+ (let [k (candidate-column-key column)]
+ (if (contains? acc k)
+ acc
+ (assoc acc k column))))
+ {}
+ columns)))
+
+(defn current-query-field-candidates
+ "Return the deduplicated field-like columns visible from the current query
+ stage.
+
+ Agent programs name semantic fields, but the current stage may expose them as
+ visible columns, fieldable columns, filterable columns, breakoutable columns,
+ or orderable columns. This helper builds the combined search space used by
+ field resolution.
+
+ Example:
+ A field that is hidden from the result set but still filterable remains
+ resolvable because this helper includes `lib/filterable-columns`."
+ [query]
+ (->> [(lib/fieldable-columns query)
+ (lib/visible-columns query)
+ (lib/filterable-columns query)
+ (lib/breakoutable-columns query)
+ (lib/orderable-columns query)]
+ (apply concat)
+ dedupe-candidate-columns
+ vec))
+
+(defn prefer-single-candidate
+ "Return the single deterministic winner from an otherwise ambiguous candidate
+ set, or nil when agent-lib should keep looking.
+
+ This is the extracted tie-breaker used by `unique-query-candidate`:
+ prefer one unaliased column over joined columns, then prefer one explicit
+ field over an FK-derived expansion.
+
+ Example:
+ Given `[Orders.ID, Products.ID via implicit join]`, the unaliased
+ `Orders.ID` candidate wins."
+ [candidates]
+ (or (when-let [unaliased (not-empty (remove column-join-alias candidates))]
+ (when-not (next unaliased)
+ (first unaliased)))
+ (when-let [explicit (not-empty (remove :fk-field-id candidates))]
+ (when-not (next explicit)
+ (first explicit)))))
+
+(defn unique-query-candidate
+ "Return a single candidate after applying agent-lib's tie-break rules.
+
+ Some lib lookups intentionally return more than one plausible column. When
+ that happens, agent-lib prefers:
+ 1. a single unaliased column over a joined one
+ 2. a single column without FK lineage over a related-field expansion
+
+ This keeps agent behavior deterministic without forcing callers to know the
+ ranking policy.
+
+ Example:
+ If candidates are `[Orders.ID, Products.ID via implicit join]`, the plain
+ `Orders.ID` column wins."
+ [candidates]
+ (or (when (= 1 (count candidates))
+ (first candidates))
+ (prefer-single-candidate (vec candidates))))
+
+(defn field-selection?
+ "Return true when `selection` behaves like a field for `with-fields`.
+
+ `with-fields` in agent-lib accepts both literal field clauses and lib column
+ maps that can be turned back into field refs. This helper centralizes that
+ check so callers do not branch on raw ref tags.
+
+ Example:
+ (field-selection? [:field {} 10])
+ ;; => true
+
+ (field-selection? {:id 10 :name \"Title\"})
+ ;; => true when `(lib/ref ...)` yields a field ref"
+ [selection]
+ (or (lib.util/field-clause? selection)
+ (when (map? selection)
+ (let [selection-ref (try
+ (lib/ref selection)
+ (catch Exception _
+ nil))]
+ (boolean
+ (or (some-> selection-ref lib/field-ref-id)
+ (some-> selection-ref lib/field-ref-name)))))))
+
+(defn resolve-aggregation-selection
+ "Resolve an aggregation selection back to the visible current-stage column
+ when one exists.
+
+ Some agent operations conceptually choose \"the aggregated result column\",
+ while the syntax still names it as an aggregation ref. When lib can map that
+ ref to the visible column, we use the column so downstream helpers treat it
+ like any other selected field.
+
+ Example:
+ Given a query with `(lib/aggregate (lib/count))`, resolving
+ `(lib/aggregation-ref query 0)` returns the visible count column instead of
+ leaving the raw ref in place."
+ [query selection]
+ (if (lib.util/clause-of-type? selection :aggregation)
+ (or (lib/find-visible-column-for-ref query selection)
+ selection)
+ selection))
diff --git a/src/metabase/agent_lib/mbql_integration/field_resolution.clj b/src/metabase/agent_lib/mbql_integration/field_resolution.clj
new file mode 100644
index 00000000000..8ff51200549
--- /dev/null
+++ b/src/metabase/agent_lib/mbql_integration/field_resolution.clj
@@ -0,0 +1,314 @@
+(ns metabase.agent-lib.mbql-integration.field-resolution
+ "Field-resolution helpers for the agent-lib MBQL bridge."
+ (:require
+ [clojure.set :as set]
+ [metabase.agent-lib.mbql-integration.common :as common]
+ [metabase.lib.core :as lib]
+ [metabase.lib.util :as lib.util]))
+
+(set! *warn-on-reflection* true)
+
+(defn fields-by-table-id
+ "Group metadata fields by table id for lookup-heavy resolution helpers."
+ [fields-by-id]
+ (reduce-kv (fn [acc _field-id field]
+ (update acc (:table-id field) (fnil conj []) field))
+ {}
+ fields-by-id))
+
+(defn field-target-table-id
+ "Return the table id reached by following a field's FK target metadata."
+ [fields-by-id field-id]
+ (when-let [target-field-id (some-> (get fields-by-id field-id) :fk-target-field-id)]
+ (:table-id (get fields-by-id target-field-id))))
+
+(defn field-path-start-ids
+ "Return the field ids that can anchor lineage for a surfaced query column."
+ [column]
+ (->> [(:source-field column)
+ (:fk-field-id column)
+ (:lib/original-fk-field-id column)
+ (:id column)]
+ (filter pos-int?)
+ distinct))
+
+(defn fk-path-to-table
+ "Return the unique FK chain from `start-field-id` to `target-table-id`, or
+ nil when none or more than one plausible path exists.
+
+ The local `walk-path` stays nested because it is the only genuinely recursive
+ helper left in this file."
+ [fields-by-id start-field-id target-table-id]
+ (let [fields-by-table (fields-by-table-id fields-by-id)]
+ (letfn [(walk-path [field-id visited depth]
+ (when (and (pos-int? field-id)
+ (not (visited field-id))
+ (< depth 4))
+ (when-let [next-table-id (field-target-table-id fields-by-id field-id)]
+ (if (= next-table-id target-table-id)
+ [[field-id]]
+ (not-empty
+ (mapcat (fn [candidate]
+ (map #(into [field-id] %)
+ (or (walk-path (:id candidate)
+ (conj visited field-id)
+ (inc depth))
+ [])))
+ (get fields-by-table next-table-id)))))))]
+ (let [paths (vec (walk-path start-field-id #{} 0))]
+ (when (= 1 (count paths))
+ (first paths))))))
+
+(defn numeric-type?
+ "True when `type` is one of the numeric MBQL types agent-lib treats as
+ interchangeable for previous-stage reuse."
+ [type]
+ (contains? #{"type/BigInteger"
+ "type/Decimal"
+ "type/Float"
+ "type/Integer"
+ "type/Number"}
+ (cond
+ (keyword? type) (str (namespace type) "/" (name type))
+ (string? type) type
+ :else nil)))
+
+(defn types-compatible?
+ "True when `raw-field` and `candidate` can safely stand in for each other by
+ type."
+ [raw-field candidate]
+ (let [raw-type ((some-fn :effective-type :base-type) raw-field)
+ candidate-type ((some-fn :effective-type :base-type) candidate)]
+ (or (nil? raw-type)
+ (nil? candidate-type)
+ (= raw-type candidate-type)
+ (and (numeric-type? raw-type)
+ (numeric-type? candidate-type)))))
+
+(defn candidate-has-resolution-lineage?
+ "True when a candidate column carries lineage metadata that makes it a better
+ resolution target than a bare id/name match."
+ [column]
+ (boolean
+ (or (:fk-field-id column)
+ (:fk-field-name column)
+ (:fk-join-alias column)
+ (:lib/original-fk-field-id column)
+ (:lib/original-fk-field-name column)
+ (:lib/original-fk-join-alias column)
+ (common/column-join-alias column)
+ (:lib/source-uuid column)
+ (:lib/expression-name column)
+ (= :source/previous-stage (:lib/source column)))))
+
+(defn source-column-field-name
+ "Return the best human-readable field name carried by a surfaced source
+ column."
+ [column]
+ ((some-fn :lib/source-column-alias
+ :name
+ :lib/original-name
+ :fk-field-name
+ :lib/original-fk-field-name)
+ column))
+
+(defn source-column-field-join-alias
+ "Return the most specific join alias associated with a surfaced source
+ column."
+ [column]
+ ((some-fn common/column-join-alias :fk-join-alias :lib/original-fk-join-alias) column))
+
+(defn source-column-original-field-id
+ "Return the original field id carried by a surfaced source column."
+ [column]
+ ((some-fn :lib/original-fk-field-id :source-field) column))
+
+(defn source-column-original-field-name
+ "Return the original field name carried by a surfaced source column."
+ [column]
+ ((some-fn :lib/original-fk-field-name :source-field-name) column))
+
+(defn source-column-original-field-join-alias
+ "Return the original join alias carried by a surfaced source column."
+ [column]
+ ((some-fn :lib/original-fk-join-alias :source-field-join-alias) column))
+
+(defn candidate-join-alias-for-field-id
+ "Return a unique join alias for columns whose lineage includes `field-id`."
+ [candidates field-id]
+ (when (pos-int? field-id)
+ (let [matches (->> candidates
+ (keep (fn [candidate]
+ (when (some #{field-id} (field-path-start-ids candidate))
+ ((some-fn common/column-join-alias
+ :fk-join-alias
+ :lib/original-fk-join-alias)
+ candidate))))
+ distinct
+ vec)]
+ (when (= 1 (count matches))
+ (first matches)))))
+
+(defn synthesize-chained-related-field
+ "Project lineage from a surfaced source column back onto the requested raw
+ field shape.
+
+ This is the agent-lib-specific bridge for multi-hop related fields until lib
+ exposes a canonical helper for the same behavior."
+ [fields-by-id candidates raw-field source-column path]
+ (let [immediate-field-id (last path)
+ immediate-field (get fields-by-id immediate-field-id)
+ original-field-id (or (source-column-original-field-id source-column)
+ (when (> (count path) 1)
+ (first path)))
+ original-field (get fields-by-id original-field-id)
+ original-field-name (or (source-column-original-field-name source-column)
+ (:name original-field))
+ immediate-field-name (or (:name immediate-field)
+ (source-column-field-name source-column))
+ immediate-join-alias (or (source-column-field-join-alias source-column)
+ (candidate-join-alias-for-field-id candidates immediate-field-id)
+ (candidate-join-alias-for-field-id candidates original-field-id))
+ original-join-alias (or (source-column-original-field-join-alias source-column)
+ (candidate-join-alias-for-field-id candidates original-field-id)
+ immediate-join-alias)]
+ (cond-> raw-field
+ immediate-field-id (assoc :fk-field-id immediate-field-id)
+ immediate-field-name (assoc :fk-field-name immediate-field-name)
+ immediate-join-alias (assoc :fk-join-alias immediate-join-alias)
+ original-field-id (assoc :lib/original-fk-field-id original-field-id)
+ original-field-name (assoc :lib/original-fk-field-name original-field-name)
+ original-join-alias (assoc :lib/original-fk-join-alias original-join-alias))))
+
+(defn multi-hop-lineage-candidate?
+ "True when a synthesized related field captures more than one hop of lineage."
+ [column]
+ (boolean
+ (or (let [immediate-field-id (:fk-field-id column)
+ original-field-id (:lib/original-fk-field-id column)]
+ (and (pos-int? immediate-field-id)
+ (pos-int? original-field-id)
+ (not= immediate-field-id original-field-id)))
+ (let [immediate-field-name (:fk-field-name column)
+ original-field-name (:lib/original-fk-field-name column)]
+ (and (string? immediate-field-name)
+ (string? original-field-name)
+ (not= immediate-field-name original-field-name)))
+ (let [immediate-join-alias (:fk-join-alias column)
+ original-join-alias (:lib/original-fk-join-alias column)]
+ (and (string? immediate-join-alias)
+ (string? original-join-alias)
+ (not= immediate-join-alias original-join-alias))))))
+
+(defn previous-stage-name-matches
+ "Return previous-stage candidates whose comparable names overlap the requested
+ raw field."
+ [raw-field previous-stage-candidates]
+ (let [raw-field-names (common/column-names raw-field)]
+ (->> previous-stage-candidates
+ (filter (fn [candidate]
+ (seq (set/intersection raw-field-names (common/column-names candidate)))))
+ vec)))
+
+(defn previous-stage-lineage-matches
+ "Return previous-stage candidates whose lineage points back to `raw-field`.
+
+ This is stricter than `types-compatible?`: appended stages may expose only
+ aggregations, and agent-lib should only treat those as substitutes for a raw
+ field when lib metadata preserves a real source-field lineage for that field."
+ [raw-field previous-stage-candidates]
+ (let [raw-field-id (:id raw-field)]
+ (->> previous-stage-candidates
+ (filter (fn [candidate]
+ (and (pos-int? raw-field-id)
+ (some #{raw-field-id} (field-path-start-ids candidate)))))
+ vec)))
+
+(defn previous-stage-aggregation-matches
+ "Return previous-stage columns whose `:lib/source-uuid` comes from an
+ aggregation over `raw-field`."
+ [query raw-field previous-stage-candidates]
+ (when-let [previous-stage-number (some-> query (lib.util/previous-stage-number -1))]
+ (let [raw-field-id (:id raw-field)
+ matching-uuids (->> (map vector
+ (lib/aggregations query previous-stage-number)
+ (lib/aggregations-metadata query previous-stage-number))
+ (keep (fn [[aggregation metadata]]
+ (when (and (pos-int? raw-field-id)
+ (some #{raw-field-id} (common/extract-field-ids aggregation)))
+ (:lib/source-uuid metadata))))
+ set)]
+ (->> previous-stage-candidates
+ (filter #(contains? matching-uuids (:lib/source-uuid %)))
+ vec))))
+
+(defn- chained-related-candidates
+ "Compute chained related-field candidates by walking FK paths from each
+ candidate back to the raw field's table."
+ [fields-by-id candidates raw-field]
+ (let [target-table-id (:table-id raw-field)]
+ (->> candidates
+ (keep (fn [column]
+ (when-let [path (some #(fk-path-to-table fields-by-id % target-table-id)
+ (field-path-start-ids column))]
+ (synthesize-chained-related-field fields-by-id candidates raw-field column path))))
+ common/dedupe-candidate-columns
+ vec)))
+
+(defn- previous-stage-candidates
+ "Compute previous-stage candidates that are type-compatible with `raw-field`."
+ [candidates raw-field]
+ (->> candidates
+ (filter #(and (= :source/previous-stage (:lib/source %))
+ (types-compatible? raw-field %)))
+ vec))
+
+(defn- try-strategies
+ "Try resolution strategies in priority order, returning the first match."
+ [strategies]
+ (some (fn [{:keys [resolve-fn]}] (resolve-fn)) strategies))
+
+(defn resolve-field-in-query
+ "Resolve a metadata field against what the current query stage actually
+ exposes.
+
+ Resolution strategies are tried in priority order; each is computed lazily
+ so we stop as soon as one succeeds."
+ ([fields-by-id query raw-field]
+ (resolve-field-in-query fields-by-id query raw-field (common/current-query-field-candidates query)))
+ ([fields-by-id query raw-field candidates]
+ (let [query (when (common/query? query) query)
+ by-id+tbl (fn [] (filterv #(and (= (:id raw-field) (:id %))
+ (= (:table-id raw-field) (:table-id %)))
+ candidates))
+ by-id (fn [] (filterv #(= (:id raw-field) (:id %)) candidates))
+ lined (fn [cs] (filterv candidate-has-resolution-lineage? cs))
+ chained (delay (chained-related-candidates fields-by-id candidates raw-field))
+ prev-stage (delay (previous-stage-candidates candidates raw-field))]
+ (or (try-strategies
+ [{:name :multi-hop-chained-lineage
+ :resolve-fn #(common/unique-query-candidate (filterv multi-hop-lineage-candidate? @chained))}
+ {:name :exact-id+table-with-lineage
+ :resolve-fn #(common/unique-query-candidate (lined (by-id+tbl)))}
+ {:name :exact-id-with-lineage
+ :resolve-fn #(common/unique-query-candidate (lined (by-id)))}
+ {:name :previous-stage-lineage-or-aggregation
+ :resolve-fn #(common/unique-query-candidate
+ (->> (concat (previous-stage-lineage-matches raw-field @prev-stage)
+ (previous-stage-aggregation-matches query raw-field @prev-stage))
+ common/dedupe-candidate-columns vec))}
+ {:name :previous-stage-name
+ :resolve-fn #(common/unique-query-candidate (previous-stage-name-matches raw-field @prev-stage))}
+ {:name :chained-related-fields
+ :resolve-fn #(common/unique-query-candidate @chained)}
+ {:name :lib-find-matching-column
+ :resolve-fn #(when query (lib/find-matching-column query -1 raw-field candidates))}
+ {:name :exact-id+table
+ :resolve-fn #(common/unique-query-candidate (by-id+tbl))}
+ {:name :exact-id
+ :resolve-fn #(common/unique-query-candidate (by-id))}])
+ (throw (ex-info (str "Field " (pr-str (:name raw-field))
+ " (id " (pr-str (:id raw-field))
+ ") is not available in the current query stage.")
+ {:field raw-field
+ :query query}))))))
diff --git a/src/metabase/agent_lib/mbql_integration/fields.clj b/src/metabase/agent_lib/mbql_integration/fields.clj
new file mode 100644
index 00000000000..0b0cdf48382
--- /dev/null
+++ b/src/metabase/agent_lib/mbql_integration/fields.clj
@@ -0,0 +1,29 @@
+(ns metabase.agent-lib.mbql-integration.fields
+ "Facade over field-resolution MBQL bridge helpers."
+ (:require
+ [metabase.agent-lib.mbql-integration.field-resolution]
+ [potemkin :as p]))
+
+(set! *warn-on-reflection* true)
+
+(p/import-vars
+ [metabase.agent-lib.mbql-integration.field-resolution
+ fields-by-table-id
+ field-target-table-id
+ field-path-start-ids
+ fk-path-to-table
+ numeric-type?
+ types-compatible?
+ candidate-has-resolution-lineage?
+ source-column-field-name
+ source-column-field-join-alias
+ source-column-original-field-id
+ source-column-original-field-name
+ source-column-original-field-join-alias
+ candidate-join-alias-for-field-id
+ synthesize-chained-related-field
+ multi-hop-lineage-candidate?
+ previous-stage-name-matches
+ previous-stage-lineage-matches
+ previous-stage-aggregation-matches
+ resolve-field-in-query])
diff --git a/src/metabase/agent_lib/mbql_integration/joins.clj b/src/metabase/agent_lib/mbql_integration/joins.clj
new file mode 100644
index 00000000000..fcb02a88305
--- /dev/null
+++ b/src/metabase/agent_lib/mbql_integration/joins.clj
@@ -0,0 +1,60 @@
+(ns metabase.agent-lib.mbql-integration.joins
+ "Explicit-vs-implicit join integration helpers for the agent-lib MBQL bridge."
+ (:require
+ [metabase.agent-lib.join-spec :as join-spec]
+ [metabase.agent-lib.mbql-integration.common :as common]
+ [metabase.agent-lib.mbql-integration.fields :as fields]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(defn implicitly-resolved-column?
+ "True when `column` represents the target side of an implicit join strongly
+ enough that agent-lib can treat an explicit join request as redundant."
+ [column target-table-id]
+ (boolean
+ (and (map? column)
+ (or (= (:table-id column) target-table-id)
+ (= :source/previous-stage (:lib/source column)))
+ (or (:fk-field-id column)
+ (:lib/original-fk-field-id column)
+ (= :source/previous-stage (:lib/source column)))
+ (nil? (:join-alias column)))))
+
+(defn- valid-implicit-join-spec?
+ "True when a parsed join spec has the shape of a potentially redundant implicit join."
+ [{:keys [target-table-id conditions fields-mode strategy alias]}]
+ (and (pos-int? target-table-id)
+ (nil? alias)
+ (or (nil? strategy) (= "left-join" strategy))
+ (join-spec/implicit-join-compatible-fields-mode? fields-mode)
+ (= 1 (count conditions))))
+
+(defn- target-field-for-join
+ "Return the field metadata for the side of the join condition that points at the
+ target table, or nil."
+ [fields-by-id target-table-id [lhs-id rhs-id]]
+ (let [lhs-table-id (some-> (get fields-by-id lhs-id) :table-id)
+ rhs-table-id (some-> (get fields-by-id rhs-id) :table-id)
+ field-id (cond
+ (= lhs-table-id target-table-id) lhs-id
+ (= rhs-table-id target-table-id) rhs-id)]
+ (some-> field-id fields-by-id)))
+
+(defn redundant-implicit-join?
+ "Return true when a structured explicit join would be redundant because the
+ current query already exposes the target side through implicit join behavior."
+ [fields-by-id current-query operation]
+ (let [join-spec (when (and (common/query? current-query)
+ (vector? operation)
+ (= "join" (syntax/canonical-op-name (first operation)))
+ (= 2 (count operation)))
+ (join-spec/parse-join-spec (second operation)))
+ target-field (when (and join-spec (valid-implicit-join-spec? join-spec))
+ (target-field-for-join fields-by-id
+ (:target-table-id join-spec)
+ (first (:conditions join-spec))))
+ target-col (when target-field
+ (fields/resolve-field-in-query fields-by-id current-query target-field))]
+ (when target-col
+ (implicitly-resolved-column? target-col (:target-table-id join-spec)))))
diff --git a/src/metabase/agent_lib/mbql_integration/orderables.clj b/src/metabase/agent_lib/mbql_integration/orderables.clj
new file mode 100644
index 00000000000..9651113b2ee
--- /dev/null
+++ b/src/metabase/agent_lib/mbql_integration/orderables.clj
@@ -0,0 +1,361 @@
+(ns metabase.agent-lib.mbql-integration.orderables
+ "Expression and orderable resolution helpers for the agent-lib MBQL bridge."
+ (:require
+ [clojure.string :as str]
+ [metabase.agent-lib.common.errors :as errors]
+ [metabase.agent-lib.mbql-integration.common :as common]
+ [metabase.lib.core :as lib]
+ [metabase.lib.expression :as lib.expression]
+ [metabase.lib.ref :as lib.ref]
+ [metabase.lib.schema.util :as lib.schema.util]
+ [metabase.lib.util :as lib.util]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private aggregation-orderable-ops
+ #{:count
+ :sum
+ :avg
+ :min
+ :max
+ :distinct
+ :median
+ :stddev
+ :var
+ :count-where
+ :sum-where
+ :distinct-where
+ :share
+ :percentile
+ :cum-count
+ :cum-sum
+ :metric})
+
+(defn- normalize-expression-comparison
+ "Strip volatile metadata from an expression tree before structural
+ comparison."
+ [expr]
+ (cond
+ (map? expr)
+ (-> expr
+ (dissoc :lib/uuid :lib/expression-name :display-name :name)
+ (update-vals normalize-expression-comparison))
+
+ (vector? expr)
+ (mapv normalize-expression-comparison expr)
+
+ (sequential? expr)
+ (mapv normalize-expression-comparison expr)
+
+ :else
+ expr))
+
+(defn matching-expression-ref
+ "Return the expression ref for the current query expression that is
+ semantically equal to `orderable`."
+ [query orderable]
+ (let [normalized-orderable (normalize-expression-comparison
+ (lib.schema.util/remove-lib-uuids orderable))]
+ (some (fn [expr]
+ (when-let [expr-name (lib.util/expression-name expr)]
+ (when (= normalized-orderable
+ (normalize-expression-comparison
+ (lib.schema.util/remove-lib-uuids expr)))
+ (lib/expression-ref query expr-name))))
+ (lib/expressions query))))
+
+(defn matching-previous-stage-column-by-name
+ "Return a unique previous-stage column whose visible names match
+ `column-name`."
+ [query column-name]
+ (let [normalized-column-name (common/normalized-name column-name)]
+ (when normalized-column-name
+ (->> [(lib/returned-columns query)
+ (common/current-query-field-candidates query)]
+ (apply concat)
+ common/dedupe-candidate-columns
+ (filter #(and (= :source/previous-stage (:lib/source %))
+ (contains? (common/column-names %) normalized-column-name)))
+ vec
+ common/unique-query-candidate))))
+
+(defn current-stage-expression-ref
+ "Return an expression ref when `expr-name` still resolves as a current-stage
+ expression name."
+ [query expr-name]
+ (when (lib.expression/maybe-resolve-expression query expr-name)
+ (lib/expression-ref query expr-name)))
+
+(defn expression-ref-or-current-stage-column
+ "Resolve an expression name to the thing the current stage can actually order
+ or project by."
+ [query expr-name]
+ (or (current-stage-expression-ref query expr-name)
+ (matching-previous-stage-column-by-name query expr-name)
+ (errors/lookup-error! (str "Expression not found in the current query stage: " (pr-str expr-name))
+ {:expression-name expr-name})))
+
+(defn ensure-query-expression-ref
+ "Ensure a query contains an expression definition for `expression`, returning
+ `[query expr-ref]`."
+ [query expression prefix]
+ (if-let [expr-ref (matching-expression-ref query expression)]
+ [query expr-ref]
+ (let [expr-name (str "__" prefix "_expression_" (inc (count (lib/expressions query))))
+ query' (lib/expression query expr-name expression)]
+ [query' (lib/expression-ref query' expr-name)])))
+
+(defn orderable-column-key
+ "Return the structural identity used when deduplicating orderable columns."
+ [column]
+ [(:id column)
+ (:name column)
+ (:lib/source column)
+ (:lib/source-uuid column)
+ (:source-field column)
+ (:fk-field-id column)
+ (:lib/original-fk-field-id column)
+ (:lib/expression-name column)])
+
+(defn current-query-orderable-candidates
+ "Return the distinct orderable columns for the current stage."
+ [query]
+ (vals
+ (reduce (fn [acc column]
+ (assoc acc (orderable-column-key column) column))
+ {}
+ (lib/orderable-columns query))))
+
+(defn column-field-ids
+ "Return every field id represented by a column's lineage metadata."
+ [column]
+ (->> [(:id column)
+ (:source-field column)
+ (:fk-field-id column)
+ (:lib/original-fk-field-id column)]
+ (filter pos-int?)
+ set))
+
+(defn breakout-expression-name
+ "Return the expression name for a breakout when the breakout is an
+ expression-ref."
+ [breakout]
+ (when (lib.util/clause-of-type? breakout :expression)
+ (lib.ref/expression-ref-name breakout)))
+
+(defn previous-stage-aggregation-column
+ "Resolve an aggregation index to the column that represents that aggregation
+ in the previous stage."
+ [query ag-index]
+ (when-let [previous-stage-number (lib.util/previous-stage-number query -1)]
+ (let [candidate-columns (concat (lib/returned-columns query)
+ (lib/filterable-columns query)
+ (lib/orderable-columns query)
+ (lib/visible-columns query))]
+ (or (try
+ (lib/find-matching-column query
+ -1
+ (lib/aggregation-ref query previous-stage-number ag-index)
+ candidate-columns)
+ (catch Exception _
+ nil))
+ (when-let [source-uuid (:lib/source-uuid
+ (get (vec (lib/aggregations-metadata query previous-stage-number))
+ ag-index))]
+ (some #(when (= source-uuid (:lib/source-uuid %))
+ %)
+ candidate-columns))))))
+
+(defn aggregation-ref-or-current-stage-column
+ "Resolve an aggregation index to the right current-stage orderable or
+ projectable value."
+ [query ag-index]
+ (if (get (vec (lib/aggregations query)) ag-index)
+ (lib/aggregation-ref query ag-index)
+ (or (previous-stage-aggregation-column query ag-index)
+ (errors/lookup-error! (str "Undefined aggregation " ag-index)
+ {:aggregation-index ag-index
+ :stage-number -1}))))
+
+(defn aggregation-signature
+ "Return the semantic signature used to compare aggregation definitions."
+ [aggregation]
+ [(first aggregation) (common/extract-field-ids aggregation)])
+
+(defn aggregation-columns
+ "Return the current-stage orderable columns backed by aggregation outputs."
+ [query]
+ (filterv #(= (:lib/source %) :source/aggregations)
+ (lib/orderable-columns query)))
+
+(defn aggregation-column-pairs
+ "Pair each current-stage aggregation with the orderable column that lib maps
+ it to, when one exists."
+ [query]
+ (let [agg-cols (aggregation-columns query)]
+ (->> (lib/aggregations query)
+ (map-indexed (fn [idx aggregation]
+ [idx aggregation
+ (lib/find-matching-column query -1 (lib/aggregation-ref query idx) agg-cols)]))
+ (filter #(nth % 2))
+ vec)))
+
+(defn requested-orderable-field-ids
+ "Return the field ids semantically referenced by an orderable request."
+ [requested]
+ (cond
+ (map? requested)
+ (column-field-ids requested)
+
+ (lib.util/field-clause? requested)
+ (common/extract-field-ids requested)
+
+ :else
+ #{}))
+
+(defn orderable-field-id->query-columns
+ "Return current-stage orderable columns whose lineage includes `field-id`."
+ [query field-id]
+ (->> (current-query-orderable-candidates query)
+ (filterv #(contains? (column-field-ids %) field-id))))
+
+(defn orderable-field-ids->query-columns
+ "Return unique current-stage orderable columns whose lineage overlaps any of
+ `field-ids`."
+ [query field-ids]
+ (->> field-ids
+ (mapcat #(orderable-field-id->query-columns query %))
+ (reduce (fn [acc column]
+ (assoc acc (orderable-column-key column) column))
+ {})
+ vals
+ vec))
+
+(defn field-id->breakout-expression-refs
+ "Return breakout expression refs whose underlying expression touches
+ `field-id`."
+ [query field-id]
+ (let [expr-fields (->> (lib/expressions query)
+ (keep (fn [expr]
+ (when-let [expr-name (lib.util/expression-name expr)]
+ [expr-name (common/extract-field-ids expr)])))
+ (into {}))]
+ (->> (lib/breakouts query)
+ (keep breakout-expression-name)
+ (filter #(contains? (get expr-fields % #{}) field-id))
+ distinct
+ (mapv #(lib/expression-ref query %)))))
+
+(defn field-ids->breakout-expression-ref
+ "Return the unique breakout expression ref implied by `field-ids`, if one can
+ be chosen unambiguously."
+ [query field-ids]
+ (let [matches (->> field-ids
+ (mapcat #(field-id->breakout-expression-refs query %))
+ (reduce (fn [acc expr-ref]
+ (assoc acc (pr-str expr-ref) expr-ref))
+ {})
+ vals
+ vec)]
+ (when (= 1 (count matches))
+ (first matches))))
+
+(defn field-id->aggregation-columns
+ "Return aggregation output columns whose defining aggregation references
+ `field-id`."
+ [query field-id]
+ (->> (aggregation-column-pairs query)
+ (keep (fn [[_idx aggregation column]]
+ (when (contains? (common/extract-field-ids aggregation) field-id)
+ column)))
+ vec))
+
+(defn field-ids->aggregation-columns
+ "Return unique aggregation output columns referenced by any of `field-ids`."
+ [query field-ids]
+ (->> field-ids
+ (mapcat #(field-id->aggregation-columns query %))
+ (reduce (fn [acc column]
+ (assoc acc (orderable-column-key column) column))
+ {})
+ vals
+ vec))
+
+(defn- sole-match
+ "Return `[query match]` when `candidates` has exactly one element, else nil."
+ [query candidates]
+ (when (= 1 (count candidates))
+ [query (first candidates)]))
+
+(defn resolve-field-like-orderable
+ "Resolve a field-like orderable request against the current query stage."
+ [query requested]
+ (let [field-ids (requested-orderable-field-ids requested)
+ query-match (orderable-field-ids->query-columns query field-ids)]
+ (or (sole-match query query-match)
+ (when-let [ref (field-ids->breakout-expression-ref query field-ids)]
+ [query ref])
+ (sole-match query (field-ids->aggregation-columns query field-ids))
+ (sole-match query (aggregation-columns query))
+ [query requested])))
+
+(defn- resolve-aggregation-orderable
+ "Resolve an aggregation-like orderable by progressively narrowing candidates."
+ [query orderable]
+ (let [agg-cols (aggregation-columns query)
+ op-name (name (first orderable))
+ by-name (filterv #(str/starts-with? (or (:name %) "") op-name) agg-cols)
+ target-sig (aggregation-signature orderable)
+ by-signature (when (> (count by-name) 1)
+ (->> (aggregation-column-pairs query)
+ (keep (fn [[_idx aggregation column]]
+ (when (and (some #{column} by-name)
+ (= target-sig (aggregation-signature aggregation)))
+ column)))
+ vec))]
+ (or (sole-match query agg-cols)
+ (sole-match query by-name)
+ (sole-match query by-signature)
+ (cond
+ (> (count by-signature) 1)
+ (errors/lookup-error! (str "Ambiguous: multiple identical '" op-name "' aggregations.")
+ {:operator op-name}
+ {:available (mapv :name agg-cols)
+ :suggestion "Use orderable-columns to pick the specific column."})
+
+ (> (count by-name) 1)
+ (errors/lookup-error! (str "No orderable column matched aggregation '" op-name "' on those fields.")
+ {:operator op-name}
+ {:available (mapv :name agg-cols)})
+
+ :else
+ (errors/lookup-error! (str "No orderable column found for aggregation '" op-name "'.")
+ {:operator op-name}
+ {:available (mapv :name agg-cols)})))))
+
+(defn resolve-orderable
+ "Resolve an agent-program orderable against the current query, returning
+ `[query resolved-orderable]`."
+ [query orderable]
+ (cond
+ (map? orderable)
+ (resolve-field-like-orderable query orderable)
+
+ (and (lib.util/clause? orderable)
+ (contains? aggregation-orderable-ops (first orderable)))
+ (resolve-aggregation-orderable query orderable)
+
+ (lib.util/field-clause? orderable)
+ (resolve-field-like-orderable query orderable)
+
+ (lib.util/clause-of-type? orderable :expression)
+ [query orderable]
+
+ (lib.util/clause-of-type? orderable :aggregation)
+ [query orderable]
+
+ (lib.util/clause? orderable)
+ (ensure-query-expression-ref query orderable "order_by")
+
+ :else
+ [query orderable]))
diff --git a/src/metabase/agent_lib/refs.clj b/src/metabase/agent_lib/refs.clj
new file mode 100644
index 00000000000..d45abc5f8f5
--- /dev/null
+++ b/src/metabase/agent_lib/refs.clj
@@ -0,0 +1,83 @@
+(ns metabase.agent-lib.refs
+ "Walk a structured program and collect every metadata id (table, field, card, metric, measure)
+ it actually references. Used by the runtime builder to scope metadata loading to entities the
+ program touches, instead of materializing the whole database."
+ (:require
+ [metabase.agent-lib.schema :as schema]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private empty-refs
+ {:table-ids #{}
+ :field-ids #{}
+ :card-ids #{}
+ :metric-ids #{}
+ :measure-ids #{}})
+
+(declare collect-program-refs)
+
+(defn- collect-source-refs
+ [acc source]
+ (if (and (map? source) (:type source))
+ (case (:type source)
+ "table" (cond-> acc (pos-int? (:id source)) (update :table-ids conj (:id source)))
+ ("card" "dataset") (cond-> acc (pos-int? (:id source)) (update :card-ids conj (:id source)))
+ "metric" (cond-> acc (pos-int? (:id source)) (update :metric-ids conj (:id source)))
+ "program" (if (:program source) (collect-program-refs acc (:program source)) acc)
+ acc)
+ acc))
+
+(defn- ref-key-for-op
+ "Map a reference operator name to the refs key it populates."
+ [op]
+ (case op
+ "table" :table-ids
+ "field" :field-ids
+ "card" :card-ids
+ "metric" :metric-ids
+ "measure" :measure-ids
+ nil))
+
+(defn- collect-form
+ [acc form]
+ (cond
+ ;; Operator tuple — `["op" arg ...]`. Reference ops have an int id as their first arg
+ ;; (after repair-normalize); other ops we just recurse into.
+ (and (vector? form) (string? (first form)))
+ (let [[op & args] form
+ ref-key (ref-key-for-op op)]
+ (if (and ref-key (pos-int? (first args)))
+ (-> (update acc ref-key conj (first args))
+ ;; Still recurse into remaining args — `(field id)` has none, but
+ ;; e.g. `(with-temporal-bucket (field id) "month")` is a different op
+ ;; whose recursion is handled by the non-ref branch below.
+ (cond-> (next args) (as-> $ (reduce collect-form $ (next args)))))
+ (reduce collect-form acc args)))
+
+ ;; Nested program literal — `{:program {...}}`.
+ (and (map? form) (schema/program-literal? form))
+ (collect-program-refs acc (:program form))
+
+ ;; Plain map — recurse into values (covers per-stage option maps, etc.).
+ (map? form)
+ (reduce-kv (fn [a _k v] (collect-form a v)) acc form)
+
+ ;; Sequential — recurse element-wise.
+ (sequential? form)
+ (reduce collect-form acc form)
+
+ :else acc))
+
+(defn collect-program-refs
+ "Walk `program` and return `{:table-ids #{...} :field-ids #{...} :card-ids #{...}
+ :metric-ids #{...} :measure-ids #{...}}` containing every numeric metadata id referenced
+ anywhere in its source or operations.
+
+ Assumes the program has been through `repair/repair-program` so that all `[op id]` reference
+ forms carry positive integer ids."
+ ([program]
+ (collect-program-refs empty-refs program))
+ ([acc program]
+ (-> acc
+ (collect-source-refs (:source program))
+ (as-> $ (reduce collect-form $ (:operations program))))))
diff --git a/src/metabase/agent_lib/repair.clj b/src/metabase/agent_lib/repair.clj
new file mode 100644
index 00000000000..cdf1b9873d7
--- /dev/null
+++ b/src/metabase/agent_lib/repair.clj
@@ -0,0 +1,10 @@
+(ns metabase.agent-lib.repair
+ "Deterministic repair helpers for structured MBQL programs."
+ (:require
+ [metabase.agent-lib.repair.context]
+ [metabase.agent-lib.repair.normalize]
+ [potemkin :as p]))
+
+(p/import-vars
+ [metabase.agent-lib.repair.normalize repair-program]
+ [metabase.agent-lib.repair.context repair-program-for-context])
diff --git a/src/metabase/agent_lib/repair/context.clj b/src/metabase/agent_lib/repair/context.clj
new file mode 100644
index 00000000000..9f8e64010b4
--- /dev/null
+++ b/src/metabase/agent_lib/repair/context.clj
@@ -0,0 +1,20 @@
+(ns metabase.agent-lib.repair.context
+ "Context-specific structured-program repair passes."
+ (:require
+ [metabase.agent-lib.repair.context.passes :as passes]
+ [metabase.agent-lib.repair.stages :as stages]))
+
+(set! *warn-on-reflection* true)
+
+(defn repair-program-for-context
+ "Apply deterministic repairs that depend on the evaluation context."
+ [program context]
+ (let [source (passes/effective-source-for-context program context)
+ operations (->> (:operations program)
+ (passes/drop-source-metric-stage-aggregates context)
+ (passes/drop-source-metric-stage-with-fields context)
+ (map #(passes/normalize-source-metric-quarter-window context %))
+ (passes/remove-redundant-operations context)
+ vec)]
+ (assoc program
+ :operations (stages/insert-stage-boundaries source operations))))
diff --git a/src/metabase/agent_lib/repair/context/passes.clj b/src/metabase/agent_lib/repair/context/passes.clj
new file mode 100644
index 00000000000..ccf0bd50516
--- /dev/null
+++ b/src/metabase/agent_lib/repair/context/passes.clj
@@ -0,0 +1,193 @@
+(ns metabase.agent-lib.repair.context.passes
+ "Context-dependent repair passes for structured programs."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [coerce-positive-int]]
+ [metabase.agent-lib.common.context :as common.context]
+ [metabase.agent-lib.common.literals :refer [now-literal?]]
+ [metabase.agent-lib.join-spec :as join-spec]
+ [metabase.agent-lib.repair.stages :as stages]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn effective-source-for-context
+ "Replace a context source with the concrete metric source when needed."
+ [program context]
+ (let [source (:source program)]
+ (if (and (map? source)
+ (= "context" (:type source))
+ (common.context/source-metric-id context))
+ {:type "metric"
+ :id (common.context/source-metric-id context)}
+ source)))
+
+(def ^:private max-tree-walk-depth
+ "Maximum depth for tree walks in repair passes."
+ 64)
+
+(defn operation-contains-canonical-op?
+ "True when `operation` contains `target-op-name` anywhere in its tree."
+ [operation target-op-name]
+ (letfn [(walk [value depth]
+ (when (<= depth max-tree-walk-depth)
+ (cond
+ (and (vector? value)
+ (= target-op-name (syntax/canonical-op-name (first value))))
+ true
+
+ (coll? value)
+ (some #(walk % (inc depth)) value)
+
+ :else
+ false)))]
+ (boolean (walk operation 0))))
+
+(defn rewrite-source-stage
+ "Rewrite only the operations that belong to the source stage."
+ [operations step]
+ (:operations
+ (reduce (fn [{:keys [operations] :as state} operation]
+ (let [op-name (when (vector? operation)
+ (syntax/canonical-op-name (first operation)))]
+ (if (= "append-stage" op-name)
+ {:in-source-stage? false
+ :operations (conj operations operation)}
+ (step state operation))))
+ {:in-source-stage? true
+ :operations []}
+ operations)))
+
+(defn drop-source-metric-stage-aggregates
+ "Drop redundant source-stage aggregates for metric contexts."
+ [context operations]
+ (if-not (common.context/source-metric-id context)
+ operations
+ (rewrite-source-stage
+ operations
+ (fn [{:keys [in-source-stage? operations] :as state} operation]
+ (let [drop? (and in-source-stage?
+ (= "aggregate" (syntax/canonical-op-name (first operation)))
+ (not (stages/aggregate-over-aggregation-ref? operation))
+ (not (operation-contains-canonical-op? operation "measure"))
+ (not (operation-contains-canonical-op? operation "metric")))]
+ (if drop?
+ state
+ (assoc state :operations (conj operations operation))))))))
+
+(defn drop-source-metric-stage-with-fields
+ "Drop source-stage `with-fields` that only restate aggregation refs."
+ [context operations]
+ (if-not (common.context/source-metric-id context)
+ operations
+ (rewrite-source-stage
+ operations
+ (fn [{:keys [in-source-stage? operations] :as state} operation]
+ (let [drop? (and in-source-stage?
+ (= "with-fields" (syntax/canonical-op-name (first operation)))
+ (stages/contains-aggregation-ref? (rest operation)))]
+ (if drop?
+ state
+ (assoc state :operations (conj operations operation))))))))
+
+(defn relative-datetime-form?
+ "True when `value` is a relative-datetime form with the given amount and unit."
+ [value amount unit]
+ (and (vector? value)
+ (= "relative-datetime" (syntax/canonical-op-name (first value)))
+ (= amount (nth value 1 nil))
+ (= unit (some-> (nth value 2 nil) syntax/raw-op-name u/lower-case-en))))
+
+(defn now-form?
+ "True when `value` is a now literal or `[\"now\"]` tuple."
+ [value]
+ (or (now-literal? value)
+ (and (vector? value)
+ (= "now" (syntax/canonical-op-name (first value))))))
+
+(defn normalize-source-metric-quarter-window
+ "Normalize rolling 3-month source-metric windows into quarter-relative bounds."
+ [context operation]
+ (if (and (common.context/source-metric-id context)
+ (vector? operation)
+ (= "filter" (syntax/canonical-op-name (first operation)))
+ (vector? (second operation))
+ (= "between" (syntax/canonical-op-name (first (second operation)))))
+ (let [[between-op field lower upper] (second operation)]
+ (if (and (relative-datetime-form? lower -3 "month")
+ (now-form? upper))
+ [(first operation)
+ [between-op field
+ ["relative-datetime" -1 "quarter"]
+ ["relative-datetime" 0 "quarter"]]]
+ operation))
+ operation))
+
+(defn- edge-value
+ [edge kebab-key snake-key]
+ (or (get edge kebab-key)
+ (get edge snake-key)))
+
+(defn- join-condition-matches-edge?
+ [source-id target-id [lhs-id rhs-id] edge]
+ (let [from-table-id (edge-value edge :from-table-id :from_table_id)
+ from-field-id (edge-value edge :from-field-id :from_field_id)
+ to-table-id (edge-value edge :to-table-id :to_table_id)
+ to-field-id (edge-value edge :to-field-id :to_field_id)]
+ (and (or (and (= source-id from-table-id)
+ (= target-id to-table-id))
+ (and (= source-id to-table-id)
+ (= target-id from-table-id)))
+ (or (and (= lhs-id from-field-id) (= rhs-id to-field-id))
+ (and (= lhs-id to-field-id) (= rhs-id from-field-id))))))
+
+(defn- valid-join-operation?
+ "True when `operation` is a two-element join tuple."
+ [operation]
+ (and (vector? operation)
+ (= "join" (syntax/canonical-op-name (first operation)))
+ (= 2 (count operation))))
+
+(defn- implicit-join-matches-context?
+ "True when a parsed join spec matches surrounding context edges."
+ [source-id context {:keys [target-table-id conditions fields-mode strategy alias]}]
+ (let [surrounding-table-ids (set (keep :id (:surrounding-tables context)))]
+ (and (pos-int? target-table-id)
+ (surrounding-table-ids target-table-id)
+ (nil? alias)
+ (or (nil? strategy) (= "left-join" strategy))
+ (join-spec/no-explicit-join-fields? fields-mode)
+ (= 1 (count conditions))
+ (some #(join-condition-matches-edge? source-id target-table-id (first conditions) %)
+ (:join-edges context)))))
+
+(defn redundant-implicit-join?
+ "True when an explicit join is redundant in the current context."
+ [context operation]
+ (let [source-id (common.context/source-table-id context)
+ join-spec (when (valid-join-operation? operation)
+ (join-spec/parse-join-spec (second operation)))]
+ (and source-id
+ join-spec
+ (implicit-join-matches-context? source-id context join-spec))))
+
+(defn- redundant-source-metric-aggregate?
+ "True when an aggregate is an exact reuse of the source metric.
+ This catches `[\"aggregate\" [\"metric\" source-id]]` which is intentionally preserved
+ by `drop-source-metric-stage-aggregates` (that pass only drops non-metric/measure aggregates)."
+ [source-id operation]
+ (and (vector? operation)
+ (= "aggregate" (syntax/canonical-op-name (first operation)))
+ (let [aggregation (second operation)]
+ (and (vector? aggregation)
+ (= "metric" (syntax/canonical-op-name (first aggregation)))
+ (= source-id (some-> (second aggregation) coerce-positive-int))))))
+
+(defn remove-redundant-operations
+ "Remove context-redundant source-metric reuse and implicit joins."
+ [context operations]
+ (remove (fn [operation]
+ (or (when-let [source-id (common.context/source-metric-id context)]
+ (redundant-source-metric-aggregate? source-id operation))
+ (redundant-implicit-join? context operation)))
+ operations))
diff --git a/src/metabase/agent_lib/repair/normalize.clj b/src/metabase/agent_lib/repair/normalize.clj
new file mode 100644
index 00000000000..b45744890b2
--- /dev/null
+++ b/src/metabase/agent_lib/repair/normalize.clj
@@ -0,0 +1,68 @@
+(ns metabase.agent-lib.repair.normalize
+ "Deterministic operator and top-level normalization for structured MBQL programs."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [coerce-positive-int]]
+ [metabase.agent-lib.common.literals :refer [scalar-literal?]]
+ [metabase.agent-lib.repair.normalize.forms :as normalize.forms]
+ [metabase.agent-lib.repair.normalize.top-level :as normalize.top-level]
+ [metabase.agent-lib.repair.stages :as stages]
+ [metabase.agent-lib.schema :as schema]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn repair-program
+ "Apply deterministic repairs for common structured program mistakes.
+
+ The local `letfn` is intentional here: `repair-node` and `repair-map-node`
+ form the minimal mutually recursive kernel for walking arbitrary nested
+ structured-program values."
+ [program]
+ (letfn [(repair-map-node [value]
+ (cond
+ (schema/program-literal? value)
+ (assoc value :program (repair-program (:program value)))
+
+ (and (map? value)
+ (contains? value :type)
+ (contains? value :id))
+ (update value :id coerce-positive-int)
+
+ :else
+ (into {}
+ (map (fn [[k v]] [k (repair-node v)]))
+ value)))
+ (repair-node [value]
+ (cond
+ (scalar-literal? value) value
+ (map? value) (repair-map-node value)
+ (syntax/boolean-wrapper-form? value)
+ (let [[raw-op arg] value
+ arg' (repair-node arg)]
+ (if (= "true" (some-> raw-op syntax/raw-op-name u/lower-case-en))
+ arg'
+ ["not" arg']))
+ (syntax/possible-operator-tuple? value) (normalize.forms/repair-operator-form repair-node value)
+ (vector? value) (mapv repair-node value)
+ (sequential? value) (mapv repair-node value)
+ :else value))]
+ (if-not (map? program)
+ program
+ (let [source (let [source (:source program)]
+ (cond
+ (not (map? source))
+ source
+
+ (= "program" (:type source))
+ (update source :program repair-program)
+
+ :else
+ (cond-> source
+ (contains? source :id) (update :id coerce-positive-int))))
+ operations (into []
+ (mapcat #(normalize.top-level/repair-top-level-operation repair-node %))
+ (:operations program))]
+ (assoc program
+ :source source
+ :operations (stages/insert-stage-boundaries source operations))))))
diff --git a/src/metabase/agent_lib/repair/normalize/forms.clj b/src/metabase/agent_lib/repair/normalize/forms.clj
new file mode 100644
index 00000000000..2fd6944ff67
--- /dev/null
+++ b/src/metabase/agent_lib/repair/normalize/forms.clj
@@ -0,0 +1,277 @@
+(ns metabase.agent-lib.repair.normalize.forms
+ "Nested operator normalization for structured program repair."
+ (:require
+ [metabase.agent-lib.common.coercions :as coercions :refer [coerce-non-negative-int
+ coerce-positive-int
+ normalize-join-conditions
+ normalize-percentile-value
+ unwrap-singleton-form]]
+ [metabase.agent-lib.common.literals :refer [iso-date-string?
+ non-blank-string?
+ now-literal?
+ null-literal?
+ quarter-label-string?
+ scalar-sequential?]]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private temporal-bucket-extraction-aliases
+ "Bucket names that should be repaired into extraction helpers."
+ {"dayofweek" "get-day-of-week"
+ "day-of-week" "get-day-of-week"
+ "hour-of-day" "get-hour"
+ "month-of-year" "get-month"
+ "quarter-of-year" "get-quarter"})
+
+(defn expression-definition-tuple?
+ "True when `value` is an inline expression definition tuple."
+ [value]
+ (and (vector? value)
+ (= "expression" (syntax/canonical-op-name (first value)))
+ (= 3 (count value))
+ (non-blank-string? (second value))))
+
+(defn- quarter-expression?
+ [value]
+ (and (vector? value)
+ (let [op-name (syntax/canonical-op-name (first value))]
+ (or (= op-name "get-quarter")
+ (and (= op-name "with-temporal-bucket")
+ (= "quarter" (nth value 2 nil)))))))
+
+(defn- normalize-quarter-filter-value
+ [lhs rhs]
+ (cond
+ (and (quarter-expression? lhs)
+ (quarter-label-string? rhs))
+ (coercions/quarter-label->number rhs)
+
+ (and (quarter-expression? lhs)
+ (scalar-sequential? rhs)
+ (every? quarter-label-string? rhs))
+ (mapv coercions/quarter-label->number rhs)
+
+ :else
+ rhs))
+
+(defn- branch-pair?
+ [v]
+ (and (vector? v) (= 2 (count v))))
+
+(defn- else-branch?
+ [v]
+ (and (branch-pair? v)
+ (= "else" (syntax/raw-op-name (first v)))))
+
+(defn- split-else-branch
+ "Peel an else-branch off the end of `branches`, merging it with `fallback`."
+ [branches fallback]
+ (if (and (seq branches) (else-branch? (last branches)))
+ [(vec (butlast branches)) (second (last branches))]
+ [branches fallback]))
+
+(defn- flat->branch-pairs
+ "Convert a flat even-length vector into branch pairs when applicable."
+ [v]
+ (when (and (vector? v)
+ (even? (count v))
+ (not-every? branch-pair? v))
+ (mapv vec (partition 2 v))))
+
+(defn- normalize-branches-fallback
+ "Uniform post-processing: split else-branch and emit `[branches]` or
+ `[branches fallback]`."
+ [branches fallback]
+ (let [[branches fallback] (split-else-branch branches fallback)]
+ (if (some? fallback)
+ [branches fallback]
+ [branches])))
+
+(defn- classify-case-args
+ "Classify the shape of `case` args into `[branches fallback]` before uniform
+ normalization, or nil if no pattern matches."
+ [args]
+ (cond
+ ;; Three bare args: pred, then, else
+ (and (= 3 (count args))
+ (not (branch-pair? (first args))))
+ [[[(first args) (second args)]] (nth args 2)]
+
+ ;; Single wrapped vector of branch pairs
+ (and (seq args)
+ (vector? (first args))
+ (every? branch-pair? (first args)))
+ [(first args) (second args)]
+
+ ;; Single wrapped triple: [pred then else]
+ (and (= 1 (count args))
+ (vector? (first args))
+ (= 3 (count (first args)))
+ (not-every? branch-pair? (first args)))
+ (let [[pred then else] (first args)]
+ [[[pred then]] else])
+
+ ;; Leading branch pairs followed by optional fallback
+ (seq (take-while branch-pair? args))
+ (let [branches (vec (take-while branch-pair? args))
+ remainder (drop (count branches) args)]
+ [branches (when (= 1 (count remainder)) (first remainder))])
+
+ ;; Flat even-length vector inside a wrapper
+ (flat->branch-pairs (first args))
+ [(flat->branch-pairs (first args)) (second args)]))
+
+(defn repair-case-args
+ "Normalize alternate `case` encodings into the canonical branches/fallback shape."
+ [args]
+ (if-let [[branches fallback] (classify-case-args args)]
+ (normalize-branches-fallback branches fallback)
+ args))
+
+(defn temporal-expression?
+ "True when `value` is a temporal literal or temporal expression tuple."
+ [value]
+ (or (now-literal? value)
+ (and (vector? value)
+ (contains? #{"absolute-datetime" "relative-datetime" "now"}
+ (syntax/canonical-op-name (first value))))))
+
+(defn- wrap-iso-date-as-absolute-datetime
+ [value]
+ (if (iso-date-string? value)
+ ["absolute-datetime" value "day"]
+ value))
+
+(defn- wrap-now-as-expression
+ [value]
+ (if (now-literal? value)
+ ["now"]
+ value))
+
+(defn repair-between-bounds
+ "Normalize temporal `between` bounds into canonical datetime helper forms.
+ Preserves degenerate (fewer than 3) arg counts so repair remains idempotent
+ when the operator is embedded in a helper that reshapes its arguments
+ (e.g. `case`)."
+ [args]
+ (if (< (count args) 3)
+ (vec args)
+ (let [[lhs lower upper] args
+ lower' (wrap-now-as-expression lower)
+ upper' (wrap-now-as-expression upper)]
+ (if (or (temporal-expression? lower')
+ (temporal-expression? upper'))
+ [lhs
+ (if (temporal-expression? upper')
+ (wrap-iso-date-as-absolute-datetime lower')
+ lower')
+ (if (temporal-expression? lower')
+ (wrap-iso-date-as-absolute-datetime upper')
+ upper')]
+ [lhs lower' upper']))))
+
+(defn repair-operator-form
+ "Normalize a nested operator form using the recursive `repair-node` callback."
+ [repair-node value]
+ (let [[raw-op & raw-args] value
+ op-name (syntax/canonical-op-name raw-op)
+ args (mapv (fn [arg]
+ (let [arg' (repair-node arg)]
+ (if (#{"in"
+ "not-in"
+ "case"
+ "with-fields"
+ "with-join-fields"
+ "with-join-conditions"} op-name)
+ arg'
+ (unwrap-singleton-form arg'))))
+ raw-args)]
+ (case op-name
+ ("field" "table" "card" "metric" "measure")
+ [op-name (coerce-positive-int (first args))]
+
+ "aggregation-ref"
+ [op-name (coerce-non-negative-int (first args))]
+
+ "with-temporal-bucket"
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[field bucket] args]
+ (if-let [extraction-op (and (string? bucket)
+ (temporal-bucket-extraction-aliases bucket))]
+ [extraction-op field]
+ [op-name field bucket])))
+
+ ("contains" "does-not-contain" "starts-with" "ends-with")
+ (into [op-name]
+ (cond-> args
+ (and (> (count args) 2)
+ (or (map? (last args))
+ (boolean? (last args))))
+ pop))
+
+ ("in" "not-in")
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[lhs rhs] args]
+ [op-name lhs (normalize-quarter-filter-value lhs rhs)]))
+
+ "="
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[lhs rhs] args
+ rhs' (normalize-quarter-filter-value lhs rhs)]
+ (if (scalar-sequential? rhs')
+ ["in" lhs (vec rhs')]
+ [op-name lhs rhs'])))
+
+ "!="
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[lhs rhs] args
+ rhs' (normalize-quarter-filter-value lhs rhs)]
+ (if (scalar-sequential? rhs')
+ ["not-in" lhs (vec rhs')]
+ [op-name lhs rhs'])))
+
+ "is"
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[lhs rhs] args]
+ (if (null-literal? rhs)
+ ["is-null" lhs]
+ ["=" lhs rhs])))
+
+ "is-not"
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[lhs rhs] args]
+ (if (null-literal? rhs)
+ ["not-null" lhs]
+ ["!=" lhs rhs])))
+
+ "case"
+ (let [args' (if (= "if" (syntax/raw-op-name raw-op))
+ (if (= 3 (count args))
+ [[[(first args) (second args)]] (nth args 2)]
+ args)
+ args)]
+ (into [op-name] (repair-case-args args')))
+
+ "between"
+ (into [op-name] (repair-between-bounds args))
+
+ "percentile"
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[expr percentile] args]
+ [op-name expr (normalize-percentile-value percentile)]))
+
+ "with-join-conditions"
+ (if (< (count args) 2)
+ (into [op-name] args)
+ (let [[join-spec conditions] args]
+ [op-name join-spec (normalize-join-conditions conditions)]))
+
+ (into [op-name] args))))
diff --git a/src/metabase/agent_lib/repair/normalize/top_level.clj b/src/metabase/agent_lib/repair/normalize/top_level.clj
new file mode 100644
index 00000000000..2c046749a92
--- /dev/null
+++ b/src/metabase/agent_lib/repair/normalize/top_level.clj
@@ -0,0 +1,154 @@
+(ns metabase.agent-lib.repair.normalize.top-level
+ "Top-level operation normalization for structured program repair."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [direction-string?
+ normalize-direction]]
+ [metabase.agent-lib.common.literals :refer [aggregation-form?]]
+ [metabase.agent-lib.repair.normalize.forms :as forms]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(defn normalize-with-fields-selection
+ "Normalize `with-fields` arguments into a single vector of selections."
+ [args]
+ (cond
+ (empty? args) []
+
+ (= 1 (count args))
+ (let [selection (first args)]
+ (if (and (sequential? selection)
+ (not (syntax/possible-operator-tuple? selection)))
+ (vec selection)
+ [selection]))
+
+ :else
+ (vec args)))
+
+(defn repair-order-by-operations
+ "Normalize one or more `order-by` arguments into canonical operations."
+ [args]
+ (if (empty? args)
+ [["order-by"]]
+ (loop [remaining args
+ repaired []]
+ (if (empty? remaining)
+ repaired
+ (let [[first-arg second-arg & tail] remaining]
+ (cond
+ (and (vector? first-arg)
+ (= 2 (count first-arg))
+ (direction-string? (first first-arg)))
+ (recur (vec (rest remaining))
+ (conj repaired ["order-by" [(normalize-direction (first first-arg))
+ (second first-arg)]]))
+
+ (direction-string? second-arg)
+ (recur (vec tail)
+ (conj repaired ["order-by" first-arg (normalize-direction second-arg)]))
+
+ :else
+ (recur (vec (rest remaining))
+ (conj repaired ["order-by" first-arg]))))))))
+
+(defn repair-repeated-single-arg-op
+ "Expand repeated top-level single-arg operations."
+ [op-name args]
+ (if (empty? args)
+ [[op-name]]
+ (mapv (fn [arg] [op-name arg]) args)))
+
+(defn repair-with-fields-operation
+ "Extract inline expressions out of `with-fields` before projection."
+ [repair-node args]
+ (let [selection (normalize-with-fields-selection args)
+ repaired-fields (mapv repair-node selection)
+ extracted-exprs (keep (fn [field]
+ (when (forms/expression-definition-tuple? field)
+ ["expression" (second field) (nth field 2)]))
+ repaired-fields)
+ projected (mapv (fn [field]
+ (if (forms/expression-definition-tuple? field)
+ ["expression-ref" (second field)]
+ field))
+ repaired-fields)]
+ (into (vec extracted-exprs)
+ [["with-fields" projected]])))
+
+(defn bare-aggregation-ref-operation?
+ "True when an operation is a bare top-level aggregation-ref."
+ [operation]
+ (and (vector? operation)
+ (= "aggregation-ref" (syntax/canonical-op-name (first operation)))))
+
+(defn- repair-with-embedded
+ "Common skeleton: combine repaired direct args with embedded ops, falling back
+ to a bare `[[op-name]]` when both are empty."
+ [op-name direct-ops embedded-ops]
+ (into (or (when (seq direct-ops) direct-ops)
+ (when (empty? embedded-ops) [[op-name]])
+ [])
+ embedded-ops))
+
+(defn- repair-direct-args
+ "Dispatch direct-arg repair by operation name."
+ [op-name direct-args]
+ (case op-name
+ "order-by"
+ (repair-order-by-operations direct-args)
+
+ ("breakout" "filter" "join")
+ (repair-repeated-single-arg-op op-name direct-args)
+
+ ("limit" "with-page")
+ [[op-name (first direct-args)]]))
+
+(defn repair-top-level-operation
+ "Normalize a top-level operation using the recursive `repair-node` callback."
+ [repair-node operation]
+ (if-not (vector? operation)
+ [operation]
+ (let [[raw-op & raw-args] operation
+ op-name (syntax/canonical-op-name raw-op)
+ args (mapv repair-node raw-args)
+ embedded-ops (mapcat #(repair-top-level-operation repair-node %)
+ (filter syntax/top-level-operation-tuple? args))
+ direct-args (vec (remove syntax/top-level-operation-tuple? args))]
+ (case op-name
+ "visualization"
+ []
+
+ "aggregate"
+ (let [keep-bare-aggregation-refs? (not-any? (complement bare-aggregation-ref-operation?)
+ direct-args)
+ direct-args' (cond->> direct-args
+ (not keep-bare-aggregation-refs?)
+ (remove bare-aggregation-ref-operation?))]
+ (repair-with-embedded op-name
+ (when (seq direct-args')
+ (repair-repeated-single-arg-op op-name direct-args'))
+ embedded-ops))
+
+ "with-fields"
+ (repair-with-fields-operation repair-node args)
+
+ ("order-by" "breakout" "filter" "join" "limit" "with-page")
+ (repair-with-embedded op-name
+ (when (seq direct-args)
+ (repair-direct-args op-name direct-args))
+ embedded-ops)
+
+ "expression"
+ (repair-with-embedded op-name
+ (when (>= (count direct-args) 2)
+ (let [[expr-name expr] (take 2 direct-args)]
+ (if (aggregation-form? expr)
+ [["aggregate" expr]]
+ [[op-name expr-name expr]])))
+ embedded-ops)
+
+ ("append-stage" "drop-stage" "drop-empty-stages")
+ [[op-name]]
+
+ (into [(into [op-name] direct-args)]
+ embedded-ops)))))
diff --git a/src/metabase/agent_lib/repair/stages.clj b/src/metabase/agent_lib/repair/stages.clj
new file mode 100644
index 00000000000..1ae56e00f65
--- /dev/null
+++ b/src/metabase/agent_lib/repair/stages.clj
@@ -0,0 +1,107 @@
+(ns metabase.agent-lib.repair.stages
+ "Stage-boundary inference and aggregation-stage helpers for structured repair."
+ (:require
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private max-repair-depth
+ "Maximum recursion depth for repair-phase tree walks."
+ 64)
+
+(defn contains-aggregation-ref?
+ "True when `value` contains an `aggregation-ref` anywhere inside it."
+ ([value]
+ (contains-aggregation-ref? value 0))
+ ([value depth]
+ (if (> depth max-repair-depth)
+ false
+ (cond
+ (and (vector? value)
+ (= "aggregation-ref" (syntax/canonical-op-name (first value))))
+ true
+
+ (map? value)
+ (boolean (some #(contains-aggregation-ref? % (inc depth)) (vals value)))
+
+ (sequential? value)
+ (boolean (some #(contains-aggregation-ref? % (inc depth)) value))
+
+ :else
+ false))))
+
+(defn aggregate-over-aggregation-ref?
+ "True when `operation` is an aggregate over an aggregation-ref."
+ [operation]
+ (and (vector? operation)
+ (= "aggregate" (syntax/canonical-op-name (first operation)))
+ (contains-aggregation-ref? (rest operation))))
+
+(defn- operation-defines-aggregation?
+ [operation]
+ (and (vector? operation)
+ (= "aggregate" (syntax/canonical-op-name (first operation)))))
+
+(defn- operation-requires-post-aggregation-stage?
+ [operation]
+ (and (vector? operation)
+ (contains-aggregation-ref? (rest operation))
+ (#{"aggregate" "expression" "with-fields"}
+ (syntax/canonical-op-name (first operation)))))
+
+(defn source-final-stage-has-aggregations?
+ "True when the source program ends in an aggregated stage."
+ [source]
+ (cond
+ (not (map? source))
+ false
+
+ (= "metric" (:type source))
+ true
+
+ (= "program" (:type source))
+ (let [nested-program (:program source)]
+ (:current-stage-has-aggregations?
+ (reduce (fn [state operation]
+ (cond
+ (and (vector? operation)
+ (= "append-stage" (syntax/canonical-op-name (first operation))))
+ (assoc state :current-stage-has-aggregations? false)
+
+ (operation-defines-aggregation? operation)
+ (assoc state :current-stage-has-aggregations? true)
+
+ :else
+ state))
+ {:current-stage-has-aggregations? (source-final-stage-has-aggregations? (:source nested-program))}
+ (:operations nested-program))))
+
+ :else
+ false))
+
+(defn insert-stage-boundaries
+ "Insert `append-stage` before operations that must run after an aggregation stage."
+ [source operations]
+ (:operations
+ (reduce (fn [{:keys [current-stage-has-aggregations? operations]} operation]
+ (let [op-name (when (vector? operation)
+ (syntax/canonical-op-name (first operation)))
+ needs-stage? (and current-stage-has-aggregations?
+ (operation-requires-post-aggregation-stage? operation)
+ (not= "append-stage"
+ (some-> operations last first syntax/canonical-op-name)))
+ operations' (cond-> operations
+ needs-stage? (conj ["append-stage"])
+ true (conj operation))]
+ (cond
+ (= "append-stage" op-name)
+ {:current-stage-has-aggregations? false
+ :operations operations'}
+
+ :else
+ {:current-stage-has-aggregations? (or (and (not needs-stage?) current-stage-has-aggregations?)
+ (operation-defines-aggregation? operation))
+ :operations operations'})))
+ {:current-stage-has-aggregations? (source-final-stage-has-aggregations? source)
+ :operations []}
+ operations)))
diff --git a/src/metabase/agent_lib/runtime.clj b/src/metabase/agent_lib/runtime.clj
new file mode 100644
index 00000000000..fd7ae195916
--- /dev/null
+++ b/src/metabase/agent_lib/runtime.clj
@@ -0,0 +1,104 @@
+(ns metabase.agent-lib.runtime
+ "Trusted MBQL helper runtime for structured MBQL program evaluation."
+ (:require
+ [metabase.agent-lib.capabilities :as capabilities]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.agent-lib.runtime.bindings :as runtime.bindings]
+ [metabase.agent-lib.runtime.fields :as runtime.fields]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]))
+
+;; Ensure all defmethod registrations are loaded.
+(comment lib/keep-me)
+
+(set! *warn-on-reflection* true)
+
+(def ^:private extra-helper-bindings
+ {;; Query-introspection helpers exposed through the shared runtime.
+ 'suggested-join-conditions lib/suggested-join-conditions
+ 'joinable-columns lib/join-fieldable-columns
+ 'joins lib/joins
+ 'visible-columns lib/visible-columns
+ 'filterable-columns lib/filterable-columns
+ 'breakoutable-columns lib/breakoutable-columns
+ 'aggregable-columns lib/aggregable-columns
+ 'orderable-columns lib/orderable-columns
+ 'expressionable-columns lib/expressionable-columns
+
+ ;; Query-dependent refs are wired at runtime build time.
+ 'aggregation-ref nil})
+
+(def trusted-helper-bindings
+ "Trusted structured helper implementations keyed by operator symbol."
+ (merge capabilities/trusted-helper-bindings
+ extra-helper-bindings))
+
+(def query-transform-symbols
+ "Top-level structured query-transform operator symbols."
+ capabilities/query-transform-symbols)
+
+(def helper-symbols
+ "All helper symbols accepted by the structured runtime."
+ (into capabilities/helper-symbols
+ (keys extra-helper-bindings)))
+(defn op-symbol
+ "Normalize helper identifiers into canonical operator symbols."
+ [op]
+ (syntax/op-symbol op))
+
+(defn- resolve-in-scope-table-ids
+ "Default fallback when no in-scope table set is provided: load every table in the database.
+ Used by tests and any code path that hasn't yet plumbed in a scoped runtime."
+ [metadata-provider]
+ (into #{} (map :id) (lib.metadata/tables metadata-provider)))
+
+(defn build-runtime
+ "Build the trusted structured-program runtime for a metadata provider.
+
+ Options:
+ :in-scope-table-ids — set of table-ids the runtime should expose for name-based lookups
+ and FK chain resolution. Defaults to every table in the database
+ (used by tests and legacy callers); production should always pass
+ the scoped set derived from the program references.
+ :extra-bindings — additional runtime bindings merged on top of the standard ones."
+ ([metadata-providerable]
+ (build-runtime metadata-providerable nil))
+ ([metadata-providerable {:keys [in-scope-table-ids extra-bindings]}]
+ (let [metadata-provider (lib.metadata/->metadata-provider metadata-providerable)
+ table-ids (or in-scope-table-ids
+ (resolve-in-scope-table-ids metadata-provider))
+ tables-by-name (runtime.fields/build-table-lookup metadata-provider table-ids)
+ {:keys [fields-by-table fields-by-id]}
+ (runtime.fields/build-field-lookups metadata-provider tables-by-name)
+ bindings (merge (assoc trusted-helper-bindings
+ 'expression-ref mbql/expression-ref-or-current-stage-column
+ 'aggregation-ref mbql/aggregation-ref-or-current-stage-column
+ 'skip-join? (fn [query operation]
+ (mbql/redundant-implicit-join? fields-by-id query operation)))
+ (runtime.bindings/make-context-bindings metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id)
+ extra-bindings)]
+ {:metadata-provider metadata-provider
+ :tables-by-name tables-by-name
+ :fields-by-table fields-by-table
+ :fields-by-id fields-by-id
+ :bindings bindings})))
+
+(defn bindings-map
+ "Return the runtime bindings map."
+ [runtime]
+ (:bindings runtime))
+
+(defn helper-fn
+ "Look up a helper implementation from a runtime by operator.
+ Throws when the binding is nil so callers get a clear error instead of an NPE."
+ [runtime op]
+ (let [sym (op-symbol op)
+ f (get (:bindings runtime) sym)]
+ (when-not f
+ (throw (ex-info (str "No runtime binding for operator: " sym)
+ {:operator sym})))
+ f))
diff --git a/src/metabase/agent_lib/runtime/bindings.clj b/src/metabase/agent_lib/runtime/bindings.clj
new file mode 100644
index 00000000000..83e9eb28898
--- /dev/null
+++ b/src/metabase/agent_lib/runtime/bindings.clj
@@ -0,0 +1,18 @@
+(ns metabase.agent-lib.runtime.bindings
+ "Context-dependent runtime bindings for structured MBQL evaluation."
+ (:require
+ [metabase.agent-lib.runtime.lookup :as runtime.lookup]
+ [metabase.agent-lib.runtime.transforms :as runtime.transforms]
+ [metabase.lib.core :as lib]))
+
+(set! *warn-on-reflection* true)
+
+(defn make-context-bindings
+ "Build query-aware structured helper bindings for the current runtime context."
+ [metadata-provider tables-by-name fields-by-table fields-by-id]
+ (merge {'query (fn [metadata] (lib/query metadata-provider metadata))}
+ (runtime.lookup/metadata-bindings metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id)
+ (runtime.transforms/query-transform-bindings)))
diff --git a/src/metabase/agent_lib/runtime/fields.clj b/src/metabase/agent_lib/runtime/fields.clj
new file mode 100644
index 00000000000..2d77016ff78
--- /dev/null
+++ b/src/metabase/agent_lib/runtime/fields.clj
@@ -0,0 +1,78 @@
+(ns metabase.agent-lib.runtime.fields
+ "Field lookup and query-stage field resolution for structured MBQL runtime bindings."
+ (:require
+ [metabase.agent-lib.common.errors :as errors]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn build-table-lookup
+ "Build a lowercase table-name lookup for `table-ids`.
+
+ Only the explicitly-requested tables are loaded — the runtime is scoped to entities the program
+ actually references (source + referenced + surrounding), not every table in the database.
+ Unknown tables are silently dropped; out-of-scope lookups will surface as nice errors at use time."
+ [metadata-provider table-ids]
+ (into {}
+ (keep (fn [tid]
+ (when-let [table (lib.metadata/table metadata-provider tid)]
+ [(u/lower-case-en (:name table)) table])))
+ table-ids))
+
+(defn build-field-lookups
+ "Build both the `fields-by-table` and `fields-by-id` lookups for the tables in `tables-by-name`.
+
+ `fields-by-table` is a `{lower-case-table-name {lower-case-field-name field}}` map.
+ `fields-by-id` is a `{field-id field}` map.
+
+ Both maps are scoped to the tables passed in — typically the in-scope set computed from the
+ evaluation context, not every table in the database."
+ [metadata-provider tables-by-name]
+ (reduce-kv (fn [acc table-name table-meta]
+ (let [fields (lib.metadata/fields metadata-provider (:id table-meta))]
+ (-> acc
+ (assoc-in [:fields-by-table table-name]
+ (into {}
+ (map (fn [field]
+ [(u/lower-case-en (:name field)) field]))
+ fields))
+ (update :fields-by-id
+ (fn [m]
+ (reduce (fn [m field] (assoc m (:id field) field)) m fields))))))
+ {:fields-by-table {} :fields-by-id {}}
+ tables-by-name))
+
+(defn- table-name-for-field-lookup
+ [metadata-provider tables-by-name table-name-or-id]
+ (if (number? table-name-or-id)
+ (if-let [table-meta (lib.metadata/table metadata-provider (int table-name-or-id))]
+ (u/lower-case-en (:name table-meta))
+ (errors/lookup-error! (str "Table not found by ID: " table-name-or-id)
+ {:table-id table-name-or-id}
+ {:available (keys tables-by-name)}))
+ (u/lower-case-en table-name-or-id)))
+
+(defn resolve-field-from-table
+ "Resolve a field by explicit table and field identifiers."
+ [metadata-provider tables-by-name fields-by-table table-name-or-id field-name]
+ (let [table-name (table-name-for-field-lookup metadata-provider tables-by-name table-name-or-id)
+ normalized-field-name (u/lower-case-en field-name)
+ table-fields (get fields-by-table table-name)]
+ (or (get table-fields normalized-field-name)
+ (errors/lookup-error! (str "Field not found: "
+ (if (number? table-name-or-id)
+ (str "(table " table-name-or-id ")")
+ table-name-or-id)
+ "." field-name)
+ {:table-name table-name
+ :field-name field-name}
+ (when table-fields
+ {:available (keys table-fields)})))))
+
+(defn resolve-field-in-query
+ "Resolve a metadata field against the columns available in the current query stage."
+ [fields-by-id query raw-field]
+ (mbql/resolve-field-in-query fields-by-id query raw-field
+ (mbql/current-query-field-candidates query)))
diff --git a/src/metabase/agent_lib/runtime/lookup.clj b/src/metabase/agent_lib/runtime/lookup.clj
new file mode 100644
index 00000000000..4f80d0a85d2
--- /dev/null
+++ b/src/metabase/agent_lib/runtime/lookup.clj
@@ -0,0 +1,160 @@
+(ns metabase.agent-lib.runtime.lookup
+ "Metadata and field lookup helpers for structured runtime bindings."
+ (:require
+ [metabase.agent-lib.common.errors :as errors]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.agent-lib.runtime.fields :as fields]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(defn lookup-table
+ "Resolve a table by numeric id or case-insensitive name."
+ [metadata-provider tables-by-name table-identifier]
+ (let [available-names (keys tables-by-name)]
+ (if (number? table-identifier)
+ (or (lib.metadata/table metadata-provider (int table-identifier))
+ (errors/lookup-error! (str "Table not found by ID: " table-identifier)
+ {:table-id table-identifier}
+ {:available available-names}))
+ (let [normalized (u/lower-case-en table-identifier)]
+ (or (get tables-by-name normalized)
+ (errors/lookup-error! (str "Table not found: " table-identifier)
+ {:table-name table-identifier}
+ {:available available-names}))))))
+
+(defn lookup-card
+ "Resolve a saved-question source, rejecting metrics."
+ [metadata-provider card-id]
+ (if (number? card-id)
+ (if-let [card-meta (lib.metadata/card metadata-provider (int card-id))]
+ (if (= (:type card-meta) :metric)
+ (errors/lookup-error! (str "Card " card-id " is a metric.")
+ {:card-id card-id}
+ {:suggestion (str "Use (metric " card-id ") instead.")})
+ card-meta)
+ (errors/lookup-error! (str "Card not found by ID: " card-id)
+ {:card-id card-id}))
+ (errors/lookup-error! (str "Card lookup requires a numeric ID, got: " (pr-str card-id))
+ {:card-id card-id})))
+
+(defn lookup-metric
+ "Resolve a metric by numeric id."
+ [metadata-provider metric-id]
+ (if (number? metric-id)
+ (or (lib.metadata/metric metadata-provider (int metric-id))
+ (errors/lookup-error! (str "Metric not found by ID: " metric-id)
+ {:metric-id metric-id}))
+ (errors/lookup-error! (str "Metric lookup requires a numeric ID, got: " (pr-str metric-id))
+ {:metric-id metric-id})))
+
+(defn lookup-measure
+ "Resolve a measure by numeric id."
+ [metadata-provider measure-id]
+ (if (number? measure-id)
+ (or (lib.metadata/measure metadata-provider (int measure-id))
+ (errors/lookup-error! (str "Measure not found by ID: " measure-id)
+ {:measure-id measure-id}))
+ (errors/lookup-error! (str "Measure lookup requires a numeric ID, got: " (pr-str measure-id))
+ {:measure-id measure-id})))
+
+(defn lookup-field
+ "Resolve a field by id, unique name, or explicit table/name pair."
+ ([metadata-provider _tables-by-name fields-by-table fields-by-id field-name-or-id]
+ (if (number? field-name-or-id)
+ (or (get fields-by-id (int field-name-or-id))
+ (lib.metadata/field metadata-provider (int field-name-or-id))
+ (errors/lookup-error! (str "Field not found by ID: " field-name-or-id)
+ {:field-id field-name-or-id}))
+ (let [field-name (u/lower-case-en field-name-or-id)
+ matches (into []
+ (keep (fn [[table-name field-map]]
+ (when-let [field (get field-map field-name)]
+ [table-name field])))
+ fields-by-table)]
+ (cond
+ (= 1 (count matches))
+ (second (first matches))
+
+ (> (count matches) 1)
+ (errors/lookup-error! (str "Ambiguous field \"" field-name-or-id "\".")
+ {:field-name field-name-or-id}
+ {:tables (mapv first matches)
+ :suggestion "Use (field \"TABLE\" \"FIELD\") to disambiguate."})
+
+ :else
+ (errors/lookup-error! (str "Field not found: \"" field-name-or-id "\".")
+ {:field-name field-name-or-id}
+ {:suggestion "Use (field \"TABLE\" \"FIELD\") with explicit table name."})))))
+ ([metadata-provider _tables-by-name fields-by-table _fields-by-id table-name-or-id field-name]
+ (fields/resolve-field-from-table metadata-provider
+ _tables-by-name
+ fields-by-table
+ table-name-or-id
+ field-name)))
+
+(defn query-relative-field
+ "Resolve a requested field against the current query stage."
+ ([metadata-provider tables-by-name fields-by-table fields-by-id query field-name-or-id]
+ (fields/resolve-field-in-query fields-by-id
+ query
+ (lookup-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ field-name-or-id)))
+ ([metadata-provider tables-by-name fields-by-table fields-by-id query table-name-or-id field-name]
+ (let [raw-field (lookup-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ table-name-or-id
+ field-name)]
+ (or (when (mbql/query? query)
+ (lib/find-matching-column query
+ -1
+ raw-field
+ (mbql/current-query-field-candidates query)))
+ (fields/resolve-field-in-query fields-by-id query raw-field)
+ raw-field))))
+
+(defn metadata-bindings
+ "Return metadata and field lookup bindings for the structured runtime."
+ [metadata-provider tables-by-name fields-by-table fields-by-id]
+ {'table (partial lookup-table metadata-provider tables-by-name)
+ 'card (partial lookup-card metadata-provider)
+ 'metric (partial lookup-metric metadata-provider)
+ 'measure (partial lookup-measure metadata-provider)
+ 'field (fn
+ ([field-name-or-id]
+ (lookup-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ field-name-or-id))
+ ([a b]
+ ;; 2-arity: (field query field-name-or-id) or (field table-name field-name).
+ ;; Queries are always maps; table names are always strings or numbers.
+ (if (map? a)
+ (query-relative-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ a
+ b)
+ (lookup-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ a
+ b)))
+ ([query table-name-or-id field-name]
+ (query-relative-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ query
+ table-name-or-id
+ field-name)))})
diff --git a/src/metabase/agent_lib/runtime/transforms.clj b/src/metabase/agent_lib/runtime/transforms.clj
new file mode 100644
index 00000000000..1d8785a5346
--- /dev/null
+++ b/src/metabase/agent_lib/runtime/transforms.clj
@@ -0,0 +1,96 @@
+(ns metabase.agent-lib.runtime.transforms
+ "Query-transform bindings for structured runtime evaluation."
+ (:require
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.lib.core :as lib]
+ [metabase.lib.util :as lib.util]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private order-wrapper-direction-key
+ :order-direction)
+
+(def ^:private order-wrapper-value-key
+ :order-value)
+
+(defn order-wrapper?
+ "True when `x` wraps an orderable together with its direction."
+ [x]
+ (and (map? x)
+ (contains? x order-wrapper-direction-key)
+ (contains? x order-wrapper-value-key)))
+
+(defn asc-orderable
+ "Wrap an orderable with ascending sort direction."
+ [orderable]
+ {order-wrapper-direction-key :asc
+ order-wrapper-value-key orderable})
+
+(defn desc-orderable
+ "Wrap an orderable with descending sort direction."
+ [orderable]
+ {order-wrapper-direction-key :desc
+ order-wrapper-value-key orderable})
+
+(defn- freshen-orderable-ref
+ "Freshen ref uuids so repeated orderables do not alias the same clause instance."
+ [resolved-orderable]
+ (if (vector? resolved-orderable)
+ (lib.util/fresh-uuids resolved-orderable)
+ resolved-orderable))
+
+(defn apply-breakout
+ "Apply a breakout, materializing expression refs when needed."
+ [query breakoutable]
+ (if (and (lib.util/clause? breakoutable)
+ (not (lib.util/ref-clause? breakoutable)))
+ (let [[query' expr-ref] (mbql/ensure-query-expression-ref query breakoutable "breakout")]
+ (lib/breakout query' expr-ref))
+ (lib/breakout query breakoutable)))
+
+(defn apply-filter
+ "Apply a filter clause to a query."
+ [query clause]
+ (lib/filter query clause))
+
+(defn apply-with-fields
+ "Apply `with-fields`, expanding raw fields into breakouts on aggregated queries."
+ [query selections]
+ (let [selections' (mapv #(mbql/resolve-aggregation-selection query %) selections)
+ field-selection? mbql/field-selection?
+ raw-fields (filterv field-selection? selections')
+ other-selections (filterv (complement field-selection?) selections')]
+ (if (and (seq (lib/aggregations query))
+ (seq raw-fields))
+ (let [query' (reduce lib/breakout query raw-fields)]
+ (if (seq other-selections)
+ (lib/with-fields query' other-selections)
+ query'))
+ (lib/with-fields query selections'))))
+
+(defn apply-order-by
+ "Apply `order-by`, resolving query-relative orderables first."
+ ([query orderable]
+ (if (order-wrapper? orderable)
+ (let [[query' resolved] (mbql/resolve-orderable query (order-wrapper-value-key orderable))]
+ (lib/order-by query'
+ (freshen-orderable-ref resolved)
+ (order-wrapper-direction-key orderable)))
+ (let [[query' resolved] (mbql/resolve-orderable query orderable)]
+ (lib/order-by query' (freshen-orderable-ref resolved)))))
+ ([query a b]
+ (if (#{:asc :desc} a)
+ (let [[query' resolved] (mbql/resolve-orderable query b)]
+ (lib/order-by query' (freshen-orderable-ref resolved) a))
+ (let [[query' resolved] (mbql/resolve-orderable query a)]
+ (lib/order-by query' (freshen-orderable-ref resolved) b)))))
+
+(defn query-transform-bindings
+ "Return query-transform bindings for the structured runtime."
+ []
+ {'asc asc-orderable
+ 'desc desc-orderable
+ 'breakout apply-breakout
+ 'filter apply-filter
+ 'with-fields apply-with-fields
+ 'order-by apply-order-by})
diff --git a/src/metabase/agent_lib/schema.clj b/src/metabase/agent_lib/schema.clj
new file mode 100644
index 00000000000..542ee83b59a
--- /dev/null
+++ b/src/metabase/agent_lib/schema.clj
@@ -0,0 +1,128 @@
+(ns metabase.agent-lib.schema
+ "Malli schemas for the structured MBQL program generation path."
+ (:require
+ [clojure.string :as str]
+ [metabase.agent-lib.capabilities :as capabilities]
+ [metabase.agent-lib.schema.top-level :as schema.top-level]
+ [metabase.util.i18n :refer [tru]]
+ [metabase.util.malli.humanize :as mu.humanize]
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.malli.schema :as ms]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private top-level-operation-values
+ capabilities/top-level-operator-values)
+
+(defn- non-blank-string?
+ [value]
+ (and (string? value) (not (str/blank? value))))
+
+(defn- nested-operator-tuple?
+ [value]
+ (and (vector? value)
+ (string? (first value))
+ (not (str/blank? (first value)))
+ (not (top-level-operation-values (first value)))
+ (not= "query" (first value))))
+
+(defn- top-level-operation-tuple?
+ [value]
+ (and (vector? value)
+ (string? (first value))
+ (top-level-operation-values (first value))))
+
+(defn program-literal?
+ "True when `value` is a nested structured-program literal."
+ [value]
+ (and (map? value)
+ (= "program" (or (:type value) (get value "type")))
+ (or (contains? value :program)
+ (contains? value "program"))))
+
+(defn- plain-map-literal?
+ [value]
+ (and (map? value)
+ (not (program-literal? value))))
+
+(def ^:private registry
+ {::scalar [:or nil? :string number? boolean?]
+ ::non-blank-string [:and
+ :string
+ [:fn
+ {:error/message "value must be a non-blank string"}
+ non-blank-string?]]
+ ::vector-literal [:vector [:ref ::node]]
+ ::page [:map {:closed true}
+ [:page ms/PositiveInt]
+ [:items ms/PositiveInt]]
+ ::source-ref [:or
+ [:tuple [:= "field"] ms/PositiveInt]
+ [:tuple [:= "table"] ms/PositiveInt]
+ [:tuple [:= "card"] ms/PositiveInt]
+ [:tuple [:= "metric"] ms/PositiveInt]]
+ ::expression-ref-form [:tuple [:= "expression-ref"] [:ref ::non-blank-string]]
+ ::aggregation-ref-form [:tuple [:= "aggregation-ref"] [:int {:min 0}]]
+ ::query-aware-ref [:or
+ [:ref ::expression-ref-form]
+ [:ref ::aggregation-ref-form]]
+ ::nested-operator-form [:and
+ [:vector {:min 1} [:ref ::node]]
+ [:fn
+ {:error/message "nested operator tuple must start with a non-blank non-transform operator string"}
+ nested-operator-tuple?]]
+ ::program-literal [:map {:closed true}
+ [:type [:= "program"]]
+ [:program [:ref ::program]]]
+ ::map-literal [:and
+ [:map-of :any [:ref ::node]]
+ [:fn
+ {:error/message "map literals cannot use the reserved `{\"type\":\"program\"}` shape unless they contain a valid nested program"}
+ plain-map-literal?]]
+ ::form [:or
+ [:ref ::source-ref]
+ [:ref ::query-aware-ref]
+ [:ref ::nested-operator-form]]
+ ::node [:or
+ [:ref ::scalar]
+ [:ref ::program-literal]
+ [:ref ::map-literal]
+ [:ref ::form]
+ [:ref ::vector-literal]]
+ ::top-level-op [:and
+ schema.top-level/query-transform-form
+ [:fn
+ {:error/message "operation tuple must start with a valid top-level operator string"}
+ top-level-operation-tuple?]]
+ ::source [:or
+ [:map {:closed true}
+ [:type [:= "context"]]
+ [:ref [:= "source"]]]
+ [:map {:closed true}
+ [:type [:enum "table" "card" "dataset" "metric"]]
+ [:id ms/PositiveInt]]
+ [:ref ::program-literal]]
+ ::program [:map {:closed true}
+ [:source [:ref ::source]]
+ [:operations [:sequential [:ref ::top-level-op]]]]})
+
+(def source-schema
+ "Malli schema for structured-program sources."
+ [:schema {:registry registry} [:ref ::source]])
+
+(def program-schema
+ "Malli schema for full structured MBQL programs."
+ [:schema {:registry registry} [:ref ::program]])
+
+(defn validated-structure
+ "Validate the structural shape of a structured program and return it unchanged."
+ [program]
+ (when-let [error (mr/explain program-schema program)]
+ (throw (ex-info (tru "Generated program has an invalid structure.")
+ {:status-code 400
+ :error :invalid-generated-program
+ :humanized (mu.humanize/humanize error)
+ :details (pr-str (mu.humanize/humanize error))
+ :schema ::program
+ :value program})))
+ program)
diff --git a/src/metabase/agent_lib/schema/top_level.clj b/src/metabase/agent_lib/schema/top_level.clj
new file mode 100644
index 00000000000..a14e5b75524
--- /dev/null
+++ b/src/metabase/agent_lib/schema/top_level.clj
@@ -0,0 +1,50 @@
+(ns metabase.agent-lib.schema.top-level
+ "Top-level operation schema helpers derived from the capability catalog."
+ (:require
+ [metabase.agent-lib.capabilities.catalog.top-level :as capabilities.top-level]
+ [metabase.util.malli.schema :as ms]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private schema-ns
+ "metabase.agent-lib.schema")
+
+(defn- schema-ref
+ [local-name]
+ [:ref (keyword schema-ns local-name)])
+
+(def ^:private top-level-operation-schemas
+ {"filter" [:tuple [:= "filter"] (schema-ref "form")]
+ "aggregate" [:tuple [:= "aggregate"] (schema-ref "form")]
+ "breakout" [:tuple [:= "breakout"] (schema-ref "form")]
+ "with-fields" [:tuple [:= "with-fields"] [:sequential (schema-ref "form")]]
+ "limit" [:tuple [:= "limit"] ms/PositiveInt]
+ "expression" [:tuple [:= "expression"] (schema-ref "non-blank-string") (schema-ref "form")]
+ "join" [:tuple [:= "join"] (schema-ref "form")]
+ "order-by" [:or
+ [:tuple [:= "order-by"] (schema-ref "form")]
+ [:tuple [:= "order-by"] (schema-ref "form") [:enum "asc" "desc"]]]
+ "append-stage" [:tuple [:= "append-stage"]]
+ "drop-stage" [:tuple [:= "drop-stage"]]
+ "drop-empty-stages" [:tuple [:= "drop-empty-stages"]]
+ "with-page" [:tuple [:= "with-page"] (schema-ref "page")]})
+
+(def ^:private top-level-op-names
+ (mapv (comp name :op) capabilities.top-level/capabilities))
+
+(defn- validate-top-level-coverage!
+ []
+ (let [defined-ops (set (keys top-level-operation-schemas))
+ catalog-ops (set top-level-op-names)]
+ (when (not= defined-ops catalog-ops)
+ (throw (ex-info "Top-level capability and schema definitions are out of sync."
+ {:defined-ops defined-ops
+ :catalog-ops catalog-ops})))))
+
+(validate-top-level-coverage!)
+
+(def query-transform-form
+ "Schema form for all valid top-level structured query transforms, ordered from the top-level capability catalog."
+ (into [:or]
+ (map top-level-operation-schemas)
+ top-level-op-names))
diff --git a/src/metabase/agent_lib/syntax.clj b/src/metabase/agent_lib/syntax.clj
new file mode 100644
index 00000000000..c00742ae2ed
--- /dev/null
+++ b/src/metabase/agent_lib/syntax.clj
@@ -0,0 +1,94 @@
+(ns metabase.agent-lib.syntax
+ "Shape-level helpers for structured MBQL operators and tuples."
+ (:require
+ [metabase.agent-lib.capabilities :as capabilities]
+ [metabase.util :as u]))
+
+(set! *warn-on-reflection* true)
+
+(def canonical-operator-aliases
+ "Legacy or alternate helper names keyed to their canonical string forms."
+ {"if" "case"
+ "relative-date" "relative-datetime"
+ "temporal-diff" "datetime-diff"
+ "count-if" "count-where"
+ "variance" "var"
+ "stddev-pop" "stddev"
+ "count-distinct" "distinct"
+ "distinct-count" "distinct"
+ "is-not-null" "not-null"
+ "|" "or"
+ "dayofweek" "get-day-of-week"
+ "day-of-week" "get-day-of-week"
+ "hour-of-day" "get-hour"
+ "month" "get-month"
+ "month-of-year" "get-month"
+ "quarter-of-year" "get-quarter"})
+
+(def top-level-operation-symbols
+ "Top-level structured program operators keyed as symbols."
+ capabilities/query-transform-symbols)
+
+(def recognized-operator-symbols
+ "All recognized operator symbols, including the synthetic `query` helper."
+ (conj capabilities/helper-symbols 'query))
+
+(def repairable-operator-names
+ "Operator names that should be normalized by the repair pass."
+ (into #{"visualization" "is" "is-not"}
+ (concat
+ (map name recognized-operator-symbols)
+ (map name top-level-operation-symbols)
+ (keys canonical-operator-aliases)
+ ["dayofweek" "day-of-week" "hour-of-day" "month-of-year" "quarter-of-year"])))
+
+(defn raw-op-name
+ "Return the raw operator name string for a string, keyword, or symbol."
+ [value]
+ (cond
+ (string? value) value
+ (keyword? value) (name value)
+ (symbol? value) (name value)
+ :else nil))
+
+(defn canonical-op-name
+ "Return the canonical string operator name for a structured helper."
+ [value]
+ (when-let [op-name (raw-op-name value)]
+ (or (canonical-operator-aliases op-name)
+ op-name)))
+
+(defn op-symbol
+ "Normalize helper identifiers into canonical operator symbols."
+ [op]
+ (some-> op canonical-op-name symbol))
+
+(defn possible-operator-tuple?
+ "True when `value` looks like a repairable structured helper tuple."
+ [value]
+ (and (vector? value)
+ (contains? repairable-operator-names
+ (or (canonical-op-name (first value))
+ (raw-op-name (first value))))))
+
+(defn boolean-wrapper-form?
+ "True when `value` is a legacy boolean wrapper tuple such as `[\"true\", expr]`."
+ [value]
+ (and (vector? value)
+ (= 2 (count value))
+ (contains? #{"true" "false"}
+ (some-> (first value) raw-op-name u/lower-case-en))))
+
+(defn top-level-operation-tuple?
+ "True when `value` is a top-level structured operation tuple."
+ [value]
+ (and (possible-operator-tuple? value)
+ (contains? top-level-operation-symbols
+ (symbol (canonical-op-name (first value))))))
+
+(defn operator-tuple?
+ "True when `value` is a recognized structured helper tuple."
+ [value]
+ (and (vector? value)
+ (string? (first value))
+ (contains? recognized-operator-symbols (op-symbol (first value)))))
diff --git a/src/metabase/agent_lib/validate.clj b/src/metabase/agent_lib/validate.clj
new file mode 100644
index 00000000000..f6f395143e0
--- /dev/null
+++ b/src/metabase/agent_lib/validate.clj
@@ -0,0 +1,38 @@
+(ns metabase.agent-lib.validate
+ "Structural and semantic validation for structured MBQL programs."
+ (:require
+ [metabase.agent-lib.repair :as repair]
+ [metabase.agent-lib.schema :as schema]
+ [metabase.agent-lib.validate.semantic :as semantic]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ContextEntitySummary
+ "Schema for a single entity summary in the evaluation context."
+ [:map
+ [:model :string]
+ [:id pos-int?]])
+
+(def EvaluationContext
+ "Schema for the evaluation context passed to validation and evaluation.
+ `:source-entity` identifies the primary data source.
+ `:surrounding-tables` provides join candidates visible in the current scope."
+ [:map
+ [:source-entity ContextEntitySummary]
+ [:referenced-entities [:sequential ContextEntitySummary]]
+ [:surrounding-tables [:sequential [:map [:id pos-int?]]]]])
+
+(mu/defn validated-program :- :map
+ "Repair and validate a structured program, returning the repaired program on success."
+ [program :- :map
+ context :- EvaluationContext]
+ (log/debug "Repairing structured program")
+ (let [program (-> program
+ repair/repair-program
+ (repair/repair-program-for-context context))]
+ (log/debug "Validating repaired program structure")
+ (schema/validated-structure program)
+ (log/debug "Validating repaired program semantics")
+ (semantic/validate-program program context)))
diff --git a/src/metabase/agent_lib/validate/context.clj b/src/metabase/agent_lib/validate/context.clj
new file mode 100644
index 00000000000..b773b80808e
--- /dev/null
+++ b/src/metabase/agent_lib/validate/context.clj
@@ -0,0 +1,76 @@
+(ns metabase.agent-lib.validate.context
+ "Context-derived validation helpers for structured programs."
+ (:require
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]))
+
+(set! *warn-on-reflection* true)
+
+(defn context-allowed-ids
+ "Derive the ids that are valid for the current evaluation context."
+ [{:keys [source-entity referenced-entities surrounding-tables measure-ids]}]
+ (let [entity-summaries (concat [source-entity] referenced-entities surrounding-tables)
+ summary-field-ids (fn [entity-summary]
+ (->> (concat
+ (:columns entity-summary)
+ (get-in entity-summary [:query_summary :returned_columns])
+ (get-in entity-summary [:query_summary :filterable_columns])
+ (:queryable_dimensions entity-summary))
+ (keep :id)
+ (filter pos-int?)))
+ source-model (:model source-entity)]
+ {:table-ids (->> (concat
+ (when (= source-model "table") [(:id source-entity)])
+ (keep #(when (= "table" (:model %)) (:id %)) referenced-entities)
+ (map :id surrounding-tables))
+ (filter pos-int?)
+ set)
+ :card-ids (->> (concat
+ (when (#{"card" "dataset"} source-model) [(:id source-entity)])
+ (keep #(when (#{"card" "dataset"} (:model %)) (:id %)) referenced-entities))
+ (filter pos-int?)
+ set)
+ :metric-ids (->> (concat
+ (when (= source-model "metric") [(:id source-entity)])
+ (keep #(when (= "metric" (:model %)) (:id %)) referenced-entities))
+ (filter pos-int?)
+ set)
+ :measure-ids (set (filter pos-int? measure-ids))
+ :field-ids (->> entity-summaries
+ (mapcat summary-field-ids)
+ set)}))
+
+(defn validate-source!
+ "Validate a structured source against the context-derived allowed ids."
+ [validate-nested-program allowed-ids path source depth state]
+ (case (:type source)
+ "context"
+ (do
+ (when-not (= "source" (:ref source))
+ (invalid-program! (conj path :ref) "context source ref must be `source`"))
+ state)
+
+ "table"
+ (do
+ (when-not (pos-int? (:id source))
+ (invalid-program! (conj path :id) "source table id must be a numeric id"))
+ (when-not ((:table-ids allowed-ids) (:id source))
+ (invalid-program! (conj path :id) "source table id is not available in the provided context"))
+ state)
+
+ ("card" "dataset")
+ (do
+ (when-not ((:card-ids allowed-ids) (:id source))
+ (invalid-program! (conj path :id) "source card id is not available in the provided context"))
+ state)
+
+ "metric"
+ (do
+ (when-not ((:metric-ids allowed-ids) (:id source))
+ (invalid-program! (conj path :id) "source metric id is not available in the provided context"))
+ state)
+
+ "program"
+ (validate-nested-program (conj path :program) (:program source) (inc depth)
+ (update state :program-nesting (fnil inc 0)))
+
+ (invalid-program! (conj path :type) "unsupported source type")))
diff --git a/src/metabase/agent_lib/validate/cross_checks.clj b/src/metabase/agent_lib/validate/cross_checks.clj
new file mode 100644
index 00000000000..0f828428036
--- /dev/null
+++ b/src/metabase/agent_lib/validate/cross_checks.clj
@@ -0,0 +1,40 @@
+(ns metabase.agent-lib.validate.cross-checks
+ "Cross-operation semantic checks for structured programs."
+ (:require
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.agent-lib.validate.walker :as walker]))
+
+(set! *warn-on-reflection* true)
+
+(defn validate-no-metric-order-by!
+ "Reject direct metric helpers nested inside `order-by`."
+ [operation-path operation]
+ (when (= 'order-by (runtime/op-symbol (first operation)))
+ (when-let [metric-path
+ (walker/find-tuple-path #(and (syntax/operator-tuple? %)
+ (= 'metric (runtime/op-symbol (first %))))
+ operation)]
+ (invalid-program! (into operation-path metric-path)
+ "metric helpers cannot be used directly inside order-by. Order by a breakout field or a non-metric aggregation instead."
+ {:operator "metric"}))))
+
+(defn validate-no-source-metric-reuse!
+ "Reject reusing the same metric already selected as the source."
+ [context operation-path operation]
+ (let [source-entity (:source-entity context)
+ source-id (:id source-entity)]
+ (when (and (= "metric" (:model source-entity))
+ (pos-int? source-id))
+ (when-let [metric-path
+ (walker/find-tuple-path #(and (syntax/operator-tuple? %)
+ (= 'metric (runtime/op-symbol (first %)))
+ (= source-id (second %)))
+ operation)]
+ (invalid-program! (into operation-path metric-path)
+ (str "source is already metric " source-id
+ ". Do not reference that same metric again inside operations; start "
+ "from the source query and build on it.")
+ {:operator "metric"
+ :id source-id})))))
diff --git a/src/metabase/agent_lib/validate/operators.clj b/src/metabase/agent_lib/validate/operators.clj
new file mode 100644
index 00000000000..80c07642485
--- /dev/null
+++ b/src/metabase/agent_lib/validate/operators.clj
@@ -0,0 +1,172 @@
+(ns metabase.agent-lib.validate.operators
+ "Operator-specific semantic validation for structured programs."
+ (:require
+ [clojure.string :as str]
+ [metabase.agent-lib.capabilities :as capabilities]
+ [metabase.agent-lib.common.context :as common.context]
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.common.literals :refer [aggregation-form?
+ bare-field-id?
+ non-blank-string?]]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private field-wrapper-required-arg-positions
+ "Argument positions that must be wrapped as `[\"field\", id]` for specific helpers."
+ '{sum [0]
+ avg [0]
+ min [0]
+ max [0]
+ distinct [0]
+ median [0]
+ stddev [0]
+ var [0]
+ percentile [0]
+ cum-sum [0]
+ sum-where [0]
+ distinct-where [0]})
+
+(defn ensure-arity!
+ "Validate a helper arity against the capability catalog."
+ [path op arg-count]
+ (when-let [allowed (capabilities/fixed-arities op)]
+ (when-not (allowed arg-count)
+ (invalid-program! path
+ (if (and (= op 'expression) (= arg-count 1))
+ (str "`expression` is a top-level operation, not a nested reference. "
+ "Use `[\"expression\", \"Name\", expr]` as its own operation, and if you need to "
+ "sort by that calculation, reuse the expression body directly inside `order-by`.")
+ (format "operator `%s` expects %s argument(s), got %s"
+ (name op)
+ (str/join " or " (sort allowed))
+ arg-count))
+ {:operator (name op)
+ :allowed (sort allowed)
+ :actual arg-count}))))
+
+(defn validate-operator-specific!
+ "Validate nested helper tuples against context-derived ids and operator rules."
+ [allowed-ids context path value]
+ (when (syntax/operator-tuple? value)
+ (let [[raw-op & args] value
+ op (runtime/op-symbol raw-op)
+ op-name (name op)]
+ (cond
+ (syntax/top-level-operation-symbols op)
+ (invalid-program! path
+ (format "`%s` is a top-level operation and cannot be nested" op-name)
+ {:operator op-name})
+
+ (= op 'query)
+ (invalid-program! path
+ "`query` cannot appear inside operations; the program source already defines the base query."
+ {:operator "query"})
+
+ :else
+ (do
+ (doseq [arg-position (get field-wrapper-required-arg-positions op)]
+ (when (bare-field-id? (nth args arg-position nil))
+ (invalid-program! (conj path (inc arg-position))
+ (str "Wrap field ids with `[\"field\", id]` inside `" op-name "`. "
+ "Do not pass bare numeric ids directly to aggregation helpers.")
+ {:operator op-name
+ :retry-category :field-wrapper
+ :value (nth args arg-position)})))
+ (case op-name
+ "case"
+ (when (some-> args second map?)
+ (when (or (contains? (second args) :default)
+ (contains? (second args) "default"))
+ (invalid-program! (conj path 2)
+ "`case` uses the fallback value itself as the optional third argument. Do not use `{\"default\": ...}`. Omit the third argument instead of using null."
+ {:operator "case"})))
+
+ "table"
+ (let [id (first args)]
+ (when-not (pos-int? id)
+ (invalid-program! (conj path 1) "table lookup requires a numeric id")))
+
+ "card"
+ (let [id (first args)]
+ (when-not (pos-int? id)
+ (invalid-program! (conj path 1) "card lookup requires a numeric id"))
+ (when-not ((:card-ids allowed-ids) id)
+ (invalid-program! (conj path 1)
+ "card id is not available in the provided context"
+ {:operator "card" :id id})))
+
+ "metric"
+ (let [id (first args)]
+ (when-not (pos-int? id)
+ (invalid-program! (conj path 1) "metric lookup requires a numeric id"))
+ (when-not ((:metric-ids allowed-ids) id)
+ (invalid-program! (conj path 1)
+ "metric id is not available in the provided context"
+ {:operator "metric" :id id})))
+
+ "measure"
+ (let [id (first args)
+ source-id (common.context/source-metric-id context)]
+ (when-not (pos-int? id)
+ (invalid-program! (conj path 1) "measure lookup requires a numeric id"))
+ (when (and source-id (= source-id id))
+ (invalid-program! (conj path 1)
+ (str "source is already metric " source-id
+ ". Metric ids are not measure ids. Do not reference that same "
+ "metric as a `measure`; start from the source query and build on it.")
+ {:operator "measure"
+ :id id
+ :source-metric-id source-id}))
+ (when-not ((:measure-ids allowed-ids) id)
+ (invalid-program! (conj path 1)
+ "measure id is not available in the provided context"
+ {:operator "measure" :id id})))
+
+ "field"
+ (cond
+ (not= 1 (count args))
+ (invalid-program! path "field lookups must use a single numeric field id")
+
+ (not (pos-int? (first args)))
+ (invalid-program! (conj path 1) "field lookup requires a numeric id"))
+
+ "expression-ref"
+ (let [expr-name (first args)]
+ (when-not (non-blank-string? expr-name)
+ (invalid-program! (conj path 1)
+ "expression-ref requires a non-blank expression name"
+ {:operator "expression-ref"})))
+
+ "aggregation-ref"
+ (let [index (first args)]
+ (when-not (and (int? index) (<= 0 index))
+ (invalid-program! (conj path 1)
+ "aggregation-ref requires a zero-based aggregation index"
+ {:operator "aggregation-ref"
+ :value index})))
+
+ nil))))))
+
+(defn validate-top-level-operation!
+ "Validate a top-level operation and its arguments."
+ [validate-node operation-path operation state]
+ (let [[raw-op & args] operation
+ op (runtime/op-symbol raw-op)]
+ (when-not (syntax/top-level-operation-symbols op)
+ (invalid-program! (conj operation-path 0)
+ "top-level operation is not allowed"
+ {:operator raw-op}))
+ (ensure-arity! operation-path op (count args))
+ (when (= op 'aggregate)
+ (doseq [[idx arg] (map-indexed vector args)]
+ (when-not (aggregation-form? arg)
+ (invalid-program! (conj operation-path (inc idx))
+ "aggregate operations require aggregation helpers like `count`, `sum`, `avg`, `percentile`, `metric`, or `measure`. Do not pass bare `aggregation-ref` or breakout clauses directly."
+ {:operator "aggregate"
+ :value arg}))))
+ (reduce (fn [state [idx arg]]
+ (validate-node (conj operation-path (inc idx)) arg 1 state))
+ state
+ (map-indexed vector args))))
diff --git a/src/metabase/agent_lib/validate/program.clj b/src/metabase/agent_lib/validate/program.clj
new file mode 100644
index 00000000000..a3a6c977058
--- /dev/null
+++ b/src/metabase/agent_lib/validate/program.clj
@@ -0,0 +1,70 @@
+(ns metabase.agent-lib.validate.program
+ "Recursive semantic program validation for agent-lib.
+
+ The local `letfn` in `validate-program-body!` is intentional: this namespace
+ is the recursive validation boundary for nested structured programs."
+ (:require
+ [clojure.string :as str]
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.schema :as schema]
+ [metabase.agent-lib.syntax :as syntax]
+ [metabase.agent-lib.validate.context :as validate.context]
+ [metabase.agent-lib.validate.cross-checks :as cross-checks]
+ [metabase.agent-lib.validate.operators :as operators]
+ [metabase.agent-lib.validate.walker :as walker]))
+
+(set! *warn-on-reflection* true)
+
+(defn validate-program-body!
+ "Validate a repaired structured program body, threading traversal state."
+ [allowed-ids context path program depth state]
+ (letfn [(semantic-visit [path value depth state]
+ (cond
+ (schema/program-literal? value)
+ (validate-program* (conj path :program) (:program value) (inc depth)
+ (update state :program-nesting (fnil inc 0)))
+
+ (vector? value)
+ (do
+ (when (and (some-> (first value) syntax/raw-op-name)
+ (str/ends-with? (some-> (first value) syntax/raw-op-name) "-ref")
+ (not (#{"expression-ref" "aggregation-ref"}
+ (some-> (first value) syntax/raw-op-name))))
+ (invalid-program! (conj path 0)
+ "unknown helper reference"
+ {:operator (syntax/raw-op-name (first value))}))
+ (when (syntax/operator-tuple? value)
+ (let [[raw-op & args] value
+ op (runtime/op-symbol raw-op)]
+ (operators/validate-operator-specific! allowed-ids context path value)
+ (operators/ensure-arity! path op (count args))))
+ state)
+
+ :else
+ state))
+ (validate-node [path value depth state]
+ (walker/walk-node path value depth state semantic-visit))
+ (validate-program* [path program depth state]
+ (let [nesting (get state :program-nesting 0)]
+ (when (> nesting walker/max-program-nesting)
+ (invalid-program! path "program source nesting exceeds maximum depth")))
+ (when (> (count (:operations program)) walker/max-operations)
+ (invalid-program! (conj path :operations) "program exceeds maximum operation count"))
+ (let [state (validate.context/validate-source! validate-program*
+ allowed-ids
+ (conj path :source)
+ (:source program)
+ depth
+ state)]
+ (reduce (fn [state [idx operation]]
+ (let [operation-path (into path [:operations idx])]
+ (cross-checks/validate-no-metric-order-by! operation-path operation)
+ (cross-checks/validate-no-source-metric-reuse! context operation-path operation)
+ (operators/validate-top-level-operation! validate-node
+ operation-path
+ operation
+ state)))
+ state
+ (map-indexed vector (:operations program)))))]
+ (validate-program* path program depth state)))
diff --git a/src/metabase/agent_lib/validate/semantic.clj b/src/metabase/agent_lib/validate/semantic.clj
new file mode 100644
index 00000000000..082cd4b615e
--- /dev/null
+++ b/src/metabase/agent_lib/validate/semantic.clj
@@ -0,0 +1,14 @@
+(ns metabase.agent-lib.validate.semantic
+ "Semantic validation rules for repaired structured MBQL programs."
+ (:require
+ [metabase.agent-lib.validate.context :as validate.context]
+ [metabase.agent-lib.validate.program :as validate.program]))
+
+(set! *warn-on-reflection* true)
+
+(defn validate-program
+ "Validate semantic structured-program rules after repair and structural validation."
+ [program context]
+ (let [allowed-ids (validate.context/context-allowed-ids context)]
+ (validate.program/validate-program-body! allowed-ids context [] program 0 {:node-count 0})
+ program))
diff --git a/src/metabase/agent_lib/validate/walker.clj b/src/metabase/agent_lib/validate/walker.clj
new file mode 100644
index 00000000000..2bb780b4ac0
--- /dev/null
+++ b/src/metabase/agent_lib/validate/walker.clj
@@ -0,0 +1,99 @@
+(ns metabase.agent-lib.validate.walker
+ "Pure traversal helpers for structured program validation."
+ (:require
+ [metabase.agent-lib.common.coercions :refer [normalize-map-key]]
+ [metabase.agent-lib.common.errors :refer [invalid-program!]]
+ [metabase.agent-lib.common.literals :refer [scalar-literal?]]
+ [metabase.agent-lib.schema :as schema]))
+
+(def max-operations
+ "Maximum number of top-level operations allowed in a structured program."
+ 32)
+
+(def max-depth
+ "Maximum nesting depth allowed in a structured program."
+ 24)
+
+(def max-node-count
+ "Maximum total node count allowed in a structured program."
+ 600)
+
+(def max-program-nesting
+ "Maximum recursive program-source nesting depth (prevents circular references)."
+ 8)
+
+(def max-string-length
+ "Maximum length of any string value in a structured program."
+ 1024)
+
+(set! *warn-on-reflection* true)
+
+(defn find-tuple-path
+ "Find the first path to a vector node matching `pred`.
+
+ The local `walk` helper stays nested because this namespace is the recursive
+ traversal boundary for validation."
+ [pred value]
+ (letfn [(walk [path node]
+ (cond
+ (and (vector? node) (pred node))
+ path
+
+ (vector? node)
+ (some identity
+ (map-indexed (fn [idx child]
+ (walk (conj path idx) child))
+ node))
+
+ (map? node)
+ (if (schema/program-literal? node)
+ (walk (conj path :program) (:program node))
+ (some identity
+ (map (fn [[k v]]
+ (walk (conj path (normalize-map-key k)) v))
+ node)))
+
+ (sequential? node)
+ (some identity
+ (map-indexed (fn [idx child]
+ (walk (conj path idx) child))
+ node))
+
+ :else
+ nil))]
+ (walk [] value)))
+
+(defn walk-node
+ "Walk a structured node in preorder, threading pure traversal state through `visit`."
+ [path value depth state visit]
+ (when (> depth max-depth)
+ (invalid-program! path "program exceeds maximum nesting depth"))
+ (let [state (update state :node-count (fnil inc 0))]
+ (when (> (:node-count state) max-node-count)
+ (invalid-program! path "program exceeds maximum node count"))
+ (when (and (string? value) (> (count value) max-string-length))
+ (invalid-program! path "string value exceeds maximum length"
+ {:length (count value) :max max-string-length}))
+ (let [state (visit path value depth state)]
+ (cond
+ (scalar-literal? value)
+ state
+
+ (map? value)
+ (if (schema/program-literal? value)
+ state
+ (reduce (fn [state [k v]]
+ (walk-node (conj path (normalize-map-key k)) v (inc depth) state visit))
+ state
+ value))
+
+ (sequential? value)
+ (reduce (fn [state [idx element]]
+ (walk-node (conj path idx) element (inc depth) state visit))
+ state
+ (map-indexed vector value))
+
+ :else
+ (invalid-program! path
+ (str "unsupported literal type " (pr-str (type value)))
+ {:value value})))))
diff --git a/src/metabase/metabot/agent/streaming.clj b/src/metabase/metabot/agent/streaming.clj
index 41cd565c2bf..437ca87a41c 100644
--- a/src/metabase/metabot/agent/streaming.clj
+++ b/src/metabase/metabot/agent/streaming.clj
@@ -25,17 +25,25 @@
(defn query->url-hash
"Convert an MBQL 4 (legacy) or MBQL 5 query to a base64-encoded URL hash.
+ When `display` is provided, includes it so the frontend renders the
+ correct visualization type instead of defaulting to table.
Used for /question# URLs."
- [query]
- (-> {:dataset_query query}
- json/encode
- (.getBytes "UTF-8")
- codecs/bytes->b64-str))
+ ([query]
+ (query->url-hash query nil))
+ ([query display]
+ (-> (cond-> {:dataset_query query}
+ display (assoc :display (name display)))
+ json/encode
+ (.getBytes "UTF-8")
+ codecs/bytes->b64-str)))
(defn query->question-url
- "Convert a query to a /question# URL."
- [query]
- (str "/question#" (query->url-hash query)))
+ "Convert a query to a /question# URL.
+ Optional `display` sets the visualization type (e.g. :line, :bar)."
+ ([query]
+ (query->question-url query nil))
+ ([query display]
+ (str "/question#" (query->url-hash query display))))
;;; Data Part Constructors
diff --git a/src/metabase/metabot/table_utils.clj b/src/metabase/metabot/table_utils.clj
index 979610d9f0f..db780537712 100644
--- a/src/metabase/metabot/table_utils.clj
+++ b/src/metabase/metabot/table_utils.clj
@@ -112,8 +112,7 @@
(mapv (fn [{:keys [id name schema description]}]
(let [table-query (lib/query mp (lib.metadata/table mp id))
cols (->> (lib/visible-columns table-query)
- (map #(metabot.tools.u/add-table-reference table-query %)))
- field-id-prefix (metabot.tools.u/table-field-id-prefix id)]
+ (map #(metabot.tools.u/add-table-reference table-query %)))]
{:id id
:type :table
:name name
@@ -122,7 +121,7 @@
:database_engine engine
:database_schema schema
:description description
- :fields (into [] (map-indexed #(metabot.tools.u/->result-column table-query %2 %1 field-id-prefix) cols))
+ :fields (mapv #(metabot.tools.u/->result-column table-query %) cols)
:metrics []}))
all-tables))))))
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 34fab60c0cf..9ffabeaf8c2 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -1,231 +1,167 @@
(ns metabase.metabot.tools.construct
"Notebook query construction tool wrappers."
(:require
- [clojure.walk :as walk]
- [malli.core :as mc]
- [malli.transform :as mtx]
+ [metabase.agent-lib.core :as agent-lib]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.metabot.agent.streaming :as streaming]
[metabase.metabot.scope :as scope]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.charts.create :as create-chart-tools]
- [metabase.metabot.tools.filters :as filter-tools]
[metabase.metabot.tools.shared.instructions :as instructions]
[metabase.metabot.tools.shared.llm-representations :as llm-rep]
- [metabase.metabot.util :as metabot-u]
+ [metabase.metabot.tools.util :as tools.u]
+ [metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
(set! *warn-on-reflection* true)
-(defn- normalize-ai-args
- "Normalize nested tool arguments to kebab-case keys and keyword enums."
- [value]
- (let [normalized (when value
- (metabot-u/recursive-update-keys value metabot-u/safe->kebab-case-en))
- enum-keys #{:operation :bucket :function :sort-order :direction :field-granularity}
- normalize-enum (fn [v]
- (cond
- (keyword? v) v
- (string? v) (case v
- "ascending" :asc
- "descending" :desc
- (keyword v))
- :else v))]
- (walk/postwalk
- (fn [x]
- (if (map? x)
- (reduce-kv (fn [m k v]
- (assoc m k (if (enum-keys k) (normalize-enum v) v)))
- {}
- x)
- x))
- normalized)))
+;;; ------------------------------------------------ Schema ------------------------------------------------
-;; Query construction is handled by construct_notebook_query for parity with ai-service.
+(def ^:private source-entity-schema
+ "Schema for source_entity — identifies the primary data source."
+ [:map
+ [:type [:enum "table" "model" "question" "metric"]]
+ [:id :int]])
-(def ^:private construct-field-schema
- [:map {:closed true}
- [:field_id :string]
- [:field_granularity {:optional true} [:maybe :string]]
- [:bucket {:optional true} [:maybe :string]]])
-
-(def ^:private construct-field-aggregation-schema
- [:map {:closed true}
- [:field_id {:optional true} :string]
- [:function [:enum "count" "count-distinct" "distinct" "sum" "min" "max" "avg"]]
- [:sort_order {:optional true} [:maybe [:enum "ascending" "descending"]]]
- [:bucket {:optional true} [:maybe :string]]])
-
-(def ^:private construct-measure-aggregation-schema
- [:map {:closed true}
- [:measure_id :int]
- [:sort_order {:optional true} [:maybe :string]]])
-
-(def ^:private construct-metric-schema
- [:or construct-field-aggregation-schema construct-measure-aggregation-schema])
-
-(def ^:private construct-order-by-schema
- [:map {:closed true}
- [:field construct-field-schema]
- [:direction :string]])
-
-;; Canonical operation values per filter_type.
-;; These appear as enum constraints in the JSON Schema sent to the LLM.
-(def ^:private multi-value-operations
- ["equals" "not-equals"
- "string-starts-with" "string-ends-with"
- "string-contains" "string-not-contains"
- ;; aliases the LLM may send from system-instruction shorthand
- "contains" "not-contains"
- "starts-with" "ends-with"])
-
-(def ^:private single-value-operations
- ["greater-than" "greater-than-or-equal"
- "less-than" "less-than-or-equal"
- ;; LLM may use equals/not-equals with a single value instead of multi_value
- "equals" "not-equals"])
-
-(def ^:private no-value-operations
- ["is-null" "is-not-null"
- "string-is-empty" "string-is-not-empty"
- "is-true" "is-false"
- ;; aliases
- "is-empty" "is-not-empty"])
-
-(def ^:private construct-multi-value-filter-schema
- [:map {:closed true
- :decode/tool filter-tools/decode-temporal-filter}
- [:filter_type [:enum "multi_value"]]
- [:field_id :string]
- [:operation (into [:enum] multi-value-operations)]
- [:bucket {:optional true} [:maybe :string]]
- [:value {:optional true} [:maybe :any]]
- [:values {:optional true} [:maybe [:sequential :any]]]])
-
-(def ^:private construct-single-value-filter-schema
- [:map {:closed true
- :decode/tool filter-tools/decode-temporal-filter}
- [:filter_type [:enum "single_value"]]
- [:field_id :string]
- [:operation (into [:enum] single-value-operations)]
- [:bucket {:optional true} [:maybe :string]]
- [:value {:optional true} [:maybe :any]]
- [:values {:optional true} [:maybe [:sequential :any]]]])
-
-(def ^:private construct-no-value-filter-schema
- [:map {:closed true}
- [:filter_type [:enum "no_value"]]
- [:field_id :string]
- [:operation (into [:enum] no-value-operations)]])
-
-(def ^:private construct-segment-filter-schema
- [:map {:closed true}
- [:filter_type [:enum "segment"]]
- [:segment_id :int]])
-
-(def ^:private construct-filter-schema
- [:or
- construct-multi-value-filter-schema
- construct-single-value-filter-schema
- construct-no-value-filter-schema
- construct-segment-filter-schema])
-
-(def ^:private construct-source-metric-schema
- [:map {:closed true}
- [:metric_id :int]])
-
-(def ^:private construct-source-model-schema
- [:map {:closed true}
- [:model_id :int]])
-
-(def ^:private construct-source-table-schema
- [:map {:closed true}
- [:table_id :int]])
+(def construct-program-schema
+ "Schema for the program parameter of construct_notebook_query.
+ Intentionally loose — agent-lib validates and repairs internally."
+ [:map
+ [:source [:map
+ [:type :string]
+ [:id {:optional true} [:maybe :int]]
+ [:ref {:optional true} [:maybe :string]]]]
+ [:operations [:sequential [:sequential :any]]]])
(def ^:private construct-visualization-schema
[:map {:closed true}
[:chart_type :string]])
-;; The LLM sometimes nests visualization inside query, so we accept it as optional
-;; in each query variant and pull it out during normalization.
-(def ^:private construct-query-metric-schema
- [:map {:closed true}
- [:query_type [:enum "metric"]]
- [:source construct-source-metric-schema]
- [:filters [:sequential construct-filter-schema]]
- [:group_by [:sequential construct-field-schema]]
- [:visualization {:optional true} construct-visualization-schema]])
-
-(def ^:private construct-query-aggregate-schema
- [:map {:closed true}
- [:query_type [:enum "aggregate"]]
- [:source [:or construct-source-model-schema construct-source-table-schema]]
- [:aggregations [:sequential construct-metric-schema]]
- [:filters [:sequential construct-filter-schema]]
- [:group_by [:sequential construct-field-schema]]
- [:limit [:maybe :int]]
- [:visualization {:optional true} construct-visualization-schema]])
-
-(def ^:private construct-query-raw-schema
+(def ^:private construct-notebook-query-args-schema
[:map {:closed true}
- [:query_type [:enum "raw"]]
- [:source [:or construct-source-model-schema construct-source-table-schema]]
- [:filters [:sequential construct-filter-schema]]
- [:fields [:sequential construct-field-schema]]
- [:order_by [:sequential construct-order-by-schema]]
- [:limit [:maybe :int]]
+ [:reasoning {:optional true} :string]
+ [:source_entity source-entity-schema]
+ [:referenced_entities {:optional true} [:maybe [:sequential source-entity-schema]]]
+ [:program construct-program-schema]
[:visualization {:optional true} construct-visualization-schema]])
-(def construct-query-schema
- "Schema for the query parameter of construct_notebook_query.
- Shared with the slackbot query tool variant."
- [:or construct-query-metric-schema construct-query-aggregate-schema construct-query-raw-schema])
-
-(def ^:private construct-operation-aliases
- {"contains" :string-contains
- "not-contains" :string-not-contains
- "starts-with" :string-starts-with
- "ends-with" :string-ends-with
- "is-empty" :string-is-empty
- "is-not-empty" :string-is-not-empty})
-
-(defn- normalize-construct-operation
- [operation]
- (let [op (cond
- (keyword? operation) operation
- (string? operation) (keyword operation)
- :else operation)]
- (get construct-operation-aliases (name op) op)))
-
-(defn- normalize-construct-filter
- [filter]
- (let [normalized (normalize-ai-args filter)
- filter-type (:filter-type normalized)
- normalized (cond-> normalized
- (:operation normalized) (update :operation normalize-construct-operation))]
- (case filter-type
- :multi-value (-> normalized
- (dissoc :filter-type :value)
- (update :values (fn [vals]
- (cond
- (sequential? vals) (vec vals)
- (some? (:value normalized)) [(:value normalized)]
- :else []))))
- :single-value (-> normalized
- (dissoc :filter-type :values)
- (assoc :value (or (:value normalized)
- (first (:values normalized)))))
- :no-value (-> normalized
- (dissoc :filter-type :value :values))
- :segment (-> normalized
- (dissoc :filter-type :field-id :operation :bucket :value :values))
- (dissoc normalized :filter-type))))
-
-(defn- normalize-construct-filters
- [filters]
- (mapv normalize-construct-filter filters))
+;;; ---------------------------------------- Source resolution ----------------------------------------
+
+(defn- resolve-source-database-id
+ "Resolve the database ID for a source_entity."
+ [{:keys [type id]}]
+ (case type
+ "table" (:db_id (tools.u/get-table id :db_id))
+ ("model" "question") (:database_id (tools.u/get-card id))
+ "metric" (:database_id (tools.u/get-card id))
+ (throw (ex-info (str "Unsupported source_entity type: " type)
+ {:agent-error? true :status-code 400}))))
+
+(defn- source-entity->model-str
+ "Map source_entity type to the model string used by agent-lib evaluation context."
+ [type]
+ (case type
+ "table" "table"
+ "model" "dataset"
+ "question" "card"
+ "metric" "metric"
+ type))
+
+(defn program-source->source-entity
+ "Convert a structured-program `:source` map (using agent-lib model strings: `table`,
+ `card`, `dataset`, `metric`) into the `source-entity` shape consumed by
+ [[execute-program]] (using metabot type strings: `table`, `question`, `model`,
+ `metric`). Throws if the source isn't one of the four database-resolved types
+ (e.g. `context` or nested `program` sources are rejected)."
+ [{:keys [type id] :as source}]
+ (let [entity-type (case type
+ "table" "table"
+ "card" "question"
+ "dataset" "model"
+ "metric" "metric"
+ (throw (ex-info (str "Unsupported program source type: " (pr-str type))
+ {:agent-error? true
+ :status-code 400
+ :source source})))]
+ {:type entity-type :id id}))
+
+(defn- source-metadata-for
+ "Resolve the lib metadata object for a source entity."
+ [type id metadata-provider]
+ (case type
+ "table" (lib.metadata/table metadata-provider id)
+ ("model" "question") (lib.metadata/card metadata-provider id)
+ "metric" (lib.metadata/card metadata-provider id)
+ nil))
+
+(defn- surrounding-tables-for
+ "Derive surrounding tables from a source query's visible columns."
+ [metadata-provider source-metadata source-id]
+ (try
+ (let [query (lib/query metadata-provider source-metadata)]
+ (->> (lib/visible-columns query)
+ (keep :table-id)
+ distinct
+ (remove #{source-id})
+ (mapv (fn [tid] {:id tid}))))
+ (catch Exception _ [])))
+
+(defn- available-measure-ids
+ "Return the set of measure IDs available on a source entity, or empty set."
+ [metadata-provider source-metadata]
+ (try
+ (->> (lib/available-measures (lib/query metadata-provider source-metadata))
+ (keep :id)
+ set)
+ (catch Exception _ #{})))
+
+(defn- build-evaluation-context
+ "Build the EvaluationContext for agent-lib from source_entity and referenced_entities."
+ [{:keys [type id]} referenced-entities metadata-provider]
+ (let [model-str (source-entity->model-str type)
+ source-metadata (source-metadata-for type id metadata-provider)
+ surrounding (surrounding-tables-for metadata-provider source-metadata id)
+ measure-ids (available-measure-ids metadata-provider source-metadata)]
+ {:source-entity {:model model-str :id id}
+ :referenced-entities (or (mapv (fn [{:keys [type id]}]
+ {:model (source-entity->model-str type) :id id})
+ referenced-entities)
+ [])
+ :surrounding-tables surrounding
+ :measure-ids measure-ids
+ :source-metadata source-metadata}))
+
+;;; ---------------------------------------- Result columns ----------------------------------------
+
+(defn- result-columns-for-query
+ "Generate result columns from a pMBQL query for LLM consumption."
+ [pmbql-query metadata-provider]
+ (let [query (lib/query metadata-provider pmbql-query)
+ cols (lib/returned-columns query)]
+ (mapv #(tools.u/->result-column query %) cols)))
+
+;;; ---------------------------------------- Query execution ----------------------------------------
+
+(defn execute-program
+ "Execute a structured program via agent-lib.
+ Returns the raw result map with :structured-output.
+ Shared between construct-notebook-query-tool and slackbot-construct-notebook-query-tool."
+ [source-entity referenced-entities program]
+ (let [database-id (resolve-source-database-id source-entity)
+ mp (lib-be/application-database-metadata-provider database-id)
+ context (build-evaluation-context source-entity referenced-entities mp)
+ pmbql-query (agent-lib/evaluate-program program mp context)
+ query-id (u/generate-nano-id)]
+ {:structured-output {:query-id query-id
+ :query pmbql-query
+ :result-columns (result-columns-for-query pmbql-query mp)}
+ :instructions (instructions/query-created-instructions-for query-id)}))
+
+;;; ---------------------------------------- Chart helpers ----------------------------------------
(defn- chart-type->keyword
[chart-type]
@@ -234,21 +170,6 @@
(string? chart-type) (keyword chart-type)
:else chart-type))
-(defn- query-type->keyword
- [query-type]
- (cond
- (keyword? query-type) query-type
- (string? query-type) (keyword query-type)
- :else query-type))
-
-(def ^:private construct-notebook-query-args-schema
- "Schema for the `construct_notebook_query` tool arguments.
- Filter sub-schemas carry `:decode/tool` transforms for temporal value coercion."
- [:map {:closed true}
- [:reasoning {:optional true} :string]
- [:query construct-query-schema]
- [:visualization {:optional true} construct-visualization-schema]])
-
(defn- structured->query-data
"Convert tool structured output to a map suitable for [[llm-rep/query->xml]].
Converts the MBQL 5 query to legacy MBQL, JSON-encodes it, and wraps result columns."
@@ -264,73 +185,28 @@
{:result_columns result-columns})}))
(defn- structured->chart-xml
- "Render the full chart XML (matching Python Visualization.llm_representation)
- for the construct_notebook_query tool result."
+ "Render the full chart XML for the construct_notebook_query tool result."
[structured chart-id chart-type]
(llm-rep/visualization->xml
{:chart-id chart-id
:queries [(structured->query-data structured)]
:visualization_settings {:chart_type (if chart-type (name chart-type) "table")}}))
-(def ^:private tool-arg-transformer
- "Malli transformer that applies `:decode/tool` transforms declared on sub-schemas.
- Used by [[decode-tool-args]] to coerce LLM arguments (e.g., date strings → integers
- for extraction buckets) before the tool function runs."
- (mtx/transformer {:name :tool}))
-
-(defn- decode-tool-args
- "Decode tool arguments through the Malli schema, applying `:decode/tool` transforms.
- This is attached as `:decode` metadata on the tool var so that [[run-tool]] calls it
- before invoking the tool function."
- [args]
- (mc/decode construct-notebook-query-args-schema args tool-arg-transformer))
-
-(defn execute-query
- "Execute a notebook query based on normalized query parameters.
- Returns the raw query result from filter-tools.
- Shared between construct-notebook-query-tool and slackbot-construct-notebook-query-tool."
- [query]
- (let [normalized-query (normalize-ai-args query)
- query-type (query-type->keyword (:query-type normalized-query))]
- (log/debug "execute-query" {:query-type query-type})
- (case query-type
- :metric
- (filter-tools/query-metric
- {:metric-id (get-in normalized-query [:source :metric-id])
- :filters (normalize-construct-filters (:filters normalized-query))
- :group-by (normalize-ai-args (:group-by normalized-query))})
- :aggregate
- (filter-tools/query-datasource
- {:model-id (get-in normalized-query [:source :model-id])
- :table-id (get-in normalized-query [:source :table-id])
- :aggregations (normalize-ai-args (:aggregations normalized-query))
- :filters (normalize-construct-filters (:filters normalized-query))
- :group-by (normalize-ai-args (:group-by normalized-query))
- :limit (:limit normalized-query)})
- :raw
- (filter-tools/query-datasource
- {:model-id (get-in normalized-query [:source :model-id])
- :table-id (get-in normalized-query [:source :table-id])
- :fields (normalize-ai-args (:fields normalized-query))
- :filters (normalize-construct-filters (:filters normalized-query))
- :order-by (normalize-ai-args (:order-by normalized-query))
- :limit (:limit normalized-query)})
- {:output (str "Unsupported query_type: " query-type)})))
+;;; ---------------------------------------- Main tool ----------------------------------------
(mu/defn ^{:tool-name "construct_notebook_query"
- :decode decode-tool-args
:scope scope/agent-notebook-create}
construct-notebook-query-tool
"Construct and visualize a notebook query from a metric, model, or table."
- [{:keys [_reasoning query visualization]} :- construct-notebook-query-args-schema]
+ [{:keys [_reasoning source_entity referenced_entities program visualization]} :- construct-notebook-query-args-schema]
(try
- (let [;; LLM sometimes nests visualization inside query — pull it out
- effective-viz (or visualization (:visualization query))
- normalized-visualization (normalize-ai-args effective-viz)
- chart-type (or (chart-type->keyword (:chart-type normalized-visualization))
- :table)
- query-result (execute-query (dissoc query :visualization))
- structured (or (:structured-output query-result) (:structured_output query-result))]
+ (let [;; LLM sometimes nests visualization inside program — pull it out
+ effective-viz (or visualization (:visualization program))
+ normalized-visualization (some-> effective-viz (update-keys (comp keyword u/->kebab-case-en name)))
+ chart-type (or (chart-type->keyword (:chart-type normalized-visualization))
+ :table)
+ query-result (execute-program source_entity referenced_entities (dissoc program :visualization))
+ structured (or (:structured-output query-result) (:structured_output query-result))]
(if (and structured (:query-id structured) (:query structured))
(let [chart-result (create-chart-tools/create-chart
{:query-id (:query-id structured)
diff --git a/src/metabase/metabot/tools/document.clj b/src/metabase/metabot/tools/document.clj
index d20dea98d08..12ee246843d 100644
--- a/src/metabase/metabot/tools/document.clj
+++ b/src/metabase/metabot/tools/document.clj
@@ -183,7 +183,8 @@
[:map {:closed true}
[:name :string]
[:description :string]
- [:query :map]
+ [:source_entity [:map [:type :string] [:id :int]]]
+ [:program :map]
[:viz_settings [:map {:closed true}
[:chart_type chart-type-enum]]]])
@@ -191,11 +192,12 @@
:scope scope/agent-document-create}
document-construct-model-chart-tool
"Construct notebook/model-backed chart draft payload for document insertion."
- [{:keys [name description query viz_settings]} :- model-chart-schema]
+ [{:keys [name description source_entity program viz_settings]} :- model-chart-schema]
(try
(let [chart-type (get viz_settings :chart_type)
result (construct-tools/construct-notebook-query-tool
- {:query query
+ {:source_entity source_entity
+ :program program
:visualization {:chart_type chart-type}})
structured (or (:structured-output result) (:structured_output result))
query-id (:query-id structured)
diff --git a/src/metabase/metabot/tools/entity_details.clj b/src/metabase/metabot/tools/entity_details.clj
index b49ac98b8d4..0ce00eaf1dc 100644
--- a/src/metabase/metabot/tools/entity_details.clj
+++ b/src/metabase/metabot/tools/entity_details.clj
@@ -130,32 +130,22 @@
visible-cols (when query-needed?
(->> (lib/visible-columns base-query)
(map #(metabot.tools.u/add-table-reference base-query %))))
- col->index (when query-needed?
- (into {} (map-indexed (fn [i col] [col i])) visible-cols))
- col-index (when query-needed?
- #(-> % (dissoc :operators :field-values) col->index))
default-temporal-breakout (when with-default-temporal-breakout?
(->> breakouts
(map #(lib/find-matching-column % visible-cols))
- (m/find-first lib.types.isa/temporal?)))
- field-id-prefix (metabot.tools.u/card-field-id-prefix id)]
+ (m/find-first lib.types.isa/temporal?)))]
(cond-> {:id id
:type :metric
:name (:name card)
:description (:description card)
- :default_time_dimension_field_id (when default-temporal-breakout
- (-> (metabot.tools.u/->result-column
- metric-query
- default-temporal-breakout
- (col-index default-temporal-breakout)
- field-id-prefix)
- :field_id))
+ :default_time_dimension_field_id (some-> default-temporal-breakout
+ (->> (metabot.tools.u/->result-column metric-query))
+ :field_id)
:verified (verified-review? id "card")}
with-queryable-dimensions?
(assoc :queryable-dimensions (into []
(comp (map #(metabot.tools.u/add-table-reference base-query %))
- (map #(metabot.tools.u/->result-column
- metric-query % (col-index %) field-id-prefix)))
+ (map #(metabot.tools.u/->result-column metric-query %)))
(->> (lib/filterable-columns base-query)
field-values-fn)))
@@ -203,13 +193,11 @@
(->> (lib/visible-columns table-query -1 {:include-implicitly-joinable? false})
field-values-fn
(map #(metabot.tools.u/add-table-reference table-query %))))
- field-id-prefix (when (or with-fields? with-related-tables?)
- (metabot.tools.u/table-field-id-prefix id))
related-tables (when with-related-tables?
- (related-tables table-query field-id-prefix with-fields? field-values-fn))]
+ (related-tables table-query with-fields? field-values-fn))]
(-> {:id id
:type :table
- :fields (into [] (map-indexed #(metabot.tools.u/->result-column table-query %2 %1 field-id-prefix)) cols)
+ :fields (mapv #(metabot.tools.u/->result-column table-query %) cols)
:name (:name base)
;; :display_name should be (lib/display-name table-query), but we want to avoid creating the query if possible
:display_name (some->> (:name base)
@@ -232,38 +220,21 @@
(defn related-tables
"Constructs a list of tables, optionally including their fields, that are related to the given query via foreign key.
Creates separate entries for each FK path when the same table is reachable through multiple foreign keys."
- [query main-field-id-prefix with-fields? field-values-fn]
- (let [all-main-cols (lib/visible-columns query)
- ;; Map [table-id fk-field-id field-name] -> index in the main query
- contextual-index (into {}
- (keep-indexed
- (fn [idx {:keys [fk-field-id table-id name]}]
- (when fk-field-id
- {[table-id fk-field-id name] idx})))
- all-main-cols)
- fk-cols (filter :fk-field-id all-main-cols)
- ;; { [table-id fk-field-id] [fk-col ...] }
- grouped-fks (group-by (juxt :table-id :fk-field-id) fk-cols)]
+ [query with-fields? field-values-fn]
+ (let [all-main-cols (lib/visible-columns query)
+ fk-cols (filter :fk-field-id all-main-cols)
+ grouped-fks (group-by (juxt :table-id :fk-field-id) fk-cols)]
(when (seq grouped-fks)
(mapv
(fn [[[table-id fk-field-id] _]]
(let [base-details (table-details table-id
- {:with-fields? with-fields?
- :field-values-fn field-values-fn
- :with-related-tables? false
- :with-metrics? false})
+ {:with-fields? with-fields?
+ :field-values-fn field-values-fn
+ :with-related-tables? false
+ :with-metrics? false})
base-table-col (lib.metadata/field query fk-field-id)
- fk-field-name (:name base-table-col)
- updated-fields
- (when with-fields?
- (->> (:fields base-details)
- (keep
- (fn [{:keys [name] :as field}]
- (when-let [idx (get contextual-index [table-id fk-field-id name])]
- (assoc field :field_id (str main-field-id-prefix idx)))))))]
- (-> (cond-> base-details
- updated-fields (assoc :fields updated-fields))
- (assoc :related_by fk-field-name))))
+ fk-field-name (:name base-table-col)]
+ (assoc base-details :related_by fk-field-name)))
grouped-fks))))
(defn- card-details
@@ -298,12 +269,11 @@
returned-fields (when with-fields?
(->> (lib/returned-columns card-query)
field-values-fn))
- field-id-prefix (metabot.tools.u/card-field-id-prefix id)
related-tables (when with-related-tables?
- (related-tables card-query field-id-prefix with-fields? field-values-fn))]
+ (related-tables card-query with-fields? field-values-fn))]
(-> {:id id
:type card-type
- :fields (into [] (map-indexed #(metabot.tools.u/->result-column card-query %2 %1 field-id-prefix)) returned-fields)
+ :fields (mapv #(metabot.tools.u/->result-column card-query %) returned-fields)
:name (:name base)
:display_name (some->> (:name base)
(u.humanization/name->human-readable-name :simple))
@@ -463,7 +433,6 @@
(defn- execute-query
[query-id query-input]
(let [normalized-query (lib-be/normalize-query query-input)
- field-id-prefix (metabot.tools.u/query-field-id-prefix query-id)
database-id (:database normalized-query)
_ (api/read-check :model/Database database-id)
mp (lib-be/application-database-metadata-provider database-id)
@@ -472,9 +441,7 @@
{:type :query
:query-id query-id
:query normalized-query
- :result-columns (into []
- (map-indexed #(metabot.tools.u/->result-column query %2 %1 field-id-prefix))
- returned-cols)}))
+ :result-columns (mapv #(metabot.tools.u/->result-column query %) returned-cols)}))
(defn get-query-details
"Get the details of a query (supports both MBQL v4 and v5)."
diff --git a/src/metabase/metabot/tools/field_stats.clj b/src/metabase/metabot/tools/field_stats.clj
index dcfd3cb4c56..09f3f8489f4 100644
--- a/src/metabase/metabot/tools/field_stats.clj
+++ b/src/metabase/metabot/tools/field_stats.clj
@@ -1,7 +1,6 @@
(ns metabase.metabot.tools.field-stats
(:require
[clojure.set :as set]
- [clojure.string :as str]
[metabase.lib.core :as lib]
[metabase.metabot.tools.util :as metabot.tools.u]
[metabase.parameters.field-values :as params.field-values]
@@ -34,56 +33,43 @@
(build-field-statistics nil fingerprint limit)))
(defn- table-field-stats
- [table-id agent-field-id limit]
+ [table-id field-id limit]
(try
- (let [field-id-prefix (metabot.tools.u/table-field-id-prefix table-id)
- ;; When the field ID belongs to a different table (e.g., agent saw a related field
- ;; listed under table 111 as t111-21 but requests it via metabase://table/173/fields/t111-21),
- ;; resolve against the correct table from the field ID prefix.
- effective-table-id (if (str/starts-with? agent-field-id field-id-prefix)
- table-id
- (let [parsed (metabot.tools.u/parse-field-id agent-field-id)]
- (if (and parsed (= "t" (:model-tag parsed)) (:model-id parsed))
- (:model-id parsed)
- table-id)))
- query (or (metabot.tools.u/table-query effective-table-id)
- (throw (ex-info (str "No table found with ID " effective-table-id)
- {:agent-error? true :status-code 404})))
- eff-prefix (metabot.tools.u/table-field-id-prefix effective-table-id)
- visible-cols (lib/visible-columns query)
- col (:column (metabot.tools.u/resolve-column {:field-id agent-field-id} eff-prefix visible-cols))]
+ (let [query (or (metabot.tools.u/table-query table-id)
+ (throw (ex-info (str "No table found with ID " table-id)
+ {:agent-error? true :status-code 404})))
+ visible-cols (lib/visible-columns query)
+ col (metabot.tools.u/find-column-by-field-id field-id visible-cols)]
{:structured-output {:result-type :field-metadata
- :field_id agent-field-id
+ :field_id field-id
:value_metadata (field-statistics col limit)}})
(catch Exception ex
(metabot.tools.u/handle-agent-error ex))))
(defn- card-field-stats
- [card-id agent-field-id limit card-type]
+ [card-id field-id limit card-type]
(try
- (let [query (or (metabot.tools.u/card-query card-id)
- (throw (ex-info (str "No " card-type " found with ID " card-id)
- {:agent-error? true :status-code 404})))
- field-id-prefix (metabot.tools.u/card-field-id-prefix card-id)
- visible-cols (lib/visible-columns query)
- col (:column (metabot.tools.u/resolve-column {:field-id agent-field-id} field-id-prefix visible-cols))]
+ (let [query (or (metabot.tools.u/card-query card-id)
+ (throw (ex-info (str "No " card-type " found with ID " card-id)
+ {:agent-error? true :status-code 404})))
+ visible-cols (lib/visible-columns query)
+ col (metabot.tools.u/find-column-by-field-id field-id visible-cols)]
{:structured-output {:result-type :field-metadata
- :field_id agent-field-id
+ :field_id field-id
:value_metadata (field-statistics col limit)}})
(catch Exception ex
(metabot.tools.u/handle-agent-error ex))))
(defn- metric-field-stats
- [metric-id agent-field-id limit]
+ [metric-id field-id limit]
(try
(let [query (or (metabot.tools.u/metric-query metric-id)
(throw (ex-info (str "No metric found with ID " metric-id)
{:agent-error? true :status-code 404})))
- field-id-prefix (metabot.tools.u/card-field-id-prefix metric-id)
filterable-cols (lib/filterable-columns query)
- col (:column (metabot.tools.u/resolve-column {:field-id agent-field-id} field-id-prefix filterable-cols))]
+ col (metabot.tools.u/find-column-by-field-id field-id filterable-cols)]
{:structured-output {:result-type :field-metadata
- :field_id agent-field-id
+ :field_id field-id
:value_metadata (field-statistics col limit)}})
(catch Exception ex
(metabot.tools.u/handle-agent-error ex))))
diff --git a/src/metabase/metabot/tools/filters.clj b/src/metabase/metabot/tools/filters.clj
deleted file mode 100644
index fcb8b5de38f..00000000000
--- a/src/metabase/metabot/tools/filters.clj
+++ /dev/null
@@ -1,675 +0,0 @@
-(ns metabase.metabot.tools.filters
- (:require
- [buddy.core.codecs :as codecs]
- [metabase.api.common :as api]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.types.isa :as lib.types.isa]
- [metabase.lib.util :as lib.util]
- [metabase.metabot.agent.streaming :as streaming]
- [metabase.metabot.tools.shared.instructions :as instructions]
- [metabase.metabot.tools.util :as metabot.tools.u]
- [metabase.util :as u]
- [metabase.util.i18n :refer [tru]]
- [metabase.util.json :as json]
- [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-;;; URL Generation for Auto-Navigation
-
-(defn- query->url-hash
- "Convert an MBQL 4 (legacy) or MBQL 5 query to a base64-encoded URL hash."
- [query]
- #_{:clj-kondo/ignore [:discouraged-var]}
- (let [dataset-query (if (and (map? query) (:lib/type query))
- (lib/->legacy-MBQL query)
- query)]
- (-> {:dataset_query dataset-query}
- json/encode
- (.getBytes "UTF-8")
- codecs/bytes->b64-str)))
-
-(defn- query->results-url
- "Convert a query to a /question# URL for navigation."
- [query]
- (str "/question#" (query->url-hash query)))
-
-;;; Filter Operations
-
-(defn- apply-filter-bucket
- [column bucket]
- (case bucket
- :second-of-minute (lib/get-second column)
- :minute-of-hour (lib/get-minute column)
- :hour-of-day (lib/get-hour column)
- :day-of-week (lib/get-day-of-week column :iso)
- :day-of-month (lib/get-day column)
- :week-of-year (lib/get-week column :iso)
- :month-of-year (lib/get-month column)
- :quarter-of-year (lib/get-quarter column)
- :year-of-era (lib/get-year column)
- ;; these below work in queries but not in the UI
- #_:millisecond
- #_:second
- #_:minute
- #_:hour
- #_:day
- #_:week
- #_:month
- #_:quarter
- #_:year
- #_:day-of-year
- (lib/with-temporal-bucket column bucket)))
-
-(defn- filter-bucketed-column
- [{:keys [column bucket]}]
- (cond-> column
- (and bucket
- (lib.types.isa/temporal? column))
- (apply-filter-bucket bucket)))
-
-(defn- apply-bucket
- [column bucket]
- (case bucket
- ;; these two work in queries but not in the UI
- :day-of-week (lib/get-day-of-week column :iso)
- :week-of-year (lib/get-week column :iso)
- #_:second-of-minute
- #_:minute-of-hour
- #_:hour-of-day
- #_:day-of-month
- #_:month-of-year
- #_:quarter-of-year
- #_:year-of-era
- #_:millisecond
- #_:second
- #_:minute
- #_:hour
- #_:day
- #_:week
- #_:month
- #_:quarter
- #_:year
- #_:day-of-year
- (lib/with-temporal-bucket column bucket)))
-
-(defn- bucketed-column
- [{:keys [column bucket]}]
- (cond-> column
- (and bucket
- (lib.types.isa/temporal? column))
- (apply-bucket bucket)))
-
-;;; Temporal bucket metadata
-;;
-;; Extraction buckets (e.g., :year-of-era) extract an integer component from a
-;; date/datetime. Truncation buckets (e.g., :day) round to a boundary and expect
-;; date/datetime string values.
-;;
-;; This metadata drives both coercion (date-string → integer for extraction buckets)
-;; and validation (range checks, rejecting negative/numeric values for truncation
-;; buckets). It is used by [[coerce-and-validate-temporal-filter]].
-
-(def ^:private extraction-bucket-specs
- "Extraction bucket → `{:min :max :hint}`.
- Filter values for extraction buckets must be integers (or integer-strings).
- Date strings are rejected — the LLM should filter date fields directly without a bucket
- for date range comparisons."
- {:year-of-era {}
- :quarter-of-year {:min 1 :max 4 :hint "quarter-of-year expects 1-4"}
- :month-of-year {:min 1 :max 12 :hint "month-of-year expects 1-12"}
- :week-of-year {:min 1 :max 53 :hint "week-of-year expects 1-53"}
- :day-of-month {:min 1 :max 31 :hint "day-of-month expects 1-31"}
- :day-of-week {:min 1 :max 7 :hint "day-of-week expects 1-7 (1=Monday, 7=Sunday)"}
- :hour-of-day {:min 0 :max 23 :hint "hour-of-day expects 0-23"}
- :minute-of-hour {:min 0 :max 59 :hint "minute-of-hour expects 0-59"}
- :second-of-minute {:min 0 :max 59 :hint "second-of-minute expects 0-59"}})
-
-(def ^:private truncation-buckets
- "Temporal buckets that truncate a date/datetime to a boundary (e.g., `:day` truncates to midnight).
- Filter values for these buckets should be date/datetime strings, not relative integers."
- #{:millisecond :second :minute :hour :day :week :month :quarter :year :day-of-year})
-
-(defn- agent-error!
- "Throw an `:agent-error?` exception (returned to the LLM so it can retry)."
- [msg]
- (throw (ex-info msg {:agent-error? true :status-code 400})))
-
-(defn- coerce-extraction-value
- "Coerce a single filter value to an integer for an extraction bucket.
- Parses integer strings but rejects date strings — extraction buckets require plain
- integers, not dates. If the LLM passes a date string like \"2025-09-01\" with bucket
- \"day-of-month\", it almost certainly meant to filter the date field directly without
- a bucket (date range comparison), not extract the day component.
- Validates that the result is within the expected range."
- [v {:keys [min max hint] :as _spec} bucket]
- (letfn [(validate-range [n]
- (when (neg? n)
- (agent-error!
- (str "Filter value " n " cannot be negative for temporal bucket '" (name bucket) "'. "
- "Temporal buckets extract components from dates (e.g., month-of-year extracts 1-12). "
- "If you need to filter relative to today (e.g., 'last 60 days'), don't use temporal "
- "buckets — filter the date field directly with a date range instead.")))
- (when (and min max (not (<= min n max)))
- (agent-error! (str "Filter value " n " is out of range for bucket '" (name bucket) "'. " hint ".")))
- n)]
- (cond
- (number? v)
- (validate-range v)
-
- (string? v)
- (try
- (validate-range (Integer/parseInt v))
- (catch NumberFormatException _
- (agent-error!
- (str "Filter with temporal bucket '" (name bucket) "' requires an integer value, "
- "got: \"" v "\". For example, 'day-of-month' expects 15, not \"2024-01-15\". "
- "If you need to filter by a date range (e.g., 'last 30 days'), don't use a "
- "temporal bucket — filter the date field directly with operation "
- "'greater-than-or-equal' and value '2025-01-15' (no bucket)."))))
-
- :else v)))
-
-(defn- validate-truncation-value
- "Reject numeric values for truncation buckets — they need date strings."
- [v bucket]
- (when (and (number? v) (or (neg? v) (< v 100)))
- (agent-error!
- (str "Filter value " v " is not valid for temporal bucket '" (name bucket) "'. "
- "Temporal buckets like '" (name bucket) "' require date/datetime string values "
- "(e.g., \"2024-01-15\"), not relative numbers. "
- "To filter for 'last 30 days', use a direct date comparison without a bucket, "
- "e.g., operation 'greater-than-or-equal' with value '2025-01-15' (the actual date)."))))
-
-(defn coerce-and-validate-temporal-filter
- "Coerce and validate temporal filter values based on the bucket type.
-
- For extraction buckets (e.g., `:year-of-era`): coerce date-string values to
- integers and validate the range.
- For truncation buckets (e.g., `:day`): reject numeric values that indicate
- the LLM is trying to use relative date math.
-
- Works on normalized filters (keyword keys and keyword buckets).
- Called from [[add-filter]].
-
- See also [[decode-temporal-filter]] for the raw (pre-normalization) variant
- used in Malli schema decode transforms."
- [{:keys [value values bucket] :as llm-filter}]
- (if-let [spec (extraction-bucket-specs bucket)]
- ;; Extraction bucket — coerce and validate values
- (let [coerce-val #(coerce-extraction-value % spec bucket)]
- (cond-> llm-filter
- value (update :value coerce-val)
- values (update :values (partial mapv coerce-val))))
- ;; Truncation bucket — validate values aren't numeric nonsense
- (if (truncation-buckets bucket)
- (do (doseq [v (or values (some-> value vector))]
- (validate-truncation-value v bucket))
- llm-filter)
- ;; No bucket or unknown — pass through
- llm-filter)))
-
-(defn decode-temporal-filter
- "Decode a raw (pre-normalization) filter map for Malli schema `:decode/tool` transforms.
-
- Operates on string keys (`\"bucket\"`, `\"value\"`, `\"values\"`) as they arrive from
- the LLM's JSON, coercing and validating temporal filter values.
-
- Example usage in a Malli schema:
-
- [:map {:decode/tool filters/decode-temporal-filter}
- [:bucket {:optional true} [:maybe :string]]
- [:value {:optional true} :any]]"
- [m]
- (let [bucket-str (or (get m "bucket") (get m :bucket))
- bucket-kw (when bucket-str (keyword bucket-str))]
- (if-not bucket-kw
- m
- (let [v-key (if (contains? m "value") "value" :value)
- vs-key (if (contains? m "values") "values" :values)
- as-norm {:bucket bucket-kw
- :value (get m v-key)
- :values (get m vs-key)}
- coerced (coerce-and-validate-temporal-filter as-norm)]
- (cond-> m
- (get m v-key) (assoc v-key (:value coerced))
- (get m vs-key) (assoc vs-key (:values coerced)))))))
-
-(def ^:private temporal-extraction-operations
- "Operations that extract an integer component from a date/datetime field (e.g. month 1-12).
- These operations inherently expect small integer values, so they must be exempt from the
- validation that rejects small integers on temporal columns."
- #{:year-equals :year-not-equals
- :quarter-equals :quarter-not-equals
- :month-equals :month-not-equals
- :day-of-week-equals :day-of-week-not-equals
- :hour-equals :hour-not-equals
- :minute-equals :minute-not-equals
- :second-equals :second-not-equals})
-
-(defn- validate-temporal-column-values
- "Reject clearly-wrong numeric values on temporal columns when no bucket is specified.
- Negative numbers and small integers (< 100) on a datetime field are almost certainly the
- LLM trying to express relative dates (e.g., -30 for 'last 30 days').
-
- This validation requires the resolved `:column` metadata, so it runs in [[add-filter]]
- rather than in the Malli decode layer.
-
- Skips validation for temporal extraction operations (e.g. `month-equals`, `year-equals`)
- because those operations explicitly expect small integer values."
- [{:keys [value values bucket operation column] :as llm-filter}]
- (when (and (nil? bucket)
- column
- (lib.types.isa/temporal? column)
- (not (temporal-extraction-operations operation)))
- (doseq [v (or values (some-> value vector))]
- (when (and (number? v) (or (neg? v) (< v 100)))
- (agent-error!
- (str "Filter value " v " is not valid for a date/datetime field. "
- "Date fields require date/datetime string values (e.g., \"2024-01-15\"), "
- "not relative numbers. "
- "To filter for 'last 30 days', compute the actual date and use "
- "operation 'greater-than-or-equal' with value '2025-01-15' (the actual date).")))))
- llm-filter)
-
-(defn- add-filter
- [query llm-filter]
- (if-let [segment-id (:segment-id llm-filter)]
- ;; Segment-based filter
- (if-let [segment (lib.metadata/segment query segment-id)]
- (lib/filter query segment)
- (throw (ex-info (tru "Segment with id {0} not found" segment-id)
- {:agent-error? true
- :status-code 404
- :segment-id segment-id})))
- ;; Standard field-based filter logic
- (let [llm-filter (-> llm-filter
- coerce-and-validate-temporal-filter
- validate-temporal-column-values)
- {:keys [operation value values]} llm-filter
- expr (filter-bucketed-column llm-filter)
- with-values-or-value (fn with-values-or-value
- ([f]
- (with-values-or-value f expr))
- ([f expr]
- (if values
- (apply f expr values)
- (f expr value))))
- string-match (fn [match-fn]
- (-> match-fn with-values-or-value lib/ignore-case))
- filter
- (case operation
- :is-null (lib/is-null expr)
- :is-not-null (lib/not-null expr)
- :string-is-empty (lib/is-empty expr)
- :string-is-not-empty (lib/not-empty expr)
- :is-true (lib/= expr true)
- :is-false (lib/= expr false)
- :equals (with-values-or-value lib/=)
- :not-equals (with-values-or-value lib/!=)
- :greater-than (lib/> expr value)
- :greater-than-or-equal (lib/>= expr value)
- :less-than (lib/< expr value)
- :less-than-or-equal (lib/<= expr value)
- :year-equals (with-values-or-value lib/= (lib/get-year expr))
- :year-not-equals (with-values-or-value lib/!= (lib/get-year expr))
- :quarter-equals (with-values-or-value lib/= (lib/get-quarter expr))
- :quarter-not-equals (with-values-or-value lib/!= (lib/get-quarter expr))
- :month-equals (with-values-or-value lib/= (lib/get-month expr))
- :month-not-equals (with-values-or-value lib/!= (lib/get-month expr))
- :day-of-week-equals (with-values-or-value lib/= (lib/get-day-of-week expr :iso))
- :day-of-week-not-equals (with-values-or-value lib/!= (lib/get-day-of-week expr :iso))
- :hour-equals (with-values-or-value lib/= (lib/get-hour expr))
- :hour-not-equals (with-values-or-value lib/!= (lib/get-hour expr))
- :minute-equals (with-values-or-value lib/= (lib/get-minute expr))
- :minute-not-equals (with-values-or-value lib/!= (lib/get-minute expr))
- :second-equals (with-values-or-value lib/= (lib/get-second expr))
- :second-not-equals (with-values-or-value lib/!= (lib/get-second expr))
- :date-equals (with-values-or-value lib/=)
- :date-not-equals (with-values-or-value lib/!=)
- :date-before (lib/< expr value)
- :date-on-or-before (lib/<= expr value)
- :date-after (lib/> expr value)
- :date-on-or-after (lib/>= expr value)
- :string-equals (with-values-or-value lib/=)
- :string-not-equals (with-values-or-value lib/!=)
- :string-contains (string-match lib/contains)
- :string-not-contains (string-match lib/does-not-contain)
- :string-starts-with (string-match lib/starts-with)
- :string-ends-with (string-match lib/ends-with)
- :number-equals (with-values-or-value lib/=)
- :number-not-equals (with-values-or-value lib/!=)
- :number-greater-than (lib/> expr value)
- :number-greater-than-or-equal (lib/>= expr value)
- :number-less-than (lib/< expr value)
- :number-less-than-or-equal (lib/<= expr value)
- (throw (ex-info (str "unknown filter operation " operation)
- {:agent-error? true :status-code 400})))]
- (lib/filter query filter))))
-
-(defn- add-breakout
- [query {:keys [column field-granularity]}]
- (let [expr (cond-> column
- (and field-granularity
- (lib.types.isa/temporal? column))
- (lib/with-temporal-bucket field-granularity))]
- (lib/breakout query expr)))
-
-(defn- query-metric*
- [{:keys [metric-id filters group-by] :as _arguments}]
- (let [card (metabot.tools.u/get-card metric-id)
- mp (lib-be/application-database-metadata-provider (:database_id card))
- base-query (->> (lib/query mp (lib.metadata/card mp metric-id))
- lib/remove-all-breakouts)
- field-id-prefix (metabot.tools.u/card-field-id-prefix metric-id)
- visible-cols (lib/visible-columns base-query)
- resolve-visible-column #(metabot.tools.u/resolve-column % field-id-prefix visible-cols)
- ;; Separate segment filters from field filters before column resolution
- resolved-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)
- query (as-> base-query $q
- (reduce add-filter $q resolved-filters)
- (reduce add-breakout
- $q
- (map #(metabot.tools.u/resolve-column % field-id-prefix visible-cols) group-by)))
- query-id (u/generate-nano-id)
- query-field-id-prefix (metabot.tools.u/query-field-id-prefix query-id)
- returned-cols (lib/returned-columns query)]
- {:type :query
- :query-id query-id
- :query query
- :result-columns (into []
- (map-indexed #(metabot.tools.u/->result-column query %2 %1 query-field-id-prefix))
- returned-cols)}))
-
-(defn query-metric
- "Create a query based on a metric.
- Returns structured output with the query and a navigate_to data part."
- [{:keys [metric-id] :as arguments}]
- (try
- (if (int? metric-id)
- (let [result (query-metric* arguments)
- results-url (query->results-url (:query result))]
- {:structured-output (assoc result :result-type :query)
- :instructions (instructions/query-created-instructions-for (:query-id result))
- :data-parts [(streaming/navigate-to-part results-url)]})
- (throw (ex-info (str "Invalid metric_id " metric-id)
- {:agent-error? true :status-code 400})))
- (catch Exception e
- (if (= (:status-code (ex-data e)) 404)
- {:output (ex-message e) :status-code 404}
- (metabot.tools.u/handle-agent-error e)))))
-
-(defn- apply-aggregation-sort-order
- "If sort-order is specified, add an order-by clause for the last aggregation in the query."
- [query sort-order]
- (if sort-order
- (let [query-aggregations (lib/aggregations query)
- last-aggregation-idx (dec (count query-aggregations))]
- (lib/order-by query (lib/aggregation-ref query last-aggregation-idx) sort-order))
- query))
-
-(defn- add-aggregation
- [query aggregation]
- (let [sort-order (:sort-order aggregation)
- query-with-aggregation
- (if-let [measure-id (:measure-id aggregation)]
- ;; Measure-based aggregation
- (if-let [measure (lib.metadata/measure query measure-id)]
- (lib/aggregate query measure)
- (throw (ex-info (tru "Measure with id {0} not found" measure-id)
- {:agent-error? true
- :status-code 404
- :measure-id measure-id})))
- ;; Field-based aggregation
- (let [agg-expr (if (= :count (:function aggregation))
- (lib/count)
- (let [expr (bucketed-column aggregation)]
- (case (:function aggregation)
- (:count-distinct :distinct) (lib/distinct expr)
- :sum (lib/sum expr)
- :min (lib/min expr)
- :max (lib/max expr)
- :avg (lib/avg expr)
- (throw (ex-info (str "Unsupported aggregation function: " (:function aggregation)
- ". Supported: count, count-distinct, sum, min, max, avg")
- {:agent-error? true
- :status-code 400})))))]
- (lib/aggregate query agg-expr)))]
- (apply-aggregation-sort-order query-with-aggregation sort-order)))
-
-(defn- resolve-aggregation-column
- "Resolve the column for an aggregation, skipping measures and field-less counts."
- [resolve-visible-column aggregation]
- (if (or (:measure-id aggregation)
- (and (= :count (:function aggregation))
- (not (:field-id aggregation))))
- aggregation
- (resolve-visible-column aggregation)))
-
-(defn- expression?
- [expr-or-column]
- (vector? expr-or-column))
-
-(defn- add-fields
- [query projection]
- (->> projection
- (map (fn [[expr-or-column expr-name]]
- (if (expression? expr-or-column)
- (lib/expression-ref query expr-name)
- ;; bucketed columns don't work in the UI
- expr-or-column)))
- (lib/with-fields query)))
-
-(defn- add-order-by [query {:keys [field direction]}]
- (lib/order-by query (:column field) direction))
-
-(defn- add-limit [query limit]
- (if limit
- (lib/limit query limit)
- query))
-
-(defn- query-model*
- [{:keys [model-id fields filters aggregations group-by order-by limit] :as _arguments}]
- (let [card (metabot.tools.u/get-card model-id)
- mp (lib-be/application-database-metadata-provider (:database_id card))
- base-query (lib/query mp (lib.metadata/card mp model-id))
- field-id-prefix (metabot.tools.u/card-field-id-prefix model-id)
- visible-cols (lib/visible-columns base-query)
- resolve-visible-column #(metabot.tools.u/resolve-column % field-id-prefix visible-cols)
- _ (log/debug "query_model field-id expectations"
- {:model-id model-id
- :field-id-prefix field-id-prefix
- :field-ids (vec (keep :field-id (concat fields filters group-by)))})
- resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})
- projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}]
- (let [column (cond-> column
- bucket (assoc :unit bucket))]
- (lib/display-name base-query -1 column :long))))
- resolve-visible-column)
- fields)
- resolved-aggregations (map (partial resolve-aggregation-column resolve-visible-column) aggregations)
- resolved-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)
- reduce-query (fn [query f coll] (reduce f query coll))
- query (-> base-query
- (reduce-query (fn [query [expr-or-column expr-name]]
- (lib/expression query expr-name expr-or-column))
- (filter (comp expression? first) projection))
- (add-fields projection)
- (reduce-query add-filter resolved-filters)
- (reduce-query add-aggregation resolved-aggregations)
- (reduce-query add-breakout (map resolve-visible-column group-by))
- (reduce-query add-order-by (map resolve-order-by-column order-by))
- (add-limit limit))
- query-id (u/generate-nano-id)
- query-field-id-prefix (metabot.tools.u/query-field-id-prefix query-id)
- returned-cols (lib/returned-columns query)]
- {:type :query
- :query-id query-id
- :query query
- :result-columns (into []
- (map-indexed #(metabot.tools.u/->result-column query %2 %1 query-field-id-prefix))
- returned-cols)}))
-
-(defn query-model
- "Create a query based on a model.
- Returns structured output with the query and a navigate_to data part."
- [{:keys [model-id] :as arguments}]
- (try
- (if (int? model-id)
- (let [result (query-model* arguments)
- results-url (query->results-url (:query result))]
- {:structured-output (assoc result :result-type :query)
- :instructions (instructions/query-created-instructions-for (:query-id result))
- :data-parts [(streaming/navigate-to-part results-url)]})
- (throw (ex-info (str "Invalid model_id " model-id)
- {:agent-error? true :status-code 400})))
- (catch Exception e
- (if (= (:status-code (ex-data e)) 404)
- {:output (ex-message e) :status-code 404}
- (metabot.tools.u/handle-agent-error e)))))
-
-(defn- resolve-datasource
- "Resolve datasource parameters to [field-id-prefix base-query] tuple.
- Accepts either {:table-id id} or {:model-id id}."
- [{:keys [table-id model-id]}]
- (cond
- model-id
- (try
- [(metabot.tools.u/card-field-id-prefix model-id) (metabot.tools.u/card-query model-id)]
- (catch clojure.lang.ExceptionInfo e
- (throw (if (= (:status-code (ex-data e)) 404)
- (ex-info (str "No model found with model_id " model-id)
- {:agent-error? true :status-code 404} e)
- e))))
-
- table-id
- (try
- [(metabot.tools.u/table-field-id-prefix table-id) (metabot.tools.u/table-query table-id)]
- (catch clojure.lang.ExceptionInfo e
- (throw (if (= (:status-code (ex-data e)) 404)
- (ex-info (str "No table found with table_id " table-id)
- {:agent-error? true :status-code 404} e)
- e))))
-
- :else
- (throw (ex-info "Either table-id or model-id must be provided" {:agent-error? true :status-code 400}))))
-
-(defn- query-datasource*
- [{:keys [fields filters aggregations group-by order-by limit] :as arguments}]
- (let [[filter-field-id-prefix base-query] (resolve-datasource arguments)
- visible-cols (lib/visible-columns base-query)
- resolve-visible-column #(metabot.tools.u/resolve-column % filter-field-id-prefix visible-cols)
- resolve-order-by-column (fn [{:keys [field direction]}] {:field (resolve-visible-column field) :direction direction})
- projection (map (comp (juxt filter-bucketed-column (fn [{:keys [column bucket]}]
- (let [column (cond-> column
- bucket (assoc :unit bucket))]
- (lib/display-name base-query -1 column :long))))
- resolve-visible-column)
- fields)
- all-aggregations (map (partial resolve-aggregation-column resolve-visible-column) aggregations)
- all-filters (map #(if (:segment-id %) % (resolve-visible-column %)) filters)
- reduce-query (fn [query f coll] (reduce f query coll))
- query (-> base-query
- (reduce-query (fn [query [expr-or-column expr-name]]
- (lib/expression query expr-name expr-or-column))
- (filter (comp expression? first) projection))
- (add-fields projection)
- (reduce-query add-filter all-filters)
- (reduce-query add-aggregation all-aggregations)
- (reduce-query add-breakout (map resolve-visible-column group-by))
- (reduce-query add-order-by (map resolve-order-by-column order-by))
- (add-limit limit))
- query-id (u/generate-nano-id)
- query-field-id-prefix (metabot.tools.u/query-field-id-prefix query-id)
- returned-cols (lib/returned-columns query)]
- {:type :query
- :query-id query-id
- :query query
- :result-columns (into []
- (map-indexed #(metabot.tools.u/->result-column query %2 %1 query-field-id-prefix))
- returned-cols)}))
-
-(defn query-datasource
- "Create a query based on a datasource (table or model)."
- [{:keys [table-id model-id] :as arguments}]
- (try
- (cond
- (and table-id model-id) (throw (ex-info "Cannot provide both table_id and model_id"
- {:agent-error? true :status-code 400}))
- (int? model-id) {:structured-output (-> (query-datasource* arguments)
- (assoc :result-type :query))}
- (int? table-id) {:structured-output (-> (query-datasource* arguments)
- (assoc :result-type :query))}
- model-id (throw (ex-info (str "Invalid model_id " model-id)
- {:agent-error? true :status-code 400}))
- table-id (throw (ex-info (str "Invalid table_id " table-id)
- {:agent-error? true :status-code 400}))
- :else (throw (ex-info "Either table_id or model_id must be provided"
- {:agent-error? true :status-code 400})))
- (catch Exception e
- (if (= (:status-code (ex-data e)) 404)
- {:output (ex-message e) :status-code 404}
- (metabot.tools.u/handle-agent-error e)))))
-
-(defn- base-query
- [data-source]
- (let [{:keys [table-id query query-id report-id]} data-source
- model-id (lib.util/legacy-string-table-id->card-id table-id)
- handle-query (fn [query query-id]
- (let [normalized-query (lib-be/normalize-query query)
- database-id (:database normalized-query)
- _ (api/read-check :model/Database database-id)
- mp (lib-be/application-database-metadata-provider database-id)]
- [(if query-id
- (metabot.tools.u/query-field-id-prefix query-id)
- metabot.tools.u/any-prefix-pattern)
- (-> (lib/query mp normalized-query) lib/append-stage)]))]
- (cond
- model-id
- (if-let [model-query (metabot.tools.u/card-query model-id)]
- [(metabot.tools.u/card-field-id-prefix model-id) model-query]
- (throw (ex-info (str "No model found with model_id " model-id)
- {:agent-error? true :status-code 404 :data-source data-source})))
-
- table-id
- (let [table-id (cond-> table-id
- (string? table-id) parse-long)]
- (if-let [table-query (metabot.tools.u/table-query table-id)]
- [(metabot.tools.u/table-field-id-prefix table-id) table-query]
- (throw (ex-info (str "No table found with table_id " table-id)
- {:agent-error? true :status-code 404 :data-source data-source}))))
-
- report-id
- (if-let [query (metabot.tools.u/card-query report-id)]
- [(metabot.tools.u/card-field-id-prefix report-id) query]
- (throw (ex-info (str "No report found with report_id " report-id)
- {:agent-error? true :status-code 404 :data-source data-source})))
-
- query
- (handle-query query query-id)
-
- :else
- (throw (ex-info "Invalid data_source"
- {:agent-error? true :status-code 400 :data-source data-source})))))
-
-(defn filter-records
- "Add `filters` to the query referenced by `data-source`"
- [{:keys [data-source filters] :as _arguments}]
- (try
- (let [[filter-field-id-prefix base] (base-query data-source)
- returned-cols (lib/returned-columns base)
- query (reduce add-filter base (map #(metabot.tools.u/resolve-column % filter-field-id-prefix returned-cols) filters))
- query-id (u/generate-nano-id)
- query-field-id-prefix (metabot.tools.u/query-field-id-prefix query-id)]
- {:structured-output
- {:result-type :query
- :type :query
- :query-id query-id
- :query query
- :result-columns (into []
- (map-indexed #(metabot.tools.u/->result-column query %2 %1 query-field-id-prefix))
- (lib/returned-columns query))}})
- (catch Exception ex
- (metabot.tools.u/handle-agent-error ex))))
diff --git a/src/metabase/metabot/tools/metadata.clj b/src/metabase/metabot/tools/metadata.clj
index 8ce39462913..f10c0d412c1 100644
--- a/src/metabase/metabot/tools/metadata.clj
+++ b/src/metabase/metabot/tools/metadata.clj
@@ -169,7 +169,7 @@
[:map {:closed true}
[:data_source [:enum "table" "model" "metric"]]
[:source_id :int]
- [:field_id :string]])
+ [:field_id [:or :int :string]]])
(mu/defn ^{:tool-name "get_field_values"
:scope scope/agent-metadata-read}
diff --git a/src/metabase/metabot/tools/slackbot_query.clj b/src/metabase/metabot/tools/slackbot_query.clj
index 317667cdb47..0f0e707ac6e 100644
--- a/src/metabase/metabot/tools/slackbot_query.clj
+++ b/src/metabase/metabot/tools/slackbot_query.clj
@@ -14,7 +14,9 @@
(def ^:private slackbot-query-schema
[:map {:closed true}
[:reasoning :string]
- [:query construct/construct-query-schema]
+ [:source_entity [:map [:type :string] [:id :int]]]
+ [:referenced_entities {:optional true} [:maybe [:sequential [:map [:type :string] [:id :int]]]]]
+ [:program construct/construct-program-schema]
[:title {:optional true} [:maybe :string]]
[:display {:optional true
:description "Visualization type for displaying the query results in Slack. Required in practice whenever the user asks for a chart or graph, and it must match any requested chart type. Valid values: 'table', 'bar', 'line', 'pie', 'area', 'row', 'scatter', 'funnel'. Use requested chart types like 'line', 'bar', 'area', 'pie', 'scatter', 'funnel', 'row', or 'table' when they fit the query. Omitting this field falls back to Metabase's default table display, so do not omit it for chart or graph requests. Only omit it when you intentionally want a plain table and the user did not request a chart type."}
@@ -24,12 +26,12 @@
:scope scope/agent-notebook-create}
slackbot-construct-notebook-query-tool
"Construct a notebook query from a metric, model, or table. The query results will be rendered as a visualization in Slack."
- [{:keys [_reasoning query title display]} :- slackbot-query-schema]
+ [{:keys [_reasoning source_entity referenced_entities program title display]} :- slackbot-query-schema]
(try
- (let [query-result (construct/execute-query query)
+ (let [query-result (construct/execute-program source_entity referenced_entities program)
structured (or (:structured-output query-result) (:structured_output query-result))]
(if (and structured (:query-id structured) (:query structured))
- (let [metabase-link (streaming/query->question-url (:query structured))
+ (let [metabase-link (streaming/query->question-url (:query structured) display)
adhoc-viz-value (cond-> {:query (:query structured)
:link metabase-link}
title (assoc :title title)
diff --git a/src/metabase/metabot/tools/util.clj b/src/metabase/metabot/tools/util.clj
index 2492b3a7e26..97b5fd159b0 100644
--- a/src/metabase/metabot/tools/util.clj
+++ b/src/metabase/metabot/tools/util.clj
@@ -1,6 +1,5 @@
(ns metabase.metabot.tools.util
(:require
- [clojure.string :as str]
[medley.core :as m]
[metabase.api.common :as api]
[metabase.audit-app.core :as audit-app]
@@ -8,9 +7,9 @@
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.types.isa :as lib.types.isa]
[metabase.premium-features.core :as premium-features]
[metabase.util :as u]
- [metabase.util.log :as log]
[toucan2.core :as t2]))
(defn handle-agent-error
@@ -23,6 +22,18 @@
status-code (assoc :status-code status-code))
(throw e))))
+(defn convert-field-type
+ "Return tool type for `column`."
+ [column]
+ (let [column (u/normalize-map column)]
+ (cond
+ (lib.types.isa/boolean? column) :boolean
+ (lib.types.isa/string-or-string-like? column) :string
+ (lib.types.isa/numeric? column) :number
+ (isa? (:effective-type column) :type/DateTime) :datetime
+ (isa? (:effective-type column) :type/Time) :time
+ (lib.types.isa/temporal? column) :date)))
+
(defn add-table-reference
"Add table-reference to columns that have FK relationships."
[query col]
@@ -33,38 +44,23 @@
(lib/display-name query)
lib/display-name-without-id))))
-(defn table-field-id-prefix
- "Return the field ID prefix for `table-id`."
- [table-id]
- (str "t" table-id "-"))
-
-(defn card-field-id-prefix
- "Return the field ID prefix for a model or a metric with ID `card-id`."
- [card-id]
- (str "c" card-id "-"))
-
-(defn query-field-id-prefix
- "Return the field ID prefix for `query-id`."
- [query-id]
- (str "q" query-id "-"))
-
-(def any-prefix-pattern
- "A prefix pattern accepting columns from any entity."
- #"^.*-(\d+)")
-
(defn ->result-column
- "Return tool result columns for `column` of `query`. The position of `column` is determined by `index`.
- The ID is then generated by prefixing the position with `field-id-prefix`."
- [query column index field-id-prefix]
+ "Return tool result column for `column` of `query`.
+ Uses the real field `:id` when available, falling back to column name for
+ expression/aggregation columns that don't have a database field ID."
+ [query column]
(let [base-type (some-> (:base-type column) u/qualified-name)
effective-type (some-> (:effective-type column) u/qualified-name)
semantic-type (some-> (:semantic-type column) u/qualified-name)
- coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)]
- (-> {:field_id (str field-id-prefix index)
- ;; Prefer desired-column-alias over source-column-alias to ensure we use deduplicated names if needed
+ coercion-strategy (some-> (:coercion-strategy column) u/qualified-name)
+ field-id (or (:id column)
+ (:lib/desired-column-alias column)
+ (:lib/source-column-alias column))]
+ (-> {:field_id field-id
:name (or (:lib/desired-column-alias column)
(:lib/source-column-alias column))
- :display_name (lib/display-name query column)}
+ :display_name (lib/display-name query column)
+ :type (convert-field-type column)}
(m/assoc-some :description (:description column)
:base_type base-type
:effective_type (when (not= effective-type base-type) effective-type)
@@ -74,77 +70,21 @@
:field_values (:field-values column)
:table_reference (:table-reference column)))))
-(defn parse-field-id
- "Parse a field-id string into its components.
-
- The field-id format is '<model-tag><model-id>-<field-index>' where:
- - model-tag is 't' for tables, 'c' for cards/models/metrics, or 'q' for ad-hoc queries
- - model-id is the numeric ID (for tables/cards) or nano-id (for queries)
- - field-index is the index within that model's visible columns
-
- Returns a map with :model-tag, :model-id, and :field-index keys, or nil if the format is invalid
- or the input is not a string.
-
- Examples:
- (parse-field-id \"t154-1\") => {:model-tag \"t\", :model-id 154, :field-index 1}
- (parse-field-id \"qpuL95JSvym3k23W1UUuog-0\") => {:model-tag \"q\", :model-id \"puL95JSvym3k23W1UUuog\", :field-index 0}
- (parse-field-id nil) => nil
- (parse-field-id \"invalid\") => nil"
- [field-id]
- (when (and field-id (string? field-id))
- (when-let [[_ model-tag model-id field-index] (re-matches #"^([tcq])(.+)-(\d+)$" field-id)]
- {:model-tag model-tag
- ;; For tables and cards, model-id should be numeric; for queries it's a nano-id string
- :model-id (if (= model-tag "q")
- model-id
- (parse-long model-id))
- :field-index (parse-long field-index)})))
-
-(defn resolve-column
- "Resolve the reference `field-id` in filter `item` by finding the column in `columns` specified by `field-id`.
-
- The field-id format is '<model-tag><model-id>-<field-index>' where:
- - model-tag is 't' for tables, 'c' for cards/models/metrics, or 'q' for ad-hoc queries
- - model-id is the numeric ID (for tables/cards) or nano-id (for queries)
- - field-index is the index within the columns array (using wide field IDs across all visible columns)
-
- The `expected-prefix` parameter validates that the field-id starts with the expected prefix (e.g., 't154-' for table 154).
- This prevents accidentally using a field-id from a different entity.
-
- For example, 't154-1' refers to the column at index 1 in the columns array,
- and 'qpuL95JSvym3k23W1UUuog-0' refers to the column at index 0."
- [{:keys [field-id] :as item} expected-prefix columns]
- (if-let [{:keys [model-tag model-id field-index]} (parse-field-id field-id)]
- (do
- ;; Validate that the field-id matches the expected prefix
- ;; Supports both string prefixes (e.g., "t154-") and regex patterns (e.g., #"^.*-(\d+)")
- (when-not (if (string? expected-prefix)
- (str/starts-with? field-id expected-prefix)
- (re-matches expected-prefix field-id))
- (log/warn "Field id prefix mismatch"
- {:field-id field-id
- :expected-prefix expected-prefix
- :model-tag model-tag
- :model-id model-id})
- (throw (ex-info (str "field " field-id " does not match expected prefix " expected-prefix)
- {:agent-error? true
- :status-code 400
- :field-id field-id
- :expected-prefix expected-prefix})))
- (if-let [column (get columns field-index)]
- (assoc item :column column)
- (throw (ex-info (str "field " field-id " not found - no column at index " field-index)
+(defn find-column-by-field-id
+ "Find a column in `columns` by its real field ID.
+ `field-id` may be an integer or a string-encoded integer.
+ Returns the matching column or throws an agent error if not found."
+ [field-id columns]
+ (let [numeric-id (cond
+ (int? field-id) field-id
+ (string? field-id) (parse-long field-id)
+ :else nil)]
+ (or (when numeric-id
+ (m/find-first #(= (:id %) numeric-id) columns))
+ (throw (ex-info (str "Field " field-id " not found")
{:agent-error? true
- :status-code 404
- :field-id field-id
- :model-tag model-tag
- :model-id model-id
- :field-index field-index
- :available-columns-count (count columns)}))))
- (throw (ex-info (str "Invalid field_id format: " field-id)
- {:agent-error? true
- :status-code 400
- :field-id field-id}))))
+ :status-code 404
+ :field-id field-id})))))
(defn schedule->schedule-map
"Convert a tool schedule map to the schedule-map format used by cron and pulse channels.
diff --git a/test/metabase/agent_api/api_test.clj b/test/metabase/agent_api/api_test.clj
index d44300594c1..6c4a6d95adf 100644
--- a/test/metabase/agent_api/api_test.clj
+++ b/test/metabase/agent_api/api_test.clj
@@ -10,7 +10,6 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.normalize :as lib.normalize]
- [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.search.ingestion :as search.ingestion]
[metabase.search.test-util :as search.tu]
[metabase.session.models.session :as session.models]
@@ -180,17 +179,15 @@
(field-values/get-or-create-full-field-values! (t2/select-one :model/Field :id field-id)))
(defn- visible-field-id
- "Find the field-id string for a field by display name within a table's visible columns."
+ "Find the real field ID for a field by display name within a table's visible columns."
[table-id field-display-name]
- (let [mp (mt/metadata-provider)
- query (lib/query mp (lib.metadata/table mp table-id))
- field-prefix (metabot.tools.u/table-field-id-prefix table-id)
- visible-cols (lib/visible-columns query)]
- (->> (keep-indexed (fn [i col]
- (when (= (lib/display-name query col) field-display-name)
- (str field-prefix i)))
- visible-cols)
- first)))
+ (let [mp (mt/metadata-provider)
+ query (lib/query mp (lib.metadata/table mp table-id))
+ visible-cols (lib/visible-columns query)]
+ (->> visible-cols
+ (filter #(= (lib/display-name query %) field-display-name))
+ first
+ :id)))
(deftest get-table-field-values-test
;; Ensure field values exist for the field we'll test
@@ -216,12 +213,12 @@
(testing "Returns 404 for non-existent table"
(is (= "Not found."
- (mt/user-http-request :crowberto :get 404 "agent/v1/table/999999/field/t999999-0/values"))))
+ (mt/user-http-request :crowberto :get 404 "agent/v1/table/999999/field/999999/values"))))
- (testing "Returns 400 for invalid field-id format"
+ (testing "Returns 404 for non-existent field"
(let [table-id (mt/id :people)]
- (is (= "Invalid field_id format: not-a-valid-id"
- (mt/user-http-request :crowberto :get 400 (format "agent/v1/table/%d/field/not-a-valid-id/values" table-id)))))))
+ (is (= "Field 999999 not found"
+ (mt/user-http-request :crowberto :get 404 (format "agent/v1/table/%d/field/999999/values" table-id)))))))
(deftest search-test
(binding [search.ingestion/*force-sync* true]
@@ -241,40 +238,35 @@
(deftest construct-query-test
(testing "Constructs a simple query from a table"
(let [table-id (mt/id :orders)
- response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id})]
+ response (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "table" :id table-id}
+ :operations []})]
(is (string? (:query response)) "Response should contain a query string")
(let [decoded (decode-query response)]
(is (= :mbql/query (lib/normalized-query-type decoded)))
(is (= (mt/id) (lib/database-id decoded)))
(is (= (mt/id :orders) (lib/primary-source-table-id decoded))))))
- (testing "Applies default limit of 200 when no limit is specified"
+ (testing "Respects explicit limit operation"
(let [table-id (mt/id :orders)
- response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id})
- decoded (decode-query response)]
- (is (= 200 (lib/current-limit decoded)))))
-
- (testing "Respects explicit limit"
- (let [table-id (mt/id :orders)
- response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id
- :limit 10})
+ response (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "table" :id table-id}
+ :operations [["limit" 10]]})
decoded (decode-query response)]
(is (= 10 (lib/current-limit decoded)))))
(testing "Returns 404 for non-existent table"
- (is (= "No table found with table_id 999999"
- (mt/user-http-request :rasta :post 404 "agent/v1/construct-query"
- {:table_id 999999})))))
+ (is (= "Not found."
+ (mt/user-http-request :rasta :post 404 "agent/v2/construct-query"
+ {:source {:type "table" :id 999999}
+ :operations []})))))
(deftest execute-query-test
(testing "Executes a query and returns results with column metadata"
(let [table-id (mt/id :orders)
- construct-resp (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id
- :limit 5})
+ construct-resp (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "table" :id table-id}
+ :operations [["limit" 5]]})
;; Streaming response returns 202 (accepted) since it starts streaming before completion
execute-resp (mt/user-http-request :rasta :post 202 "agent/v1/execute"
{:query (:query construct-resp)})]
@@ -289,9 +281,9 @@
(testing "Enforces agent query row limit even when query specifies a higher limit"
(let [table-id (mt/id :orders)
- construct-resp (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id
- :limit 300})
+ construct-resp (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "table" :id table-id}
+ :operations [["limit" 300]]})
execute-resp (mt/user-http-request :rasta :post 202 "agent/v1/execute"
{:query (:query construct-resp)})]
(is (=? {:status "completed" :row_count 200}
@@ -315,12 +307,11 @@
(testing "Returns 404 for non-existent metric"
(is (= "Not found."
- (mt/user-http-request :rasta :get 404 "agent/v1/metric/999999/field/c999999-0/values"))))
+ (mt/user-http-request :rasta :get 404 "agent/v1/metric/999999/field/999999/values"))))
- (testing "Returns 400 for field-id from wrong entity type"
- ;; Using a table field-id (t-prefix) when querying a metric should fail
- (is (re-find #"does not match expected prefix"
- (mt/user-http-request :rasta :get 400 (format "agent/v1/metric/%d/field/t123-0/values" (:id metric))))))))
+ (testing "Returns 404 for non-existent field on metric"
+ (is (= "Field 999999 not found"
+ (mt/user-http-request :rasta :get 404 (format "agent/v1/metric/%d/field/999999/values" (:id metric))))))))
(deftest construct-metric-query-test
(mt/with-temp [:model/Card metric {:name "Test Metric"
@@ -328,8 +319,9 @@
:database_id (mt/id)
:dataset_query (orders-count-query)}]
(testing "Constructs a query from a metric"
- (let [response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:metric_id (:id metric)})]
+ (let [response (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "metric" :id (:id metric)}
+ :operations []})]
(is (string? (:query response)) "Response should contain a query string")
(let [decoded (decode-query response)]
(is (= :mbql/query (lib/normalized-query-type decoded)))
@@ -337,28 +329,17 @@
(testing "Returns 404 for non-existent metric"
(is (= "Not found."
- (mt/user-http-request :rasta :post 404 "agent/v1/construct-query"
- {:metric_id 999999}))))))
+ (mt/user-http-request :rasta :post 404 "agent/v2/construct-query"
+ {:source {:type "metric" :id 999999}
+ :operations []}))))))
(deftest construct-query-with-count-aggregation-test
- (testing "Count aggregation without field_id produces a valid query"
- (let [table-id (mt/id :orders)
- response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id
- :aggregations [{:function "count"}]
- :limit 10})]
- (is (string? (:query response)))
- (let [decoded (decode-query response)]
- (is (= 1 (count (lib/aggregations decoded)))))))
-
- (testing "Count aggregation with field_id still works"
+ (testing "Count aggregation produces a valid query"
(let [table-id (mt/id :orders)
- table (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))
- field-id (-> table :fields first :field_id)
- response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id
- :aggregations [{:function "count" :field_id field-id}]
- :limit 10})]
+ response (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "table" :id table-id}
+ :operations [["aggregate" ["count"]]
+ ["limit" 10]]})]
(is (string? (:query response)))
(let [decoded (decode-query response)]
(is (= 1 (count (lib/aggregations decoded))))))))
@@ -366,14 +347,11 @@
(deftest construct-query-with-filters-test
(testing "Constructs a query with filters"
(let [table-id (mt/id :orders)
- ;; Get table details to find a valid field_id
- table (mt/user-http-request :rasta :get 200 (str "agent/v1/table/" table-id))
- field-id (-> table :fields first :field_id)
- response (mt/user-http-request :rasta :post 200 "agent/v1/construct-query"
- {:table_id table-id
- :filters [{:field_id field-id
- :operation "is-not-null"}]
- :limit 10})]
+ field-id (mt/id :orders :id)
+ response (mt/user-http-request :rasta :post 200 "agent/v2/construct-query"
+ {:source {:type "table" :id table-id}
+ :operations [["filter" ["not-null" ["field" field-id]]]
+ ["limit" 10]]})]
(is (string? (:query response)))
(let [decoded (decode-query response)]
(is (seq (lib/filters decoded)) "Query should have filters")))))
@@ -401,10 +379,10 @@
(testing "Returns results for a table query"
(let [table-id (mt/id :orders)
field-id (visible-field-id table-id "ID")
- response (mt/user-http-request :rasta :post 202 "agent/v1/query"
- {:table_id table-id
- :order_by [{:field {:field_id field-id} :direction "asc"}]
- :limit 5})]
+ response (mt/user-http-request :rasta :post 202 "agent/v2/query"
+ {:source {:type "table" :id table-id}
+ :operations [["order-by" ["field" field-id]]
+ ["limit" 5]]})]
(is (=? {:status "completed"
:row_count 5
:continuation_token string?
@@ -415,11 +393,11 @@
(testing "Continuation token returns next page of results"
(let [table-id (mt/id :orders)
field-id (visible-field-id table-id "ID")
- page1 (mt/user-http-request :rasta :post 202 "agent/v1/query"
- {:table_id table-id
- :order_by [{:field {:field_id field-id} :direction "asc"}]
- :limit 5})
- page2 (mt/user-http-request :rasta :post 202 "agent/v1/query"
+ page1 (mt/user-http-request :rasta :post 202 "agent/v2/query"
+ {:source {:type "table" :id table-id}
+ :operations [["order-by" ["field" field-id]]
+ ["limit" 5]]})
+ page2 (mt/user-http-request :rasta :post 202 "agent/v2/query"
{:continuation_token (:continuation_token page1)})]
(is (=? {:row_count 5
:continuation_token string?
@@ -436,16 +414,16 @@
(testing "No continuation_token when all rows are returned"
(is (=? {:status "completed"
:continuation_token nil?}
- (mt/user-http-request :rasta :post 202 "agent/v1/query"
- {:table_id (mt/id :orders)
- :aggregations [{:function "count"}]}))))
+ (mt/user-http-request :rasta :post 202 "agent/v2/query"
+ {:source {:type "table" :id (mt/id :orders)}
+ :operations [["aggregate" ["count"]]]}))))
(testing "Constraint cap limits results to 200 rows"
(is (=? {:status "completed"
:row_count (fn [n] (<= n 200))}
- (mt/user-http-request :rasta :post 202 "agent/v1/query"
- {:table_id (mt/id :orders)
- :limit 1000})))))
+ (mt/user-http-request :rasta :post 202 "agent/v2/query"
+ {:source {:type "table" :id (mt/id :orders)}
+ :operations [["limit" 1000]]})))))
(deftest combined-query-metric-test
(mt/with-temp [:model/Card metric {:name "Test Metric"
@@ -455,8 +433,9 @@
(testing "Returns results for a metric query"
(is (=? {:status "completed"
:row_count pos?}
- (mt/user-http-request :rasta :post 202 "agent/v1/query"
- {:metric_id (:id metric)}))))))
+ (mt/user-http-request :rasta :post 202 "agent/v2/query"
+ {:source {:type "metric" :id (:id metric)}
+ :operations []}))))))
(deftest search-finds-metrics-test
(binding [search.ingestion/*force-sync* true]
diff --git a/test/metabase/agent_lib/capabilities/catalog_test.clj b/test/metabase/agent_lib/capabilities/catalog_test.clj
new file mode 100644
index 00000000000..7e1391b7969
--- /dev/null
+++ b/test/metabase/agent_lib/capabilities/catalog_test.clj
@@ -0,0 +1,43 @@
+(ns metabase.agent-lib.capabilities.catalog-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.capabilities :as capabilities]
+ [metabase.agent-lib.capabilities.catalog :as catalog]
+ [metabase.agent-lib.capabilities.catalog.aggregation :as aggregation]
+ [metabase.agent-lib.capabilities.catalog.expressions :as expressions]
+ [metabase.agent-lib.capabilities.catalog.filtering :as filtering]
+ [metabase.agent-lib.capabilities.catalog.joins :as joins]
+ [metabase.agent-lib.capabilities.catalog.ordering :as ordering]
+ [metabase.agent-lib.capabilities.catalog.sources :as sources]
+ [metabase.agent-lib.capabilities.catalog.top-level :as top-level]))
+
+(deftest ^:parallel grouped-capability-catalog-preserves-review-groups-and-order-test
+ (is (= [:top-level
+ :sources
+ :filtering
+ :aggregation
+ :breakout-ordering
+ :expressions
+ :joins]
+ (mapv :group catalog/grouped-capability-catalog)))
+ (is (= [top-level/capabilities
+ sources/capabilities
+ filtering/capabilities
+ aggregation/capabilities
+ ordering/capabilities
+ expressions/capabilities
+ joins/capabilities]
+ (mapv :capabilities catalog/grouped-capability-catalog))))
+
+(deftest ^:parallel raw-capability-catalog-flattens-the-grouped-catalog-test
+ (let [expected (into []
+ cat
+ [top-level/capabilities
+ sources/capabilities
+ filtering/capabilities
+ aggregation/capabilities
+ ordering/capabilities
+ expressions/capabilities
+ joins/capabilities])]
+ (is (= expected catalog/raw-capability-catalog))
+ (is (= expected capabilities/raw-capability-catalog))))
diff --git a/test/metabase/agent_lib/common/coercions_test.clj b/test/metabase/agent_lib/common/coercions_test.clj
new file mode 100644
index 00000000000..2552c24a968
--- /dev/null
+++ b/test/metabase/agent_lib/common/coercions_test.clj
@@ -0,0 +1,113 @@
+(ns metabase.agent-lib.common.coercions-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.common.coercions :as coercions]))
+
+(deftest ^:parallel quarter-label->number-test
+ (testing "converts Q labels to integers"
+ (is (= 1 (coercions/quarter-label->number "Q1")))
+ (is (= 2 (coercions/quarter-label->number "q2")))
+ (is (= 3 (coercions/quarter-label->number " Q3 ")))
+ (is (= 4 (coercions/quarter-label->number "Q4"))))
+ (testing "returns nil for invalid inputs"
+ (is (nil? (coercions/quarter-label->number "Q5")))
+ (is (nil? (coercions/quarter-label->number "Q0")))
+ (is (nil? (coercions/quarter-label->number "abc")))
+ (is (nil? (coercions/quarter-label->number nil)))))
+
+(deftest ^:parallel parse-int-string-test
+ (testing "parses valid integer strings"
+ (is (= 42 (coercions/parse-int-string "42")))
+ (is (= -7 (coercions/parse-int-string "-7")))
+ (is (= 0 (coercions/parse-int-string "0"))))
+ (testing "returns nil for invalid inputs"
+ (is (nil? (coercions/parse-int-string "abc")))
+ (is (nil? (coercions/parse-int-string "3.14")))
+ (is (nil? (coercions/parse-int-string "")))
+ (is (nil? (coercions/parse-int-string nil)))))
+
+(deftest ^:parallel coerce-positive-int-test
+ (testing "passes through positive integers"
+ (is (= 5 (coercions/coerce-positive-int 5))))
+ (testing "coerces positive integer strings"
+ (is (= 5 (coercions/coerce-positive-int "5")))
+ (is (= 100 (coercions/coerce-positive-int "100"))))
+ (testing "does not coerce non-positive strings"
+ (is (= "0" (coercions/coerce-positive-int "0")))
+ (is (= "-1" (coercions/coerce-positive-int "-1"))))
+ (testing "passes through non-string values"
+ (is (= :kw (coercions/coerce-positive-int :kw)))
+ (is (= -1 (coercions/coerce-positive-int -1)))))
+
+(deftest ^:parallel coerce-non-negative-int-test
+ (testing "passes through non-negative integers"
+ (is (= 0 (coercions/coerce-non-negative-int 0)))
+ (is (= 5 (coercions/coerce-non-negative-int 5))))
+ (testing "coerces non-negative integer strings"
+ (is (= 0 (coercions/coerce-non-negative-int "0")))
+ (is (= 42 (coercions/coerce-non-negative-int "42"))))
+ (testing "does not coerce negative strings"
+ (is (= "-1" (coercions/coerce-non-negative-int "-1"))))
+ (testing "passes through non-string values"
+ (is (= :kw (coercions/coerce-non-negative-int :kw)))))
+
+(deftest ^:parallel direction-string-test
+ (testing "recognizes direction strings"
+ (is (true? (coercions/direction-string? "asc")))
+ (is (true? (coercions/direction-string? "desc")))
+ (is (true? (coercions/direction-string? "ASC")))
+ (is (true? (coercions/direction-string? :desc))))
+ (testing "rejects non-direction values"
+ (is (false? (coercions/direction-string? "up")))
+ (is (false? (coercions/direction-string? nil)))))
+
+(deftest ^:parallel normalize-direction-test
+ (testing "normalizes to lowercase string"
+ (is (= "asc" (coercions/normalize-direction "ASC")))
+ (is (= "desc" (coercions/normalize-direction :desc)))
+ (is (= "asc" (coercions/normalize-direction 'asc))))
+ (testing "returns nil for nil"
+ (is (nil? (coercions/normalize-direction nil)))))
+
+(deftest ^:parallel unwrap-singleton-form-test
+ (testing "unwraps nested singletons but stops at scalar-sequential"
+ (is (= [5] (coercions/unwrap-singleton-form [[[[5]]]]))))
+ (testing "stops at operator tuples"
+ (is (= ["field" 1] (coercions/unwrap-singleton-form [[["field" 1]]]))))
+ (testing "passes through non-singleton vectors"
+ (is (= [1 2] (coercions/unwrap-singleton-form [1 2]))))
+ (testing "passes through scalars"
+ (is (= "hello" (coercions/unwrap-singleton-form "hello")))
+ (is (= 42 (coercions/unwrap-singleton-form 42)))))
+
+(deftest ^:parallel normalize-percentile-value-test
+ (testing "converts whole-number percentages to fractions"
+ (is (= 0.9 (coercions/normalize-percentile-value 90)))
+ (is (= 0.5 (coercions/normalize-percentile-value 50)))
+ (is (= 1.0 (coercions/normalize-percentile-value 100))))
+ (testing "passes through values already in fraction form"
+ (is (= 0.9 (coercions/normalize-percentile-value 0.9)))
+ (is (= 0.5 (coercions/normalize-percentile-value 0.5))))
+ (testing "boundary: 1.0 is already a fraction"
+ (is (= 1.0 (coercions/normalize-percentile-value 1.0))))
+ (testing "passes through non-numbers"
+ (is (= "abc" (coercions/normalize-percentile-value "abc")))))
+
+(deftest ^:parallel normalize-join-conditions-test
+ (testing "unwraps extra singleton nesting"
+ (is (= [["=" ["field" 1] ["field" 2]]]
+ (coercions/normalize-join-conditions [[["=" ["field" 1] ["field" 2]]]]))))
+ (testing "leaves properly-structured conditions alone"
+ (is (= [["=" ["field" 1] ["field" 2]]]
+ (coercions/normalize-join-conditions [["=" ["field" 1] ["field" 2]]]))))
+ (testing "handles deeply nested singleton wrapping"
+ (is (= [["=" ["field" 1] ["field" 2]]]
+ (coercions/normalize-join-conditions [[[["=" ["field" 1] ["field" 2]]]]])))))
+
+(deftest ^:parallel normalize-map-key-test
+ (testing "converts string keys to keywords"
+ (is (= :foo (coercions/normalize-map-key "foo"))))
+ (testing "passes through keyword keys"
+ (is (= :bar (coercions/normalize-map-key :bar))))
+ (testing "passes through non-string keys"
+ (is (= 42 (coercions/normalize-map-key 42)))))
diff --git a/test/metabase/agent_lib/common/context_test.clj b/test/metabase/agent_lib/common/context_test.clj
new file mode 100644
index 00000000000..c6e61bea941
--- /dev/null
+++ b/test/metabase/agent_lib/common/context_test.clj
@@ -0,0 +1,25 @@
+(ns metabase.agent-lib.common.context-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.common.context :as context]))
+
+(deftest ^:parallel source-metric-id-test
+ (testing "returns id for metric source entities"
+ (is (= 42 (context/source-metric-id {:source-entity {:model "metric" :id 42}}))))
+ (testing "returns nil for table source entities"
+ (is (nil? (context/source-metric-id {:source-entity {:model "table" :id 1}}))))
+ (testing "returns nil for non-positive ids"
+ (is (nil? (context/source-metric-id {:source-entity {:model "metric" :id 0}})))
+ (is (nil? (context/source-metric-id {:source-entity {:model "metric" :id -1}}))))
+ (testing "returns nil for missing source-entity"
+ (is (nil? (context/source-metric-id {})))))
+
+(deftest ^:parallel source-table-id-test
+ (testing "returns id for table source entities"
+ (is (= 1 (context/source-table-id {:source-entity {:model "table" :id 1}}))))
+ (testing "returns nil for metric source entities"
+ (is (nil? (context/source-table-id {:source-entity {:model "metric" :id 42}}))))
+ (testing "returns nil for non-positive ids"
+ (is (nil? (context/source-table-id {:source-entity {:model "table" :id 0}}))))
+ (testing "returns nil for missing source-entity"
+ (is (nil? (context/source-table-id {})))))
diff --git a/test/metabase/agent_lib/common/literals_test.clj b/test/metabase/agent_lib/common/literals_test.clj
new file mode 100644
index 00000000000..eaf67789243
--- /dev/null
+++ b/test/metabase/agent_lib/common/literals_test.clj
@@ -0,0 +1,133 @@
+(ns metabase.agent-lib.common.literals-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.common.literals :as literals]))
+
+(deftest ^:parallel scalar-literal-test
+ (testing "nil is a scalar literal"
+ (is (true? (literals/scalar-literal? nil))))
+ (testing "strings are scalar literals"
+ (is (true? (literals/scalar-literal? "hello")))
+ (is (true? (literals/scalar-literal? ""))))
+ (testing "numbers are scalar literals"
+ (is (true? (literals/scalar-literal? 42)))
+ (is (true? (literals/scalar-literal? 3.14)))
+ (is (true? (literals/scalar-literal? 0))))
+ (testing "booleans are scalar literals"
+ (is (true? (literals/scalar-literal? true)))
+ (is (true? (literals/scalar-literal? false))))
+ (testing "maps and vectors are not scalar literals"
+ (is (false? (literals/scalar-literal? {})))
+ (is (false? (literals/scalar-literal? [])))
+ (is (false? (literals/scalar-literal? :keyword)))))
+
+(deftest ^:parallel non-blank-string-test
+ (testing "non-blank strings"
+ (is (true? (literals/non-blank-string? "hello")))
+ (is (true? (literals/non-blank-string? " x "))))
+ (testing "blank strings"
+ (is (false? (literals/non-blank-string? "")))
+ (is (false? (literals/non-blank-string? " "))))
+ (testing "non-strings"
+ (is (false? (literals/non-blank-string? nil)))
+ (is (false? (literals/non-blank-string? 42)))))
+
+(deftest ^:parallel quarter-label-string-test
+ (testing "valid quarter labels"
+ (is (some? (literals/quarter-label-string? "Q1")))
+ (is (some? (literals/quarter-label-string? "Q4")))
+ (is (some? (literals/quarter-label-string? "q2")))
+ (is (some? (literals/quarter-label-string? " Q3 "))))
+ (testing "invalid quarter labels"
+ (is (nil? (literals/quarter-label-string? "Q5")))
+ (is (nil? (literals/quarter-label-string? "Q0")))
+ (is (nil? (literals/quarter-label-string? "Q")))
+ (is (nil? (literals/quarter-label-string? "")))
+ (is (not (literals/quarter-label-string? nil)))))
+
+(deftest ^:parallel now-literal-test
+ (testing "case-insensitive now"
+ (is (true? (literals/now-literal? "now")))
+ (is (true? (literals/now-literal? "NOW")))
+ (is (true? (literals/now-literal? " now "))))
+ (testing "not-now values"
+ (is (false? (literals/now-literal? "not-now")))
+ (is (false? (literals/now-literal? nil)))
+ (is (false? (literals/now-literal? 42)))))
+
+(deftest ^:parallel iso-date-string-test
+ (testing "valid ISO dates"
+ (is (some? (literals/iso-date-string? "2024-01-15")))
+ (is (some? (literals/iso-date-string? " 2024-12-31 "))))
+ (testing "invalid date formats"
+ (is (nil? (literals/iso-date-string? "2024-1-15")))
+ (is (nil? (literals/iso-date-string? "not-a-date")))
+ (is (not (literals/iso-date-string? nil)))))
+
+(deftest ^:parallel null-literal-test
+ (testing "nil is null"
+ (is (true? (literals/null-literal? nil))))
+ (testing "string null variants"
+ (is (true? (literals/null-literal? "null")))
+ (is (true? (literals/null-literal? "NULL"))))
+ (testing "non-null values"
+ (is (false? (literals/null-literal? "not null")))
+ (is (false? (literals/null-literal? 0)))
+ (is (false? (literals/null-literal? false)))))
+
+(deftest ^:parallel scalar-sequential-test
+ (testing "vector of scalars"
+ (is (true? (literals/scalar-sequential? [1 2 3])))
+ (is (true? (literals/scalar-sequential? ["a" "b"])))
+ (is (true? (literals/scalar-sequential? [nil true 42 "x"]))))
+ (testing "empty vector is scalar-sequential"
+ (is (true? (literals/scalar-sequential? []))))
+ (testing "rejects operator tuples"
+ (is (false? (literals/scalar-sequential? ["count"])))
+ (is (false? (literals/scalar-sequential? ["field" 1]))))
+ (testing "rejects non-sequential"
+ (is (false? (literals/scalar-sequential? "abc")))
+ (is (false? (literals/scalar-sequential? 42)))))
+
+(deftest ^:parallel aggregation-form-test
+ (testing "recognizes canonical aggregation operators"
+ (is (some? (literals/aggregation-form? ["count"])))
+ (is (some? (literals/aggregation-form? ["sum" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["avg" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["min" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["max" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["distinct" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["median" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["stddev" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["var" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["count-where" ["field" 1]])))
+ (is (some? (literals/aggregation-form? ["percentile" ["field" 1] 0.9])))
+ (is (some? (literals/aggregation-form? ["metric" 42]))))
+ (testing "rejects non-aggregation operators"
+ (is (nil? (literals/aggregation-form? ["field" 1])))
+ (is (nil? (literals/aggregation-form? ["=" 1 2]))))
+ (testing "rejects non-vectors"
+ (is (not (literals/aggregation-form? "count")))))
+
+(deftest ^:parallel default-map-test
+ (testing "maps with :default key"
+ (is (true? (literals/default-map? {:default nil})))
+ (is (true? (literals/default-map? {"default" 42}))))
+ (testing "maps without default key"
+ (is (false? (literals/default-map? {:other 1})))
+ (is (false? (literals/default-map? {}))))
+ (testing "non-maps"
+ (is (false? (literals/default-map? nil)))
+ (is (false? (literals/default-map? [])))))
+
+(deftest ^:parallel bare-field-id-test
+ (testing "positive integers"
+ (is (true? (literals/bare-field-id? 1)))
+ (is (true? (literals/bare-field-id? 42))))
+ (testing "zero and negatives"
+ (is (false? (literals/bare-field-id? 0)))
+ (is (false? (literals/bare-field-id? -1))))
+ (testing "non-integers"
+ (is (false? (literals/bare-field-id? 3.14)))
+ (is (false? (literals/bare-field-id? "42")))
+ (is (false? (literals/bare-field-id? nil)))))
diff --git a/test/metabase/agent_lib/eval/args_test.clj b/test/metabase/agent_lib/eval/args_test.clj
new file mode 100644
index 00000000000..db33242b318
--- /dev/null
+++ b/test/metabase/agent_lib/eval/args_test.clj
@@ -0,0 +1,30 @@
+(ns metabase.agent-lib.eval.args-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.eval.args :as eval.args]))
+
+(deftest ^:parallel normalize-helper-args-flattens-in-lists-test
+ (is (= [["field" 10] "a" "b"]
+ (eval.args/normalize-helper-args [:operations 0 1]
+ 'in
+ [["field" 10] ["a" "b"]]))))
+
+(deftest ^:parallel normalize-helper-args-normalizes-enums-test
+ (is (= [["aggregation-ref" 0] :desc]
+ (eval.args/normalize-helper-args [:operations 1]
+ 'order-by
+ [["aggregation-ref" 0] "desc"])))
+ (is (= [["field" 10] -30 :day 0 :day]
+ (eval.args/normalize-helper-args [:operations 2]
+ 'relative-time-interval
+ [["field" 10] -30 "day" 0 "day"]))))
+
+(deftest ^:parallel normalize-helper-args-normalizes-percentiles-and-join-fields-test
+ (is (= [["field" 12] 0.9]
+ (eval.args/normalize-helper-args [:operations 3]
+ 'percentile
+ [["field" 12] 90])))
+ (is (= [["join-clause" ["table" 1]] :all]
+ (eval.args/normalize-helper-args [:operations 4]
+ 'with-join-fields
+ [["join-clause" ["table" 1]] "all"]))))
diff --git a/test/metabase/agent_lib/eval/invoke_test.clj b/test/metabase/agent_lib/eval/invoke_test.clj
new file mode 100644
index 00000000000..ad881918fd1
--- /dev/null
+++ b/test/metabase/agent_lib/eval/invoke_test.clj
@@ -0,0 +1,54 @@
+(ns metabase.agent-lib.eval.invoke-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.eval.invoke :as eval.invoke]
+ [metabase.agent-lib.test-util :as agent-lib.tu]))
+
+(defn- runtime-with-bindings
+ [bindings]
+ {:bindings bindings})
+
+(deftest invoke-helper-rejects-unknown-operators-test
+ (let [runtime (runtime-with-bindings {'known identity})]
+ (try
+ (eval.invoke/invoke-helper! runtime [:operations 0] 'missing [])
+ (is false "expected unknown helper to fail")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (= "missing" (:operator (ex-data e))))))))
+
+(deftest invoke-query-aware-helper-requires-current-query-test
+ (let [runtime (runtime-with-bindings {'filter (fn [& _] :ok)})]
+ (try
+ (eval.invoke/invoke-query-aware-helper! runtime nil [:operations 0] 'filter [])
+ (is false "expected query-aware helper to require current query")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"`filter` requires the current query state"
+ (:details (ex-data e))))))))
+
+(deftest invoke-query-aware-helper-shapes-expression-ref-errors-test
+ (let [runtime (runtime-with-bindings {'expression-ref (fn [& _]
+ (throw (ex-info "missing expression" {})))})]
+ (try
+ (eval.invoke/invoke-query-aware-helper! runtime {:query true} [:operations 0] 'expression-ref ["Net"])
+ (is false "expected expression-ref failure")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (= "expression-ref" (:operator (ex-data e))))
+ (is (= "Net" (:name (ex-data e))))))))
+
+(deftest invoke-field-helper-prefers-query-aware-arity-test
+ (let [runtime (runtime-with-bindings {'field (fn [query arg] [query arg])})]
+ (is (= [:current-query 42]
+ (eval.invoke/invoke-field-helper! runtime :current-query [:operations 0] [42])))))
+
+(deftest ensure-query-result-recognizes-pmbql-queries-test
+ (testing "non-query values are rejected"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"must return a pMBQL query map"
+ (eval.invoke/ensure-query-result! :not-a-query))))
+ (testing "queries pass through unchanged"
+ (let [query (agent-lib.tu/query-for-table :orders)]
+ (is (= query (eval.invoke/ensure-query-result! query))))))
diff --git a/test/metabase/agent_lib/eval/source_test.clj b/test/metabase/agent_lib/eval/source_test.clj
new file mode 100644
index 00000000000..66c46784a69
--- /dev/null
+++ b/test/metabase/agent_lib/eval/source_test.clj
@@ -0,0 +1,59 @@
+(ns metabase.agent-lib.eval.source-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.eval.source :as source]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.test-util :as tu]
+ [metabase.lib.test-metadata :as meta]))
+
+(deftest ^:parallel resolve-source-context-test
+ (testing "context source returns the source from runtime bindings"
+ (let [base-query (tu/query-for-table :orders)
+ rt (runtime/build-runtime meta/metadata-provider {:extra-bindings {'source base-query}})
+ result (source/resolve-source identity rt [:source] {:type "context" :ref "source"})]
+ (is (= base-query result)))))
+
+(deftest resolve-source-context-missing-test
+ (testing "context source throws when unavailable in runtime bindings"
+ (let [rt (runtime/build-runtime meta/metadata-provider)
+ error (try
+ (source/resolve-source identity rt [:source] {:type "context" :ref "source"})
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (re-find #"context source is unavailable"
+ (:details (ex-data error)))))))
+
+(deftest ^:parallel resolve-source-table-test
+ (testing "table source calls table lookup with id and returns a query"
+ (let [rt (runtime/build-runtime meta/metadata-provider)
+ result (source/resolve-source identity rt [:source] {:type "table" :id (meta/id :orders)})]
+ (is (map? result)))))
+
+(deftest resolve-source-unsupported-type-test
+ (testing "unsupported source type throws"
+ (let [rt (runtime/build-runtime meta/metadata-provider)
+ error (try
+ (source/resolve-source identity rt [:source] {:type "unknown"})
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (re-find #"unsupported source type"
+ (:details (ex-data error)))))))
+
+(deftest ^:parallel resolve-source-program-test
+ (testing "program source recurses via evaluate-program callback"
+ (let [rt (runtime/build-runtime meta/metadata-provider)
+ inner-result {:result :from-inner-program}
+ evaluate-fn (fn [_path program]
+ (when (= {:source {:type "context" :ref "source"}
+ :operations [["limit" 5]]}
+ program)
+ inner-result))
+ result (source/resolve-source evaluate-fn rt [:source]
+ {:type "program"
+ :program {:source {:type "context" :ref "source"}
+ :operations [["limit" 5]]}})]
+ (is (= inner-result result)))))
diff --git a/test/metabase/agent_lib/eval_test.clj b/test/metabase/agent_lib/eval_test.clj
new file mode 100644
index 00000000000..fb8624dd48f
--- /dev/null
+++ b/test/metabase/agent_lib/eval_test.clj
@@ -0,0 +1,679 @@
+(ns metabase.agent-lib.eval-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.core :as program]
+ [metabase.agent-lib.test-util :as tu]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.test-util :as lib.tu]
+ [metabase.lib.util :as lib.util]))
+
+(deftest ^:parallel expression-ref-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Net Amount"
+ ["-" ["field" (meta/id :orders :total)]
+ ["field" (meta/id :orders :discount)]]]
+ ["order-by" ["desc" ["expression-ref" "Net Amount"]]]
+ ["limit" 10]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/expression "Net Amount"
+ (lib/- (meta/field-metadata :orders :total)
+ (meta/field-metadata :orders :discount)))
+ (as-> query
+ (lib/order-by query (lib/expression-ref query "Net Amount") :desc))
+ (lib/limit 10))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel expression-ref-breakout-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Hour of Day" ["get-hour" ["field" (meta/id :orders :created-at)]]]
+ ["aggregate" ["count"]]
+ ["breakout" ["expression-ref" "Hour of Day"]]
+ ["order-by" ["expression-ref" "Hour of Day"]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/expression "Hour of Day" (lib/get-hour (meta/field-metadata :orders :created-at)))
+ (lib/aggregate (lib/count)))
+ expected (as-> base-query query
+ (let [expr-ref (lib/expression-ref query "Hour of Day")]
+ (-> query
+ (lib/breakout expr-ref)
+ (lib/order-by (lib.util/fresh-uuids expr-ref)))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel raw-expression-breakout-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["breakout" ["get-day-of-week" ["field" (meta/id :orders :created-at)]]]
+ ["order-by" ["get-day-of-week" ["field" (meta/id :orders :created-at)]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/expression "__breakout_expression_1"
+ (lib/get-day-of-week (meta/field-metadata :orders :created-at)))
+ (lib/aggregate (lib/count)))
+ expected (as-> base-query query
+ (let [expr-ref (lib/expression-ref query "__breakout_expression_1")]
+ (-> query
+ (lib/breakout expr-ref)
+ (lib/order-by (lib.util/fresh-uuids expr-ref)))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel raw-field-order-by-reuses-matching-breakout-expression-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["breakout" ["get-hour" ["field" (meta/id :orders :created-at)]]]
+ ["order-by" ["field" (meta/id :orders :created-at)]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/expression "__breakout_expression_1"
+ (lib/get-hour (meta/field-metadata :orders :created-at)))
+ (lib/aggregate (lib/count)))
+ expected (as-> base-query query
+ (let [expr-ref (lib/expression-ref query "__breakout_expression_1")]
+ (-> query
+ (lib/breakout expr-ref)
+ (lib/order-by (lib.util/fresh-uuids expr-ref)))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel raw-expression-order-by-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Net Amount"
+ ["-" ["field" (meta/id :orders :total)]
+ ["field" (meta/id :orders :discount)]]]
+ ["order-by" ["desc"
+ ["-" ["field" (meta/id :orders :total)]
+ ["field" (meta/id :orders :discount)]]]]
+ ["limit" 10]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/expression "Net Amount"
+ (lib/- (meta/field-metadata :orders :total)
+ (meta/field-metadata :orders :discount)))
+ (as-> query
+ (lib/order-by query (lib/expression-ref query "Net Amount") :desc))
+ (lib/limit 10))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel expression-projection-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Net Amount"
+ ["-" ["field" (meta/id :orders :total)]
+ ["field" (meta/id :orders :discount)]]]
+ ["with-fields" [["field" (meta/id :orders :id)]
+ ["expression-ref" "Net Amount"]]]
+ ["order-by" ["desc" ["expression-ref" "Net Amount"]]]
+ ["limit" 10]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/expression "Net Amount"
+ (lib/- (meta/field-metadata :orders :total)
+ (meta/field-metadata :orders :discount)))
+ (as-> query
+ (-> query
+ (lib/with-fields [(meta/field-metadata :orders :id)
+ (lib/expression-ref query "Net Amount")])
+ (lib/order-by (lib/expression-ref query "Net Amount") :desc)))
+ (lib/limit 10))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel same-stage-aggregation-ref-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["aggregate" ["sum" ["field" (meta/id :venues :price)]]]
+ ["breakout" ["field" (meta/id :venues :category-id)]]
+ ["filter" [">" ["aggregation-ref" 1] 10]]
+ ["order-by" ["desc" ["aggregation-ref" 1]]]]}
+ expected (-> (tu/query-for-table :venues)
+ (lib/aggregate (lib/count))
+ (lib/aggregate (lib/sum (meta/field-metadata :venues :price)))
+ (lib/breakout (meta/field-metadata :venues :category-id))
+ (as-> query
+ (-> query
+ (lib/filter (lib/> (lib/aggregation-ref query 1) 10))
+ (lib/order-by (lib/aggregation-ref query 1) :desc))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :venues))
+ tu/comparable-query)))))
+
+(deftest ^:parallel join-with-explicit-fields-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-fields"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ [["=" ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]]
+ [["field" (meta/id :categories :name)]]]]]}
+ base-query (lib.tu/venues-query)
+ expected (-> base-query
+ (lib/join (-> (lib/join-clause (meta/table-metadata :categories))
+ (lib/with-join-conditions
+ [(lib/= (tu/current-query-column base-query (meta/id :venues :category-id))
+ (tu/current-query-column base-query (meta/id :categories :id)))])
+ (lib/with-join-fields
+ [(tu/current-query-column base-query (meta/id :categories :name))]))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :venues :categories))
+ tu/comparable-query)))))
+
+(deftest ^:parallel implicit-join-field-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["with-fields" [["field" (meta/id :orders :id)]
+ ["field" (meta/id :products :title)]]]
+ ["order-by" ["asc" ["field" (meta/id :products :title)]]]]}
+ base-query (tu/query-for-table :orders)
+ expected (-> base-query
+ (lib/with-fields [(tu/current-query-column base-query (meta/id :orders :id))
+ (tu/current-query-column base-query (meta/id :products :title))])
+ (lib/order-by (tu/current-query-column base-query (meta/id :products :title)) :asc))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders :products))
+ tu/comparable-query)))))
+
+(deftest ^:parallel implicit-join-field-outside-context-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["with-fields" [["field" (meta/id :orders :id)]
+ ["field" (meta/id :products :title)]]]
+ ["order-by" ["asc" ["field" (meta/id :products :title)]]]]}
+ base-query (tu/query-for-table :orders)
+ expected (-> base-query
+ (lib/with-fields [(tu/current-query-column base-query (meta/id :orders :id))
+ (tu/current-query-column base-query (meta/id :products :title))])
+ (lib/order-by (tu/current-query-column base-query (meta/id :products :title)) :asc))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel explicit-join-table-outside-context-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-fields"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ [["=" ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]]
+ [["field" (meta/id :categories :name)]]]]
+ ["filter" ["=" ["field" (meta/id :categories :name)] "Mexican"]]]}
+ base-query (lib.tu/venues-query)
+ expected (as-> base-query query
+ (lib/join query
+ (-> (lib/join-clause (meta/table-metadata :categories))
+ (lib/with-join-conditions
+ [(lib/= (tu/current-query-column base-query (meta/id :venues :category-id))
+ (tu/current-query-column base-query (meta/id :categories :id)))])
+ (lib/with-join-fields
+ [(tu/current-query-column base-query (meta/id :categories :name))])))
+ (lib/filter query (lib/= (tu/current-query-column query (meta/id :categories :name)) "Mexican")))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :venues))
+ tu/comparable-query)))))
+
+(deftest ^:parallel redundant-explicit-join-to-implicit-related-table-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :products)]]
+ [["=" ["field" (meta/id :orders :product-id)]
+ ["field" (meta/id :products :id)]]]]]
+ ["filter" ["=" ["field" (meta/id :products :title)] "Widget"]]
+ ["aggregate" ["count"]]]}
+ context (tu/table-context-with-join-edges :orders
+ [:products]
+ [(tu/join-edge :orders :product-id :products :id)])
+ base-query (tu/query-for-table :orders)
+ expected (-> base-query
+ (lib/filter (lib/= (tu/current-query-column base-query (meta/id :products :title)) "Widget"))
+ (lib/aggregate (lib/count)))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider context)
+ tu/comparable-query)))))
+
+(deftest ^:parallel implicit-join-filter-field-uses-source-field-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["contains" ["field" (meta/id :products :title)] "widget"]]
+ ["aggregate" ["count"]]]}
+ base-query (tu/query-for-table :orders)
+ expected (-> base-query
+ (lib/filter (lib/contains (tu/current-query-column base-query (meta/id :products :title)) "widget"))
+ (lib/aggregate (lib/count)))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders :products))
+ tu/comparable-query)))))
+
+(deftest ^:parallel explicit-join-with-field-selection-is-not-dropped-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-fields"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ [["=" ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]]
+ [["field" (meta/id :categories :name)]]]]
+ ["filter" ["=" ["field" (meta/id :categories :name)] "Mexican"]]]}
+ context (tu/table-context-with-join-edges :venues
+ [:categories]
+ [(tu/join-edge :venues :category-id :categories :id)])
+ base-query (lib.tu/venues-query)
+ expected (as-> base-query query
+ (lib/join query
+ (-> (lib/join-clause (meta/table-metadata :categories))
+ (lib/with-join-conditions
+ [(lib/= (tu/current-query-column base-query (meta/id :venues :category-id))
+ (tu/current-query-column base-query (meta/id :categories :id)))])
+ (lib/with-join-fields
+ [(tu/current-query-column base-query (meta/id :categories :name))])))
+ (lib/filter query (lib/= (tu/current-query-column query (meta/id :categories :name)) "Mexican")))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider context)
+ tu/comparable-query)))))
+
+(deftest ^:parallel append-stage-explicit-related-join-falls-back-to-implicit-related-field-test
+ (let [implicit-program {:source {:type "context" :ref "source"}
+ :operations [["breakout" ["field" (meta/id :orders :product-id)]]
+ ["append-stage"]
+ ["breakout" ["field" (meta/id :products :title)]]]}
+ explicit-program {:source {:type "context" :ref "source"}
+ :operations [["breakout" ["field" (meta/id :orders :product-id)]]
+ ["append-stage"]
+ ["join"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :products)]]
+ [["=" ["field" (meta/id :orders :product-id)]
+ ["field" (meta/id :products :id)]]]]]
+ ["breakout" ["field" (meta/id :products :title)]]]}
+ context (tu/table-context :orders :products)]
+ (is (= (-> (program/evaluate-program implicit-program meta/metadata-provider context)
+ tu/comparable-query)
+ (-> (program/evaluate-program explicit-program meta/metadata-provider context)
+ tu/comparable-query)))))
+
+(deftest ^:parallel bare-order-by-field-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["with-fields" [["field" (meta/id :orders :id)]
+ ["field" (meta/id :products :title)]]]
+ ["order-by" ["field" (meta/id :products :title)]]]}
+ base-query (tu/query-for-table :orders)
+ expected (-> base-query
+ (lib/with-fields [(tu/current-query-column base-query (meta/id :orders :id))
+ (tu/current-query-column base-query (meta/id :products :title))])
+ (lib/order-by (tu/current-query-column base-query (meta/id :products :title)) :asc))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders :products))
+ tu/comparable-query)))))
+
+(deftest ^:parallel append-stage-raw-field-falls-back-to-previous-stage-column-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["field" (meta/id :orders :total)]]]
+ ["append-stage"]
+ ["order-by" ["desc" ["field" (meta/id :orders :total)]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/avg (meta/field-metadata :orders :total)))
+ (lib/append-stage))
+ expected (lib/order-by base-query (tu/previous-stage-aggregation-column base-query 0) :desc)]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel aggregate-order-by-raw-field-falls-back-to-aggregation-column-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["min" ["field" (meta/id :orders :total)]]]
+ ["order-by" ["field" (meta/id :orders :total)]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/min (meta/field-metadata :orders :total))))
+ expected (lib/order-by base-query (tu/current-aggregation-orderable base-query 0))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel grouped-raw-field-order-by-falls-back-to-aggregation-column-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["field" (meta/id :orders :product-id)]]
+ ["order-by" ["desc" ["field" (meta/id :orders :total)]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/avg (meta/field-metadata :orders :total)))
+ (lib/breakout (meta/field-metadata :orders :product-id)))
+ expected (lib/order-by base-query (tu/current-aggregation-orderable base-query 0) :desc)]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel append-stage-aggregate-raw-field-falls-back-to-previous-stage-column-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["field" (meta/id :orders :total)]]]
+ ["append-stage"]
+ ["aggregate" ["avg" ["field" (meta/id :orders :total)]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/avg (meta/field-metadata :orders :total)))
+ (lib/append-stage))
+ expected (lib/aggregate base-query (lib/avg (tu/previous-stage-aggregation-column base-query 0)))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel append-stage-filter-on-unavailable-raw-field-is-rejected-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["append-stage"]
+ ["filter" [">" ["field" (meta/id :orders :total)] 10]]]}
+ error (try
+ (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (re-find #"not available in the current query stage"
+ (:details (ex-data error))))))
+
+(deftest ^:parallel relative-date-alias-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" [">="
+ ["field" (meta/id :orders :created-at)]
+ ["relative-date" -90 "day"]]]
+ ["limit" 10]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/filter (lib/>= (meta/field-metadata :orders :created-at)
+ (lib/relative-datetime -90 :day)))
+ (lib/limit 10))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel aggregate-over-aggregation-ref-source-program-parity-test
+ (let [program {:source {:type "program"
+ :program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["field" (meta/id :orders :created-at)]]]}}
+ :operations [["aggregate" ["stddev" ["aggregation-ref" 0]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :total)))
+ (lib/breakout (meta/field-metadata :orders :created-at))
+ (lib/append-stage))
+ expected (lib/aggregate base-query (lib/stddev (tu/previous-stage-aggregation-column base-query 0)))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel repaired-program-evaluation-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["is-not-null" ["field" (meta/id :orders :discount)]]]
+ ["filter" ["=" ["field" (meta/id :orders :product-id)] [1 2]]]
+ ["aggregate" ["count-if" ["=" ["field" (meta/id :orders :product-id)] 1]]
+ ["variance" ["field" (meta/id :orders :total)]]]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/filter (lib/not-null (meta/field-metadata :orders :discount)))
+ (lib/filter (lib/in (meta/field-metadata :orders :product-id) 1 2))
+ (lib/aggregate (lib/count-where (lib/= (meta/field-metadata :orders :product-id) 1)))
+ (lib/aggregate (lib/var (meta/field-metadata :orders :total))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel datetime-diff-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Days Since Order"
+ ["datetime-diff" ["now"]
+ ["field" (meta/id :orders :created-at)] "day"]]
+ ["with-fields" [["field" (meta/id :orders :id)]
+ ["expression-ref" "Days Since Order"]]]
+ ["limit" 10]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/expression "Days Since Order"
+ (lib/expression-clause :datetime-diff
+ [(lib/now)
+ (meta/field-metadata :orders :created-at)
+ :day]
+ nil))
+ (as-> query
+ (lib/with-fields query [(meta/field-metadata :orders :id)
+ (lib/expression-ref query "Days Since Order")]))
+ (lib/limit 10))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel later-stage-aggregation-ref-expression-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["field" (meta/id :orders :product-id)]]
+ ["append-stage"]
+ ["expression" "Revenue K" ["/" ["aggregation-ref" 0] 1000]]
+ ["with-fields" [["field" (meta/id :orders :product-id)]
+ ["expression-ref" "Revenue K"]]]
+ ["order-by" ["desc" ["expression-ref" "Revenue K"]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :total)))
+ (lib/breakout (meta/field-metadata :orders :product-id))
+ lib/append-stage)
+ agg-column (tu/previous-stage-aggregation-column base-query 0)
+ expected (-> base-query
+ (lib/expression "Revenue K" (lib// agg-column 1000))
+ (as-> query
+ (-> query
+ (lib/with-fields [(tu/current-query-column query (meta/id :orders :product-id))
+ (lib/expression-ref query "Revenue K")])
+ (lib/order-by (lib/expression-ref query "Revenue K") :desc))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel expression-ref-falls-back-to-previous-stage-column-after-append-stage-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["aggregate" ["sum" ["field" (meta/id :orders :discount)]]]
+ ["append-stage"]
+ ["expression" "Variance" ["-" ["aggregation-ref" 0] ["aggregation-ref" 1]]]
+ ["append-stage"]
+ ["aggregate" ["avg" ["expression-ref" "Variance"]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :total)))
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :discount)))
+ lib/append-stage)
+ variance-stage (lib/expression base-query
+ "Variance"
+ (lib/- (tu/previous-stage-aggregation-column base-query 0)
+ (tu/previous-stage-aggregation-column base-query 1)))
+ appended (lib/append-stage variance-stage)
+ expected (lib/aggregate appended
+ (lib/avg (tu/current-query-column-by-name appended "Variance")))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel aggregated-with-fields-raw-fields-become-breakouts-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["field" (meta/id :orders :total)]]]
+ ["with-fields" [["field" (meta/id :orders :product-id)]
+ ["field" (meta/id :orders :id)]]]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/avg (meta/field-metadata :orders :total)))
+ (lib/breakout (meta/field-metadata :orders :product-id))
+ (lib/breakout (meta/field-metadata :orders :id)))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel aggregated-with-fields-aggregation-ref-selection-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["with-fields" [["aggregation-ref" 0]]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/count)))
+ appended (lib/append-stage base-query)
+ expected (lib/with-fields appended [(tu/previous-stage-aggregation-column appended 0)])]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel nested-source-program-parity-test
+ (let [program {:source {:type "program"
+ :program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["<" ["field" (meta/id :orders :total)] 100]]
+ ["with-fields" [["field" (meta/id :orders :id)]
+ ["field" (meta/id :orders :total)]]]]}}
+ :operations [["limit" 5]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/filter (lib/< (meta/field-metadata :orders :total) 100))
+ (lib/with-fields [(meta/field-metadata :orders :id)
+ (meta/field-metadata :orders :total)])
+ (lib/limit 5))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel nested-join-source-program-parity-test
+ (let [nested-program {:source {:type "table" :id (meta/id :products)}
+ :operations [["with-fields" [["field" (meta/id :products :id)]
+ ["field" (meta/id :products :title)]]]]}
+ program {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-conditions"
+ ["join-clause" {:type "program" :program nested-program}]
+ [["=" ["field" (meta/id :orders :product-id)]
+ ["field" (meta/id :products :id)]]]]]]}
+ joined-query (-> (tu/query-for-table :products)
+ (lib/with-fields [(meta/field-metadata :products :id)
+ (meta/field-metadata :products :title)]))
+ base-query (tu/query-for-table :orders)
+ expected (-> base-query
+ (lib/join (-> (lib/join-clause joined-query)
+ (lib/with-join-conditions
+ [(lib/= (tu/current-query-column base-query (meta/id :orders :product-id))
+ (tu/current-query-column base-query (meta/id :products :id)))]))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders :products))
+ tu/comparable-query)))))
+
+(deftest ^:parallel with-page-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["with-page" {:page 2 :items 25}]]}
+ expected (lib/with-page (tu/query-for-table :orders) {:page 2 :items 25})]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel stage-ops-parity-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["append-stage"]
+ ["drop-empty-stages"]
+ ["append-stage"]
+ ["with-page" {:page 2 :items 25}]
+ ["drop-stage"]]}
+ expected (-> (tu/query-for-table :orders)
+ lib/append-stage
+ lib/drop-empty-stages
+ lib/append-stage
+ (lib/with-page {:page 2 :items 25})
+ lib/drop-stage)]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel invalid-case-default-object-test
+ (let [total-field-id (meta/id :orders :total)
+ case-expr ["case" [[[">" ["field" total-field-id] 100]
+ ["field" total-field-id]]]
+ {:default nil}]
+ program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Maybe Revenue" case-expr]]}
+ error (try
+ (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (= "case" (:operator (ex-data error))))
+ (is (re-find #"fallback value itself as the optional third argument"
+ (:details (ex-data error))))
+ (is (re-find #"Omit the third argument instead of using null"
+ (:details (ex-data error))))))
+
+(deftest ^:parallel invalid-bare-field-id-in-aggregation-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" (meta/id :orders :total)]]]}
+ error (try
+ (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (= "sum" (:operator (ex-data error))))
+ (is (= :field-wrapper (:retry-category (ex-data error))))
+ (is (re-find #"Wrap field ids with `\[\"field\", id\]` inside `sum`"
+ (:details (ex-data error))))))
+
+(deftest ^:parallel invalid-unknown-ref-helper-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Percentage Share" ["sum-ref"]]]}
+ error (try
+ (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (= "sum-ref" (:operator (ex-data error))))
+ (is (re-find #"unknown helper reference"
+ (:details (ex-data error))))))
+
+(deftest ^:parallel conditional-average-case-parity-test
+ (let [total-field-id (meta/id :orders :total)
+ case-expr ["case" [[[">" ["field" total-field-id] 100]
+ ["field" total-field-id]]]]
+ program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Large Total" case-expr]
+ ["aggregate" ["avg" ["expression-ref" "Large Total"]]]]}
+ expected (-> (tu/query-for-table :orders)
+ (lib/expression "Large Total"
+ (lib/case [[(lib/> (meta/field-metadata :orders :total) 100)
+ (meta/field-metadata :orders :total)]]))
+ (as-> query
+ (lib/aggregate query (lib/avg (lib/expression-ref query "Large Total")))))]
+ (is (= (tu/comparable-query expected)
+ (-> (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ tu/comparable-query)))))
+
+(deftest ^:parallel offset-expression-defers-to-lib-core-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["with-temporal-bucket" ["field" (meta/id :orders :created-at)] "month"]]
+ ["order-by" ["asc"
+ ["with-temporal-bucket"
+ ["field" (meta/id :orders :created-at)] "month"]]]
+ ["append-stage"]
+ ["expression" "Prev Revenue" ["offset" ["aggregation-ref" 0] -1]]]}
+ base-query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :total)))
+ (lib/breakout (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :month))
+ (lib/order-by (lib/with-temporal-bucket (meta/field-metadata :orders :created-at) :month) :asc)
+ lib/append-stage)
+ agg-column (tu/previous-stage-aggregation-column base-query 0)
+ lib-error (try
+ (-> base-query
+ (lib/expression "Prev Revenue" (lib/offset agg-column -1)))
+ nil
+ (catch Exception e
+ e))
+ program-error (try
+ (program/evaluate-program program meta/metadata-provider (tu/table-context :orders))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ e))]
+ (testing "structured evaluation should defer to lib.core instead of rejecting this pattern preemptively"
+ (is lib-error)
+ (is program-error)
+ (is (re-find #"non-aggregation expression" (ex-message lib-error)))
+ (is (re-find #"non-aggregation expression" (:details (ex-data program-error))))
+ (is (not (re-find #"named expressions must be non-aggregation only"
+ (:details (ex-data program-error)))))
+ (is (= :invalid-generated-program (:error (ex-data program-error)))))))
diff --git a/test/metabase/agent_lib/fuzz_test.clj b/test/metabase/agent_lib/fuzz_test.clj
new file mode 100644
index 00000000000..c80f05434b8
--- /dev/null
+++ b/test/metabase/agent_lib/fuzz_test.clj
@@ -0,0 +1,253 @@
+(ns metabase.agent-lib.fuzz-test
+ "Property-based tests for agent-lib repair, validation, and error handling.
+ Generators produce random program-like structures (both valid and invalid)
+ to verify that the pipeline never crashes — it either succeeds or throws
+ a structured `:invalid-generated-program` error."
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [clojure.test.check.clojure-test :refer [defspec]]
+ [clojure.test.check.generators :as gen]
+ [clojure.test.check.properties :as prop]
+ [metabase.agent-lib.repair :as repair]
+ [metabase.agent-lib.schema :as schema]
+ [metabase.agent-lib.validate :as validate]
+ [metabase.lib.test-metadata :as meta]))
+
+(set! *warn-on-reflection* true)
+
+;; --- Generators ---
+
+(def ^:private gen-field-id
+ "Generator for field ids from test metadata."
+ (gen/elements [(meta/id :orders :id)
+ (meta/id :orders :total)
+ (meta/id :orders :product-id)
+ (meta/id :orders :user-id)
+ (meta/id :orders :created-at)
+ (meta/id :products :title)
+ (meta/id :products :category)
+ 99999]))
+
+(def ^:private gen-table-id
+ (gen/elements [(meta/id :orders)
+ (meta/id :products)
+ (meta/id :people)
+ 99999]))
+
+(def ^:private gen-op-name
+ "Generator for operator names — mix of valid and invalid."
+ (gen/elements ["count" "sum" "avg" "min" "max" "distinct" "field"
+ "=" "!=" "<" ">" "in" "between" "contains"
+ "expression-ref" "aggregation-ref"
+ "case" "coalesce" "+" "-" "*" "/"
+ "with-temporal-bucket" "relative-datetime" "now"
+ ;; invalid operators to exercise error paths
+ "bogus" "YOLO" ""]))
+
+(def ^:private gen-top-level-op-name
+ (gen/elements ["filter" "aggregate" "breakout" "order-by"
+ "expression" "with-fields" "limit" "join"
+ "append-stage"
+ ;; invalid to exercise error paths
+ "bogus-top" ""]))
+
+(def ^:private gen-scalar
+ (gen/one-of [gen/small-integer
+ (gen/double* {:NaN? false :infinite? false})
+ gen/boolean
+ (gen/not-empty gen/string-alphanumeric)
+ (gen/return nil)]))
+
+(defn- gen-node
+ "Recursive generator for program nodes. `depth` limits nesting."
+ [depth]
+ (if (zero? depth)
+ gen-scalar
+ (gen/one-of
+ [gen-scalar
+ ;; field ref
+ (gen/fmap (fn [id] ["field" id]) gen-field-id)
+ ;; nested operator
+ (gen/fmap (fn [[op args]] (into [op] args))
+ (gen/tuple gen-op-name
+ (gen/vector (gen-node (dec depth)) 0 3)))
+ ;; vector of nodes
+ (gen/vector (gen-node (dec depth)) 0 3)])))
+
+(def ^:private gen-operation
+ "Generator for top-level operations."
+ (gen/fmap (fn [[op args]] (into [op] args))
+ (gen/tuple gen-top-level-op-name
+ (gen/vector (gen-node 3) 0 4))))
+
+(def ^:private gen-source
+ "Generator for program sources."
+ (gen/one-of
+ [(gen/return {:type "context" :ref "source"})
+ (gen/fmap (fn [id] {:type "table" :id id}) gen-table-id)
+ (gen/fmap (fn [id] {:type "metric" :id id}) (gen/elements [1 42 99999]))
+ (gen/fmap (fn [id] {:type "card" :id id}) (gen/elements [1 99999]))
+ ;; invalid source types
+ (gen/return {:type "nonexistent"})
+ (gen/return {})
+ (gen/return nil)]))
+
+(def ^:private gen-program
+ "Generator for program-like structures."
+ (gen/fmap (fn [[source ops]] {:source source :operations (vec ops)})
+ (gen/tuple gen-source
+ (gen/vector gen-operation 0 6))))
+
+(def ^:private gen-garbage
+ "Generator for completely random values — not program-shaped at all."
+ (gen/one-of
+ [gen/small-integer
+ gen/string-alphanumeric
+ gen/boolean
+ (gen/return nil)
+ (gen/vector gen/small-integer 0 5)
+ (gen/map gen/keyword gen/small-integer {:max-elements 3})]))
+
+;; --- Helpers ---
+
+(defn- orders-context
+ []
+ {:source-entity {:model "table" :id (meta/id :orders)
+ :columns (mapv (fn [f] {:id (meta/id :orders f)})
+ (meta/fields :orders))}
+ :referenced-entities []
+ :surrounding-tables [{:id (meta/id :products)}
+ {:id (meta/id :people)}]
+ :measure-ids []})
+
+(defn- structured-error?
+ "True when `e` is the standard `:invalid-generated-program` error
+ or a Malli input validation error (from mu/defn schemas)."
+ [e]
+ (let [data (ex-data e)]
+ (or (= :invalid-generated-program (:error data))
+ (= :metabase.util.malli.fn/invalid-input (:type data))
+ (= :metabase.util.malli.fn/invalid-output (:type data)))))
+
+;; --- Properties ---
+
+(defspec repair-never-crashes 200
+ (prop/for-all [program gen-program]
+ (let [result (try
+ (repair/repair-program program)
+ ::ok
+ (catch Exception _
+ ::ok))]
+ (= ::ok result))))
+
+(defspec repair-returns-map-for-map-input 200
+ (prop/for-all [program gen-program]
+ (map? (repair/repair-program program))))
+
+(defspec repair-preserves-source-key 200
+ (prop/for-all [program gen-program]
+ (contains? (repair/repair-program program) :source)))
+
+(defspec repair-preserves-operations-key 200
+ (prop/for-all [program gen-program]
+ (contains? (repair/repair-program program) :operations)))
+
+(defspec repair-is-idempotent 200
+ (prop/for-all [program gen-program]
+ (= (repair/repair-program program)
+ (repair/repair-program (repair/repair-program program)))))
+
+(defspec validation-never-crashes-with-unstructured-error 200
+ (prop/for-all [program gen-program]
+ (try
+ (validate/validated-program program (orders-context))
+ true
+ (catch clojure.lang.ExceptionInfo e
+ (structured-error? e))
+ (catch Exception _
+ ;; Non-ExceptionInfo exceptions are acceptable from downstream
+ ;; libs (e.g. Malli schema validation); the key property is
+ ;; agent-lib code itself doesn't throw raw exceptions.
+ true))))
+
+(defspec garbage-input-to-repair-never-crashes 100
+ (prop/for-all [garbage gen-garbage]
+ (try
+ (repair/repair-program garbage)
+ true
+ (catch Exception _
+ true))))
+
+(defspec garbage-input-to-validation-never-crashes-with-unstructured-error 100
+ (prop/for-all [garbage gen-garbage]
+ (try
+ (validate/validated-program garbage (orders-context))
+ true
+ (catch clojure.lang.ExceptionInfo e
+ (structured-error? e))
+ (catch Exception _
+ true))))
+
+(defspec schema-validation-rejects-or-accepts-cleanly 200
+ (prop/for-all [program gen-program]
+ (let [repaired (repair/repair-program program)]
+ (try
+ (schema/validated-structure repaired)
+ true
+ (catch clojure.lang.ExceptionInfo e
+ (structured-error? e))))))
+
+;; --- Deterministic edge-case tests ---
+
+(deftest ^:parallel deeply-nested-operations-do-not-overflow-test
+ (let [deep-op (reduce (fn [inner _] ["+" inner 1])
+ ["field" (meta/id :orders :total)]
+ (range 50))
+ program {:source {:type "table" :id (meta/id :orders)}
+ :operations [["filter" [">" deep-op 0]]]}]
+ (testing "repair handles deeply nested operations"
+ (is (map? (repair/repair-program program))))
+ (testing "validation rejects or accepts without crashing"
+ (try
+ (validate/validated-program program (orders-context))
+ (is true)
+ (catch clojure.lang.ExceptionInfo e
+ (is (structured-error? e)))))))
+
+(deftest ^:parallel empty-and-minimal-programs-test
+ (testing "empty operations"
+ (let [program {:source {:type "table" :id (meta/id :orders)}
+ :operations []}]
+ (is (= program (validate/validated-program program (orders-context))))))
+ (testing "nil operations repairs to empty"
+ (let [repaired (repair/repair-program {:source {:type "context" :ref "source"}
+ :operations nil})]
+ (is (vector? (:operations repaired)))))
+ (testing "missing operations key repairs gracefully"
+ (let [repaired (repair/repair-program {:source {:type "context" :ref "source"}})]
+ (is (map? repaired)))))
+
+(deftest ^:parallel many-operations-test
+ (let [ops (vec (repeat 50 ["aggregate" ["count"]]))
+ program {:source {:type "table" :id (meta/id :orders)}
+ :operations ops}]
+ (testing "programs exceeding max-operations are rejected"
+ (try
+ (validate/validated-program program (orders-context))
+ (is false "expected rejection")
+ (catch clojure.lang.ExceptionInfo e
+ (is (structured-error? e))
+ (is (re-find #"maximum operation count"
+ (:details (ex-data e)))))))))
+
+(deftest ^:parallel repair-idempotence-degenerate-helpers-test
+ (testing "degenerate operators inside `case` do not grow extra args on each repair pass (regression)"
+ (doseq [op ["in" "not-in" "=" "!=" "is" "is-not"
+ "between" "with-temporal-bucket" "percentile" "with-join-conditions"]]
+ (testing op
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["count" ["case" [op]]]]]}
+ once (repair/repair-program program)
+ twice (repair/repair-program once)]
+ (is (= once twice)
+ (str "repair of `[\"case\" [\"" op "\"]]` should be idempotent")))))))
diff --git a/test/metabase/agent_lib/guards_test.clj b/test/metabase/agent_lib/guards_test.clj
new file mode 100644
index 00000000000..288f8d5f613
--- /dev/null
+++ b/test/metabase/agent_lib/guards_test.clj
@@ -0,0 +1,199 @@
+(ns metabase.agent-lib.guards-test
+ "Tests for security and robustness guards added to agent-lib."
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.common.errors :as errors]
+ [metabase.agent-lib.repair.context.passes :as passes]
+ [metabase.agent-lib.repair.stages :as repair.stages]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.validate :as validate]
+ [metabase.agent-lib.validate.context :as validate.context]
+ [metabase.agent-lib.validate.walker :as walker]
+ [metabase.lib.test-metadata :as meta]))
+
+(set! *warn-on-reflection* true)
+
+(defn- orders-context
+ "Minimal context with only the orders table allowed."
+ []
+ {:source-entity {:model "table" :id (meta/id :orders)
+ :columns (mapv (fn [f] {:id (meta/id :orders f)})
+ (meta/fields :orders))}
+ :referenced-entities []
+ :surrounding-tables []
+ :measure-ids []})
+
+(defn- ex-details
+ "Extract the :details string from an ExceptionInfo thrown by invalid-program!."
+ [^Exception e]
+ (:details (ex-data e)))
+
+;; --- P0: Table source validation ---
+
+(deftest validate-source-rejects-table-not-in-context-test
+ (let [allowed-ids (validate.context/context-allowed-ids (orders-context))]
+ (testing "table in context passes"
+ (is (= {}
+ (validate.context/validate-source!
+ (fn [& _] {})
+ allowed-ids
+ [:source]
+ {:type "table" :id (meta/id :orders)}
+ 0
+ {}))))
+ (testing "table NOT in context is rejected"
+ (try
+ (validate.context/validate-source!
+ (fn [& _] {})
+ allowed-ids
+ [:source]
+ {:type "table" :id 99999}
+ 0
+ {})
+ (is false "expected unknown table to be rejected")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"not available in the provided context" (ex-details e))))))))
+
+(deftest ^:parallel validated-program-rejects-unknown-table-source-test
+ (let [program {:source {:type "table" :id 99999}
+ :operations []}
+ context (orders-context)]
+ (try
+ (validate/validated-program program context)
+ (is false "expected unknown table to be rejected")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"not available in the provided context" (ex-details e)))))))
+
+;; --- P1: Runtime nil-guard ---
+
+(deftest ^:parallel helper-fn-rejects-nil-binding-test
+ (let [rt {:bindings {'known identity 'nil-bound nil}}]
+ (testing "non-nil binding returns the function"
+ (is (fn? (runtime/helper-fn rt 'known))))
+ (testing "nil binding throws"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"No runtime binding for operator"
+ (runtime/helper-fn rt 'nil-bound))))
+ (testing "missing binding throws"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"No runtime binding for operator"
+ (runtime/helper-fn rt 'totally-missing))))))
+
+;; --- P1: Program nesting depth ---
+
+(deftest ^:parallel validated-program-rejects-deeply-nested-program-sources-test
+ (let [leaf {:source {:type "table" :id (meta/id :orders)} :operations []}
+ deep-program (reduce (fn [inner _]
+ {:source {:type "program" :program inner}
+ :operations []})
+ leaf
+ (range (+ walker/max-program-nesting 2)))
+ context (orders-context)]
+ (try
+ (validate/validated-program deep-program context)
+ (is false "expected deep nesting to be rejected")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"program source nesting exceeds maximum depth" (ex-details e)))))))
+
+;; --- P2: Depth-limited contains-aggregation-ref? ---
+
+(deftest ^:parallel contains-aggregation-ref-respects-depth-limit-test
+ (testing "shallow aggregation-ref is found"
+ (is (true? (repair.stages/contains-aggregation-ref?
+ ["aggregate" ["sum" ["aggregation-ref" 0]]]))))
+ (testing "deeply nested structure does not overflow"
+ (let [deep (reduce (fn [inner _] [inner]) ["aggregation-ref" 0] (range 100))]
+ ;; Returns false (too deep) rather than overflowing the stack
+ (is (false? (repair.stages/contains-aggregation-ref? deep))))))
+
+;; --- P2: Depth-limited tree walk in passes ---
+
+(deftest ^:parallel operation-contains-canonical-op-respects-depth-limit-test
+ (testing "shallow match is found"
+ (is (true? (passes/operation-contains-canonical-op?
+ ["aggregate" ["sum" ["metric" 1]]]
+ "metric"))))
+ (testing "deeply nested structure does not overflow"
+ (let [deep (reduce (fn [inner _] [inner]) ["metric" 1] (range 100))]
+ ;; Returns false (too deep) rather than overflowing the stack
+ (is (false? (passes/operation-contains-canonical-op? deep "metric"))))))
+
+;; --- P2: String length validation ---
+
+(deftest ^:parallel walk-node-rejects-overlong-strings-test
+ (let [long-string (apply str (repeat (inc walker/max-string-length) "x"))]
+ (try
+ (walker/walk-node [:test] long-string 0 {} (fn [_ _ _ state] state))
+ (is false "expected overlong string to be rejected")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"string value exceeds maximum length" (ex-details e))))))
+ (testing "strings within limit pass"
+ (is (= {:node-count 1}
+ (walker/walk-node [:test] "short" 0 {} (fn [_ _ _ state] state))))))
+
+;; --- P2: Error messages don't leak schema ---
+
+(deftest ^:parallel lookup-table-error-keeps-schema-out-of-message-test
+ (let [rt (runtime/build-runtime meta/metadata-provider)]
+ (try
+ ((runtime/helper-fn rt 'table) "nonexistent_table")
+ (is false "expected table lookup to fail")
+ (catch clojure.lang.ExceptionInfo e
+ (testing "message does not enumerate available tables"
+ (is (not (re-find #"orders" (ex-message e)))))
+ (testing "recovery map carries available names for LLM agents"
+ (is (seq (get-in (ex-data e) [:recovery :available]))))))))
+
+;; --- Empty operations list ---
+
+(deftest ^:parallel validate-empty-operations-test
+ (let [program {:source {:type "table" :id (meta/id :orders)} :operations []}
+ context (orders-context)]
+ (is (= program (validate/validated-program program context)))))
+
+;; --- Structured error helpers ---
+
+(deftest ^:parallel wrap-runtime-error-preserves-recovery-from-cause-test
+ (let [cause (ex-info "Table not found: foo"
+ {:table-name "foo"
+ :recovery {:available ["orders" "products"]}})
+ wrapped (errors/wrap-runtime-error [:source] "table" cause)]
+ (testing "standard fields are set"
+ (is (= :invalid-generated-program (:error (ex-data wrapped))))
+ (is (= "table" (:operator (ex-data wrapped)))))
+ (testing "recovery from cause is preserved"
+ (is (= {:available ["orders" "products"]}
+ (:recovery (ex-data wrapped)))))
+ (testing "cause chain is preserved"
+ (is (= cause (.getCause ^Exception wrapped))))))
+
+(deftest ^:parallel wrap-runtime-error-promotes-legacy-available-key-test
+ (let [cause (ex-info "Field not found" {:field-name "foo" :available ["a" "b"]})
+ wrapped (errors/wrap-runtime-error [:ops 0] "field" cause)]
+ (testing "top-level :available is promoted into :recovery"
+ (is (= ["a" "b"] (get-in (ex-data wrapped) [:recovery :available]))))))
+
+(deftest ^:parallel recovery-summary-formats-hints-test
+ (testing "available values"
+ (is (= "Available: orders, products"
+ (errors/recovery-summary {:available ["orders" "products"]}))))
+ (testing "tables for disambiguation"
+ (is (= "Matching tables: orders, people"
+ (errors/recovery-summary {:tables ["orders" "people"]}))))
+ (testing "suggestion"
+ (is (= "Use (field \"TABLE\" \"FIELD\") to disambiguate."
+ (errors/recovery-summary {:suggestion "Use (field \"TABLE\" \"FIELD\") to disambiguate."}))))
+ (testing "combined"
+ (let [summary (errors/recovery-summary {:available ["a" "b"]
+ :suggestion "Try again."})]
+ (is (re-find #"Available: a, b" summary))
+ (is (re-find #"Try again\." summary))))
+ (testing "empty recovery returns nil"
+ (is (nil? (errors/recovery-summary {})))
+ (is (nil? (errors/recovery-summary nil)))))
diff --git a/test/metabase/agent_lib/join_spec_test.clj b/test/metabase/agent_lib/join_spec_test.clj
new file mode 100644
index 00000000000..7a1b48cb012
--- /dev/null
+++ b/test/metabase/agent_lib/join_spec_test.clj
@@ -0,0 +1,32 @@
+(ns metabase.agent-lib.join-spec-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.join-spec :as join-spec]
+ [metabase.lib.test-metadata :as meta]))
+
+(deftest parse-join-spec-normalizes-nested-join-wrappers-test
+ (let [join-form ["with-join-alias"
+ ["with-join-strategy"
+ ["with-join-fields"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ [["="
+ ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]]
+ "none"]
+ "left-join"]
+ "cat"]
+ expected {:target-table-id (meta/id :categories)
+ :conditions [[(meta/id :venues :category-id)
+ (meta/id :categories :id)]]
+ :fields-mode "none"
+ :strategy "left-join"
+ :alias "cat"}]
+ (is (= expected (join-spec/parse-join-spec join-form)))))
+
+(deftest join-fields-mode-helpers-test
+ (is (true? (join-spec/no-explicit-join-fields? nil)))
+ (is (true? (join-spec/no-explicit-join-fields? "none")))
+ (is (false? (join-spec/no-explicit-join-fields? "all")))
+ (is (true? (join-spec/implicit-join-compatible-fields-mode? "all")))
+ (is (false? (join-spec/implicit-join-compatible-fields-mode? [["field" 1]]))))
diff --git a/test/metabase/agent_lib/mbql_integration_test.clj b/test/metabase/agent_lib/mbql_integration_test.clj
new file mode 100644
index 00000000000..7243d1e19ee
--- /dev/null
+++ b/test/metabase/agent_lib/mbql_integration_test.clj
@@ -0,0 +1,266 @@
+(ns metabase.agent-lib.mbql-integration-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.agent-lib.runtime :as runtime]
+ [metabase.agent-lib.test-util :as tu]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.util :as lib.schema.util]
+ [metabase.lib.test-metadata :as meta]))
+
+(def ^:private mbql-ns 'metabase.agent-lib.mbql-integration)
+
+(defn- runtime-fields-by-id
+ []
+ (:fields-by-id (runtime/build-runtime meta/metadata-provider)))
+
+(defn- comparable-ref
+ [ref-clause]
+ (lib.schema.util/remove-lib-uuids ref-clause))
+
+(defn- call-private
+ [sym & args]
+ (apply (var-get (ns-resolve mbql-ns sym)) args))
+
+(defn- current-query-field
+ [query table-name field-name]
+ (or (lib/find-matching-column query
+ -1
+ (meta/field-metadata table-name field-name)
+ (mbql/current-query-field-candidates query))
+ (throw (ex-info "Expected field to be available in query context"
+ {:table-name table-name
+ :field-name field-name
+ :query query}))))
+
+(defn- current-aggregation-column
+ [query idx]
+ (or (lib/find-matching-column query -1 (lib/aggregation-ref query idx) (lib/orderable-columns query))
+ (throw (ex-info "Expected aggregation column in query context"
+ {:aggregation-index idx
+ :query query}))))
+
+(deftest query?-uses-normalized-query-type-test
+ (is (true? (mbql/query? (tu/query-for-table :orders))))
+ (is (false? (mbql/query? (meta/table-metadata :orders)))))
+
+(deftest extracted-field-resolution-helper-test
+ (let [fields-by-id (runtime-fields-by-id)
+ orders-query (tu/query-for-table :orders)
+ by-table (call-private 'fields-by-table-id fields-by-id)
+ query-candidates (mbql/current-query-field-candidates orders-query)
+ product-id-field-id (meta/id :orders :product-id)
+ venues-category-id (meta/id :venues :category-id)
+ category-id-field-id (meta/id :products :category)
+ product-table-id (meta/id :products)
+ category-table-id (meta/id :categories)]
+ (testing "candidate preference is explicit and deterministic"
+ (is (= {:id 10 :name "ID"}
+ (call-private 'prefer-single-candidate
+ [{:id 10 :name "ID" :lib/join-alias "Products"}
+ {:id 10 :name "ID"}])))
+ (is (= {:id 20 :name "Category"}
+ (call-private 'prefer-single-candidate
+ [{:id 20 :name "Category" :fk-field-id 21}
+ {:id 20 :name "Category"}]))))
+ (testing "metadata grouping and FK target resolution use the extracted helpers"
+ (is (some #(= product-id-field-id (:id %))
+ (get by-table (meta/id :orders))))
+ (is (= product-table-id
+ (call-private 'field-target-table-id fields-by-id product-id-field-id)))
+ (is (nil? (call-private 'field-target-table-id fields-by-id (meta/id :orders :total)))))
+ (testing "lineage helpers expose the path anchors and recursive FK traversal"
+ (is (= [product-id-field-id category-id-field-id 999]
+ (vec (call-private 'field-path-start-ids
+ {:source-field product-id-field-id
+ :fk-field-id category-id-field-id
+ :lib/original-fk-field-id category-id-field-id
+ :id 999}))))
+ (is (= [product-id-field-id]
+ (call-private 'fk-path-to-table fields-by-id product-id-field-id product-table-id)))
+ (is (= [venues-category-id]
+ (call-private 'fk-path-to-table fields-by-id venues-category-id category-table-id)))
+ (is (nil? (call-private 'fk-path-to-table fields-by-id product-id-field-id category-table-id))))
+ (testing "type compatibility and lineage ranking helpers are independently testable"
+ (is (true? (call-private 'numeric-type? :type/Float)))
+ (is (false? (call-private 'numeric-type? :type/Text)))
+ (is (true? (call-private 'types-compatible?
+ {:effective-type :type/Float}
+ {:effective-type :type/Integer})))
+ (is (false? (call-private 'types-compatible?
+ {:effective-type :type/Text}
+ {:effective-type :type/Integer})))
+ (is (true? (call-private 'candidate-has-resolution-lineage?
+ {:fk-field-id category-id-field-id})))
+ (is (false? (call-private 'candidate-has-resolution-lineage?
+ {:id product-id-field-id}))))
+ (testing "source-column accessors and synthesized multi-hop lineage are namespace-level helpers"
+ (let [source-column {:lib/source-column-alias "Category"
+ :name "CATEGORY"
+ :fk-field-name "Category"
+ :fk-join-alias "Products"
+ :source-field category-id-field-id
+ :lib/original-fk-field-id product-id-field-id
+ :lib/original-fk-field-name "Product ID"
+ :lib/original-fk-join-alias nil}
+ raw-field {:id category-id-field-id
+ :name "Category"
+ :table-id category-table-id}
+ synthesized (call-private 'synthesize-chained-related-field
+ fields-by-id
+ query-candidates
+ raw-field
+ source-column
+ [product-id-field-id category-id-field-id])]
+ (is (= "Category" (call-private 'source-column-field-name source-column)))
+ (is (= "Products" (call-private 'source-column-field-join-alias source-column)))
+ (is (= product-id-field-id (call-private 'source-column-original-field-id source-column)))
+ (is (= "Product ID" (call-private 'source-column-original-field-name source-column)))
+ (is (nil? (call-private 'source-column-original-field-join-alias source-column)))
+ (is (= "Products"
+ (call-private 'candidate-join-alias-for-field-id
+ [{:source-field category-id-field-id
+ :lib/join-alias "Products"}]
+ category-id-field-id)))
+ (is (= category-id-field-id (:fk-field-id synthesized)))
+ (is (= product-id-field-id (:lib/original-fk-field-id synthesized)))
+ (is (= "Products" (:fk-join-alias synthesized)))
+ (is (true? (call-private 'multi-hop-lineage-candidate? synthesized)))
+ (is (false? (call-private 'multi-hop-lineage-candidate?
+ {:fk-field-id category-id-field-id
+ :lib/original-fk-field-id category-id-field-id})))))
+ (testing "previous-stage matching is a separate unit from the main resolver"
+ (is (= [{:name "Average of Total"
+ :source-field (meta/id :orders :total)
+ :lib/source :source/previous-stage}]
+ (call-private 'previous-stage-lineage-matches
+ {:id (meta/id :orders :total)
+ :name "Total"}
+ [{:name "Average of Total"
+ :source-field (meta/id :orders :total)
+ :lib/source :source/previous-stage}
+ {:name "Count"
+ :lib/source :source/previous-stage}])))
+ (let [appended-query (-> orders-query
+ (lib/aggregate (lib/avg (meta/field-metadata :orders :total)))
+ (lib/append-stage))
+ previous-stage-columns (->> (mbql/current-query-field-candidates appended-query)
+ (filter #(= :source/previous-stage (:lib/source %)))
+ vec)]
+ (is (= ["avg"]
+ (mapv :name
+ (call-private 'previous-stage-aggregation-matches
+ appended-query
+ {:id (meta/id :orders :total)
+ :name "Total"}
+ previous-stage-columns)))))
+ (is (= [{:name "NET AMOUNT" :lib/source :source/previous-stage}]
+ (call-private 'previous-stage-name-matches
+ {:name "Net Amount"}
+ [{:name "NET AMOUNT" :lib/source :source/previous-stage}
+ {:name "Other" :lib/source :source/previous-stage}]))))))
+
+(deftest extracted-orderable-helper-test
+ (let [orders-query (tu/query-for-table :orders)
+ breakout-query (-> orders-query
+ (lib/expression "__breakout_expression_1"
+ (lib/get-hour (meta/field-metadata :orders :created-at)))
+ (lib/aggregate (lib/count))
+ (as-> query
+ (lib/breakout query (lib/expression-ref query "__breakout_expression_1"))))
+ aggregation-query (-> orders-query
+ (lib/aggregate (lib/sum (meta/field-metadata :orders :total)))
+ (lib/aggregate (lib/count)))
+ total-field-id (meta/id :orders :total)
+ created-at-field-id (meta/id :orders :created-at)
+ breakout-expression-ref (lib/expression-ref breakout-query "__breakout_expression_1")
+ aggregation-column (current-aggregation-column aggregation-query 0)]
+ (testing "aggregation inventory helpers are directly unit-tested"
+ (is (= 2 (count (call-private 'aggregation-columns aggregation-query))))
+ (is (= [0 1]
+ (mapv first (call-private 'aggregation-column-pairs aggregation-query)))))
+ (testing "requested field ids and current-stage field matching are separate helpers"
+ (is (= #{total-field-id}
+ (call-private 'requested-orderable-field-ids (meta/field-metadata :orders :total))))
+ (is (= #{created-at-field-id}
+ (call-private 'requested-orderable-field-ids (lib/ref (meta/field-metadata :orders :created-at)))))
+ (is (= #{}
+ (call-private 'requested-orderable-field-ids "not-a-field")))
+ (is (some #(= total-field-id (:id %))
+ (call-private 'orderable-field-id->query-columns orders-query total-field-id)))
+ (is (some #(= total-field-id (:id %))
+ (call-private 'orderable-field-ids->query-columns orders-query #{total-field-id}))))
+ (testing "breakout and aggregation matching helpers are individually covered"
+ (is (= [(comparable-ref breakout-expression-ref)]
+ (mapv comparable-ref
+ (call-private 'field-id->breakout-expression-refs breakout-query created-at-field-id))))
+ (is (= (comparable-ref breakout-expression-ref)
+ (comparable-ref
+ (call-private 'field-ids->breakout-expression-ref breakout-query #{created-at-field-id}))))
+ (is (= [aggregation-column]
+ (call-private 'field-id->aggregation-columns aggregation-query total-field-id)))
+ (is (= [aggregation-column]
+ (call-private 'field-ids->aggregation-columns aggregation-query #{total-field-id}))))
+ (testing "field-like orderable resolution is directly testable after extraction"
+ (let [[_ breakout-orderable] (call-private 'resolve-field-like-orderable
+ breakout-query
+ (meta/field-metadata :orders :created-at))
+ [_ aggregation-orderable] (call-private 'resolve-field-like-orderable
+ aggregation-query
+ (meta/field-metadata :orders :total))]
+ (is (= (comparable-ref breakout-expression-ref)
+ (comparable-ref breakout-orderable)))
+ (is (= aggregation-column aggregation-orderable))))))
+
+(deftest implicitly-resolved-column-helper-test
+ (is (true? (call-private 'implicitly-resolved-column?
+ {:table-id 100
+ :fk-field-id 20}
+ 100)))
+ (is (true? (call-private 'implicitly-resolved-column?
+ {:table-id 200
+ :lib/source :source/previous-stage}
+ 100)))
+ (is (false? (call-private 'implicitly-resolved-column?
+ {:table-id 100
+ :join-alias "Products"
+ :fk-field-id 20}
+ 100))))
+
+(deftest resolve-aggregation-selection-uses-visible-column-match-test
+ (let [query (-> (tu/query-for-table :orders)
+ (lib/aggregate (lib/count)))
+ selection (lib/aggregation-ref query 0)]
+ (is (= (lib/find-visible-column-for-ref query selection)
+ (mbql/resolve-aggregation-selection query selection)))))
+
+(deftest expression-ref-or-current-stage-column-uses-existing-expression-test
+ (let [query (-> (tu/query-for-table :orders)
+ (lib/expression "Net Amount"
+ (lib/- (meta/field-metadata :orders :total)
+ (meta/field-metadata :orders :discount))))]
+ (is (= (comparable-ref (lib/expression-ref query "Net Amount"))
+ (comparable-ref (mbql/expression-ref-or-current-stage-column query "Net Amount"))))))
+
+(deftest resolve-orderable-reuses-breakout-expression-test
+ (let [query (-> (tu/query-for-table :orders)
+ (lib/expression "__breakout_expression_1"
+ (lib/get-hour (meta/field-metadata :orders :created-at)))
+ (lib/aggregate (lib/count))
+ (as-> query
+ (lib/breakout query (lib/expression-ref query "__breakout_expression_1"))))
+ [_ resolved-orderable] (mbql/resolve-orderable query (meta/field-metadata :orders :created-at))]
+ (is (= (comparable-ref (lib/expression-ref query "__breakout_expression_1"))
+ (comparable-ref resolved-orderable)))))
+
+(deftest redundant-implicit-join-detects-existing-implicit-join-test
+ (let [fields-by-id (:fields-by-id (runtime/build-runtime meta/metadata-provider))
+ base-query (tu/query-for-table :orders)
+ query (lib/with-fields base-query [(current-query-field base-query :products :title)])
+ operation ["join"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :products)]]
+ [["=" ["field" (meta/id :orders :product-id)]
+ ["field" (meta/id :products :id)]]]]]]
+ (testing "implicit join coverage is centralized in mbql-integration"
+ (is (true? (mbql/redundant-implicit-join? fields-by-id query operation))))))
diff --git a/test/metabase/agent_lib/repair/context/passes_test.clj b/test/metabase/agent_lib/repair/context/passes_test.clj
new file mode 100644
index 00000000000..b10502dbd94
--- /dev/null
+++ b/test/metabase/agent_lib/repair/context/passes_test.clj
@@ -0,0 +1,64 @@
+(ns metabase.agent-lib.repair.context.passes-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.repair.context.passes :as passes]
+ [metabase.agent-lib.test-util :as agent-lib.tu]
+ [metabase.lib.test-metadata :as meta]))
+
+(deftest ^:parallel effective-source-for-context-rewrites-metric-context-source-test
+ (let [program {:source {:type "context" :ref "source"}}
+ context {:source-entity {:model "metric" :id 42}}]
+ (is (= {:type "metric" :id 42}
+ (passes/effective-source-for-context program context)))))
+
+(deftest ^:parallel rewrite-source-stage-only-touches-the-first-stage-test
+ (let [operations [["aggregate" ["count"]]
+ ["append-stage"]
+ ["aggregate" ["sum" ["field" 1]]]]
+ rewritten (passes/rewrite-source-stage
+ operations
+ (fn [{:keys [in-source-stage? operations] :as state} operation]
+ (assoc state
+ :operations
+ (conj operations
+ (if in-source-stage?
+ ["seen" operation]
+ operation)))))]
+ (is (= [["seen" ["aggregate" ["count"]]]
+ ["append-stage"]
+ ["aggregate" ["sum" ["field" 1]]]]
+ rewritten))))
+
+(deftest ^:parallel normalize-source-metric-quarter-window-rewrites-rolling-quarter-filter-test
+ (let [context {:source-entity {:model "metric" :id 42}}
+ operation ["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -3 "month"]
+ ["now"]]]
+ normalized (passes/normalize-source-metric-quarter-window context operation)]
+ (is (= ["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -1 "quarter"]
+ ["relative-datetime" 0 "quarter"]]]
+ normalized))))
+
+(deftest ^:parallel remove-redundant-operations-drops-source-metric-and-implicit-join-noise-test
+ (let [metric-context {:source-entity {:model "metric" :id 7}}
+ table-context {:source-entity {:model "table" :id (meta/id :venues)}
+ :surrounding-tables [(agent-lib.tu/entity-summary :categories)]
+ :join-edges [(agent-lib.tu/join-edge :venues :category-id :categories :id)]}
+ metric-aggregate ["aggregate" ["metric" 7]]
+ explicit-join ["join"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ [["=" ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]]]
+ kept-operation ["filter" ["=" ["field" (meta/id :venues :name)] "Red Medicine"]]]
+ (testing "source-metric reuse is dropped in metric contexts"
+ (is (= [kept-operation]
+ (vec (passes/remove-redundant-operations metric-context
+ [metric-aggregate kept-operation])))))
+ (testing "implicit joins already represented by the context are dropped for table sources"
+ (is (= [kept-operation]
+ (vec (passes/remove-redundant-operations table-context
+ [explicit-join kept-operation])))))))
diff --git a/test/metabase/agent_lib/repair/normalize/forms_test.clj b/test/metabase/agent_lib/repair/normalize/forms_test.clj
new file mode 100644
index 00000000000..2727abbad05
--- /dev/null
+++ b/test/metabase/agent_lib/repair/normalize/forms_test.clj
@@ -0,0 +1,137 @@
+(ns metabase.agent-lib.repair.normalize.forms-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.repair.normalize.forms :as forms]))
+
+(deftest ^:parallel expression-definition-tuple-test
+ (testing "recognizes expression definitions"
+ (is (true? (forms/expression-definition-tuple? ["expression" "Net Total" ["-" ["field" 1] 10]]))))
+ (testing "rejects wrong operator"
+ (is (false? (forms/expression-definition-tuple? ["filter" "Name" ["-" 1 2]]))))
+ (testing "rejects wrong arity"
+ (is (false? (forms/expression-definition-tuple? ["expression" "Name"])))
+ (is (false? (forms/expression-definition-tuple? ["expression" "Name" 1 2]))))
+ (testing "rejects blank name"
+ (is (false? (forms/expression-definition-tuple? ["expression" "" ["-" 1 2]]))))
+ (testing "rejects non-vectors"
+ (is (false? (forms/expression-definition-tuple? "expression")))))
+
+(deftest ^:parallel temporal-expression-test
+ (testing "now literal is temporal"
+ (is (true? (forms/temporal-expression? "now")))
+ (is (true? (forms/temporal-expression? "NOW"))))
+ (testing "temporal tuples are temporal"
+ (is (true? (forms/temporal-expression? ["absolute-datetime" "2024-01-01" "day"])))
+ (is (true? (forms/temporal-expression? ["relative-datetime" -3 "month"])))
+ (is (true? (forms/temporal-expression? ["now"]))))
+ (testing "non-temporal values"
+ (is (false? (forms/temporal-expression? "hello")))
+ (is (false? (forms/temporal-expression? 42)))
+ (is (false? (forms/temporal-expression? ["field" 1])))))
+
+(deftest ^:parallel repair-case-args-three-bare-args-test
+ (testing "three bare args become branch pair + fallback"
+ (is (= [[["pred" "then"]] "else"]
+ (forms/repair-case-args ["pred" "then" "else"])))))
+
+(deftest ^:parallel repair-case-args-wrapped-branch-pairs-test
+ (testing "single wrapped vector of branch pairs"
+ (is (= [[[[">" 1] "a"] [["<" 0] "b"]]]
+ (forms/repair-case-args [[[">" 1] "a"] [["<" 0] "b"]])))))
+
+(deftest ^:parallel repair-case-args-wrapped-branch-pairs-with-fallback-test
+ (testing "wrapped branches with fallback"
+ (is (= [[[[">" 1] "a"]] "default"]
+ (forms/repair-case-args [[[">" 1] "a"] "default"])))))
+
+(deftest ^:parallel repair-case-args-leading-branch-pairs-test
+ (testing "leading branch pairs followed by fallback"
+ (is (= [[[[">" 1] "a"]] "default"]
+ (forms/repair-case-args [[[">" 1] "a"] "default"])))))
+
+(deftest ^:parallel repair-case-args-flat-even-vector-test
+ (testing "flat even-length vector inside wrapper"
+ (is (= [[[1 2] [3 4]]]
+ (forms/repair-case-args [[1 2 3 4]])))))
+
+(deftest ^:parallel repair-case-args-else-branch-test
+ (testing "else branch is split out"
+ (is (= [[[[">" 1] "a"]] "fallback"]
+ (forms/repair-case-args [[[">" 1] "a"] ["else" "fallback"]])))))
+
+(deftest ^:parallel repair-between-bounds-test
+ (testing "wraps ISO date when other bound is temporal"
+ (is (= [["field" 1] ["relative-datetime" -3 "month"] ["absolute-datetime" "2025-09-30" "day"]]
+ (forms/repair-between-bounds
+ [["field" 1] ["relative-datetime" -3 "month"] "2025-09-30"]))))
+ (testing "wraps now literal into expression"
+ (is (= [["field" 1] ["relative-datetime" -3 "month"] ["now"]]
+ (forms/repair-between-bounds
+ [["field" 1] ["relative-datetime" -3 "month"] "now"]))))
+ (testing "leaves non-temporal bounds unchanged"
+ (is (= [["field" 1] 10 20]
+ (forms/repair-between-bounds [["field" 1] 10 20])))))
+
+(deftest ^:parallel repair-operator-form-field-coercion-test
+ (testing "coerces field id string to integer"
+ (is (= ["field" 42]
+ (forms/repair-operator-form identity ["field" "42"])))))
+
+(deftest ^:parallel repair-operator-form-is-null-test
+ (testing "is with null becomes is-null"
+ (is (= ["is-null" ["field" 1]]
+ (forms/repair-operator-form identity ["is" ["field" 1] "null"]))))
+ (testing "is with non-null becomes ="
+ (is (= ["=" ["field" 1] "hello"]
+ (forms/repair-operator-form identity ["is" ["field" 1] "hello"])))))
+
+(deftest ^:parallel repair-operator-form-is-not-null-test
+ (testing "is-not with null becomes not-null"
+ (is (= ["not-null" ["field" 1]]
+ (forms/repair-operator-form identity ["is-not" ["field" 1] nil]))))
+ (testing "is-not with non-null becomes !="
+ (is (= ["!=" ["field" 1] "hello"]
+ (forms/repair-operator-form identity ["is-not" ["field" 1] "hello"])))))
+
+(deftest ^:parallel repair-operator-form-eq-with-sequential-promotes-to-in-test
+ (testing "= with sequential RHS promotes to in"
+ (is (= ["in" ["field" 1] ["a" "b"]]
+ (forms/repair-operator-form identity ["=" ["field" 1] ["a" "b"]])))))
+
+(deftest ^:parallel repair-operator-form-neq-with-sequential-promotes-to-not-in-test
+ (testing "!= with sequential RHS promotes to not-in"
+ (is (= ["not-in" ["field" 1] ["a" "b"]]
+ (forms/repair-operator-form identity ["!=" ["field" 1] ["a" "b"]])))))
+
+(deftest ^:parallel repair-operator-form-contains-strips-boolean-flag-test
+ (testing "contains strips trailing boolean option"
+ (is (= ["contains" ["field" 1] "text"]
+ (forms/repair-operator-form identity ["contains" ["field" 1] "text" false]))))
+ (testing "contains strips trailing map option"
+ (is (= ["contains" ["field" 1] "text"]
+ (forms/repair-operator-form identity ["contains" ["field" 1] "text" {:case-sensitive false}])))))
+
+(deftest ^:parallel repair-operator-form-with-temporal-bucket-extraction-alias-test
+ (testing "with-temporal-bucket rewrites extraction aliases"
+ (is (= ["get-hour" ["field" 1]]
+ (forms/repair-operator-form identity ["with-temporal-bucket" ["field" 1] "hour-of-day"])))
+ (is (= ["get-day-of-week" ["field" 1]]
+ (forms/repair-operator-form identity ["with-temporal-bucket" ["field" 1] "day-of-week"]))))
+ (testing "non-extraction buckets pass through"
+ (is (= ["with-temporal-bucket" ["field" 1] "month"]
+ (forms/repair-operator-form identity ["with-temporal-bucket" ["field" 1] "month"])))))
+
+(deftest ^:parallel repair-operator-form-percentile-test
+ (testing "normalizes percentile scale"
+ (is (= ["percentile" ["field" 1] 0.9]
+ (forms/repair-operator-form identity ["percentile" ["field" 1] 90])))))
+
+(deftest ^:parallel repair-operator-form-case-from-if-test
+ (testing "if alias normalizes to case with three args"
+ (is (= ["case" [["> 1" "then"]] "else"]
+ (forms/repair-operator-form identity ["if" "> 1" "then" "else"])))))
+
+(deftest ^:parallel repair-operator-form-passthrough-test
+ (testing "unrecognized-but-valid operators pass through"
+ (is (= ["+" 1 2]
+ (forms/repair-operator-form identity ["+" 1 2])))))
diff --git a/test/metabase/agent_lib/repair/normalize/top_level_test.clj b/test/metabase/agent_lib/repair/normalize/top_level_test.clj
new file mode 100644
index 00000000000..56e6ef2b657
--- /dev/null
+++ b/test/metabase/agent_lib/repair/normalize/top_level_test.clj
@@ -0,0 +1,27 @@
+(ns metabase.agent-lib.repair.normalize.top-level-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.repair.normalize.top-level :as top-level]))
+
+(deftest ^:parallel normalize-with-fields-selection-flattens-single-sequential-arg-test
+ (is (= [["field" 1] ["field" 2]]
+ (top-level/normalize-with-fields-selection [[["field" 1] ["field" 2]]]))))
+
+(deftest ^:parallel repair-order-by-operations-normalizes-both-order-by-encodings-test
+ (is (= [["order-by" ["desc" ["field" 1]]]
+ ["order-by" ["field" 2] "asc"]]
+ (top-level/repair-order-by-operations [["desc" ["field" 1]]
+ ["field" 2]
+ "asc"]))))
+
+(deftest ^:parallel repair-with-fields-operation-extracts-inline-expressions-test
+ (is (= [["expression" "Net" ["-" ["field" 1] 10]]
+ ["with-fields" [["field" 1] ["expression-ref" "Net"]]]]
+ (top-level/repair-with-fields-operation identity
+ [[["field" 1]
+ ["expression" "Net" ["-" ["field" 1] 10]]]]))))
+
+(deftest ^:parallel repair-top-level-operation-rewrites-aggregation-expressions-test
+ (is (= [["aggregate" ["count"]]]
+ (top-level/repair-top-level-operation identity
+ ["expression" "Total" ["count"]]))))
diff --git a/test/metabase/agent_lib/repair/stages_test.clj b/test/metabase/agent_lib/repair/stages_test.clj
new file mode 100644
index 00000000000..c3d2a9323ad
--- /dev/null
+++ b/test/metabase/agent_lib/repair/stages_test.clj
@@ -0,0 +1,23 @@
+(ns metabase.agent-lib.repair.stages-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.repair.stages :as repair.stages]
+ [metabase.lib.test-metadata :as meta]))
+
+(deftest insert-stage-boundaries-adds-post-aggregation-stage-test
+ (let [source {:type "context" :ref "source"}
+ operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["expression" "Revenue K" ["/" ["aggregation-ref" 0] 1000]]]
+ expected [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["append-stage"]
+ ["expression" "Revenue K" ["/" ["aggregation-ref" 0] 1000]]]]
+ (is (= expected
+ (repair.stages/insert-stage-boundaries source operations)))))
+
+(deftest insert-stage-boundaries-respects-existing-boundaries-test
+ (let [source {:type "context" :ref "source"}
+ operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["append-stage"]
+ ["filter" [">" ["aggregation-ref" 0] 100]]]]
+ (is (= operations
+ (repair.stages/insert-stage-boundaries source operations)))))
diff --git a/test/metabase/agent_lib/repair_test.clj b/test/metabase/agent_lib/repair_test.clj
new file mode 100644
index 00000000000..508a74ef1c5
--- /dev/null
+++ b/test/metabase/agent_lib/repair_test.clj
@@ -0,0 +1,199 @@
+(ns metabase.agent-lib.repair-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.repair :as repair]
+ [metabase.lib.test-metadata :as meta]))
+
+(deftest ^:parallel repair-program-normalization-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"] ["variance" ["field" (meta/id :orders :total)]]]
+ ["with-fields" [["field" (meta/id :orders :id)]
+ ["expression" "Net Total" ["-" ["field" (meta/id :orders :total)] 10]]]]
+ ["filter" ["is-not" ["field" (meta/id :orders :total)] "null"]]
+ ["filter" ["=" ["field" (meta/id :orders :product-id)] ["1" "2"]]]
+ ["breakout"
+ ["with-temporal-bucket" ["field" (meta/id :orders :created-at)] "day-of-week"]]
+ ["order-by" ["field" (meta/id :orders :created-at)]
+ ["desc" ["field" (meta/id :orders :total)]]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["aggregate" ["var" ["field" (meta/id :orders :total)]]]
+ ["expression" "Net Total" ["-" ["field" (meta/id :orders :total)] 10]]
+ ["with-fields" [["field" (meta/id :orders :id)]
+ ["expression-ref" "Net Total"]]]
+ ["filter" ["not-null" ["field" (meta/id :orders :total)]]]
+ ["filter" ["in" ["field" (meta/id :orders :product-id)] ["1" "2"]]]
+ ["breakout" ["get-day-of-week" ["field" (meta/id :orders :created-at)]]]
+ ["order-by" ["field" (meta/id :orders :created-at)]]
+ ["order-by" ["desc" ["field" (meta/id :orders :total)]]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-quarter-and-diff-normalization-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["="
+ ["with-temporal-bucket"
+ ["field" (meta/id :orders :created-at)] "quarter"]
+ "Q1"]]
+ ["breakout"
+ ["with-temporal-bucket"
+ ["field" (meta/id :orders :created-at)] "hour-of-day"]]
+ ["expression" "Quarter Number"
+ ["quarter-of-year" ["field" (meta/id :orders :created-at)]]]
+ ["expression" "Days Open"
+ ["temporal-diff" ["now"]
+ ["field" (meta/id :orders :created-at)] "day"]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["filter" ["="
+ ["with-temporal-bucket"
+ ["field" (meta/id :orders :created-at)] "quarter"]
+ 1]]
+ ["breakout"
+ ["get-hour" ["field" (meta/id :orders :created-at)]]]
+ ["expression" "Quarter Number"
+ ["get-quarter" ["field" (meta/id :orders :created-at)]]]
+ ["expression" "Days Open"
+ ["datetime-diff" ["now"]
+ ["field" (meta/id :orders :created-at)] "day"]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-normalizes-if-and-between-bounds-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -3 "month"]
+ "2025-09-30"]]
+ ["expression" "Accepted Count"
+ ["if" ["=" ["field" (meta/id :orders :user-id)] 1] 1 0]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -3 "month"]
+ ["absolute-datetime" "2025-09-30" "day"]]]
+ ["expression" "Accepted Count"
+ ["case" [[["=" ["field" (meta/id :orders :user-id)] 1] 1]] 0]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-normalizes-now-bounds-case-ternaries-and-text-filter-flags-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["contains" ["field" (meta/id :orders :product-id)] "friend" false]]
+ ["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -3 "month"]
+ "now"]]
+ ["expression" "Accepted Count"
+ ["case" [["true" ["field" (meta/id :orders :user-id)]]
+ 1
+ 0]]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["filter" ["contains" ["field" (meta/id :orders :product-id)] "friend"]]
+ ["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -3 "month"]
+ ["now"]]]
+ ["expression" "Accepted Count"
+ ["case" [[["field" (meta/id :orders :user-id)] 1]] 0]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-unwraps-join-condition-wrapper-test
+ (let [wrapped-conditions [[["="
+ ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]]
+ expected-conditions [["="
+ ["field" (meta/id :venues :category-id)]
+ ["field" (meta/id :categories :id)]]]
+ program {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ wrapped-conditions]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["join"
+ ["with-join-conditions"
+ ["join-clause" ["table" (meta/id :categories)]]
+ expected-conditions]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-rewrites-aggregation-expression-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Total Opportunities" ["count"]]
+ ["expression" "Won" ["count-if" ["field" (meta/id :orders :discount)]]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]
+ ["aggregate" ["count-where" ["field" (meta/id :orders :discount)]]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-drops-stray-bare-aggregation-ref-in-aggregate-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"] ["aggregation-ref" 0]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["count"]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-normalizes-percentile-scale-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["percentile" ["field" (meta/id :orders :total)] 90]]]}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["percentile" ["field" (meta/id :orders :total)] 0.9]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-inserts-append-stage-before-aggregate-on-aggregation-ref-test
+ (let [program {:source {:type "program"
+ :program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["field" (meta/id :orders :created-at)]]]}}
+ :operations [["aggregate" ["stddev" ["aggregation-ref" 0]]]]}
+ expected {:source {:type "program"
+ :program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["sum" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["field" (meta/id :orders :created-at)]]]}}
+ :operations [["append-stage"]
+ ["aggregate" ["stddev" ["aggregation-ref" 0]]]]}]
+ (is (= expected (repair/repair-program program)))))
+
+(deftest ^:parallel repair-program-for-context-drops-source-stage-aggregates-for-metric-sources-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["field" (meta/id :orders :total)]]]
+ ["breakout" ["field" (meta/id :orders :product-id)]]
+ ["append-stage"]
+ ["aggregate" ["avg" ["aggregation-ref" 0]]]]}
+ context {:source-entity {:model "metric" :id 42}
+ :referenced-entities []
+ :surrounding-tables []
+ :join-edges []}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["breakout" ["field" (meta/id :orders :product-id)]]
+ ["append-stage"]
+ ["aggregate" ["avg" ["aggregation-ref" 0]]]]}]
+ (is (= expected (repair/repair-program-for-context program context)))))
+
+(deftest ^:parallel repair-program-for-context-drops-source-stage-with-fields-over-aggregation-ref-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["=" ["field" (meta/id :products :category)] "Gizmo"]]
+ ["with-fields" [["field" (meta/id :products :category)]
+ ["aggregation-ref" 0]]]]}
+ context {:source-entity {:model "metric" :id 83}
+ :referenced-entities []
+ :surrounding-tables []
+ :measure-ids []}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["filter" ["=" ["field" (meta/id :products :category)] "Gizmo"]]]}]
+ (is (= expected (repair/repair-program-for-context program context)))))
+
+(deftest ^:parallel repair-program-for-context-normalizes-last-quarter-metric-window-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -3 "month"]
+ "now"]]
+ ["aggregate" ["count"]]]}
+ context {:source-entity {:model "metric" :id 42}
+ :referenced-entities []
+ :surrounding-tables []
+ :measure-ids []}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["filter" ["between"
+ ["field" (meta/id :orders :created-at)]
+ ["relative-datetime" -1 "quarter"]
+ ["relative-datetime" 0 "quarter"]]]]}]
+ (is (= expected
+ (repair/repair-program-for-context (repair/repair-program program) context)))))
diff --git a/test/metabase/agent_lib/runtime/fields_test.clj b/test/metabase/agent_lib/runtime/fields_test.clj
new file mode 100644
index 00000000000..25daa5b58de
--- /dev/null
+++ b/test/metabase/agent_lib/runtime/fields_test.clj
@@ -0,0 +1,145 @@
+(ns metabase.agent-lib.runtime.fields-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.mbql-integration :as mbql]
+ [metabase.agent-lib.runtime.fields :as runtime.fields]))
+
+(deftest resolve-field-in-query-prefers-chained-lineage-over-raw-exact-match-test
+ (let [raw-field {:id 300 :table-id 30 :name "name" :base-type :type/Text}
+ raw-exact {:id 300 :table-id 30 :name "name" :base-type :type/Text}
+ source-column {:id 200 :table-id 20 :name "customer_id" :base-type :type/Integer}
+ fields-by-id {200 {:id 200 :table-id 20 :fk-target-field-id 301}
+ 301 {:id 301 :table-id 30}}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [raw-exact source-column])]
+ (runtime.fields/resolve-field-in-query fields-by-id ::query raw-field))]
+ (is (= 300 (:id resolved-field)))
+ (is (= 200 (:fk-field-id resolved-field)))
+ (is (= "customer_id" (:fk-field-name resolved-field)))))
+
+(deftest resolve-field-in-query-preserves-upstream-lineage-when-chaining-related-fields-test
+ (let [raw-field {:id 2455 :table-id 127 :name "first_name" :base-type :type/Text}
+ source-column {:id 2488
+ :table-id 133
+ :name "customer_id"
+ :lib/source-column-alias "customer_id"
+ :base-type :type/Integer
+ :fk-field-id 2503}
+ fields-by-id {2503 {:id 2503 :table-id 116 :name "order_id" :fk-target-field-id 2499}
+ 2499 {:id 2499 :table-id 133 :name "id"}
+ 2488 {:id 2488 :table-id 133 :name "customer_id" :fk-target-field-id 2456}
+ 2456 {:id 2456 :table-id 127 :name "id"}}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [source-column])]
+ (runtime.fields/resolve-field-in-query fields-by-id ::query raw-field))]
+ (is (= 2455 (:id resolved-field)))
+ (is (= 2488 (:fk-field-id resolved-field)))
+ (is (= "customer_id" (:fk-field-name resolved-field)))
+ (is (= 2503 (:lib/original-fk-field-id resolved-field)))))
+
+(deftest resolve-field-in-query-preserves-original-source-field-lineage-when-chaining-related-fields-test
+ (let [raw-field {:id 2455 :table-id 127 :name "first_name" :base-type :type/Text}
+ source-column {:id 2488
+ :table-id 133
+ :name "customer_id"
+ :lib/source-column-alias "customer_id"
+ :base-type :type/Integer
+ :source-field 2503
+ :source-field-name "order_id"}
+ fields-by-id {2488 {:id 2488 :table-id 133 :fk-target-field-id 2456}
+ 2456 {:id 2456 :table-id 127}}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [source-column])]
+ (runtime.fields/resolve-field-in-query fields-by-id ::query raw-field))]
+ (is (= 2455 (:id resolved-field)))
+ (is (= 2488 (:fk-field-id resolved-field)))
+ (is (= "customer_id" (:fk-field-name resolved-field)))
+ (is (= 2503 (:lib/original-fk-field-id resolved-field)))
+ (is (= "order_id" (:lib/original-fk-field-name resolved-field)))))
+
+(deftest resolve-field-in-query-preserves-upstream-join-alias-when-chaining-related-fields-test
+ (let [raw-field {:id 2455 :table-id 127 :name "first_name" :base-type :type/Text}
+ source-column {:id 2488
+ :table-id 133
+ :name "customer_id"
+ :base-type :type/Integer
+ :fk-field-id 2503
+ :fk-join-alias "order__via__order_id"}
+ fields-by-id {2503 {:id 2503 :table-id 116 :fk-target-field-id 2499}
+ 2499 {:id 2499 :table-id 133}
+ 2488 {:id 2488 :table-id 133 :fk-target-field-id 2456}
+ 2456 {:id 2456 :table-id 127}}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [source-column])]
+ (runtime.fields/resolve-field-in-query fields-by-id ::query raw-field))]
+ (is (= "order__via__order_id" (:fk-join-alias resolved-field)))
+ (is (= "order__via__order_id" (:lib/original-fk-join-alias resolved-field)))))
+
+(deftest resolve-field-in-query-prefers-chained-lineage-over-exact-partial-related-field-test
+ (let [raw-field {:id 2455 :table-id 127 :name "first_name" :base-type :type/Text}
+ exact-partial {:id 2455
+ :table-id 127
+ :name "first_name"
+ :base-type :type/Text
+ :source-field 2488
+ :source-field-name "customer_id"}
+ source-column {:id 2488
+ :table-id 133
+ :name "customer_id"
+ :base-type :type/Integer
+ :source-field 2503
+ :source-field-name "order_id"
+ :fk-join-alias "order__via__order_id"}
+ fields-by-id {2503 {:id 2503 :table-id 116 :name "order_id" :fk-target-field-id 2499}
+ 2499 {:id 2499 :table-id 133 :name "id"}
+ 2488 {:id 2488 :table-id 133 :name "customer_id" :fk-target-field-id 2456}
+ 2456 {:id 2456 :table-id 127 :name "id"}}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [exact-partial source-column])]
+ (runtime.fields/resolve-field-in-query fields-by-id ::query raw-field))]
+ (is (= 2455 (:id resolved-field)))
+ (is (= 2488 (:fk-field-id resolved-field)))
+ (is (= "customer_id" (:fk-field-name resolved-field)))
+ (is (= 2503 (:lib/original-fk-field-id resolved-field)))
+ (is (= "order_id" (:lib/original-fk-field-name resolved-field)))
+ (is (= "order__via__order_id" (:fk-join-alias resolved-field)))
+ (is (= "order__via__order_id" (:lib/original-fk-join-alias resolved-field)))))
+
+(deftest resolve-field-in-query-prefers-previous-stage-column-over-raw-exact-match-test
+ (let [raw-field {:id 282 :table-id 88 :name "utilization_percentage" :base-type :type/Decimal}
+ raw-exact {:id 282 :table-id 88 :name "utilization_percentage" :base-type :type/Decimal}
+ previous-stage {:name "utilization_percentage"
+ :base-type :type/Decimal
+ :effective-type :type/Decimal
+ :lib/source :source/previous-stage}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [raw-exact previous-stage])]
+ (runtime.fields/resolve-field-in-query {} ::query raw-field))]
+ (is (= previous-stage resolved-field))))
+
+(deftest resolve-field-in-query-prefers-previous-stage-name-match-test
+ (let [raw-field {:id 468 :table-id 21 :name "name" :base-type :type/Text}
+ previous-event-uri {:name "event_uri"
+ :base-type :type/Text
+ :effective-type :type/Text
+ :lib/source :source/previous-stage}
+ previous-name {:name "name"
+ :base-type :type/Text
+ :effective-type :type/Text
+ :lib/source :source/previous-stage}
+ resolved-field (with-redefs [mbql/current-query-field-candidates
+ (constantly [previous-event-uri previous-name])]
+ (runtime.fields/resolve-field-in-query {} ::query raw-field))]
+ (is (= previous-name resolved-field))))
+
+(deftest resolve-field-in-query-rejects-unrelated-previous-stage-numeric-column-test
+ (let [raw-field {:id 282 :table-id 88 :name "total" :base-type :type/Decimal}
+ previous-stage {:name "count"
+ :base-type :type/Integer
+ :effective-type :type/Integer
+ :lib/source :source/previous-stage}]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"not available in the current query stage"
+ (with-redefs [mbql/current-query-field-candidates
+ (constantly [previous-stage])]
+ (runtime.fields/resolve-field-in-query {} ::query raw-field))))))
diff --git a/test/metabase/agent_lib/runtime/lookup_test.clj b/test/metabase/agent_lib/runtime/lookup_test.clj
new file mode 100644
index 00000000000..8ba9a5708e9
--- /dev/null
+++ b/test/metabase/agent_lib/runtime/lookup_test.clj
@@ -0,0 +1,71 @@
+(ns metabase.agent-lib.runtime.lookup-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.runtime.bindings :as runtime.bindings]
+ [metabase.agent-lib.runtime.fields :as runtime.fields]
+ [metabase.agent-lib.runtime.lookup :as runtime.lookup]
+ [metabase.agent-lib.test-util :as agent-lib.tu]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-metadata :as meta]))
+
+(defn- lookup-env
+ []
+ (let [metadata-provider meta/metadata-provider
+ all-table-ids (map :id (lib.metadata/tables metadata-provider))
+ tables-by-name (runtime.fields/build-table-lookup metadata-provider all-table-ids)
+ {:keys [fields-by-table fields-by-id]}
+ (runtime.fields/build-field-lookups metadata-provider tables-by-name)]
+ {:metadata-provider metadata-provider
+ :tables-by-name tables-by-name
+ :fields-by-table fields-by-table
+ :fields-by-id fields-by-id}))
+
+(deftest ^:parallel lookup-table-resolves-by-id-and-name-test
+ (let [{:keys [metadata-provider tables-by-name]} (lookup-env)
+ by-name (runtime.lookup/lookup-table metadata-provider tables-by-name "ORDERS")
+ by-id (runtime.lookup/lookup-table metadata-provider tables-by-name (:id by-name))]
+ (is (= (:id by-name) (:id by-id)))
+ (is (= (meta/id :orders) (:id by-name)))))
+
+(deftest ^:parallel lookup-field-rejects-ambiguous-name-test
+ (let [{:keys [metadata-provider tables-by-name fields-by-table fields-by-id]} (lookup-env)]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Ambiguous field"
+ (runtime.lookup/lookup-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ "id")))))
+
+(deftest ^:parallel query-relative-field-prefers-current-query-columns-test
+ (let [{:keys [metadata-provider tables-by-name fields-by-table fields-by-id]} (lookup-env)
+ query (-> (agent-lib.tu/query-for-table :orders)
+ (lib/breakout (meta/field-metadata :orders :product-id)))
+ raw-field (meta/field-metadata :orders :product-id)
+ resolved (runtime.lookup/query-relative-field metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id
+ query
+ "orders"
+ "PRODUCT_ID")]
+ (is (= (:id raw-field) (:id resolved)))
+ (is (:lib/source resolved))))
+
+(deftest ^:parallel make-context-bindings-composes-lookup-and-transform-helpers-test
+ (let [{:keys [metadata-provider tables-by-name fields-by-table fields-by-id]} (lookup-env)
+ bindings (runtime.bindings/make-context-bindings metadata-provider
+ tables-by-name
+ fields-by-table
+ fields-by-id)]
+ (testing "lookup bindings"
+ (is (= (meta/id :orders)
+ (:id ((get bindings 'table) "orders"))))
+ (is (= (meta/id :orders :id)
+ (:id ((get bindings 'field) "orders" "id")))))
+ (testing "transform bindings"
+ (is (contains? bindings 'breakout))
+ (is (contains? bindings 'order-by))
+ (is (contains? bindings 'query)))))
diff --git a/test/metabase/agent_lib/runtime/transforms_test.clj b/test/metabase/agent_lib/runtime/transforms_test.clj
new file mode 100644
index 00000000000..90d8e83c2c1
--- /dev/null
+++ b/test/metabase/agent_lib/runtime/transforms_test.clj
@@ -0,0 +1,52 @@
+(ns metabase.agent-lib.runtime.transforms-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.runtime.transforms :as runtime.transforms]
+ [metabase.agent-lib.test-util :as agent-lib.tu]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.util :as lib.util]))
+
+(deftest ^:parallel order-wrapper-helpers-test
+ (is (runtime.transforms/order-wrapper? (runtime.transforms/asc-orderable :x)))
+ (is (runtime.transforms/order-wrapper? (runtime.transforms/desc-orderable :x)))
+ (is (not (runtime.transforms/order-wrapper? :x))))
+
+(deftest ^:parallel apply-breakout-materializes-expression-ref-test
+ (let [query (agent-lib.tu/query-for-table :orders)
+ expr (lib/get-hour (meta/field-metadata :orders :created-at))
+ expected (-> query
+ (lib/expression "__breakout_expression_1" expr)
+ ((fn [query']
+ (lib/breakout query' (lib/expression-ref query' "__breakout_expression_1")))))
+ actual (runtime.transforms/apply-breakout query expr)]
+ (is (= (agent-lib.tu/comparable-query expected)
+ (agent-lib.tu/comparable-query actual)))))
+
+(deftest ^:parallel apply-with-fields-breaks-out-raw-fields-on-aggregated-query-test
+ (let [query (-> (agent-lib.tu/query-for-table :orders)
+ (lib/aggregate (lib/count)))
+ selection (meta/field-metadata :orders :id)
+ expected (lib/breakout query selection)
+ actual (runtime.transforms/apply-with-fields query [selection])]
+ (is (= (agent-lib.tu/comparable-query expected)
+ (agent-lib.tu/comparable-query actual)))))
+
+(deftest ^:parallel apply-order-by-supports-direction-wrappers-test
+ (let [query (agent-lib.tu/query-for-table :orders)
+ field (meta/field-metadata :orders :created-at)
+ expected (lib/order-by query (lib.util/fresh-uuids field) :desc)
+ actual (runtime.transforms/apply-order-by query
+ (runtime.transforms/desc-orderable field))]
+ (is (= (agent-lib.tu/comparable-query expected)
+ (agent-lib.tu/comparable-query actual)))))
+
+(deftest ^:parallel query-transform-bindings-expose-all-transform-helpers-test
+ (let [bindings (runtime.transforms/query-transform-bindings)]
+ (testing "all extracted transform helpers are reachable through the binding map"
+ (is (contains? bindings 'asc))
+ (is (contains? bindings 'desc))
+ (is (contains? bindings 'breakout))
+ (is (contains? bindings 'filter))
+ (is (contains? bindings 'with-fields))
+ (is (contains? bindings 'order-by)))))
diff --git a/test/metabase/agent_lib/schema/top_level_test.clj b/test/metabase/agent_lib/schema/top_level_test.clj
new file mode 100644
index 00000000000..9db47cc241a
--- /dev/null
+++ b/test/metabase/agent_lib/schema/top_level_test.clj
@@ -0,0 +1,30 @@
+(ns metabase.agent-lib.schema.top-level-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.capabilities.catalog.top-level :as capabilities.top-level]
+ [metabase.agent-lib.schema :as schema]))
+
+(def ^:private sample-operations
+ {"filter" ["filter" ["=" ["field" 1] 1]]
+ "aggregate" ["aggregate" ["count"]]
+ "breakout" ["breakout" ["field" 1]]
+ "with-fields" ["with-fields" [["field" 1]]]
+ "limit" ["limit" 10]
+ "expression" ["expression" "Net Amount" ["+" ["field" 1] 1]]
+ "join" ["join" ["join-clause" ["table" 2]]]
+ "order-by" ["order-by" ["field" 1] "desc"]
+ "append-stage" ["append-stage"]
+ "drop-stage" ["drop-stage"]
+ "drop-empty-stages" ["drop-empty-stages"]
+ "with-page" ["with-page" {:page 2 :items 25}]})
+
+(deftest ^:parallel top-level-capability-catalog-and-schema-samples-stay-in-sync-test
+ (let [capability-ops (mapv (comp name :op) capabilities.top-level/capabilities)]
+ (is (= (set capability-ops)
+ (set (keys sample-operations))))
+ (doseq [op-name capability-ops]
+ (testing op-name
+ (let [program {:source {:type "table" :id 1}
+ :operations [(get sample-operations op-name)]}]
+ (is (= program
+ (schema/validated-structure program))))))))
diff --git a/test/metabase/agent_lib/syntax_test.clj b/test/metabase/agent_lib/syntax_test.clj
new file mode 100644
index 00000000000..579d24ecb73
--- /dev/null
+++ b/test/metabase/agent_lib/syntax_test.clj
@@ -0,0 +1,103 @@
+(ns metabase.agent-lib.syntax-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.syntax :as syntax]))
+
+(deftest ^:parallel raw-op-name-test
+ (testing "returns string for strings"
+ (is (= "filter" (syntax/raw-op-name "filter"))))
+ (testing "returns name for keywords"
+ (is (= "filter" (syntax/raw-op-name :filter))))
+ (testing "returns name for symbols"
+ (is (= "filter" (syntax/raw-op-name 'filter))))
+ (testing "returns nil for other types"
+ (is (nil? (syntax/raw-op-name nil)))
+ (is (nil? (syntax/raw-op-name 42)))
+ (is (nil? (syntax/raw-op-name [])))))
+
+(deftest ^:parallel canonical-op-name-test
+ (testing "resolves aliases to canonical names"
+ (is (= "case" (syntax/canonical-op-name "if")))
+ (is (= "var" (syntax/canonical-op-name "variance")))
+ (is (= "relative-datetime" (syntax/canonical-op-name "relative-date")))
+ (is (= "datetime-diff" (syntax/canonical-op-name "temporal-diff")))
+ (is (= "count-where" (syntax/canonical-op-name "count-if")))
+ (is (= "distinct" (syntax/canonical-op-name "count-distinct")))
+ (is (= "distinct" (syntax/canonical-op-name "distinct-count")))
+ (is (= "not-null" (syntax/canonical-op-name "is-not-null")))
+ (is (= "get-day-of-week" (syntax/canonical-op-name "dayofweek")))
+ (is (= "get-day-of-week" (syntax/canonical-op-name "day-of-week")))
+ (is (= "get-hour" (syntax/canonical-op-name "hour-of-day")))
+ (is (= "get-month" (syntax/canonical-op-name "month-of-year")))
+ (is (= "get-quarter" (syntax/canonical-op-name "quarter-of-year"))))
+ (testing "passes through canonical names unchanged"
+ (is (= "filter" (syntax/canonical-op-name "filter")))
+ (is (= "count" (syntax/canonical-op-name "count"))))
+ (testing "works with keywords and symbols"
+ (is (= "case" (syntax/canonical-op-name :if)))
+ (is (= "var" (syntax/canonical-op-name 'variance))))
+ (testing "returns nil for nil"
+ (is (nil? (syntax/canonical-op-name nil)))))
+
+(deftest ^:parallel op-symbol-test
+ (testing "normalizes to canonical symbol"
+ (is (= 'case (syntax/op-symbol "if")))
+ (is (= 'var (syntax/op-symbol "variance")))
+ (is (= 'filter (syntax/op-symbol "filter"))))
+ (testing "works with keywords and symbols"
+ (is (= 'case (syntax/op-symbol :if)))
+ (is (= 'filter (syntax/op-symbol 'filter))))
+ (testing "returns nil for nil"
+ (is (nil? (syntax/op-symbol nil)))))
+
+(deftest ^:parallel possible-operator-tuple-test
+ (testing "true for recognized operator tuples"
+ (is (true? (syntax/possible-operator-tuple? ["filter" ["=" ["field" 1] 2]])))
+ (is (true? (syntax/possible-operator-tuple? ["count"])))
+ (is (true? (syntax/possible-operator-tuple? ["aggregate" ["count"]]))))
+ (testing "true for alias operators"
+ (is (true? (syntax/possible-operator-tuple? ["if" true 1 0])))
+ (is (true? (syntax/possible-operator-tuple? ["variance" ["field" 1]]))))
+ (testing "false for non-vectors"
+ (is (false? (syntax/possible-operator-tuple? "filter")))
+ (is (false? (syntax/possible-operator-tuple? {:op "filter"}))))
+ (testing "false for unrecognized operators"
+ (is (false? (syntax/possible-operator-tuple? ["bogus-op" 1 2])))))
+
+(deftest ^:parallel boolean-wrapper-form-test
+ (testing "true for boolean wrappers"
+ (is (true? (syntax/boolean-wrapper-form? ["true" ["field" 1]])))
+ (is (true? (syntax/boolean-wrapper-form? ["false" ["field" 1]])))
+ (is (true? (syntax/boolean-wrapper-form? ["TRUE" ["field" 1]]))))
+ (testing "false for wrong arity"
+ (is (false? (syntax/boolean-wrapper-form? ["true"])))
+ (is (false? (syntax/boolean-wrapper-form? ["true" 1 2]))))
+ (testing "false for non-boolean strings"
+ (is (false? (syntax/boolean-wrapper-form? ["maybe" ["field" 1]]))))
+ (testing "false for non-vectors"
+ (is (false? (syntax/boolean-wrapper-form? "true")))))
+
+(deftest ^:parallel top-level-operation-tuple-test
+ (testing "true for top-level operations"
+ (is (true? (syntax/top-level-operation-tuple? ["filter" ["=" ["field" 1] 2]])))
+ (is (true? (syntax/top-level-operation-tuple? ["aggregate" ["count"]])))
+ (is (true? (syntax/top-level-operation-tuple? ["breakout" ["field" 1]])))
+ (is (true? (syntax/top-level-operation-tuple? ["order-by" ["field" 1]])))
+ (is (true? (syntax/top-level-operation-tuple? ["limit" 10]))))
+ (testing "false for nested-only operators"
+ (is (false? (syntax/top-level-operation-tuple? ["=" ["field" 1] 2])))
+ (is (false? (syntax/top-level-operation-tuple? ["field" 1])))
+ (is (false? (syntax/top-level-operation-tuple? ["count"])))))
+
+(deftest ^:parallel operator-tuple-test
+ (testing "true for recognized string-led operator tuples"
+ (is (true? (syntax/operator-tuple? ["count"])))
+ (is (true? (syntax/operator-tuple? ["field" 1])))
+ (is (true? (syntax/operator-tuple? ["=" ["field" 1] 2]))))
+ (testing "false for non-string first element"
+ (is (false? (syntax/operator-tuple? [:count])))
+ (is (false? (syntax/operator-tuple? [42]))))
+ (testing "false for unrecognized operators"
+ (is (false? (syntax/operator-tuple? ["bogus" 1 2]))))
+ (testing "false for non-vectors"
+ (is (false? (syntax/operator-tuple? "count")))))
diff --git a/test/metabase/agent_lib/test_util.clj b/test/metabase/agent_lib/test_util.clj
new file mode 100644
index 00000000000..306e24b34f4
--- /dev/null
+++ b/test/metabase/agent_lib/test_util.clj
@@ -0,0 +1,110 @@
+(ns metabase.agent-lib.test-util
+ (:require
+ [metabase.lib.aggregation :as lib.aggregation]
+ [metabase.lib.core :as lib]
+ [metabase.lib.schema.util :as lib.schema.util]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.util :as lib.util]
+ [metabase.util :as u]))
+
+(defn entity-summary
+ [table-name]
+ {:model "table"
+ :id (meta/id table-name)
+ :columns (mapv (fn [field-name]
+ {:id (meta/id table-name field-name)})
+ (meta/fields table-name))})
+
+(defn table-context
+ [table-name & surrounding-table-names]
+ {:source-entity (entity-summary table-name)
+ :source-metadata (meta/table-metadata table-name)
+ :referenced-entities []
+ :referenced-metadata []
+ :surrounding-tables (mapv entity-summary surrounding-table-names)
+ :surrounding-table-metadata (mapv meta/table-metadata surrounding-table-names)
+ :join-edges []})
+
+(defn table-context-with-join-edges
+ [table-name surrounding-table-names join-edges]
+ {:source-entity (entity-summary table-name)
+ :source-metadata (meta/table-metadata table-name)
+ :referenced-entities []
+ :referenced-metadata []
+ :surrounding-tables (mapv entity-summary surrounding-table-names)
+ :surrounding-table-metadata (mapv meta/table-metadata surrounding-table-names)
+ :join-edges join-edges})
+
+(defn join-edge
+ [from-table from-field to-table to-field]
+ {:from_table_id (meta/id from-table)
+ :from_field_id (meta/id from-table from-field)
+ :to_table_id (meta/id to-table)
+ :to_field_id (meta/id to-table to-field)})
+
+(defn query-for-table
+ [table-name]
+ (lib/query meta/metadata-provider (meta/table-metadata table-name)))
+
+(defn comparable-query
+ [query]
+ {:database (:database query)
+ :legacy (-> query
+ lib/->legacy-MBQL
+ lib.schema.util/remove-lib-uuids)})
+
+(defn current-query-column
+ [query field-id]
+ (or (some #(when (= field-id (:id %)) %)
+ (distinct
+ (concat (lib/visible-columns query)
+ (lib/filterable-columns query)
+ (lib/breakoutable-columns query)
+ (lib/orderable-columns query))))
+ (throw (ex-info "Expected field to be available in query context"
+ {:field-id field-id
+ :query query}))))
+
+(defn current-query-column-by-name
+ [query column-name]
+ (let [normalized-column-name (u/lower-case-en column-name)]
+ (or (some (fn [column]
+ (when (contains? (->> [(:name column)
+ (:lib/source-column-alias column)
+ (:lib/ref-name column)]
+ (keep identity)
+ (map u/lower-case-en)
+ set)
+ normalized-column-name)
+ column))
+ (distinct
+ (concat (lib/returned-columns query)
+ (lib/visible-columns query)
+ (lib/filterable-columns query)
+ (lib/breakoutable-columns query)
+ (lib/orderable-columns query))))
+ (throw (ex-info "Expected named column to be available in query context"
+ {:column-name column-name
+ :query query})))))
+
+(defn previous-stage-aggregation-column
+ [query ag-index]
+ (let [previous-stage-number (lib.util/previous-stage-number query -1)
+ source-uuid (:lib/source-uuid
+ (get (vec (lib.aggregation/aggregations-metadata query previous-stage-number))
+ ag-index))
+ candidate-columns (concat (lib/filterable-columns query)
+ (lib/orderable-columns query)
+ (lib/visible-columns query))]
+ (or (some #(when (= source-uuid (:lib/source-uuid %)) %) candidate-columns)
+ (throw (ex-info "Expected a previous-stage aggregation column"
+ {:query query
+ :aggregation-index ag-index})))))
+
+(defn current-aggregation-orderable
+ [query ag-index]
+ (or (lib/find-matching-column query
+ -1
+ (lib/aggregation-ref query ag-index)
+ (lib/orderable-columns query))
+ (lib/aggregation-ref query ag-index)))
diff --git a/test/metabase/agent_lib/validate/cross_checks_test.clj b/test/metabase/agent_lib/validate/cross_checks_test.clj
new file mode 100644
index 00000000000..7e8bd08f511
--- /dev/null
+++ b/test/metabase/agent_lib/validate/cross_checks_test.clj
@@ -0,0 +1,58 @@
+(ns metabase.agent-lib.validate.cross-checks-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.validate.cross-checks :as cross-checks]))
+
+(deftest validate-no-metric-order-by-test
+ (testing "rejects metric helper inside order-by"
+ (let [error (try
+ (cross-checks/validate-no-metric-order-by!
+ [:operations 0]
+ ["order-by" ["metric" 42]])
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (re-find #"metric helpers cannot be used directly inside order-by"
+ (:details (ex-data error))))))
+ (testing "accepts field inside order-by"
+ (is (nil? (cross-checks/validate-no-metric-order-by!
+ [:operations 0]
+ ["order-by" ["field" 1]]))))
+ (testing "ignores non-order-by operations"
+ (is (nil? (cross-checks/validate-no-metric-order-by!
+ [:operations 0]
+ ["filter" ["metric" 42]])))))
+
+(deftest validate-no-source-metric-reuse-test
+ (testing "rejects reuse of source metric id"
+ (let [context {:source-entity {:model "metric" :id 42}
+ :referenced-entities []
+ :surrounding-tables []}
+ error (try
+ (cross-checks/validate-no-source-metric-reuse!
+ context
+ [:operations 0]
+ ["aggregate" ["metric" 42]])
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (re-find #"source is already metric 42"
+ (:details (ex-data error))))))
+ (testing "accepts different metric id"
+ (let [context {:source-entity {:model "metric" :id 42}
+ :referenced-entities []
+ :surrounding-tables []}]
+ (is (nil? (cross-checks/validate-no-source-metric-reuse!
+ context
+ [:operations 0]
+ ["aggregate" ["metric" 99]])))))
+ (testing "no-op for table sources"
+ (let [context {:source-entity {:model "table" :id 1}
+ :referenced-entities []
+ :surrounding-tables []}]
+ (is (nil? (cross-checks/validate-no-source-metric-reuse!
+ context
+ [:operations 0]
+ ["aggregate" ["metric" 42]]))))))
diff --git a/test/metabase/agent_lib/validate/operators_test.clj b/test/metabase/agent_lib/validate/operators_test.clj
new file mode 100644
index 00000000000..2b9ca9bcd24
--- /dev/null
+++ b/test/metabase/agent_lib/validate/operators_test.clj
@@ -0,0 +1,54 @@
+(ns metabase.agent-lib.validate.operators-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.validate.operators :as operators]))
+
+(deftest ensure-arity-rejects-invalid-expression-reference-arity-test
+ (try
+ (operators/ensure-arity! [:operations 0] 'expression 1)
+ (is false "expected invalid expression arity")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"top-level operation" (:details (ex-data e)))))))
+
+(deftest validate-operator-specific-rejects-bare-field-ids-in-aggregations-test
+ (try
+ (operators/validate-operator-specific! {:table-ids #{}
+ :card-ids #{}
+ :metric-ids #{}
+ :measure-ids #{}
+ :field-ids #{}}
+ {:source-entity {:model "table" :id 1}}
+ [:operations 0 1]
+ ["sum" 42])
+ (is false "expected bare field id validation failure")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (= :field-wrapper (:retry-category (ex-data e)))))))
+
+(deftest validate-operator-specific-rejects-source-metric-as-measure-test
+ (try
+ (operators/validate-operator-specific! {:table-ids #{}
+ :card-ids #{}
+ :metric-ids #{}
+ :measure-ids #{118}
+ :field-ids #{}}
+ {:source-entity {:model "metric" :id 118}}
+ [:operations 0 1]
+ ["measure" 118])
+ (is false "expected source-metric measure validation failure")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"Metric ids are not measure ids" (:details (ex-data e)))))))
+
+(deftest validate-top-level-operation-rejects-non-aggregation-aggregate-args-test
+ (try
+ (operators/validate-top-level-operation! (fn [path arg depth state]
+ (assoc state :visited [path arg depth]))
+ [:operations 0]
+ ["aggregate" ["field" 10]]
+ {:node-count 0})
+ (is false "expected aggregate validation failure")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"require aggregation helpers" (:details (ex-data e)))))))
diff --git a/test/metabase/agent_lib/validate/semantic_test.clj b/test/metabase/agent_lib/validate/semantic_test.clj
new file mode 100644
index 00000000000..52a05ac9184
--- /dev/null
+++ b/test/metabase/agent_lib/validate/semantic_test.clj
@@ -0,0 +1,39 @@
+(ns metabase.agent-lib.validate.semantic-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase.agent-lib.test-util :as tu]
+ [metabase.agent-lib.validate.semantic :as semantic]))
+
+(deftest ^:parallel validate-program-passes-valid-program-test
+ (testing "valid program passes through unchanged"
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["=" ["field" 1] "hello"]]
+ ["limit" 10]]}
+ context (tu/table-context :orders)]
+ (is (= program (semantic/validate-program program context))))))
+
+(deftest validate-program-rejects-unknown-table-source-test
+ (testing "unknown table source raises invalid-generated-program"
+ (let [program {:source {:type "table" :id 999999}
+ :operations [["limit" 10]]}
+ context (tu/table-context :orders)
+ error (try
+ (semantic/validate-program program context)
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error)))))))
+
+(deftest validate-program-rejects-metric-in-order-by-test
+ (testing "metric helper inside order-by is rejected"
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["order-by" ["metric" 42]]]}
+ context (tu/table-context :orders)
+ error (try
+ (semantic/validate-program program context)
+ nil
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is error)
+ (is (= :invalid-generated-program (:error (ex-data error))))
+ (is (re-find #"metric helpers cannot be used directly inside order-by"
+ (:details (ex-data error)))))))
diff --git a/test/metabase/agent_lib/validate_test.clj b/test/metabase/agent_lib/validate_test.clj
new file mode 100644
index 00000000000..91bb23f414a
--- /dev/null
+++ b/test/metabase/agent_lib/validate_test.clj
@@ -0,0 +1,44 @@
+(ns metabase.agent-lib.validate-test
+ (:require
+ [clojure.test :refer [deftest is]]
+ [metabase.agent-lib.validate :as validate]))
+
+(deftest ^:parallel validate-program-inserts-append-stage-for-context-metric-source-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["aggregation-ref" 0]]]]}
+ context {:source-entity {:model "metric" :id 42}
+ :source-metadata nil
+ :referenced-entities []
+ :surrounding-tables []
+ :measure-ids []}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["append-stage"]
+ ["aggregate" ["avg" ["aggregation-ref" 0]]]]}]
+ (is (= expected (validate/validated-program program context)))))
+
+(deftest ^:parallel validate-program-inserts-append-stage-for-expression-over-aggregation-ref-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["expression" "Churn Rate" ["*" ["aggregation-ref" 0] 100]]]}
+ context {:source-entity {:model "metric" :id 42}
+ :source-metadata nil
+ :referenced-entities []
+ :surrounding-tables []
+ :measure-ids []}
+ expected {:source {:type "context" :ref "source"}
+ :operations [["append-stage"]
+ ["expression" "Churn Rate" ["*" ["aggregation-ref" 0] 100]]]}]
+ (is (= expected (validate/validated-program program context)))))
+
+(deftest ^:parallel metric-source-measure-guidance-test
+ (let [program {:source {:type "context" :ref "source"}
+ :operations [["aggregate" ["avg" ["measure" 118]]]]}
+ context {:source-entity {:model "metric" :id 118}
+ :referenced-entities []
+ :surrounding-tables []
+ :measure-ids []}]
+ (try
+ (validate/validated-program program context)
+ (is false "expected metric/measure validation to fail")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= :invalid-generated-program (:error (ex-data e))))
+ (is (re-find #"Metric ids are not measure ids" (:details (ex-data e))))))))
diff --git a/test/metabase/mcp/api_test.clj b/test/metabase/mcp/api_test.clj
index fb5cbd9b775..6cc8452cc54 100644
--- a/test/metabase/mcp/api_test.clj
+++ b/test/metabase/mcp/api_test.clj
@@ -405,8 +405,11 @@
(reset! streamed? true)
(original-fn response))]
(let [[session-id _] (initialize!)
- construct-data (call-tool session-id "construct_query" {:table_id (mt/id :orders) :limit 5})
- execute-data (call-tool session-id "execute_query" {:query (:query construct-data)})]
+ construct-data (call-tool session-id "construct_query"
+ {:source {:type "table" :id (mt/id :orders)}
+ :operations [["limit" 5]]})
+ execute-data (call-tool session-id "execute_query"
+ {:query (:query construct-data)})]
(is (true? @streamed?) "execute_query should use the streaming response path")
(is (=? {:status "completed"
:row_count 5
diff --git a/test/metabase/metabot/agent/core_test.clj b/test/metabase/metabot/agent/core_test.clj
index b26692aca4e..56b569fc990 100644
--- a/test/metabase/metabot/agent/core_test.clj
+++ b/test/metabase/metabot/agent/core_test.clj
@@ -278,18 +278,15 @@
:keyword_queries ["orders"]
:entity_types ["table"]}}
{:type :usage :usage {:promptTokens 100 :completionTokens 20} :model "test" :id "msg-1"}]
- ;; Iteration 2: Construct a simple raw query (no fields/aggregations = select all)
+ ;; Iteration 2: Construct a simple query via agent-lib program
[{:type :start :id "msg-2"}
{:type :tool-input
:id "call-construct-1"
:function "construct_notebook_query"
:arguments {:reasoning "User wants to see orders"
- :query {:query_type "raw"
- :source {:table_id orders-table-id}
- :filters []
- :fields []
- :order_by []
- :limit 10}
+ :source_entity {:type "table" :id orders-table-id}
+ :program {:source {:type "table" :id orders-table-id}
+ :operations [["limit" 10]]}
:visualization {:chart_type "table"}}}
{:type :usage :usage {:promptTokens 200 :completionTokens 30} :model "test" :id "msg-2"}]
;; Iteration 3: Final text response
diff --git a/test/metabase/metabot/agent/tool_output_test.clj b/test/metabase/metabot/agent/tool_output_test.clj
index 33ebd664952..5721e41d342 100644
--- a/test/metabase/metabot/agent/tool_output_test.clj
+++ b/test/metabase/metabot/agent/tool_output_test.clj
@@ -37,14 +37,16 @@
#"#metabase"
#"clojure\.lang\."])
-(defn- find-field-index
- "Find the index of a field by name in a table's visible columns."
+(defn- find-field-id
+ "Find the real field ID by name in a table's visible columns."
[table-id field-name]
(let [mp (mt/metadata-provider)
query (lib/query mp (lib.metadata/table mp table-id))
cols (lib/visible-columns query -1 {:include-implicitly-joinable? false})]
- (some (fn [[i col]] (when (= field-name (:name col)) i))
- (map-indexed vector cols))))
+ (->> cols
+ (filter #(= field-name (:name %)))
+ first
+ :id)))
(defn- assert-formatted-output
"Assert that a tool result contains a well-formatted :output string.
@@ -92,10 +94,10 @@
["read_resource: table field values"
#(let [table-id (mt/id :orders)
- idx (find-field-index table-id "QUANTITY")]
+ field-id (find-field-id table-id "QUANTITY")]
(ensure-fresh-field-values! (mt/id :orders :quantity))
(resource-tools/read-resource-tool
- {:uris [(str "metabase://table/" table-id "/fields/t" table-id "-" idx)]}))
+ {:uris [(str "metabase://table/" table-id "/fields/" field-id)]}))
#"<field-metadata\b"]])
(defn- read-resource-metric-invocations [metric-id]
@@ -254,12 +256,12 @@
(mt/test-driver :h2
(mt/with-current-user (mt/user->id :crowberto)
(let [table-id (mt/id :orders)
- idx (find-field-index table-id "QUANTITY")]
+ field-id (find-field-id table-id "QUANTITY")]
(ensure-fresh-field-values! (mt/id :orders :quantity))
(let [result (metadata-tools/get-field-values-tool
{:data_source "table"
:source_id table-id
- :field_id (str "t" table-id "-" idx)})]
+ :field_id field-id})]
(assert-formatted-structured result "get_field_values: table field" #"<field-metadata\b")))))))
(deftest get-field-values-metric-structured-output-test
@@ -274,10 +276,10 @@
:database_id (mt/id)
:name "Test Metric For Fields"
:type :metric}]
- ;; Use the card-field-id-prefix format: c<id>-<index>
+ ;; Use real field ID — first filterable column of the metric
;; Temp metrics may not have fingerprints, so field metadata may be "No metadata available"
;; — still a valid formatted string, just not XML-wrapped
- (let [field-id (str "c" metric-id "-0")
+ (let [field-id (mt/id :orders :total)
result (metadata-tools/get-field-values-tool
{:data_source "metric"
:source_id metric-id
@@ -297,9 +299,10 @@
:database_id (mt/id)
:name "Test Model For Fields"
:type :model}]
+ ;; Use real field ID — first returned column of the model
;; Temp models may not have fingerprints, so we accept either
;; <field-metadata> XML or "No metadata available" fallback
- (let [field-id (str "c" model-id "-0")
+ (let [field-id (mt/id :orders :id)
result (metadata-tools/get-field-values-tool
{:data_source "model"
:source_id model-id
@@ -319,9 +322,8 @@
:database_id (mt/id)
:name "Test Model Field Values"
:type :model}]
- ;; field index 1 = :quantity (second field)
(ensure-fresh-field-values! (mt/id :orders :quantity))
- (let [field-id (str "c" model-id "-1")
+ (let [field-id (mt/id :orders :quantity)
result (resource-tools/read-resource-tool
{:uris [(str "metabase://model/" model-id "/fields/" field-id)]})]
(assert-formatted-output result "model field values" #"<field-metadata\b"))))))))
@@ -338,9 +340,9 @@
:database_id (mt/id)
:name "Test Metric Dim Values"
:type :metric}]
- ;; dimension index 0 = first filterable column
+ ;; Use a real filterable field ID from the metric's source table
;; Temp metrics may not have fingerprints — "No metadata available" is also valid
- (let [field-id (str "c" metric-id "-0")
+ (let [field-id (mt/id :orders :id)
result (resource-tools/read-resource-tool
{:uris [(str "metabase://metric/" metric-id "/dimensions/" field-id)]})]
(assert-formatted-output result "metric dimension values" #"(?:<field-metadata\b|No metadata available)"))))))))
diff --git a/test/metabase/metabot/example_question_generator_test.clj b/test/metabase/metabot/example_question_generator_test.clj
index c63f0bde106..3c4d2b175cb 100644
--- a/test/metabase/metabot/example_question_generator_test.clj
+++ b/test/metabase/metabot/example_question_generator_test.clj
@@ -184,4 +184,5 @@
"duration_ms" nat-int?
"source" "example_question_generation_batch"
"tag" "example-question-generation"}}]
- (snowplow-test/pop-event-data-and-user-id!))))))))))
+ (->> (snowplow-test/pop-event-data-and-user-id!)
+ (remove #(= "new_instance_created" (get-in % [:data "event"])))))))))))))
diff --git a/test/metabase/metabot/tools/charts/edit_test.clj b/test/metabase/metabot/tools/charts/edit_test.clj
index 88a4413a98a..45d1c823c89 100644
--- a/test/metabase/metabot/tools/charts/edit_test.clj
+++ b/test/metabase/metabot/tools/charts/edit_test.clj
@@ -70,19 +70,18 @@
{[{{table :structured-output} :content}] :resources}
(tools.resources/read-resource-tool {:uris [table-fields-uri]})
+ table-id (:id table)
+ category-field-id (some (fn [{:keys [display_name field_id]}]
+ (when (= "Category" display_name)
+ field_id))
+ (:fields table))
;; (1) construct a query
construct-result (tools.construct/construct-notebook-query-tool
- {:query
- {:query_type "aggregate"
- :source {:table_id (:id table)}
- :aggregations [{:function "count"}]
- :filters []
- :group_by [{:field_id (some (fn [{:keys [display_name field_id]}]
- (when (= "Category" display_name)
- field_id))
- (:fields table))}]
- :limit nil
- :visualization {:chart_type "bar"}}})
+ {:source_entity {:type "table" :id table-id}
+ :program {:source {:type "table" :id table-id}
+ :operations [["aggregate" ["count"]]
+ ["breakout" ["field" category-field-id]]]}
+ :visualization {:chart_type "bar"}})
query-id (get-in construct-result [:structured-output :query-id])
query (get-in construct-result [:structured-output :query])
chart-id (get-in construct-result [:structured-output :chart-id])]
diff --git a/test/metabase/metabot/tools/document_test.clj b/test/metabase/metabot/tools/document_test.clj
index 4eac3eb6521..7b5473f3b87 100644
--- a/test/metabase/metabot/tools/document_test.clj
+++ b/test/metabase/metabot/tools/document_test.clj
@@ -125,7 +125,8 @@
(let [result (document-tools/document-construct-model-chart-tool
{:name "Test Name"
:description "Test Desc"
- :query {:source {:model_id 4}}
+ :source_entity {:type "model" :id 4}
+ :program {:source {:type "context" :ref "source"} :operations []}
:viz_settings {:chart_type "bar"}})
structured (:structured-output result)]
(is (true? (:final-response? result)))
diff --git a/test/metabase/metabot/tools/field_stats_test.clj b/test/metabase/metabot/tools/field_stats_test.clj
index 94abfd1c57c..29fa7264cc0 100644
--- a/test/metabase/metabot/tools/field_stats_test.clj
+++ b/test/metabase/metabot/tools/field_stats_test.clj
@@ -1,10 +1,7 @@
(ns metabase.metabot.tools.field-stats-test
(:require
[clojure.test :refer :all]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
[metabase.metabot.tools.field-stats :as metabot.tools.field-stats]
- [metabase.metabot.tools.util :as metabot.tools.u]
[metabase.test :as mt]
[metabase.warehouse-schema.models.field-values :as field-values]
[toucan2.core :as t2]))
@@ -17,35 +14,14 @@
:type)))
(is (= 1 (t2/count :model/FieldValues :field_id field-id :type :full))))
-(defn- table-query
- [metadata-provider table-id]
- (lib/query metadata-provider (lib.metadata/table metadata-provider table-id)))
-
-(defn- query-field-id [query field-id-prefix field-display-name columns-fn]
- (->> (keep-indexed (fn [i col]
- (when (= (lib/display-name query col) field-display-name)
- i))
- (columns-fn query))
- first
- (str field-id-prefix)))
-
-(defn- visible-field-id [query field-id-prefix field-display-name]
- (query-field-id query field-id-prefix field-display-name lib/visible-columns))
-
-(defn- filterable-field-id [query field-id-prefix field-display-name]
- (query-field-id query field-id-prefix field-display-name lib/filterable-columns))
-
(deftest field-values-table-test
(ensure-fresh-field-values! (mt/id :people :state))
(ensure-fresh-field-values! (mt/id :products :category))
- (let [mp (mt/metadata-provider)
- people-id (mt/id :people)
- people-query (table-query mp people-id)
- birth-date-id (visible-field-id people-query (metabot.tools.u/table-field-id-prefix people-id) "Birth Date")
- state-id (visible-field-id people-query (metabot.tools.u/table-field-id-prefix people-id) "State")
- products-id (mt/id :products)
- products-query (table-query mp products-id)
- category-id (visible-field-id products-query (metabot.tools.u/table-field-id-prefix products-id) "Category")]
+ (let [birth-date-id (mt/id :people :birth_date)
+ state-id (mt/id :people :state)
+ people-id (mt/id :people)
+ products-id (mt/id :products)
+ category-id (mt/id :products :category)]
(testing "No read permission results in an error."
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
(metabot.tools.field-stats/field-values
@@ -86,13 +62,9 @@
(ensure-fresh-field-values! (mt/id :products :category))
(mt/with-temp [:model/Card {model-id :id} {:dataset_query (mt/mbql-query orders)
:type :model}]
- (let [mp (mt/metadata-provider)
- model-query (lib/query mp (lib.metadata/card mp model-id))
- card-field-id-prefix (metabot.tools.u/card-field-id-prefix model-id)
- ;; All fields use c<card-id> syntax with indices from model's visible-columns
- quantity-id (visible-field-id model-query card-field-id-prefix "Quantity")
- state-id (visible-field-id model-query card-field-id-prefix "State")
- category-id (visible-field-id model-query card-field-id-prefix "Category")]
+ (let [quantity-id (mt/id :orders :quantity)
+ state-id (mt/id :people :state)
+ category-id (mt/id :products :category)]
(testing "No read permission results in an error."
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
(metabot.tools.field-stats/field-values
@@ -131,12 +103,8 @@
:breakout [$quantity
!year.user_id->people.birth_date]})
:type :metric}]
- (let [mp (mt/metadata-provider)
- metric-query (lib/query mp (lib.metadata/metric mp metric-id))
- card-field-id-prefix (metabot.tools.u/card-field-id-prefix metric-id)
- ;; All fields use c<card-id> syntax with indices from metric's filterable-columns
- quantity-id (filterable-field-id metric-query card-field-id-prefix "Quantity")
- birth-date-id (filterable-field-id metric-query card-field-id-prefix "Birth Date")]
+ (let [quantity-id (mt/id :orders :quantity)
+ birth-date-id (mt/id :people :birth_date)]
(testing "No read permission results in an error."
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
(metabot.tools.field-stats/field-values
diff --git a/test/metabase/metabot/tools/filters_test.clj b/test/metabase/metabot/tools/filters_test.clj
deleted file mode 100644
index f2bb3a9c9f4..00000000000
--- a/test/metabase/metabot/tools/filters_test.clj
+++ /dev/null
@@ -1,959 +0,0 @@
-(ns metabase.metabot.tools.filters-test
- (:require
- [clojure.string :as str]
- [clojure.test :refer :all]
- [medley.core :as m]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.convert :as lib.convert]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.test-util :as lib.tu]
- [metabase.metabot.tools.entity-details :as metabot.tools.entity-details]
- [metabase.metabot.tools.filters :as metabot.tools.filters]
- [metabase.metabot.tools.test-util :as tools.tu]
- [metabase.test :as mt]
- [metabase.util :as u]))
-
-(defn- by-name
- [dimensions dimension-name]
- (let [pred (if (string? dimension-name)
- (comp #{dimension-name} :display_name)
- (every-pred (comp #{(first dimension-name)} :table_reference)
- (comp #{(second dimension-name)} :display_name)))]
- (m/find-first pred dimensions)))
-
-(deftest ^:parallel query-metric-test
- (let [mp (mt/metadata-provider)
- created-at-meta (lib.metadata/field mp (mt/id :orders :created_at))
- metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- (lib/aggregate (lib/avg (lib.metadata/field mp (mt/id :orders :subtotal))))
- (lib/breakout (lib/with-temporal-bucket created-at-meta :month)))
- legacy-metric-query (lib.convert/->legacy-MBQL metric-query)]
- (mt/with-temp [:model/Card {metric-id :id} {:dataset_query legacy-metric-query
- :database_id (mt/id)
- :name "Average Order Value"
- :description "The average subtotal of orders."
- :type :metric}]
- (testing "User has to have execution rights."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters []
- :group-by []}))))
- (mt/with-current-user (mt/user->id :crowberto)
- (let [metric-details (metabot.tools.entity-details/metric-details metric-id)
- ->field-id #(u/prog1 (-> metric-details :queryable-dimensions (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Trivial query works."
- (let [result (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters []
- :group-by []})
- actual-query (get-in result [:structured-output :query])
- expected-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- (lib/aggregate (lib/ensure-uuid [:metric {} metric-id])))]
- (is (= :query (get-in result [:structured-output :type])))
- (is (string? (get-in result [:structured-output :query-id])))
- (is (tools.tu/query= expected-query actual-query))))
- (testing "Filtering and grouping works and ignores bucketing for non-temporal columns."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table (mt/id :orders)
- :aggregation [[:metric {} metric-id]]
- :breakout [[:field {:source-field (mt/id :orders :product_id)}
- (mt/id :products :category)]]
- :filters
- [[:= {} [:field {:source-field (mt/id :orders :user_id)}
- (mt/id :people :state)]
- "TX"]
- [:> {} [:field {} (mt/id :orders :discount)]
- 3]]}]}}}
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id ["User" "State"])
- :operation :string-equals
- :value "TX"}
- {:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]
- :group-by [{:field-id (->field-id ["Product" "Category"])
- :field-granularity :year}]}))))
- (testing "Temporal bucketing works for temporal columns."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table (mt/id :orders)
- :aggregation [[:metric {} metric-id]]
- :filters [[:!= {}
- [:get-week {} [:field {} (mt/id :orders :created_at)] :iso]
- 1 2 3]
- [:= {}
- [:get-month {} [:field {} (mt/id :orders :created_at)]]
- 6 7 8]]
- :breakout [[:field {:temporal-unit :week}
- (mt/id :orders :created_at)]]}]}}}
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id "Created At")
- :bucket :week-of-year
- :operation :not-equals
- :values [1 2 3]}
- {:field-id (->field-id "Created At")
- :bucket :month-of-year
- :operation :equals
- :values [6 7 8]}]
- :group-by [{:field-id (->field-id "Created At")
- :field-granularity :week}]}))))
- (testing "Multi-value filtering works"
- (let [state-id (->field-id ["User" "State"])]
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table (mt/id :orders)
- :aggregation [[:metric {} metric-id]]
- :filters
- [[:contains {:case-sensitive false}
- [:field {:source-field (mt/id :orders :user_id)}
- (mt/id :people :state)]
- "o" "e"]
- [:does-not-contain {:case-sensitive false}
- [:field {:source-field (mt/id :orders :user_id)}
- (mt/id :people :state)]
- "y"]
- [:starts-with {:case-sensitive false}
- [:field {:source-field (mt/id :orders :user_id)}
- (mt/id :people :state)]
- "A" "G"]
- [:ends-with {:case-sensitive false}
- [:field {:source-field (mt/id :orders :user_id)}
- (mt/id :people :state)]
- "e" "f" "i" "T" "x"]
- [:!= {}
- [:field {} (mt/id :orders :discount)]
- 3 42]]}]}}}
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id state-id
- :operation :string-contains
- :values ["o" "e"]}
- {:field-id state-id
- :operation :string-not-contains
- :value "y"}
- {:field-id state-id
- :operation :string-starts-with
- :values ["A" "G"]}
- {:field-id state-id
- :operation :string-ends-with
- :values ["e" "f" "i" "T" "x"]}
- {:field-id (->field-id "Discount")
- :operation :not-equals
- :values [3 42]}]}))))))
- (testing "Missing metric results in an error."
- (is (= {:output "Not found."
- :status-code 404}
- (metabot.tools.filters/query-metric {:metric-id Integer/MAX_VALUE}))))
- (testing "Invalid metric-id results in an error."
- (is (= {:output (str "Invalid metric_id " metric-id)
- :status-code 400}
- (metabot.tools.filters/query-metric {:metric-id (str metric-id)}))))))))
-
-(deftest query-metric-temporal-value-validation-test
- (let [mp (mt/metadata-provider)
- created-at-meta (lib.metadata/field mp (mt/id :orders :created_at))
- metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- (lib/aggregate (lib/avg (lib.metadata/field mp (mt/id :orders :subtotal))))
- (lib/breakout (lib/with-temporal-bucket created-at-meta :month)))
- legacy-metric-query (lib.convert/->legacy-MBQL metric-query)]
- (mt/with-temp [:model/Card {metric-id :id} {:dataset_query legacy-metric-query
- :database_id (mt/id)
- :name "Average Order Value"
- :description "The average subtotal of orders."
- :type :metric}]
- (mt/with-current-user (mt/user->id :crowberto)
- (let [metric-details (metabot.tools.entity-details/metric-details metric-id)
- ->field-id #(u/prog1 (-> metric-details :queryable-dimensions (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Negative value on temporal field without bucket is rejected"
- (let [result (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id "Created At")
- :operation :greater-than-or-equal
- :value -30}]
- :group-by []})]
- (is (= 400 (:status-code result)))
- (is (str/includes? (:output result) "not valid for a date/datetime field"))))
- (testing "Small positive integer on temporal field without bucket is rejected"
- (let [result (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id "Created At")
- :operation :greater-than-or-equal
- :value 7}]
- :group-by []})]
- (is (= 400 (:status-code result)))
- (is (str/includes? (:output result) "not valid for a date/datetime field"))))
- (testing "Date string on temporal field without bucket passes validation"
- (is (=? {:structured-output {:type :query}}
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id "Created At")
- :operation :greater-than-or-equal
- :value "2024-01-01"}]
- :group-by []}))))
- (testing "Large integer (year) on temporal field without bucket passes validation"
- (is (=? {:structured-output {:type :query}}
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id "Created At")
- :operation :greater-than-or-equal
- :value 2024}]
- :group-by []}))))
- (testing "Numeric value on non-temporal field without bucket passes validation"
- (is (=? {:structured-output {:type :query}}
- (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:field-id (->field-id "Discount")
- :operation :greater-than-or-equal
- :value -5}]
- :group-by []})))))))))
-
-(deftest ^:parallel query-model-test
- (mt/with-temp [:model/Card {model-id :id} {:dataset_query (mt/mbql-query orders {})
- :database_id (mt/id)
- :name "Orders Model"
- :description "The _real_ orders."
- :type :model}]
- (testing "User has to have execution rights."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/query-model
- {:model-id model-id
- :filters []
- :group-by []}))))
- (mt/with-current-user (mt/user->id :crowberto)
- (let [model-details (-> (metabot.tools.entity-details/get-table-details {:entity-type :model
- :entity-id model-id})
- :structured-output)
- ->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))
- order-created-at-field-id (->field-id "Created At")]
- (testing "Trivial query works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id}]}}}
- (metabot.tools.filters/query-model
- {:model-id model-id
- :filters []
- :group-by []}))))
- (testing "Filtering, aggregation and grouping works and ignores bucketing for non-temporal columns."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :aggregation [[:sum {} [:field {} "SUBTOTAL"]]]
- :breakout [[:field {} "PRODUCT_ID"]]
- :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}
- (metabot.tools.filters/query-model
- {:model-id model-id
- :filters [{:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]
- :aggregations [{:field-id (->field-id "Subtotal")
- :bucket :day-of-month ; ignored
- :function :sum}]
- :group-by [{:field-id (->field-id "Product ID")
- :field-granularity :year}]}))))
- (testing "Temporal bucketing works for temporal columns."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :aggregation [[:min {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]
- [:avg {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]
- [:max {} [:field {:temporal-unit :hour-of-day} "CREATED_AT"]]]
- :filters [[:!= {} [:get-week {} [:field {} "CREATED_AT"] :iso] 1 2 3]
- [:= {} [:get-month {} [:field {} "CREATED_AT"]] 6 7 8]]
- :breakout [[:field {} "PRODUCT_ID"]
- [:field {:temporal-unit :week} "CREATED_AT"]]}]}}}
- (metabot.tools.filters/query-model
- {:model-id model-id
- :filters [{:field-id order-created-at-field-id
- :bucket :week-of-year
- :operation :not-equals
- :values [1 2 3]}
- {:field-id order-created-at-field-id
- :bucket :month-of-year
- :operation :equals
- :values [6 7 8]}]
- :aggregations [{:field-id order-created-at-field-id
- :bucket :hour-of-day
- :function :min}
- {:field-id order-created-at-field-id
- :bucket :hour-of-day
- :function :avg}
- {:field-id order-created-at-field-id
- :bucket :hour-of-day
- :function :max}]
- :group-by [{:field-id (->field-id "Product ID")}
- {:field-id order-created-at-field-id
- :field-granularity :week}]}))))
- ;;
- ;; TODO (Cam 6/19/25) -- disabled for now since this was generating invalid queries with duplicate expression
- ;; names. Previously this was ok I think because the MBQL 5 didn't enforce it but when I added that this started
- ;; failing
- ;;
- #_(testing "Fields can be selected"
- (is (=? {:structured-output
- {:type :query,
- :query-id string?
- :query (mt/mbql-query orders
- {:source-table model-card-id
- :expressions {"Created At: Day of month"
- [:get-day [:field "CREATED_AT" {:base-type :type/DateTimeWithLocalTZ}]],
- "Created At: Day of week"
- [:get-day-of-week [:field "CREATED_AT" {}] :iso]}
- :fields [[:expression "Created At: Day of month" {:base-type :type/Integer}]
- [:expression "Created At: Day of week" {:base-type :type/Integer}]
- [:field "TOTAL" {:base-type :type/Float}]]
- :filter [:!= [:field "USER_ID" {}] 3 42]})}}
- (metabot.tools.filters/query-model
- {:model-id model-id
- :fields [{:field-id order-created-at-field-id
- :bucket :day-of-month}
- {:field-id order-created-at-field-id
- :bucket :day-of-week}
- {:field-id (->field-id "Total")}]
- :filters [{:field-id (->field-id "User ID")
- :operation :not-equals
- :values [3 42]}]}))))
- (testing "With empty or missing fields and no summary, all fields are returned"
- (let [expected-query {:structured-output
- {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :filters [[:!= {} [:field {} "USER_ID"] 3 42]]}]}}}
- input {:model-id model-id
- :filters [{:field-id (->field-id "User ID")
- :operation :not-equals
- :values [3 42]}]}]
- (are [input] (=? expected-query (metabot.tools.filters/query-model input))
- input
- (assoc input :fields nil)
- (assoc input :fields [])))))
- (testing "Missing model results in an error."
- (is (= {:output "Not found."
- :status-code 404}
- (metabot.tools.filters/query-model {:model-id Integer/MAX_VALUE}))))
- (testing "Invalid model-id results in an error."
- (is (= {:output (str "Invalid model_id " model-id)
- :status-code 400}
- (metabot.tools.filters/query-model {:model-id (str model-id)})))))))
-
-(deftest ^:parallel filter-records-table-test
- (testing "User has to have execution rights, otherwise the table should be invisible."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/filter-records
- {:data-source {:table-id (mt/id :orders)}
- :filters []}))))
- (mt/with-current-user (mt/user->id :crowberto)
- (let [mp (mt/metadata-provider)
- table-id (mt/id :orders)
- table-details (#'metabot.tools.entity-details/table-details table-id {:metadata-provider mp})
- ->field-id #(u/prog1 (-> table-details :fields (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Trivial query works."
- (let [result (metabot.tools.filters/filter-records
- {:data-source {:table-id table-id}
- :filters []})
- actual-query (get-in result [:structured-output :query])
- expected-query (lib/query mp (lib.metadata/table mp table-id))]
- (is (= :query (get-in result [:structured-output :type])))
- (is (string? (get-in result [:structured-output :query-id])))
- (is (tools.tu/query= expected-query actual-query))))
- (testing "Filtering works."
- (let [result (metabot.tools.filters/filter-records
- {:data-source {:table-id table-id}
- :filters [{:field-id (->field-id "Created At")
- :bucket :day-of-week
- :operation :equals
- :values [1 7]}
- {:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]})
- actual-query (get-in result [:structured-output :query])
- orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- created-at-col (lib.metadata/field mp (mt/id :orders :created_at))
- discount-col (lib.metadata/field mp (mt/id :orders :discount))
- expected-query (-> orders-query
- (lib/filter (lib/= (lib/get-day-of-week created-at-col :iso) 1 7))
- (lib/filter (lib/> discount-col 3)))]
- (is (= :query (get-in result [:structured-output :type])))
- (is (string? (get-in result [:structured-output :query-id])))
- (is (tools.tu/query= expected-query actual-query)))))
- (testing "Missing table results in an error."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found."
- (metabot.tools.filters/filter-records
- {:data-source {:table-id Integer/MAX_VALUE}}))))))
-
-(deftest ^:parallel filter-records-model-test
- (let [mp (mt/metadata-provider)
- query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- legacy-query (lib.convert/->legacy-MBQL query)]
- (mt/with-temp [:model/Card {model-id :id} {:dataset_query legacy-query
- :database_id (mt/id)
- :name "Orders"
- :description "The orders."
- :type :model}]
- (testing "User has to have execution rights."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/filter-records
- {:data-source {:table-id (str "card__" model-id)}
- :filters []}))))
- (mt/with-current-user (mt/user->id :crowberto)
- (let [model-details (#'metabot.tools.entity-details/card-details model-id)
- table-id (str "card__" model-id)
- ->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Trivial query works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id}]}}}
- (metabot.tools.filters/filter-records
- {:data-source {:table-id table-id}
- :filters []}))))
- (testing "Filtering works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}
- (metabot.tools.filters/filter-records
- {:data-source {:table-id table-id}
- :filters [{:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]})))))
- (testing "Missing table results in an error."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found."
- (metabot.tools.filters/filter-records
- {:data-source {:table-id (str "card__" Integer/MAX_VALUE)}}))))))))
-
-(deftest ^:parallel filter-records-report-test
- (let [mp (mt/metadata-provider)
- table-id (mt/id :orders)
- query (lib/query mp (lib.metadata/table mp table-id))
- legacy-query (lib.convert/->legacy-MBQL query)]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query legacy-query
- :database_id (mt/id)
- :name "Orders"
- :description "The orders."
- :type :question}]
- (testing "User has to have execution rights."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/filter-records
- {:data-source {:report-id card-id}
- :filters []}))))
- (mt/with-current-user (mt/user->id :crowberto)
- (let [report-details (#'metabot.tools.entity-details/card-details card-id)
- ->field-id #(u/prog1 (-> report-details :fields (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Trivial query works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id}]}}}
- (metabot.tools.filters/filter-records
- {:data-source {:report-id card-id}
- :filters []}))))
- (testing "Filtering works."
- (is (=? {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id
- :filters [[:> {} [:field {} (mt/id :orders :discount)] 3]]}]}}}
- (metabot.tools.filters/filter-records
- {:data-source {:report-id card-id}
- :filters [{:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]})))))
- (testing "Missing table results in an error."
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Not found."
- (metabot.tools.filters/filter-records
- {:data-source {:report-id Integer/MAX_VALUE}}))))))))
-
-(deftest ^:parallel filter-records-query-test
- (let [query-id (u/generate-nano-id)
- mp (mt/metadata-provider)
- table-id (mt/id :orders)
- query (lib/query mp (lib.metadata/table mp table-id))
- legacy-query (lib.convert/->legacy-MBQL query)
- query-details (mt/with-current-user (mt/user->id :crowberto)
- (#'metabot.tools.entity-details/execute-query query-id legacy-query))
- ->field-id #(u/prog1 (-> query-details :result-columns (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))
- input {:data-source {:query-id query-id}
- :filters [{:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}]}
- expected {:structured-output {:type :query,
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id}
- {:lib/type :mbql.stage/mbql
- :filters [[:> {} [:field {} "DISCOUNT"] 3]]}]}}}]
- (mt/with-current-user (mt/user->id :crowberto)
- (testing "Filtering works."
- (testing "new tool call with query and query-id"
- (is (=? expected
- (metabot.tools.filters/filter-records
- (assoc input :data-source (select-keys query-details [:query :query-id]))))))
- (testing "new tool call with just query"
- (is (=? expected
- (metabot.tools.filters/filter-records
- (assoc input :data-source (select-keys query-details [:query]))))))
- (testing "MBQL v5 query works"
- (is (=? expected
- (metabot.tools.filters/filter-records
- (assoc input :data-source {:query query})))))))))
-
-(deftest ^:parallel query-datasource-table-test
- (mt/with-current-user (mt/user->id :crowberto)
- (let [mp (mt/metadata-provider)
- table-id (mt/id :orders)
- table-details (#'metabot.tools.entity-details/table-details table-id {:metadata-provider mp})
- ->field-id #(u/prog1 (-> table-details :fields (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Basic table query works"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id}]}}}
- (metabot.tools.filters/query-datasource
- {:table-id table-id}))))
-
- (testing "Table query with fields selection"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id
- :fields [[:field {} (mt/id :orders :created_at)]
- [:field {} (mt/id :orders :total)]]}]}}}
- (metabot.tools.filters/query-datasource
- {:table-id table-id
- :fields [{:field-id (->field-id "Created At")}
- {:field-id (->field-id "Total")}]}))))
-
- (testing "Table query with filters"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id
- :filters [[:> {} [:field {} (mt/id :orders :discount)] 3]
- [:= {} [:field {} (mt/id :orders :user_id)] 10]]}]}}}
- (metabot.tools.filters/query-datasource
- {:table-id table-id
- :filters [{:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 3}
- {:field-id (->field-id "User ID")
- :operation :equals
- :value 10}]}))))
-
- (testing "Table query with aggregations and grouping"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id
- :aggregation [[:sum {} [:field {} (mt/id :orders :subtotal)]]
- [:avg {} [:field {} (mt/id :orders :discount)]]]
- :breakout [[:field {} (mt/id :orders :product_id)]]}]}}}
- (metabot.tools.filters/query-datasource
- {:table-id table-id
- :aggregations [{:field-id (->field-id "Subtotal")
- :function :sum}
- {:field-id (->field-id "Discount")
- :function :avg}]
- :group-by [{:field-id (->field-id "Product ID")}]}))))
-
- (testing "Table query with order by and limit"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id
- :order-by [[:asc {} [:field {} (mt/id :orders :created_at)]]
- [:desc {} [:field {} (mt/id :orders :total)]]]
- :limit 100}]}}}
- (metabot.tools.filters/query-datasource
- {:table-id table-id
- :order-by [{:field {:field-id (->field-id "Created At")}
- :direction :asc}
- {:field {:field-id (->field-id "Total")}
- :direction :desc}]
- :limit 100}))))
-
- (testing "Table query with temporal bucketing"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-table table-id
- :aggregation [[:count {}]]
- :breakout [[:field {:temporal-unit :month} (mt/id :orders :created_at)]]}]}}}
- (metabot.tools.filters/query-datasource
- {:table-id table-id
- :aggregations [{:field-id (->field-id "Product ID")
- :function :count}]
- :group-by [{:field-id (->field-id "Created At")
- :field-granularity :month}]})))))))
-
-(deftest ^:parallel query-datasource-model-test
- (mt/with-temp [:model/Card {model-id :id} {:dataset_query (mt/mbql-query orders {})
- :database_id (mt/id)
- :name "Orders Model"
- :description "Test model for orders"
- :type :model}]
- (mt/with-current-user (mt/user->id :crowberto)
- (let [model-details (-> (metabot.tools.entity-details/get-table-details {:entity-type :model
- :entity-id model-id})
- :structured-output)
- ->field-id #(u/prog1 (-> model-details :fields (by-name %) :field_id)
- (when-not <>
- (throw (ex-info (str "Column " % " not found") {:column %}))))]
- (testing "Basic model query works"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id}]}}}
- (metabot.tools.filters/query-datasource
- {:model-id model-id}))))
-
- (testing "Model query with fields selection"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :fields [[:field {} "CREATED_AT"]
- [:field {} "TOTAL"]]}]}}}
- (metabot.tools.filters/query-datasource
- {:model-id model-id
- :fields [{:field-id (->field-id "Created At")}
- {:field-id (->field-id "Total")}]}))))
-
- (testing "Model query with filters"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :filters [[:> {} [:field {} "DISCOUNT"] 5]
- [:< {} [:field {} "SUBTOTAL"] 100]]}]}}}
- (metabot.tools.filters/query-datasource
- {:model-id model-id
- :filters [{:field-id (->field-id "Discount")
- :operation :number-greater-than
- :value 5}
- {:field-id (->field-id "Subtotal")
- :operation :number-less-than
- :value 100}]}))))
-
- (testing "Model query with aggregations and grouping"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :aggregation [[:count {}]
- [:max {} [:field {} "TOTAL"]]]
- :breakout [[:field {} "USER_ID"]]}]}}}
- (metabot.tools.filters/query-datasource
- {:model-id model-id
- :aggregations [{:field-id (->field-id "Product ID")
- :function :count}
- {:field-id (->field-id "Total")
- :function :max}]
- :group-by [{:field-id (->field-id "User ID")}]}))))
-
- (testing "Model query with order by and limit"
- (is (=? {:structured-output {:type :query
- :query-id string?
- :query {:database (mt/id)
- :lib/type :mbql/query
- :stages [{:lib/type :mbql.stage/mbql
- :source-card model-id
- :order-by [[:desc {} [:field {} "CREATED_AT"]]]
- :limit 50}]}}}
- (metabot.tools.filters/query-datasource
- {:model-id model-id
- :order-by [{:field {:field-id (->field-id "Created At")}
- :direction :desc}]
- :limit 50}))))))))
-
-(deftest ^:parallel query-datasource-validation-test
- (mt/with-current-user (mt/user->id :crowberto)
- (testing "Error when neither table-id nor model-id provided"
- (is (= {:output "Either table_id or model_id must be provided"
- :status-code 400}
- (metabot.tools.filters/query-datasource {}))))
-
- (testing "Error when both table-id and model-id provided"
- (is (= {:output "Cannot provide both table_id and model_id"
- :status-code 400}
- (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :model-id 123}))))
-
- (testing "Error with invalid table-id"
- (is (= {:output "Invalid table_id not-a-number"
- :status-code 400}
- (metabot.tools.filters/query-datasource
- {:table-id "not-a-number"}))))
-
- (testing "Error with invalid model-id"
- (is (= {:output "Invalid model_id not-a-number"
- :status-code 400}
- (metabot.tools.filters/query-datasource
- {:model-id "not-a-number"}))))
-
- (testing "Error with non-existent table"
- (is (= {:output (str "No table found with table_id " Integer/MAX_VALUE)
- :status-code 404}
- (metabot.tools.filters/query-datasource
- {:table-id Integer/MAX_VALUE}))))
-
- (testing "Error with non-existent model"
- (is (= {:output (str "No model found with model_id " Integer/MAX_VALUE)
- :status-code 404}
- (metabot.tools.filters/query-datasource
- {:model-id Integer/MAX_VALUE}))))))
-
-(deftest ^:parallel query-datasource-permissions-test
- (mt/with-temp [:model/Card {model-id :id} {:dataset_query (mt/mbql-query orders {})
- :database_id (mt/id)
- :name "Orders Model"
- :type :model}]
- (testing "User without permissions gets error for table"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)}))))
-
- (testing "User without permissions gets error for model"
- (is (thrown-with-msg? clojure.lang.ExceptionInfo #"You don't have permissions to do that."
- (metabot.tools.filters/query-datasource
- {:model-id model-id}))))))
-
-(defn- mbql-measure-definition
- "Create an MBQL5 measure definition with a sum aggregation."
- [table-id field-id]
- (let [mp (mt/metadata-provider)
- table (lib.metadata/table mp table-id)
- query (lib/query mp table)
- field (lib.metadata/field mp field-id)]
- (lib/aggregate query (lib/sum field))))
-
-(defn- mbql-segment-definition
- "Create an MBQL5 segment definition with a filter."
- [table-id field-id value]
- (let [mp (mt/metadata-provider)
- table (lib.metadata/table mp table-id)
- query (lib/query mp table)
- field (lib.metadata/field mp field-id)]
- (lib/filter query (lib/> field value))))
-
-(defn- add-mock-segment
- "Add a mock segment to a metadata provider."
- [base-mp segment-id table-id definition]
- (lib.tu/mock-metadata-provider
- base-mp
- {:segments [{:lib/type :metadata/segment
- :id segment-id
- :name (str "Mock Segment " segment-id)
- :table-id table-id
- :archived false
- :definition definition}]}))
-
-(defn- add-mock-measure
- "Add a mock measure to a metadata provider."
- [base-mp measure-id table-id definition]
- (lib.tu/mock-metadata-provider
- base-mp
- {:measures [{:lib/type :metadata/measure
- :id measure-id
- :name (str "Mock Measure " measure-id)
- :table-id table-id
- :archived false
- :definition definition}]}))
-
-(deftest query-metric-with-segment-filter-test
- (let [mp (mt/metadata-provider)
- created-at-meta (lib.metadata/field mp (mt/id :orders :created_at))
- metric-query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- (lib/aggregate (lib/sum (lib.metadata/field mp (mt/id :orders :total))))
- (lib/breakout (lib/with-temporal-bucket created-at-meta :month)))
- legacy-metric-query (lib.convert/->legacy-MBQL metric-query)
- segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :total) 50)
- mock-segment-id 1
- mock-mp (add-mock-segment mp mock-segment-id (mt/id :orders) segment-def)]
- (mt/with-temp [:model/Card {metric-id :id} {:dataset_query legacy-metric-query
- :database_id (mt/id)
- :name "Total Orders"
- :type :metric}]
- (mt/with-current-user (mt/user->id :crowberto)
- (testing "query-metric with segment filter applies the segment"
- (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
- (let [result (metabot.tools.filters/query-metric
- {:metric-id metric-id
- :filters [{:segment-id mock-segment-id}]
- :group-by []})
- query (get-in result [:structured-output :query])
- filters (get-in query [:stages 0 :filters])]
- (is (some? (:structured-output result)))
- (is (= :query (get-in result [:structured-output :type])))
- (is (=? [[:segment {} mock-segment-id]] filters)))))))))
-
-(deftest query-datasource-with-measure-aggregation-test
- (let [mp (mt/metadata-provider)
- measure-def (mbql-measure-definition (mt/id :orders) (mt/id :orders :total))
- mock-measure-id 1
- mock-mp (add-mock-measure mp mock-measure-id (mt/id :orders) measure-def)]
- (mt/with-current-user (mt/user->id :crowberto)
- (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
- (testing "query-datasource with measure aggregation"
- (let [result (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :aggregations [{:measure-id mock-measure-id}]})
- query (get-in result [:structured-output :query])
- aggregations (get-in query [:stages 0 :aggregation])]
- (is (some? (:structured-output result)))
- (is (= :query (get-in result [:structured-output :type])))
- ;; Check that a measure aggregation is present
- (is (some #(and (vector? %) (= :measure (first %))) aggregations))))
-
- (testing "query-datasource with measure aggregation and sort order"
- (let [result (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :aggregations [{:measure-id mock-measure-id
- :sort-order :desc}]})
- query (get-in result [:structured-output :query])
- order-by (get-in query [:stages 0 :order-by])]
- (is (some? order-by))
- (is (some #(= :desc (first %)) order-by))))))))
-
-(deftest query-datasource-with-segment-filter-test
- (let [mp (mt/metadata-provider)
- segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :total) 100)
- mock-segment-id 1
- mock-mp (add-mock-segment mp mock-segment-id (mt/id :orders) segment-def)]
- (mt/with-current-user (mt/user->id :crowberto)
- (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
- (testing "query-datasource with segment filter"
- (let [result (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :filters [{:segment-id mock-segment-id}]})
- query (get-in result [:structured-output :query])
- filters (get-in query [:stages 0 :filters])]
- (is (some? (:structured-output result)))
- (is (= :query (get-in result [:structured-output :type])))
- ;; Check that a segment filter is present
- (is (some #(and (vector? %) (= :segment (first %))) filters))))))))
-
-(deftest query-datasource-with-measures-and-segments-test
- (let [mp (mt/metadata-provider)
- measure-def (mbql-measure-definition (mt/id :orders) (mt/id :orders :total))
- segment-def (mbql-segment-definition (mt/id :orders) (mt/id :orders :discount) 0)
- mock-measure-id 1
- mock-segment-id 1
- mock-mp (-> mp
- (add-mock-measure mock-measure-id (mt/id :orders) measure-def)
- (add-mock-segment mock-segment-id (mt/id :orders) segment-def))]
- (mt/with-current-user (mt/user->id :crowberto)
- (with-redefs [lib-be/application-database-metadata-provider (constantly mock-mp)]
- (testing "query-datasource with both measure aggregation and segment filter"
- (let [result (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :aggregations [{:measure-id mock-measure-id}]
- :filters [{:segment-id mock-segment-id}]})
- query (get-in result [:structured-output :query])
- aggregations (get-in query [:stages 0 :aggregation])
- filters (get-in query [:stages 0 :filters])]
- (is (some? (:structured-output result)))
- ;; Check that both measure and segment are present
- (is (some #(and (vector? %) (= :measure (first %))) aggregations))
- (is (some #(and (vector? %) (= :segment (first %))) filters))))))))
-
-(deftest segment-not-found-test
- (let [mp (mt/metadata-provider)
- non-existent-segment-id 99999]
- (mt/with-current-user (mt/user->id :crowberto)
- (with-redefs [lib-be/application-database-metadata-provider (constantly mp)]
- (testing "query-datasource with non-existent segment returns error"
- (let [result (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :filters [{:segment-id non-existent-segment-id}]})]
- (is (nil? (:structured-output result)))
- (is (string? (:output result)))
- (is (str/includes? (:output result) "Segment"))
- (is (str/includes? (:output result) "not found"))))))))
-
-(deftest measure-not-found-test
- (let [mp (mt/metadata-provider)
- non-existent-measure-id 99999]
- (mt/with-current-user (mt/user->id :crowberto)
- (with-redefs [lib-be/application-database-metadata-provider (constantly mp)]
- (testing "query-datasource with non-existent measure returns error"
- (let [result (metabot.tools.filters/query-datasource
- {:table-id (mt/id :orders)
- :aggregations [{:measure-id non-existent-measure-id}]})]
- (is (nil? (:structured-output result)))
- (is (string? (:output result)))
- (is (str/includes? (:output result) "Measure"))
- (is (str/includes? (:output result) "not found"))))))))
diff --git a/test/metabase/metabot/tools/util_test.clj b/test/metabase/metabot/tools/util_test.clj
index 226a7f6fd51..a5ac74aa3ac 100644
--- a/test/metabase/metabot/tools/util_test.clj
+++ b/test/metabase/metabot/tools/util_test.clj
@@ -242,170 +242,38 @@
(is (thrown? clojure.lang.ExceptionInfo
(metabot.tools.util/get-table inactive-table-id)))))))
-(deftest parse-field-id-test
- (testing "parse-field-id parses valid field IDs correctly"
- (testing "table field IDs with numeric table ID"
- (is (= {:model-tag "t", :model-id 154, :field-index 1}
- (metabot.tools.util/parse-field-id "t154-1")))
- (is (= {:model-tag "t", :model-id 1, :field-index 0}
- (metabot.tools.util/parse-field-id "t1-0")))
- (is (= {:model-tag "t", :model-id 999, :field-index 42}
- (metabot.tools.util/parse-field-id "t999-42"))))
-
- (testing "card/model/metric field IDs with numeric card ID"
- (is (= {:model-tag "c", :model-id 125, :field-index 7}
- (metabot.tools.util/parse-field-id "c125-7")))
- (is (= {:model-tag "c", :model-id 2, :field-index 0}
- (metabot.tools.util/parse-field-id "c2-0"))))
-
- (testing "query field IDs with nano-id string"
- (is (= {:model-tag "q", :model-id "puL95JSvym3k23W1UUuog", :field-index 0}
- (metabot.tools.util/parse-field-id "qpuL95JSvym3k23W1UUuog-0")))
- (is (= {:model-tag "q", :model-id "abc123XYZ", :field-index 5}
- (metabot.tools.util/parse-field-id "qabc123XYZ-5"))))
-
- (testing "query field IDs with nano-id containing dashes"
- (is (= {:model-tag "q", :model-id "wG9GfYTcE-wKTg3wlZyuc", :field-index 6}
- (metabot.tools.util/parse-field-id "qwG9GfYTcE-wKTg3wlZyuc-6")))
- (is (= {:model-tag "q", :model-id "a-b-c", :field-index 0}
- (metabot.tools.util/parse-field-id "qa-b-c-0"))))))
-
-(deftest resolve-column-test
- (testing "resolve-column resolves field IDs to columns"
- (let [columns [{:name "ID" :table-id 1 :type :number} ; index 0
- {:name "NAME" :table-id 1 :type :string} ; index 1
- {:name "EMAIL" :table-id 1 :type :string} ; index 2
- {:name "USER_ID" :table-id 2 :type :number} ; index 0 for table 2
- {:name "TOTAL" :table-id 2 :type :number}]] ; index 1 for table 2
- (testing "resolves table field IDs correctly"
- (let [item {:field-id "t1-0" :operation "equals"}
- result (metabot.tools.util/resolve-column item "t1-" columns)]
- (is (= "ID" (get-in result [:column :name])))
- (is (= 1 (get-in result [:column :table-id])))
- (is (= :number (get-in result [:column :type])))))
-
- (testing "resolves field IDs using flat index into columns vector"
- (let [item {:field-id "t1-3" :operation "equals"}
- result (metabot.tools.util/resolve-column item "t1-" columns)]
- (is (= "USER_ID" (get-in result [:column :name])))
- (is (= 2 (get-in result [:column :table-id])))))
-
- (testing "resolves card field IDs using flat index"
- (let [item {:field-id "c125-1" :operation "equals"}
- result (metabot.tools.util/resolve-column item "c125-" columns)]
- (is (= "NAME" (get-in result [:column :name])))
- (is (= 1 (get-in result [:column :table-id])))))
-
- (testing "resolves query field IDs using flat index"
- (let [item {:field-id "qabc123-2" :operation "equals"}
- result (metabot.tools.util/resolve-column item "qabc123-" columns)]
- (is (= "EMAIL" (get-in result [:column :name])))))))
-
- (testing "resolve-column throws for invalid field IDs"
- (let [columns [{:name "ID" :table-id 1 :type :number}
- {:name "NAME" :table-id 1 :type :string}]]
- (testing "throws for invalid field ID format"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Invalid field_id format"
- (metabot.tools.util/resolve-column {:field-id "invalid"} "t1-" columns)))
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Invalid field_id format"
- (metabot.tools.util/resolve-column {:field-id "t154/1"} "t154-" columns))))
-
- (testing "throws when field index is out of bounds"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"field t1-10 not found"
- (metabot.tools.util/resolve-column {:field-id "t1-10"} "t1-" columns))))
-
- (testing "throws when field ID prefix doesn't match expected prefix"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"does not match expected prefix"
- (metabot.tools.util/resolve-column {:field-id "t999-0"} "t1-" columns)))))))
-
-;;; Nil handling tests - important for robustness against LLM edge cases
-
-(deftest parse-field-id-nil-handling-test
- (testing "returns nil for nil input (no NPE)"
- (is (nil? (metabot.tools.util/parse-field-id nil))))
-
- (testing "returns nil for empty string"
- (is (nil? (metabot.tools.util/parse-field-id ""))))
-
- (testing "returns nil for non-string input"
- (is (nil? (metabot.tools.util/parse-field-id 123)))
- (is (nil? (metabot.tools.util/parse-field-id {:field-id "t1-0"})))
- (is (nil? (metabot.tools.util/parse-field-id [:t 1 0]))))
-
- (testing "returns nil for invalid format strings"
- (is (nil? (metabot.tools.util/parse-field-id "invalid")))
- (is (nil? (metabot.tools.util/parse-field-id "x1-0"))) ; invalid model tag
- (is (nil? (metabot.tools.util/parse-field-id "t-1"))) ; missing model-id
- (is (nil? (metabot.tools.util/parse-field-id "t1-"))) ; missing field-index
- (is (nil? (metabot.tools.util/parse-field-id "t1"))) ; no separator
- (is (nil? (metabot.tools.util/parse-field-id "-1-0"))) ; missing model tag
- ))
-
-(deftest resolve-column-nil-handling-test
- (testing "throws informative error for nil field-id"
- (let [columns [{:name "ID" :table-id 1 :type :number}]]
+(deftest find-column-by-field-id-test
+ (testing "finds column by integer field ID"
+ (let [columns [{:id 301 :name "ID"}
+ {:id 302 :name "NAME"}
+ {:id 303 :name "EMAIL"}]]
+ (is (= {:id 301 :name "ID"} (metabot.tools.util/find-column-by-field-id 301 columns)))
+ (is (= {:id 303 :name "EMAIL"} (metabot.tools.util/find-column-by-field-id 303 columns)))))
+
+ (testing "finds column by string-encoded field ID"
+ (let [columns [{:id 301 :name "ID"}
+ {:id 302 :name "NAME"}]]
+ (is (= {:id 302 :name "NAME"} (metabot.tools.util/find-column-by-field-id "302" columns)))))
+
+ (testing "throws agent error when field ID not found"
+ (let [columns [{:id 301 :name "ID"}]]
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
- #"Invalid field_id format"
- (metabot.tools.util/resolve-column {:field-id nil} "t1-" columns)))))
+ #"Field 999 not found"
+ (metabot.tools.util/find-column-by-field-id 999 columns)))))
- (testing "throws informative error for empty field-id"
- (let [columns [{:name "ID" :table-id 1 :type :number}]]
+ (testing "throws for nil field ID"
+ (let [columns [{:id 301 :name "ID"}]]
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
- #"Invalid field_id format"
- (metabot.tools.util/resolve-column {:field-id ""} "t1-" columns)))))
+ #"not found"
+ (metabot.tools.util/find-column-by-field-id nil columns)))))
- (testing "throws informative error for numeric field-id"
- (let [columns [{:name "ID" :table-id 1 :type :number}]]
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"Invalid field_id format"
- (metabot.tools.util/resolve-column {:field-id 123} "t1-" columns)))))
-
- (testing "error data contains agent-error? flag for user-friendly handling"
- (let [columns [{:name "ID" :table-id 1 :type :number}]]
+ (testing "error data contains agent-error? flag"
+ (let [columns [{:id 301 :name "ID"}]]
(try
- (metabot.tools.util/resolve-column {:field-id nil} "t1-" columns)
- (is false "Expected exception to be thrown")
+ (metabot.tools.util/find-column-by-field-id 999 columns)
+ (is false "Expected exception")
(catch clojure.lang.ExceptionInfo e
(is (:agent-error? (ex-data e)))
- (is (= nil (:field-id (ex-data e)))))))))
-
-(deftest resolve-column-edge-cases-test
- (testing "handles empty columns vector"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"not found"
- (metabot.tools.util/resolve-column {:field-id "t1-0"} "t1-" []))))
-
- (testing "handles columns vector with nil entries at accessed index"
- ;; Index 0 is nil, so get returns nil and throws an error
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"not found"
- (metabot.tools.util/resolve-column {:field-id "t1-0"} "t1-" [nil {:name "NAME"}]))))
-
- (testing "handles columns vector with nil entries at other indices"
- ;; Access index 1 which has valid data, index 0 is nil but not accessed
- (let [columns [nil {:name "NAME" :table-id 1 :type :string}]
- result (metabot.tools.util/resolve-column {:field-id "t1-1"} "t1-" columns)]
- (is (some? (:column result)))
- (is (= "NAME" (get-in result [:column :name])))))
-
- (testing "preserves other item keys when resolving"
- (let [columns [{:name "ID" :table-id 1 :type :number}]
- item {:field-id "t1-0" :operation "equals" :value 42 :custom-key "preserved"}
- result (metabot.tools.util/resolve-column item "t1-" columns)]
- (is (= "equals" (:operation result)))
- (is (= 42 (:value result)))
- (is (= "preserved" (:custom-key result)))
- (is (some? (:column result))))))
+ (is (= 404 (:status-code (ex-data e)))))))))
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index 1b5f53dbbd7..f7326f7a8b4 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -6,7 +6,7 @@
[metabase.metabot.scope :as scope]
[metabase.metabot.tools :as agent-tools]
[metabase.metabot.tools.charts.create :as create-chart-tools]
- [metabase.metabot.tools.filters :as filter-tools]))
+ [metabase.metabot.tools.construct :as construct]))
(deftest all-tools-test
(testing "profile tools are vars with required metadata"
@@ -113,14 +113,16 @@
(is (= "search" (:tool-name m)))
(is (some? (:schema m))))))
-(deftest construct-notebook-query-tool-raw-table-test
- (testing "construct_notebook_query maps raw table query args"
- (let [captured (atom nil)
- chart-called (atom nil)]
- (with-redefs [filter-tools/query-datasource (fn [args]
- (reset! captured args)
- {:structured-output {:query-id "q-1"
- :query {:database 1}}})
+(deftest construct-notebook-query-tool-test
+ (testing "construct_notebook_query evaluates a program and creates a chart"
+ (let [program-captured (atom nil)
+ chart-called (atom nil)]
+ (with-redefs [construct/execute-program (fn [_source-entity _referenced-entities program]
+ (reset! program-captured program)
+ {:structured-output {:query-id "q-1"
+ :query {:database 1}
+ :result-columns []}
+ :instructions "Query created."})
create-chart-tools/create-chart (fn [args]
(reset! chart-called args)
{:chart-id "c-1"
@@ -132,20 +134,15 @@
:url "/question#hash"}]})]
(let [result (agent-tools/construct-notebook-query-tool
{:reasoning "check seats"
- :query {:query_type "raw"
- :source {:table_id 6}
- :filters [{:filter_type "multi_value"
- :field_id "t6-1"
- :operation "equals"
- :values ["a" "b"]}]
- :fields [{:field_id "t6-1"}]
- :order_by [{:field {:field_id "t6-1"} :direction "desc"}]
- :limit 10}
+ :source_entity {:type "table" :id 6}
+ :program {:source {:type "context" :ref "source"}
+ :operations [["filter" ["=" ["field" 301] "a"]]
+ ["with-fields" [["field" 301]]]
+ ["order-by" ["field" 301] "desc"]
+ ["limit" 10]]}
:visualization {:chart_type "table"}})]
- (is (= 6 (:table-id @captured)))
- (is (= "t6-1" (get-in @captured [:fields 0 :field-id])))
- (is (= ["a" "b"] (get-in @captured [:filters 0 :values])))
- (is (= :desc (get-in @captured [:order-by 0 :direction])))
+ (is (= "context" (get-in @program-captured [:source :type])))
+ (is (= 4 (count (:operations @program-captured))))
(is (= "c-1" (get-in result [:structured-output :chart-id])))
(is (= "q-1" (get-in result [:structured-output :query-id])))
(is (= :table (get @chart-called :chart-type)))