The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
BOT-1524 Save tables used in a metabot conversation (#74254)
applebyd4238abb10d1
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 2a415cb35a1..61bdbd4ceed 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -95,6 +95,7 @@"MetabotMessage""MetabotPermissions""MetabotSourceFeedback"+ "MetabotUsedTable""ModelIndex""ModelIndexValue""ModerationReview"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 99ae9f98970..09ffee3979e 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -69,6 +69,7 @@:model/MetabotMessage:model/MetabotPermissions:model/MetabotSourceFeedback+ :model/MetabotUsedTable:model/ModelIndex:model/ModelIndexValue:model/ModerationReviewdiff --git a/resources/migrations/061/20260515_metabot_used_table.yaml b/resources/migrations/061/20260515_metabot_used_table.yamlnew file mode 100644index 00000000000..7404ede3ad8--- /dev/null+++ b/resources/migrations/061/20260515_metabot_used_table.yaml@@ -0,0 +1,87 @@+## Persist tables referenced by Metabot-generated queries++databaseChangeLog:+ - changeSet:+ id: v61.zk7xq4+ author: appleby+ comment: metabot_used_table table+ changes:+ - createTable:+ tableName: metabot_used_table+ remarks: Record of which database tables Metabot referenced in generated a queries. Source cards, models, and metrics are recursively expanded to their underlying tables before insertion.+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ remarks: Auto-incrementing primary key+ - column:+ name: message_id+ type: int+ constraints:+ nullable: false+ remarks: 'FK to the Metabot message whose query referenced this table'+ - column:+ name: table_id+ type: int+ constraints:+ nullable: false+ remarks: 'FK to the underlying database table the generated query references'+ - column:+ name: created_at+ type: ${timestamp_type}+ defaultValueComputed: current_timestamp+ constraints:+ nullable: false+ remarks: 'When this reference was recorded'++ - changeSet:+ id: v61.zk7xq4-uq+ author: appleby+ comment: Unique constraint on metabot_used_table (message_id, table_id)+ changes:+ - addUniqueConstraint:+ tableName: metabot_used_table+ columnNames: message_id, table_id+ constraintName: uq_metabot_used_table_message_table++ - changeSet:+ id: v61.zk7xq4-idx-table+ author: appleby+ comment: Index on metabot_used_table.table_id for reverse reads and FK lookups+ changes:+ - createIndex:+ tableName: metabot_used_table+ indexName: idx_metabot_used_table_table_id+ columns:+ - column:+ name: table_id++ - changeSet:+ id: v61.zk7xq4-fk-msg+ author: appleby+ comment: FK from metabot_used_table.message_id to metabot_message+ changes:+ - addForeignKeyConstraint:+ baseTableName: metabot_used_table+ baseColumnNames: message_id+ constraintName: fk_metabot_used_table_message_id+ referencedTableName: metabot_message+ referencedColumnNames: id+ onDelete: CASCADE++ - changeSet:+ id: v61.zk7xq4-fk-table+ author: appleby+ comment: FK from metabot_used_table.table_id to metabase_table+ changes:+ - addForeignKeyConstraint:+ baseTableName: metabot_used_table+ baseColumnNames: table_id+ constraintName: fk_metabot_used_table_table_id+ referencedTableName: metabase_table+ referencedColumnNames: id+ onDelete: CASCADEdiff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 3d8fa3952a4..53e1b7290c6 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -642,6 +642,21 @@(prometheus/counter :metabase-metabot/turn-started{:description "A metabot turn was started (user row + assistant placeholder inserted)":labels [:profile-id]})+ (prometheus/counter :metabase-metabot/used-tables-extraction-total+ {:description "Number of used-tables extractions attempted."})+ (prometheus/counter :metabase-metabot/used-tables-extraction-errors+ {:description "Number of used-tables extractions that threw an unhandled exception."})+ (prometheus/counter :metabase-metabot/used-tables-extraction-warnings+ {:description "Number of used-table extraction failures caught and logged (may be >1 per invocation)."+ :labels [:reason]})+ (prometheus/counter :metabase-metabot/used-tables-extraction-dropped+ {:description "Number of used-tables extraction tasks dropped because the background executor's queue was full."})+ (prometheus/counter :metabase-metabot/used-tables-extraction-timeouts+ {:description "Number of used-tables extractions abandoned because they exceeded the extraction timeout."})+ (prometheus/histogram :metabase-metabot/used-tables-extraction-duration-ms+ {:description "Duration in milliseconds of used-tables extraction."+ ;; 1ms -> 30s+ :buckets [1 5 10 25 50 100 250 500 1000 2500 5000 10000 30000]});; release dashboard metrics(prometheus/counter :metabase-sync/failures{:description "Number of sync operation failures."diff --git a/src/metabase/cmd/copy.clj b/src/metabase/cmd/copy.cljindex 1135bc75afc..fa3ea135334 100644--- a/src/metabase/cmd/copy.clj+++ b/src/metabase/cmd/copy.clj@@ -136,6 +136,7 @@:model/MetabotMessage:model/MetabotFeedback:model/MetabotSourceFeedback+ :model/MetabotUsedTable:model/MetabotPrompt](when config/ee-available?[:model/MetabotPermissionsdiff --git a/src/metabase/metabot/models/metabot_used_table.clj b/src/metabase/metabot/models/metabot_used_table.cljnew file mode 100644index 00000000000..d8941a312a0--- /dev/null+++ b/src/metabase/metabot/models/metabot_used_table.clj@@ -0,0 +1,12 @@+(ns metabase.metabot.models.metabot-used-table+ "Persist which database tables Metabot referenced when generating a query.+ Source cards / models / metrics are recursively expanded to their underlying+ tables by [[metabase.metabot.used-tables]] before insertion."+ (:require+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/MetabotUsedTable [_model] :metabot_used_table)++(doto :model/MetabotUsedTable+ (derive :metabase/model))diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.cljindex 71cef823b9a..06a30c73e25 100644--- a/src/metabase/metabot/persistence.clj+++ b/src/metabase/metabot/persistence.clj@@ -7,6 +7,7 @@[metabase.metabot.agent.streaming :as streaming][metabase.metabot.provider-util :as provider-util][metabase.metabot.settings :as metabot.settings]+ [metabase.metabot.used-tables :as used-tables][metabase.util :as u][metabase.util.json :as json][metabase.util.log :as log]@@ -234,10 +235,12 @@(= "state" (:data-type %)))parts)usage (extract-usage parts)- content (->> parts+ ;; used-table extraction needs the value before `strip-tool-output-bloat` so it can see keys that strip+ ;; discards, e.g. `:transform`+ pre-strip (->> parts(remove #(#{:start :usage :finish :error} (:type %)))- (filter streaming/persistable-data-part?)- (mapv strip-tool-output-bloat))]+ (filter streaming/persistable-data-part?))+ content (mapv strip-tool-output-bloat pre-strip)](analytics/observe! :metabase-metabot/message-persist-bytes{:profile-id (or profile-id "unknown")}(u/string-byte-count (json/encode content)))@@ -254,7 +257,11 @@:finished (boolean finished?):error (safe-encode-error error)}slack-msg-id (assoc :slack_msg_id slack-msg-id)- channel-id (assoc :channel_id channel-id))))))+ channel-id (assoc :channel_id channel-id))))+ ;; Hand the (potentially slow) used-table extraction + insert off to a background worker *after* the message+ ;; UPDATE transaction commits, so it neither blocks nor fails the turn. The assistant row already exists, so its+ ;; `message_id` FK is valid even before the UPDATE completes.+ (used-tables/record-used-tables! assistant-msg-id pre-strip)))(defn set-response-slack-msg-id!"Backfill slack_msg_id on a MetabotMessage by primary key."diff --git a/src/metabase/metabot/used_tables.clj b/src/metabase/metabot/used_tables.cljnew file mode 100644index 00000000000..a26b65598f8--- /dev/null+++ b/src/metabase/metabot/used_tables.clj@@ -0,0 +1,417 @@+(ns metabase.metabot.used-tables+ "Extract table references from queries generated by the tool calls in a metabot message.++ Walks `:tool-input` / `:tool-output` pairs, dispatches on tool name:+ - `construct_notebook_query` — query in `:structured-output.query`.+ - `create_sql_query` / `replace_sql_query` / `edit_sql_query` — native SQL.+ - `write_transform_sql` — native query in `:structured-output.transform.source.query`.+ - `write_transform_python` — table ids declared explicitly in the+ `:source_tables` tool-input argument.++ For both native and mbql queries, we recursively expand card references to extract underlying source tables.++ - Collects table and card ids via [[metabase.lib.core/all-referenced-entity-ids]]. The metadata provider is shared+ across the whole extraction via [[metabase.lib-be.core/with-metadata-provider-cache]].+ - All tool-call pairs in a single message feed into one shared BFS, so a card referenced from multiple pairs is+ resolved at most once per extraction. This matters for iterative-refinement patterns where the agent regenerates+ the same query shape and sources or template-tags the same upstream card on each pass.+ - Treats `:metric` refs as cards for recursion. Enqueueing their `:dataset_query` collapses metric references down+ to their underlying source tables alongside questions and models.+ - For native stages, runs the sql parser via [[metabase.metabot.query-analyzer/tables-for-native]] to recover+ tables inside raw SQL text.+ - Visited card ids are tracked to break cycles."+ (:require+ [clojure.set :as set]+ [metabase.analytics-interface.core :as analytics]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.metabot.query-analyzer :as nqa]+ [metabase.metabot.tools :as metabot.tools]+ [metabase.util :as u]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent ArrayBlockingQueue Callable ExecutionException ExecutorService Future RejectedExecutionException ThreadFactory ThreadPoolExecutor TimeoutException TimeUnit)))++(set! *warn-on-reflection* true)++(defn- record-extraction-warning!+ "Increment the used-tables-extraction-warnings prometheus counter."+ [reason]+ (analytics/inc! :metabase-metabot/used-tables-extraction-warnings {:reason reason}))++(def ^:private notebook-tool-name "construct_notebook_query")+(def ^:private transform-sql-tool-name "write_transform_sql")+(def ^:private transform-python-tool-name "write_transform_python")++(def ^:private transform-tool-names+ #{transform-sql-tool-name transform-python-tool-name})++(def ^:private tracked-tool-names+ "Union of every tool name whose successful invocation we mine for table references. We keep the transform tools+ out of [[metabase.metabot.tools/query-generation-tool-names]] because the metabot_analytics module makes shape+ assumptions about that set (`:query-content`/`:database`/etc. in structured-output) that transform outputs do not+ satisfy."+ (into transform-tool-names metabot.tools/query-generation-tool-names))++(defn- tool-output->structured+ [result]+ (when (map? result)+ (or (:structured-output result)+ (:structured_output result))))++(defn- successful-tracked-output?+ "Is this a successful invocation of a tool in tracked-tool-names?"+ [input-block output-block]+ (and output-block+ (contains? tracked-tool-names (:function input-block))+ (not (:error output-block))+ (some? (tool-output->structured (:result output-block)))))++(defn- ->mbql5-query+ "Convert a raw query (legacy or MBQL 5) into an MBQL 5 query with `metadata-providerable` attached.+ Returns nil on conversion failure."+ [metadata-providerable raw-query]+ (try+ (lib/query metadata-providerable raw-query)+ (catch Exception e+ (record-extraction-warning! :query-build)+ (log/warn e "Failed to convert query to MBQL 5")+ (log/debugf "Failed to convert query to MBQL 5: %s" (pr-str raw-query))+ nil)))++(defn- native-tables+ "If `query` is native, run `tables-for-native` against it and return its underlying table ids."+ [query]+ (if (lib/native? query)+ (try+ (let [result (nqa/tables-for-native query)]+ (if (:error result)+ (do (record-extraction-warning! :native-parse)+ (log/warnf "tables-for-native error: %s" (:error result))+ (log/debugf "tables-for-native error: %s; query: %s" (:error result) (pr-str query))+ #{})+ (into #{} (keep :table-id) (:tables result))))+ (catch Exception e+ (record-extraction-warning! :native-parse)+ (log/warn e "Failed to extract tables from native SQL")+ (log/debugf "Failed to extract tables from native SQL: %s" (pr-str query))+ #{}))+ #{}))++(defn- query-tables-and-cards+ "Walk one query and return `{:tables #{table-id ...} :cards #{card-id ...}}`.++ `:tables` includes `:source-table` ids, template-tag table ids, the parent tables of implicitly-joined field refs,+ and any tables found by `native-tables`.++ `:cards` includes `:source-card`, template-tag `:card-id`, and `:metric` ref ids."+ [query]+ (try+ (let [ids (lib/all-referenced-entity-ids [query])]+ {:tables (set/union (:table ids)+ (native-tables query))+ :cards (set/union (:card ids) (:metric ids))})+ (catch Exception e+ (record-extraction-warning! :query-walk)+ (log/warn e "Failed to walk query for used-table extraction")+ (log/debugf "Failed to walk query for used-table extraction: %s" (pr-str query))+ {:tables #{} :cards #{}})))++(defn- card-query+ "Look up the Card with `card-id` via `metadata-providerable` and return its underlying query. Warns and returns+ nil if the card is missing or the conversion fails."+ [metadata-providerable card-id]+ (try+ (if-let [card (lib.metadata/card metadata-providerable card-id)]+ (lib/card->underlying-query metadata-providerable card)+ (do (record-extraction-warning! :card-missing)+ (log/warnf "Card %s referenced by Metabot query was not found" card-id)+ nil))+ (catch Exception e+ (record-extraction-warning! :card-lookup)+ (log/warnf e "Failed to look up card %s for used-table extraction" card-id)+ nil)))++(defn- collect-tables+ "BFS over a queue of MBQL queries, accumulating referenced table ids.++ Tracks visited card ids to break cycles and to skip redundant work when the same card is referenced from multiple+ seed queries.++ Called once per [[extract-used-tables]] invocation with seed queries gathered from every tool-call pair, so the+ `visited` set is shared across pairs."+ [initial-queries]+ (loop [tables #{}+ queue (vec initial-queries)+ visited #{}]+ (if (empty? queue)+ tables+ (let [query (first queue)+ queue' (subvec queue 1)+ {ts :tables cs :cards} (query-tables-and-cards query)+ new-cards (set/difference cs visited)+ new-queries (keep #(card-query query %) new-cards)]+ (recur (into tables ts)+ (into queue' new-queries)+ (into visited new-cards))))))++(defn- notebook-starting-query+ "Starting query for a `construct_notebook_query` tool output."+ [metadata-providerable structured]+ (some->> (:query structured)+ (->mbql5-query metadata-providerable)))++(defn- sql-raw-text+ "Best-effort raw SQL text for a SQL tool call."+ [tool-name arguments structured]+ (or (:query-content structured)+ (case tool-name+ "create_sql_query" (:sql_query arguments)+ "replace_sql_query" (:new_query arguments)+ nil)))++(defn- sql-starting-query+ "Starting query for an SQL tool output.+ Prefers the structured output's full `:query` map; falls back to building a native query from the structured+ output's `:database` field plus the tool's raw SQL text."+ [metadata-providerable tool-name arguments structured]+ (or (notebook-starting-query metadata-providerable structured)+ (when-let [db-id (:database structured)]+ (when-let [sql (sql-raw-text tool-name arguments structured)]+ (when (seq sql)+ (try+ (lib/native-query (lib.metadata/->metadata-provider metadata-providerable db-id) sql)+ (catch Exception e+ (record-extraction-warning! :native-build)+ (log/warnf e "Failed to build native query for %s from raw SQL on database %s" tool-name db-id)+ (log/debugf "Failed to build native query for %s from raw SQL on database %s: %s" tool-name db-id sql)+ nil)))))))++(defn- transform-sql-starting-query+ "Starting query for a `write_transform_sql` tool output."+ [metadata-providerable structured]+ (some->> (get-in structured [:transform :source :query])+ (->mbql5-query metadata-providerable)))++(defn- transform-declared-table-ids+ "Read `:table_id` values from a `write_transform_*` tool's `:source_tables` argument.+ Required by the Python tool's schema; absent from the SQL tool's schema."+ [arguments]+ (into #{} (keep :table_id) (:source_tables arguments)))++(def ^:private empty-seeds {:queries [] :tables #{}})++(defn- pair->seeds+ "Return `{:queries [...] :tables #{...}}` for one `(tool-input, tool-output)` pair.++ `:queries` are MBQL queries to feed into the shared BFS in [[extract-used-tables]].++ `:tables` are already-known table ids. Currently only populated by the Python transform tool, which declares its+ sources in the `:source_tables` argument rather than embedding them in a query."+ [metadata-providerable input-block output-block]+ (let [tool-name (:function input-block)+ arguments (:arguments input-block)+ structured (tool-output->structured (:result output-block))]+ (cond+ (= tool-name notebook-tool-name)+ (if-let [start (notebook-starting-query metadata-providerable structured)]+ {:queries [start] :tables #{}}+ empty-seeds)++ (= tool-name transform-sql-tool-name)+ {:queries (if-let [start (transform-sql-starting-query metadata-providerable structured)]+ [start]+ [])+ :tables (transform-declared-table-ids arguments)}++ (= tool-name transform-python-tool-name)+ {:queries [] :tables (transform-declared-table-ids arguments)}++ :else+ (if-let [start (sql-starting-query metadata-providerable tool-name arguments structured)]+ {:queries [start] :tables #{}}+ empty-seeds))))++(defn- ->rows+ [message-id table-ids]+ (mapv (fn [id] {:message_id message-id :table_id id})+ table-ids))++(defn- extract-used-tables+ "Walk the `parts` vector and return rows for `metabot_used_table`.++ One row per distinct `(message-id, table-id)` pair. Returns `[]` when nothing was referenced.++ Runs a single BFS across all tool-call pairs, so a card referenced from multiple pairs is resolved at most once per+ call.++ Must run on the pre-strip `parts` before [[metabase.metabot.persistence/strip-tool-output-bloat]] trims tool-output;+ the transform path needs the un-trimmed `:transform` key, which is not+ in [[metabase.metabot.persistence/persisted-structured-output-keys]]."+ ([message-id parts]+ (lib-be/with-metadata-provider-cache+ (extract-used-tables lib-be/application-database-metadata-provider message-id parts)))+ ([metadata-providerable message-id parts]+ (let [outputs-by-id (into {}+ (comp (filter #(= :tool-output (:type %)))+ (map (juxt :id identity)))+ parts)+ pairs (into []+ (comp (filter #(= :tool-input (:type %)))+ (keep (fn [input]+ (let [output (get outputs-by-id (:id input))]+ (when (successful-tracked-output? input output)+ [input output])))))+ parts)+ {:keys [queries tables]}+ (reduce (fn [acc [input output]]+ (let [seeds (pair->seeds metadata-providerable input output)]+ (-> acc+ (update :queries into (:queries seeds))+ (update :tables into (:tables seeds)))))+ empty-seeds+ pairs)+ all-tables (into tables (collect-tables queries))]+ (->rows message-id all-tables))))++(def ^:dynamic *run-synchronously?*+ "When true, [[record-used-tables!]] extracts and inserts on the calling thread instead of the background+ executor. Bound to true in tests so assertions can read `metabot_used_table` rows immediately after a turn is+ finalized, and so the work participates in the test's rollback transaction."+ false)++(def ^:private extraction-timeout-ms+ "Hard cap on a single used-table extraction. On timeout we abandon the work and persist no rows for the message.+ This is safeguard against bugs or pathological queries. Under normal circumstances, even for large queries,+ extraction should not take more than a few seconds."+ (* 15 1000))++(def ^:private executor-queue-capacity+ "Max number of tasks allowed to sit in either pool's queue. Beyond this, new submissions are rejected.+ 256 caps worst-case retained heap in the low hundreds of MiB; the intention is that the queues are normally+ ~empty (extraction is far faster than turns arrive), and this only kicks in under a regression/stall/abuse."+ 256)++(def ^:private pool-size+ "Max threads in each of [[waiter-executor]] and [[worker-executor]]. Caps concurrent extractions:+ every [[waiter-executor]] has a matching [[worker-executor]]. Threads are created on demand and reaped+ after [[idle-timeout-ms]]."+ 4)++(def ^:private idle-timeout-ms+ "How long an idle thread lives before being reaped. With `allowCoreThreadTimeOut`, this applies to every thread,+ so a pool that has been idle this long scales back down to zero threads."+ (* 5 60 1000))++(defn- thread-pool-executor+ "Pool of up to [[pool-size]] daemon threads named `thread-name`, created on demand.++ Backed by a bounded queue of [[executor-queue-capacity]]. The default `AbortPolicy` throws+ `RejectedExecutionException` once the queue is full. Idle threads are reaped after [[idle-timeout-ms]]."+ ^ThreadPoolExecutor [thread-name]+ (doto (ThreadPoolExecutor.+ (int pool-size)+ (int pool-size)+ (long idle-timeout-ms) TimeUnit/MILLISECONDS+ (ArrayBlockingQueue. (int executor-queue-capacity))+ (reify ThreadFactory+ (newThread [_ r]+ (doto (Thread. r)+ (.setName thread-name)+ (.setDaemon true)))))+ (.allowCoreThreadTimeOut true)))++(defonce ^:private waiter-executor+ ;; Outer pool: runs the `extract-and-insert!` tasks submitted by `record-used-tables!`. Each task blocks waiting+ ;; (with a timeout) on the result of an extraction submitted to `worker-executor`. Bounding this pool caps how many+ ;; turns we process concurrently; the bounded queue caps how many pending requests we retain. The default+ ;; `AbortPolicy` rejects beyond that, which `record-used-tables!` catches to drop the task. Keeping the work on+ ;; these bounded pools (rather than `u/with-timeout`'s unbounded shared `future` pool) means a stall can't spill+ ;; onto threads shared with the rest of the app.+ (delay (thread-pool-executor "metabot-used-tables-waiter")))++(defonce ^:private worker-executor+ ;; Inner pool: runs the (usually fast, but potentially slow) BFS+sql parse. Kept separate from `waiter-executor` so+ ;; a burst of waiters can never starve the workers doing the extractions. Sized to match `waiter-executor` so every+ ;; waiter has a worker. Rows are keyed by distinct `message_id`, so concurrent inserts don't contend.+ (delay (thread-pool-executor "metabot-used-tables-worker")))++(defn- extract-used-tables-async-with-timeout+ "Run [[extract-used-tables]] on [[worker-executor]] and block the calling [[waiter-executor]] thread on the result for+ up to [[extraction-timeout-ms]]. On timeout the work is cancelled and a `TimeoutException` is thrown. Any extraction+ failure is unwrapped from its `ExecutionException` and rethrown."+ [message-id parts]+ (let [fut ^Future (.submit ^ExecutorService @worker-executor+ ^Callable (bound-fn* #(extract-used-tables message-id parts)))]+ (try+ (.get fut extraction-timeout-ms TimeUnit/MILLISECONDS)+ (catch ExecutionException e+ (throw (.getCause e)))+ (catch TimeoutException e+ (.cancel fut true)+ (throw e)))))++(defn- extract-used-tables-with-timing!+ "Like the 2-arity [[extract-used-tables]] but records Prometheus timing/failure metrics and caps runtime at+ [[extraction-timeout-ms]]. Returns nil (persisting no rows) and counts a timeout if the cap is exceeded.++ When not running synchronously the work is handed off via [[extract-used-tables-async-with-timeout]]. The+ timeout/handoff is skipped when [[*run-synchronously?*]] is set (tests), so extraction stays on the calling thread+ and inside the test transaction."+ [message-id parts]+ (let [start (u/start-timer)]+ (analytics/inc! :metabase-metabot/used-tables-extraction-total)+ (try+ (let [result (if *run-synchronously?*+ (extract-used-tables message-id parts)+ (extract-used-tables-async-with-timeout message-id parts))]+ (analytics/observe! :metabase-metabot/used-tables-extraction-duration-ms (long (u/since-ms start)))+ result)+ (catch TimeoutException _+ (analytics/inc! :metabase-metabot/used-tables-extraction-timeouts)+ (analytics/observe! :metabase-metabot/used-tables-extraction-duration-ms (long (u/since-ms start)))+ (log/warnf "Used-table extraction for message %s exceeded the %sms timeout; abandoning extraction"+ message-id extraction-timeout-ms)+ nil)+ (catch Throwable t+ (analytics/inc! :metabase-metabot/used-tables-extraction-errors)+ (analytics/observe! :metabase-metabot/used-tables-extraction-duration-ms (long (u/since-ms start)))+ (throw t)))))++(defn- extract-and-insert!+ "Extract used tables from `parts` and insert `metabot_used_table` rows for `message-id`.+ Best-effort: any `Exception` (extraction *or* insert) is logged and swallowed, since this runs+ off the turn-finalization path and must never surface to the user. For example, a table+ deleted between query generation and now would make the row's `table_id` FK invalid."+ [message-id parts]+ (log/with-thread-context {:metabot_message_id message-id}+ (try+ (let [rows (extract-used-tables-with-timing! message-id parts)]+ (when (seq rows)+ (t2/insert! :model/MetabotUsedTable rows)))+ (catch Exception e+ (log/warn e "Failed to record metabot used tables for message" message-id)))))++(defn record-used-tables!+ "Extract used tables from `parts` and persist `metabot_used_table` rows for `message-id`.++ The extraction can be slow for large native queries, so by default this is handed off to a background+ [[waiter-executor]] and returns immediately. Callers must already have committed the message row so the+ `message_id` FK is valid.++ When the background executor's bounded queue is full, the task is dropped (logged + counted), rather than blocking+ turn finalization or growing the queue without bound.++ Binding [[*run-synchronously?*]] true runs the work inline on the calling thread instead (used in tests)."+ [message-id parts]+ (if *run-synchronously?*+ (extract-and-insert! message-id parts)+ (try+ (.submit ^ExecutorService @waiter-executor+ ^Callable (bound-fn* #(extract-and-insert! message-id parts)))+ (catch RejectedExecutionException _+ (analytics/inc! :metabase-metabot/used-tables-extraction-dropped)+ (log/warnf "used-tables extraction queue full (capacity %d); dropping extraction for message %s"+ executor-queue-capacity message-id)))))diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex ad5d4d1f162..d905f02c4fd 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -63,6 +63,7 @@:model/MetabotFeedback metabase.metabot.models.metabot-feedback:model/MetabotMessage metabase.metabot.models.metabot-message:model/MetabotSourceFeedback metabase.metabot.models.metabot-source-feedback+ :model/MetabotUsedTable metabase.metabot.models.metabot-used-table:model/AiUsageLog metabase.metabot.models.ai-usage-log:model/DataComplexityScore metabase-enterprise.data-complexity-score.models.data-complexity-score:model/MetabotGroupLimit metabase-enterprise.metabot.models.metabot-group-limitdiff --git a/test/metabase/metabot/models/metabot_used_table_test.clj b/test/metabase/metabot/models/metabot_used_table_test.cljnew file mode 100644index 00000000000..a7126e24f80--- /dev/null+++ b/test/metabase/metabot/models/metabot_used_table_test.clj@@ -0,0 +1,68 @@+(ns metabase.metabot.models.metabot-used-table-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(deftest insert-and-round-trip-test+ (testing "inserting a row with a real table_id succeeds and round-trips"+ (mt/with-temp [:model/User {user-id :id} {}+ :model/MetabotConversation {conversation-id :id} {:user_id user-id}+ :model/MetabotMessage {msg-id :id} {:conversation_id conversation-id}]+ (let [table-id (mt/id :orders)]+ (t2/insert! :model/MetabotUsedTable+ {:message_id msg-id :table_id table-id})+ (is (=? {:message_id msg-id :table_id table-id}+ (t2/select-one :model/MetabotUsedTable :message_id msg-id)))))))++(deftest unique-constraint-test+ (testing "inserting two rows with the same (message_id, table_id) throws"+ (mt/with-temp [:model/User {user-id :id} {}+ :model/MetabotConversation {conversation-id :id} {:user_id user-id}+ :model/MetabotMessage {msg-id :id} {:conversation_id conversation-id}]+ (let [table-id (mt/id :orders)]+ (t2/insert! :model/MetabotUsedTable {:message_id msg-id :table_id table-id})+ ;; Wrap the failing INSERT in its own transaction so the outer+ ;; `mt/with-temp` transaction stays viable for cleanup.+ (is (thrown? Exception+ (t2/with-transaction [_conn]+ (t2/insert! :model/MetabotUsedTable+ {:message_id msg-id :table_id table-id}))))))))++(deftest unknown-table-rejected-test+ (testing "inserting with a table_id that does not exist violates the FK"+ (mt/with-temp [:model/User {user-id :id} {}+ :model/MetabotConversation {conversation-id :id} {:user_id user-id}+ :model/MetabotMessage {msg-id :id} {:conversation_id conversation-id}]+ (let [absent-table-id (+ 1000000 (rand-int 1000000))]+ (is (thrown? Exception+ (t2/with-transaction [_conn]+ (t2/insert! :model/MetabotUsedTable+ {:message_id msg-id :table_id absent-table-id}))))+ (is (zero? (t2/count :model/MetabotUsedTable :message_id msg-id)))))))++(deftest cascade-from-message-test+ (testing "deleting a MetabotMessage cascades to its used-table rows"+ (mt/with-temp [:model/User {user-id :id} {}+ :model/MetabotConversation {conversation-id :id} {:user_id user-id}+ :model/MetabotMessage {msg-id :id} {:conversation_id conversation-id}]+ (t2/insert! :model/MetabotUsedTable+ [{:message_id msg-id :table_id (mt/id :orders)}+ {:message_id msg-id :table_id (mt/id :people)}])+ (is (= 2 (t2/count :model/MetabotUsedTable :message_id msg-id)))+ (t2/delete! :model/MetabotMessage :id msg-id)+ (is (zero? (t2/count :model/MetabotUsedTable :message_id msg-id))))))++(deftest cascade-from-table-test+ (testing "deleting the referenced Table cascades to its used-table rows"+ (mt/with-temp [:model/User {user-id :id} {}+ :model/Database {db-id :id} {}+ :model/Table {table-id :id} {:db_id db-id}+ :model/MetabotConversation {conversation-id :id} {:user_id user-id}+ :model/MetabotMessage {msg-id :id} {:conversation_id conversation-id}]+ (t2/insert! :model/MetabotUsedTable {:message_id msg-id :table_id table-id})+ (is (= 1 (t2/count :model/MetabotUsedTable :message_id msg-id)))+ (t2/delete! :model/Table :id table-id)+ (is (zero? (t2/count :model/MetabotUsedTable :message_id msg-id))))))diff --git a/test/metabase/metabot/persistence_test.clj b/test/metabase/metabot/persistence_test.cljindex 63218e9cbfb..c1de0e09a42 100644--- a/test/metabase/metabot/persistence_test.clj+++ b/test/metabase/metabot/persistence_test.clj@@ -1,10 +1,16 @@(ns metabase.metabot.persistence-test(:require[clojure.test :refer [deftest is testing use-fixtures]]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.metabot.persistence :as metabot-persistence]+ [metabase.metabot.query-analyzer :as nqa]+ [metabase.metabot.used-tables :as used-tables][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.test.util :as tu][metabase.util.json :as json]+ [metabase.util.log.capture :as log.capture][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -120,7 +126,7 @@(is (= {:ok true} (json/decode+kw (:result (nth result 2)))))))(deftest start-turn-persists-slack-conversation-metadata-test- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))team-id "T123"channel-id "C123"@@ -163,7 +169,7 @@(testing "start-turn! lands slack-team-id / channel-id / slack-thread-ts on the conversation row,and slack-msg-id / channel-id / user-id on the user message row, plus user-id on theassistant placeholder row"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))team-id "T-NATIVE"channel-id "C-NATIVE"@@ -204,7 +210,7 @@(deftest start-turn-does-not-overwrite-slack-metadata-on-subsequent-turns-test(testing "conversation-level slack metadata is set once and never overwritten by a later turn"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))](mt/with-current-user (mt/user->id :rasta)(metabot-persistence/start-turn!@@ -226,7 +232,7 @@(deftest start-turn-returns-assistant-pk-and-external-id-test(testing "start-turn! returns the assistant placeholder PK and a fresh external_id;inserts exactly one user row + one assistant placeholder row"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))](mt/with-current-user (mt/user->id :rasta)(let [{:keys [assistant-msg-id assistant-external-id]}@@ -247,7 +253,7 @@(deftest finalize-assistant-turn-updates-placeholder-in-place-test(testing "finalize-assistant-turn! UPDATEs the placeholder; row count and created_at unchanged"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))](mt/with-current-user (mt/user->id :rasta)(let [{:keys [assistant-msg-id]} (metabot-persistence/start-turn!@@ -272,7 +278,7 @@(deftest finalize-assistant-turn-passes-through-aborted-and-errored-test(testing "finalize-assistant-turn! preserves :finished? false and JSON-encodes :error"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](mt/with-current-user (mt/user->id :rasta)(testing "aborted: finished false flows through"(let [conversation-id (str (random-uuid))@@ -300,7 +306,7 @@(deftest start-turn-pins-ordering-under-abort-then-retry-test(testing "an aborted turn whose finalize fires after a retry still sorts before the retry's rows"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))](mt/with-current-user (mt/user->id :rasta);; Turn A: starts and is left in-flight@@ -336,7 +342,7 @@(deftest start-turn-user-and-placeholder-share-created-at-test(testing "user-message and assistant-placeholder rows inserted by start-turn! share an instant;readers must tiebreak on :id to preserve user-before-assistant ordering"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))](mt/with-current-user (mt/user->id :rasta)(metabot-persistence/start-turn!@@ -352,7 +358,7 @@(deftest conversation-detail-drops-errored-pair-under-created-at-collision-test(testing "drop-errored-pairs works correctly when the errored user/asst rows share created_at"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))](mt/with-current-user (mt/user->id :rasta);; Errored turn: user-msg and asst-row share created_at via start-turn!'s transaction.@@ -398,7 +404,7 @@(deftest conversation-detail-filters-soft-deleted-messages-and-orders-ascending-test(testing "conversation-detail returns only non-deleted messages, ordered by :created_at ascending"- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [conversation-id (str (random-uuid))user-id (mt/user->id :rasta)now (java.time.OffsetDateTime/now)@@ -575,7 +581,7 @@"stringified :data still carries enough info for triage"))))))(deftest finalize-assistant-turn-persists-finished-and-error-test- (mt/with-model-cleanup [:model/MetabotMessage [:model/MetabotConversation :created_at]]+ (t2/with-transaction [_conn nil {:rollback-only true}](testing "default: finished true, no error"(let [[row] (start-and-finalize!)](is (=? {:finished true :error nil} row))))@@ -626,3 +632,198 @@(mapv :message (metabot-persistence/messages->chat-messages[user-msg errored] {:include-errored? true})))"audit read keeps both rows; the empty-data stub renders so the FE has somewhere to hang the error alert"))))))++;;; ---------------------------------------- used-table recording ----------------------------------------++(defn- ->notebook-parts+ "Build a minimal `construct_notebook_query` tool-input/output pair whose query references the given table id."+ [call-id source-table-id]+ (let [mp (mt/metadata-provider)+ query (lib/query mp (lib.metadata/table mp source-table-id))]+ [{:type :tool-input+ :id call-id+ :function "construct_notebook_query"+ :arguments {:reasoning "x"}}+ {:type :tool-output+ :id call-id+ :result {:output "<result>...</result>"+ :structured-output {:query-id "qid"+ :query query}}}]))++(deftest finalize-records-used-tables-test+ (testing "finalize-assistant-turn! inserts metabot_used_table rows for successful query-generating tool calls"+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [table-id (mt/id :orders)+ conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id+ "internal"+ {:role "user" :content "go"})]+ (binding [used-tables/*run-synchronously?* true]+ (metabot-persistence/finalize-assistant-turn!+ conversation-id+ assistant-msg-id+ (into [{:type :text :text "ok"}] (->notebook-parts "c1" table-id))))+ (is (=? [{:message_id assistant-msg-id+ :table_id table-id}]+ (t2/select :model/MetabotUsedTable :message_id assistant-msg-id))))))))++(deftest finalize-records-used-tables-in-background-test+ (testing "with the default (non-synchronous) path, finalize-assistant-turn! eventually records used tables on the background worker"+ ;; No rollback transaction and no `*run-synchronously?*` binding: the extraction + insert run on+ ;; the background executor (its own connection), so we commit real rows and clean up in `finally`.+ (mt/with-current-user (mt/user->id :rasta)+ (let [table-id (mt/id :orders)+ conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id+ "internal"+ {:role "user" :content "go"})]+ (try+ (metabot-persistence/finalize-assistant-turn!+ conversation-id+ assistant-msg-id+ (into [{:type :text :text "ok"}] (->notebook-parts "c1" table-id)))+ (is (=? [{:message_id assistant-msg-id+ :table_id table-id}]+ (tu/poll-until 5000+ (seq (t2/select :model/MetabotUsedTable :message_id assistant-msg-id)))))+ (finally+ ;; CASCADE from metabot_message cleans up the used-table row.+ (t2/delete! :model/MetabotMessage :conversation_id conversation-id)+ (t2/delete! :model/MetabotConversation :id conversation-id)))))))++(defn- ->transform-python-parts+ "Build a `write_transform_python` tool-input/output pair that declares `table-id` as its single source table.+ Used to verify the end-to-end transform extraction path."+ [call-id table-id]+ [{:type :tool-input+ :id call-id+ :function "write_transform_python"+ :arguments {:transform_name "T"+ :edit_action {:mode "replace"+ :new_content "def transform(): pass"}+ :source_tables [{:alias "t"+ :table_id table-id+ :schema "PUBLIC"+ :database_id (mt/id)}]}}+ {:type :tool-output+ :id call-id+ :result {:output "ok"+ :structured-output {:transform {:source {:type "python"}}+ :thinking "x"+ :message "Transform Python updated successfully."}}}])++(defn- ->transform-sql-parts+ "Build a `write_transform_sql` tool-input/output pair whose suggested transform's `[:source :query]` is a native query.+ The structured-output's `:transform` key is dropped by `strip-tool-output-bloat`, so this pair exercises finalize's+ pre-strip extraction path."+ [call-id db-id sql]+ (let [query (lib/native-query (mt/metadata-provider) sql)]+ [{:type :tool-input+ :id call-id+ :function "write_transform_sql"+ :arguments {:database_id db-id+ :edit_action {:mode "replace" :new_content sql}}}+ {:type :tool-output+ :id call-id+ :result {:output "ok"+ :structured-output+ {:transform {:id nil+ :name "T"+ :description ""+ :target {:type "table" :name "" :database db-id :schema nil}+ :source {:type "query"+ :query query}}+ :thinking "x"+ :message "Transform SQL updated successfully."}}}]))++(deftest finalize-records-used-tables-for-python-transform-test+ (testing "finalize-assistant-turn! records `metabot_used_table` rows for a write_transform_python tool call."+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [table-id (mt/id :orders)+ conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id+ "transforms_codegen"+ {:role "user" :content "make a transform"})]+ (binding [used-tables/*run-synchronously?* true]+ (metabot-persistence/finalize-assistant-turn!+ conversation-id+ assistant-msg-id+ (into [{:type :text :text "ok"}] (->transform-python-parts "t1" table-id))))+ (is (=? [{:message_id assistant-msg-id+ :table_id table-id}]+ (t2/select :model/MetabotUsedTable :message_id assistant-msg-id))))))))++(deftest finalize-records-used-tables-for-sql-transform-test+ (testing "finalize-assistant-turn! parses the SQL transform's native query."+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [orders-id (mt/id :orders)+ conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id+ "transforms_codegen"+ {:role "user" :content "make a SQL transform"})]+ (binding [used-tables/*run-synchronously?* true]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _] {:tables [{:table-id orders-id}]})]+ (metabot-persistence/finalize-assistant-turn!+ conversation-id+ assistant-msg-id+ (into [{:type :text :text "ok"}]+ (->transform-sql-parts "t1" (mt/id) "SELECT * FROM orders")))))+ (is (=? [{:message_id assistant-msg-id :table_id orders-id}]+ (t2/select :model/MetabotUsedTable :message_id assistant-msg-id))))))))++(deftest finalize-records-nothing-without-query-tools-test+ (testing "finalize-assistant-turn! inserts no used-table rows for text-only or non-query tool turns"+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id+ "internal"+ {:role "user" :content "go"})]+ (binding [used-tables/*run-synchronously?* true]+ (metabot-persistence/finalize-assistant-turn!+ conversation-id+ assistant-msg-id+ [{:type :text+ :text "just text"}+ {:type :tool-input+ :id "n1"+ :function "navigate_user"+ :arguments {}}+ {:type :tool-output+ :id "n1"+ :result {:output "ok"}}]))+ (is (zero? (t2/count :model/MetabotUsedTable :message_id assistant-msg-id))))))))++(deftest insert-failure-does-not-fail-finalize-test+ (testing "a failed used-table INSERT is logged and finalize still completes"+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (mt/with-current-user (mt/user->id :rasta)+ (let [conversation-id (str (random-uuid))+ {:keys [assistant-msg-id]} (metabot-persistence/start-turn!+ conversation-id+ "internal"+ {:role "user" :content "go"})]+ (with-redefs [used-tables/extract-used-tables-with-timing! (fn [_ _]+ [{:message_id assistant-msg-id :table_id 1}])+ t2/insert! (fn [& _]+ (throw (ex-info "boom" {})))]+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.used-tables :warn]]+ (binding [used-tables/*run-synchronously?* true]+ (metabot-persistence/finalize-assistant-turn!+ conversation-id+ assistant-msg-id+ [{:type :text :text "ok"}]))+ (is (=? {:finished true+ :data [{:type "text" :text "ok"}]}+ (t2/select-one :model/MetabotMessage assistant-msg-id))+ "message UPDATE still landed")+ (is (some #(re-find #"Failed to record metabot used tables" (:message %)) (logs))+ "warn line captured"))))))))diff --git a/test/metabase/metabot/used_tables_test.clj b/test/metabase/metabot/used_tables_test.cljnew file mode 100644index 00000000000..189c4fe1a26--- /dev/null+++ b/test/metabase/metabot/used_tables_test.clj@@ -0,0 +1,750 @@+(ns metabase.metabot.used-tables-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [medley.core :as m]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]+ [metabase.metabot.query-analyzer :as nqa]+ [metabase.metabot.used-tables :as used-tables]+ [metabase.test :as mt]+ [metabase.util.log.capture :as log.capture])+ (:import+ (java.util.concurrent ArrayBlockingQueue CountDownLatch ThreadPoolExecutor TimeUnit)))++(set! *warn-on-reflection* true)++;;; ---------------------------------------- metadata provider helpers ----------------------------------------++(defn- mock-card+ "Build a card map suitable for [[lib.tu/mock-metadata-provider]]."+ ([id query]+ (mock-card id query :question))+ ([id query card-type]+ {:lib/type :metadata/card+ :id id+ :name (str "Card " id)+ :database-id (meta/id)+ :type card-type+ :dataset-query query}))++(defn- mp+cards+ "Compose `meta/metadata-provider` with a mock provider containing `cards`."+ [cards]+ (lib.tu/mock-metadata-provider meta/metadata-provider {:cards cards}))++;;; ---------------------------------------- query construction helpers ----------------------------------------++(defn- table-query+ "Query whose first stage's source is `table-id`, attached to `metadata-provider`."+ ([table-id]+ (table-query meta/metadata-provider table-id))+ ([metadata-provider table-id]+ (lib/query metadata-provider (lib.metadata/table metadata-provider table-id))))++(defn- card-query+ "Query whose first stage's source is the given Card, attached to `metadata-provider`."+ [metadata-provider card-id]+ (lib/query metadata-provider (lib.metadata/card metadata-provider card-id)))++(defn- native-query+ "Native query against the test database."+ ([sql]+ (native-query meta/metadata-provider sql))+ ([metadata-provider sql]+ (lib/native-query metadata-provider sql)))++;;; ---------------------------------------- tool-input/output helpers ----------------------------------------++(defn- notebook-input+ "Build a `construct_notebook_query` tool-input part."+ [id]+ {:type :tool-input+ :id id+ :function "construct_notebook_query"+ :arguments {:reasoning "x"}})++(defn- notebook-output+ "Build a `construct_notebook_query` tool-output part with the given query."+ [id query]+ {:type :tool-output+ :id id+ :result {:output "<result>...</result>"+ :structured-output {:query-id "qid"+ :query query}}})++(defn- sql-input+ ([id]+ (sql-input id "create_sql_query" {:database_id 1 :sql_query "SELECT 1"}))+ ([id fn-name args]+ {:type :tool-input+ :id id+ :function fn-name+ :arguments args}))++(defn- sql-output+ "Build a SQL tool-output. `query` is an already-constructed native query+ (built via [[native-query]] in the test)."+ [id sql db-id query]+ {:type :tool-output+ :id id+ :result {:output "<result>...</result>"+ :structured-output {:query-id "qid"+ :query-content sql+ :database db-id+ :query query}}})++(defn- table-ids+ "Return the set of `:table_id`s in the resulting rows."+ [rows]+ (into #{} (map :table_id) rows))++;;; ---------------------------------------- empty/no-op (no DB) ----------------------------------------++(deftest ^:parallel empty-parts-returns-nothing-test+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 1 []))))++(deftest ^:parallel non-query-tool-skipped-test+ (testing "tool calls outside `query-generation-tool-names` produce no rows"+ (let [parts [{:type :tool-input+ :id "n1"+ :function "navigate_user"+ :arguments {}}+ {:type :tool-output+ :id "n1"+ :result {:output "ok"+ :structured-output {:query-id "ignored"}}}]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 1 parts))))))++(deftest ^:parallel errored-tool-output-skipped-test+ (testing "tool outputs with :error are dropped even with structured-output"+ (let [parts [(notebook-input "c1")+ (-> (notebook-output "c1" (table-query (meta/id :orders)))+ (assoc :error "exploded"))]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 1 parts))))))++(deftest ^:parallel missing-structured-output-skipped-test+ (testing "tool outputs without :structured-output are dropped"+ (let [parts [(notebook-input "c1")+ {:type :tool-output+ :id "c1"+ :result {:output "<result>failed</result>"}}]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 1 parts))))))++(deftest ^:parallel orphan-tool-input-skipped-test+ (testing "tool-inputs without a matching tool-output yield no rows"+ (let [parts [(notebook-input "c1")]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 1 parts))))))++;;; ---------------------------------------- notebook ----------------------------------------++(deftest ^:parallel notebook-source-table-test+ (testing "construct_notebook_query with :source-table yields one table row"+ (let [parts [(notebook-input "c1")+ (notebook-output "c1" (table-query (meta/id :orders)))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows)))))++(deftest ^:parallel notebook-source-card-question-expanded-test+ (testing ":source-card pointing at a question Card collapses to the question's underlying table"+ (let [card-id 1+ mp (mp+cards [(mock-card card-id (table-query (meta/id :orders)) :question)])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp card-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows)))))++(deftest ^:parallel notebook-source-card-model-expanded-test+ (testing ":source-card pointing at a model Card collapses to the model's underlying table"+ (let [card-id 1+ mp (mp+cards [(mock-card card-id (table-query (meta/id :orders)) :model)])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp card-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows)))))++(deftest ^:parallel notebook-source-card-metric-expanded-test+ (testing ":source-card pointing at a metric Card collapses to the metric's underlying table"+ (let [card-id 1+ mp (mp+cards [(mock-card card-id+ (-> (table-query (meta/id :orders))+ (lib/aggregate (lib/count)))+ :metric)])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp card-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows)))))++(deftest ^:parallel notebook-recurses-through-card-on-card-test+ (testing "card A on card B on table T collapses all the way down to T"+ (let [a-id 1+ mp-b (mp+cards [(mock-card 2 (table-query (meta/id :orders)))])+ mp (mp+cards [(mock-card 2 (table-query (meta/id :orders)))+ (mock-card a-id (card-query mp-b 2))])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp a-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows)))))++(deftest ^:parallel notebook-deleted-card-warns-test+ (testing "a :source-card id that doesn't exist yields no row and logs a warn"+ (let [card-id 1+ absent-id 2+ mp0 (mp+cards [(mock-card 1 (table-query (meta/id :orders)))+ (mock-card 2 (table-query (meta/id :products)))])+ ;; query references a card which does not exist in new mock mp+ mp (mp+cards [(mock-card card-id (card-query mp0 2))])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp card-id))]]+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.used-tables :warn]]+ (let [rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= [] rows))+ (is (some #(re-find (re-pattern (str absent-id)) (:message %)) (logs))+ "warn line mentions the missing card id"))))))++;;; ---------------------------------------- nested cards (recursion) ----------------------------------------++(deftest ^:parallel notebook-three-levels-of-nesting-test+ (testing "card -> card -> card -> table collapses all the way down to the table"+ (let [a-id 1+ mp-c (mp+cards [(mock-card 3 (table-query (meta/id :orders)))])+ mp-b (mp+cards [(mock-card 3 (table-query (meta/id :orders)))+ (mock-card 2 (card-query mp-c 3))])+ mp (mp+cards [(mock-card 3 (table-query (meta/id :orders)))+ (mock-card 2 (card-query mp-c 3))+ (mock-card a-id (card-query mp-b 2))])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp a-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders)}+ (table-ids rows))))))++(deftest ^:parallel notebook-model-on-question-test+ (testing "a model whose source is a question Card still resolves to the underlying table"+ (let [m-id 1+ mp-q (mp+cards [(mock-card 2 (table-query (meta/id :orders)))])+ mp (mp+cards [(mock-card 2 (table-query (meta/id :orders)))+ (mock-card m-id (card-query mp-q 2) :model)])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp m-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders)}+ (table-ids rows))))))++(deftest ^:parallel notebook-question-on-model-test+ (testing "a question whose source is a model resolves to the underlying table"+ (let [q-id 1+ mp-m (mp+cards [(mock-card 2 (table-query (meta/id :orders)) :model)])+ mp (mp+cards [(mock-card 2 (table-query (meta/id :orders)) :model)+ (mock-card q-id (card-query mp-m 2))])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp q-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders)}+ (table-ids rows))))))++(deftest ^:parallel notebook-multi-stage-source-table-test+ (testing "a query with multiple stages still surfaces the first-stage :source-table"+ (let [query (-> (table-query (meta/id :orders))+ (lib/aggregate (lib/count))+ lib/append-stage)+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= #{(meta/id :orders)}+ (table-ids rows))))))++;;; ---------------------------------------- joins ----------------------------------------++(deftest ^:parallel notebook-explicit-join-against-table-test+ (testing "an explicit join against another table surfaces both the source and joined tables"+ (let [mp meta/metadata-provider+ query (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/join (lib.metadata/table mp (meta/id :people))))+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :people)}+ (table-ids rows))))))++(deftest ^:parallel notebook-explicit-join-against-card-test+ (testing "an explicit join against a question Card surfaces both the source table and the card's underlying table"+ (let [card-id 1+ mp (mp+cards [(mock-card card-id (table-query (meta/id :people)))])+ query (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/join (lib.metadata/card mp card-id)))+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :people)}+ (table-ids rows))))))++(deftest ^:parallel notebook-explicit-join-against-model-test+ (testing "an explicit join against a model Card surfaces both the source table and the model's underlying table"+ (let [model-id 1+ mp (mp+cards [(mock-card model-id (table-query (meta/id :people)) :model)])+ query (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/join (lib.metadata/card mp model-id)))+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :people)}+ (table-ids rows))))))++(deftest ^:parallel notebook-recurses-into-card-with-joins-test+ (testing "recursing into a Card whose own query has joins picks up all of the card's tables"+ (let [card-query-with-join (-> (table-query (meta/id :orders))+ (lib/join (lib.metadata/table meta/metadata-provider (meta/id :people))))+ card-id 1+ mp (mp+cards [(mock-card card-id card-query-with-join)])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp card-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :people)}+ (table-ids rows))))))++(deftest ^:parallel notebook-source-table-with-join-to-card-on-card-test+ (testing "an outer query with both :source-table and a join to a chain of cards captures everything"+ (let [mid-id 1+ mp-leaf (mp+cards [(mock-card 2 (table-query (meta/id :people)))])+ mp (mp+cards [(mock-card 2 (table-query (meta/id :people)))+ (mock-card mid-id (card-query mp-leaf 2))])+ query (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/join (lib.metadata/card mp mid-id)))+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :people)}+ (table-ids rows))))))++(deftest ^:parallel notebook-implicit-join-via-source-field-breakout-test+ (testing "a breakout with :source-field (implicit join) surfaces the joined table"+ (let [base (-> (table-query (meta/id :orders))+ (lib/aggregate (lib/count)))+ ;; pick the implicitly-joinable products.category column reachable+ ;; via the FK orders.product_id -> products.id+ cat-col (m/find-first #(= (:id %) (meta/id :products :category))+ (lib/breakoutable-columns base))+ query (lib/breakout base cat-col)+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :products)}+ (table-ids rows))))))++(deftest ^:parallel notebook-implicit-join-via-source-field-filter-test+ (testing "an implicit-join field reference inside a filter clause also surfaces the joined table"+ (let [base (table-query (meta/id :orders))+ cat-col (m/find-first #(= (:id %) (meta/id :products :category))+ (lib/filterable-columns base))+ query (lib/filter base (lib/= cat-col "Widget"))+ parts [(notebook-input "c1")+ (notebook-output "c1" query)]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :products)}+ (table-ids rows))))))++(deftest ^:parallel notebook-implicit-join-inside-nested-card-test+ (testing "implicit joins inside a recursively-walked card's dataset_query also surface"+ (let [base (-> (table-query (meta/id :orders))+ (lib/aggregate (lib/count)))+ cat-col (m/find-first #(= (:id %) (meta/id :products :category))+ (lib/breakoutable-columns base))+ inner (lib/breakout base cat-col)+ card-id 1+ mp (mp+cards [(mock-card card-id inner)])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp card-id))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(meta/id :orders) (meta/id :products)}+ (table-ids rows))))))++;;; ---------------------------------------- native (SQL) ----------------------------------------++(deftest ^:parallel sql-extracts-native-tables-test+ (testing "native SQL is parsed and the underlying tables come through"+ (let [order-id (meta/id :orders)+ sql "SELECT * FROM orders"]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _]+ {:tables [{:table-id order-id}]})]+ (let [parts [(sql-input "s1" "create_sql_query" {:database_id (meta/id) :sql_query sql})+ (sql-output "s1" sql (meta/id) (native-query sql))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= [{:message_id 99+ :table_id order-id}]+ rows)))))))++(deftest ^:parallel sql-template-tag-card-expanded-test+ (testing "card-type template tag in native SQL recursively expands to its underlying table"+ (let [card-id 1+ mp (mp+cards [(mock-card card-id (table-query (meta/id :orders)))])]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _] {:tables []})]+ (let [;; `{{#N}}` syntax produces a card-type template tag with :card-id N+ sql (format "SELECT * FROM {{#%s}}" card-id)+ parts [(sql-input "s1" "create_sql_query" {:database_id (meta/id) :sql_query sql})+ (sql-output "s1" sql (meta/id) (native-query mp sql))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (some #(= % {:message_id 99+ :table_id (meta/id :orders)})+ rows)))))))++(deftest ^:parallel sql-template-tag-native-card-test+ (testing "template tag pointing at a native-query card recurses and parses the inner SQL"+ (let [orders-id (meta/id :orders)+ people-id (meta/id :people)+ inner-sql "SELECT * FROM people"+ inner-tables (atom nil)+ card-id 1+ ;; build the inner native query attached to the base provider, then mock the card+ inner-query (native-query inner-sql)+ mp (mp+cards [(mock-card card-id inner-query)])+ ;; outer SQL must contain the `{{#N}}` template tag for the card so that+ ;; `lib/native-query` auto-extracts a card-type tag pointing at the card+ outer-sql (format "SELECT * FROM {{#%s}}" card-id)]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [query & _]+ ;; production extractor passes a query, not raw SQL+ (let [sql (lib/raw-native-query query)]+ (cond+ (= sql outer-sql) {:tables [{:table-id orders-id}]}+ (= sql inner-sql) (do (reset! inner-tables true)+ {:tables [{:table-id people-id}]})+ :else {:tables []})))]+ (let [parts [(sql-input "s1" "create_sql_query"+ {:database_id (meta/id) :sql_query outer-sql})+ (sql-output "s1" outer-sql (meta/id) (native-query mp outer-sql))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (true? @inner-tables) "sql parser was invoked on the inner card's native SQL")+ (is (= #{orders-id people-id}+ (table-ids rows))+ "outer and inner native tables both make it into the result"))))))++(deftest ^:parallel sql-parse-error-logged-test+ (testing "tables-for-native error yields no rows and logs a warn"+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _]+ {:error :query-analysis.error/parse-failed})]+ (let [sql "SELEKT bad"+ parts [(sql-input "s1" "create_sql_query" {:database_id (meta/id) :sql_query sql})+ (sql-output "s1" sql (meta/id) (native-query sql))]]+ (log.capture/with-log-messages-for-level [logs [metabase.metabot.used-tables :warn]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)))+ (is (some #(re-find #"tables-for-native error" (:message %)) (logs))))))))++;;; ---------------------------------------- native (SQL) — real tables-for-native ----------------------------------+;;;+;;; The tests above all stub `nqa/tables-for-native` to isolate the extractor logic from the sql parser. These two+;;; tests run the real parser end-to-end against the H2 test-data DB so we catch integration issues. They need the+;;; real application DB metadata provider, so they cannot be `^:parallel`.++(deftest sql-real-tables-for-native-join-with-where-test+ (testing "real tables-for-native against a plain JOIN + WHERE: both tables come through end-to-end"+ (let [sql "SELECT o.id FROM orders o JOIN products p ON o.product_id = p.id WHERE p.category = 'Widget'"+ mp (mt/metadata-provider)+ parts [(sql-input "s1" "create_sql_query" {:database_id (mt/id) :sql_query sql})+ (sql-output "s1" sql (mt/id) (lib/native-query mp sql))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(mt/id :orders) (mt/id :products)}+ (table-ids rows))))))++(deftest sql-real-tables-for-native-with-card-template-tag-test+ (testing "real tables-for-native against SQL that references a saved card via {{#card-id}} template tag"+ (mt/with-temp [:model/Card {card-id :id}+ {:type :question+ :database_id (mt/id)+ :dataset_query (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id :products))))}]+ (let [tag-name (str "#" card-id)+ sql (format "SELECT * FROM orders WHERE product_id IN (SELECT id FROM {{%s}})" tag-name)+ mp (mt/metadata-provider)+ parts [(sql-input "s1" "create_sql_query" {:database_id (mt/id) :sql_query sql})+ (sql-output "s1" sql (mt/id) (lib/native-query mp sql))]+ rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= #{(mt/id :orders) (mt/id :products)}+ (table-ids rows)))))))++;;; ---------------------------------------- transforms ----------------------------------------++(defn- transform-sql-input+ "Build a `write_transform_sql` tool-input part."+ ([id]+ (transform-sql-input id {}))+ ([id arguments]+ {:type :tool-input+ :id id+ :function "write_transform_sql"+ :arguments (merge {:database_id 1+ :edit_action {:mode "replace" :new_content "SELECT 1"}}+ arguments)}))++(defn- transform-sql-output+ "Build a `write_transform_sql` tool-output part."+ [id db-id query]+ {:type :tool-output+ :id id+ :result {:output "ok"+ :structured-output+ {:transform {:id nil+ :name "T"+ :description ""+ :target {:type "table" :name "" :database db-id :schema nil}+ :source {:type "query"+ :query query}}+ :thinking "x"+ :message "Transform SQL updated successfully."}}})++(defn- transform-python-input+ "Build a `write_transform_python` tool-input part with the given `source_tables` entries."+ [id source-tables]+ {:type :tool-input+ :id id+ :function "write_transform_python"+ :arguments {:transform_name "T"+ :edit_action {:mode "replace" :new_content "def transform(): pass"}+ :source_tables (vec source-tables)}})++(defn- transform-python-output+ "Build a `write_transform_python` tool-output.+ The structured `:transform`is intentionally sparse.+ Extraction reads `:source_tables` from the *arguments*, not from the structured output."+ [id]+ {:type :tool-output+ :id id+ :result {:output "ok"+ :structured-output {:transform {:source {:type "python"}}+ :thinking "x"+ :message "Transform Python updated successfully."}}})++(deftest ^:parallel transform-sql-extracts-tables-from-structured-output-test+ (testing "write_transform_sql walks the suggested transform's native query"+ (let [orders-id (meta/id :orders)+ sql "SELECT * FROM orders"]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _] {:tables [{:table-id orders-id}]})]+ (let [parts [(transform-sql-input "t1")+ (transform-sql-output "t1" (meta/id) (native-query sql))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= [{:message_id 99 :table_id orders-id}]+ rows)))))))++(deftest ^:parallel transform-sql-defensively-reads-source-tables-arg-test+ (testing "write_transform_sql also picks up `:source_tables` from arguments when present"+ (let [orders-id (meta/id :orders)+ people-id (meta/id :people)+ sql "SELECT * FROM orders"]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _] {:tables [{:table-id orders-id}]})]+ (let [parts [(transform-sql-input "t1"+ {:source_tables [{:alias "o" :table_id orders-id+ :schema "PUBLIC" :database_id (meta/id)}+ {:alias "p" :table_id people-id+ :schema "PUBLIC" :database_id (meta/id)}]})+ (transform-sql-output "t1" (meta/id) (native-query sql))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= #{orders-id people-id}+ (table-ids rows))+ "sql parsed `orders` and arg-declared `people` both surface, deduped")+ (is (= 2 (count rows))))))))++(deftest ^:parallel transform-sql-errored-output-skipped-test+ (testing "an errored write_transform_sql output yields no rows"+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _] {:tables [{:table-id (meta/id :orders)}]})]+ (let [parts [(transform-sql-input "t1")+ (-> (transform-sql-output "t1" (meta/id) (native-query "SELECT 1"))+ (assoc :error "boom"))]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)))))))++(deftest ^:parallel transform-sql-without-query-source-yields-nothing-test+ (testing "no rows if the suggested transform has no [:source :query] or :source_tables"+ (let [parts [(transform-sql-input "t1")+ {:type :tool-output+ :id "t1"+ :result {:output "ok"+ :structured-output {:transform {:source {:type "query"}}+ :thinking "x"+ :message "ok"}}}]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 99 parts))))))++(deftest ^:parallel transform-python-extracts-declared-source-tables-test+ (testing "write_transform_python yields one row per declared `:source_tables` entry"+ (let [orders-id (meta/id :orders)+ people-id (meta/id :people)+ parts [(transform-python-input+ "t1"+ [{:alias "o" :table_id orders-id :schema "PUBLIC" :database_id (meta/id)}+ {:alias "p" :table_id people-id :schema "PUBLIC" :database_id (meta/id)}])+ (transform-python-output "t1")]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= #{orders-id people-id}+ (table-ids rows)))+ (is (= 2 (count rows))))))++(deftest ^:parallel transform-python-empty-source-tables-yields-nothing-test+ (testing "write_transform_python with an empty source_tables list yields no rows"+ (let [parts [(transform-python-input "t1" [])+ (transform-python-output "t1")]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 99 parts))))))++(deftest ^:parallel transform-python-errored-output-skipped-test+ (testing "an errored write_transform_python output yields no rows, even if arguments declare tables"+ (let [parts [(transform-python-input+ "t1"+ [{:alias "o" :table_id (meta/id :orders) :schema "PUBLIC" :database_id (meta/id)}])+ (-> (transform-python-output "t1")+ (assoc :error "boom"))]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 99 parts))))))++(deftest ^:parallel transform-python-orphan-input-yields-nothing-test+ (testing "a write_transform_python tool-input without a matching tool-output yields no rows"+ (let [parts [(transform-python-input+ "t1"+ [{:alias "o" :table_id (meta/id :orders) :schema "PUBLIC" :database_id (meta/id)}])]]+ (is (= [] (#'used-tables/extract-used-tables meta/metadata-provider 99 parts))))))++;;; ---------------------------------------- combined tool calls ----------------------------------------++(deftest ^:parallel notebook-and-sql-tool-calls-combine-test+ (testing "a turn that mixes construct_notebook_query with create_sql_query yields the union of their tables"+ (let [orders-id (meta/id :orders)+ people-id (meta/id :people)+ sql "SELECT * FROM people"]+ (mt/with-dynamic-fn-redefs [nqa/tables-for-native (fn [_ & _]+ {:tables [{:table-id people-id}]})]+ (let [parts [(notebook-input "n1")+ (notebook-output "n1" (table-query orders-id))+ (sql-input "s1" "create_sql_query" {:database_id (meta/id) :sql_query sql})+ (sql-output "s1" sql (meta/id) (native-query sql))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= #{orders-id people-id}+ (table-ids rows))))))))++(deftest ^:parallel cross-pair-card-resolved-once-test+ (testing "a card referenced from multiple tool-call pairs is resolved at most once per extraction"+ (let [c-id 1+ mp (mp+cards [(mock-card c-id (table-query (meta/id :orders)))])+ parts [(notebook-input "n1")+ (notebook-output "n1" (card-query mp c-id))+ (notebook-input "n2")+ (notebook-output "n2" (card-query mp c-id))+ (notebook-input "n3")+ (notebook-output "n3" (card-query mp c-id))]+ calls (atom [])+ orig (mt/original-fn #'used-tables/card-query)]+ (mt/with-dynamic-fn-redefs [used-tables/card-query (fn [m cid]+ (swap! calls conj cid)+ (orig m cid))]+ (let [rows (#'used-tables/extract-used-tables mp 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows))+ (is (= [c-id] @calls)))))))++;;; ---------------------------------------- dedupe / stamping ----------------------------------------++(deftest ^:parallel dedupes-within-message-test+ (testing "two notebook tool calls referencing the same table yield a single row"+ (let [parts [(notebook-input "c1")+ (notebook-output "c1" (table-query (meta/id :orders)))+ (notebook-input "c2")+ (notebook-output "c2" (table-query (meta/id :orders)))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 99 parts)]+ (is (= [{:message_id 99+ :table_id (meta/id :orders)}]+ rows)))))++(deftest ^:parallel stamps-message-id-test+ (testing "every returned row has :message_id set to the value passed in"+ (let [parts [(notebook-input "c1")+ (notebook-output "c1" (table-query (meta/id :orders)))+ (notebook-input "c2")+ (notebook-output "c2" (table-query (meta/id :people)))]+ rows (#'used-tables/extract-used-tables meta/metadata-provider 42 parts)]+ (is (every? #(= 42 (:message_id %)) rows))+ (is (= 2 (count rows))))))++;;; ---------------------------------------- prometheus metrics ----------------------------------------+;;; `extract-used-tables-with-timing!` wraps the 2-arity entry point. We redef `extract-used-tables` to isolate the+;;; wrapper's metric contract from query analysis. One shared `with-prometheus-system!` (it's slow to set up); metrics+;;; are cleared between `testing` blocks so each starts from zero. Not `^:parallel` (installs a reporter, redefs a var).++(def ^:private extraction-metrics+ [:metabase-metabot/used-tables-extraction-total+ :metabase-metabot/used-tables-extraction-errors+ :metabase-metabot/used-tables-extraction-duration-ms+ :metabase-metabot/used-tables-extraction-warnings+ :metabase-metabot/used-tables-extraction-dropped+ :metabase-metabot/used-tables-extraction-timeouts])++(deftest extraction-metrics-test+ (mt/with-prometheus-system! [_ system]+ ;; mt/with-prometheus-system! is slow, so combine all test in a single deftest and clear! metrics between cases.+ (let [reset! #(run! prometheus/clear! extraction-metrics)]+ (testing "extract-used-tables-with-timing! increments the total counter, leaves errors at 0, and returns the body"+ (mt/with-dynamic-fn-redefs [used-tables/extract-used-tables (fn [_message-id _parts] [::row])]+ (is (= [::row] (#'used-tables/extract-used-tables-with-timing! 1 [])))+ (is (= 1.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-total)))+ (is (= 0.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-errors)))))+ (reset!)+ (testing "extract-used-tables-with-timing! increments the errors counter and rethrows when extraction throws"+ (mt/with-dynamic-fn-redefs [used-tables/extract-used-tables (fn [_message-id _parts] (throw (ex-info "boom" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"boom"+ (#'used-tables/extract-used-tables-with-timing! 1 [])))+ (is (= 1.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-total)))+ (is (= 1.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-errors)))))+ (reset!)+ (testing "extract-used-tables-with-timing! observes the duration histogram"+ (mt/with-dynamic-fn-redefs [used-tables/extract-used-tables (fn [_message-id _parts] (Thread/sleep 1) [])]+ (#'used-tables/extract-used-tables-with-timing! 1 [])+ (is (pos? (:sum (mt/metric-value system :metabase-metabot/used-tables-extraction-duration-ms))))))+ (reset!)+ (testing "caught exception increments the warnings counter (by :reason) without touching errors"+ ;; query references a card absent from the provider -> card-query warns :card-missing and recovers (no rows)+ (let [mp0 (mp+cards [(mock-card 1 (table-query (meta/id :orders)))+ (mock-card 2 (table-query (meta/id :products)))])+ mp (mp+cards [(mock-card 1 (card-query mp0 2))])+ parts [(notebook-input "c1")+ (notebook-output "c1" (card-query mp 1))]]+ (is (= [] (#'used-tables/extract-used-tables mp 99 parts)))+ (is (= 0.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-errors)))+ (is (= 1.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-warnings+ {:reason :card-missing})))))+ (reset!)+ (testing "record-used-tables! drops the task (counter + warning, no throw) when the bounded queue is full"+ ;; Saturate a real bounded pool sized 1 thread + 1 queue slot. With corePoolSize=1 the first submission+ ;; is handed straight to the (only) worker, which parks on `release`; the second fills the single queue+ ;; slot; the third can be neither run nor queued, so the default AbortPolicy rejects it -> dropped.+ (let [release (CountDownLatch. 1)+ full (ThreadPoolExecutor. 1 1 0 TimeUnit/MILLISECONDS (ArrayBlockingQueue. 1))]+ (try+ (with-redefs [used-tables/waiter-executor (delay full)+ used-tables/extract-and-insert! (fn [_ _] (.await release))]+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.used-tables :warn]]+ (used-tables/record-used-tables! 1 []) ; accepted: runs, parks on `release`+ (used-tables/record-used-tables! 2 []) ; accepted: occupies the lone queue slot+ (is (nil? (used-tables/record-used-tables! 3 []))) ; rejected: nowhere to run or queue+ (is (= 1.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-dropped)))+ (is (some #(re-find #"queue full" (:message %)) (messages)))))+ (finally+ (.countDown release)+ (.shutdownNow full)))))+ (reset!)+ (testing "extract-used-tables-with-timing! counts a timeout, warns, and returns nil when extraction exceeds the cap"+ ;; Shrink extraction-timeout-ms and make extraction outlast it: the async path's `.get` times out, the worker+ ;; future is cancelled, and the wrapper swallows the TimeoutException (counter + warning, returns nil).+ (with-redefs [used-tables/extraction-timeout-ms 50]+ (mt/with-dynamic-fn-redefs [used-tables/extract-used-tables (fn [_message-id _parts]+ (Thread/sleep 10000)+ [::never])]+ (log.capture/with-log-messages-for-level [messages [metabase.metabot.used-tables :warn]]+ (is (nil? (#'used-tables/extract-used-tables-with-timing! 1 [])))+ (is (= 1.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-timeouts)))+ (is (= 0.0 (mt/metric-value system :metabase-metabot/used-tables-extraction-errors)))+ (is (some #(re-find #"exceeded the .* timeout" (:message %)) (messages))))))))))diff --git a/test/metabase/slackbot/api_test.clj b/test/metabase/slackbot/api_test.cljindex ff6da3e8442..7fcb48a5d40 100644--- a/test/metabase/slackbot/api_test.clj+++ b/test/metabase/slackbot/api_test.clj@@ -179,7 +179,8 @@(testing "POST /events with app_mention uses visible channel reply (not streaming)"(tu/with-slackbot-setup(let [mock-ai-text "Here is your answer"- event-body tu/base-mention-event]+ channel-id "C-MENTION-TEST"+ event-body (assoc-in tu/base-mention-event [:event :channel] channel-id)](tu/with-slackbot-mocks{:ai-text mock-ai-text}(fn [{:keys [post-calls stream-calls stop-stream-calls]}]@@ -187,8 +188,12 @@(tu/slack-request-options event-body)event-body)](is (= "ok" response))- (u/poll {:thunk #(>= (count @post-calls) 1)- :done? true?+ ;; Poll on the slack_msg_id backfill rather than the post call. The backfill runs after+ ;; post-thread-reply returns, so waiting on post-calls alone races the DB update in CI.+ (u/poll {:thunk #(t2/select-one :model/MetabotMessage+ :channel_id channel-id :role "assistant"+ :slack_msg_id [:not= nil])+ :done? some?:timeout-ms 5000})(testing "a single threaded reply is posted with the answer"(is (= 1 (count @post-calls)))@@ -198,7 +203,7 @@(is (empty? @stream-calls))(is (empty? @stop-stream-calls)))(testing "assistant message in DB has slack_msg_id backfilled"- (let [msg (t2/select-one :model/MetabotMessage :channel_id "C123" :role "assistant")]+ (let [msg (t2/select-one :model/MetabotMessage :channel_id channel-id :role "assistant")](is (some? (:slack_msg_id msg))))))))))))(deftest stream-start-failure-test