Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

BOT-1956: Render MBQL transform sources as portable-repr JSON for Metabot (#79575)

Andrei Simionescu28d47ffafe12
diff --git a/src/metabase/metabot/agent/user_context.clj b/src/metabase/metabot/agent/user_context.clj
index 44989ef6e17..de825dd8811 100644
--- a/src/metabase/metabot/agent/user_context.clj
+++ b/src/metabase/metabot/agent/user_context.clj
@@ -250,22 +250,8 @@
te/lines)))))
(defn- transform-query-source-text
- "Format a transform's `:query` source for the LLM.
-
- When the source carries a query map with a `:database` key, we normalise it and export to
- the same canonical portable representations form the `construct_notebook_query` tool
- consumes (rendered as a JSON code block). Both structured (`mbql.stage/mbql`) and native
- (`mbql.stage/native`) stages go through this path - the latter is intentional: the repr
- export preserves portable `card-id` / `snippet-id` references inside `template-tags`, and
- stays in lockstep with the freshly-built-query payloads `construct_notebook_query` returns
- to the LLM.
-
- Pre-resolved string sources (`:query` is itself a string, or carries `:query-content` -
- the SQL-tool's already-rendered shape) pass through unchanged: there's no map to
- normalise.
-
- Falls back to a `pprint`'d query map only as a last resort, when repr export is
- unavailable (e.g. a partially-broken `dataset_query`)."
+ "Format a transform's `:query` source for the LLM; the rendering and fallback contract
+ lives in [[llm-shape/export-query-for-llm]]."
[source]
(llm-shape/export-query-for-llm (:query source)))
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 41803b63a2a..ed6a5740ca0 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -179,7 +179,7 @@
(def ^:private permission-aware-content-store
"ContentStore for agent query construction. Alias for
[[shared.content-store/default-store]] — the chokepoint wrapper applies `api/read-check` to
- every lookup whenever `api/*current-user-id*` is bound, symmetrically across all five
+ every lookup whenever `api/*current-user-id*` is bound, symmetrically across all six
ContentStore methods. The unchecked underlying store rejects non-NanoID entity-id values."
shared.content-store/default-store)
diff --git a/src/metabase/metabot/tools/shared/content_store.clj b/src/metabase/metabot/tools/shared/content_store.clj
index 6fff5688ac2..e2dc18c9cd4 100644
--- a/src/metabase/metabot/tools/shared/content_store.clj
+++ b/src/metabase/metabot/tools/shared/content_store.clj
@@ -13,7 +13,7 @@
`default-store` is the standard agent / tool-path store — `unchecked-app-db-content-store`
wrapped with `read-checked`. Use this as the content store for any agent or tool path that
- runs under an authenticated request. The symmetry across all five methods (both
+ runs under an authenticated request. The symmetry across all six methods (both
`-by-entity-id` import-direction and `-by-id` export-direction) is intentional: the export
direction may surface entity_ids of Cards / Measures / Segments referenced inside an
exported query body, and a missing check on those branches is exactly the N1 ACL gap that
@@ -36,12 +36,13 @@
(defn read-checked
"Wrap `store` so every lookup applies `api/read-check` when `api/*current-user-id*` is
- bound. Symmetric across all five `ContentStore` methods."
+ bound. Symmetric across all six `ContentStore` methods."
[store]
(reify resolve.mp/ContentStore
(card-by-entity-id [_ eid] (maybe-read-check (resolve.mp/card-by-entity-id store eid)))
(measure-by-entity-id [_ eid] (maybe-read-check (resolve.mp/measure-by-entity-id store eid)))
(segment-by-entity-id [_ eid] (maybe-read-check (resolve.mp/segment-by-entity-id store eid)))
+ (card-by-id [_ id] (maybe-read-check (resolve.mp/card-by-id store id)))
(measure-by-id [_ id] (maybe-read-check (resolve.mp/measure-by-id store id)))
(segment-by-id [_ id] (maybe-read-check (resolve.mp/segment-by-id store id)))))
diff --git a/src/metabase/metabot/tools/shared/llm_shape.clj b/src/metabase/metabot/tools/shared/llm_shape.clj
index 768aa2de3ec..29b215834d6 100644
--- a/src/metabase/metabot/tools/shared/llm_shape.clj
+++ b/src/metabase/metabot/tools/shared/llm_shape.clj
@@ -77,6 +77,47 @@
(seq data))
(str "```json\n" (json/encode data {:pretty true}) "\n```")))
+(defn- query-edn-fallback
+ "Pretty-printed EDN of `query`, minus the `:lib/metadata` provider `normalize-query` attaches."
+ [query]
+ (u/pprint-to-str (cond-> query (map? query) (dissoc :lib/metadata))))
+
+(defn export-query-for-llm
+ "Render a `query` (legacy or pMBQL map, or a pre-resolved string) for the LLM. A query
+ map with a `:database` is normalized and exported to the portable representations form
+ the `construct_notebook_query` tool consumes (a JSON code block); pre-resolved string
+ sources pass through; a `pprint`'d map is the last-resort fallback. A permission-refused
+ export renders nothing at all rather than the fallback."
+ [query]
+ (cond
+ (string? query) query
+ (string? (:query-content query)) (:query-content query)
+ (and (map? query) (:database query))
+ (try
+ (let [normalized (lib-be/normalize-query query)
+ mp (lib-be/application-database-metadata-provider (:database normalized))
+ exported (repr.resolve/export-query mp normalized shared.content-store/default-store)]
+ (or (repr-data->llm-block exported)
+ (query-edn-fallback normalized)))
+ (catch Exception e
+ (when-not (= 403 (:status-code (ex-data e)))
+ (log/debugf "Failed to export query for LLM, using EDN fallback: %s" (ex-message e))
+ (query-edn-fallback query))))
+ (string? (get-in query [:native :query])) (get-in query [:native :query])
+ (map? query) (query-edn-fallback query)
+ :else (some-> query str)))
+
+(defn transform-query->text
+ "Render a transform source query for model context: native SQL verbatim, anything else
+ through [[export-query-for-llm]]. Portable JSON gets boundary newlines so its Markdown
+ fence remains valid when the result is interpolated inside an XML `<query>` element."
+ [query]
+ (or (when (map? query) (metabot.u/extract-sql-content query))
+ (when-let [text (export-query-for-llm query)]
+ (if (str/starts-with? text "```json\n")
+ (format "\n%s\n" text)
+ text))))
+
(defn escape-xml
"Escape XML special characters in a string.
Only needed for content that bypasses Selmer's auto-escaping (marked with |safe) or is interpolated
@@ -889,7 +930,7 @@
:transform_description description
:transform_source_type (some-> (:type source) clojure.core/name)
:transform_source_database (when-let [db (:source-database source)] (str db))
- :transform_source_query (metabot.u/transform-query->text (:query source))
+ :transform_source_query (transform-query->text (:query source))
:transform_target (when target (pr-str target))}))
(def formatters
@@ -992,28 +1033,3 @@
(str "<" tag (when-not (str/blank? attrs) (str " " attrs)) ">"
(escape-xml description) "</" tag ">")
(str "<" tag (when-not (str/blank? attrs) (str " " attrs)) "/>"))))
-
-(defn export-query-for-llm
- "Render a `query` (legacy or pMBQL map, or a pre-resolved string) for the LLM. A query
- map with a `:database` is normalized and exported to the portable representations form
- the `construct_notebook_query` tool consumes (a JSON code block); pre-resolved string
- sources pass through; a `pprint`'d map is the last-resort fallback."
- [query]
- (cond
- (string? query) query
- (string? (:query-content query)) (:query-content query)
- (and (map? query) (:database query))
- (try
- (let [normalized (lib-be/normalize-query query)
- database-id (:database normalized)
- mp (when database-id
- (lib-be/application-database-metadata-provider database-id))
- exported (some->> mp (#(repr.resolve/try-export-query % normalized shared.content-store/default-store)))]
- (if exported
- (str "```json\n" (json/encode exported {:pretty true}) "\n```")
- (u/pprint-to-str normalized)))
- (catch Exception _
- (u/pprint-to-str query)))
- (string? (get-in query [:native :query])) (get-in query [:native :query])
- (map? query) (u/pprint-to-str query)
- :else (some-> query str)))
diff --git a/src/metabase/metabot/tools/transforms.clj b/src/metabase/metabot/tools/transforms.clj
index 638cb8c148d..9afdc3b6a56 100644
--- a/src/metabase/metabot/tools/transforms.clj
+++ b/src/metabase/metabot/tools/transforms.clj
@@ -10,7 +10,6 @@
[metabase.metabot.tools.shared.llm-shape :as llm-shape]
[metabase.metabot.tools.transforms.write :as transforms-write-tools]
[metabase.metabot.tools.util :as metabot.tools.u]
- [metabase.metabot.util :as metabot.u]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.transforms.core :as transforms]
[metabase.util.log :as log]
@@ -22,9 +21,9 @@
;;; Formatting helpers
;;; ──────────────────────────────────────────────────────────────────
-;; Hand-built rather than `metabot.u/xml` so query and body content stays verbatim:
-;; `clojure.data.xml` escapes `<`/`>`/`&`, and an escaped query breaks `old_string`
-;; matching when the model quotes it back to the write tools.
+;; Hand-built rather than `clojure.data.xml` so query and body content stays verbatim:
+;; data.xml escapes `<`/`>`/`&`, and an escaped query breaks `old_string` matching when
+;; the model quotes it back to the write tools.
(defn- format-transform-details-output
[{:keys [id description source target] :as transform}]
(let [{source-type :type, :keys [query body source-database]} source]
@@ -33,8 +32,8 @@
(str " <description>" (llm-shape/escape-xml-content description) "</description>"))
(when source
(str " <source type=\"" (some-> source-type name) "\">"))
- (when query
- (str " <query>" (metabot.u/transform-query->text query) "</query>"))
+ (when-let [query-text (some-> query llm-shape/transform-query->text)]
+ (str " <query>" query-text "</query>"))
(when body
(str " <body>" body "</body>"))
(when source-database
diff --git a/src/metabase/metabot/util.clj b/src/metabase/metabot/util.clj
index a5b8754b135..97d4ff47fd0 100644
--- a/src/metabase/metabot/util.clj
+++ b/src/metabase/metabot/util.clj
@@ -56,16 +56,3 @@
(when (= 1 (count (get query "stages")))
(get-in query ["stages" 0 "native"]))
(get-in query ["native" "query"])))
-
-(defn transform-query->text
- "Render a transform source query for model context: the native SQL when the query
- has any, otherwise the EDN of the query with its metadata provider stripped so the
- provider is never printed into the context window. String queries (legacy data and
- orphaned sources, which skip normalization) pass through verbatim."
- [query]
- (when query
- (cond
- (string? query) query
- (map? query) (or (extract-sql-content query)
- (pr-str (dissoc query :lib/metadata)))
- :else (pr-str query))))
diff --git a/src/metabase/models/serialization/resolve/mp.clj b/src/metabase/models/serialization/resolve/mp.clj
index 3ad60441629..ff29e223864 100644
--- a/src/metabase/models/serialization/resolve/mp.clj
+++ b/src/metabase/models/serialization/resolve/mp.clj
@@ -6,10 +6,10 @@
`lib.metadata/MetadataProvider`, which may be the live application-DB-backed provider, a
test-only mock provider, or any cached variant.
- It does, however, touch the application DB for *Metabase-model* lookups (cards by
- `entity_id`, etc.) because the lib metadata protocol doesn't support filtering by
- `entity_id`, and cards in any case live in the application DB independently of whose
- warehouse the metadata provider points at.
+ Its default [[ContentStore]] does touch the application DB for *Metabase-model* lookups
+ (cards by `entity_id`, etc.) because the lib metadata protocol doesn't support filtering
+ by `entity_id`, and cards in any case live in the application DB independently of whose
+ warehouse the metadata provider points at. Callers can supply a different store.
Primary consumer: the agent-lib representations pipeline, which converts LLM-authored
portable MBQL queries (with FK paths like `[DB, SCHEMA, TABLE, FIELD]`) into numeric-ID
@@ -19,9 +19,7 @@
* `import-table-fk`, `import-field-fk`, `export-table-fk`, `export-field-fk` for
warehouse metadata.
* `import-fk-keyed` / `export-fk-keyed` for `:model/Database` by `:name`.
- * `import-fk` for `Card` / `:model/Card` by `entity_id` (source-card and metric refs).
- * `export-fk` for `Card` / `:model/Card` by `entity_id` (exporting final pMBQL back to
- portable representations YAML).
+ * `import-fk` / `export-fk` for `Card`, `Measure`, and `Segment` references by `entity_id`.
Everything else throws `:not-implemented-yet` for now.
@@ -29,14 +27,13 @@
The resolver has two orthogonal lookup responsibilities:
* **Warehouse metadata** (databases, tables, fields) is resolved through a
`lib.metadata/MetadataProvider`.
- * **Metabase content / assets** (cards, snippets, segments, …) is resolved through a
- [[ContentStore]] on import, where callers may need permission-aware lookups. Exporting
- card ids uses the same database-scoped metadata provider, which already contains card
- metadata in app-backed and mock-provider contexts. The default app-DB-backed store goes
- through `serdes/lookup-by-id`, but a different store (e.g. backed by the checker's YAML
- index, an in-memory test fixture, or a snapshot) can be supplied to make import usable
- without an application database."
+ * **Metabase content / assets** (cards, measures, segments, …) is resolved through a
+ [[ContentStore]] in both directions, where callers may need permission-aware lookups.
+ The default store is app-DB-backed; a different store (e.g. backed by the checker's YAML
+ index, an in-memory test fixture, or a snapshot) can be supplied for contexts without an
+ application database."
(:require
+ [clojure.string :as str]
[metabase.app-db.core :as mdb]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
@@ -278,11 +275,11 @@
(into [(db-name metadata-provider) (:schema table) (:name table)] chain)))
;;; ============================================================
-;;; Content store - Metabase asset lookups (cards, etc.) by portable id
+;;; Content store - Metabase asset lookups by portable entity id or numeric id
;;; ============================================================
(p.types/defprotocol+ ContentStore
- "Lookup of Metabase content (\"assets\") by portable id.
+ "Lookup of Metabase content (\"assets\") by portable entity id or numeric id.
Kept separate from the warehouse-metadata `MetadataProvider` so the resolver can be reused in
contexts without an application database (e.g. the serdes checker, in-memory tests)."
@@ -296,6 +293,11 @@
(segment-by-entity-id [this entity-id]
"Return the segment row for the given portable `entity_id`, or nil. Same contract as
`measure-by-entity-id`.")
+ (card-by-id [this card-id]
+ "Return the card row for the given numeric id, or nil. Used by the export direction
+ (a numeric `source-card` / metric ref → its portable `entity_id`). The returned map must
+ carry `:entity_id` (or `:entity-id`) and `:database_id` (or `:database-id`) for the
+ cross-database guard.")
(measure-by-id [this measure-id]
"Return the measure row for the given numeric id, or nil. Same contract as
`measure-by-entity-id`; used by the export direction (`[:measure {} <id>]` →
@@ -311,13 +313,14 @@
wrap this with `metabase.metabot.tools.shared.content-store/read-checked`** (or use
`shared.content-store/default-store`, which is the wrapped form).
- Default [[ContentStore]] backed by the Metabase application database via
- `serdes/lookup-by-id`. Use this in production code paths that already have an app DB; pass a
- different store implementation when running without one (checker, isolated tests).
+ Default [[ContentStore]] backed by the Metabase application database. Portable entity-id
+ lookups use `serdes/lookup-by-id`; numeric export lookups use direct Toucan selects. Use this
+ in production code paths that already have an app DB; pass a different store implementation
+ when running without one (checker, isolated tests).
- Gated on [[resolve/entity-id?]]: LLM-authored entity-id values are untrusted, so anything that
- isn't a 21-char NanoID short-circuits to `nil` and the caller surfaces a clear `:unknown-…`
- agent error."
+ Portable entity-id lookups are gated on [[resolve/entity-id?]]: LLM-authored values are
+ untrusted, so anything that isn't a 21-char NanoID short-circuits to `nil` and the caller
+ surfaces a clear `:unknown-…` agent error."
(reify ContentStore
(card-by-entity-id [_ entity-id]
(when (resolve/entity-id? entity-id)
@@ -328,6 +331,13 @@
(segment-by-entity-id [_ entity-id]
(when (resolve/entity-id? entity-id)
(serdes/lookup-by-id 'Segment entity-id)))
+ (card-by-id [_ card-id]
+ (when card-id
+ ;; `api/read-check` for Cards needs only the parent collection. Avoid loading and
+ ;; transforming the entire dataset_query just to export one stable identifier.
+ ;; `:card_schema` must ride along: selecting `:database_id` makes the after-select
+ ;; treat this as a full card row and demand it.
+ (t2/select-one [:model/Card :id :entity_id :collection_id :database_id :card_schema] :id card-id)))
(measure-by-id [_ measure-id]
(when measure-id
(t2/select-one [:model/Measure :id :entity_id :table_id] :id measure-id)))
@@ -394,18 +404,19 @@
(defn- export-card-by-id
"Resolve a saved question / model / metric by numeric id to its portable `entity_id`.
- Uses the metadata provider rather than the generic app-DB serdes resolver so exporting a
- final pMBQL query can stay paired with the same database-scoped provider used for table and
- field FK export. Guards against accidental cross-database card refs."
- [metadata-provider card-id]
+ The row comes from `content-store`, not the metadata provider: the provider is
+ permission-agnostic, so a read-checked store is what keeps an unreadable Card's stable id
+ out of the export. The stored `database_id` guards against cross-database card refs, like
+ `table-belongs-to-current-database?` does for measures and segments."
+ [metadata-provider content-store card-id]
(when card-id
(let [current-db-id (:id (lib.metadata/database metadata-provider))
- card (lib.metadata.protocols/card metadata-provider card-id)
- card-db-id (or (:database-id card) (:database_id card))
- entity-id (or (:entity-id card) (:entity_id card))]
+ card (card-by-id content-store card-id)
+ card-db-id (when card (or (:database-id card) (:database_id card)))
+ entity-id (when card (or (:entity-id card) (:entity_id card)))]
(cond
(nil? card)
- (throw (ex-info (tru "No saved question, model, or metric found with id {0} in metadata provider." card-id)
+ (throw (ex-info (tru "No saved question, model, or metric found with id {0}." card-id)
{:status-code 400
:error :unknown-card-id
:card-id card-id}))
@@ -419,7 +430,7 @@
:card-database-id card-db-id
:expected-database current-db-id}))
- (not (and (string? entity-id) (seq entity-id)))
+ (or (not (string? entity-id)) (str/blank? entity-id))
(throw (ex-info (tru "Saved question, model, or metric id {0} does not have an entity_id, so it cannot be exported as a portable representation." card-id)
{:status-code 400
:error :missing-card-entity-id
@@ -609,10 +620,10 @@
with `metabase.metabot.tools.shared.content-store/read-checked`.
Implemented methods:
- * `import-table-fk`, `import-field-fk` (Phase 1).
- * `import-fk-keyed` for `:model/Database` by `:name` (Phase 1 - needed because
+ * `import-table-fk`, `import-field-fk`.
+ * `import-fk-keyed` for `:model/Database` by `:name` (needed because
`resolve/import-mbql` dispatches on `:database` keys).
- * `import-fk` for `Card` / `:model/Card` by `entity_id` (Phase 2, step 11).
+ * `import-fk` for `Card`, `Measure`, and `Segment` by `entity_id`.
Other methods throw `:not-implemented-yet`."
([metadata-provider]
@@ -643,7 +654,10 @@
(:id (find-field metadata-provider path)))))))
(defn export-resolver
- "Build a `SerdesExportResolver` backed by `metadata-provider`.
+ "Build a `SerdesExportResolver` backed by `metadata-provider` and `content-store`.
+
+ The 1-arity form uses [[unchecked-app-db-content-store]]. Agent-facing callers must pass
+ an explicit permission-aware store so exported content entity IDs are read-checked.
Implemented methods:
* `export-table-fk`, `export-field-fk` for warehouse metadata.
@@ -659,7 +673,7 @@
(export-fk [_ id model]
(cond
(nil? id) nil
- (card-model? model) (export-card-by-id metadata-provider id)
+ (card-model? model) (export-card-by-id metadata-provider content-store id)
(measure-model? model) (export-measure-by-id metadata-provider content-store id)
(segment-model? model) (export-segment-by-id metadata-provider content-store id)
:else (not-implemented! :export-fk)))
diff --git a/test/metabase/agent_lib/representations/resolve_test.clj b/test/metabase/agent_lib/representations/resolve_test.clj
index afaca10c90e..99db664398f 100644
--- a/test/metabase/agent_lib/representations/resolve_test.clj
+++ b/test/metabase/agent_lib/representations/resolve_test.clj
@@ -451,6 +451,9 @@
eid))
(measure-by-entity-id [_ eid] (when (= eid measure-entity-id) {:id 77 :table_id 10}))
(segment-by-entity-id [_ _] nil)
+ (card-by-id [_ id] (get {900 {:id 900 :database_id 1 :entity_id metric-entity-id}
+ 902 {:id 902 :database_id 1 :entity_id datetime-metric-entity-id}}
+ id))
(measure-by-id [_ _] nil)
(segment-by-id [_ _] nil)))
@@ -587,27 +590,35 @@
:source-table 99999}]}]
(is (nil? (repr.resolve/try-export-query mp bogus))))))
+(def ^:private source-card-export-query
+ {:lib/type :mbql/query
+ :database 1
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card 900}]})
+
+(defn- recording-export-content-store [calls]
+ (reify resolve.mp/ContentStore
+ (card-by-entity-id [_ _] nil)
+ (measure-by-entity-id [_ _] nil)
+ (segment-by-entity-id [_ _] nil)
+ (card-by-id [_ id]
+ (swap! calls conj id)
+ {:id id :database_id 1 :entity_id metric-entity-id})
+ (measure-by-id [_ _] nil)
+ (segment-by-id [_ _] nil)))
+
(deftest export-query-3-arity-threads-content-store-test
- (testing (str "export-query / try-export-query accept an explicit content-store as their "
- "third argument. The store is used for Card / Measure / Segment id → "
- "entity_id lookups (the N1 chokepoint). Pass an erroring store and the "
- "export-side `:export-fk` lookups should fail through that store, not "
- "through the default unchecked one. We use a basic 1-stage query (no card "
- "ref) so we can prove the 3-arity wiring works regardless of whether the "
- "particular query needs a content-store lookup.")
- (let [parsed {"lib/type" "mbql/query"
- "database" "Sample"
- "stages" [{"lib/type" "mbql.stage/mbql"
- "source-table" ["Sample" "PUBLIC" "ORDERS"]
- "aggregation" [["count" {}]]}]}
- q (repr.resolve/resolve-query mp parsed)]
- (testing "3-arity export-query mirrors 2-arity for queries with no Card / Measure / Segment refs"
- (is (= (repr.resolve/export-query mp q)
- (repr.resolve/export-query mp q
- @(requiring-resolve
- 'metabase.models.serialization.resolve.mp/unchecked-app-db-content-store)))))
- (testing "3-arity try-export-query likewise"
- (is (= (repr.resolve/try-export-query mp q)
- (repr.resolve/try-export-query mp q
- @(requiring-resolve
- 'metabase.models.serialization.resolve.mp/unchecked-app-db-content-store))))))))
+ (testing "export-query uses its explicit content store for source-card export"
+ (let [calls (atom [])
+ store (recording-export-content-store calls)
+ exported (repr.resolve/export-query mp-with-metric source-card-export-query store)]
+ (is (= metric-entity-id (get-in exported ["stages" 0 "source-card"])))
+ (is (= [900] @calls)))))
+
+(deftest try-export-query-3-arity-threads-content-store-test
+ (testing "try-export-query uses its explicit content store for source-card export"
+ (let [calls (atom [])
+ store (recording-export-content-store calls)
+ exported (repr.resolve/try-export-query mp-with-metric source-card-export-query store)]
+ (is (= metric-entity-id (get-in exported ["stages" 0 "source-card"])))
+ (is (= [900] @calls)))))
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index a3ab3e4a8a1..1dd2b3ed1ed 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -19,7 +19,7 @@
[metabase.metabot.tools.construct :as construct]
[metabase.metabot.tools.entity-details :as entity-details]
[metabase.models.interface :as mi]
- [metabase.models.serialization :as serdes]))
+ [metabase.models.serialization.resolve.mp :as resolve.mp]))
(set! *warn-on-reflection* true)
@@ -872,15 +872,25 @@
:result-metadata [{:name "ID" :base-type :type/Integer}
{:name "TOTAL" :base-type :type/Float}]}]}))
-(defn- lookup-card-stub [model eid]
- (when (and (or (= model 'Card) (= model :model/Card))
- (= eid card-entity-id))
- {:id 500 :database_id 1 :entity_id eid}))
+(defn- stub-content-store
+ "Stands in for the app-DB store over a mock metadata provider, serving `row` by entity id and
+ by numeric id. It answers from `row` rather than delegating to
+ [[metabase.models.serialization/lookup-by-id]], so code that bypassed the store could not
+ satisfy these tests by reaching the serdes resolver directly."
+ [row]
+ (reify resolve.mp/ContentStore
+ (card-by-entity-id [_ eid] (when (= eid (:entity_id row)) row))
+ (measure-by-entity-id [_ _] nil)
+ (segment-by-entity-id [_ _] nil)
+ (card-by-id [_ id] (when (= id (:id row)) row))
+ (measure-by-id [_ _] nil)
+ (segment-by-id [_ _] nil)))
(defn- with-card-mp-and-stubs! [f]
(with-redefs [lib-be/application-database-metadata-provider (fn [_] mp-with-card)
construct/resolve-database-id-from-first-stage (fn [_] 1)
- serdes/lookup-by-id lookup-card-stub
+ construct/permission-aware-content-store (stub-content-store
+ {:id 500 :database_id 1 :entity_id card-entity-id})
api/read-check allow-read-check
api/query-check allow-read-check]
(f)))
@@ -1084,15 +1094,11 @@
:aggregation [[:sum {}
[:field {:base-type :type/Float} 101]]]}]}}]}))
-(defn- lookup-metric-stub [model eid]
- (when (and (or (= model 'Card) (= model :model/Card))
- (= eid metric-entity-id))
- {:id 900 :database_id 1 :entity_id eid}))
-
(defn- with-metric-mp-and-stubs! [f]
(with-redefs [lib-be/application-database-metadata-provider (fn [_] mp-with-metric)
construct/resolve-database-id-from-first-stage (fn [_] 1)
- serdes/lookup-by-id lookup-metric-stub
+ construct/permission-aware-content-store (stub-content-store
+ {:id 900 :database_id 1 :entity_id metric-entity-id})
api/read-check allow-read-check
api/query-check allow-read-check]
(f)))
@@ -1197,9 +1203,8 @@
(defn- with-joined-metric-mp-and-stubs! [f]
(with-redefs [lib-be/application-database-metadata-provider (fn [_] mp-metric)
construct/resolve-database-id-from-first-stage (fn [_] 1)
- serdes/lookup-by-id (fn [model eid]
- (when (and (#{'Card :model/Card} model) (= eid metric-eid))
- {:id 700 :database_id 1 :entity_id eid}))
+ construct/permission-aware-content-store (stub-content-store
+ {:id 700 :database_id 1 :entity_id metric-eid})
api/read-check allow-read-check
api/query-check allow-read-check
;; `metric-details` gates its base table and every surfaced column on app-db
diff --git a/test/metabase/metabot/tools/shared/content_store_test.clj b/test/metabase/metabot/tools/shared/content_store_test.clj
index bdca1750a6b..2cc9857cb39 100644
--- a/test/metabase/metabot/tools/shared/content_store_test.clj
+++ b/test/metabase/metabot/tools/shared/content_store_test.clj
@@ -7,27 +7,30 @@
must keep working without an authenticated user. The wrapper short-circuits
`api/read-check` and returns the inner store's row unchanged.
- 2. **Read-checked when a user is bound** — every method (5 total, symmetric across import-
- and export-direction lookups) routes through `api/read-check`. A user without read
+ 2. **Read-checked when a user is bound** — all six methods (symmetric across import- and
+ export-direction lookups) route through `api/read-check`. A user without read
perms gets a 403; the underlying store is consulted for the existence check, and
unknown / `nil` returns pass through cleanly so the per-model resolver can emit its
`:unknown-…` agent error."
(:require
[clojure.test :refer :all]
[metabase.api.common :as api]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.metabot.tools.shared.content-store :as shared.content-store]
+ [metabase.models.serialization.resolve :as resolve]
[metabase.models.serialization.resolve.mp :as resolve.mp]
[metabase.test :as mt]))
(defn- record-store
- "A fake ContentStore that returns a fixed row for any of the 5 methods. Returning a fixed
- row whose `:row-id` is the requested argument lets tests assert both that the inner
- method was invoked (`:row-id` carries over) and what `:can-read?` posture each row has."
- [{:keys [card measure-eid segment-eid measure-id segment-id]}]
+ "A fake ContentStore that returns the configured row for each of its six methods."
+ [{:keys [card measure-eid segment-eid card-id measure-id segment-id]}]
(reify resolve.mp/ContentStore
(card-by-entity-id [_ _eid] card)
(measure-by-entity-id [_ _eid] measure-eid)
(segment-by-entity-id [_ _eid] segment-eid)
+ (card-by-id [_ _id] card-id)
(measure-by-id [_ _id] measure-id)
(segment-by-id [_ _id] segment-id)))
@@ -36,16 +39,18 @@
;;; ============================================================
(deftest pass-through-when-no-user-bound-test
- (testing "with api/*current-user-id* unbound, every method returns the inner store's row unchanged"
+ (testing "with api/*current-user-id* unbound, all six methods return the inner store's row unchanged"
;; Plain Clojure maps don't satisfy `mi/can-read?`, so a `read-check` call would throw.
;; The fact that these assertions pass proves no read-check fires.
(let [row {:opaque :marker}
- store (record-store {:card row :measure-eid row :segment-eid row :measure-id row :segment-id row})
+ store (record-store {:card row :measure-eid row :segment-eid row
+ :card-id row :measure-id row :segment-id row})
gated (shared.content-store/read-checked store)]
(is (nil? api/*current-user-id*) "precondition: no user bound")
(is (= row (resolve.mp/card-by-entity-id gated "x")))
(is (= row (resolve.mp/measure-by-entity-id gated "x")))
(is (= row (resolve.mp/segment-by-entity-id gated "x")))
+ (is (= row (resolve.mp/card-by-id gated 1)))
(is (= row (resolve.mp/measure-by-id gated 1)))
(is (= row (resolve.mp/segment-by-id gated 1))))))
@@ -62,6 +67,7 @@
(is (nil? (resolve.mp/card-by-entity-id gated "x")))
(is (nil? (resolve.mp/measure-by-entity-id gated "x")))
(is (nil? (resolve.mp/segment-by-entity-id gated "x")))
+ (is (nil? (resolve.mp/card-by-id gated 1)))
(is (nil? (resolve.mp/measure-by-id gated 1)))
(is (nil? (resolve.mp/segment-by-id gated 1)))))))
@@ -70,13 +76,14 @@
;;; ============================================================
(deftest applies-read-check-when-user-bound-test
- (testing (str "with api/*current-user-id* bound, every method routes the inner row through "
+ (testing (str "with api/*current-user-id* bound, all six methods route the inner row through "
"`api/read-check`. Using a row that doesn't satisfy `can-read?` proves the "
"check fires on each branch — we stub `api/read-check` to a recording fn so "
"we don't need a real auth setup, and assert it was called with the row.")
(let [calls (atom [])
row {:opaque :marker}
- store (record-store {:card row :measure-eid row :segment-eid row :measure-id row :segment-id row})
+ store (record-store {:card row :measure-eid row :segment-eid row
+ :card-id row :measure-id row :segment-id row})
gated (shared.content-store/read-checked store)]
(mt/with-dynamic-fn-redefs [api/read-check (fn [obj] (swap! calls conj obj) obj)]
(binding [api/*current-user-id* 1]
@@ -89,19 +96,22 @@
(testing "segment-by-entity-id"
(is (= row (resolve.mp/segment-by-entity-id gated "x")))
(is (= row (last @calls))))
+ (testing "card-by-id"
+ (is (= row (resolve.mp/card-by-id gated 1)))
+ (is (= row (last @calls))))
(testing "measure-by-id"
(is (= row (resolve.mp/measure-by-id gated 1)))
(is (= row (last @calls))))
(testing "segment-by-id"
(is (= row (resolve.mp/segment-by-id gated 1)))
(is (= row (last @calls))))
- (testing "every one of the five methods invoked read-check exactly once"
- (is (= 5 (count @calls)))))))))
+ (testing "all six methods invoke read-check exactly once"
+ (is (= 6 (count @calls)))))))))
(deftest propagates-read-check-403-test
(testing "if read-check throws (403), the wrapper propagates the exception unchanged"
(let [row {:opaque :marker}
- store (record-store {:card row :measure-id row})
+ store (record-store {:card row :card-id row :measure-id row})
gated (shared.content-store/read-checked store)]
(mt/with-dynamic-fn-redefs [api/read-check (fn [_]
(throw (ex-info "Forbidden" {:status-code 403})))]
@@ -112,12 +122,13 @@
(is false "expected throw")
(catch clojure.lang.ExceptionInfo e
(is (= 403 (:status-code (ex-data e)))))))
- (testing "export-direction branch (the N1 gap)"
- (try
- (resolve.mp/measure-by-id gated 1)
- (is false "expected throw")
- (catch clojure.lang.ExceptionInfo e
- (is (= 403 (:status-code (ex-data e))))))))))))
+ (testing "export-direction lookups"
+ (doseq [lookup [resolve.mp/card-by-id resolve.mp/measure-by-id]]
+ (try
+ (lookup gated 1)
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= 403 (:status-code (ex-data e)))))))))))))
;;; ============================================================
;;; default-store integration shape
@@ -132,3 +143,30 @@
(is (nil? (resolve.mp/card-by-entity-id shared.content-store/default-store "not-a-nanoid")))
(is (nil? (resolve.mp/measure-by-entity-id shared.content-store/default-store "not-a-nanoid")))
(is (nil? (resolve.mp/segment-by-entity-id shared.content-store/default-store "not-a-nanoid")))))
+
+;;; ============================================================
+;;; Card export through the default store
+;;; ============================================================
+
+(deftest export-card-entity-id-is-permission-gated-test
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-temp [:model/Collection {collection-id :id} {}
+ :model/Card {card-id :id, entity-id :entity_id}
+ {:collection_id collection-id
+ :database_id (mt/id)
+ :dataset_query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}]
+ (let [resolver (resolve.mp/export-resolver
+ (lib-be/application-database-metadata-provider (mt/id))
+ shared.content-store/default-store)
+ export! #(resolve/export-fk resolver card-id 'Card)]
+ (testing "a user who can read the source Card gets its entity_id"
+ (mt/with-current-user (mt/user->id :crowberto)
+ (is (= entity-id (export!)))))
+ (testing "a user who cannot read the source Card's collection gets a 403, not the entity_id"
+ (mt/with-current-user (mt/user->id :rasta)
+ (try
+ (export!)
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= 403 (:status-code (ex-data e))))))))))))
diff --git a/test/metabase/metabot/tools/shared/llm_shape_test.clj b/test/metabase/metabot/tools/shared/llm_shape_test.clj
index 74527078aef..c0300298af9 100644
--- a/test/metabase/metabot/tools/shared/llm_shape_test.clj
+++ b/test/metabase/metabot/tools/shared/llm_shape_test.clj
@@ -2,7 +2,11 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase.metabot.tools.shared.llm-shape :as llm-shape]))
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.metabot.tools.shared.llm-shape :as llm-shape]
+ [metabase.test :as mt]
+ [metabase.util.json :as json]))
(deftest ^:parallel escape-xml-test
(testing "escape-xml handles special characters"
@@ -909,8 +913,84 @@
(is (str/includes? xml "related_by_field_name=\"user_id\" related_by_field_id=\"303\""))
(is (str/includes? xml "related_by_field_name=\"review_id\" related_by_field_id=\"404\""))))))
+(deftest ^:parallel transform-query->text-native-test
+ (testing "native SQL renders verbatim"
+ (are [query] (= "SELECT 1" (llm-shape/transform-query->text query))
+ {:stages [{:lib/type :mbql.stage/native :native "SELECT 1"}]}
+ {:native {:query "SELECT 1"}}
+ "SELECT 1")))
+
+(deftest ^:parallel transform-query->text-orphaned-source-test
+ (testing "orphaned sources skip normalization and still render as SQL through their string keys"
+ (are [query] (= "SELECT 1" (llm-shape/transform-query->text query))
+ {"database" nil "native" {"query" "SELECT 1"}}
+ {"database" nil "stages" [{"native" "SELECT 1"}]})))
+
+(deftest ^:parallel transform-query->text-nil-test
+ (testing "nil stays nil"
+ (is (nil? (llm-shape/transform-query->text nil)))))
+
+(deftest ^:parallel transform-query->text-portable-mbql-test
+ (testing "an MBQL query exports to portable form, referencing tables by FK path"
+ (let [mp (mt/metadata-provider)
+ text (llm-shape/transform-query->text
+ (lib/query mp (lib.metadata/table mp (mt/id :products))))
+ exported (json/decode (second (re-find #"(?s)```json\n(.*)\n```" text)))]
+ (is (= [(:name (lib.metadata/database mp)) "PUBLIC" "PRODUCTS"]
+ (get-in exported ["stages" 0 "source-table"])))
+ (is (not (contains? exported "lib/metadata"))))))
+
+(deftest ^:parallel transform-query->text-export-failure-fallback-test
+ (testing "a query that fails to export falls back to EDN without the metadata provider"
+ (let [text (llm-shape/transform-query->text
+ {:lib/type :mbql/query
+ :database (mt/id)
+ :stages [{:lib/type :mbql.stage/mbql :source-table Integer/MAX_VALUE}]})]
+ (is (str/includes? text ":mbql.stage/mbql"))
+ (is (not (str/includes? text ":lib/metadata"))))))
+
+(deftest ^:parallel transform-query->text-existing-metadata-fallback-test
+ (testing "the EDN fallback strips an existing provider even when there is no database to normalize"
+ (let [text (llm-shape/transform-query->text
+ {:lib/type :mbql/query
+ :lib/metadata :fake-provider
+ :stages [{:lib/type :mbql.stage/mbql}]})]
+ (is (str/includes? text ":mbql.stage/mbql"))
+ (is (not (str/includes? text ":lib/metadata"))))))
+
+(deftest ^:parallel transform-query->text-multi-stage-test
+ (testing "a native stage followed by an MBQL stage renders every stage, not the first stage's SQL"
+ (let [mp (mt/metadata-provider)
+ text (llm-shape/transform-query->text
+ (lib/append-stage (lib/native-query mp "SELECT * FROM PRODUCTS")))
+ exported (json/decode (second (re-find #"(?s)```json\n(.*)\n```" text)))]
+ (is (= 2 (count (get exported "stages"))))
+ (is (= "SELECT * FROM PRODUCTS" (get-in exported ["stages" 0 "native"]))))))
+
+(deftest transform-query->text-source-card-permission-test
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-temp [:model/Collection {collection-id :id} {}
+ :model/Card {card-id :id, entity-id :entity_id}
+ {:collection_id collection-id
+ :database_id (mt/id)
+ :dataset_query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}]
+ (let [query {:lib/type :mbql/query
+ :database (mt/id)
+ :stages [{:lib/type :mbql.stage/mbql
+ :source-card card-id}]}
+ render-as (fn [user-id]
+ (mt/with-current-user user-id
+ (llm-shape/transform-query->text query)))
+ readable (render-as (mt/user->id :crowberto))
+ exported (json/decode (second (re-find #"(?s)```json\n(.*)\n```" readable)))]
+ (testing "a user who can read the source Card gets its portable entity id"
+ (is (= entity-id (get-in exported ["stages" 0 "source-card"]))))
+ (testing "a user who cannot read the source Card gets nothing at all"
+ (is (nil? (render-as (mt/user->id :rasta)))))))))
+
(deftest ^:parallel transform->xml-source-query-test
- (testing "a normalized (map) source query renders as verbatim SQL text"
+ (testing "a native source query renders as verbatim SQL text"
(let [xml (llm-shape/transform->xml
{:id 7
:name "Orders rollup"
@@ -918,13 +998,13 @@
:query {:stages [{:lib/type :mbql.stage/native
:native "SELECT * FROM orders WHERE total < 100"}]}}})]
(is (str/includes? xml "<query>SELECT * FROM orders WHERE total < 100</query>"))))
- (testing "a notebook-built source query renders as EDN with the metadata provider stripped"
+ (testing "an MBQL source query renders as a JSON code block"
(let [xml (llm-shape/transform->xml
{:id 8
:name "Notebook rollup"
:source {:type :query
- :query {:lib/type :mbql/query
- :lib/metadata :fake-metadata-provider
- :stages [{:lib/type :mbql.stage/mbql :source-table 1}]}}})]
- (is (str/includes? xml ":mbql.stage/mbql"))
- (is (not (str/includes? xml ":lib/metadata"))))))
+ :query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :products)))}})]
+ (is (str/includes? xml "<query>\n```json"))
+ (is (str/includes? xml "```\n</query>"))
+ (is (str/includes? xml "\"PRODUCTS\"")))))
diff --git a/test/metabase/metabot/tools/transforms_test.clj b/test/metabase/metabot/tools/transforms_test.clj
index 679a82b5833..9bc4d87c429 100644
--- a/test/metabase/metabot/tools/transforms_test.clj
+++ b/test/metabase/metabot/tools/transforms_test.clj
@@ -11,42 +11,77 @@
[metabase.metabot.tools.transforms :as agent-transforms]
[metabase.metabot.tools.transforms.write :as transforms-write]
[metabase.premium-features.core :as premium-features]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.util.json :as json]))
;;; ----------------------------------- read tool integration tests ---------------------------------------------------
-(deftest get-transform-details-tool-test
+(def ^:private gadget-sql
+ "SELECT * FROM products WHERE price < 100 AND category <> 'Widget'")
+
+(deftest get-transform-details-native-query-test
(mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
(mt/with-current-user (mt/user->id :crowberto)
- (testing "query source renders as verbatim SQL text, unescaped"
- (mt/with-temp [:model/Transform {transform-id :id}
- {:name "Gadget Products"
- :source {:type "query"
- :query (lib/native-query (mt/metadata-provider)
- "SELECT * FROM products WHERE price < 100 AND category <> 'Widget'")}}]
- (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
- (is (str/includes? output "name=\"Gadget Products\""))
- (is (str/includes? output "<source type=\"query\">"))
- (is (str/includes? output "<query>SELECT * FROM products WHERE price < 100 AND category <> 'Widget'</query>")))))
- (testing "notebook-built source falls back to EDN with the metadata provider stripped"
- (mt/with-temp [:model/Transform {transform-id :id}
- {:name "Notebook Products"
- :source {:type "query"
- :query (lib/query (mt/metadata-provider)
- (lib.metadata/table (mt/metadata-provider) (mt/id :products)))}}]
- (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
- (is (str/includes? output ":lib/type"))
- (is (not (str/includes? output ":lib/metadata"))))))
- (testing "python source renders its body and source database"
- (mt/with-temp [:model/Transform {transform-id :id}
- {:name "Gadget Metrics"
- :source {:type "python"
- :source-database (mt/id)
- :body "import pandas as pd"}}]
- (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
- (is (str/includes? output "<source type=\"python\">"))
- (is (str/includes? output "<body>import pandas as pd</body>"))
- (is (str/includes? output (str "<database>" (mt/id) "</database>")))))))))
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Gadget Products"
+ :source {:type "query"
+ :query (lib/native-query (mt/metadata-provider) gadget-sql)}}]
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output "name=\"Gadget Products\""))
+ (is (str/includes? output "<source type=\"query\">"))
+ (is (str/includes? output (str "<query>" gadget-sql "</query>"))))))))
+
+(deftest get-transform-details-notebook-query-test
+ (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Notebook Products"
+ :source {:type "query"
+ :query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :products)))}}]
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output "<query>\n```json"))
+ (is (str/includes? output "```\n</query>"))
+ (let [exported (json/decode (second (re-find #"(?s)```json\n(.*)\n```" output)))]
+ (is (= [(:name (lib.metadata/database (mt/metadata-provider))) "PUBLIC" "PRODUCTS"]
+ (get-in exported ["stages" 0 "source-table"])))
+ (is (not-any? #(and (map? %) (contains? % "lib/metadata"))
+ (tree-seq coll? seq exported)))))))))
+
+(deftest get-transform-details-python-source-test
+ (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id}
+ {:name "Gadget Metrics"
+ :source {:type "python"
+ :source-database (mt/id)
+ :body "import pandas as pd"}}]
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output "<source type=\"python\">"))
+ (is (str/includes? output "<body>import pandas as pd</body>"))
+ (is (str/includes? output (str "<database>" (mt/id) "</database>"))))))))
+
+(deftest get-transform-details-source-card-permission-test
+ (mt/with-premium-features #{:transforms-basic :transforms-python :hosting}
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (mt/with-full-data-perms-for-all-users!
+ (mt/with-data-analyst-role! (mt/user->id :rasta)
+ (mt/with-temp [:model/Collection {collection-id :id} {}
+ :model/Card {card-id :id, entity-id :entity_id}
+ {:collection_id collection-id
+ :database_id (mt/id)
+ :dataset_query (lib/query (mt/metadata-provider)
+ (lib.metadata/table (mt/metadata-provider) (mt/id :orders)))}
+ :model/Transform {transform-id :id}
+ {:name "Private source Card"
+ :source {:type "query"
+ :query (lib/query (mt/metadata-provider)
+ (lib.metadata/card (mt/metadata-provider) card-id))}}]
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [{:keys [output]} (agent-transforms/get-transform-details-tool {:transform_id transform-id})]
+ (is (str/includes? output "name=\"Private source Card\""))
+ (is (not (str/includes? output entity-id)))
+ (is (not (str/includes? output "<query>")))))))))))
;;; ----------------------------------- write tool integration tests --------------------------------------------------
diff --git a/test/metabase/metabot/util_test.clj b/test/metabase/metabot/util_test.clj
index 0dca1d009ba..031be696b9e 100644
--- a/test/metabase/metabot/util_test.clj
+++ b/test/metabase/metabot/util_test.clj
@@ -1,37 +1,32 @@
(ns metabase.metabot.util-test
(:require
- [clojure.string :as str]
[clojure.test :refer :all]
[metabase.metabot.util :as metabot.u]))
-(deftest ^:parallel transform-query->text-test
- (testing "native query renders as its SQL"
- (is (= "SELECT 1"
- (metabot.u/transform-query->text {:stages [{:lib/type :mbql.stage/native
- :native "SELECT 1"}]}))))
- (testing "legacy format renders as its SQL"
- (is (= "SELECT 2"
- (metabot.u/transform-query->text {:native {:query "SELECT 2"}}))))
- (testing "notebook-built query renders as EDN without the metadata provider"
- (let [text (metabot.u/transform-query->text {:lib/type :mbql/query
- :lib/metadata :fake-provider
- :stages [{:lib/type :mbql.stage/mbql}]})]
- (is (str/includes? text ":mbql.stage/mbql"))
- (is (not (str/includes? text ":lib/metadata")))))
- (testing "raw string query passes through verbatim"
- (is (= "SELECT * FROM legacy" (metabot.u/transform-query->text "SELECT * FROM legacy"))))
- (testing "orphaned source (string keys, skipped normalization) renders as its SQL"
- (is (= "SELECT 3"
- (metabot.u/transform-query->text {"database" nil
- "native" {"query" "SELECT 3"}})))
- (is (= "SELECT 4"
- (metabot.u/transform-query->text {"database" nil
- "stages" [{"native" "SELECT 4"}]}))))
- (testing "multi-stage query falls through to EDN rather than dropping stages"
- (let [text (metabot.u/transform-query->text {:stages [{:lib/type :mbql.stage/native
- :native "SELECT 5"}
- {:lib/type :mbql.stage/mbql}]})]
- (is (not= "SELECT 5" text))
- (is (str/includes? text ":mbql.stage/mbql"))))
- (testing "nil stays nil"
- (is (nil? (metabot.u/transform-query->text nil)))))
+(deftest ^:parallel extract-sql-content-native-test
+ (testing "extracts SQL from normalized pMBQL and legacy query shapes"
+ (are [expected query] (= expected (metabot.u/extract-sql-content query))
+ "SELECT 1" {:stages [{:lib/type :mbql.stage/native
+ :native "SELECT 1"}]}
+ "SELECT 2" {:native {:query "SELECT 2"}})))
+
+(deftest ^:parallel extract-sql-content-orphaned-query-test
+ (testing "extracts SQL from orphaned string-keyed query shapes"
+ (are [expected query] (= expected (metabot.u/extract-sql-content query))
+ "SELECT 3" {"database" nil
+ "native" {"query" "SELECT 3"}}
+ "SELECT 4" {"database" nil
+ "stages" [{"native" "SELECT 4"}]})))
+
+(deftest ^:parallel extract-sql-content-multi-stage-test
+ (testing "does not mistake the first stage of a multi-stage query for the whole query"
+ (are [query] (nil? (metabot.u/extract-sql-content query))
+ {:stages [{:lib/type :mbql.stage/native :native "SELECT 5"}
+ {:lib/type :mbql.stage/mbql}]}
+ {"stages" [{"native" "SELECT 6"}
+ {"lib/type" "mbql.stage/mbql"}]})))
+
+(deftest ^:parallel extract-sql-content-non-native-test
+ (testing "a non-native MBQL query has no SQL content"
+ (is (nil? (metabot.u/extract-sql-content
+ {:stages [{:lib/type :mbql.stage/mbql :source-table 1}]})))))
diff --git a/test/metabase/models/serialization/resolve/mp_test.clj b/test/metabase/models/serialization/resolve/mp_test.clj
index 90cfd735cca..6677ecc7239 100644
--- a/test/metabase/models/serialization/resolve/mp_test.clj
+++ b/test/metabase/models/serialization/resolve/mp_test.clj
@@ -98,6 +98,20 @@
:type :metric
:entity-id metric-entity-id}]}))
+(def ^:private cards-content-store
+ "Serves the [[mp-with-cards]] cards by numeric id. Card export reads `entity_id` through a
+ ContentStore rather than the permission-agnostic metadata provider, so a provider-only
+ fixture needs a matching store."
+ (reify resolve.mp/ContentStore
+ (card-by-entity-id [_ _entity-id] nil)
+ (measure-by-entity-id [_ _entity-id] nil)
+ (segment-by-entity-id [_ _entity-id] nil)
+ (card-by-id [_ card-id] (get {500 {:id 500 :database_id 1 :entity_id card-entity-id}
+ 501 {:id 501 :database_id 1 :entity_id metric-entity-id}}
+ card-id))
+ (measure-by-id [_ _measure-id] nil)
+ (segment-by-id [_ _segment-id] nil)))
+
;;; ============================================================
;;; import-table-fk
;;; ============================================================
@@ -409,7 +423,7 @@
(is (= path (resolve/export-field-fk er (resolve/import-field-fk ir path)))))))
(deftest ^:parallel export-database-and-card-fks-test
- (let [r (resolve.mp/export-resolver mp-with-cards)]
+ (let [r (resolve.mp/export-resolver mp-with-cards cards-content-store)]
(testing "database id exports to the provider's database name"
(is (= "Sample" (resolve/export-fk-keyed r 1 :model/Database :name)))
(is (= "Sample" (resolve/export-fk-keyed r 1 'Database :name))))
@@ -423,7 +437,7 @@
(deftest ^:parallel export-mbql-with-mp-resolver-round-trip-shape-test
(testing "final numeric pMBQL exports back to portable DB/table/field/card references"
- (let [r (resolve.mp/export-resolver mp-with-cards)
+ (let [r (resolve.mp/export-resolver mp-with-cards cards-content-store)
exported (resolve/export-mbql
r
{:lib/type :mbql/query
@@ -442,7 +456,7 @@
(is (string? (get-in exported [:stages 0 :fields 0 1 :lib/uuid])))
(is (string? (get-in exported [:stages 0 :aggregation 0 1 :lib/uuid])))))
(testing "source-card map keys export through the Card entity_id path"
- (let [r (resolve.mp/export-resolver mp-with-cards)]
+ (let [r (resolve.mp/export-resolver mp-with-cards cards-content-store)]
(is (= {:source-card card-entity-id}
(resolve/export-mbql r {:source-card 500}))))))
@@ -691,9 +705,48 @@
(get entity-id->card entity-id))
(measure-by-entity-id [_ _entity-id] nil)
(segment-by-entity-id [_ _entity-id] nil)
+ (card-by-id [_ card-id]
+ (first (filter #(= card-id (:id %)) (vals entity-id->card))))
(measure-by-id [_ _measure-id] nil)
(segment-by-id [_ _segment-id] nil)))
+(deftest ^:parallel export-fk-card-via-custom-content-store-test
+ (testing "Card export gets the entity id from the supplied store, including kebab-case rows"
+ (let [store-entity-id "storeEntityId12345678"
+ store (map-content-store
+ {"lookup-key"
+ {:id 500 :database_id 1 :entity-id store-entity-id}})
+ er (resolve.mp/export-resolver mp-with-cards store)
+ exported (resolve/export-fk er 500 'Card)]
+ (is (= store-entity-id exported))
+ (is (not= card-entity-id exported)
+ "the metadata provider's permission-agnostic entity id is not used"))))
+
+(deftest ^:parallel export-fk-card-cross-database-test
+ (testing "a stored card pinned to another database is rejected"
+ (let [store (map-content-store {"lookup-key" {:id 500 :database_id 999 :entity_id card-entity-id}})
+ er (resolve.mp/export-resolver mp-with-cards store)]
+ (try
+ (resolve/export-fk er 500 'Card)
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (let [d (ex-data e)]
+ (is (= :cross-database-card (:error d)))
+ (is (= 999 (:card-database-id d)))))))))
+
+(deftest ^:parallel export-fk-card-missing-entity-id-test
+ (testing "a card the store does not know is distinct from one with a blank entity id"
+ (doseq [[rows expected-error] {{} :unknown-card-id
+ {"lookup-key" {:id 500 :database_id 1}} :missing-card-entity-id
+ {"lookup-key" {:id 500 :database_id 1 :entity_id ""}} :missing-card-entity-id
+ {"lookup-key" {:id 500 :database_id 1 :entity_id " \t"}} :missing-card-entity-id}]
+ (let [er (resolve.mp/export-resolver mp-with-cards (map-content-store rows))]
+ (try
+ (resolve/export-fk er 500 'Card)
+ (is false "expected throw")
+ (catch clojure.lang.ExceptionInfo e
+ (is (= expected-error (:error (ex-data e))))))))))
+
(deftest ^:parallel import-fk-card-via-custom-content-store-happy-path-test
(testing "a custom ContentStore lets the resolver work without an app DB"
(let [card-eid "abcdefghijabcdefghij1"