Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Catch fields reached through a foreign key to the wrong table (#85051)

Andrei Simionescu4bf7eb06ba8f
diff --git a/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj b/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj
new file mode 100644
index 00000000000..0ae23a61295
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot/tools/construct_test.clj
@@ -0,0 +1,108 @@
+(ns metabase-enterprise.metabot.tools.construct-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [metabase-enterprise.sandbox.test-util :as sandbox.tu]
+ [metabase.api.common :as api]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.metabot.metadata-perms :as metabot.perms]
+ [metabase.metabot.tools.construct :as construct]
+ [metabase.permissions.core :as perms]
+ [metabase.query-processor :as qp]
+ [metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.test :as mt]
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
+
+(defn- sandbox-query [table fields]
+ (let [mp (mt/metadata-provider)]
+ (lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id table)))
+ (mapv #(lib.metadata/field mp (mt/id table %)) fields))))
+
+(defn- source-field-query [key-table key-column target-table]
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ field #(vector db "PUBLIC" %1 %2)]
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table [db "PUBLIC" "CHECKINS"]
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field (field key-table key-column)} (field target-table "NAME")]]}]}))
+
+(defn- assert-private-field-error! [key-table key-column target-table hidden-texts]
+ (let [build! #(construct/execute-representations-query (source-field-query key-table key-column target-table))
+ error (try (build!) (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :wrong-source-field (:error (ex-data error))))
+ (doseq [hidden-text hidden-texts]
+ (is (not (str/includes? (or (ex-message error) "") hidden-text)))
+ (is (not (str/includes? (pr-str (ex-data error)) hidden-text))))
+ (mt/with-dynamic-fn-redefs [construct/assert-source-fields-reach-their-tables! identity]
+ (let [query (get-in (build!) [:structured-output :query])]
+ (is (map? query))
+ (is (thrown? Exception (qp/process-query query)))))))
+
+(deftest source-field-errors-respect-column-sandboxes-test
+ (sandbox.tu/with-gtaps! {:gtaps {:checkins {:query (sandbox-query :checkins [:id :date])}}}
+ (doseq [table [:users :categories]]
+ (perms/set-table-permission! &group (mt/id table) :perms/create-queries :query-builder))
+ (mt/with-temp-vals-in-db :model/Database (mt/id) {:name "Sandbox check-ins"}
+ (let [table-id (mt/id :checkins)
+ allowed (get (metabot.perms/sandbox-restricted-fields #{table-id}) table-id)]
+ (is (= #{(mt/id :checkins :id) (mt/id :checkins :date)} allowed))
+ (is (= #{table-id (mt/id :users)} (metabot.perms/queryable-table-ids #{table-id (mt/id :users)}))))
+ (testing "the direct hint never names a hidden candidate key"
+ (assert-private-field-error! "CHECKINS" "DATE" "USERS" ["USER_ID"]))
+ (testing "the error does not export or confirm an authored hidden key"
+ (assert-private-field-error! "CHECKINS" "USER_ID" "CATEGORIES" ["USER_ID"])))))
+
+(deftest source-field-errors-respect-intermediate-column-sandboxes-test
+ (sandbox.tu/with-gtaps! {:gtaps {:venues {:query (sandbox-query :venues [:id :name])}}}
+ (doseq [table [:checkins :categories]]
+ (perms/set-table-permission! &group (mt/id table) :perms/create-queries :query-builder))
+ (mt/with-temp-vals-in-db :model/Database (mt/id) {:name "Sandbox venues"}
+ (assert-private-field-error! "CHECKINS" "VENUE_ID" "CATEGORIES" ["has a foreign key to"]))))
+
+(deftest source-field-errors-respect-target-column-sandboxes-test
+ (sandbox.tu/with-gtaps! {:gtaps {:categories {:query (sandbox-query :categories [:id])}}}
+ (doseq [table [:checkins :venues]]
+ (perms/set-table-permission! &group (mt/id table) :perms/create-queries :query-builder))
+ (mt/with-temp-vals-in-db :model/Database (mt/id) {:name "Sandbox categories"}
+ (assert-private-field-error! "CHECKINS" "VENUE_ID" "CATEGORIES" ["CATEGORIES" "VENUE_ID"]))))
+
+(deftest source-field-errors-respect-foreign-key-target-sandboxes-test
+ (sandbox.tu/with-gtaps! {:gtaps {:venues {:query (sandbox-query :venues [:name :category_id])}}}
+ (doseq [table [:checkins :categories]]
+ (perms/set-table-permission! &group (mt/id table) :perms/create-queries :query-builder))
+ (mt/with-temp-vals-in-db :model/Database (mt/id) {:name "Sandbox venue keys"}
+ (assert-private-field-error! "CHECKINS" "VENUE_ID" "CATEGORIES" ["VENUES" "has a foreign key to"]))))
+
+(deftest construct-query-with-native-sandbox-test
+ (sandbox.tu/with-gtaps! {:gtaps {:checkins {:query (lib/native-query (mt/metadata-provider) "SELECT * FROM CHECKINS")}}}
+ (perms/set-table-permission! &group (mt/id :venues) :perms/create-queries :query-builder)
+ (mt/with-temp-vals-in-db :model/Database (mt/id) {:name "Native sandbox"}
+ (let [card-id (t2/select-one-fn :card_id :model/Sandbox :table_id (mt/id :checkins) :group_id (u/the-id &group))
+ card-before (t2/select-one :model/Card card-id)
+ snapshot #(hash-map :executions (t2/count :model/QueryExecution) :cache (t2/count :model/QueryCache))
+ before (snapshot)
+ build! #(construct/execute-representations-query (source-field-query "CHECKINS" "VENUE_ID" "VENUES"))
+ original-execute qp.pipeline/*execute*
+ warehouse (atom [])
+ observe-execute (fn [& args]
+ (swap! warehouse conj {:user api/*current-user-id* :admin api/*is-superuser?*})
+ (apply original-execute args))
+ control (binding [qp.pipeline/*execute* observe-execute]
+ (mt/with-dynamic-fn-redefs [construct/assert-source-fields-reach-their-tables! identity]
+ (build!)))
+ card-after-control (t2/select-one :model/Card card-id)
+ actual (binding [qp.pipeline/*execute* observe-execute] (build!))]
+ (is (nil? (:result_metadata card-before)))
+ (is (= card-before card-after-control))
+ (is (map? (get-in actual [:structured-output :query])))
+ (is (empty? @warehouse) "Construction must not execute warehouse SQL")
+ (is (not-any? :admin @warehouse) "Construction must not execute SQL as admin")
+ (is (= card-before (t2/select-one :model/Card card-id)) "Construction must not write Card metadata")
+ (is (= before (snapshot)))
+ (let [control-result (qp/process-query (get-in control [:structured-output :query]))
+ actual-result (qp/process-query (get-in actual [:structured-output :query]))]
+ (is (= :completed (:status actual-result)))
+ (is (seq (mt/rows actual-result)))
+ (is (= (set (mt/rows control-result)) (set (mt/rows actual-result)))))))))
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index 8761e7c0188..fb2d53e09b0 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -15,13 +15,16 @@
[metabase.lib.schema]
[metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.util :as lib.util]
+ [metabase.lib.walk :as lib.walk]
[metabase.metabot.agent.links :as links]
[metabase.metabot.agent.streaming :as streaming]
[metabase.metabot.db :as metabot.db]
+ [metabase.metabot.metadata-perms :as metabot.perms]
[metabase.metabot.scope :as scope]
[metabase.metabot.tmpl :as te]
[metabase.metabot.tools.charts.create :as create-chart-tools]
[metabase.metabot.tools.core :as tools]
+ [metabase.metabot.tools.entity-details :as entity-details]
[metabase.metabot.tools.shared :as shared]
[metabase.metabot.tools.shared.content-store :as shared.content-store]
[metabase.metabot.tools.shared.instructions :as instructions]
@@ -30,6 +33,7 @@
[metabase.models.interface :as mi]
[metabase.models.serialization.resolve :as serdes.resolve]
[metabase.models.serialization.resolve.mp :as resolve.mp]
+ [metabase.query-processor.core :as qp]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
@@ -645,6 +649,127 @@
:status-code 400
:error :missing-source-in-first-stage})))))
+;;; ---------------------------------------- Implicit joins ----------------------------------------
+
+(defn- field-ref-for-error
+ "Return `field-ref` with portable field paths unless the caller accepts numeric IDs."
+ [metadata-provider field-ref]
+ (if serdes.resolve/*numeric-ids-allowed?*
+ field-ref
+ (serdes.resolve/export-mbql (resolve.mp/export-resolver metadata-provider permission-aware-content-store) field-ref)))
+
+(defn- foreign-key-target-table-id
+ [metadata-provider field]
+ (some->> (:fk-target-field-id field) (lib.metadata/field metadata-provider) :table-id))
+
+(defn- direct-source-field-hint
+ [metadata-provider columns table-id table-name]
+ (when-let [field-ids (seq (distinct (keep (fn [column]
+ (when (and (= table-id (:table-id column))
+ (nil? (:fk-join-alias column)))
+ (:fk-field-id column)))
+ columns)))]
+ (let [fields (lib.metadata/bulk-metadata metadata-provider :metadata/column field-ids)
+ queryable-tables (metabot.perms/queryable-table-ids (into #{} (keep :table-id) fields))
+ fields (->> fields
+ (filter #(contains? queryable-tables (:table-id %)))
+ (filter #(visible-field metadata-provider (:id %)))
+ entity-details/permission-filter-columns
+ (filter #(= table-id (foreign-key-target-table-id metadata-provider %))))]
+ (when (seq fields)
+ (format "Use `source-field` %s to reach %s directly."
+ (str/join " or " (map #(pr-str (nth (field-ref-for-error metadata-provider [:field {} (:id %)]) 2)) fields))
+ table-name)))))
+
+(defn- intermediate-source-field
+ [metadata-provider from-table-id to-table-id]
+ (let [fields (->> (resolve.mp/outbound-fks-from-table metadata-provider from-table-id)
+ (map :source-field)
+ (filter #(visible-field metadata-provider (:id %)))
+ entity-details/permission-filter-columns
+ (filter #(= to-table-id (foreign-key-target-table-id metadata-provider %))))]
+ (when (= 1 (count fields))
+ (first fields))))
+
+(defn- wrong-source-field-error
+ "Return an agent error for a field reached through an invalid foreign key."
+ [metadata-provider [_ {:keys [source-field source-field-join-alias]} target] table-id fk-table-id columns]
+ (let [table-name (pr-str (:name (lib.metadata.protocols/table metadata-provider table-id)))
+ ;; The metadata provider skips permission checks, so name the key's table only if the user may query it
+ fk-table-name (when (and fk-table-id (contains? (metabot.perms/queryable-table-ids #{fk-table-id}) fk-table-id))
+ (pr-str (:name (lib.metadata.protocols/table metadata-provider fk-table-id))))]
+ (ex-info (str (if fk-table-name
+ (format "Field %s is on table %s, but its `source-field` %s is a foreign key to table %s, so the field cannot be reached through it."
+ (pr-str target) table-name (pr-str source-field) fk-table-name)
+ (format "Field %s is on table %s, but its `source-field` %s is not a foreign key to that table, so the field cannot be reached through it."
+ (pr-str target) table-name (pr-str source-field)))
+ " "
+ (or (direct-source-field-hint metadata-provider columns table-id table-name)
+ (when fk-table-name
+ (when-let [field (intermediate-source-field metadata-provider fk-table-id table-id)]
+ (format "%s has a foreign key to %s: join %s explicitly on %s%s, then reference the field with `source-field` %s and `source-field-join-alias` set to the new join's alias."
+ fk-table-name table-name fk-table-name (pr-str source-field)
+ (if source-field-join-alias
+ (format " (set `join-alias` %s on this source key)" (pr-str source-field-join-alias))
+ "")
+ (pr-str (nth (field-ref-for-error metadata-provider [:field {} (:id field)]) 2)))))
+ (format "Use a `source-field` that is a foreign key to %s, or join %s explicitly and reference the field with its `join-alias`."
+ table-name table-name)))
+ {:agent-error? true
+ :status-code 400
+ :error :wrong-source-field})))
+
+(defn- source-field-contexts
+ [query]
+ (let [contexts (volatile! [])]
+ (lib.walk/walk-clauses
+ (qp/drop-fields-in-summaries query)
+ (fn [query path-type path clause]
+ (when (and (lib.util/clause? clause)
+ (= :field (first clause))
+ (:source-field (second clause))
+ (nil? (:join-alias (second clause))))
+ (let [stage-path (if (= path-type :lib.walk/join) (lib.walk/join-parent-stage-path path) path)]
+ (vswap! contexts conj
+ (lib.walk/apply-f-for-stage-at-path
+ (fn [query stage-number]
+ {:query query :stage-number stage-number :field-ref clause})
+ query stage-path))))
+ nil))
+ @contexts))
+
+(defn- wrong-source-field?
+ [metadata-provider [_ {:keys [source-field]} target]]
+ (when (and (pos-int? source-field) (pos-int? target))
+ (let [field (lib.metadata/field metadata-provider target)
+ key-field (lib.metadata/field metadata-provider source-field)
+ fk-table-id (foreign-key-target-table-id metadata-provider key-field)]
+ (and (:table-id field)
+ key-field
+ (if (:fk-target-field-id key-field)
+ (and fk-table-id (not= (:table-id field) fk-table-id))
+ true)))))
+
+(defn- assert-source-fields-reach-their-tables!
+ "Reject implicit field references whose source field is not a foreign key to the field's table."
+ [pmbql-query]
+ (doseq [{:keys [query stage-number field-ref]} (source-field-contexts pmbql-query)
+ :let [mp (lib.metadata/->metadata-provider query)]
+ :when (wrong-source-field? mp field-ref)]
+ (let [[_ {:keys [source-field]} target] field-ref
+ field-ids (set [source-field target])
+ fields (->> (lib.metadata/bulk-metadata mp :metadata/column field-ids)
+ entity-details/permission-filter-columns
+ (u/index-by :id))
+ _visible (when-not (every? fields field-ids)
+ (throw (ex-info "The implicit field reference cannot be resolved with the fields available to you. Use a visible foreign key or an explicit join."
+ {:agent-error? true :status-code 400 :error :wrong-source-field})))
+ table-id (:table-id (get fields target))
+ fk-field (get fields source-field)
+ fk-table-id (foreign-key-target-table-id mp fk-field)
+ field-ref (field-ref-for-error mp field-ref)]
+ (throw (wrong-source-field-error mp field-ref table-id fk-table-id (lib/visible-columns query stage-number))))))
+
;;; ---------------------------------------- Result columns ----------------------------------------
(defn- result-columns-for-query
@@ -858,11 +983,12 @@
6. Resolve portable FKs to numeric IDs and normalize through `lib.schema/query` against the
metadata-provider.
6.5. Backstop-gate the resolved query: mirror the FE `canRun` schema validation
- ([[query-not-runnable-explanation]]), run the FE expression editor's own
+ ([[query-not-runnable-explanation]]), check that implicit field references use a foreign key to their table
+ ([[assert-source-fields-reach-their-tables!]]), run the FE expression editor's own
diagnostics over every custom column / aggregation / filter
([[repr.repair/assert-editor-accepts-expressions!]]), then reject a temporal bucket
sitting on a column that is not temporal
- ([[repr.repair/assert-temporal-buckets-on-temporal-columns!]]). Any of the three
+ ([[repr.repair/assert-temporal-buckets-on-temporal-columns!]]). Any of these
failing is a retryable `:agent-error?` - success on a query the editor rejects is
BOT-1442.
7. Export that final numeric MBQL 5 back to the portable form for the LLM-facing
@@ -915,6 +1041,7 @@
{:agent-error? true
:error :query-not-runnable
:status-code 400})))
+ _fks-ok (assert-source-fields-reach-their-tables! pmbql-query)
;; The FE expression editor rejects some shapes the (type-check-suppressed) canRun
;; gate above accepts - `offset` in a filter, window functions nested in
;; aggregations, cyclic expression refs, type-incompatible arguments. Must run
diff --git a/src/metabase/metabot/tools/recoverable/pipeline.clj b/src/metabase/metabot/tools/recoverable/pipeline.clj
index 86cef63ad58..0afc96c1228 100644
--- a/src/metabase/metabot/tools/recoverable/pipeline.clj
+++ b/src/metabase/metabot/tools/recoverable/pipeline.clj
@@ -216,6 +216,10 @@
"No foreign key connects the two tables, so they cannot be joined implicitly."
[metric-dimensions-step])
+(defpipeline-error wrong-source-field!
+ "A field reference's `source-field` is not a foreign key to the table its column is on."
+ [list-fks-step])
+
;;; Saved entities
(defpipeline-error unknown-card!
diff --git a/src/metabase/query_processor/core.clj b/src/metabase/query_processor/core.clj
index 3908a81eb57..23f5e915c5e 100644
--- a/src/metabase/query_processor/core.clj
+++ b/src/metabase/query_processor/core.clj
@@ -14,6 +14,7 @@
[metabase.query-processor.metadata :as qp.metadata]
[metabase.query-processor.middleware.cache.impl :as qp.cache.impl]
[metabase.query-processor.middleware.constraints :as qp.constraints]
+ [metabase.query-processor.middleware.drop-fields-in-summaries :as qp.drop-fields]
[metabase.query-processor.middleware.limit :as qp.limit]
[metabase.query-processor.middleware.process-userland-query :as qp.process-userland-query]
[metabase.query-processor.pivot :as qp.pivot]
@@ -52,6 +53,9 @@
;; Constraints — metabase.query-processor.middleware.constraints
[qp.constraints
default-query-constraints]
+ ;; Summary fields – metabase.query-processor.middleware.drop-fields-in-summaries
+ [qp.drop-fields
+ drop-fields-in-summaries]
;; Limit — metabase.query-processor.middleware.limit
[qp.limit
disable-max-results]
diff --git a/test/metabase/metabot/tools_test.clj b/test/metabase/metabot/tools_test.clj
index aff65e2f664..04af9263c50 100644
--- a/test/metabase/metabot/tools_test.clj
+++ b/test/metabase/metabot/tools_test.clj
@@ -1,10 +1,12 @@
(ns metabase.metabot.tools-test
(:require
+ [clojure.edn :as edn]
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.api-scope.core :as api-scope]
[metabase.channel.settings :as channel.settings]
[metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
[metabase.metabot.agent.profiles :as profiles]
[metabase.metabot.scope :as scope]
@@ -16,6 +18,9 @@
[metabase.metabot.tools.legacy :as tools.legacy]
[metabase.metabot.tools.shared :as shared]
[metabase.notification.models :as models.notification]
+ [metabase.permissions.core :as perms]
+ [metabase.permissions.models.permissions-group :as perms-group]
+ [metabase.query-processor :as qp]
[metabase.search.entity-retrieval.core :as entity-retrieval]
[metabase.test :as mt]
[metabase.util.json :as json]
@@ -258,6 +263,532 @@
(is (= {:chart? true, :shown? true, :query-id "q-1", :run-it? false}
(construct! #{"construct_notebook_query"} {:show_chart false}))))))
+(defn- source-field-query-without-validation! [construct!]
+ (let [resolved (atom nil)]
+ (mt/with-dynamic-fn-redefs [construct/assert-source-fields-reach-their-tables! (fn [query] (reset! resolved query))]
+ (try (construct!) (catch clojure.lang.ExceptionInfo _)))
+ (is (map? @resolved))
+ @resolved))
+
+(defn- source-field-qp-control! [construct!]
+ (qp/process-query (source-field-query-without-validation! construct!)))
+
+(defn- assert-source-field-qp-failure! [construct!]
+ (let [query (source-field-query-without-validation! construct!)
+ result (try (qp/process-query query) (catch Exception e e))]
+ (is (or (instance? Exception result) (= :failed (:status result)))
+ (if (instance? Exception result) (ex-message result)
+ (str "QP accepted the rejected query: " (select-keys result [:status :row_count]))))))
+
+(deftest construct-notebook-query-tool-wrong-source-field-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table column] [db "PUBLIC" table column])
+ venue-id (fk "CHECKINS" "VENUE_ID")
+ venues-join {:alias "Venues"
+ :strategy "left-join"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table [db "PUBLIC" "VENUES"]}]
+ :conditions [["=" {}
+ ["field" {} venue-id]
+ ["field" {:join-alias "Venues"} (fk "VENUES" "ID")]]]}
+ args (fn [stage]
+ {:query {:lib/type "mbql/query"
+ :stages [(merge {:lib/type "mbql.stage/mbql"
+ :source-table [db "PUBLIC" "CHECKINS"]
+ :aggregation [["count" {}]]}
+ stage)]}
+ :title "Check-ins"
+ :description "Check-ins by category."})
+ refusal (fn [opts field]
+ (assert-source-field-qp-failure!
+ #(construct/execute-representations-query (:query (args {:breakout [["field" opts field]]}))))
+ (test-util/tool-failure agent-tools/construct-notebook-query-tool
+ (args {:breakout [["field" opts field]]})
+ #{"read_resource"}))
+ refused (fn [& sentences]
+ {:class :recoverable
+ :code :metabase.metabot.tools.recoverable.pipeline/wrong-source-field
+ :text (str (apply str sentences) "\n"
+ "Call `read_resource` with `metabase://table/<numeric id>/fields` for the source "
+ "table to list the available foreign-key columns.")})]
+ (mt/with-current-user (mt/user->id :rasta)
+ (testing "a field reached through a foreign key to another table is refused, naming the path that reaches it"
+ (doseq [opts [{:source-field venue-id} {:source-field venue-id :join-alias nil}]]
+ (testing (pr-str opts)
+ (is (= (refused "Field " (pr-str (fk "CATEGORIES" "NAME")) " is on table \"CATEGORIES\", but its "
+ "`source-field` " (pr-str venue-id) " is a foreign key to table \"VENUES\", so the "
+ "field cannot be reached through it. \"VENUES\" has a foreign key to \"CATEGORIES\": "
+ "join \"VENUES\" explicitly on " (pr-str venue-id) ", then reference the field with "
+ "`source-field` " (pr-str (fk "VENUES" "CATEGORY_ID")) " and `source-field-join-alias` "
+ "set to the new join's alias.")
+ (refusal opts (fk "CATEGORIES" "NAME")))))))
+ (testing "a source-field that is not a foreign key is refused"
+ (is (= (refused "Field " (pr-str (fk "VENUES" "NAME")) " is on table \"VENUES\", but its `source-field` "
+ (pr-str (fk "CHECKINS" "DATE")) " is not a foreign key to that table, so the field cannot "
+ "be reached through it. Use `source-field` " (pr-str venue-id) " to reach \"VENUES\" directly.")
+ (refusal {:source-field (fk "CHECKINS" "DATE")} (fk "VENUES" "NAME")))))
+ (testing (str "a one-hop reference builds and runs, as do the path the refusal names and a reference through "
+ "an explicit join, whose `source-field` the query processor ignores")
+ (doseq [stage [{:breakout [["field" {:source-field venue-id} (fk "VENUES" "NAME")]]}
+ {:joins [venues-join]
+ :breakout [["field" {:source-field (fk "VENUES" "CATEGORY_ID") :source-field-join-alias "Venues"}
+ (fk "CATEGORIES" "NAME")]]}
+ {:joins [venues-join]
+ :breakout [["field" {:join-alias "Venues" :source-field (fk "CHECKINS" "DATE")}
+ (fk "VENUES" "NAME")]]}]]
+ (testing (pr-str stage)
+ (is (seq (mt/rows (qp/process-query
+ (get-in (test-util/call-tool agent-tools/construct-notebook-query-tool (args stage))
+ [:structured-output :query])))))))))
+ (testing "the table the foreign key points at is not named when the user cannot query it"
+ (mt/with-no-data-perms-for-all-users!
+ (perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)
+ (doseq [table [:checkins :categories]]
+ (perms/set-table-permission! (perms-group/all-users) (mt/id table) :perms/create-queries :query-builder))
+ (mt/with-current-user (mt/user->id :rasta)
+ (is (= (refused "Field " (pr-str (fk "CATEGORIES" "NAME")) " is on table \"CATEGORIES\", but its "
+ "`source-field` " (pr-str venue-id) " is not a foreign key to that table, so the field "
+ "cannot be reached through it. Use a `source-field` that is a foreign key to "
+ "\"CATEGORIES\", or join \"CATEGORIES\" explicitly and reference the field with its "
+ "`join-alias`.")
+ (refusal {:source-field venue-id} (fk "CATEGORIES" "NAME")))))))))
+
+(defn- join-on-field [db alias table field]
+ {:alias alias
+ :strategy "left-join"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table [db "PUBLIC" table]}]
+ :conditions [["=" {}
+ field
+ ["field" {:join-alias alias} [db "PUBLIC" table "ID"]]]]})
+
+(defn- checkins-venues-join [db]
+ (join-on-field db "Venues" "VENUES" ["field" {} [db "PUBLIC" "CHECKINS" "VENUE_ID"]]))
+
+(defn- construct-checkins-query [db stages]
+ (construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages (-> (mapv #(merge {:lib/type "mbql.stage/mbql"} %) stages)
+ (assoc-in [0 :source-table] [db "PUBLIC" "CHECKINS"]))}))
+
+(deftest construct-notebook-query-summary-fields-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table column] [db "PUBLIC" table column])
+ wrong ["field" {:source-field (fk "CHECKINS" "VENUE_ID")} (fk "CATEGORIES" "NAME")]
+ non-fk ["field" {:source-field (fk "CHECKINS" "DATE")} (fk "VENUES" "NAME")]
+ breakout ["field" {} (fk "CHECKINS" "VENUE_ID")]]
+ (mt/with-current-user (mt/user->id :rasta)
+ (doseq [stage [{:aggregation [["count" {}]] :fields [wrong]}
+ {:aggregation [["count" {}]] :fields [non-fk]}
+ {:breakout [breakout] :fields [wrong]}]]
+ (testing (pr-str stage)
+ (let [build! #(construct-checkins-query db [stage])
+ control (source-field-qp-control! build!)
+ actual (qp/process-query (get-in (build!) [:structured-output :query]))]
+ (is (= :completed (:status control) (:status actual)))
+ (is (seq (mt/rows actual)))
+ (is (= (set (mt/rows control)) (set (mt/rows actual))))
+ (when (:aggregation stage)
+ (is (= [[1000]] (mt/rows actual))))))))))
+
+(deftest construct-notebook-query-supported-source-fields-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table column] [db "PUBLIC" table column])
+ venue-id (fk "CHECKINS" "VENUE_ID")
+ category-id (fk "VENUES" "CATEGORY_ID")
+ category (fk "CATEGORIES" "NAME")
+ breakout (fn [opts field] {:aggregation [["count" {}]] :breakout [["field" opts field]]})
+ venues-join (checkins-venues-join db)]
+ (mt/with-current-user (mt/user->id :rasta)
+ (testing "available keys work in later stages and through an explicit join"
+ (doseq [stages [[(assoc (breakout {:source-field category-id :source-field-join-alias "Other"} category)
+ :joins [venues-join])]
+ [(assoc (breakout {:source-field category-id :source-field-name "CATEGORY_ID"
+ :source-field-join-alias "Other"} category) :joins [venues-join])]
+ [{:fields [["field" {} venue-id]]}
+ (breakout {:source-field venue-id} (fk "VENUES" "NAME"))]
+ [{:fields [["field" {} (fk "CHECKINS" "ID")]]}
+ (breakout {:source-field venue-id} (fk "VENUES" "NAME"))]
+ [(assoc (breakout {:source-field venue-id} (fk "VENUES" "NAME"))
+ :joins [venues-join])]
+ [(assoc (breakout {:source-field category-id :source-field-join-alias "Venues"} category)
+ :joins [venues-join])]
+ [(assoc (breakout {:source-field category-id} category) :joins [venues-join])]
+ [(breakout {:source-field venue-id} (fk "VENUES" "NAME"))
+ {:fields [["field" {:source-field venue-id} (fk "VENUES" "NAME")]]}]]]
+ (testing (pr-str stages)
+ (let [rows (mt/rows (qp/process-query (get-in (construct-checkins-query db stages) [:structured-output :query])))]
+ (is (seq rows))
+ (when (= "Other" (get-in stages [0 :breakout 0 1 :source-field-join-alias]))
+ (let [corrected (assoc-in stages [0 :breakout 0 1 :source-field-join-alias] "Venues")]
+ (is (= (set rows) (set (mt/rows (qp/process-query
+ (get-in (construct-checkins-query db corrected) [:structured-output :query])))))))))))))))
+
+(deftest construct-notebook-query-wrong-source-field-context-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ correct ["field" {:source-field [db "PUBLIC" "VENUES" "CATEGORY_ID"]
+ :source-field-join-alias "Venues"}
+ [db "PUBLIC" "CATEGORIES" "NAME"]]
+ wrong-key [db "PUBLIC" "CHECKINS" "VENUE_ID"]
+ wrong ["field" {:source-field wrong-key} [db "PUBLIC" "CATEGORIES" "NAME"]]
+ build! #(construct-checkins-query db [{:joins [(checkins-venues-join db)]
+ :aggregation [["count" {}]] :breakout %}])]
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [control (mt/rows (source-field-qp-control! #(build! [correct])))]
+ (is (seq control))
+ (is (= (set control) (set (mt/rows (qp/process-query (get-in (build! [correct]) [:structured-output :query])))))))
+ (doseq [breakouts [[correct wrong] [wrong correct]]]
+ (assert-source-field-qp-failure! #(build! breakouts))
+ (let [error (try (build! breakouts) (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :wrong-source-field (:error (ex-data error))))
+ (is (str/includes? (or (ex-message error) "") (str "but its `source-field` " (pr-str wrong-key))))
+ (is (not (str/includes? (or (ex-message error) "")
+ (str "but its `source-field` " (pr-str (get-in correct [1 :source-field])))))))))))
+
+(deftest construct-notebook-query-direct-source-field-hint-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table column] [db "PUBLIC" table column])
+ user-id (fk "CHECKINS" "USER_ID")
+ query (fn [source-field]
+ (construct-checkins-query
+ db [{:aggregation [["count" {}]]
+ :breakout [["field" {:source-field source-field} (fk "USERS" "NAME")]]}]))]
+ (mt/with-current-user (mt/user->id :rasta)
+ (assert-source-field-qp-failure! #(query (fk "CHECKINS" "VENUE_ID")))
+ (let [e (try
+ (query (fk "CHECKINS" "VENUE_ID"))
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :wrong-source-field (:error (ex-data e))))
+ (is (str/includes? (or (ex-message e) "") (str "Use `source-field` " (pr-str user-id)))))
+ (testing "the direct key suggested by the hint builds and runs"
+ (is (seq (mt/rows (qp/process-query (get-in (query user-id) [:structured-output :query])))))))))
+
+(deftest construct-notebook-query-hidden-parent-key-hint-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))]
+ (mt/with-temp [:model/Field parent {:table_id (mt/id :venues)
+ :name "HIDDEN_PAYLOAD"
+ :base_type :type/JSON
+ :database_type "JSON"
+ :visibility_type "sensitive"
+ :active true}]
+ (mt/with-temp-vals-in-db :model/Field (mt/id :venues :category_id) {:parent_id (:id parent)}
+ (mt/with-current-user (mt/user->id :rasta)
+ (doseq [[table key] [["CHECKINS" "VENUE_ID"] ["VENUES" "ID"]]]
+ (testing table
+ (let [build! #(construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-table [db "PUBLIC" table]
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field [db "PUBLIC" table key]}
+ [db "PUBLIC" "CATEGORIES" "NAME"]]]}]})
+ error (try (build!) (catch clojure.lang.ExceptionInfo e e))]
+ (assert-source-field-qp-failure! build!)
+ (is (= :wrong-source-field (:error (ex-data error))))
+ (is (not (re-find #"HIDDEN_PAYLOAD|CATEGORY_ID" (str (ex-message error) (ex-data error)))))))))))))
+
+(defn- follow-source-field-hint! [db stage message explicit-joins]
+ (let [target (get-in stage [:breakout 0 2])]
+ (cond
+ (re-find #"Use `source-field` (\[[^\]]+\]) to reach" message)
+ (let [[_ key] (re-find #"Use `source-field` (\[[^\]]+\]) to reach" message)]
+ (assoc stage :breakout [["field" {:source-field (edn/read-string key)} target]]))
+
+ (re-find #"join \"([^\"]+)\" explicitly on (\[[^\]]+\])" message)
+ (let [[_ table key] (re-find #"join \"([^\"]+)\" explicitly on (\[[^\]]+\])" message)
+ [_ key-alias] (re-find #"set `join-alias` \"([^\"]+)\"" message)
+ [_ next-key] (re-find #"reference the field with `source-field` (\[[^\]]+\])" message)
+ join (join-on-field db "Hint" table
+ ["field" (cond-> {} key-alias (assoc :join-alias key-alias)) (edn/read-string key)])]
+ (-> stage
+ (update :joins (fnil conj []) join)
+ (assoc :breakout [["field" (if next-key
+ {:source-field (edn/read-string next-key) :source-field-join-alias "Hint"}
+ {})
+ target]])))
+
+ :else
+ (do
+ (is (str/includes? message "join \"CATEGORIES\" explicitly"))
+ (assoc stage :joins explicit-joins :breakout [["field" {:join-alias "Categories"} target]])))))
+
+(defn- assert-category-hint! [db source-kind source prefix-field stage explicit-joins expected-hint]
+ (let [query (fn [stage]
+ {:lib/type "mbql/query"
+ :stages (if (= source-kind :later-stage)
+ [(merge {:lib/type "mbql.stage/mbql" :aggregation [["count" {}]]
+ :breakout [["field" {} prefix-field]]} source)
+ (assoc stage :lib/type "mbql.stage/mbql")]
+ [(merge {:lib/type "mbql.stage/mbql"} source stage)])})
+ build! #(construct/execute-representations-query (query %))
+ expected-stage (assoc stage :joins explicit-joins
+ :breakout [["field" {:join-alias "Categories"} (get-in stage [:breakout 0 2])]])
+ expected (qp/process-query (get-in (build! expected-stage) [:structured-output :query]))
+ error (try (build! stage) (catch clojure.lang.ExceptionInfo e e))
+ message (or (ex-message error) "")]
+ (is (= :completed (:status expected)))
+ (is (= 28 (count (mt/rows expected))))
+ (is (= :wrong-source-field (:error (ex-data error))))
+ (is (str/includes? message expected-hint))
+ (assert-source-field-qp-failure! #(build! stage))
+ (let [corrected (follow-source-field-hint! db stage message explicit-joins)
+ result (qp/process-query (get-in (build! corrected) [:structured-output :query]))]
+ (is (= :completed (:status result)))
+ (is (= 28 (count (mt/rows result))))
+ (is (= (set (mt/rows expected)) (set (mt/rows result)))))))
+
+(deftest construct-notebook-query-category-hints-test
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [mp (mt/metadata-provider)
+ db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table field] [db "PUBLIC" table field])
+ venues (checkins-venues-join db)
+ checkins (assoc-in (join-on-field db "Checkins" "CHECKINS" ["field" {} (fk "USERS" "ID")])
+ [:conditions 0 3 2] (fk "CHECKINS" "USER_ID"))
+ aliased-venues (assoc-in venues [:conditions 0 2 1 :join-alias] "Checkins")
+ categories #(join-on-field db "Categories" "CATEGORIES" ["field" % (fk "VENUES" "CATEGORY_ID")])
+ generic-hint "Use a `source-field` that is a foreign key to \"CATEGORIES\", or join \"CATEGORIES\" explicitly and reference the field with its `join-alias`."
+ intermediate-hint (str "then reference the field with `source-field` " (pr-str (fk "VENUES" "CATEGORY_ID"))
+ " and `source-field-join-alias` set to the new join's alias.")]
+ (doseq [{:keys [label table prefix-key key key-alias joins explicit-joins hint native-hint]}
+ [{:label "non-FK venue key" :table :venues :prefix-key :category_id :key (fk "VENUES" "ID")
+ :hint (str "Use `source-field` " (pr-str (fk "VENUES" "CATEGORY_ID")) " to reach \"CATEGORIES\" directly.")
+ :native-hint generic-hint
+ :explicit-joins [(categories {})]}
+ {:label "intermediate join" :table :checkins :prefix-key :venue_id :key (fk "CHECKINS" "VENUE_ID")
+ :hint intermediate-hint
+ :explicit-joins [venues (categories {:join-alias "Venues"})]}
+ {:label "explicit target join" :table :checkins :prefix-key :venue_id :key (fk "CHECKINS" "DATE")
+ :hint generic-hint
+ :explicit-joins [venues (categories {:join-alias "Venues"})]}
+ {:label "aliased intermediate join" :table :users :prefix-key :id :key (fk "CHECKINS" "VENUE_ID")
+ :hint (str "(set `join-alias` \"Checkins\" on this source key), " intermediate-hint)
+ :key-alias "Checkins" :joins [checkins]
+ :explicit-joins [checkins aliased-venues (categories {:join-alias "Venues"})]}]
+ source-kind [:table :question :model :later-stage :native-question :native-model]]
+ (testing (str label " from " source-kind)
+ (let [table-meta (lib.metadata/table mp (mt/id table))
+ table-name (:name table-meta)
+ source {:source-table [db "PUBLIC" table-name]}
+ stage (cond-> {:aggregation [["count" {}]]
+ :breakout [["field" (cond-> {:source-field key}
+ key-alias (assoc :source-field-join-alias key-alias))
+ (fk "CATEGORIES" "NAME")]]}
+ (seq joins) (assoc :joins joins))
+ check! #(assert-category-hint! db source-kind % (fk table-name (:name (lib.metadata/field mp (mt/id table prefix-key))))
+ stage explicit-joins
+ (if (#{:native-question :native-model} source-kind) (or native-hint hint) hint))]
+ (if (#{:table :later-stage} source-kind)
+ (check! source)
+ (let [native? (#{:native-question :native-model} source-kind)
+ card-query (if native?
+ (lib/native-query mp (str "SELECT * FROM " table-name))
+ (lib/with-fields (lib/query mp table-meta)
+ (mapv #(lib.metadata/field mp %) (distinct [(mt/id table :id) (mt/id table prefix-key)]))))
+ result (qp/process-query card-query)]
+ (is (= :completed (:status result)))
+ (mt/with-temp [:model/Card {eid :entity_id}
+ {:name (str label " " source-kind) :type (if (#{:model :native-model} source-kind) :model :question)
+ :dataset_query card-query :result_metadata (get-in result [:data :cols])}]
+ (check! {:source-card eid}))))))))))
+
+(deftest construct-notebook-query-model-source-field-hint-test
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [mp (mt/metadata-provider)
+ db (t2/select-one-fn :name :model/Database (mt/id))
+ key-id (mt/id :checkins :venue_id)
+ original-target (t2/select-one-fn :fk_target_field_id :model/Field key-id)
+ key-path [db "PUBLIC" "CHECKINS" "VENUE_ID"]
+ target-path [db "PUBLIC" "VENUES" "NAME"]
+ query (lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id :checkins)))
+ [(lib.metadata/field mp key-id)])
+ result (qp/process-query query)]
+ (is (= :completed (:status result)))
+ (mt/with-temp [:model/Card {eid :entity_id card-id :id}
+ {:name "Stored venue key" :type :model :dataset_query query :result_metadata (get-in result [:data :cols])}]
+ (let [metadata-before (t2/select-one-fn :result_metadata :model/Card card-id)
+ build! #(construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-card eid
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field key-path} target-path]]}]})]
+ (try
+ (let [updated (mt/user-http-request :crowberto :put 200 (str "field/" key-id)
+ {:fk_target_field_id (mt/id :users :id)})
+ error (try (build!) (catch clojure.lang.ExceptionInfo e e))]
+ (is (= (mt/id :users :id) (:id (:target updated))))
+ (is (= metadata-before (t2/select-one-fn :result_metadata :model/Card card-id)))
+ (is (= :wrong-source-field (:error (ex-data error))))
+ (assert-source-field-qp-failure! build!)
+ (is (not (str/includes? (or (ex-message error) "")
+ (str "Use `source-field` " (pr-str key-path) " to reach \"VENUES\" directly."))))
+ (is (str/includes? (or (ex-message error) "") "join \"VENUES\" explicitly"))
+ (let [corrected (construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-card eid
+ :joins [(checkins-venues-join db)] :aggregation [["count" {}]]
+ :breakout [["field" {:join-alias "Venues"} target-path]]}]})
+ rows (qp/process-query (get-in corrected [:structured-output :query]))]
+ (is (= :completed (:status rows)))
+ (is (= 100 (count (mt/rows rows))))))
+ (finally
+ (mt/user-http-request :crowberto :put 200 (str "field/" key-id)
+ {:fk_target_field_id original-target}))))))))
+
+(deftest construct-notebook-query-source-field-join-alias-hint-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table column] [db "PUBLIC" table column])
+ checkins-join {:alias "Checkins"
+ :strategy "left-join"
+ :stages [{:lib/type "mbql.stage/mbql" :source-table [db "PUBLIC" "CHECKINS"]}]
+ :conditions [["=" {}
+ ["field" {} (fk "USERS" "ID")]
+ ["field" {:join-alias "Checkins"} (fk "CHECKINS" "USER_ID")]]]}
+ query (fn [joins opts]
+ (construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-table [db "PUBLIC" "USERS"]
+ :joins joins
+ :aggregation [["count" {}]]
+ :breakout [["field" opts (fk "CATEGORIES" "NAME")]]}]}))]
+ (mt/with-current-user (mt/user->id :rasta)
+ (assert-source-field-qp-failure!
+ #(query [checkins-join] {:source-field (fk "CHECKINS" "VENUE_ID") :source-field-join-alias "Checkins"}))
+ (let [e (try
+ (query [checkins-join] {:source-field (fk "CHECKINS" "VENUE_ID")
+ :source-field-join-alias "Checkins"})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :wrong-source-field (:error (ex-data e))))
+ (is (str/includes? (or (ex-message e) "") "(set `join-alias` \"Checkins\" on this source key)")))
+ (testing "the suggested join runs with an explicit or inferred alias for its key"
+ (let [rows (mapv (fn [join-alias]
+ (let [venues-join (cond-> (checkins-venues-join db)
+ join-alias (assoc-in [:conditions 0 2 1 :join-alias] join-alias))
+ result (query [checkins-join venues-join]
+ {:source-field (fk "VENUES" "CATEGORY_ID")
+ :source-field-join-alias "Venues"})]
+ (mt/rows (qp/process-query (get-in result [:structured-output :query])))))
+ [nil "Checkins"])]
+ (is (= 28 (count (first rows))))
+ (is (= (set (first rows)) (set (second rows)))))))))
+
+(deftest construct-notebook-query-dependent-implicit-joins-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ fk (fn [table field] [db "PUBLIC" table field])
+ build! #(construct-checkins-query
+ db [{:aggregation [["count" {}]]
+ :breakout [["field" {:source-field (fk "CHECKINS" "VENUE_ID")} (fk "VENUES" "NAME")]
+ ["field" {:source-field (fk "VENUES" "CATEGORY_ID")
+ :source-field-join-alias "VENUES__via__VENUE_ID"}
+ (fk "CATEGORIES" "NAME")]]}])]
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [control (mt/rows (source-field-qp-control! build!))]
+ (is (seq control))
+ (is (= (set control) (set (mt/rows (qp/process-query (get-in (build!) [:structured-output :query]))))))))))
+
+(deftest construct-notebook-query-native-source-card-implicit-join-test
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [mp (mt/metadata-provider)
+ native (lib/native-query mp "SELECT VENUE_ID FROM CHECKINS")
+ result (qp/process-query native)
+ columns (get-in result [:data :cols])
+ db (t2/select-one-fn :name :model/Database (mt/id))]
+ (is (seq columns))
+ (is (every? (comp nil? :id) columns))
+ (mt/with-temp [:model/Card {eid :entity_id} {:name "Native venue keys" :dataset_query native :result_metadata columns}]
+ (let [build! #(construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-card eid
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field [db "PUBLIC" "CHECKINS" "VENUE_ID"]}
+ [db "PUBLIC" "VENUES" "NAME"]]]}]})
+ control (mt/rows (source-field-qp-control! build!))]
+ (is (seq control))
+ (is (= (set control) (set (mt/rows (qp/process-query (get-in (build!) [:structured-output :query])))))))))))
+
+(deftest construct-notebook-query-joined-key-propagation-test
+ (let [db (t2/select-one-fn :name :model/Database (mt/id))
+ stages [{:fields [["field" {} [db "PUBLIC" "CHECKINS" "ID"]]]
+ :joins [(assoc (checkins-venues-join db) :fields "none")]}
+ {:aggregation [["count" {}]]
+ :breakout [["field" {:source-field [db "PUBLIC" "VENUES" "CATEGORY_ID"]
+ :source-field-name "Venues__CATEGORY_ID"}
+ [db "PUBLIC" "CATEGORIES" "NAME"]]]}]
+ run! (fn []
+ (mt/rows (qp/process-query
+ (get-in (construct-checkins-query db stages) [:structured-output :query]))))]
+ (mt/with-current-user (mt/user->id :rasta)
+ (is (seq (run!))))))
+
+(deftest construct-notebook-query-source-card-key-propagation-test
+ (let [mp (mt/metadata-provider)
+ db (t2/select-one-fn :name :model/Database (mt/id))
+ raw (lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id :checkins)))
+ [(lib.metadata/field mp (mt/id :checkins :id))])
+ run! (fn [card-eid]
+ (get-in (construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql"
+ :source-card card-eid
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field [db "PUBLIC" "CHECKINS" "VENUE_ID"]}
+ [db "PUBLIC" "VENUES" "NAME"]]]}]})
+ [:structured-output :query]))]
+ (mt/with-current-user (mt/user->id :rasta)
+ (mt/with-temp [:model/Card {card-eid :entity_id} {:name "Check-ins source" :dataset_query raw}]
+ (is (seq (mt/rows (qp/process-query (run! card-eid)))))))))
+
+(deftest construct-notebook-query-source-card-field-permissions-test
+ (let [mp (mt/metadata-provider)
+ db (t2/select-one-fn :name :model/Database (mt/id))
+ query (lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ [(lib.metadata/field mp (mt/id :venues :name))])]
+ (mt/with-temp [:model/Card {card-eid :entity_id} {:name "Venue names" :dataset_query query}]
+ (mt/with-no-data-perms-for-all-users!
+ (perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)
+ (perms/set-table-permission! (perms-group/all-users) (mt/id :checkins) :perms/create-queries :query-builder)
+ (mt/with-current-user (mt/user->id :rasta)
+ (let [build! #(construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-card card-eid
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field [db "PUBLIC" "CHECKINS" "DATE"]
+ :base-type "type/Text"} "NAME"]]}]})
+ control (mt/rows (source-field-qp-control! build!))]
+ (is (seq control))
+ (is (= (set control) (set (mt/rows (qp/process-query (get-in (build!) [:structured-output :query]))))))))))))
+
+(deftest construct-notebook-query-source-card-hint-permissions-test
+ (let [mp (mt/metadata-provider)
+ db (t2/select-one-fn :name :model/Database (mt/id))
+ category [db "PUBLIC" "VENUES" "CATEGORY_ID"]
+ target [db "PUBLIC" "VENUES" "NAME"]
+ query (lib/with-fields (lib/query mp (lib.metadata/table mp (mt/id :checkins)))
+ [(lib.metadata/field mp (mt/id :checkins :venue_id))])]
+ (mt/with-temp [:model/Card {card-eid :entity_id} {:name "Venue keys" :dataset_query query}]
+ (mt/with-no-data-perms-for-all-users!
+ (perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/view-data :unrestricted)
+ (perms/set-table-permission! (perms-group/all-users) (mt/id :venues) :perms/create-queries :query-builder)
+ (mt/with-current-user (mt/user->id :rasta)
+ (assert-source-field-qp-failure!
+ #(construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-card card-eid
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field category} target]]}]}))
+ (let [e (try
+ (construct/execute-representations-query
+ {:lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/mbql" :source-card card-eid
+ :aggregation [["count" {}]]
+ :breakout [["field" {:source-field category} target]]}]})
+ (catch clojure.lang.ExceptionInfo e e))]
+ (is (= :wrong-source-field (:error (ex-data e))))
+ (is (= (str "Field " (pr-str target) " is on table \"VENUES\", but its `source-field` "
+ (pr-str category) " is not a foreign key to that table, so the field cannot be reached through it. "
+ "Use a `source-field` that is a foreign key to \"VENUES\", or join \"VENUES\" explicitly "
+ "and reference the field with its `join-alias`.")
+ (ex-message e)))))))))
+
(defn- construct-tool-failure-for-thrown
"Run `construct_notebook_query` with `execute-representations-query` throwing `e`, and return the
failure the model would see."