Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Type and check by-name refs inside joins (#85736)

Chris Truter714a9e49117e
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.clj
index ebc767e9bb9..04821d8bd29 100644
--- a/src/metabase/agent_lib/representations/repair.clj
+++ b/src/metabase/agent_lib/representations/repair.clj
@@ -2667,9 +2667,9 @@
;;; Strategy:
;;; * Walk stages left-to-right.
;;; * Before processing stage `i` (i ≥ 1), build a mini-query consisting of stages[0..i-1]
-;;; and resolve it. Call `lib/returned-columns` on the result; index by `:name`.
-;;; * Walk stage[i] (skipping descent into `joins` subtrees - those have their own
-;;; resolution context). For every `["field" opts <string>]` clause:
+;;; and resolve it. Match refs to its `lib/returned-columns` by the names lib resolves.
+;;; * Walk stage[i], including its joins' `conditions` but not their [[hidden-join-keys]]: a
+;;; join's own `stages` have their own resolution context. For every `["field" opts <string>]` clause:
;;; - If `opts` already has `"base-type"`, leave it alone (idempotent).
;;; - If the name is unknown to the previous stage, leave it alone (the resolver will
;;; report the real error with a better message).
@@ -2686,6 +2686,35 @@
(let [joins (get stage "joins")]
(into #{} (keep #(when (map? %) (get % "alias"))) (when (sequential? joins) joins))))
+(defn- hidden-join-keys
+ "Return the keys of `join` that the cross-stage passes leave alone.
+ These are its own `stages`, and its `fields` when it has no alias."
+ [join]
+ ;; Refs in an alias-less join's `fields` can't be told from refs to the previous stage, and validation reports the
+ ;; missing alias.
+ (cond-> ["stages"]
+ (not (non-blank-string? (get join "alias"))) (conj "fields")))
+
+(defn- without-join-stages
+ "Return `stage` without its joins' [[hidden-join-keys]], keeping the stage's own clauses and the rest of each join."
+ [stage]
+ (cond-> stage
+ (sequential? (get stage "joins"))
+ (update "joins" (partial mapv #(if (map? %) (apply dissoc % (hidden-join-keys %)) %)))))
+
+(defn- postwalk-outside-join-stages
+ "Apply `f` with `walk/postwalk` to the parts of `stage` that [[without-join-stages]] keeps.
+ Each join's [[hidden-join-keys]] come back unchanged."
+ [f stage]
+ (let [joins (get stage "joins")]
+ (cond-> (walk/postwalk f (without-join-stages stage))
+ (sequential? joins)
+ (update "joins" #(mapv (fn [walked join]
+ (cond-> walked
+ (map? join) (merge (select-keys join (hidden-join-keys join)))))
+ %
+ joins)))))
+
(defn- string-cross-stage-field-clause?
"True if `v` is a string-named field ref to a previous-stage column of a stage whose joins are
aliased `join-aliases`. A ref through one of those aliases names a join's column instead."
@@ -2708,17 +2737,30 @@
(if (keyword? et) (subs (str et) 1) (str et))))))))
(defn- column-descriptors
- "Build the cross-stage match oracle from a `lib/returned-columns` sequence: a vector of
- `{:name <machine-name>, :display-name <display-name>, :id <field-id>|nil, :types {\"base-type\" …}|nil}`
- descriptors. `:types` is nil when the column exposes no usable `base-type`. Carrying the
- `:display-name` lets [[match-cross-stage-column]] recover a ref the LLM wrote with the UI label
- (e.g. `\"Max of Instance Has SC\"`) instead of the machine name (`max`)."
+ "Build the cross-stage match oracle from a `lib/returned-columns` sequence, as a vector of descriptors:
+
+ {:name <machine name>
+ :desired-alias <alias>
+ :deduplicated-name <name>
+ :display-name <display name>
+ :join-alias <alias of the join the column came through> | nil
+ :id <field id> | nil
+ :types {\"base-type\" …} | nil}
+
+ The desired alias and the deduplicated name are the names lib resolves a by-name ref against.
+ The display name lets [[match-cross-stage-column]] recover a ref the LLM wrote with the UI label, such as
+ `\"Max of Instance Has SC\"`, instead of the machine name `max`.
+ The join alias says which of the previous stage's joins the column came through.
+ The types are nil when the column has no usable `base-type`."
[cols]
(mapv (fn [col]
- {:name (:name col)
- :display-name (:display-name col)
- :id (:id col)
- :types (types-from-column col)})
+ {:name (:name col)
+ :desired-alias (:lib/desired-column-alias col)
+ :deduplicated-name (:lib/deduplicated-name col)
+ :display-name (:display-name col)
+ :join-alias (:lib/original-join-alias col)
+ :id (:id col)
+ :types (types-from-column col)})
cols))
(defn- resolved-columns
@@ -2768,34 +2810,50 @@
(-> s u/lower-case-en (str/replace #"[-\s]+" "_"))))
(defn- match-cross-stage-column
- "Find the column descriptor in `cols` (a vector of [[column-descriptors]]) that `col-name`
- refers to. Returns `[canonical-machine-name types]` or `nil` when no column matches.
+ "Find the column in `cols`, a vector of [[column-descriptors]], that `col-name` refers to.
+ Returns `[name types]`, where `name` is the spelling the ref should carry, or nil when no column matches.
Tries, in order:
- 1. an exact `:name` match;
- 2. the double-quote-stripped `:name` (only when stripping actually resolves);
+ 1. an exact match on a name lib resolves: the desired alias, then the deduplicated name, then `:name`.
+ The ref keeps its spelling, since a nested join's column is `P__NAME` to lib even when its `:name` is `NAME`;
+ 2. the same match with surrounding double quotes stripped, giving the stripped spelling;
3. a loose `:name` match that folds case and hyphens/whitespace to underscores, but **only
when exactly one** real column matches - so `count-where` resolves to `count_where`;
4. a loose `:display-name` match (same folding, same one-hit guard) - so a ref the LLM
wrote with the UI label (`\"Max of Instance Has SC\"`) recovers the machine name (`max`).
- Genuine ambiguity (more than one loose hit) is left unmatched at each layer. A column whose
- name legitimately needs quotes (vanishingly rare) is never clobbered."
- [cols col-name]
- (let [stripped (strip-surrounding-double-quotes col-name)
- exact (fn [n] (some #(when (= n (:name %)) %) cols))
- loose-on (fn [key-fn raw]
+ Genuine ambiguity (more than one loose hit) is left unmatched at each layer.
+ A column whose name legitimately needs quotes (vanishingly rare) is never clobbered.
+ With `join-alias`, each exact match tries `join-alias__name` before the name itself, keeping the ref's spelling.
+ Loose matches through `join-alias` use a spelling that resolves back to the same column."
+ ([cols col-name]
+ (match-cross-stage-column cols col-name nil))
+ ([cols col-name join-alias]
+ (let [stripped (strip-surrounding-double-quotes col-name)
+ find-on (fn [k n] (some #(when (= n (k %)) %) cols))
+ exact (fn [n] (some #(find-on % n) [:desired-alias :deduplicated-name :name]))
+ lib-match (fn [n] (or (when join-alias (exact (str join-alias "__" n)))
+ (exact n)))
+ loose-on (fn [key-fn raw]
(let [target (normalize-col-key raw)
hits (when target
(filter #(= target (normalize-col-key (key-fn %))) cols))]
(when (= 1 (count hits))
- (first hits))))
- match (or (exact col-name)
- (when (not= stripped col-name) (exact stripped))
- (loose-on :name stripped)
- (loose-on :display-name stripped))]
- (when match
- [(:name match) (:types match)])))
+ (first hits))))]
+ (if-let [[spelling match] (or (some->> (lib-match col-name) (vector col-name))
+ (when (not= stripped col-name)
+ (some->> (lib-match stripped) (vector stripped))))]
+ [spelling (:types match)]
+ (when-let [match (or (loose-on :name stripped)
+ (loose-on :display-name stripped))]
+ (let [desired-alias (:desired-alias match)
+ prefix (when join-alias (str join-alias "__"))
+ spelling (if (and prefix desired-alias (str/starts-with? desired-alias prefix))
+ (subs desired-alias (count prefix))
+ (:name match))]
+ ;; A deduplicated name can denote another column after lib adds the join prefix.
+ (when (or (nil? join-alias) (= match (lib-match spelling)))
+ [spelling (:types match)])))))))
(defn- maybe-fill-cross-stage-types
"Given a cross-stage field clause and a name→types map, return the clause with its column name
@@ -2803,40 +2861,49 @@
column) and `base-type` / `effective-type` filled in (when missing and known).
Idempotent: once the name is canonical it matches exactly on the next pass (no rename), and
- once `base-type` is present it is left alone."
- [clause cols]
- (let [opts (nth clause 1)
- col-name (nth clause 2)]
- (if-let [[canonical-name types] (match-cross-stage-column cols col-name)]
- (cond-> clause
- ;; Canonicalise the name when quote-stripping / display-name recovery was needed to
- ;; match. A bare `base-type` stamp without this would leave the ref pointing at a
- ;; non-existent column.
- (not= canonical-name col-name)
- (assoc 2 canonical-name)
- ;; Stamp inferred types only when the LLM didn't author a `base-type` already and the
- ;; matched column actually exposes one.
- (and types (not (contains? opts "base-type")))
- (assoc 1 (merge opts types)))
- clause)))
+ once `base-type` is present it is left alone.
+ The optional `join-alias` enables previous-stage join lookup in [[match-cross-stage-column]]."
+ ([clause cols]
+ (maybe-fill-cross-stage-types clause cols nil))
+ ([clause cols join-alias]
+ (let [opts (nth clause 1)
+ col-name (nth clause 2)]
+ (if-let [[canonical-name types] (match-cross-stage-column cols col-name join-alias)]
+ (cond-> clause
+ ;; Canonicalise the name when quote-stripping / display-name recovery was needed to
+ ;; match. A bare `base-type` stamp without this would leave the ref pointing at a
+ ;; non-existent column.
+ (not= canonical-name col-name)
+ (assoc 2 canonical-name)
+ ;; Stamp inferred types only when the LLM didn't author a `base-type` already and the
+ ;; matched column actually exposes one.
+ (and types (not (contains? opts "base-type")))
+ (assoc 1 (merge opts types)))
+ clause))))
+
+(defn- cols-through-alias
+ "Return the columns in `cols` that `field-ref` can name: those that came through its join alias, or all of them
+ when it has none.
+ Returns nil when no column came through its join alias."
+ [cols field-ref]
+ (if-let [alias (get (nth field-ref 1) "join-alias")]
+ (not-empty (filterv #(= alias (:join-alias %)) cols))
+ cols))
(defn- infer-cross-stage-field-types-in-stage
- "Walk one stage and stamp inferred types into every string-named field reference that lacks
- `\"base-type\"`. Skips descent into `\"joins\"` subtrees - join stages have their own
- resolution context (the join's own `stages`) and shouldn't reach into their parent stage's
- previous-stage columns."
+ "Stamp types from `cols` on every by-name field ref in `stage` that lacks `\"base-type\"`, including the
+ refs in its joins' `conditions`.
+ A ref through a join alias matches only the columns that came through it, and is left for validation when none did.
+ Skips each join's [[hidden-join-keys]]; its own `stages` resolve on their own and can't reach this stage's columns."
[stage cols]
- (let [joins (get stage "joins")
- aliases (stage-join-aliases stage)
- stage' (cond-> stage (contains? stage "joins") (dissoc "joins"))
- walked (walk/postwalk
- (fn [node]
- (if (string-cross-stage-field-clause? aliases node)
- (maybe-fill-cross-stage-types node cols)
- node))
- stage')]
- (cond-> walked
- (contains? stage "joins") (assoc "joins" joins))))
+ (let [aliases (stage-join-aliases stage)]
+ (postwalk-outside-join-stages
+ (fn [node]
+ (if-let [ref-cols (when (string-cross-stage-field-clause? aliases node)
+ (cols-through-alias cols node))]
+ (maybe-fill-cross-stage-types node ref-cols (get-in node [1 "join-alias"]))
+ node))
+ stage)))
(defn- mini-resolved-columns-for-source-card
"Resolve a single-stage query consisting only of `{:source-card <entity-id>}` (plus the
@@ -2918,29 +2985,9 @@
(contains? (first stages) "source-table")
(not-any? #(contains? (first stages) %) ["aggregation" "breakout" "fields"]))))
-(defn- without-join-stages
- "`stage` with each join's own `stages` removed, leaving the clauses that may reference a join's
- columns: the stage's own and the joins' `conditions`."
- [stage]
- (cond-> stage
- (sequential? (get stage "joins"))
- (update "joins" (partial mapv #(cond-> % (map? %) (dissoc "stages"))))))
-
-(defn- postwalk-outside-join-stages
- "`walk/postwalk` `f` over the parts of `stage` that [[without-join-stages]] keeps."
- [f stage]
- (let [joins (get stage "joins")]
- (cond-> (walk/postwalk f (without-join-stages stage))
- (sequential? joins)
- (update "joins" #(mapv (fn [walked join]
- (cond-> walked
- (and (map? join) (contains? join "stages")) (assoc "stages" (get join "stages"))))
- %
- joins)))))
-
(defn- join-alias-refs
"`stage`'s join-alias string refs (`:named`) and concrete-column join-alias refs (`:path`), each
- grouped by alias, outside the joins' own `stages`."
+ grouped by alias, outside the joins' [[hidden-join-keys]]."
[stage]
(let [stage' (without-join-stages stage)
alias #(get (nth % 1) "join-alias")]
@@ -3084,24 +3131,23 @@
(not (contains? (nth node 1) "base-type"))))
(defn- stage-has-unstamped-cross-stage-ref?
- "Cheap structural pre-scan: does `stage` contain any [[unstamped-cross-stage-ref?]] outside
- its `joins` subtrees? Lets the assert pass skip the (resolving) confirmation step entirely
- on the happy path, where every ref already carries a stamped `base-type`."
+ "True if `stage` has an [[unstamped-cross-stage-ref?]] outside its joins' [[hidden-join-keys]].
+ Lets the assert pass skip the resolving confirmation step on the happy path, where every ref is already typed."
[stage]
- (let [aliases (stage-join-aliases stage)
- stage' (cond-> stage (contains? stage "joins") (dissoc "joins"))]
- (some? (match/match-one stage'
+ (let [aliases (stage-join-aliases stage)]
+ (some? (match/match-one (without-join-stages stage)
(node :guard (unstamped-cross-stage-ref? aliases node)) true))))
(defn- first-unresolved-cross-stage-ref
- "Return the first [[unstamped-cross-stage-ref?]] clause in `stage` that matches no column in
- `cols`, or nil. Skips `joins` subtrees (their own resolution context)."
+ "Return the first [[unstamped-cross-stage-ref?]] clause in `stage` that matches no column in `cols`, or nil.
+ A ref through a join alias matches only the columns that came through it, and is skipped when none did.
+ Skips each join's [[hidden-join-keys]]."
[stage cols]
- (let [aliases (stage-join-aliases stage)
- stage' (cond-> stage (contains? stage "joins") (dissoc "joins"))]
- (match/match-one stage'
+ (let [aliases (stage-join-aliases stage)]
+ (match/match-one (without-join-stages stage)
(node :guard (and (unstamped-cross-stage-ref? aliases node)
- (nil? (match-cross-stage-column cols (nth node 2)))))
+ (when-let [ref-cols (cols-through-alias cols node)]
+ (nil? (match-cross-stage-column ref-cols (nth node 2) (get-in node [1 "join-alias"]))))))
node)))
(defn- assert-cross-stage-refs-resolved*
@@ -3118,17 +3164,93 @@
(get stage "source-card") (mini-resolved-columns-for-source-card mp query idx content-store)
(pos? idx) (mini-resolved-columns mp query idx content-store))]
(when-let [bad (first-unresolved-cross-stage-ref stage cols)]
- (throw (ex-info
- (format "No column named %s is available to reference in this stage. Reference a column by its machine name (e.g. an aggregation is `max`, `count`, `sum` - never its display label like `Max of X`), not a UI label. Available column names: %s."
- (quoting/quoted (nth bad 2))
- (str/join ", " (map (comp quoting/quoted :name) cols)))
- {:agent-error? true
- :error :unresolved-cross-stage-field
- :stage idx
- :clause bad
- :available (mapv :name cols)}))))))
+ (let [alias (get (nth bad 1) "join-alias")
+ available (cols-through-alias cols bad)]
+ (throw (ex-info
+ (format "No column named %s is available to reference in this stage. Reference a column by its machine name (e.g. an aggregation is `max`, `count`, `sum` - never its display label like `Max of X`), not a UI label. Available column names: %s."
+ (cond-> (quoting/quoted (nth bad 2))
+ alias (str " from join " (quoting/quoted alias)))
+ (str/join ", " (map (comp quoting/quoted :name) available)))
+ {:agent-error? true
+ :error :unresolved-cross-stage-field
+ :stage idx
+ :clause bad
+ :available (mapv :name available)})))))))
query)
+;;; ============================================================
+;;; Passes 4.5 to 5.7 over every join's own stages
+;;;
+;;; A join's `stages` are a query of their own, with the same kinds of by-name refs.
+;;; Until they are typed, the join can't be resolved, so Pass 4.7 can't type or check refs to its columns either.
+;;; The passes run over each join's stages first, innermost join first.
+;;; An agent error raised inside a join says which join it came from.
+;;; ============================================================
+
+(declare infer-field-types*)
+
+(defn- infer-join-field-types
+ "Return `join` with Passes 4.5 to 5.7 run over its own `stages`, as a query against `database`.
+ Rethrows an agent error from inside with a message naming the join and `stage-idx`, and with `:join-alias`
+ and `:join-stage` added to its data."
+ [join database mp content-store stage-idx]
+ (if-not (and (map? join) (vector? (get join "stages")))
+ join
+ (try
+ (assoc join "stages" (-> (cond-> {"lib/type" "mbql/query", "stages" (get join "stages")}
+ database (assoc "database" database))
+ (infer-field-types* mp content-store)
+ (get "stages")))
+ (catch clojure.lang.ExceptionInfo e
+ (if (:agent-error? (ex-data e))
+ (let [alias (get join "alias")
+ where (if (non-blank-string? alias)
+ (format "In join %s of stage %d" (quoting/quoted alias) stage-idx)
+ (format "In a join with no alias in stage %d" stage-idx))]
+ (throw (ex-info (str where ": " (ex-message e))
+ (assoc (ex-data e) :join-alias alias :join-stage stage-idx)
+ e)))
+ (throw e))))))
+
+(defn- with-alias-on-join-fields
+ "Return `join` with its alias on each by-name field ref in its `fields`, as lib's join normalization does.
+ Leaves path refs alone."
+ [join]
+ (let [alias (when (map? join) (get join "alias"))]
+ (cond-> join
+ (and (non-blank-string? alias) (sequential? (get join "fields")))
+ (update "fields" (partial mapv #(cond-> %
+ ;; no pass mistakes a path ref for a previous-stage ref, so lib aliases those
+ (match/matches? % ["field" (_ :guard map?) (_ :guard string?)])
+ (assoc-in [1 "join-alias"] alias)))))))
+
+(defn- infer-join-field-types-in-stage
+ "Return `stage`, the stage at `idx` of `query`, with [[with-alias-on-join-fields]] and then
+ [[infer-join-field-types]] applied to each of its joins."
+ [query mp content-store idx stage]
+ (if-not (and (map? stage) (sequential? (get stage "joins")))
+ stage
+ (let [database (get query "database")
+ stage-idx (repr/original-stage-index query idx)]
+ ;; Aliasing the `fields` refs first lets Pass 4.7 type them from the join, and keeps Pass 5 from
+ ;; taking them for refs to the previous stage.
+ (update stage "joins" (partial mapv #(-> %
+ with-alias-on-join-fields
+ (infer-join-field-types database mp content-store stage-idx)))))))
+
+(defn- infer-field-types*
+ "Run Passes 4.5, 4.7, 5 and 5.7 over `query`, after running them over every join's own stages.
+ No-op without `mp`."
+ [query mp content-store]
+ (if-not (and mp (map? query) (vector? (get query "stages")))
+ query
+ (-> query
+ (update "stages" #(into [] (map-indexed (partial infer-join-field-types-in-stage query mp content-store)) %))
+ (infer-source-card-field-types* mp content-store)
+ (repair-join-alias-refs* mp content-store)
+ (infer-cross-stage-field-types* mp content-store)
+ (assert-cross-stage-refs-resolved* mp content-store))))
+
;;; ============================================================
;;; Pass 6 -- friendly error messages for silently-accepted-but-wrong shapes.
;;;
@@ -3454,21 +3576,23 @@
a saved question / model (`source-card:`), using the card's resolved returned columns.
Must run *before* Pass 5: a `source-card` stage's own bare-name field refs need
`base-type` before that stage can be mini-resolved as part of a later stage's prefix.
+ 4.7. infer `base-type` / `effective-type` on join-alias string field references from the
+ referenced join's own resolved returned columns, and raise an `:agent-error?` for a
+ join-alias ref naming a column its join doesn't return. Runs before Pass 5 so a stage
+ carrying such a ref can still be mini-resolved as a prefix.
5. infer `base-type` / `effective-type` on cross-stage field references
(`[\"field\" {} \"<column-name>\"]` in a non-first stage), by mini-resolving the
prefix of stages and reading the returned columns' metadata. When the prefix includes
a `source-card` stage, this relies on Pass 4.5 having already typed that stage's own
field refs - otherwise the prefix fails to resolve/schema-validate and the mini-resolve
silently no-ops (BOT-1604).
- 4.7. infer `base-type` / `effective-type` on join-alias string field references from the
- referenced join's own resolved returned columns, and raise an `:agent-error?` for a
- join-alias ref naming a column its join doesn't return. Runs before Pass 5 so a stage
- carrying such a ref can still be mini-resolved as a prefix.
5.7. assert that every string-named cross-stage / source-card field ref resolved to a real
column (i.e. Pass 4.5 / 5 stamped its `base-type`). A ref that matched nothing is the
LLM naming a column that doesn't exist (often a display label); raise an `:agent-error?`
naming it and listing the valid columns, instead of letting it pass through into a
schema-invalid, non-runnable query (BOT-1442).
+ Passes 4.5 to 5.7 run over every join's own stages first, innermost first, since a join
+ can't be resolved until its own by-name refs are typed.
Pass 4, Pass 4.5, Pass 4.7, Pass 5, and Pass 5.7 require `mp` (a `MetadataProvider`); they are
best-effort no-ops when `mp` can't resolve the relevant pieces (so the subsequent
@@ -3510,10 +3634,7 @@
hoist-temporal-buckets*
(resolve-source-field-join-alias* mp content-store)
(resolve-implicit-joins* mp content-store)
- (infer-source-card-field-types* mp content-store)
- (repair-join-alias-refs* mp content-store)
- (infer-cross-stage-field-types* mp content-store)
- (assert-cross-stage-refs-resolved* mp content-store)
+ (infer-field-types* mp content-store)
friendly-errors*)))
;;; ============================================================
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index d5e02a8d4b9..ebf05a2159a 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -943,7 +943,7 @@
"a reference inside it could not be resolved - often a field reference to a column, join alias, or expression that does not exist. Refer to a join's or an earlier stage's columns by their exact machine name (an aggregation is `min`, `count`, `sum`)"
(= :base-type missing)
- "a field reference by name is missing `base-type`, usually because the name matches no real column"
+ "a field reference by name is missing `base-type` because its type could not be inferred - check that the name is a column this stage or join can reference"
missing
(format "missing `%s`" (name missing))
@@ -956,21 +956,50 @@
(= "non-aggregation expression" message)
(format "%s: aggregation and window functions such as `offset` cannot be custom columns - move them to `aggregation:` or `order-by:`" message))))
+(defn- problem-rank
+ "Return where `error` sorts among the listed problems: 0 when [[not-runnable-problem]] states its cause,
+ 1 for Malli's own message, and 2 for a missing `base-type`."
+ [error]
+ (cond
+ (and (= :malli.core/missing-key (:type error))
+ (= :base-type (peek (:in error)))) 2
+ (not-runnable-problem error) 0
+ :else 1))
+
+(def ^:private max-not-runnable-problems 3)
+
+(defn- not-runnable-problem-lines
+ "Return one line per distinct problem among `errors`, in [[problem-rank]] order.
+ Each line gives the problem's first path, how many other places have it, and the problem.
+ Lines past [[max-not-runnable-problems]] are replaced by a count.
+ Paths use the original stage indexes recorded in `repaired-query`."
+ [errors repaired-query]
+ (let [located (for [error (sort-by problem-rank errors)]
+ {:path (repr/query-path (repr/original-query-path repaired-query (:in error)))
+ :problem (or (not-runnable-problem error) (me/error-message error))})
+ paths-of (group-by :problem located)
+ lines (for [problem (distinct (map :problem located))
+ :let [[path & others] (distinct (map :path (paths-of problem)))]]
+ (str path
+ (when others
+ (format " (and %d other place%s)" (count others) (if (= 1 (count others)) "" "s")))
+ ": " problem))
+ omitted (- (count lines) max-not-runnable-problems)]
+ (cond-> (vec (take max-not-runnable-problems lines))
+ (pos? omitted) (conj (format "and %d more problem%s" omitted (if (= 1 omitted) "" "s"))))))
+
(defn- query-not-runnable-explanation
"Explain why `pmbql-query` fails the query builder's `canRun` check, or return nil.
- Problems that [[not-runnable-problem]] can explain come first. Add the usual causes if it cannot
- explain any of them. `repaired-query` supplies the original stage indexes and defaults to `pmbql-query`."
+ Each distinct problem is listed once, with how many places have it, in [[problem-rank]] order.
+ The usual causes are added when [[not-runnable-problem]] cannot explain any of them.
+ The optional `repaired-query` supplies the original stage indexes and defaults to `pmbql-query`."
([pmbql-query] (query-not-runnable-explanation pmbql-query pmbql-query))
([pmbql-query repaired-query]
(binding [lib.schema.expression/*suppress-expression-type-check?* true]
(when-let [explanation (mr/explain :metabase.lib.schema/query pmbql-query)]
- (let [errors (sort-by (comp nil? not-runnable-problem) (:errors explanation))]
- (cond-> (or (some->> errors
- (map #(str (repr/query-path (repr/original-query-path repaired-query (:in %))) ": "
- (or (not-runnable-problem %) (me/error-message %))))
- distinct
- (take 3)
+ (let [errors (:errors explanation)]
+ (cond-> (or (some->> (not-runnable-problem-lines errors repaired-query)
seq
(str/join "; "))
"the query does not match the query builder's schema")
diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.clj
index 2b6ac4a7fdf..e3c07c25bac 100644
--- a/test/metabase/agent_lib/representations/repair_test.clj
+++ b/test/metabase/agent_lib/representations/repair_test.clj
@@ -4120,7 +4120,7 @@
(orders-stage {"aggregation" [["min" {} (orders-col "TOTAL")]]
"breakout" [(orders-col "USER_ID")]}))
-(def ^:private user-id-by-name (typed ["field" {} "USER_ID"] "type/Integer"))
+(def ^:private user-id-by-name ["field" {} "USER_ID"])
(deftest ^:parallel join-alias-field-type-scoped-to-the-stage-with-the-join-test
(let [joined-here (query min-total-by-user-stage
@@ -4145,6 +4145,162 @@
(testing "the ref isn't typed from the previous stage's same-named column, and nothing is raised"
(is (= min-ref (get-in (repair* q) ["stages" 1 "filters" 0 2]))))))
+(deftest ^:parallel join-stages-field-types-test
+ (let [two-stage (fn [column] [first-orders-stage (stage {"filters" [["not-null" {} ["field" {} column]]]})])
+ joined (fn [join-stages field-ref] (query (orders-stage {"joins" [(join "Two" join-stages)]
+ "filters" [["not-null" {} field-ref]]})))]
+ (testing "a by-name ref in a join's second stage is typed, so a ref to the join's column is too"
+ (let [out (repair* (joined (two-stage "min") (join-ref "Two" "min")))]
+ (is (= "type/Date" (base-type (get-in out ["stages" 0 "joins" 0 "stages" 1 "filters" 0 2]))))
+ (is (= "type/Date" (base-type (get-in out ["stages" 0 "filters" 0 2]))))
+ (is (= out (repair* out)))))
+ (testing "a by-name ref to a nested join's column is typed, and so is the ref to the outer join"
+ (let [inner (orders-stage {"joins" [(join "First" [first-orders-stage])]
+ "filters" [["not-null" {} (join-ref "First" "min")]]})
+ out (repair* (query (orders-stage {"joins" [(join "Outer" [inner])]
+ "filters" [["not-null" {} (join-ref "Outer" "TOTAL")]]})))]
+ (is (= "type/Date" (base-type (get-in out ["stages" 0 "joins" 0 "stages" 0 "filters" 0 2]))))
+ (is (= "type/Float" (base-type (get-in out ["stages" 0 "filters" 0 2]))))))
+ (testing "a bad name inside a join is reported with the join it is in"
+ (let [e (repair-error (joined (two-stage "nope") (join-ref "Two" "min")))]
+ (is (=? {:error :unresolved-cross-stage-field
+ :join-alias "Two"
+ :join-stage 0
+ :stage 1}
+ e))
+ (is (str/starts-with? (:message e) "In join \"Two\" of stage 0: No column named \"nope\""))))
+ (testing "a bad name inside a join with no alias is reported as being in such a join"
+ (let [no-alias (-> (join "Two" (two-stage "nope")) (dissoc "alias") (assoc "conditions" []))]
+ (is (str/starts-with? (:message (repair-error (query (orders-stage {"joins" [no-alias]}))))
+ "In a join with no alias in stage 0: No column named \"nope\""))))))
+
+(deftest ^:parallel join-fields-ref-without-alias-test
+ (let [join-fields (fn [previous-stage]
+ (get-in (repair* (query previous-stage
+ (stage {"joins" [(assoc (join "First" [first-orders-stage] :lhs user-id-by-name)
+ "fields" [["field" {} "min"]])]})))
+ ["stages" 1 "joins" 0 "fields"]))]
+ (testing "a by-name ref in a join's fields gets the join's alias and is typed from the join"
+ (is (= [["field" {"join-alias" "First", "base-type" "type/Date", "effective-type" "type/Date"} "min"]]
+ (join-fields min-total-by-user-stage))))
+ (testing "the previous stage not having the column doesn't matter"
+ (is (= "type/Date"
+ (base-type (first (join-fields (orders-stage {"aggregation" [["count" {}]]
+ "breakout" [(orders-col "USER_ID")]})))))))))
+
+(deftest ^:parallel alias-less-join-fields-left-for-validation-test
+ (let [fields-of (fn [column]
+ (let [no-alias {"strategy" "left-join"
+ "stages" [first-orders-stage]
+ "conditions" [["=" {} user-id-by-name (orders-col "USER_ID")]]
+ "fields" [["field" {} column]]}]
+ (get-in (repair* (query (orders-stage {"aggregation" [["count" {}]]
+ "breakout" [(orders-col "USER_ID")]})
+ (stage {"joins" [no-alias]})))
+ ["stages" 1 "joins" 0 "fields"])))]
+ (testing "a by-name ref in an alias-less join's fields stays untyped, whether or not the previous stage has it"
+ (doseq [column ["count" "min"]]
+ (is (= [["field" {} column]] (fields-of column)))))))
+
+(def ^:private prev-joined-stage
+ "Orders with a `Prev` join of each user's first order, returning all its columns."
+ (orders-stage {"joins" [(assoc (join "Prev" [first-orders-stage]) "fields" "all")]}))
+
+(deftest ^:parallel previous-stage-ref-in-join-condition-test
+ (let [joined-on (fn [lhs] (query first-orders-stage (stage {"joins" [(join "First" [first-orders-stage] :lhs lhs)]})))]
+ (testing "a by-name ref to the previous stage in a join's conditions is typed from that stage"
+ (is (= "type/Integer"
+ (base-type (get-in (repair* (joined-on ["field" {} "USER_ID"])) ["stages" 1 "joins" 0 "conditions" 0 2])))))
+ (testing "a bad name there raises the cross-stage error"
+ (is (=? {:error :unresolved-cross-stage-field, :clause ["field" {} "nope"]}
+ (repair-error (joined-on ["field" {} "nope"])))))
+ (testing "a ref there through a join alias the previous stage doesn't have is left for validation"
+ (doseq [column ["USER_ID" "nope"]
+ :let [field-ref ["field" {"join-alias" "first"} column]]]
+ (is (= field-ref (get-in (repair* (joined-on field-ref)) ["stages" 1 "joins" 0 "conditions" 0 2])))))
+ (let [prev-joined-on (fn [lhs]
+ (query prev-joined-stage (stage {"joins" [(join "First" [first-orders-stage] :lhs lhs)]})))]
+ (testing "a ref there through a join of the previous stage is typed from that stage"
+ (is (= "type/Date"
+ (base-type (get-in (repair* (prev-joined-on ["field" {"join-alias" "Prev"} "min"]))
+ ["stages" 1 "joins" 0 "conditions" 0 2])))))
+ (testing "a ref there through that join to a column the previous stage deduplicates is typed from the join"
+ (is (= "type/Integer"
+ (base-type (get-in (repair* (prev-joined-on ["field" {"join-alias" "Prev"} "USER_ID"]))
+ ["stages" 1 "joins" 0 "conditions" 0 2])))))
+ (testing "a ref there through that join to a column only outside it raises the cross-stage error"
+ (let [e (repair-error (prev-joined-on ["field" {"join-alias" "Prev"} "TOTAL"]))]
+ (is (=? {:error :unresolved-cross-stage-field
+ :clause ["field" {"join-alias" "Prev"} "TOTAL"]
+ :available ["USER_ID_2" "min"]}
+ e))
+ (is (str/starts-with? (:message e) "No column named \"TOTAL\" from join \"Prev\" is available")))))))
+
+(deftest ^:parallel previous-stage-join-alias-ref-test
+ (let [filtered-on (fn [field-ref] (query prev-joined-stage (stage {"filters" [["not-null" {} field-ref]]})))]
+ (testing "a ref through a join of the previous stage is typed from that join's columns"
+ (is (= "type/Date"
+ (base-type (get-in (repair* (filtered-on ["field" {"join-alias" "Prev"} "min"]))
+ ["stages" 1 "filters" 0 2])))))
+ (testing "a ref through that join to a column only outside it raises the cross-stage error"
+ (is (=? {:error :unresolved-cross-stage-field, :clause ["field" {"join-alias" "Prev"} "TOTAL"]}
+ (repair-error (filtered-on ["field" {"join-alias" "Prev"} "TOTAL"])))))))
+
+(deftest ^:parallel previous-stage-join-alias-deduplicated-aggregation-test
+ (let [min-ref (join-ref "Prev" "min")
+ q (query min-total-by-user-stage
+ (stage {"joins" [(assoc (join "Prev" [first-orders-stage] :lhs user-id-by-name)
+ "fields" "all")]})
+ (stage {"filters" [["not-null" {} min-ref]]
+ "joins" [(assoc-in (join "Next" [first-orders-stage] :lhs min-ref)
+ ["conditions" 0 3] (join-ref "Next" "min"))]}))
+ out (repair* q)]
+ (testing "a deduplicated join column is typed through its alias in filters and join conditions"
+ (doseq [path [["filters" 0 2] ["joins" 0 "conditions" 0 2]]]
+ (is (=? (typed min-ref "type/Date")
+ (get-in out (into ["stages" 2] path))))))
+ (testing "a quoted name is unquoted without adding the join prefix to the ref"
+ (is (= out (repair* (assoc-in q ["stages" 2 "filters" 0 2 2] "\"min\"")))))
+ (is (= out (repair* out)))))
+
+(deftest ^:parallel previous-stage-join-alias-prefixed-name-precedence-test
+ (let [joined (update first-orders-stage "aggregation" conj ["max" {"name" "min_2"} (orders-col "TOTAL")])
+ q (query min-total-by-user-stage
+ (stage {"joins" [(assoc (join "Prev" [joined] :lhs user-id-by-name) "fields" "all")]})
+ (stage {"filters" [["not-null" {} (join-ref "Prev" "min_2")]]}))]
+ (testing "the joined min_2 is a Float, even though the joined Date min is deduplicated to min_2"
+ (is (= "type/Float" (base-type (get-in (repair* q) ["stages" 2 "filters" 0 2])))))))
+
+(deftest ^:parallel previous-stage-join-alias-display-name-collision-test
+ (let [joined (update first-orders-stage "aggregation" conj ["max" {"name" "min_2"} (orders-col "TOTAL")])]
+ (doseq [column-name ["Min of Created At" "min-of-created-at"]]
+ (let [q (query min-total-by-user-stage
+ (stage {"joins" [(assoc (join "Prev" [joined] :lhs user-id-by-name) "fields" "all")]})
+ (stage {"filters" [["not-null" {} (join-ref "Prev" column-name)]]}))
+ out (repair* q)]
+ (is (=? (typed (join-ref "Prev" "min") "type/Date") (get-in out ["stages" 2 "filters" 0 2])))
+ (is (= out (repair* out)))))))
+
+(deftest ^:parallel previous-stage-join-alias-custom-display-name-test
+ (let [joined (assoc-in prev-joined-stage ["joins" 0 "stages" 0 "breakout" 0 1 "display-name"] "Account key")
+ q (query joined (stage {"filters" [["not-null" {} (join-ref "Prev" "USER_ID")]]}))]
+ (is (= "type/Integer" (base-type (get-in (repair* q) ["stages" 1 "filters" 0 2]))))))
+
+(deftest ^:parallel previous-stage-join-alias-nested-column-test
+ (let [self-joined [(orders-stage {"joins" [(assoc (join "Self" [(orders-stage {})]) "fields" "all")]})]
+ field-ref (join-ref "Outer" "Self__TOTAL")
+ q (query (orders-stage {"joins" [(assoc (join "Outer" self-joined) "fields" "all")]})
+ (stage {"filters" [["not-null" {} field-ref]]}))
+ out (repair* q)]
+ (is (=? (typed field-ref "type/Float")
+ (get-in out ["stages" 1 "filters" 0 2])))
+ (is (= out (repair* out)))))
+
+(deftest ^:parallel unresolved-cross-stage-ref-with-no-columns-test
+ (testing "a ref without a join alias is reported when the previous stage returns no columns"
+ (is (= ["field" {} "nope"]
+ (#'repair/first-unresolved-cross-stage-ref {"filters" [["not-null" {} ["field" {} "nope"]]]} [])))))
+
(deftest ^:parallel join-alias-field-type-runs-before-cross-stage-inference-test
(testing "a stage carrying a join-alias ref still resolves as the next stage's prefix"
(let [out (repair* (query (orders-stage {"joins" [(join "First" [first-orders-stage])]
@@ -4213,6 +4369,23 @@
(base-type (get-in (repair* (first-orders-filtering (typed (join-ref "First" "min") "type/Text")))
["stages" 0 "filters" 0 2]))))))
+(deftest ^:parallel join-alias-field-type-nested-join-column-names-test
+ (let [self-joined [(orders-stage {"joins" [(assoc (join "Self" [(orders-stage {})]) "fields" "all")]})]
+ filtering (fn [field-ref] (query (orders-stage {"joins" [(join "Outer" self-joined)]
+ "filters" [["not-null" {} field-ref]]})))
+ filtered (fn [field-ref] (get-in (repair* (filtering field-ref)) ["stages" 0 "filters" 0 2]))]
+ (testing "a nested join's column is typed by each name lib resolves, and kept as written"
+ (doseq [column-name ["Self__TOTAL" "TOTAL_2"]]
+ (is (= (typed (join-ref "Outer" column-name) "type/Float")
+ (filtered (join-ref "Outer" column-name))))))
+ (testing "a quoted desired alias is unquoted and typed"
+ (is (= (typed (join-ref "Outer" "Self__TOTAL") "type/Float")
+ (filtered (join-ref "Outer" "\"Self__TOTAL\"")))))
+ (testing "a ref the LLM typed itself by the desired alias is accepted"
+ (is (= "type/Number" (base-type (filtered (typed (join-ref "Outer" "Self__TOTAL") "type/Number"))))))
+ (testing "a name the join doesn't return is still reported"
+ (is (= [[0 "Outer" ["Self__NOPE"]]] (join-problems (filtering (join-ref "Outer" "Self__NOPE"))))))))
+
(deftest ^:parallel join-alias-field-type-unknown-column-quotes-available-columns-test
(testing "the names of the columns a join returns reach the message quoted"
(let [narrow (orders-stage {"fields" [(orders-col "USER_ID") (orders-col odd-column-name)]})
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index 1eb6a34b98b..1174432e91a 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -3327,6 +3327,23 @@
(is (=? {:agent-error? true :error :query-not-runnable :status-code 400} (ex-data e)))
(is (str/includes? (ex-message e) "could not be resolved")))))))
+(deftest nested-expression-unresolved-reference-explained-test
+ (with-mp-and-stubs!
+ (fn []
+ (let [q {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "expressions" {"Broken" ["+" {}
+ ["length" {}
+ ["replace" {}
+ ["substring" {} ["field" {"join-alias" "Nope"} "NAME"] 1 10]
+ "-" ""]]
+ 1]}}]}
+ e (editor-rejection q)]
+ (is (=? {:agent-error? true, :error :query-not-runnable} (ex-data e)))
+ (is (re-find #"Failing paths: [^;]+could not be resolved" (ex-message e)))))))
+
(deftest path-ref-to-dropped-join-column-rejected-before-sql-test
(testing "a path ref to a column the grouped join drops fails in repair, not in the database"
(with-mp-and-stubs!
@@ -3354,11 +3371,22 @@
hint #"\. This usually means a field reference is missing its type"]
(testing "identical problems are listed once"
(is (= "stages[0].alias: missing `alias`" (explain (missing :alias) (missing :alias)))))
- (testing "at most three problems are listed"
- (is (= 3 (count (str/split (apply explain (map missing [:a :b :c :d])) #"; ")))))
+ (testing "one problem in several places is listed once, with a count of the others"
+ (is (str/starts-with? (explain (invalid :a) (invalid :b) (invalid :c))
+ "stages[0].a (and 2 other places): should be an integer.")))
+ (testing "at most three problems are listed, and the rest are counted"
+ (is (str/ends-with? (apply explain (map missing [:a :b :c :d :e])) "; and 2 more problems")))
(testing "problems with a known cause are listed before Malli's own messages"
(is (str/starts-with? (explain (invalid :a) (invalid :b) (invalid :c) (missing :alias))
- "stages[0].alias: missing `alias`; stages[0].a: should be an integer")))
+ "stages[0].alias: missing `alias`; stages[0].a (and 2 other places): should be an integer")))
+ (testing "Malli's own messages are listed before a guessed cause, however often it repeats"
+ (let [untyped (fn [i] {:type :malli.core/missing-key, :in [:stages 0 :filters i 2 1 :base-type]})
+ no-join {:schema (mc/schema [:fn {:error/message "no join named \"first\""} any?])
+ :value 1
+ :in [:stages]}]
+ (is (str/starts-with? (apply explain (conj (mapv untyped (range 4)) no-join))
+ (str "stages: no join named \"first\"; "
+ "stages[0].filters[0][2][1].base-type (and 3 other places): a field reference by name is missing `base-type`")))))
(testing "the usual causes are added only when no problem has a known cause"
(is (re-find hint (explain (invalid :a))))
(is (not (re-find hint (explain (invalid :a) (missing :alias))))))