The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Type and check by-name refs inside joins (#85736)
Chris Truter714a9e49117e
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.cljindex ebc767e9bb9..04821d8bd29 100644--- a/src/metabase/agent_lib/representations/repair.clj+++ b/src/metabase/agent_lib/representations/repair.clj@@ -2667,9 +2667,9 @@;;; Strategy:;;; * Walk stages left-to-right.;;; * Before processing stage `i` (i ≥ 1), build a mini-query consisting of stages[0..i-1]-;;; and resolve it. Call `lib/returned-columns` on the result; index by `:name`.-;;; * Walk stage[i] (skipping descent into `joins` subtrees - those have their own-;;; resolution context). For every `["field" opts <string>]` clause:+;;; and resolve it. Match refs to its `lib/returned-columns` by the names lib resolves.+;;; * Walk stage[i], including its joins' `conditions` but not their [[hidden-join-keys]]: a+;;; join's own `stages` have their own resolution context. For every `["field" opts <string>]` clause:;;; - If `opts` already has `"base-type"`, leave it alone (idempotent).;;; - If the name is unknown to the previous stage, leave it alone (the resolver will;;; report the real error with a better message).@@ -2686,6 +2686,35 @@(let [joins (get stage "joins")](into #{} (keep #(when (map? %) (get % "alias"))) (when (sequential? joins) joins))))+(defn- hidden-join-keys+ "Return the keys of `join` that the cross-stage passes leave alone.+ These are its own `stages`, and its `fields` when it has no alias."+ [join]+ ;; Refs in an alias-less join's `fields` can't be told from refs to the previous stage, and validation reports the+ ;; missing alias.+ (cond-> ["stages"]+ (not (non-blank-string? (get join "alias"))) (conj "fields")))++(defn- without-join-stages+ "Return `stage` without its joins' [[hidden-join-keys]], keeping the stage's own clauses and the rest of each join."+ [stage]+ (cond-> stage+ (sequential? (get stage "joins"))+ (update "joins" (partial mapv #(if (map? %) (apply dissoc % (hidden-join-keys %)) %)))))++(defn- postwalk-outside-join-stages+ "Apply `f` with `walk/postwalk` to the parts of `stage` that [[without-join-stages]] keeps.+ Each join's [[hidden-join-keys]] come back unchanged."+ [f stage]+ (let [joins (get stage "joins")]+ (cond-> (walk/postwalk f (without-join-stages stage))+ (sequential? joins)+ (update "joins" #(mapv (fn [walked join]+ (cond-> walked+ (map? join) (merge (select-keys join (hidden-join-keys join)))))+ %+ joins)))))+(defn- string-cross-stage-field-clause?"True if `v` is a string-named field ref to a previous-stage column of a stage whose joins arealiased `join-aliases`. A ref through one of those aliases names a join's column instead."@@ -2708,17 +2737,30 @@(if (keyword? et) (subs (str et) 1) (str et))))))))(defn- column-descriptors- "Build the cross-stage match oracle from a `lib/returned-columns` sequence: a vector of- `{:name <machine-name>, :display-name <display-name>, :id <field-id>|nil, :types {\"base-type\" …}|nil}`- descriptors. `:types` is nil when the column exposes no usable `base-type`. Carrying the- `:display-name` lets [[match-cross-stage-column]] recover a ref the LLM wrote with the UI label- (e.g. `\"Max of Instance Has SC\"`) instead of the machine name (`max`)."+ "Build the cross-stage match oracle from a `lib/returned-columns` sequence, as a vector of descriptors:++ {:name <machine name>+ :desired-alias <alias>+ :deduplicated-name <name>+ :display-name <display name>+ :join-alias <alias of the join the column came through> | nil+ :id <field id> | nil+ :types {\"base-type\" …} | nil}++ The desired alias and the deduplicated name are the names lib resolves a by-name ref against.+ The display name lets [[match-cross-stage-column]] recover a ref the LLM wrote with the UI label, such as+ `\"Max of Instance Has SC\"`, instead of the machine name `max`.+ The join alias says which of the previous stage's joins the column came through.+ The types are nil when the column has no usable `base-type`."[cols](mapv (fn [col]- {:name (:name col)- :display-name (:display-name col)- :id (:id col)- :types (types-from-column col)})+ {:name (:name col)+ :desired-alias (:lib/desired-column-alias col)+ :deduplicated-name (:lib/deduplicated-name col)+ :display-name (:display-name col)+ :join-alias (:lib/original-join-alias col)+ :id (:id col)+ :types (types-from-column col)})cols))(defn- resolved-columns@@ -2768,34 +2810,50 @@(-> s u/lower-case-en (str/replace #"[-\s]+" "_"))))(defn- match-cross-stage-column- "Find the column descriptor in `cols` (a vector of [[column-descriptors]]) that `col-name`- refers to. Returns `[canonical-machine-name types]` or `nil` when no column matches.+ "Find the column in `cols`, a vector of [[column-descriptors]], that `col-name` refers to.+ Returns `[name types]`, where `name` is the spelling the ref should carry, or nil when no column matches.Tries, in order:- 1. an exact `:name` match;- 2. the double-quote-stripped `:name` (only when stripping actually resolves);+ 1. an exact match on a name lib resolves: the desired alias, then the deduplicated name, then `:name`.+ The ref keeps its spelling, since a nested join's column is `P__NAME` to lib even when its `:name` is `NAME`;+ 2. the same match with surrounding double quotes stripped, giving the stripped spelling;3. a loose `:name` match that folds case and hyphens/whitespace to underscores, but **onlywhen exactly one** real column matches - so `count-where` resolves to `count_where`;4. a loose `:display-name` match (same folding, same one-hit guard) - so a ref the LLMwrote with the UI label (`\"Max of Instance Has SC\"`) recovers the machine name (`max`).- Genuine ambiguity (more than one loose hit) is left unmatched at each layer. A column whose- name legitimately needs quotes (vanishingly rare) is never clobbered."- [cols col-name]- (let [stripped (strip-surrounding-double-quotes col-name)- exact (fn [n] (some #(when (= n (:name %)) %) cols))- loose-on (fn [key-fn raw]+ Genuine ambiguity (more than one loose hit) is left unmatched at each layer.+ A column whose name legitimately needs quotes (vanishingly rare) is never clobbered.+ With `join-alias`, each exact match tries `join-alias__name` before the name itself, keeping the ref's spelling.+ Loose matches through `join-alias` use a spelling that resolves back to the same column."+ ([cols col-name]+ (match-cross-stage-column cols col-name nil))+ ([cols col-name join-alias]+ (let [stripped (strip-surrounding-double-quotes col-name)+ find-on (fn [k n] (some #(when (= n (k %)) %) cols))+ exact (fn [n] (some #(find-on % n) [:desired-alias :deduplicated-name :name]))+ lib-match (fn [n] (or (when join-alias (exact (str join-alias "__" n)))+ (exact n)))+ loose-on (fn [key-fn raw](let [target (normalize-col-key raw)hits (when target(filter #(= target (normalize-col-key (key-fn %))) cols))](when (= 1 (count hits))- (first hits))))- match (or (exact col-name)- (when (not= stripped col-name) (exact stripped))- (loose-on :name stripped)- (loose-on :display-name stripped))]- (when match- [(:name match) (:types match)])))+ (first hits))))]+ (if-let [[spelling match] (or (some->> (lib-match col-name) (vector col-name))+ (when (not= stripped col-name)+ (some->> (lib-match stripped) (vector stripped))))]+ [spelling (:types match)]+ (when-let [match (or (loose-on :name stripped)+ (loose-on :display-name stripped))]+ (let [desired-alias (:desired-alias match)+ prefix (when join-alias (str join-alias "__"))+ spelling (if (and prefix desired-alias (str/starts-with? desired-alias prefix))+ (subs desired-alias (count prefix))+ (:name match))]+ ;; A deduplicated name can denote another column after lib adds the join prefix.+ (when (or (nil? join-alias) (= match (lib-match spelling)))+ [spelling (:types match)])))))))(defn- maybe-fill-cross-stage-types"Given a cross-stage field clause and a name→types map, return the clause with its column name@@ -2803,40 +2861,49 @@column) and `base-type` / `effective-type` filled in (when missing and known).Idempotent: once the name is canonical it matches exactly on the next pass (no rename), and- once `base-type` is present it is left alone."- [clause cols]- (let [opts (nth clause 1)- col-name (nth clause 2)]- (if-let [[canonical-name types] (match-cross-stage-column cols col-name)]- (cond-> clause- ;; Canonicalise the name when quote-stripping / display-name recovery was needed to- ;; match. A bare `base-type` stamp without this would leave the ref pointing at a- ;; non-existent column.- (not= canonical-name col-name)- (assoc 2 canonical-name)- ;; Stamp inferred types only when the LLM didn't author a `base-type` already and the- ;; matched column actually exposes one.- (and types (not (contains? opts "base-type")))- (assoc 1 (merge opts types)))- clause)))+ once `base-type` is present it is left alone.+ The optional `join-alias` enables previous-stage join lookup in [[match-cross-stage-column]]."+ ([clause cols]+ (maybe-fill-cross-stage-types clause cols nil))+ ([clause cols join-alias]+ (let [opts (nth clause 1)+ col-name (nth clause 2)]+ (if-let [[canonical-name types] (match-cross-stage-column cols col-name join-alias)]+ (cond-> clause+ ;; Canonicalise the name when quote-stripping / display-name recovery was needed to+ ;; match. A bare `base-type` stamp without this would leave the ref pointing at a+ ;; non-existent column.+ (not= canonical-name col-name)+ (assoc 2 canonical-name)+ ;; Stamp inferred types only when the LLM didn't author a `base-type` already and the+ ;; matched column actually exposes one.+ (and types (not (contains? opts "base-type")))+ (assoc 1 (merge opts types)))+ clause))))++(defn- cols-through-alias+ "Return the columns in `cols` that `field-ref` can name: those that came through its join alias, or all of them+ when it has none.+ Returns nil when no column came through its join alias."+ [cols field-ref]+ (if-let [alias (get (nth field-ref 1) "join-alias")]+ (not-empty (filterv #(= alias (:join-alias %)) cols))+ cols))(defn- infer-cross-stage-field-types-in-stage- "Walk one stage and stamp inferred types into every string-named field reference that lacks- `\"base-type\"`. Skips descent into `\"joins\"` subtrees - join stages have their own- resolution context (the join's own `stages`) and shouldn't reach into their parent stage's- previous-stage columns."+ "Stamp types from `cols` on every by-name field ref in `stage` that lacks `\"base-type\"`, including the+ refs in its joins' `conditions`.+ A ref through a join alias matches only the columns that came through it, and is left for validation when none did.+ Skips each join's [[hidden-join-keys]]; its own `stages` resolve on their own and can't reach this stage's columns."[stage cols]- (let [joins (get stage "joins")- aliases (stage-join-aliases stage)- stage' (cond-> stage (contains? stage "joins") (dissoc "joins"))- walked (walk/postwalk- (fn [node]- (if (string-cross-stage-field-clause? aliases node)- (maybe-fill-cross-stage-types node cols)- node))- stage')]- (cond-> walked- (contains? stage "joins") (assoc "joins" joins))))+ (let [aliases (stage-join-aliases stage)]+ (postwalk-outside-join-stages+ (fn [node]+ (if-let [ref-cols (when (string-cross-stage-field-clause? aliases node)+ (cols-through-alias cols node))]+ (maybe-fill-cross-stage-types node ref-cols (get-in node [1 "join-alias"]))+ node))+ stage)))(defn- mini-resolved-columns-for-source-card"Resolve a single-stage query consisting only of `{:source-card <entity-id>}` (plus the@@ -2918,29 +2985,9 @@(contains? (first stages) "source-table")(not-any? #(contains? (first stages) %) ["aggregation" "breakout" "fields"]))))-(defn- without-join-stages- "`stage` with each join's own `stages` removed, leaving the clauses that may reference a join's- columns: the stage's own and the joins' `conditions`."- [stage]- (cond-> stage- (sequential? (get stage "joins"))- (update "joins" (partial mapv #(cond-> % (map? %) (dissoc "stages"))))))--(defn- postwalk-outside-join-stages- "`walk/postwalk` `f` over the parts of `stage` that [[without-join-stages]] keeps."- [f stage]- (let [joins (get stage "joins")]- (cond-> (walk/postwalk f (without-join-stages stage))- (sequential? joins)- (update "joins" #(mapv (fn [walked join]- (cond-> walked- (and (map? join) (contains? join "stages")) (assoc "stages" (get join "stages"))))- %- joins)))))-(defn- join-alias-refs"`stage`'s join-alias string refs (`:named`) and concrete-column join-alias refs (`:path`), each- grouped by alias, outside the joins' own `stages`."+ grouped by alias, outside the joins' [[hidden-join-keys]]."[stage](let [stage' (without-join-stages stage)alias #(get (nth % 1) "join-alias")]@@ -3084,24 +3131,23 @@(not (contains? (nth node 1) "base-type"))))(defn- stage-has-unstamped-cross-stage-ref?- "Cheap structural pre-scan: does `stage` contain any [[unstamped-cross-stage-ref?]] outside- its `joins` subtrees? Lets the assert pass skip the (resolving) confirmation step entirely- on the happy path, where every ref already carries a stamped `base-type`."+ "True if `stage` has an [[unstamped-cross-stage-ref?]] outside its joins' [[hidden-join-keys]].+ Lets the assert pass skip the resolving confirmation step on the happy path, where every ref is already typed."[stage]- (let [aliases (stage-join-aliases stage)- stage' (cond-> stage (contains? stage "joins") (dissoc "joins"))]- (some? (match/match-one stage'+ (let [aliases (stage-join-aliases stage)]+ (some? (match/match-one (without-join-stages stage)(node :guard (unstamped-cross-stage-ref? aliases node)) true))))(defn- first-unresolved-cross-stage-ref- "Return the first [[unstamped-cross-stage-ref?]] clause in `stage` that matches no column in- `cols`, or nil. Skips `joins` subtrees (their own resolution context)."+ "Return the first [[unstamped-cross-stage-ref?]] clause in `stage` that matches no column in `cols`, or nil.+ A ref through a join alias matches only the columns that came through it, and is skipped when none did.+ Skips each join's [[hidden-join-keys]]."[stage cols]- (let [aliases (stage-join-aliases stage)- stage' (cond-> stage (contains? stage "joins") (dissoc "joins"))]- (match/match-one stage'+ (let [aliases (stage-join-aliases stage)]+ (match/match-one (without-join-stages stage)(node :guard (and (unstamped-cross-stage-ref? aliases node)- (nil? (match-cross-stage-column cols (nth node 2)))))+ (when-let [ref-cols (cols-through-alias cols node)]+ (nil? (match-cross-stage-column ref-cols (nth node 2) (get-in node [1 "join-alias"]))))))node)))(defn- assert-cross-stage-refs-resolved*@@ -3118,17 +3164,93 @@(get stage "source-card") (mini-resolved-columns-for-source-card mp query idx content-store)(pos? idx) (mini-resolved-columns mp query idx content-store))](when-let [bad (first-unresolved-cross-stage-ref stage cols)]- (throw (ex-info- (format "No column named %s is available to reference in this stage. Reference a column by its machine name (e.g. an aggregation is `max`, `count`, `sum` - never its display label like `Max of X`), not a UI label. Available column names: %s."- (quoting/quoted (nth bad 2))- (str/join ", " (map (comp quoting/quoted :name) cols)))- {:agent-error? true- :error :unresolved-cross-stage-field- :stage idx- :clause bad- :available (mapv :name cols)}))))))+ (let [alias (get (nth bad 1) "join-alias")+ available (cols-through-alias cols bad)]+ (throw (ex-info+ (format "No column named %s is available to reference in this stage. Reference a column by its machine name (e.g. an aggregation is `max`, `count`, `sum` - never its display label like `Max of X`), not a UI label. Available column names: %s."+ (cond-> (quoting/quoted (nth bad 2))+ alias (str " from join " (quoting/quoted alias)))+ (str/join ", " (map (comp quoting/quoted :name) available)))+ {:agent-error? true+ :error :unresolved-cross-stage-field+ :stage idx+ :clause bad+ :available (mapv :name available)})))))))query)+;;; ============================================================+;;; Passes 4.5 to 5.7 over every join's own stages+;;;+;;; A join's `stages` are a query of their own, with the same kinds of by-name refs.+;;; Until they are typed, the join can't be resolved, so Pass 4.7 can't type or check refs to its columns either.+;;; The passes run over each join's stages first, innermost join first.+;;; An agent error raised inside a join says which join it came from.+;;; ============================================================++(declare infer-field-types*)++(defn- infer-join-field-types+ "Return `join` with Passes 4.5 to 5.7 run over its own `stages`, as a query against `database`.+ Rethrows an agent error from inside with a message naming the join and `stage-idx`, and with `:join-alias`+ and `:join-stage` added to its data."+ [join database mp content-store stage-idx]+ (if-not (and (map? join) (vector? (get join "stages")))+ join+ (try+ (assoc join "stages" (-> (cond-> {"lib/type" "mbql/query", "stages" (get join "stages")}+ database (assoc "database" database))+ (infer-field-types* mp content-store)+ (get "stages")))+ (catch clojure.lang.ExceptionInfo e+ (if (:agent-error? (ex-data e))+ (let [alias (get join "alias")+ where (if (non-blank-string? alias)+ (format "In join %s of stage %d" (quoting/quoted alias) stage-idx)+ (format "In a join with no alias in stage %d" stage-idx))]+ (throw (ex-info (str where ": " (ex-message e))+ (assoc (ex-data e) :join-alias alias :join-stage stage-idx)+ e)))+ (throw e))))))++(defn- with-alias-on-join-fields+ "Return `join` with its alias on each by-name field ref in its `fields`, as lib's join normalization does.+ Leaves path refs alone."+ [join]+ (let [alias (when (map? join) (get join "alias"))]+ (cond-> join+ (and (non-blank-string? alias) (sequential? (get join "fields")))+ (update "fields" (partial mapv #(cond-> %+ ;; no pass mistakes a path ref for a previous-stage ref, so lib aliases those+ (match/matches? % ["field" (_ :guard map?) (_ :guard string?)])+ (assoc-in [1 "join-alias"] alias)))))))++(defn- infer-join-field-types-in-stage+ "Return `stage`, the stage at `idx` of `query`, with [[with-alias-on-join-fields]] and then+ [[infer-join-field-types]] applied to each of its joins."+ [query mp content-store idx stage]+ (if-not (and (map? stage) (sequential? (get stage "joins")))+ stage+ (let [database (get query "database")+ stage-idx (repr/original-stage-index query idx)]+ ;; Aliasing the `fields` refs first lets Pass 4.7 type them from the join, and keeps Pass 5 from+ ;; taking them for refs to the previous stage.+ (update stage "joins" (partial mapv #(-> %+ with-alias-on-join-fields+ (infer-join-field-types database mp content-store stage-idx)))))))++(defn- infer-field-types*+ "Run Passes 4.5, 4.7, 5 and 5.7 over `query`, after running them over every join's own stages.+ No-op without `mp`."+ [query mp content-store]+ (if-not (and mp (map? query) (vector? (get query "stages")))+ query+ (-> query+ (update "stages" #(into [] (map-indexed (partial infer-join-field-types-in-stage query mp content-store)) %))+ (infer-source-card-field-types* mp content-store)+ (repair-join-alias-refs* mp content-store)+ (infer-cross-stage-field-types* mp content-store)+ (assert-cross-stage-refs-resolved* mp content-store))))+;;; ============================================================;;; Pass 6 -- friendly error messages for silently-accepted-but-wrong shapes.;;;@@ -3454,21 +3576,23 @@a saved question / model (`source-card:`), using the card's resolved returned columns.Must run *before* Pass 5: a `source-card` stage's own bare-name field refs need`base-type` before that stage can be mini-resolved as part of a later stage's prefix.+ 4.7. infer `base-type` / `effective-type` on join-alias string field references from the+ referenced join's own resolved returned columns, and raise an `:agent-error?` for a+ join-alias ref naming a column its join doesn't return. Runs before Pass 5 so a stage+ carrying such a ref can still be mini-resolved as a prefix.5. infer `base-type` / `effective-type` on cross-stage field references(`[\"field\" {} \"<column-name>\"]` in a non-first stage), by mini-resolving theprefix of stages and reading the returned columns' metadata. When the prefix includesa `source-card` stage, this relies on Pass 4.5 having already typed that stage's ownfield refs - otherwise the prefix fails to resolve/schema-validate and the mini-resolvesilently no-ops (BOT-1604).- 4.7. infer `base-type` / `effective-type` on join-alias string field references from the- referenced join's own resolved returned columns, and raise an `:agent-error?` for a- join-alias ref naming a column its join doesn't return. Runs before Pass 5 so a stage- carrying such a ref can still be mini-resolved as a prefix.5.7. assert that every string-named cross-stage / source-card field ref resolved to a realcolumn (i.e. Pass 4.5 / 5 stamped its `base-type`). A ref that matched nothing is theLLM naming a column that doesn't exist (often a display label); raise an `:agent-error?`naming it and listing the valid columns, instead of letting it pass through into aschema-invalid, non-runnable query (BOT-1442).+ Passes 4.5 to 5.7 run over every join's own stages first, innermost first, since a join+ can't be resolved until its own by-name refs are typed.Pass 4, Pass 4.5, Pass 4.7, Pass 5, and Pass 5.7 require `mp` (a `MetadataProvider`); they arebest-effort no-ops when `mp` can't resolve the relevant pieces (so the subsequent@@ -3510,10 +3634,7 @@hoist-temporal-buckets*(resolve-source-field-join-alias* mp content-store)(resolve-implicit-joins* mp content-store)- (infer-source-card-field-types* mp content-store)- (repair-join-alias-refs* mp content-store)- (infer-cross-stage-field-types* mp content-store)- (assert-cross-stage-refs-resolved* mp content-store)+ (infer-field-types* mp content-store)friendly-errors*)));;; ============================================================diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex d5e02a8d4b9..ebf05a2159a 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -943,7 +943,7 @@"a reference inside it could not be resolved - often a field reference to a column, join alias, or expression that does not exist. Refer to a join's or an earlier stage's columns by their exact machine name (an aggregation is `min`, `count`, `sum`)"(= :base-type missing)- "a field reference by name is missing `base-type`, usually because the name matches no real column"+ "a field reference by name is missing `base-type` because its type could not be inferred - check that the name is a column this stage or join can reference"missing(format "missing `%s`" (name missing))@@ -956,21 +956,50 @@(= "non-aggregation expression" message)(format "%s: aggregation and window functions such as `offset` cannot be custom columns - move them to `aggregation:` or `order-by:`" message))))+(defn- problem-rank+ "Return where `error` sorts among the listed problems: 0 when [[not-runnable-problem]] states its cause,+ 1 for Malli's own message, and 2 for a missing `base-type`."+ [error]+ (cond+ (and (= :malli.core/missing-key (:type error))+ (= :base-type (peek (:in error)))) 2+ (not-runnable-problem error) 0+ :else 1))++(def ^:private max-not-runnable-problems 3)++(defn- not-runnable-problem-lines+ "Return one line per distinct problem among `errors`, in [[problem-rank]] order.+ Each line gives the problem's first path, how many other places have it, and the problem.+ Lines past [[max-not-runnable-problems]] are replaced by a count.+ Paths use the original stage indexes recorded in `repaired-query`."+ [errors repaired-query]+ (let [located (for [error (sort-by problem-rank errors)]+ {:path (repr/query-path (repr/original-query-path repaired-query (:in error)))+ :problem (or (not-runnable-problem error) (me/error-message error))})+ paths-of (group-by :problem located)+ lines (for [problem (distinct (map :problem located))+ :let [[path & others] (distinct (map :path (paths-of problem)))]]+ (str path+ (when others+ (format " (and %d other place%s)" (count others) (if (= 1 (count others)) "" "s")))+ ": " problem))+ omitted (- (count lines) max-not-runnable-problems)]+ (cond-> (vec (take max-not-runnable-problems lines))+ (pos? omitted) (conj (format "and %d more problem%s" omitted (if (= 1 omitted) "" "s"))))))+(defn- query-not-runnable-explanation"Explain why `pmbql-query` fails the query builder's `canRun` check, or return nil.- Problems that [[not-runnable-problem]] can explain come first. Add the usual causes if it cannot- explain any of them. `repaired-query` supplies the original stage indexes and defaults to `pmbql-query`."+ Each distinct problem is listed once, with how many places have it, in [[problem-rank]] order.+ The usual causes are added when [[not-runnable-problem]] cannot explain any of them.+ The optional `repaired-query` supplies the original stage indexes and defaults to `pmbql-query`."([pmbql-query] (query-not-runnable-explanation pmbql-query pmbql-query))([pmbql-query repaired-query](binding [lib.schema.expression/*suppress-expression-type-check?* true](when-let [explanation (mr/explain :metabase.lib.schema/query pmbql-query)]- (let [errors (sort-by (comp nil? not-runnable-problem) (:errors explanation))]- (cond-> (or (some->> errors- (map #(str (repr/query-path (repr/original-query-path repaired-query (:in %))) ": "- (or (not-runnable-problem %) (me/error-message %))))- distinct- (take 3)+ (let [errors (:errors explanation)]+ (cond-> (or (some->> (not-runnable-problem-lines errors repaired-query)seq(str/join "; "))"the query does not match the query builder's schema")diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.cljindex 2b6ac4a7fdf..e3c07c25bac 100644--- a/test/metabase/agent_lib/representations/repair_test.clj+++ b/test/metabase/agent_lib/representations/repair_test.clj@@ -4120,7 +4120,7 @@(orders-stage {"aggregation" [["min" {} (orders-col "TOTAL")]]"breakout" [(orders-col "USER_ID")]}))-(def ^:private user-id-by-name (typed ["field" {} "USER_ID"] "type/Integer"))+(def ^:private user-id-by-name ["field" {} "USER_ID"])(deftest ^:parallel join-alias-field-type-scoped-to-the-stage-with-the-join-test(let [joined-here (query min-total-by-user-stage@@ -4145,6 +4145,162 @@(testing "the ref isn't typed from the previous stage's same-named column, and nothing is raised"(is (= min-ref (get-in (repair* q) ["stages" 1 "filters" 0 2]))))))+(deftest ^:parallel join-stages-field-types-test+ (let [two-stage (fn [column] [first-orders-stage (stage {"filters" [["not-null" {} ["field" {} column]]]})])+ joined (fn [join-stages field-ref] (query (orders-stage {"joins" [(join "Two" join-stages)]+ "filters" [["not-null" {} field-ref]]})))]+ (testing "a by-name ref in a join's second stage is typed, so a ref to the join's column is too"+ (let [out (repair* (joined (two-stage "min") (join-ref "Two" "min")))]+ (is (= "type/Date" (base-type (get-in out ["stages" 0 "joins" 0 "stages" 1 "filters" 0 2]))))+ (is (= "type/Date" (base-type (get-in out ["stages" 0 "filters" 0 2]))))+ (is (= out (repair* out)))))+ (testing "a by-name ref to a nested join's column is typed, and so is the ref to the outer join"+ (let [inner (orders-stage {"joins" [(join "First" [first-orders-stage])]+ "filters" [["not-null" {} (join-ref "First" "min")]]})+ out (repair* (query (orders-stage {"joins" [(join "Outer" [inner])]+ "filters" [["not-null" {} (join-ref "Outer" "TOTAL")]]})))]+ (is (= "type/Date" (base-type (get-in out ["stages" 0 "joins" 0 "stages" 0 "filters" 0 2]))))+ (is (= "type/Float" (base-type (get-in out ["stages" 0 "filters" 0 2]))))))+ (testing "a bad name inside a join is reported with the join it is in"+ (let [e (repair-error (joined (two-stage "nope") (join-ref "Two" "min")))]+ (is (=? {:error :unresolved-cross-stage-field+ :join-alias "Two"+ :join-stage 0+ :stage 1}+ e))+ (is (str/starts-with? (:message e) "In join \"Two\" of stage 0: No column named \"nope\""))))+ (testing "a bad name inside a join with no alias is reported as being in such a join"+ (let [no-alias (-> (join "Two" (two-stage "nope")) (dissoc "alias") (assoc "conditions" []))]+ (is (str/starts-with? (:message (repair-error (query (orders-stage {"joins" [no-alias]}))))+ "In a join with no alias in stage 0: No column named \"nope\""))))))++(deftest ^:parallel join-fields-ref-without-alias-test+ (let [join-fields (fn [previous-stage]+ (get-in (repair* (query previous-stage+ (stage {"joins" [(assoc (join "First" [first-orders-stage] :lhs user-id-by-name)+ "fields" [["field" {} "min"]])]})))+ ["stages" 1 "joins" 0 "fields"]))]+ (testing "a by-name ref in a join's fields gets the join's alias and is typed from the join"+ (is (= [["field" {"join-alias" "First", "base-type" "type/Date", "effective-type" "type/Date"} "min"]]+ (join-fields min-total-by-user-stage))))+ (testing "the previous stage not having the column doesn't matter"+ (is (= "type/Date"+ (base-type (first (join-fields (orders-stage {"aggregation" [["count" {}]]+ "breakout" [(orders-col "USER_ID")]})))))))))++(deftest ^:parallel alias-less-join-fields-left-for-validation-test+ (let [fields-of (fn [column]+ (let [no-alias {"strategy" "left-join"+ "stages" [first-orders-stage]+ "conditions" [["=" {} user-id-by-name (orders-col "USER_ID")]]+ "fields" [["field" {} column]]}]+ (get-in (repair* (query (orders-stage {"aggregation" [["count" {}]]+ "breakout" [(orders-col "USER_ID")]})+ (stage {"joins" [no-alias]})))+ ["stages" 1 "joins" 0 "fields"])))]+ (testing "a by-name ref in an alias-less join's fields stays untyped, whether or not the previous stage has it"+ (doseq [column ["count" "min"]]+ (is (= [["field" {} column]] (fields-of column)))))))++(def ^:private prev-joined-stage+ "Orders with a `Prev` join of each user's first order, returning all its columns."+ (orders-stage {"joins" [(assoc (join "Prev" [first-orders-stage]) "fields" "all")]}))++(deftest ^:parallel previous-stage-ref-in-join-condition-test+ (let [joined-on (fn [lhs] (query first-orders-stage (stage {"joins" [(join "First" [first-orders-stage] :lhs lhs)]})))]+ (testing "a by-name ref to the previous stage in a join's conditions is typed from that stage"+ (is (= "type/Integer"+ (base-type (get-in (repair* (joined-on ["field" {} "USER_ID"])) ["stages" 1 "joins" 0 "conditions" 0 2])))))+ (testing "a bad name there raises the cross-stage error"+ (is (=? {:error :unresolved-cross-stage-field, :clause ["field" {} "nope"]}+ (repair-error (joined-on ["field" {} "nope"])))))+ (testing "a ref there through a join alias the previous stage doesn't have is left for validation"+ (doseq [column ["USER_ID" "nope"]+ :let [field-ref ["field" {"join-alias" "first"} column]]]+ (is (= field-ref (get-in (repair* (joined-on field-ref)) ["stages" 1 "joins" 0 "conditions" 0 2])))))+ (let [prev-joined-on (fn [lhs]+ (query prev-joined-stage (stage {"joins" [(join "First" [first-orders-stage] :lhs lhs)]})))]+ (testing "a ref there through a join of the previous stage is typed from that stage"+ (is (= "type/Date"+ (base-type (get-in (repair* (prev-joined-on ["field" {"join-alias" "Prev"} "min"]))+ ["stages" 1 "joins" 0 "conditions" 0 2])))))+ (testing "a ref there through that join to a column the previous stage deduplicates is typed from the join"+ (is (= "type/Integer"+ (base-type (get-in (repair* (prev-joined-on ["field" {"join-alias" "Prev"} "USER_ID"]))+ ["stages" 1 "joins" 0 "conditions" 0 2])))))+ (testing "a ref there through that join to a column only outside it raises the cross-stage error"+ (let [e (repair-error (prev-joined-on ["field" {"join-alias" "Prev"} "TOTAL"]))]+ (is (=? {:error :unresolved-cross-stage-field+ :clause ["field" {"join-alias" "Prev"} "TOTAL"]+ :available ["USER_ID_2" "min"]}+ e))+ (is (str/starts-with? (:message e) "No column named \"TOTAL\" from join \"Prev\" is available")))))))++(deftest ^:parallel previous-stage-join-alias-ref-test+ (let [filtered-on (fn [field-ref] (query prev-joined-stage (stage {"filters" [["not-null" {} field-ref]]})))]+ (testing "a ref through a join of the previous stage is typed from that join's columns"+ (is (= "type/Date"+ (base-type (get-in (repair* (filtered-on ["field" {"join-alias" "Prev"} "min"]))+ ["stages" 1 "filters" 0 2])))))+ (testing "a ref through that join to a column only outside it raises the cross-stage error"+ (is (=? {:error :unresolved-cross-stage-field, :clause ["field" {"join-alias" "Prev"} "TOTAL"]}+ (repair-error (filtered-on ["field" {"join-alias" "Prev"} "TOTAL"])))))))++(deftest ^:parallel previous-stage-join-alias-deduplicated-aggregation-test+ (let [min-ref (join-ref "Prev" "min")+ q (query min-total-by-user-stage+ (stage {"joins" [(assoc (join "Prev" [first-orders-stage] :lhs user-id-by-name)+ "fields" "all")]})+ (stage {"filters" [["not-null" {} min-ref]]+ "joins" [(assoc-in (join "Next" [first-orders-stage] :lhs min-ref)+ ["conditions" 0 3] (join-ref "Next" "min"))]}))+ out (repair* q)]+ (testing "a deduplicated join column is typed through its alias in filters and join conditions"+ (doseq [path [["filters" 0 2] ["joins" 0 "conditions" 0 2]]]+ (is (=? (typed min-ref "type/Date")+ (get-in out (into ["stages" 2] path))))))+ (testing "a quoted name is unquoted without adding the join prefix to the ref"+ (is (= out (repair* (assoc-in q ["stages" 2 "filters" 0 2 2] "\"min\"")))))+ (is (= out (repair* out)))))++(deftest ^:parallel previous-stage-join-alias-prefixed-name-precedence-test+ (let [joined (update first-orders-stage "aggregation" conj ["max" {"name" "min_2"} (orders-col "TOTAL")])+ q (query min-total-by-user-stage+ (stage {"joins" [(assoc (join "Prev" [joined] :lhs user-id-by-name) "fields" "all")]})+ (stage {"filters" [["not-null" {} (join-ref "Prev" "min_2")]]}))]+ (testing "the joined min_2 is a Float, even though the joined Date min is deduplicated to min_2"+ (is (= "type/Float" (base-type (get-in (repair* q) ["stages" 2 "filters" 0 2])))))))++(deftest ^:parallel previous-stage-join-alias-display-name-collision-test+ (let [joined (update first-orders-stage "aggregation" conj ["max" {"name" "min_2"} (orders-col "TOTAL")])]+ (doseq [column-name ["Min of Created At" "min-of-created-at"]]+ (let [q (query min-total-by-user-stage+ (stage {"joins" [(assoc (join "Prev" [joined] :lhs user-id-by-name) "fields" "all")]})+ (stage {"filters" [["not-null" {} (join-ref "Prev" column-name)]]}))+ out (repair* q)]+ (is (=? (typed (join-ref "Prev" "min") "type/Date") (get-in out ["stages" 2 "filters" 0 2])))+ (is (= out (repair* out)))))))++(deftest ^:parallel previous-stage-join-alias-custom-display-name-test+ (let [joined (assoc-in prev-joined-stage ["joins" 0 "stages" 0 "breakout" 0 1 "display-name"] "Account key")+ q (query joined (stage {"filters" [["not-null" {} (join-ref "Prev" "USER_ID")]]}))]+ (is (= "type/Integer" (base-type (get-in (repair* q) ["stages" 1 "filters" 0 2]))))))++(deftest ^:parallel previous-stage-join-alias-nested-column-test+ (let [self-joined [(orders-stage {"joins" [(assoc (join "Self" [(orders-stage {})]) "fields" "all")]})]+ field-ref (join-ref "Outer" "Self__TOTAL")+ q (query (orders-stage {"joins" [(assoc (join "Outer" self-joined) "fields" "all")]})+ (stage {"filters" [["not-null" {} field-ref]]}))+ out (repair* q)]+ (is (=? (typed field-ref "type/Float")+ (get-in out ["stages" 1 "filters" 0 2])))+ (is (= out (repair* out)))))++(deftest ^:parallel unresolved-cross-stage-ref-with-no-columns-test+ (testing "a ref without a join alias is reported when the previous stage returns no columns"+ (is (= ["field" {} "nope"]+ (#'repair/first-unresolved-cross-stage-ref {"filters" [["not-null" {} ["field" {} "nope"]]]} [])))))+(deftest ^:parallel join-alias-field-type-runs-before-cross-stage-inference-test(testing "a stage carrying a join-alias ref still resolves as the next stage's prefix"(let [out (repair* (query (orders-stage {"joins" [(join "First" [first-orders-stage])]@@ -4213,6 +4369,23 @@(base-type (get-in (repair* (first-orders-filtering (typed (join-ref "First" "min") "type/Text")))["stages" 0 "filters" 0 2]))))))+(deftest ^:parallel join-alias-field-type-nested-join-column-names-test+ (let [self-joined [(orders-stage {"joins" [(assoc (join "Self" [(orders-stage {})]) "fields" "all")]})]+ filtering (fn [field-ref] (query (orders-stage {"joins" [(join "Outer" self-joined)]+ "filters" [["not-null" {} field-ref]]})))+ filtered (fn [field-ref] (get-in (repair* (filtering field-ref)) ["stages" 0 "filters" 0 2]))]+ (testing "a nested join's column is typed by each name lib resolves, and kept as written"+ (doseq [column-name ["Self__TOTAL" "TOTAL_2"]]+ (is (= (typed (join-ref "Outer" column-name) "type/Float")+ (filtered (join-ref "Outer" column-name))))))+ (testing "a quoted desired alias is unquoted and typed"+ (is (= (typed (join-ref "Outer" "Self__TOTAL") "type/Float")+ (filtered (join-ref "Outer" "\"Self__TOTAL\"")))))+ (testing "a ref the LLM typed itself by the desired alias is accepted"+ (is (= "type/Number" (base-type (filtered (typed (join-ref "Outer" "Self__TOTAL") "type/Number"))))))+ (testing "a name the join doesn't return is still reported"+ (is (= [[0 "Outer" ["Self__NOPE"]]] (join-problems (filtering (join-ref "Outer" "Self__NOPE"))))))))+(deftest ^:parallel join-alias-field-type-unknown-column-quotes-available-columns-test(testing "the names of the columns a join returns reach the message quoted"(let [narrow (orders-stage {"fields" [(orders-col "USER_ID") (orders-col odd-column-name)]})diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.cljindex 1eb6a34b98b..1174432e91a 100644--- a/test/metabase/metabot/tools/construct_representations_test.clj+++ b/test/metabase/metabot/tools/construct_representations_test.clj@@ -3327,6 +3327,23 @@(is (=? {:agent-error? true :error :query-not-runnable :status-code 400} (ex-data e)))(is (str/includes? (ex-message e) "could not be resolved")))))))+(deftest nested-expression-unresolved-reference-explained-test+ (with-mp-and-stubs!+ (fn []+ (let [q {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "expressions" {"Broken" ["+" {}+ ["length" {}+ ["replace" {}+ ["substring" {} ["field" {"join-alias" "Nope"} "NAME"] 1 10]+ "-" ""]]+ 1]}}]}+ e (editor-rejection q)]+ (is (=? {:agent-error? true, :error :query-not-runnable} (ex-data e)))+ (is (re-find #"Failing paths: [^;]+could not be resolved" (ex-message e)))))))+(deftest path-ref-to-dropped-join-column-rejected-before-sql-test(testing "a path ref to a column the grouped join drops fails in repair, not in the database"(with-mp-and-stubs!@@ -3354,11 +3371,22 @@hint #"\. This usually means a field reference is missing its type"](testing "identical problems are listed once"(is (= "stages[0].alias: missing `alias`" (explain (missing :alias) (missing :alias)))))- (testing "at most three problems are listed"- (is (= 3 (count (str/split (apply explain (map missing [:a :b :c :d])) #"; ")))))+ (testing "one problem in several places is listed once, with a count of the others"+ (is (str/starts-with? (explain (invalid :a) (invalid :b) (invalid :c))+ "stages[0].a (and 2 other places): should be an integer.")))+ (testing "at most three problems are listed, and the rest are counted"+ (is (str/ends-with? (apply explain (map missing [:a :b :c :d :e])) "; and 2 more problems")))(testing "problems with a known cause are listed before Malli's own messages"(is (str/starts-with? (explain (invalid :a) (invalid :b) (invalid :c) (missing :alias))- "stages[0].alias: missing `alias`; stages[0].a: should be an integer")))+ "stages[0].alias: missing `alias`; stages[0].a (and 2 other places): should be an integer")))+ (testing "Malli's own messages are listed before a guessed cause, however often it repeats"+ (let [untyped (fn [i] {:type :malli.core/missing-key, :in [:stages 0 :filters i 2 1 :base-type]})+ no-join {:schema (mc/schema [:fn {:error/message "no join named \"first\""} any?])+ :value 1+ :in [:stages]}]+ (is (str/starts-with? (apply explain (conj (mapv untyped (range 4)) no-join))+ (str "stages: no join named \"first\"; "+ "stages[0].filters[0][2][1].base-type (and 3 other places): a field reference by name is missing `base-type`")))))(testing "the usual causes are added only when no problem has a known cause"(is (re-find hint (explain (invalid :a))))(is (not (re-find hint (explain (invalid :a) (missing :alias))))))