Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Give path refs in a join's fields the join's alias (#85805)

Chris Truterc5cdc7a2ffac
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.clj
index 04821d8bd29..c53e4fc90f1 100644
--- a/src/metabase/agent_lib/representations/repair.clj
+++ b/src/metabase/agent_lib/representations/repair.clj
@@ -2656,7 +2656,7 @@
mp import-resolver export-resolver))))
;;; ============================================================
-;;; Pass 4 -- infer base-type / effective-type on cross-stage field references
+;;; Passes 4.5 and 5 -- infer base-type / effective-type on cross-stage field references
;;;
;;; A cross-stage field reference looks like `["field" {opts} "<column-name>"]` where the
;;; third element is a **string** (the column's name in the previous stage's output) rather
@@ -2763,15 +2763,19 @@
:types (types-from-column col)})
cols))
+(defn- stages-query
+ "Return `stages` as a standalone query against `database`, which may be nil."
+ [database stages]
+ (cond-> {"lib/type" "mbql/query", "stages" stages}
+ database (assoc "database" database)))
+
(defn- resolved-columns
"Resolve `stages` as a standalone query against `database` and return their returned columns as
a vector of [[column-descriptors]], or nil when anything in the resolve path throws. `what`
names the stages in the debug log."
[mp database stages content-store what]
(try
- (let [query (cond-> {"lib/type" "mbql/query" "stages" stages}
- database (assoc "database" database))
- resolved (repr.resolve/resolve-query mp query content-store)]
+ (let [resolved (repr.resolve/resolve-query mp (stages-query database stages) content-store)]
(column-descriptors (lib/returned-columns (lib/query mp resolved))))
(catch Exception e
(log/debugf "[repr-repair] resolve of %s failed; skipping field-type inference: %s" what (ex-message e))
@@ -2817,9 +2821,12 @@
1. an exact match on a name lib resolves: the desired alias, then the deduplicated name, then `:name`.
The ref keeps its spelling, since a nested join's column is `P__NAME` to lib even when its `:name` is `NAME`;
2. the same match with surrounding double quotes stripped, giving the stripped spelling;
- 3. a loose `:name` match that folds case and hyphens/whitespace to underscores, but **only
- when exactly one** real column matches - so `count-where` resolves to `count_where`;
- 4. a loose `:display-name` match (same folding, same one-hit guard) - so a ref the LLM
+ 3. with `join-alias`, a loose match on the desired alias without its `join-alias__` prefix, which folds case and
+ hyphens/whitespace to underscores, but **only when exactly one** real column matches - so `Sum` finds a
+ joined `sum` that the previous stage deduplicated to `sum_2`.
+ It comes first because, as with exact matches, the join's own name wins over a deduplicated `:name`;
+ 4. the same loose match on `:name` - so `count-where` resolves to `count_where`;
+ 5. a loose `:display-name` match (same folding, same one-hit guard) - so a ref the LLM
wrote with the UI label (`\"Max of Instance Has SC\"`) recovers the machine name (`max`).
Genuine ambiguity (more than one loose hit) is left unmatched at each layer.
@@ -2829,52 +2836,51 @@
([cols col-name]
(match-cross-stage-column cols col-name nil))
([cols col-name join-alias]
- (let [stripped (strip-surrounding-double-quotes col-name)
- find-on (fn [k n] (some #(when (= n (k %)) %) cols))
- exact (fn [n] (some #(find-on % n) [:desired-alias :deduplicated-name :name]))
- lib-match (fn [n] (or (when join-alias (exact (str join-alias "__" n)))
- (exact n)))
- loose-on (fn [key-fn raw]
- (let [target (normalize-col-key raw)
- hits (when target
- (filter #(= target (normalize-col-key (key-fn %))) cols))]
- (when (= 1 (count hits))
- (first hits))))]
- (if-let [[spelling match] (or (some->> (lib-match col-name) (vector col-name))
- (when (not= stripped col-name)
- (some->> (lib-match stripped) (vector stripped))))]
+ (let [stripped (strip-surrounding-double-quotes col-name)
+ find-on (fn [k n] (some #(when (= n (k %)) %) cols))
+ exact (fn [n] (some #(find-on % n) [:desired-alias :deduplicated-name :name]))
+ prefix (when join-alias (str join-alias "__"))
+ unprefixed (fn [col]
+ (let [desired-alias (:desired-alias col)]
+ (when (and prefix desired-alias (str/starts-with? desired-alias prefix))
+ (subs desired-alias (count prefix)))))
+ lib-match (fn [n] (or (when prefix (exact (str prefix n)))
+ (exact n)))
+ loose-on (fn [key-fn raw]
+ (let [target (normalize-col-key raw)
+ hits (when target
+ (filter #(= target (normalize-col-key (key-fn %))) cols))]
+ (when (= 1 (count hits))
+ (first hits))))]
+ (if-let [[spelling match] (some #(when-let [match (lib-match %)] [% match])
+ (distinct [col-name stripped]))]
[spelling (:types match)]
- (when-let [match (or (loose-on :name stripped)
+ (when-let [match (or (loose-on unprefixed stripped)
+ (loose-on :name stripped)
(loose-on :display-name stripped))]
- (let [desired-alias (:desired-alias match)
- prefix (when join-alias (str join-alias "__"))
- spelling (if (and prefix desired-alias (str/starts-with? desired-alias prefix))
- (subs desired-alias (count prefix))
- (:name match))]
+ (let [spelling (or (unprefixed match) (:name match))]
;; A deduplicated name can denote another column after lib adds the join prefix.
(when (or (nil? join-alias) (= match (lib-match spelling)))
[spelling (:types match)])))))))
(defn- maybe-fill-cross-stage-types
- "Given a cross-stage field clause and a name→types map, return the clause with its column name
- canonicalised (surrounding double-quotes stripped when that is what makes it match a real
- column) and `base-type` / `effective-type` filled in (when missing and known).
+ "Return the by-name field `clause` with the spelling [[match-cross-stage-column]] finds for it in `cols`,
+ and with `base-type` / `effective-type` filled in when missing and known.
+ Returns `clause` unchanged when no column matches.
- Idempotent: once the name is canonical it matches exactly on the next pass (no rename), and
- once `base-type` is present it is left alone.
+ Idempotent: once renamed, the ref matches exactly on the next pass, and a present `base-type` is kept.
The optional `join-alias` enables previous-stage join lookup in [[match-cross-stage-column]]."
([clause cols]
(maybe-fill-cross-stage-types clause cols nil))
([clause cols join-alias]
(let [opts (nth clause 1)
col-name (nth clause 2)]
- (if-let [[canonical-name types] (match-cross-stage-column cols col-name join-alias)]
+ (if-let [[spelling types] (match-cross-stage-column cols col-name join-alias)]
(cond-> clause
- ;; Canonicalise the name when quote-stripping / display-name recovery was needed to
- ;; match. A bare `base-type` stamp without this would leave the ref pointing at a
- ;; non-existent column.
- (not= canonical-name col-name)
- (assoc 2 canonical-name)
+ ;; Rename the ref when it matched only after quote-stripping or loose matching.
+ ;; A `base-type` stamp alone would leave it pointing at a column that doesn't exist.
+ (not= spelling col-name)
+ (assoc 2 spelling)
;; Stamp inferred types only when the LLM didn't author a `base-type` already and the
;; matched column actually exposes one.
(and types (not (contains? opts "base-type")))
@@ -2955,7 +2961,13 @@
;;; query, and its returned columns both type the string refs and check every join-alias ref
;;; against what the join actually returns. A concrete-column ref to a column a grouped join
;;; doesn't return, or a string ref the LLM typed itself with a name the join doesn't return,
-;;; passes lib and only fails in SQL, so both are caught here too. A ref whose alias
+;;; passes lib and only fails in SQL, so both are caught here too. A plain table join returns
+;;; every column of its table, so a concrete-column ref resolves it only when the ref names another
+;;; table's column. A string ref resolves it as it would any join.
+;;; Lib compiles such a ref against the joined table's column of the same name if there is one,
+;;; giving a wrong answer, and the database rejects it otherwise.
+;;; The refs in a join's own `fields` get its alias first, so they are typed and checked the same way.
+;;; A ref whose alias
;;; names no join on its stage is a previous-stage ref, typed and checked by Passes 4.5 / 5 / 5.7;
;;; refs whose join or field can't be resolved are left for the resolver to report.
;;; ============================================================
@@ -2985,6 +2997,15 @@
(contains? (first stages) "source-table")
(not-any? #(contains? (first stages) %) ["aggregation" "breakout" "fields"]))))
+(defn- other-table-path-refs
+ "The `path-refs` into plain-table `join` whose field belongs to another table, or nil if there are none.
+ A ref is skipped when its field or the join's table can't be resolved."
+ [mp import-resolver join path-refs]
+ (when-let [table-id (try-resolve-source-table-id import-resolver (get-in join ["stages" 0 "source-table"]))]
+ (not-empty (filterv #(when-let [field-table-id (try-resolve-field-target-table-id mp import-resolver (nth % 2))]
+ (not= field-table-id table-id))
+ path-refs))))
+
(defn- join-alias-refs
"`stage`'s join-alias string refs (`:named`) and concrete-column join-alias refs (`:path`), each
grouped by alias, outside the joins' [[hidden-join-keys]]."
@@ -2995,22 +3016,21 @@
:path (group-by alias (match/match-many stage' (node :guard (join-alias-path-field-clause? node)) node))}))
(defn- bad-join-alias-refs
- "The distinct `named-refs` naming no column in `cols`, followed by the distinct `path-refs` whose
- field matches no column of `cols` by id, or by name among the columns without an id. A path ref
- whose field can't be resolved is skipped."
+ "Returns `{:named [...], :path [...]}`: the distinct `named-refs` naming no column in `cols`, and the distinct
+ `path-refs` whose field matches no column of `cols` by id, or by name among the columns without an id.
+ A path ref whose field can't be resolved is skipped."
[mp cols named-refs path-refs import-resolver]
(let [returned-ids (set (keep :id cols))
- id-less-names (set (keep #(when-not (:id %) (:name %)) cols))]
- (distinct
- (concat
- (filter #(nil? (match-cross-stage-column cols (nth % 2))) named-refs)
- (filter #(when-let [field (some->> (try-resolve-field-id @import-resolver (nth % 2))
- (lib.metadata.protocols/field mp))]
- ;; like lib, fall back to the field's name only for a column without an id, e.g.
- ;; one returned through a native card
- (and (not (contains? returned-ids (:id field)))
- (not (contains? id-less-names (:name field)))))
- path-refs)))))
+ id-less-names (set (keep #(when-not (:id %) (:name %)) cols))
+ bad-named? #(nil? (match-cross-stage-column cols (nth % 2)))
+ bad-path? #(when-let [field (some->> (try-resolve-field-id @import-resolver (nth % 2))
+ (lib.metadata.protocols/field mp))]
+ ;; like lib, fall back to the field's name only for a column without an id, e.g. one
+ ;; returned through a native card
+ (and (not (contains? returned-ids (:id field)))
+ (not (contains? id-less-names (:name field)))))]
+ {:named (into [] (comp (filter bad-named?) (distinct)) named-refs)
+ :path (into [] (comp (filter bad-path?) (distinct)) path-refs)}))
(defn- field-target-name
"The column a field ref's target denotes: the name itself, or `TABLE.FIELD` for a portable path
@@ -3030,14 +3050,24 @@
joins (get stage "joins")
checks (vec (for [join (when (and (or (seq named) (seq path)) (sequential? joins)) joins)
:let [alias (when (map? join) (get join "alias"))
+ plain? (plain-table-join? join)
named-refs (get named alias)
- path-refs (when-not (plain-table-join? join) (get path alias))]
+ path-refs (cond->> (get path alias)
+ ;; a plain table join returns all its table's columns, so it
+ ;; is resolved only for a ref to another table's column
+ plain?
+ (other-table-path-refs mp @import-resolver join))]
:when (or named-refs path-refs)
:let [cols (when (vector? (get join "stages"))
(resolved-columns mp (get query "database") (get join "stages") content-store
(format "join %s" (pr-str alias))))]
:when cols]
- {:alias alias :cols cols :named-refs named-refs :path-refs path-refs}))
+ {:alias alias
+ :cols cols
+ :named-refs named-refs
+ :path-refs path-refs
+ :plain? plain?
+ :field-refs (set (when (sequential? (get join "fields")) (get join "fields")))}))
alias->cols (into {} (map (juxt :alias :cols)) checks)]
{:stage (cond->> stage
(seq alias->cols)
@@ -3047,34 +3077,62 @@
(alias->cols (get (nth node 1) "join-alias")))]
(maybe-fill-cross-stage-types node cols)
node))))
- :problems (vec (for [{:keys [alias cols named-refs path-refs]} checks
- :let [bad (bad-join-alias-refs mp cols named-refs path-refs import-resolver)]
+ :problems (vec (for [{:keys [alias cols named-refs path-refs plain? field-refs]} checks
+ :let [{bad-named :named, bad-path :path}
+ (bad-join-alias-refs mp cols named-refs path-refs import-resolver)
+ bad (into bad-named bad-path)]
:when (seq bad)]
- {:stage (repr/original-stage-index query idx)
- :join-alias alias
- :clauses (vec bad)
- :columns (vec (distinct (map #(field-target-name mp (nth % 2)) bad)))
- :available (mapv :name cols)}))}))
+ ;; a plain table join's path refs are all to other tables, see `path-refs` above
+ {:stage (repr/original-stage-index query idx)
+ :join-alias alias
+ :clauses bad
+ :columns (vec (distinct (map #(field-target-name mp (nth % 2)) bad)))
+ :available (mapv :name cols)
+ :named? (boolean (seq bad-named))
+ :plain? plain?
+ :other-table? (boolean (and plain? (some (complement field-refs) bad-path)))
+ :other-table-in-fields? (boolean (and plain? (some field-refs bad-path)))}))}))
(defn- unresolved-join-alias-fields-error
+ "The `:agent-error?` for `problems`, the join-alias refs naming columns their joins don't return.
+ Its message lists them, then gives the advice each problem's `:named?`, `:plain?`, `:other-table?` and
+ `:other-table-in-fields?` call for."
[problems]
+ ;; Only the advice that applies: a ref to another table through a plain table join has nothing to do with display
+ ;; labels or grouped joins. Outside the join, repair routes that ref through a foreign key once `join-alias` goes, or
+ ;; lib does when given both `source-field` options. A join's own `fields` can't take another table's column at all:
+ ;; repair puts the alias back, and lib rejects an implicitly joined column there.
(ex-info
- (format "Some join column references name a column their join doesn't return: %s. Reference a join column by its machine name (e.g. an aggregation is `min`, `count`, `sum` - never its display label like `Min of X`), and only columns the join returns: a grouped join returns just its breakouts and aggregations."
- (str/join "; " (for [{:keys [stage join-alias columns available]} problems]
- (format "stage %s, join %s doesn't return %s (available: %s)"
- stage
- (quoting/quoted join-alias)
- (str/join ", " (map quoting/quoted columns))
- (str/join ", " (map quoting/quoted available))))))
+ (str (format "Some join column references name a column their join doesn't return: %s."
+ (str/join "; " (for [{:keys [stage join-alias columns available]} problems]
+ (format "stage %s, join %s doesn't return %s (available: %s)"
+ stage
+ (quoting/quoted join-alias)
+ (str/join ", " (map quoting/quoted columns))
+ (str/join ", " (map quoting/quoted available))))))
+ (when (some :named? problems)
+ (str " Reference a join column by its machine name (e.g. an aggregation is `min`, `count`, `sum` - never its"
+ " display label like `Min of X`)."))
+ (when (some (complement :plain?) problems)
+ " Reference only columns the join returns: a grouped join returns just its breakouts and aggregations.")
+ (when (some #(or (:other-table? %) (:other-table-in-fields? %)) problems)
+ " A join of a single table returns only that table's columns.")
+ (when (some :other-table? problems)
+ (str " To use another table's column, drop `join-alias` so the table is reached through a foreign key, or set"
+ " both `source-field` (the foreign key column on the joined table) and `source-field-join-alias` (the"
+ " join's alias)."))
+ (when (some :other-table-in-fields? problems)
+ " To show another table's column, add a join on that table and list the column in that join's `fields`."))
{:agent-error? true
:error :unresolved-join-alias-field
:problems problems}))
(defn- repair-join-alias-refs*
- "Pass 4.7: type the join-alias string field refs of every stage from the referenced joins'
- returned columns, and raise one `:agent-error?` listing every join-alias ref, across all stages,
- that names a column its join doesn't return. Only joins named by a string ref, or by a
- concrete-column ref into a join that isn't a [[plain-table-join?]], are resolved."
+ "Pass 4.7: type the join-alias string field refs of every stage from the referenced joins' returned columns.
+ Raises one `:agent-error?` listing every join-alias ref, across all stages, that names a column its join doesn't
+ return.
+ Only joins named by a string ref or a concrete-column ref are resolved.
+ A concrete-column ref resolves a [[plain-table-join?]] only when it names another table's column."
[query mp content-store]
(if-not (and mp (map? query) (vector? (get query "stages")))
query
@@ -3113,7 +3171,7 @@
;;; Pass 5.7 -- assert cross-stage / source-card string refs resolved
;;;
;;; A `["field" {} "<name>"]` ref into a previous stage or a source-card must name a real
-;;; output column. The inference passes above (5 / 5.5) stamp `base-type` onto every such ref
+;;; output column. The inference passes above (4.5 / 5) stamp `base-type` onto every such ref
;;; that matches; anything still missing `base-type` afterward matched nothing - the LLM
;;; named a column that doesn't exist (typically a UI display label like "Max of X" instead of
;;; the machine name "max"). lib's resolver passes a typeless string-named field ref through
@@ -3197,8 +3255,7 @@
(if-not (and (map? join) (vector? (get join "stages")))
join
(try
- (assoc join "stages" (-> (cond-> {"lib/type" "mbql/query", "stages" (get join "stages")}
- database (assoc "database" database))
+ (assoc join "stages" (-> (stages-query database (get join "stages"))
(infer-field-types* mp content-store)
(get "stages")))
(catch clojure.lang.ExceptionInfo e
@@ -3213,15 +3270,14 @@
(throw e))))))
(defn- with-alias-on-join-fields
- "Return `join` with its alias on each by-name field ref in its `fields`, as lib's join normalization does.
- Leaves path refs alone."
+ "Return `join` with its alias on each field ref in its `fields`.
+ Lib's join normalization later adds the alias to every clause there."
[join]
(let [alias (when (map? join) (get join "alias"))]
(cond-> join
(and (non-blank-string? alias) (sequential? (get join "fields")))
(update "fields" (partial mapv #(cond-> %
- ;; no pass mistakes a path ref for a previous-stage ref, so lib aliases those
- (match/matches? % ["field" (_ :guard map?) (_ :guard string?)])
+ (match/matches? % ["field" (_ :guard map?) _])
(assoc-in [1 "join-alias"] alias)))))))
(defn- infer-join-field-types-in-stage
@@ -3578,7 +3634,9 @@
`base-type` before that stage can be mini-resolved as part of a later stage's prefix.
4.7. infer `base-type` / `effective-type` on join-alias string field references from the
referenced join's own resolved returned columns, and raise an `:agent-error?` for a
- join-alias ref naming a column its join doesn't return. Runs before Pass 5 so a stage
+ join-alias ref naming a column its join doesn't return, including a concrete-column ref
+ to another table through a plain table join. The refs in a join's own `fields` get the
+ join's alias first, so they are checked too. Runs before Pass 5 so a stage
carrying such a ref can still be mini-resolved as a prefix.
5. infer `base-type` / `effective-type` on cross-stage field references
(`[\"field\" {} \"<column-name>\"]` in a non-first stage), by mini-resolving the
diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.clj
index ebf05a2159a..7b1164d9782 100644
--- a/src/metabase/metabot/tools/construct.clj
+++ b/src/metabase/metabot/tools/construct.clj
@@ -979,14 +979,15 @@
:problem (or (not-runnable-problem error) (me/error-message error))})
paths-of (group-by :problem located)
lines (for [problem (distinct (map :problem located))
- :let [[path & others] (distinct (map :path (paths-of problem)))]]
+ :let [[path & others] (distinct (map :path (paths-of problem)))
+ n-others (count others)]]
(str path
(when others
- (format " (and %d other place%s)" (count others) (if (= 1 (count others)) "" "s")))
+ (format " (and %d other %s)" n-others (u/format-plural n-others "place")))
": " problem))
omitted (- (count lines) max-not-runnable-problems)]
(cond-> (vec (take max-not-runnable-problems lines))
- (pos? omitted) (conj (format "and %d more problem%s" omitted (if (= 1 omitted) "" "s"))))))
+ (pos? omitted) (conj (format "and %d more %s" omitted (u/format-plural omitted "problem"))))))
(defn- query-not-runnable-explanation
"Explain why `pmbql-query` fails the query builder's `canRun` check, or return nil.
diff --git a/src/metabase/metabot/tools/recoverable/pipeline.clj b/src/metabase/metabot/tools/recoverable/pipeline.clj
index 74337d35279..69e446891f7 100644
--- a/src/metabase/metabot/tools/recoverable/pipeline.clj
+++ b/src/metabase/metabot/tools/recoverable/pipeline.clj
@@ -267,8 +267,8 @@
(defpipeline-error unresolved-join-alias-field!
"A reference through a join alias named a column that join does not return."
- ;; No step: the message already lists the columns the join returns, and for a grouped join those
- ;; are its breakouts and aggregations, not a table's fields.
+ ;; No step: the message already lists the columns the join returns, and says how to reach another
+ ;; table's column through a plain table join.
[])
(defrecoverable column-not-returned!
diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.clj
index e3c07c25bac..f06afa47951 100644
--- a/test/metabase/agent_lib/representations/repair_test.clj
+++ b/test/metabase/agent_lib/representations/repair_test.clj
@@ -1535,7 +1535,7 @@
"source-table" ["Sample" "PUBLIC" "ORDERS"]
"joins" [join]}]}
output (repair/repair trivial-mp input)]
- (is (= [field] (get-in output ["stages" 0 "joins" 0 "fields"])))))
+ (is (= [(assoc-in field [1 "join-alias"] "P")] (get-in output ["stages" 0 "joins" 0 "fields"])))))
(testing "join `fields:` enum value (\"all\" / \"none\") is left alone"
(doseq [enum-value ["all" "none"]]
(let [stage {"lib/type" "mbql.stage/mbql"
@@ -4175,18 +4175,26 @@
"In a join with no alias in stage 0: No column named \"nope\""))))))
(deftest ^:parallel join-fields-ref-without-alias-test
- (let [join-fields (fn [previous-stage]
- (get-in (repair* (query previous-stage
- (stage {"joins" [(assoc (join "First" [first-orders-stage] :lhs user-id-by-name)
- "fields" [["field" {} "min"]])]})))
- ["stages" 1 "joins" 0 "fields"]))]
+ (let [fields-query (fn [previous-stage fields]
+ (query previous-stage
+ (stage {"joins" [(assoc (join "First" [first-orders-stage] :lhs user-id-by-name)
+ "fields" fields)]})))
+ join-fields (fn [previous-stage fields]
+ (get-in (repair* (fields-query previous-stage fields)) ["stages" 1 "joins" 0 "fields"]))]
(testing "a by-name ref in a join's fields gets the join's alias and is typed from the join"
(is (= [["field" {"join-alias" "First", "base-type" "type/Date", "effective-type" "type/Date"} "min"]]
- (join-fields min-total-by-user-stage))))
+ (join-fields min-total-by-user-stage [["field" {} "min"]]))))
(testing "the previous stage not having the column doesn't matter"
(is (= "type/Date"
(base-type (first (join-fields (orders-stage {"aggregation" [["count" {}]]
- "breakout" [(orders-col "USER_ID")]})))))))))
+ "breakout" [(orders-col "USER_ID")]})
+ [["field" {} "min"]]))))))
+ (testing "a path ref in a join's fields gets the join's alias too"
+ (is (= [(assoc-in (orders-col "USER_ID") [1 "join-alias"] "First")]
+ (join-fields min-total-by-user-stage [(orders-col "USER_ID")]))))
+ (testing "a path ref there to a column the grouped join doesn't return is reported"
+ (is (= [[1 "First" ["ORDERS.TOTAL"]]]
+ (join-problems (fields-query min-total-by-user-stage [(orders-col "TOTAL")])))))))
(deftest ^:parallel alias-less-join-fields-left-for-validation-test
(let [fields-of (fn [column]
@@ -4263,13 +4271,27 @@
(is (= out (repair* (assoc-in q ["stages" 2 "filters" 0 2 2] "\"min\"")))))
(is (= out (repair* out)))))
+(deftest ^:parallel previous-stage-join-alias-deduplicated-loose-name-test
+ (let [q (query min-total-by-user-stage
+ (stage {"joins" [(assoc (join "Prev" [first-orders-stage] :lhs user-id-by-name) "fields" "all")]})
+ (stage {"filters" [["not-null" {} (join-ref "Prev" "Min")]]}))
+ out (repair* q)]
+ (testing "a loosely spelled ref to a deduplicated join column matches the name after the join prefix"
+ (is (=? (typed (join-ref "Prev" "min") "type/Date") (get-in out ["stages" 2 "filters" 0 2]))))
+ (is (= out (repair* out)))))
+
(deftest ^:parallel previous-stage-join-alias-prefixed-name-precedence-test
(let [joined (update first-orders-stage "aggregation" conj ["max" {"name" "min_2"} (orders-col "TOTAL")])
q (query min-total-by-user-stage
(stage {"joins" [(assoc (join "Prev" [joined] :lhs user-id-by-name) "fields" "all")]})
(stage {"filters" [["not-null" {} (join-ref "Prev" "min_2")]]}))]
(testing "the joined min_2 is a Float, even though the joined Date min is deduplicated to min_2"
- (is (= "type/Float" (base-type (get-in (repair* q) ["stages" 2 "filters" 0 2])))))))
+ (is (= "type/Float" (base-type (get-in (repair* q) ["stages" 2 "filters" 0 2])))))
+ (testing "a loose spelling of min_2 also finds the Float"
+ (doseq [column-name ["Min_2" "MIN-2"]
+ :let [loose (assoc-in q ["stages" 2 "filters" 0 2 2] column-name)]]
+ (is (=? (typed (join-ref "Prev" "min_2") "type/Float")
+ (get-in (repair* loose) ["stages" 2 "filters" 0 2])))))))
(deftest ^:parallel previous-stage-join-alias-display-name-collision-test
(let [joined (update first-orders-stage "aggregation" conj ["max" {"name" "min_2"} (orders-col "TOTAL")])]
@@ -4332,6 +4354,67 @@
(query (orders-stage {"joins" [(join "Plain" [(orders-stage {})])]
"filters" [["not-null" {} (join-ref "Plain" (conj orders "TOTAL"))]]}))))))))
+(def ^:private people ["Sample" "PUBLIC" "PEOPLE"])
+
+(def ^:private people-id (conj people "ID"))
+
+(def ^:private plain-join (join "Plain" [(orders-stage {})]))
+
+(defn- plain-filtering [target]
+ (query (orders-stage {"joins" [plain-join]
+ "filters" [["not-null" {} (join-ref "Plain" target)]]})))
+
+(defn- advice-flags
+ "The flags set on each problem in the join-alias error `q`'s repair raises, which pick the advice in its message."
+ [q]
+ (mapv #(set (filter % [:named? :plain? :other-table? :other-table-in-fields?])) (:problems (repair-error q))))
+
+(defn- plain-join-fields [target]
+ (query (orders-stage {"joins" [(assoc plain-join "fields" [["field" {} target]])]})))
+
+(deftest ^:parallel plain-table-join-other-table-path-ref-test
+ (testing "a path ref through a plain table join to another table's column of the same name raises"
+ (is (= [[0 "Plain" ["PEOPLE.ID"]]] (join-problems (plain-filtering people-id)))))
+ (testing "a path ref there in the join's own fields raises too"
+ (is (= [[0 "Plain" ["PEOPLE.ID"]]] (join-problems (plain-join-fields people-id)))))
+ (testing "a path ref to the joined table's own column is accepted"
+ (is (nil? (repair-error (plain-filtering (conj orders "ID"))))))
+ (testing "a path ref to a column of a join nested in the joined stage is accepted"
+ (let [inner (assoc (join "People" [(stage {"source-table" people})])
+ "conditions" [["=" {} (orders-col "USER_ID") (join-ref "People" people-id)]]
+ "fields" "all")
+ outer (join "Outer" [(orders-stage {"joins" [inner]})])]
+ (is (nil? (repair-error (query (orders-stage {"joins" [outer]
+ "filters" [["not-null" {} (join-ref "Outer" people-id)]]}))))))))
+
+(deftest ^:parallel join-alias-field-advice-test
+ (let [message (comp :message repair-error)]
+ (testing "a ref to another table through a plain table join is told how to reach that table, and nothing else"
+ (let [q (plain-filtering people-id)
+ m (message q)]
+ (is (= [#{:plain? :other-table?}] (advice-flags q)))
+ (is (str/includes? m "drop `join-alias`"))
+ (is (str/includes? m "set both `source-field`"))
+ (is (not-any? #(str/includes? m %) ["display label" "grouped join" "add a join"]))))
+ (testing "a ref to another table in a plain table join's own fields is told to join that table"
+ (let [q (plain-join-fields people-id)
+ m (message q)]
+ (is (= [#{:plain? :other-table-in-fields?}] (advice-flags q)))
+ (is (str/includes? m "add a join on that table"))
+ (is (not-any? #(str/includes? m %) ["drop `join-alias`" "display label" "grouped join"]))))
+ (testing "a bad name through a plain table join gets only the advice about names"
+ (let [q (plain-filtering "nope")
+ m (message q)]
+ (is (= [#{:named? :plain?}] (advice-flags q)))
+ (is (str/includes? m "display label"))
+ (is (not-any? #(str/includes? m %) ["grouped join" "source-field"]))))
+ (testing "problems in a grouped and a plain table join get the advice for each"
+ (let [q (query (orders-stage {"joins" [(join "First" [first-orders-stage]) plain-join]
+ "filters" [["not-null" {} (join-ref "First" "nope")]
+ ["not-null" {} (join-ref "Plain" people-id)]]}))]
+ (is (= [#{:named?} #{:plain? :other-table?}] (advice-flags q)))
+ (is (every? #(str/includes? (message q) %) ["display label" "grouped join" "set both `source-field`"]))))))
+
(deftest ^:parallel join-alias-field-type-idempotent-test
(let [once (repair* first-order-query)]
(is (= once (repair* once)))))
@@ -4348,7 +4431,9 @@
:error :unresolved-join-alias-field
:problems [{:stage 0 :join-alias "First" :clauses [total-ref] :columns ["TOTAL"] :available ["USER_ID" "min"]}]}
e))
- (is (re-find #"join \"First\" doesn't return \"TOTAL\" \(available: \"USER_ID\", \"min\"\)" (:message e)))))
+ (is (re-find #"join \"First\" doesn't return \"TOTAL\" \(available: \"USER_ID\", \"min\"\)" (:message e)))
+ (is (str/includes? (:message e) "a grouped join returns just its breakouts and aggregations"))
+ (is (not (str/includes? (:message e) "source-field-join-alias")))))
(testing "every bad ref is reported in one error, across refs, joins, and stages"
(is (= [[0 "First" ["nope" "ORDERS.TOTAL"]] [0 "Second" ["nada"]] [1 "First" ["zilch"]]]
(join-problems
diff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.clj
index 1174432e91a..94f9413660e 100644
--- a/test/metabase/metabot/tools/construct_representations_test.clj
+++ b/test/metabase/metabot/tools/construct_representations_test.clj
@@ -3374,6 +3374,9 @@
(testing "one problem in several places is listed once, with a count of the others"
(is (str/starts-with? (explain (invalid :a) (invalid :b) (invalid :c))
"stages[0].a (and 2 other places): should be an integer.")))
+ (testing "a problem in one other place, and one more problem, are counted in the singular"
+ (is (str/starts-with? (explain (invalid :a) (invalid :b)) "stages[0].a (and 1 other place): "))
+ (is (str/ends-with? (apply explain (map missing [:a :b :c :d])) "; and 1 more problem")))
(testing "at most three problems are listed, and the rest are counted"
(is (str/ends-with? (apply explain (map missing [:a :b :c :d :e])) "; and 2 more problems")))
(testing "problems with a known cause are listed before Malli's own messages"