The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Keep query error paths tied to the original stages (#84909)
Andrei Simionescu04a6f8fbba2a
diff --git a/src/metabase/agent_lib/representations.clj b/src/metabase/agent_lib/representations.cljindex 463f213de0a..d7d7bdc0b03 100644--- a/src/metabase/agent_lib/representations.clj+++ b/src/metabase/agent_lib/representations.clj@@ -226,22 +226,40 @@"Malli schema for a single MBQL stage in the portable representations form."[:schema {:registry registry} [:ref ::stage]])+(defn original-stage-index+ "Return the original index of a repaired stage, or `stage-idx` if repair metadata is missing."+ [query stage-idx]+ (get (::original-stage-indexes (meta query)) stage-idx stage-idx))++(defn original-query-path+ "Translate a repaired query path to the stage indexes in the original input."+ [query path]+ (let [[stage-key stage-idx] path]+ (cond-> path+ (and (#{:stages "stages"} stage-key) (integer? stage-idx))+ (assoc 1 (original-stage-index query stage-idx)))))+(defn validate-query- "Validate a string-keyed portable representations query against [[query-schema]] and return it- unchanged.-- Used as a sanity check between the repair and resolve passes."- [parsed]- (when-let [error (mr/explain query-schema parsed)]- (throw (ex-info (tru "Representations query has an invalid structure.")- {:status-code 400- :error :invalid-representations-query- :error-code :invalid-representations-query- :humanized (mu.humanize/humanize error)- :details (pr-str (mu.humanize/humanize error))- :schema ::query- :response/keys #{:humanized :details}})))- parsed)+ "Validate a portable representations query against [[query-schema]] and return it unchanged.++ `parsed` must have string keys. Error paths use the original stage indexes from `repaired-query`,+ which defaults to `parsed`."+ ([parsed] (validate-query parsed parsed))+ ([parsed repaired-query]+ (when-let [error (mr/explain query-schema parsed)]+ (let [humanized (mu.humanize/humanize+ (update error :errors (fn [errors]+ (mapv #(update % :in (partial original-query-path repaired-query))+ errors))))]+ (throw (ex-info (tru "Representations query has an invalid structure.")+ {:status-code 400+ :error :invalid-representations-query+ :error-code :invalid-representations-query+ :humanized humanized+ :details (pr-str humanized)+ :schema ::query+ :response/keys #{:humanized :details}}))))+ parsed))(defn query-path"Format a query location as a dotted path with bracketed indices, or `(root)` for an empty path."diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.cljindex 08faeb70729..1b7779db0b2 100644--- a/src/metabase/agent_lib/representations/repair.clj+++ b/src/metabase/agent_lib/representations/repair.clj@@ -30,6 +30,7 @@(:require[clojure.string :as str][clojure.walk :as walk]+ [metabase.agent-lib.representations :as repr][metabase.agent-lib.representations.resolve :as repr.resolve][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]@@ -2166,14 +2167,22 @@:stage stage}))))))))(defn- split-post-agg-filters*- "Top-level pass: walk every stage in the query, expanding any post-agg-filter stage- in-place to `[stage0 stage1]`. Re-runs are no-ops (idempotent: split stages no longer- match the trigger). Split failures carry the unsplit query as `:query` in their ex-data."+ "Split query stages that filter aggregation results.++ Repeated calls leave already split stages unchanged. The `::repr/original-stage-indexes` metadata+ maps each resulting stage to its index in the original input. Split failures carry the unsplit query as `:query`."[query](if-not (and (map? query) (vector? (get query "stages")))query(try- (update query "stages" #(into [] (mapcat split-post-agg-filter-stage) %))+ (let [stages (get query "stages")+ original-indexes (or (::repr/original-stage-indexes (meta query)) (range (count stages)))+ stage-groups (mapv split-post-agg-filter-stage stages)+ stage-indexes (into [] (mapcat (fn [idx group] (repeat (count group) idx))+ original-indexes stage-groups))]+ (-> query+ (assoc "stages" (into [] cat stage-groups))+ (vary-meta assoc ::repr/original-stage-indexes stage-indexes)))(catch clojure.lang.ExceptionInfo e(throw (ex-info (ex-message e) (assoc (ex-data e) :query query) e))))))@@ -3033,7 +3042,7 @@:problems (vec (for [{:keys [alias cols named-refs path-refs]} checks:let [bad (bad-join-alias-refs mp cols named-refs path-refs import-resolver)]:when (seq bad)]- {:stage idx+ {:stage (repr/original-stage-index query idx):join-alias alias:clauses (vec bad):columns (vec (distinct (map #(field-target-name mp (nth % 2)) bad)))@@ -3388,7 +3397,7 @@(when (and (map? query) (vector? (get query "stages")))(doseq [[idx stage] (map-indexed vector (get query "stages"))](when (map? stage)- (aggregation-entry-not-aggregation-error! stage idx)))+ (aggregation-entry-not-aggregation-error! stage (repr/original-stage-index query idx))))(case-default-in-opts-error! query)(sexp-legacy-op-as-clause-error! query)(blank-expression-ref-error! query)@@ -3437,6 +3446,8 @@(defn repair"Run the repair pipeline on a parsed (string-keyed, portable) representations query.+ The `::repr/original-stage-indexes` metadata maps each returned stage to its index in the original input.+Passes:1. ensure every clause vector has an options map at position 2;1.5. normalise `expressions:` shape - accept map form `{Name: clause, …}` or the@@ -3549,41 +3560,40 @@;;; ============================================================(defn assert-editor-accepts-expressions!- "Run the FE expression editor's own validation, [[metabase.lib.expression/diagnose-expression]],- over every custom column, aggregation, and filter of the resolved `pmbql-query`, throwing a- retryable `:agent-error?` on the first clause it rejects. Returns `pmbql-query`.-- Catches shapes that are schema-legal yet broken in the editor: `offset` in a filter, window- functions nested inside aggregation functions, cyclic expression references, type-incompatible- arguments, standalone constants without `:expression-literals` support.-- Unlike the rest of this namespace this runs after resolve. Call it only on a query that already- passed the suppressed-type-check `::lib.schema/query` validation - `diagnose-expression`- validates its query argument against that same schema, and its raw validation error would- otherwise preempt the friendly message."- [pmbql-query]- (doseq [stage-idx (range (lib/stage-count pmbql-query))- [mode clauses] [[:expression (lib/expressions pmbql-query stage-idx)]- [:aggregation (lib/aggregations pmbql-query stage-idx)]- [:filter (lib/filters pmbql-query stage-idx)]]- [pos clause] (map-indexed vector clauses)]- (when-let [{:keys [message]} (lib.expression/diagnose-expression- pmbql-query stage-idx mode clause- (when (#{:expression :aggregation} mode) pos))]- (let [where (case mode- :expression "custom column (`expressions:`)"- :aggregation "`aggregation:`"- :filter "`filters:`")]- (throw (ex-info- (tru "The query builder''s expression editor rejects a {0} clause in stage {1}: {2}. The query would build but could not be visualized or saved - rewrite the offending clause."- where stage-idx (str/replace message #"\.$" ""))- {:agent-error? true- :error :expression-editor-rejection- :status-code 400- :stage stage-idx- :mode mode- :clause clause})))))- pmbql-query)+ "Validate resolved query expressions with the query builder's expression editor.++ Check custom columns, aggregations and filters with [[metabase.lib.expression/diagnose-expression]],+ and throw a retryable `:agent-error?` for the first rejected clause. Return `pmbql-query` if every+ clause passes.++ `pmbql-query` must already pass `::lib.schema/query` validation with expression type checks+ suppressed. Otherwise, schema errors can hide the clause's error message.+ `repaired-query` supplies the original stage indexes for diagnostics and defaults to `pmbql-query`."+ ([pmbql-query] (assert-editor-accepts-expressions! pmbql-query pmbql-query))+ ([pmbql-query repaired-query]+ (doseq [stage-idx (range (lib/stage-count pmbql-query))+ [mode clauses] [[:expression (lib/expressions pmbql-query stage-idx)]+ [:aggregation (lib/aggregations pmbql-query stage-idx)]+ [:filter (lib/filters pmbql-query stage-idx)]]+ [pos clause] (map-indexed vector clauses)]+ (when-let [{:keys [message]} (lib.expression/diagnose-expression+ pmbql-query stage-idx mode clause+ (when (#{:expression :aggregation} mode) pos))]+ (let [original-idx (repr/original-stage-index repaired-query stage-idx)+ where (case mode+ :expression "custom column (`expressions:`)"+ :aggregation "`aggregation:`"+ :filter "`filters:`")]+ (throw (ex-info+ (tru "The query builder''s expression editor rejects a {0} clause in stage {1}: {2}. The query would build but could not be visualized or saved - rewrite the offending clause."+ where original-idx (str/replace message #"\.$" ""))+ {:agent-error? true+ :error :expression-editor-rejection+ :status-code 400+ :stage original-idx+ :mode mode+ :clause clause})))))+ pmbql-query))(defn- unbucketed-ref"`clause` with its `temporal-unit`, and the type the bucket stamped on it, taken off.diff --git a/src/metabase/metabot/tools/construct.clj b/src/metabase/metabot/tools/construct.cljindex 5290b616fdf..0e566e25389 100644--- a/src/metabase/metabot/tools/construct.clj+++ b/src/metabase/metabot/tools/construct.clj@@ -865,23 +865,25 @@(tru "{0}: aggregation and window functions such as `offset` cannot be custom columns - move them to `aggregation:` or `order-by:`" (me/error-message error)))))(defn- query-not-runnable-explanation- "When `pmbql-query` would make the FE's `canRun` gate return false, return an LLM-facing- description of where and why; nil when the query is runnable. Problems [[not-runnable-problem]]- explains are listed first, and the usual causes are added when it explains none."- [pmbql-query]- (binding [lib.schema.expression/*suppress-expression-type-check?* true]- (when-let [explanation (mr/explain :metabase.lib.schema/query pmbql-query)]- (let [errors (sort-by (comp nil? not-runnable-problem) (:errors explanation))]- (cond-> (or (some->> errors- (map #(str (repr/query-path (:in %)) ": "- (or (not-runnable-problem %) (me/error-message %))))- distinct- (take 3)- seq- (str/join "; "))- (tru "the query does not match the query builder''s schema"))- (not-any? not-runnable-problem errors)- (str ". " (tru "This usually means a field reference is missing its type or names a column that does not exist, or an aggregation or window function such as `offset` is in `expressions:` instead of `aggregation:` or `order-by:`")))))))+ "Explain why `pmbql-query` fails the query builder's `canRun` check, or return nil.++ Problems that [[not-runnable-problem]] can explain come first. Add the usual causes if it cannot+ explain any of them. `repaired-query` supplies the original stage indexes and defaults to `pmbql-query`."+ ([pmbql-query] (query-not-runnable-explanation pmbql-query pmbql-query))+ ([pmbql-query repaired-query]+ (binding [lib.schema.expression/*suppress-expression-type-check?* true]+ (when-let [explanation (mr/explain :metabase.lib.schema/query pmbql-query)]+ (let [errors (sort-by (comp nil? not-runnable-problem) (:errors explanation))]+ (cond-> (or (some->> errors+ (map #(str (repr/query-path (repr/original-query-path repaired-query (:in %))) ": "+ (or (not-runnable-problem %) (me/error-message %))))+ distinct+ (take 3)+ seq+ (str/join "; "))+ (tru "the query does not match the query builder''s schema"))+ (not-any? not-runnable-problem errors)+ (str ". " (tru "This usually means a field reference is missing its type or names a column that does not exist, or an aggregation or window function such as `offset` is in `expressions:` instead of `aggregation:` or `order-by:`"))))))))(defn- incompatible-metrics"Return the required sources of incompatible metrics in the first stage, sorted by metric ID.@@ -1049,7 +1051,7 @@_perms (check-source-table-query-permissions! mp repaired checked)_validated (repr/validate-query repaired)pmbql-query (repr.resolve/resolve-query mp repaired permission-aware-content-store)- _runnable (when-let [why (query-not-runnable-explanation pmbql-query)]+ _runnable (when-let [why (query-not-runnable-explanation pmbql-query repaired)](throw (ex-info (tru "The constructed query is not runnable - it would fail the query builder''s validation, so it cannot be visualized or saved. Failing paths: {0}."why){:agent-error? true@@ -1061,7 +1063,7 @@;; aggregations, cyclic expression refs, type-incompatible arguments. Must run;; after the `_runnable` gate: `diagnose-expression` itself validates its query;; argument against the same schema.- _editor-ok (repr.repair/assert-editor-accepts-expressions! pmbql-query)+ _editor-ok (repr.repair/assert-editor-accepts-expressions! pmbql-query repaired)_buckets-ok (repr.repair/assert-temporal-buckets-on-temporal-columns! pmbql-query);; Check sources after schema validation so an incompatible metric produces a retryable error before execution._metrics-ok (when-let [bad (incompatible-metrics mp pmbql-query)]diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.cljindex 55b49aa5343..1cf1ad952ff 100644--- a/test/metabase/agent_lib/representations/repair_test.clj+++ b/test/metabase/agent_lib/representations/repair_test.clj@@ -3384,6 +3384,25 @@(is (= 2 (count (get once "stages"))))(is (= once twice)))))+(deftest ^:parallel error-paths-preserve-original-stages-test+ (let [post-agg-stage {"lib/type" "mbql.stage/mbql"+ "aggregation" [["count" {}]]+ "filters" [[">" {} ["aggregation" {} 0] 10]]}+ query {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ post-agg-stage+ post-agg-stage+ {"lib/type" "mbql.stage/mbql"}]}+ repaired (repair/repair trivial-mp query)+ expected (mapv #(vector :stages % :filters 0) [0 1 1 2 2 3])]+ (doseq [q [repaired (repair/repair trivial-mp repaired)]]+ (is (= 6 (count (get q "stages"))))+ (is (= expected (mapv #(repr/original-query-path q [:stages % :filters 0]) (range 6))))+ (testing "only the parent stage index changes for a nested join path"+ (is (= [:stages 3 :joins 0 :stages 1 :filters 0]+ (repr/original-query-path q [:stages 5 :joins 0 :stages 1 :filters 0])))))))+(deftest ^:parallel split-post-agg-handles-distinct-and-count-where-test(testing "distinct and count-where get correct lib column names"(let [build (fn [agg]diff --git a/test/metabase/agent_lib/representations/resolve_test.clj b/test/metabase/agent_lib/representations/resolve_test.cljindex e93c37dc8cc..e4956248727 100644--- a/test/metabase/agent_lib/representations/resolve_test.clj+++ b/test/metabase/agent_lib/representations/resolve_test.clj@@ -635,6 +635,25 @@#"/ expects a number"(repr.repair/assert-editor-accepts-expressions! q))))))+(deftest editor-errors-use-query-stage-metadata-test+ (let [parsed {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ {"lib/type" "mbql.stage/mbql"+ "aggregation" [["count" {}]]+ "filters" [[">" {} ["aggregation" {} 0] 10]]}+ {"lib/type" "mbql.stage/mbql"+ "filters" [[">" {} ["offset" {} ["field" {} "count"] -1] 1]]}]}+ repaired (repr.repair/repair mp parsed)+ query (with-meta (repr.resolve/resolve-query mp repaired) (meta repaired))+ error (try+ (repr.repair/assert-editor-accepts-expressions! query)+ (catch clojure.lang.ExceptionInfo e e))]+ (is (= 4 (lib/stage-count query)))+ (is (= :expression-editor-rejection (:error (ex-data error))))+ (is (= 2 (:stage (ex-data error))))+ (is (re-find #"rejects a `filters:` clause in stage 2:" (ex-message error)))))+(deftest annotate-metric-ref-types-leaves-hard-cases-untouched-test(let [agg-query (fn [agg]{:lib/type :mbql/querydiff --git a/test/metabase/metabot/tools/construct_representations_test.clj b/test/metabase/metabot/tools/construct_representations_test.cljindex 45a18577ec4..eb1356cdc69 100644--- a/test/metabase/metabot/tools/construct_representations_test.clj+++ b/test/metabase/metabot/tools/construct_representations_test.clj@@ -3279,3 +3279,74 @@(catch clojure.lang.ExceptionInfo e e))](is (= (str "Representations query has an invalid structure.\n" hint)(tools.error/recoverable-text (tools.error/classify error) #{}))))))))++(deftest error-locations-after-stage-split-test+ (with-mp-and-stubs!+ (fn []+ (doseq [[stage-kvs error-code expected]+ [[{"filters" {"op" ">"}}+ :invalid-representations-query+ "Invalid at stages[2].filters: invalid type. Fix the named paths."]+ [{"breakout" [{"op" "field"}]}+ :invalid-representations-query+ (str "Invalid at stages[2].breakout[0]: should be a vector, "+ "clause must be [operator-string, options-map, …args]. Fix the named paths.")]+ [{"expressions" {"Previous" ["offset" {} ["field" {} "count"] -1]}}+ :query-not-runnable+ "Failing paths: stages[2].expressions[0]:"]]]+ (let [query (query-data+ {"lib/type" "mbql/query"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]}+ {"lib/type" "mbql.stage/mbql"+ "aggregation" [["count" {}]]+ "filters" [[">" {} ["aggregation" {} 0] 10]]}+ (merge {"lib/type" "mbql.stage/mbql"} stage-kvs)]})+ error (try+ (tools/with-pipeline-errors (construct/execute-representations-query query))+ (catch clojure.lang.ExceptionInfo e e))+ classified (tools.error/classify error)]+ (is (= :recoverable (:class classified)))+ (is (= (keyword "metabase.metabot.tools.recoverable.pipeline" (name error-code))+ (:code classified)))+ (when (contains? stage-kvs "breakout")+ (is (vector? (get-in (ex-data (ex-cause error)) [:humanized "stages" 2 "breakout"]))))+ (is (str/includes? (tools.error/recoverable-text classified #{}) expected)))))))++(deftest diagnostic-stage-numbers-after-split-test+ (with-mp-and-stubs!+ (fn []+ (doseq [split? [false true]+ [stage-kvs error-code expected]+ [[{"aggregation" [["field" {} "count"]]}+ :aggregation-entry-not-aggregation+ "Stage 1 aggregation entry 0 is a `field` clause"]+ [{"filters" [[">" {} ["offset" {} ["field" {} "count"] -1] 1]]}+ :expression-editor-rejection+ "rejects a `filters:` clause in stage 1: OFFSET is not supported in custom filters"]+ [{"joins" [{"alias" "P"+ "strategy" "left-join"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "PRODUCTS"]}]+ "conditions" [["=" {} ["field" {} "PRODUCT_ID"]+ ["field" {"join-alias" "P"} ["Sample" "PUBLIC" "PRODUCTS" "ID"]]]]}]+ "breakout" [["field" {"join-alias" "P"} "NOPE"]]}+ :unresolved-join-alias-field+ "stage 1, join \"P\" doesn't return \"NOPE\""]]]+ (testing (str error-code ", split: " split?)+ (let [query (query-data+ {"lib/type" "mbql/query"+ "stages" [(cond-> {"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "aggregation" [["count" {}]]+ "breakout" [["field" {} ["Sample" "PUBLIC" "ORDERS" "PRODUCT_ID"]]]}+ split? (assoc "filters" [[">" {} ["aggregation" {} 0] 10]]))+ (merge {"lib/type" "mbql.stage/mbql"} stage-kvs)]})+ error (try+ (tools/with-pipeline-errors (construct/execute-representations-query query))+ (catch clojure.lang.ExceptionInfo e e))+ classified (tools.error/classify error)]+ (is (= :recoverable (:class classified)))+ (is (= (keyword "metabase.metabot.tools.recoverable.pipeline" (name error-code))+ (:code classified)))+ (is (str/includes? (tools.error/recoverable-text classified #{}) expected))))))))