The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add agent-lib repair pass to fixup invalid lib/type for join clauses (#76309)
appleby00f5a7c258bf
diff --git a/src/metabase/agent_lib/representations.clj b/src/metabase/agent_lib/representations.cljindex 530b3ee3dd9..6269940dbaa 100644--- a/src/metabase/agent_lib/representations.clj+++ b/src/metabase/agent_lib/representations.clj@@ -194,6 +194,7 @@clause-shape?]]::join [:map{:closed false}+ ["lib/type" [:= "mbql/join"]]["stages" [:sequential {:min 1} [:ref ::stage]]]["conditions" [:sequential {:min 1} [:ref ::clause]]]["alias" :string]diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.cljindex 552c84f86dc..777a4ff30e0 100644--- a/src/metabase/agent_lib/representations/repair.clj+++ b/src/metabase/agent_lib/representations/repair.clj@@ -7,7 +7,7 @@patch up the sort of shape drift that LLMs routinely produce:* missing `{}` options on clauses (LLMs forget to emit empty maps);- * missing `\"lib/type\"` marker on stages and on the top-level query;+ * missing `\"lib/type\"` marker on stages, joins, and the top-level query;* stage / clause shape normalisations covering common LLM-authored variants(see the per-pass docstrings below).@@ -388,6 +388,27 @@node))form))+;;; ============================================================+;;; Pass 1.87 -- rewrite known misspelled `lib/type` markers to their canonical value.+;;;+;;; Pass 2's `infer-*` helpers only FILL a missing marker; they never rewrite a present one.+;;; This pass handles the present-but-wrong case via a small alias table, exactly like the+;;; `rewrite-operator-name-aliases*` / `rewrite-temporal-bucket-aliases*` passes.+;;; ============================================================++(def ^:private lib-type-aliases+ "Known LLM misspellings of a `lib/type` marker -> its canonical value."+ {"mbql.join/join" "mbql/join"})++(defn- rewrite-lib-type-aliases*+ [form]+ (walk/postwalk+ (fn [node]+ (if-let [canonical (and (map? node) (lib-type-aliases (get node "lib/type")))]+ (assoc node "lib/type" canonical)+ node))+ form))+;;; ============================================================;;; Pass 1.88 -- merge a trailing extra options-map into the position-1 options.;;;@@ -1221,13 +1242,26 @@(contains? m "database")(contains? m "stages")))+(defn- join-like-map?+ "A map that looks like an explicit join: it carries join-only keys (`conditions`, or+ `alias`+`stages`) that never appear on a stage or top-level query. We key on these rather+ than on `\"fields\"` (which a join shares with a stage) so a join is not mistaken for a+ stage by [[stage-like-map?]]."+ [m]+ (and (map? m)+ (or (contains? m "conditions")+ (and (contains? m "alias")+ (contains? m "stages")))))+(defn- stage-like-map?"A map that looks like an MBQL stage: has `\"source-table\"`, `\"source-card\"`, or any of thestage-body keys (`filters`, `aggregation`, `breakout`, `order-by`, `fields`, `joins`,- `expressions`, `limit`). Not a top-level query."+ `expressions`, `limit`). Not a top-level query and not an explicit join (a join can carry+ `\"fields\"`, so we exclude it explicitly)."[m](and (map? m)(not (top-level-query-map? m))+ (not (join-like-map? m))(boolean(some #(contains? m %)["source-table" "source-card" "filters" "aggregation"@@ -1238,6 +1272,11 @@(assoc m "lib/type" "mbql/query")m))+(defn- infer-join-lib-type [m]+ (if (and (join-like-map? m) (not (contains? m "lib/type")))+ (assoc m "lib/type" "mbql/join")+ m))+(defn- infer-stage-lib-type [m](if (and (stage-like-map? m) (not (contains? m "lib/type")))(assoc m "lib/type" "mbql.stage/mbql")@@ -1247,7 +1286,7 @@(walk/postwalk(fn [node](if (map? node)- (-> node infer-query-lib-type infer-stage-lib-type)+ (-> node infer-query-lib-type infer-join-lib-type infer-stage-lib-type)node))form))@@ -2559,6 +2598,7 @@rewrite-operator-name-aliases*rewrite-temporal-bucket-aliases*rewrite-direction-aliases*+ rewrite-lib-type-aliases*merge-trailing-options*merge-string-filter-trailing-options*wrap-iso-date-bounds*@@ -2586,9 +2626,11 @@1.75. strip stray surrounding double-quotes from the string segments of `field` clauses'portable-FK vector targets, e.g. `\"col\"` → `col` (cross-stage string targets are leftto the resolution-aware cross-stage matching in pass 5);+ 1.87. rewrite a known-misspelled `\"lib/type\"` marker to its canonical value (e.g. the+ join slip `\"mbql.join/join\"` → `\"mbql/join\"`);1.88. merge a trailing extra options-map back into position-1 options on fixed-aritytuple clauses (e.g. `[\"time-interval\" {} <expr> -1 \"month\" {\"include-current\" true}]`);- 2. fill in missing `\"lib/type\"` markers on the query and stages;+ 2. fill in missing `\"lib/type\"` markers on the query, joins, and stages;3. rewrite inline aggregation expressions in `order-by` to aggregation references whenthey match an aggregation in the same stage's `aggregation:` list (synthesising thereferenced aggregation's `lib/uuid` if needed);diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.cljindex 02f44dba3d3..926df085ba4 100644--- a/test/metabase/agent_lib/representations/repair_test.clj+++ b/test/metabase/agent_lib/representations/repair_test.clj@@ -930,6 +930,46 @@twice (repair/repair trivial-mp once)](is (= once twice)))))+;;; ============================================================+;;; Pass 1.87 - rewrite misspelled `lib/type` aliases+;;; ============================================================++(def ^:private lib-type-join-query+ "ORDERS with an explicit join to PRODUCTS. The join `lib/type` is filled in by each test."+ {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "joins" [{"alias" "P"+ "fields" "none"+ "conditions" [["=" {}+ ["field" {} ["Sample" "PUBLIC" "ORDERS" "PRODUCT_ID"]]+ ["field" {"join-alias" "P"} ["Sample" "PUBLIC" "PRODUCTS" "ID"]]]]+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "PRODUCTS"]}]}]}]})++(defn- join-lib-type [q]+ (get-in q ["stages" 0 "joins" 0 "lib/type"]))++(defn- with-join-lib-type [v]+ (assoc-in lib-type-join-query ["stages" 0 "joins" 0 "lib/type"] v))++(deftest ^:parallel rewrite-join-lib-type-test+ (testing "a join with the misspelled `mbql.join/join` marker is rewritten to `mbql/join`"+ (let [output (repair/repair trivial-mp (with-join-lib-type "mbql.join/join"))]+ (is (= "mbql/join" (join-lib-type output))))))++(deftest ^:parallel rewrite-join-lib-type-preserves-canonical-test+ (testing "a join that already has the canonical `mbql/join` marker is left unchanged"+ (let [output (repair/repair trivial-mp (with-join-lib-type "mbql/join"))]+ (is (= "mbql/join" (join-lib-type output))))))++(deftest ^:parallel rewrite-join-lib-type-idempotent-test+ (testing "join `lib/type` correction is a fixed point"+ (let [once (repair/repair trivial-mp (with-join-lib-type "mbql.join/join"))+ twice (repair/repair trivial-mp once)]+ (is (= once twice)))))+;;; ============================================================;;; Pass 2 - fill in missing `lib/type`;;; ============================================================@@ -960,6 +1000,11 @@(let [input {"foo" "bar"}](is (= input (repair/repair trivial-mp input))))))+(deftest ^:parallel add-join-lib-type-test+ (testing "a join with no lib/type gets mbql/join"+ (let [output (repair/repair trivial-mp (update-in lib-type-join-query ["stages" 0 "joins" 0] dissoc "lib/type"))]+ (is (= "mbql/join" (join-lib-type output))))))+;;; ============================================================;;; Pass 1.9 - stamp top-level `database:` from the first stage's source;;;diff --git a/test/metabase/agent_lib/representations_test.clj b/test/metabase/agent_lib/representations_test.cljindex e5e7edcaccc..df8889cf9c8 100644--- a/test/metabase/agent_lib/representations_test.clj+++ b/test/metabase/agent_lib/representations_test.clj@@ -11,7 +11,7 @@;;; external-query->portable;;; ============================================================-(deftest external-query->portable-test+(deftest ^:parallel external-query->portable-test(testing "stringifies map keys preserving namespace"(is (= {"lib/type" "mbql/query""database" "Sample"@@ -45,7 +45,7 @@;;; validate-external-query;;; ============================================================-(deftest validate-external-query-happy-path-test+(deftest ^:parallel validate-external-query-happy-path-test(testing "valid external-query with string-valued enums"(let [q {:lib/type "mbql/query":database "Sample"@@ -61,7 +61,7 @@(is (= ["Sample" "PUBLIC" "ORDERS"](get-in decoded [:stages 0 :source-table])))))))-(deftest validate-external-query-error-paths-test+(deftest ^:parallel validate-external-query-error-paths-test(testing "missing :stages"(is (thrown-with-msg? clojure.lang.ExceptionInfo#"invalid structure"@@ -79,7 +79,7 @@(catch clojure.lang.ExceptionInfo e(ex-data e))))-(deftest validate-query-happy-path-test+(deftest ^:parallel validate-query-happy-path-test(testing "minimal valid portable form"(let [parsed {"lib/type" "mbql/query""database" "Sample"@@ -94,7 +94,7 @@"aggregation" [["count" {}]]}]}](is (= parsed (repr/validate-query parsed))))))-(deftest validate-query-error-paths-test+(deftest ^:parallel validate-query-error-paths-test(testing "missing lib/type on query"(let [parsed {"database" "Sample""stages" [{"lib/type" "mbql.stage/mbql"@@ -122,3 +122,38 @@"stages" [{"lib/type" "mbql.stage/mbql""source-table" ["Sample" "PUBLIC" "ORDERS"]"aggregation" [["" {}]]}]})))))++(defn- join-query-with-lib-type+ "A minimal portable query with a single explicit join with `join-lib-type`.+ `join-lib-type` is spliced into the join (or, when `nil`, the `lib/type` key is omitted)."+ [join-lib-type]+ (let [condition ["=" {}+ ["field" {} ["Sample" "PUBLIC" "ORDERS" "PRODUCT_ID"]]+ ["field" {"join-alias" "P"} ["Sample" "PUBLIC" "PRODUCTS" "ID"]]]+ join (cond-> {"alias" "P"+ "conditions" [condition]+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "PRODUCTS"]}]}+ join-lib-type (assoc "lib/type" join-lib-type))]+ {"lib/type" "mbql/query"+ "database" "Sample"+ "stages" [{"lib/type" "mbql.stage/mbql"+ "source-table" ["Sample" "PUBLIC" "ORDERS"]+ "joins" [join]}]}))++(deftest ^:parallel validate-query-canonical-join-lib-type-test+ (testing "a join with the canonical `mbql/join` lib/type validates"+ (let [parsed (join-query-with-lib-type "mbql/join")]+ (is (= parsed (repr/validate-query parsed))))))++(deftest ^:parallel validate-query-misspelled-join-lib-type-test+ (testing "BOT-1657: a join with the misspelled `mbql.join/join` lib/type is rejected"+ (let [data (validate-query-error (join-query-with-lib-type "mbql.join/join"))]+ (is (= :invalid-representations-query (:error data)))+ (is (= 400 (:status-code data))))))++(deftest ^:parallel validate-query-missing-join-lib-type-test+ (testing "a join with no lib/type is rejected"+ (let [data (validate-query-error (join-query-with-lib-type nil))]+ (is (= :invalid-representations-query (:error data)))+ (is (= 400 (:status-code data))))))