Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add agent-lib repair pass to fixup invalid lib/type for join clauses (#76309)

appleby00f5a7c258bf
diff --git a/src/metabase/agent_lib/representations.clj b/src/metabase/agent_lib/representations.clj
index 530b3ee3dd9..6269940dbaa 100644
--- a/src/metabase/agent_lib/representations.clj
+++ b/src/metabase/agent_lib/representations.clj
@@ -194,6 +194,7 @@
clause-shape?]]
::join [:map
{:closed false}
+ ["lib/type" [:= "mbql/join"]]
["stages" [:sequential {:min 1} [:ref ::stage]]]
["conditions" [:sequential {:min 1} [:ref ::clause]]]
["alias" :string]
diff --git a/src/metabase/agent_lib/representations/repair.clj b/src/metabase/agent_lib/representations/repair.clj
index 552c84f86dc..777a4ff30e0 100644
--- a/src/metabase/agent_lib/representations/repair.clj
+++ b/src/metabase/agent_lib/representations/repair.clj
@@ -7,7 +7,7 @@
patch up the sort of shape drift that LLMs routinely produce:
* missing `{}` options on clauses (LLMs forget to emit empty maps);
- * missing `\"lib/type\"` marker on stages and on the top-level query;
+ * missing `\"lib/type\"` marker on stages, joins, and the top-level query;
* stage / clause shape normalisations covering common LLM-authored variants
(see the per-pass docstrings below).
@@ -388,6 +388,27 @@
node))
form))
+;;; ============================================================
+;;; Pass 1.87 -- rewrite known misspelled `lib/type` markers to their canonical value.
+;;;
+;;; Pass 2's `infer-*` helpers only FILL a missing marker; they never rewrite a present one.
+;;; This pass handles the present-but-wrong case via a small alias table, exactly like the
+;;; `rewrite-operator-name-aliases*` / `rewrite-temporal-bucket-aliases*` passes.
+;;; ============================================================
+
+(def ^:private lib-type-aliases
+ "Known LLM misspellings of a `lib/type` marker -> its canonical value."
+ {"mbql.join/join" "mbql/join"})
+
+(defn- rewrite-lib-type-aliases*
+ [form]
+ (walk/postwalk
+ (fn [node]
+ (if-let [canonical (and (map? node) (lib-type-aliases (get node "lib/type")))]
+ (assoc node "lib/type" canonical)
+ node))
+ form))
+
;;; ============================================================
;;; Pass 1.88 -- merge a trailing extra options-map into the position-1 options.
;;;
@@ -1221,13 +1242,26 @@
(contains? m "database")
(contains? m "stages")))
+(defn- join-like-map?
+ "A map that looks like an explicit join: it carries join-only keys (`conditions`, or
+ `alias`+`stages`) that never appear on a stage or top-level query. We key on these rather
+ than on `\"fields\"` (which a join shares with a stage) so a join is not mistaken for a
+ stage by [[stage-like-map?]]."
+ [m]
+ (and (map? m)
+ (or (contains? m "conditions")
+ (and (contains? m "alias")
+ (contains? m "stages")))))
+
(defn- stage-like-map?
"A map that looks like an MBQL stage: has `\"source-table\"`, `\"source-card\"`, or any of the
stage-body keys (`filters`, `aggregation`, `breakout`, `order-by`, `fields`, `joins`,
- `expressions`, `limit`). Not a top-level query."
+ `expressions`, `limit`). Not a top-level query and not an explicit join (a join can carry
+ `\"fields\"`, so we exclude it explicitly)."
[m]
(and (map? m)
(not (top-level-query-map? m))
+ (not (join-like-map? m))
(boolean
(some #(contains? m %)
["source-table" "source-card" "filters" "aggregation"
@@ -1238,6 +1272,11 @@
(assoc m "lib/type" "mbql/query")
m))
+(defn- infer-join-lib-type [m]
+ (if (and (join-like-map? m) (not (contains? m "lib/type")))
+ (assoc m "lib/type" "mbql/join")
+ m))
+
(defn- infer-stage-lib-type [m]
(if (and (stage-like-map? m) (not (contains? m "lib/type")))
(assoc m "lib/type" "mbql.stage/mbql")
@@ -1247,7 +1286,7 @@
(walk/postwalk
(fn [node]
(if (map? node)
- (-> node infer-query-lib-type infer-stage-lib-type)
+ (-> node infer-query-lib-type infer-join-lib-type infer-stage-lib-type)
node))
form))
@@ -2559,6 +2598,7 @@
rewrite-operator-name-aliases*
rewrite-temporal-bucket-aliases*
rewrite-direction-aliases*
+ rewrite-lib-type-aliases*
merge-trailing-options*
merge-string-filter-trailing-options*
wrap-iso-date-bounds*
@@ -2586,9 +2626,11 @@
1.75. strip stray surrounding double-quotes from the string segments of `field` clauses'
portable-FK vector targets, e.g. `\"col\"` → `col` (cross-stage string targets are left
to the resolution-aware cross-stage matching in pass 5);
+ 1.87. rewrite a known-misspelled `\"lib/type\"` marker to its canonical value (e.g. the
+ join slip `\"mbql.join/join\"` → `\"mbql/join\"`);
1.88. merge a trailing extra options-map back into position-1 options on fixed-arity
tuple clauses (e.g. `[\"time-interval\" {} <expr> -1 \"month\" {\"include-current\" true}]`);
- 2. fill in missing `\"lib/type\"` markers on the query and stages;
+ 2. fill in missing `\"lib/type\"` markers on the query, joins, and stages;
3. rewrite inline aggregation expressions in `order-by` to aggregation references when
they match an aggregation in the same stage's `aggregation:` list (synthesising the
referenced aggregation's `lib/uuid` if needed);
diff --git a/test/metabase/agent_lib/representations/repair_test.clj b/test/metabase/agent_lib/representations/repair_test.clj
index 02f44dba3d3..926df085ba4 100644
--- a/test/metabase/agent_lib/representations/repair_test.clj
+++ b/test/metabase/agent_lib/representations/repair_test.clj
@@ -930,6 +930,46 @@
twice (repair/repair trivial-mp once)]
(is (= once twice)))))
+;;; ============================================================
+;;; Pass 1.87 - rewrite misspelled `lib/type` aliases
+;;; ============================================================
+
+(def ^:private lib-type-join-query
+ "ORDERS with an explicit join to PRODUCTS. The join `lib/type` is filled in by each test."
+ {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "joins" [{"alias" "P"
+ "fields" "none"
+ "conditions" [["=" {}
+ ["field" {} ["Sample" "PUBLIC" "ORDERS" "PRODUCT_ID"]]
+ ["field" {"join-alias" "P"} ["Sample" "PUBLIC" "PRODUCTS" "ID"]]]]
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "PRODUCTS"]}]}]}]})
+
+(defn- join-lib-type [q]
+ (get-in q ["stages" 0 "joins" 0 "lib/type"]))
+
+(defn- with-join-lib-type [v]
+ (assoc-in lib-type-join-query ["stages" 0 "joins" 0 "lib/type"] v))
+
+(deftest ^:parallel rewrite-join-lib-type-test
+ (testing "a join with the misspelled `mbql.join/join` marker is rewritten to `mbql/join`"
+ (let [output (repair/repair trivial-mp (with-join-lib-type "mbql.join/join"))]
+ (is (= "mbql/join" (join-lib-type output))))))
+
+(deftest ^:parallel rewrite-join-lib-type-preserves-canonical-test
+ (testing "a join that already has the canonical `mbql/join` marker is left unchanged"
+ (let [output (repair/repair trivial-mp (with-join-lib-type "mbql/join"))]
+ (is (= "mbql/join" (join-lib-type output))))))
+
+(deftest ^:parallel rewrite-join-lib-type-idempotent-test
+ (testing "join `lib/type` correction is a fixed point"
+ (let [once (repair/repair trivial-mp (with-join-lib-type "mbql.join/join"))
+ twice (repair/repair trivial-mp once)]
+ (is (= once twice)))))
+
;;; ============================================================
;;; Pass 2 - fill in missing `lib/type`
;;; ============================================================
@@ -960,6 +1000,11 @@
(let [input {"foo" "bar"}]
(is (= input (repair/repair trivial-mp input))))))
+(deftest ^:parallel add-join-lib-type-test
+ (testing "a join with no lib/type gets mbql/join"
+ (let [output (repair/repair trivial-mp (update-in lib-type-join-query ["stages" 0 "joins" 0] dissoc "lib/type"))]
+ (is (= "mbql/join" (join-lib-type output))))))
+
;;; ============================================================
;;; Pass 1.9 - stamp top-level `database:` from the first stage's source
;;;
diff --git a/test/metabase/agent_lib/representations_test.clj b/test/metabase/agent_lib/representations_test.clj
index e5e7edcaccc..df8889cf9c8 100644
--- a/test/metabase/agent_lib/representations_test.clj
+++ b/test/metabase/agent_lib/representations_test.clj
@@ -11,7 +11,7 @@
;;; external-query->portable
;;; ============================================================
-(deftest external-query->portable-test
+(deftest ^:parallel external-query->portable-test
(testing "stringifies map keys preserving namespace"
(is (= {"lib/type" "mbql/query"
"database" "Sample"
@@ -45,7 +45,7 @@
;;; validate-external-query
;;; ============================================================
-(deftest validate-external-query-happy-path-test
+(deftest ^:parallel validate-external-query-happy-path-test
(testing "valid external-query with string-valued enums"
(let [q {:lib/type "mbql/query"
:database "Sample"
@@ -61,7 +61,7 @@
(is (= ["Sample" "PUBLIC" "ORDERS"]
(get-in decoded [:stages 0 :source-table])))))))
-(deftest validate-external-query-error-paths-test
+(deftest ^:parallel validate-external-query-error-paths-test
(testing "missing :stages"
(is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid structure"
@@ -79,7 +79,7 @@
(catch clojure.lang.ExceptionInfo e
(ex-data e))))
-(deftest validate-query-happy-path-test
+(deftest ^:parallel validate-query-happy-path-test
(testing "minimal valid portable form"
(let [parsed {"lib/type" "mbql/query"
"database" "Sample"
@@ -94,7 +94,7 @@
"aggregation" [["count" {}]]}]}]
(is (= parsed (repr/validate-query parsed))))))
-(deftest validate-query-error-paths-test
+(deftest ^:parallel validate-query-error-paths-test
(testing "missing lib/type on query"
(let [parsed {"database" "Sample"
"stages" [{"lib/type" "mbql.stage/mbql"
@@ -122,3 +122,38 @@
"stages" [{"lib/type" "mbql.stage/mbql"
"source-table" ["Sample" "PUBLIC" "ORDERS"]
"aggregation" [["" {}]]}]})))))
+
+(defn- join-query-with-lib-type
+ "A minimal portable query with a single explicit join with `join-lib-type`.
+ `join-lib-type` is spliced into the join (or, when `nil`, the `lib/type` key is omitted)."
+ [join-lib-type]
+ (let [condition ["=" {}
+ ["field" {} ["Sample" "PUBLIC" "ORDERS" "PRODUCT_ID"]]
+ ["field" {"join-alias" "P"} ["Sample" "PUBLIC" "PRODUCTS" "ID"]]]
+ join (cond-> {"alias" "P"
+ "conditions" [condition]
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "PRODUCTS"]}]}
+ join-lib-type (assoc "lib/type" join-lib-type))]
+ {"lib/type" "mbql/query"
+ "database" "Sample"
+ "stages" [{"lib/type" "mbql.stage/mbql"
+ "source-table" ["Sample" "PUBLIC" "ORDERS"]
+ "joins" [join]}]}))
+
+(deftest ^:parallel validate-query-canonical-join-lib-type-test
+ (testing "a join with the canonical `mbql/join` lib/type validates"
+ (let [parsed (join-query-with-lib-type "mbql/join")]
+ (is (= parsed (repr/validate-query parsed))))))
+
+(deftest ^:parallel validate-query-misspelled-join-lib-type-test
+ (testing "BOT-1657: a join with the misspelled `mbql.join/join` lib/type is rejected"
+ (let [data (validate-query-error (join-query-with-lib-type "mbql.join/join"))]
+ (is (= :invalid-representations-query (:error data)))
+ (is (= 400 (:status-code data))))))
+
+(deftest ^:parallel validate-query-missing-join-lib-type-test
+ (testing "a join with no lib/type is rejected"
+ (let [data (validate-query-error (join-query-with-lib-type nil))]
+ (is (= :invalid-representations-query (:error data)))
+ (is (= 400 (:status-code data))))))