Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix code that populates :used_tables in metabot context (#65787)

applebyab6c2ab72d6a
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index ccc6560903d..95342774948 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1584,6 +1584,8 @@
:api #{metabase-enterprise.ai-sql-fixer.api}
:uses #{api
driver
+ lib
+ lib-be
query-processor
util
enterprise/metabot-v3}}
diff --git a/enterprise/backend/src/metabase_enterprise/ai_sql_fixer/api.clj b/enterprise/backend/src/metabase_enterprise/ai_sql_fixer/api.clj
index 49897d5b19c..21b9df94dfe 100644
--- a/enterprise/backend/src/metabase_enterprise/ai_sql_fixer/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/ai_sql_fixer/api.clj
@@ -5,6 +5,8 @@
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.driver.util :as driver.u]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.util.malli.schema :as ms]))
@@ -19,11 +21,12 @@
[:database ms/PositiveInt]]]
[:error_message :string]]]
(qp.perms/check-current-user-has-adhoc-native-query-perms query)
- (let [driver (-> query :database driver.u/database->driver)]
- (-> (metabot-v3/fix-sql {:sql (-> query :native :query)
+ (let [driver (-> query :database driver.u/database->driver)
+ normalized-query (lib-be/normalize-query query)]
+ (-> (metabot-v3/fix-sql {:sql (lib/raw-native-query normalized-query)
:dialect driver
:error_message error_message
- :schema_ddl (metabot-v3/schema-sample query)})
+ :schema_ddl (metabot-v3/schema-sample normalized-query)})
(select-keys [:fixes]))))
(def ^{:arglists '([request respond raise])} routes
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
index e81327d69e5..184b8dae875 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
@@ -5,6 +5,8 @@
[metabase-enterprise.metabot-v3.table-utils :as table-utils]
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.config.core :as config]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -65,9 +67,11 @@
"transform" (-> item :source :query)
"adhoc" (-> item :query)
(-> item :query))]
- (when (and (#{:native "native"} (:type query))
- (:database query))
- query)))
+ ;; Draft transforms might not have a database yet. Check this before attempting to normalize the query.
+ (when (:database query)
+ (when-let [normalized-query (lib-be/normalize-query query)]
+ (when (lib/native-only-query? normalized-query)
+ normalized-query)))))
(defn- database-tables-for-context
"Get database tables formatted for metabot context. Only includes tables used in the query, formatted for API output.
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution.clj
index 745d65b543d..973bd7c5127 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.metabot-v3.query-analyzer.parameter-substitution
"Replace {{variable}}s and {{field filters}} in SQL queries with parse-able equivalents."
(:require
+ [metabase.lib.core :as lib]
[metabase.query-processor.compile :as qp.compile]))
(def default-values
@@ -56,6 +57,8 @@
"Given a native dataset_query, return a `{:query \"<SQL string>\"}` where the SQL no longer has Metabase-specific
template tags."
[query]
- (if-let [name->tag (seq (get-in query [:native :template-tags]))]
- (qp.compile/compile (assoc-in query [:native :template-tags] (update-vals name->tag tag-default)))
- (:native query)))
+ (if-let [name->tag (seq (lib/template-tags query))]
+ (-> query
+ (lib/with-template-tags (update-vals name->tag tag-default))
+ qp.compile/compile)
+ {:query (lib/raw-native-query query)}))
diff --git a/enterprise/backend/test/metabase_enterprise/ai_sql_fixer/api_test.clj b/enterprise/backend/test/metabase_enterprise/ai_sql_fixer/api_test.clj
index 8ee83aa733f..b9fe297292f 100644
--- a/enterprise/backend/test/metabase_enterprise/ai_sql_fixer/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/ai_sql_fixer/api_test.clj
@@ -3,14 +3,15 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase-enterprise.metabot-v3.table-utils :as table-utils]
+ [metabase.lib.core :as lib]
[metabase.test :as mt]))
(deftest ^:parallel schema-sample-test
(mt/test-driver :h2
- (let [query {:database (mt/id)
- :native {:query (str "SELECT * FROM x.orders1"
- " INNER JOIN products ON orders1.product_id = products.id"
- " INNER JOIN ANALYTIC_EVENT ON true")}}
+ (let [query (lib/native-query (mt/metadata-provider)
+ (str "SELECT * FROM x.orders1"
+ " INNER JOIN products ON orders1.product_id = products.id"
+ " INNER JOIN ANALYTIC_EVENT ON true"))
normalize #(into #{} (str/split % #"(?<=;)\n"))]
(is (= (-> (str "CREATE TABLE PUBLIC.PRODUCTS (\n"
" ID BIGINT,\n"
@@ -46,8 +47,7 @@
(deftest ^:parallel no-used-table-test
(mt/test-driver :postgres
- (let [query {:database (mt/id)
- :native {:query "SELECT * FROM x.orders1"}}
+ (let [query (lib/native-query (mt/metadata-provider) "SELECT * FROM x.orders1")
normalize #(into #{} (str/split % #"(?<=;)\n"))]
(is (= (-> (str "CREATE TABLE public.orders (\n"
" id int4,\n"
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj
index b37a9998a93..3fcdec2fe7c 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/context_test.clj
@@ -5,8 +5,12 @@
[metabase-enterprise.metabot-v3.context :as context]
[metabase-enterprise.metabot-v3.table-utils :as table-utils]
[metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]
[metabase.test :as mt]))
+(def ^:private users-native-query (lib/native-query meta/metadata-provider "SELECT * FROM users"))
+(def ^:private users-mbql-query (lib/query meta/metadata-provider (meta/table-metadata :users)))
+
(deftest database-tables-for-context-prioritization
(let [used [{:id 1 :name "used1"} {:id 2 :name "used2"}]
extra [{:id 3 :name "extra1"} {:id 4 :name "extra2"}]
@@ -17,7 +21,7 @@
non-priority (remove #(priority-ids (:id %)) all-tables)
result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
(take all-tables-limit result)))]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
(is (= used result) "Should only return used tables, in order")
(is (= (count used) (count result)) "Should return all used tables")
(is (not-any? #(= 3 (:id %)) result) "Should not include extra tables")
@@ -34,7 +38,7 @@
non-priority (remove #(priority-ids (:id %)) all-tables)
result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
(take all-tables-limit result)))]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
(is (= used result) "Should return all used tables")
(is (= (count used) (count result)) "Should return all used tables")
(is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates
@@ -44,7 +48,7 @@
(with-redefs [table-utils/used-tables (fn [_] [])
table-utils/enhanced-database-tables (fn [_ opts]
(take (:all-tables-limit opts 100) extra))]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
(is (empty? result) "Should return empty when no used tables")))))
(deftest database-tables-for-context-used-tables-exact-limit
@@ -57,7 +61,7 @@
non-priority (remove #(priority-ids (:id %)) all-tables)
result (concat priority-tables (take (- all-tables-limit (count priority-tables)) non-priority))]
(take all-tables-limit result)))]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}, :all-tables-limit 3})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query, :all-tables-limit 3})]
(is (= used result) "Should be exactly the used tables, in order")
(is (= (count used) (count result)))
(is (= (count (distinct (map :id result))) (count result))))))) ; no duplicates
@@ -66,7 +70,7 @@
(testing "Returns empty when table-utils/enhanced-database-tables throws"
(with-redefs [table-utils/used-tables (fn [_] [{:id 1}])
table-utils/enhanced-database-tables (fn [_ _] (throw (Exception. "boom")))]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query})]
(is (empty? result))))))
(deftest database-tables-for-context-nil-used-tables
@@ -74,7 +78,7 @@
(with-redefs [table-utils/used-tables (fn [_] nil)
table-utils/enhanced-database-tables (fn [_ opts]
(take (:all-tables-limit opts 100) [{:id 1} {:id 2}]))]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query})]
(is (empty? result) "Should return empty when used-tables is nil")))))
(deftest database-tables-for-context-duplicate-ids
@@ -83,7 +87,7 @@
dup [{:id 1} {:id 1} {:id 2}]]
(with-redefs [table-utils/used-tables (fn [_] used)
table-utils/enhanced-database-tables (fn [_ _] dup)]
- (let [result (#'context/database-tables-for-context {:query {:database 123 :native {:query "SELECT *"}}})]
+ (let [result (#'context/database-tables-for-context {:query users-native-query})]
(is (= [1 2] (map :id result)) "Should preserve order and remove duplicates"))))))
;; --- enhance-context-with-schema tests ---
@@ -92,19 +96,19 @@
(let [mock-tables [{:id 1 :name "table1"} {:id 2 :name "table2"}]]
(with-redefs [context/database-tables-for-context (fn [_] mock-tables)]
(testing "Enhances context with schema for native queries"
- (let [input {:user_is_viewing [{:query {:type :native :database 123}}]}
+ (let [input {:user_is_viewing [{:query users-native-query}]}
result (#'context/enhance-context-with-schema input)]
(is (= (get-in result [:user_is_viewing 0 :used_tables]) mock-tables)))))))
(deftest enhance-context-with-schema-non-native
(testing "Does not enhance context for non-native queries"
- (let [input {:user_is_viewing [{:query {:type :query :database 123}}]}
+ (let [input {:user_is_viewing [{:query users-mbql-query}]}
result (#'context/enhance-context-with-schema input)]
(is (nil? (get-in result [:user_is_viewing 0 :used_tables]))))))
(deftest enhance-context-with-schema-no-database
(testing "Does not enhance context if no database present"
- (let [input {:user_is_viewing [{:query {:type :native}}]}
+ (let [input {:user_is_viewing [{:query (dissoc users-native-query :database)}]}
result (#'context/enhance-context-with-schema input)]
(is (nil? (get-in result [:user_is_viewing 0 :used_tables]))))))
@@ -112,9 +116,7 @@
(testing "Enhances context with schema for complete native query structure"
(let [mock-tables [{:id 1 :name "users"} {:id 2 :name "orders"}]]
(with-redefs [context/database-tables-for-context (fn [_] mock-tables)]
- (let [input {:user_is_viewing [{:query {:type :native
- :database 123
- :native {:query "SELECT * FROM users WHERE active = true"}}}]}
+ (let [input {:user_is_viewing [{:query users-native-query}]}
result (#'context/enhance-context-with-schema input)]
(is (= (get-in result [:user_is_viewing 0 :used_tables]) mock-tables)))))))
@@ -124,11 +126,7 @@
(with-redefs [context/database-tables-for-context (fn [_]
(swap! call-count inc)
[{:id 1 :name "should-not-be-used"}])]
- (let [input {:user_is_viewing [{:query {:type :query
- :database 123
- :query {:source-table 456
- :aggregation [[:count]]
- :breakout [[:field 789 nil]]}}}]}
+ (let [input {:user_is_viewing [{:query users-mbql-query}]}
result (#'context/enhance-context-with-schema input)]
(is (nil? (get-in result [:user_is_viewing 0 :used_tables]))
"Should not add database_schema for MBQL queries")
@@ -188,7 +186,7 @@
(testing "Annotates draft native transform with source_type :native"
(let [input {:user_is_viewing [{:type "transform"
:source {:type :query
- :query (lib/native-query (mt/metadata-provider) "SELECT * FROM users")}}]}
+ :query users-native-query}}]}
result (#'context/annotate-transform-source-types input)]
(is (= :native (get-in result [:user_is_viewing 0 :source_type]))))))
@@ -196,7 +194,7 @@
(testing "Annotates draft MBQL transform with source_type :mbql"
(let [input {:user_is_viewing [{:type "transform"
:source {:type :query
- :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}]}
+ :query users-mbql-query}}]}
result (#'context/annotate-transform-source-types input)]
(is (= :mbql (get-in result [:user_is_viewing 0 :source_type]))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution_test.clj
new file mode 100644
index 00000000000..7073d7a604e
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/query_analyzer/parameter_substitution_test.clj
@@ -0,0 +1,51 @@
+(ns metabase-enterprise.metabot-v3.query-analyzer.parameter-substitution-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase-enterprise.metabot-v3.query-analyzer.parameter-substitution :as nqa.sub]
+ [metabase.lib.core :as lib]
+ [metabase.lib.test-metadata :as meta]))
+
+(deftest replace-tags-without-template-tags-test
+ (testing "Query without template tags returns raw SQL"
+ (let [query (lib/native-query meta/metadata-provider "SELECT * FROM venues")]
+ (is (= {:query "SELECT * FROM venues"}
+ (nqa.sub/replace-tags query))))))
+
+(deftest replace-tags-with-text-tag-test
+ (testing "Query with :text template tag gets default value substituted"
+ (let [query (lib/native-query meta/metadata-provider "SELECT * FROM venues WHERE name = {{name}}")]
+ (is (=? {:query "SELECT * FROM venues WHERE name = ?"
+ :params ["sample text"]}
+ (nqa.sub/replace-tags query))))))
+
+(deftest replace-tags-with-number-tag-test
+ (testing "Query with :number template tag gets default value substituted"
+ (let [query (lib/native-query meta/metadata-provider "SELECT * FROM venues WHERE id = {{id}}")
+ tags (-> (lib/template-tags query)
+ (update-vals #(assoc % :type :number)))
+ query-with-tags (lib/with-template-tags query tags)]
+ (is (=? {:query "SELECT * FROM venues WHERE id = 1"
+ :params []}
+ (nqa.sub/replace-tags query-with-tags))))))
+
+(deftest replace-tags-with-date-tag-test
+ (testing "Query with :date template tag gets default value substituted"
+ (let [query (lib/native-query meta/metadata-provider "SELECT * FROM venues WHERE created_at = {{date}}")
+ tags (-> (lib/template-tags query)
+ (update-vals #(assoc % :type :date)))
+ query-with-tags (lib/with-template-tags query tags)]
+ (is (=? {:query "SELECT * FROM venues WHERE created_at = ?"
+ :params [#t "2024-01-09"]}
+ (nqa.sub/replace-tags query-with-tags))))))
+
+(deftest replace-tags-with-multiple-tags-test
+ (testing "Query with multiple template tags get default values substituted"
+ (let [query (lib/native-query meta/metadata-provider
+ "SELECT * FROM venues WHERE id = {{id}} AND name = {{name}}")
+ tags (-> (lib/template-tags query)
+ (update "id" #(assoc % :type :number))
+ (update "name" #(assoc % :type :text)))
+ query-with-tags (lib/with-template-tags query tags)]
+ (is (=? {:query "SELECT * FROM venues WHERE id = 1 AND name = ?"
+ :params ["sample text"]}
+ (nqa.sub/replace-tags query-with-tags))))))
diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/table_utils_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/table_utils_test.clj
index 8b934cc518a..638c772c84f 100644
--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/table_utils_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/table_utils_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer :all]
[metabase-enterprise.metabot-v3.query-analyzer :as query-analyzer]
[metabase-enterprise.metabot-v3.table-utils :as table-utils]
+ [metabase.lib.core :as lib]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -160,7 +161,7 @@
(deftest used-tables-test
(testing "used-tables function"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}
:model/Table {} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}]
@@ -170,7 +171,8 @@
(with-redefs [query-analyzer/tables-for-native
(fn [_query & _opts]
{:tables [{:table "users" :table-id table1-id :schema "public"}]})]
- (let [query {:database db-id :native {:query "SELECT * FROM users"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM users"))
tables (table-utils/used-tables query)]
(is (seq tables))
(is (some #(= table1-id (:id %)) tables))))))
@@ -181,7 +183,8 @@
(with-redefs [query-analyzer/tables-for-native
(fn [_query & _opts]
{:tables [{:table "user" :schema "public"}]})] ; "user" should match "users"
- (let [query {:database db-id :native {:query "SELECT * FROM user"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM user"))
tables (table-utils/used-tables query)]
(is (seq tables))
;; Should find the fuzzy match for "users" table
@@ -191,7 +194,8 @@
(mt/with-current-user (mt/user->id :crowberto)
(with-redefs [query-analyzer/tables-for-native
(fn [_query & _opts] {:tables []})]
- (let [query {:database db-id :native {:query "SELECT 1"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT 1"))
tables (table-utils/used-tables query)]
(is (empty? tables))))))
@@ -201,7 +205,8 @@
(fn [_query & _opts]
{:tables [{:table "users" :table-id table1-id :schema "public"} ; recognized
{:table "order" :schema "public"}]})] ; unrecognized, should match "orders"
- (let [query {:database db-id :native {:query "SELECT * FROM users JOIN order ON ..."}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM users JOIN order ON ..."))
tables (table-utils/used-tables query)]
(is (>= (count tables) 1)) ; At least the recognized table
(is (some #(= table1-id (:id %)) tables)))))))))
@@ -290,7 +295,7 @@
(testing "used-tables handles query analyzer exceptions"
(with-redefs [query-analyzer/tables-for-native
(fn [_query & _opts] (throw (Exception. "Query analysis failed")))]
- (let [query {:database 1 :native {:query "SELECT * FROM users"}}]
+ (let [query (lib/native-query (mt/metadata-provider) "SELECT * FROM users")]
(is (thrown? Exception (table-utils/used-tables query))))))
(testing "database-tables with inactive tables"
@@ -381,14 +386,15 @@
(deftest schema-sample-basic-functionality-test
(testing "schema-sample with fewer tables than limit"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}
:model/Table {table2-id :id} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}
:model/Field {} {:table_id table1-id, :name "id", :database_type "INTEGER"}
:model/Field {} {:table_id table1-id, :name "name", :database_type "VARCHAR(255)"}
:model/Field {} {:table_id table2-id, :name "id", :database_type "INTEGER"}
:model/Field {} {:table_id table2-id, :name "total", :database_type "DECIMAL(10,2)"}]
- (let [query {:database db-id :native {:query "SELECT * FROM users"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM users"))
ddl (table-utils/schema-sample query {:all-tables-limit 10})]
(testing "returns DDL string"
(is (string? ddl)))
@@ -406,7 +412,7 @@
(deftest schema-sample-query-based-selection-test
(testing "schema-sample with more tables than limit - uses query-based selection"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}
:model/Table {table2-id :id} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}
:model/Table {table3-id :id} {:db_id db-id, :name "products", :schema "public", :active true, :visibility_type nil}
@@ -420,7 +426,8 @@
(with-redefs [query-analyzer/tables-for-native
(fn [_query & _opts]
{:tables [{:table "users" :table-id table1-id :schema "public"}]})]
- (let [query {:database db-id :native {:query "SELECT * FROM users"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM users"))
;; Set limit to 2, but we have 4 tables, so it should use query-based selection
ddl (table-utils/schema-sample query {:all-tables-limit 2})]
(testing "returns DDL string"
@@ -432,10 +439,11 @@
(deftest schema-sample-default-limit-test
(testing "schema-sample with default all-tables-limit"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "table1", :schema nil, :active true, :visibility_type nil}
:model/Field {} {:table_id table1-id, :name "col1", :database_type "TEXT"}]
- (let [query {:database db-id :native {:query "SELECT * FROM table1"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM table1"))
ddl (table-utils/schema-sample query)] ; No options, uses default limit
(testing "returns DDL string"
(is (string? ddl)))
@@ -446,10 +454,11 @@
(deftest schema-sample-special-characters-test
(testing "schema-sample handles tables with special characters in names"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "user-profiles", :schema "my schema", :active true, :visibility_type nil}
:model/Field {} {:table_id table1-id, :name "user id", :database_type "INTEGER"}]
- (let [query {:database db-id :native {:query "SELECT * FROM \"user-profiles\""}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM \"user-profiles\""))
ddl (table-utils/schema-sample query)]
(testing "escapes table and schema names with special characters"
(is (re-find #"\"my schema\"\.\"user-profiles\"" ddl)))
@@ -458,8 +467,9 @@
(deftest schema-sample-empty-database-test
(testing "schema-sample handles empty database"
- (mt/with-temp [:model/Database {db-id :id} {}]
- (let [query {:database db-id :native {:query "SELECT 1"}}
+ (mt/with-temp [:model/Database db {}]
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT 1"))
ddl (table-utils/schema-sample query)]
(testing "returns empty string for no tables"
(is (string? ddl))
@@ -467,12 +477,13 @@
(deftest schema-sample-table-visibility-test
(testing "schema-sample excludes inactive and hidden tables"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "active_table", :active true, :visibility_type nil}
:model/Table {} {:db_id db-id, :name "inactive_table", :active false, :visibility_type nil}
:model/Table {} {:db_id db-id, :name "hidden_table", :active true, :visibility_type "hidden"}
:model/Field {} {:table_id table1-id, :name "id", :database_type "INTEGER"}]
- (let [query {:database db-id :native {:query "SELECT * FROM active_table"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM active_table"))
ddl (table-utils/schema-sample query)]
(testing "includes only active, visible tables"
(is (re-find #"active_table" ddl))
@@ -481,16 +492,17 @@
(deftest schema-sample-empty-tables-test
(testing "schema-sample handles tables without fields"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {} {:db_id db-id, :name "empty_table", :schema "public", :active true, :visibility_type nil}]
- (let [query {:database db-id :native {:query "SELECT * FROM empty_table"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM empty_table"))
ddl (table-utils/schema-sample query)]
(testing "creates DDL for table without fields"
(is (re-find #"CREATE TABLE public\.empty_table \(\n\);" ddl)))))))
(deftest schema-sample-limit-exceeded-fuzzy-matching-test
(testing "schema-sample when limit exceeded, uses fuzzy matching for query tables"
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "users", :schema "public", :active true, :visibility_type nil}
:model/Table {table2-id :id} {:db_id db-id, :name "user_profiles", :schema "public", :active true, :visibility_type nil}
:model/Table {table3-id :id} {:db_id db-id, :name "products", :schema "public", :active true, :visibility_type nil}
@@ -504,7 +516,8 @@
(fn [_query & _opts]
{:tables [{:table "users" :table-id table1-id :schema "public"}
{:table "products" :table-id table3-id :schema "public"}]})]
- (let [query {:database db-id :native {:query "SELECT * FROM users JOIN products"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM users JOIN products"))
ddl (table-utils/schema-sample query {:all-tables-limit 1})]
(testing "includes tables that match query"
(is (re-find #"users" ddl))
@@ -514,7 +527,7 @@
(deftest schema-sample-fuzzy-matching-similar-names-test
(testing "schema-sample fuzzy matching behavior with similar table names"
;; This test documents the fuzzy matching behavior when limit is exceeded
- (mt/with-temp [:model/Database {db-id :id} {}
+ (mt/with-temp [:model/Database {db-id :id :as db} {}
:model/Table {table1-id :id} {:db_id db-id, :name "order", :schema "public", :active true, :visibility_type nil}
:model/Table {table2-id :id} {:db_id db-id, :name "orders", :schema "public", :active true, :visibility_type nil}
:model/Table {table3-id :id} {:db_id db-id, :name "order_items", :schema "public", :active true, :visibility_type nil}
@@ -526,7 +539,8 @@
(fn [_query & _opts]
;; Return "order" which will fuzzy match multiple tables
{:tables [{:table "order" :schema "public"}]})]
- (let [query {:database db-id :native {:query "SELECT * FROM order"}}
+ (let [query (mt/with-db db
+ (lib/native-query (mt/metadata-provider) "SELECT * FROM order"))
ddl (table-utils/schema-sample query {:all-tables-limit 1})]
(testing "fuzzy matches similar table names"
;; "order" will match both "order" and "orders" due to fuzzy matching