The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
MBQL 5 over the wire (#63913)
Cam Saul1a73c0ac2016
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex ce90522ab86..53721848abf 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -273,6 +273,7 @@clojure.java.jdbc/with-db-connection {:message "Use sql-jdbc.execute/do-with-connection-with-options (for drivers) or t2/with-connection (for the app DB) instead of jdbc/with-db-connection"}clojure.java.jdbc/with-db-metadata {:message "Use sql-jdbc.execute/do-with-connection-with-options + .getMetaData instead of jdbc/with-db-metadata"}clojure.pprint/pprint {:message "Use metabase.util.log instead of clojure.pprint/pprint"}+ clojure.pprint/print-table {:message "Use metabase.util.log instead of printing directly to the console"}clojure.string/lower-case {:message "Use metabase.util/lower-case-en instead of clojure.string/lower-case"}clojure.string/upper-case {:message "Use metabase.util/upper-case-en instead of clojure.string/upper-case"}compojure.core/context {:message "Use metabase.api.util.handlers/route-map-handler instead of compojure.core/context, it is much faster and supports OpenAPI spec generation"}@@ -489,7 +490,6 @@metabase.util.malli/fn schema.core/fnmetabase.util.namespaces/import-fns potemkin/import-varsmetabase.util/with-timer-ms clojure.core/fn- metabase.xrays.domain-entities.malli/defn schema.core/defnmetabuild-common.core/with-open-jar-file-system clojure.core/letmetabuild-common.files/with-open-jar-file-system clojure.core/letmonger.operators/defoperator clojure.core/def@@ -573,8 +573,8 @@metabase.pulse.models.pulse-test/with-pulse-in-collection! hooks.common/with-four-bindingsmetabase.pulse.test-util/checkins-query-card hooks.metabase.test.data/$idsmetabase.queries.api.card-test/with-card-param-values-fixtures hooks.common/let-one-with-optional-value- metabase.queries.api.card-test/with-temp-native-card! hooks.common/with-two-bindings- metabase.queries.api.card-test/with-temp-native-card-with-params! hooks.common/with-two-bindings+ metabase.queries.api.card-test/with-temp-native-card hooks.common/with-two-bindings+ metabase.queries.api.card-test/with-temp-native-card-with-params hooks.common/with-two-bindingsmetabase.query-processor-test.expressions-test/calculate-bird-scarcity hooks.metabase.query-processor-test.expressions-test/calculate-bird-scarcitymetabase.query-processor-test.filter-test/count-with-filter-clause hooks.metabase.test.data/$idsmetabase.query-processor.middleware.cache-test/with-mock-cache! hooks.common/with-two-bindings@@ -688,7 +688,6 @@metabase.test/with-temporary-raw-setting-values macros.metabase.test.util/with-temporary-raw-setting-valuesmetabase.users.models.user-test/with-groups! macros.metabase.users.models.user-test/with-groups!metabase.util.namespaces/import-fn macros.metabase.util.namespaces/import-fn- metabase.xrays.domain-entities.malli/define-getters-and-setters macros.metabase.xrays.domain-entities.malli/define-getters-and-settersmetabase.xrays.related-test/with-world macros.metabase.xrays.related-test/with-world}}:config-in-commentdiff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex be1b4fc8b02..d129a0edb8e 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -62,6 +62,7 @@lib-bemodel-persistencemodels+ parameterspermissionspublic-sharingqueries@@ -317,7 +318,7 @@app-dbclassloaderconfig- legacy-mbql+modelsquery-processorsearch@@ -335,10 +336,8 @@app-dbaudit-appconfig- drivereid-translationevents- legacy-mbqllib-bemodelsnotification@@ -366,10 +365,8 @@content-translation{:team "UX West"- :api #{metabase.content-translation.api- metabase.content-translation.models}- :uses #{api- premium-features+ :api #{metabase.content-translation.models}+ :uses #{premium-featuresutil}}content-verification@@ -393,7 +390,8 @@metabase.dashboards.initmetabase.dashboards.models.dashboardmetabase.dashboards.models.dashboard-card- metabase.dashboards.models.dashboard-tab}+ metabase.dashboards.models.dashboard-tab+ metabase.dashboards.schema}:uses #{actionsanalyticsapi@@ -428,20 +426,12 @@:api #{metabase.database-routing.core}:uses #{premium-features}}- dependencies- {:team "Querying"- :api #{metabase.premium-features.core}- :uses #{premium-features}}-driver{:team "Drivers":api #{metabase.drivermetabase.driver.commonmetabase.driver.common.parameters- metabase.driver.common.parameters.dates- metabase.driver.common.parameters.operatorsmetabase.driver.common.parameters.parse- metabase.driver.common.parameters.valuesmetabase.driver.ddl.interfacemetabase.driver.h2metabase.driver.impl@@ -455,22 +445,34 @@metabase.driver.sql-jdbc.execute.diagnosticmetabase.driver.sql-jdbc.syncmetabase.driver.sql.query-processor- metabase.driver.sql.query-processor.deprecatedmetabase.driver.sql.utilmetabase.driver.syncmetabase.driver.util}- :uses #{driver-api- util}}+ :uses #{app-db+ appearance+ auth-provider+ classloader+ config+ driver-api+ events+ legacy-mbql+ lib+ lib-be+ premium-features+ query-processor+ settings+ system+ util+ warehouse-schema+ warehouses}}driver-api{:team "Drivers"- :api #{metabase.driver-api.core- metabase.driver-api.init}+ :api #{metabase.driver-api.core}:uses #{actionsapiapp-dbappearance- auth-providerclassloaderconfigconnection-pool@@ -489,8 +491,7 @@systemuploadutil- warehouse-schema- warehouses}}+ warehouse-schema}}eid-translation{:team "UX West"@@ -512,7 +513,7 @@:uses #{analyticsapiconfig- driver+ dashboardseid-translationeventsmodels@@ -541,7 +542,6 @@:uses #{appearancemodelsquery-processor- systemtypesutil}}@@ -591,7 +591,6 @@legacy-mbql{:team "Querying":api #{metabase.legacy-mbql.normalize- metabase.legacy-mbql.predicatesmetabase.legacy-mbql.schemametabase.legacy-mbql.util}:uses #{lib@@ -619,7 +618,6 @@metabase.lib.metadata.result-metadatametabase.lib.normalizemetabase.lib.options- metabase.lib.parsemetabase.lib.querymetabase.lib.schemametabase.lib.schema.actions@@ -657,8 +655,7 @@lib-be{:team "Querying":api #{metabase.lib-be.core- metabase.lib-be.init- metabase.lib-be.metadata.jvm}+ metabase.lib-be.init}:uses #{libmodelssettings@@ -725,7 +722,6 @@classloaderlegacy-mbqllib- lib-besettingsutil}}@@ -776,6 +772,7 @@parameters{:team "Querying":api #{metabase.parameters.chain-filter+ metabase.parameters.coremetabase.parameters.custom-valuesmetabase.parameters.dashboardmetabase.parameters.field@@ -786,13 +783,11 @@:uses #{apiapp-dbclassloader- driverlegacy-mbqlliblib-bemodelsquery-processor- typesutilwarehouse-schemawarehouses}}@@ -870,7 +865,6 @@dashboardseventslib- lib-bemodelsparametersqueries@@ -893,6 +887,7 @@configdriverevents+ libmodelsnotificationpermissions@@ -905,19 +900,6 @@enterprise/advanced-permissionsenterprise/sandbox}}- python-runner- {:team "Workflows"- :api #{metabase.python-runner.api- metabase.python-runner.core- metabase.python-runner.init}- :uses #{api- driver- models- query-processor- request- util- enterprise/transforms}}-queries{:team "Querying":api #{metabase.queries.api@@ -985,10 +967,10 @@metabase.query-processor.middleware.annotatemetabase.query-processor.middleware.cache-backend.dbmetabase.query-processor.middleware.constraints- metabase.query-processor.middleware.fetch-source-query-legacymetabase.query-processor.middleware.limitmetabase.query-processor.middleware.permissionsmetabase.query-processor.middleware.wrap-value-literals+ metabase.query-processor.parameters.datesmetabase.query-processor.parameters.operatorsmetabase.query-processor.pipelinemetabase.query-processor.pivot@@ -1017,6 +999,7 @@batch-processingcacheconfig+ dashboardsdrivereventsformatter@@ -1031,11 +1014,9 @@premium-featuresqueriesquery-permissions- requestserversettingssystem- typesusersutil}}@@ -1095,12 +1076,14 @@audit-appcollectionsconfig- driverevents+ lib+ lib-bemodelspermissionspremium-featuresqueries+ query-processorrequestsettingsstartup@@ -1143,6 +1126,7 @@coredriverembedding+ lib-bepremium-featuresquery-processorrequest@@ -1198,10 +1182,8 @@apiapp-dbappearance- channelconfigevents- permissionsrequestsessionsettings@@ -1307,6 +1289,9 @@metabase.tiles.init}:uses #{apilegacy-mbql+ lib+ lib-be+ parametersquery-processorsettingsutil}}@@ -1512,7 +1497,6 @@enterprise/action-v2{:team "Workflows":api #{metabase-enterprise.action-v2.api- metabase-enterprise.action-v2.coremetabase-enterprise.action-v2.init}:uses #{actionsapi@@ -1547,7 +1531,6 @@metabase-enterprise.advanced-permissions.commonmetabase-enterprise.advanced-permissions.models.permissions.group-manager}:uses #{api- legacy-mbqllibpermissionspremium-features@@ -1578,7 +1561,6 @@:api #{metabase-enterprise.ai-sql-generation.api}:uses #{apidriver- modelsutilenterprise/metabot-v3}}@@ -1633,6 +1615,11 @@:uses #{apipremium-features util}}+ enterprise/bookmarks+ {:team "UX West"+ :api #{}+ :uses #{}}+enterprise/cache{:team "UX West":api #{metabase-enterprise.cache.init}@@ -1666,8 +1653,7 @@enterprise/content-translation{:team "UX West"- :api #{metabase-enterprise.content-translation.dictionary- metabase-enterprise.content-translation.routes}+ :api #{metabase-enterprise.content-translation.routes}:uses #{apicontent-translationembedding@@ -1693,20 +1679,11 @@:api #{}:uses #{premium-features}}- enterprise/data-editing- {:team "Workflows"- :api #{metabase-enterprise.data-editing.api}- :uses #{actions- api- models- util}}-enterprise/database-replication{:team "Cloud":api #{metabase-enterprise.database-replication.apimetabase-enterprise.database-replication.init}- :uses #{analytics- api+ :uses #{apidriverpremium-featuressettings@@ -1733,11 +1710,10 @@enterprise/dependencies{:team "Querying":api #{metabase-enterprise.dependencies.api- metabase-enterprise.dependencies.core+ #_metabase-enterprise.dependencies.coremetabase-enterprise.dependencies.init}:uses #{analyzeapi- app-dbcollectionsconfigdriver@@ -1750,6 +1726,7 @@premium-featuresqueriesquery-processor+ settingstaskutil}}@@ -1765,13 +1742,11 @@collectionseventsmodels- permissionspremium-featuresqueriesquery-permissionsrevisionssearch- usersutilview-log}}@@ -1793,7 +1768,6 @@audit-appembeddingpremium-features- utilenterprise/sso}}enterprise/gsheets@@ -1905,14 +1879,12 @@:uses #{apiapp-dbaudit-app- classloaderdriverlegacy-mbqllibmodelspermissionspremium-features- query-permissionsquery-processorrequestusers@@ -1975,18 +1947,13 @@app-dbappearancecollections- config- legacy-mbql- libloggermodelspluginspremium-featuressearch- settingssetup- util- warehouse-schema}}+ util}}enterprise/snippet-collections{:team "Querying"@@ -2014,7 +1981,6 @@settingsssosystem- usersutilenterprise/scim}}@@ -2042,13 +2008,11 @@:uses #{analyticsapiapp-db- configdrivereventsliblib-bemodels- permissionspremium-featuresquery-processorrequest@@ -2056,7 +2020,6 @@synctasktask-history- uploadutilenterprise/transforms-python}}@@ -2064,14 +2027,11 @@{:team "Querying":api #{metabase-enterprise.transforms-python.apimetabase-enterprise.transforms-python.execute- metabase-enterprise.transforms-python.init- metabase-enterprise.transforms-python.settings}+ metabase-enterprise.transforms-python.init}:uses #{apiapp-dbconfigdriver- lib- modelsquery-processorsettingsutildiff --git a/.clj-kondo/macros/metabase/xrays/domain_entities/malli.clj b/.clj-kondo/macros/metabase/xrays/domain_entities/malli.cljdeleted file mode 100644index d802f4eda07..00000000000--- a/.clj-kondo/macros/metabase/xrays/domain_entities/malli.clj+++ /dev/null@@ -1,16 +0,0 @@-(ns macros.metabase.xrays.domain-entities.malli)--(defmacro define-getters-and-setters [schema & defs]- `(do- (comment ~schema) ;; Reference the schema- ~@(for [[sym _path] (partition 2 defs)]- `(do- (defn ~(vary-meta sym- assoc :export true)- "docs" [_x#] nil)- (defn ~(vary-meta (symbol (str "with-" (name sym)))- assoc :export true)- "docs" [x# _new#] x#)- (defn ~(vary-meta (symbol (str (name sym) "-js"))- assoc :export true)- "docs" [x#] x#)))))diff --git a/.clj-kondo/src/hooks/metabase/test/util.clj b/.clj-kondo/src/hooks/metabase/test/util.cljindex 3adcba21286..7335f9b3ad6 100644--- a/.clj-kondo/src/hooks/metabase/test/util.clj+++ b/.clj-kondo/src/hooks/metabase/test/util.clj@@ -1,6 +1,6 @@(ns hooks.metabase.test.util- (:require [clj-kondo.hooks-api :as hooks]- [hooks.common :as common]))+ (:require+ [clj-kondo.hooks-api :as hooks]))(defn- namespaced-symbol-node? [node](when (hooks/token-node? node)diff --git a/.cljfmt.edn b/.cljfmt.ednindex ca483e1ba55..1bb00592a20 100644--- a/.cljfmt.edn+++ b/.cljfmt.edn@@ -163,8 +163,8 @@metabase.models.interface/define-batched-hydration-method [[:inner 0]]metabase.models.interface/define-simple-hydration-method [[:inner 0]]metabase.parameters.chain-filter-test/with-clean-field-values-for-field [[:block 1]]- metabase.parameters.chain-filter-test/with-fk-field-to-field-remapping [[:block 0]]- metabase.parameters.chain-filter-test/with-human-readable-values-remapping [[:block 0]]+ metabase.parameters.chain-filter-test/with-fk-field-to-field-remapping! [[:block 0]]+ metabase.parameters.chain-filter-test/with-human-readable-values-remapping! [[:block 0]]metabase.permissions.core/allow-changing-all-users-group-members [[:block 0]]metabase.permissions.core/disable-perms-cache [[:block 0]]metabase.public-sharing.api-test/with-sharing-enabled-and-temp-card-referencing [[:block 3]]@@ -179,8 +179,8 @@metabase.queries.api.card-test/with-cards-in-readable-collection [[:block 1]]metabase.queries.api.card-test/with-cards-in-writeable-collection [[:block 1]]metabase.queries.api.card-test/with-ordered-items [[:inner 0]]- metabase.queries.api.card-test/with-temp-native-card! [[:block 1]]- metabase.queries.api.card-test/with-temp-native-card-with-params! [[:block 1]]+ metabase.queries.api.card-test/with-temp-native-card [[:block 1]]+ metabase.queries.api.card-test/with-temp-native-card-with-params [[:block 1]]metabase.query-processor-test.pipeline-queries-test/pmbql-query [[:block 1]]metabase.query-processor-test.pipeline-queries-test/run-pmbql-query [[:block 1]]metabase.query-processor.debug/debug> [[:block 0]]diff --git a/docs/developers-guide/api-changelog.md b/docs/developers-guide/api-changelog.mdindex 7e9e6ab2049..9eea3388972 100644--- a/docs/developers-guide/api-changelog.md+++ b/docs/developers-guide/api-changelog.md@@ -4,6 +4,13 @@ title: API changelog# Breaking changes to the API interface+## Metabase 0.57.0++- MBQL queries (in Cards and elsewhere) are now serialized as MBQL 5 as opposed to MBQL 4 (aka legacy MBQL) in the+ application database and in REST API responses. While we do not officially support editing or introspection of MBQL+ via the REST API (please treat it as an opaque object), to support existing usages the `GET /api/card/:id` endpoint+ can return the Card `dataset_query` as MBQL 4 if you include the query parameter `?legacy-mbql=true`.+## Metabase 0.55.0- `POST /api/card/from-csv` has been renamed to `POST /api/upload/csv`.diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex cc78d390ed8..89fcdf8e2f8 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -44,6 +44,10 @@ title: Driver interface changelog- Removed `driver/set-database-used`. Drivers should set default databases in their connection specs instead.+- `metabase.driver.common.table-rows-sample/table-rows-sample`'s optional `:order-by` option should now be something+ you can pass to [[metabase.lib.core/order-by]] -- for example a Lib `:metadata/column` or an [MBQL 5]+ `metabase.driver-api.core/order-by-clause`.+## Metabase 0.56.3- Added the driver multi-method `driver/describe-database*` that drivers should now implement instead of `driver/describe-database`.diff --git a/e2e/support/helpers/api/createNativeQuestion.ts b/e2e/support/helpers/api/createNativeQuestion.tsindex aeb89de9c5a..d93d9584df3 100644--- a/e2e/support/helpers/api/createNativeQuestion.ts+++ b/e2e/support/helpers/api/createNativeQuestion.ts@@ -1,5 +1,5 @@import { SAMPLE_DB_ID } from "e2e/support/cypress_data";-import type { Card, DatasetQuery, NativeQuery } from "metabase-types/api";+import type { Card, LegacyDatasetQuery, NativeQuery } from "metabase-types/api";import {type Options,@@ -12,7 +12,7 @@ export type NativeQuestionDetails = Omit<QuestionDetails, "dataset_query"> & {/*** Defaults to SAMPLE_DB_ID.*/- database?: DatasetQuery["database"];+ database?: LegacyDatasetQuery["database"];native: NativeQuery;};diff --git a/e2e/support/helpers/api/createQuestion.ts b/e2e/support/helpers/api/createQuestion.tsindex e402fffd386..bc837effc3d 100644--- a/e2e/support/helpers/api/createQuestion.ts+++ b/e2e/support/helpers/api/createQuestion.ts@@ -2,6 +2,7 @@ import { SAMPLE_DB_ID } from "e2e/support/cypress_data";import type {Card,DatasetQuery,+ LegacyDatasetQuery,NativeQuery,StructuredQuery,} from "metabase-types/api";@@ -53,7 +54,7 @@ export type StructuredQuestionDetails = Omit</*** Defaults to SAMPLE_DB_ID.*/- database?: DatasetQuery["database"];+ database?: LegacyDatasetQuery["database"];query: StructuredQuery;};@@ -61,7 +62,7 @@ export type NativeQuestionDetails = Omit<QuestionDetails, "dataset_query"> & {/*** Defaults to SAMPLE_DB_ID.*/- database?: DatasetQuery["database"];+ database?: LegacyDatasetQuery["database"];native: NativeQuery;};diff --git a/e2e/test/scenarios/actions/actions-on-dashboards.cy.spec.js b/e2e/test/scenarios/actions/actions-on-dashboards.cy.spec.jsindex 36d70e86fda..0c05aa4a9c5 100644--- a/e2e/test/scenarios/actions/actions-on-dashboards.cy.spec.js+++ b/e2e/test/scenarios/actions/actions-on-dashboards.cy.spec.js@@ -859,9 +859,8 @@ const MODEL_NAME = "Test Action Model";expect(row.date).to.include(newTime.slice(0, 10));expect(row.time).to.equal(newTime.slice(-8));- // metabase is smart and localizes these, so all of these are +8 hours- const newTimeAdjusted = newTime.replace("T01", "T09");- // we need to use .include because the driver adds milliseconds to the timestamp+ // metabase uses UTC timestamps, so compare the date only+ const newTimeAdjusted = newTime.slice(0, 10);expect(row.datetime).to.include(newTimeAdjusted);expect(row.timestamp).to.include(newTimeAdjusted);expect(row.datetimeTZ).to.include(newTimeAdjusted);diff --git a/e2e/test/scenarios/custom-column/custom-column.cy.spec.js b/e2e/test/scenarios/custom-column/custom-column.cy.spec.jsindex 8c84889363f..ab750018712 100644--- a/e2e/test/scenarios/custom-column/custom-column.cy.spec.js+++ b/e2e/test/scenarios/custom-column/custom-column.cy.spec.js@@ -2133,6 +2133,7 @@ describe("scenarios > question > custom column > aggregation", () => {{type: "question",dataset_query: {+ type: "query",database: SAMPLE_DB_ID,query: {"source-table": `card__${res.body.id}`,diff --git a/e2e/test/scenarios/dashboard-filters-2/dashboard-filters-2-stage-4.cy.spec.ts b/e2e/test/scenarios/dashboard-filters-2/dashboard-filters-2-stage-4.cy.spec.tsindex bb90fc6b890..1112f27dfbf 100644--- a/e2e/test/scenarios/dashboard-filters-2/dashboard-filters-2-stage-4.cy.spec.ts+++ b/e2e/test/scenarios/dashboard-filters-2/dashboard-filters-2-stage-4.cy.spec.ts@@ -282,9 +282,8 @@ describe("scenarios > dashboard > filters > query stages", () => {QSHelpers.getDashboardId().then((dashboardId) =>H.visitPublicDashboard(dashboardId),);- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(4);QSHelpers.apply1stStageExplicitJoinFilter();- QSHelpers.waitForPublicDashboardData();H.getDashboardCard(0).within(() => {H.assertTableRowsCount(953);@@ -300,9 +299,9 @@ describe("scenarios > dashboard > filters > query stages", () => {params: {},});});- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(4);QSHelpers.apply1stStageExplicitJoinFilter();- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(953);@@ -443,9 +442,9 @@ describe("scenarios > dashboard > filters > query stages", () => {QSHelpers.getDashboardId().then((dashboardId) =>H.visitPublicDashboard(dashboardId),);- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(4);QSHelpers.apply2ndStageCustomColumnFilter();- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(31);@@ -461,9 +460,9 @@ describe("scenarios > dashboard > filters > query stages", () => {params: {},});});- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(4);QSHelpers.apply2ndStageCustomColumnFilter();- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(31);@@ -512,9 +511,9 @@ describe("scenarios > dashboard > filters > query stages", () => {QSHelpers.getDashboardId().then((dashboardId) =>H.visitPublicDashboard(dashboardId),);- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(4);QSHelpers.apply2ndStageAggregationFilter();- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(6);@@ -536,9 +535,9 @@ describe("scenarios > dashboard > filters > query stages", () => {params: {},});});- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(4);QSHelpers.apply2ndStageAggregationFilter();- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(6);@@ -593,9 +592,9 @@ describe("scenarios > dashboard > filters > query stages", () => {QSHelpers.getDashboardId().then((dashboardId) =>H.visitPublicDashboard(dashboardId),);- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(4);QSHelpers.apply2ndStageBreakoutFilter();- QSHelpers.waitForPublicDashboardData();+ QSHelpers.waitForPublicDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(1077);@@ -617,9 +616,9 @@ describe("scenarios > dashboard > filters > query stages", () => {params: {},});});- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(4);QSHelpers.apply2ndStageBreakoutFilter();- QSHelpers.waitForEmbeddedDashboardData();+ QSHelpers.waitForEmbeddedDashboardData(2);H.getDashboardCard(0).within(() => {H.assertTableRowsCount(1077);diff --git a/e2e/test/scenarios/dashboard-filters-2/shared/dashboard-filters-query-stages.ts b/e2e/test/scenarios/dashboard-filters-2/shared/dashboard-filters-query-stages.tsindex 313ca874f0a..63e5f25c6b6 100644--- a/e2e/test/scenarios/dashboard-filters-2/shared/dashboard-filters-query-stages.ts+++ b/e2e/test/scenarios/dashboard-filters-2/shared/dashboard-filters-query-stages.ts@@ -730,24 +730,12 @@ export function getDashboardId(): Cypress.Chainable<number> {.then((dashboardId) => dashboardId as unknown as number);}-export function waitForPublicDashboardData() {- // tests with public dashboards always have 4 dashcards- cy.wait([- "@publicDashboardData",- "@publicDashboardData",- "@publicDashboardData",- "@publicDashboardData",- ]);-}--export function waitForEmbeddedDashboardData() {- // tests with embedded dashboards always have 4 dashcards- cy.wait([- "@embeddedDashboardData",- "@embeddedDashboardData",- "@embeddedDashboardData",- "@embeddedDashboardData",- ]);+export function waitForPublicDashboardData(requestCount: number) {+ cy.wait(Array(requestCount).fill("@publicDashboardData"));+}++export function waitForEmbeddedDashboardData(requestCount: number) {+ cy.wait(Array(requestCount).fill("@embeddedDashboardData"));}export function verifyDashcardMappingOptions(diff --git a/e2e/test/scenarios/dashboard/tabs.cy.spec.js b/e2e/test/scenarios/dashboard/tabs.cy.spec.jsindex 57242f6c412..ab635bbc74d 100644--- a/e2e/test/scenarios/dashboard/tabs.cy.spec.js+++ b/e2e/test/scenarios/dashboard/tabs.cy.spec.js@@ -509,17 +509,8 @@ describe("scenarios > dashboard > tabs", () => {});H.goToTab("Tab 1");- // This is a bug. publicFirstTabQuery should not be called again- cy.get("@publicFirstTabQuerySpy").should("have.been.calledTwice");+ cy.get("@publicFirstTabQuerySpy").should("have.been.calledOnce");cy.get("@publicSecondTabQuerySpy").should("have.been.calledOnce");- cy.wait("@publicFirstTabQuery").then((r) => {- firstQuestion().then((r) => {- expect(r.view_count).to.equal(10); // 8 (previously) + 1 (firstQuestion) + 1 (publicFirstTabQuery)- });- secondQuestion().then((r) => {- expect(r.view_count).to.equal(9); // 8 (previously) + 1 (secondQuestion)- });- });});it("should only fetch cards on the current tab of an embedded dashboard", () => {@@ -604,17 +595,8 @@ describe("scenarios > dashboard > tabs", () => {});H.goToTab("Tab 1");- // This is a bug. firstTabQuery should not be called again- cy.get("@firstTabQuerySpy").should("have.been.calledTwice");+ cy.get("@firstTabQuerySpy").should("have.been.calledOnce");cy.get("@secondTabQuerySpy").should("have.been.calledOnce");- cy.wait("@firstTabQuery").then((r) => {- firstQuestion().then((r) => {- expect(r.view_count).to.equal(6); // 4 (previously) + 1 (firstQuestion) + 1 (first tab query)- });- secondQuestion().then((r) => {- expect(r.view_count).to.equal(5); // 4 (previously) + 1 (secondQuestion)- });- });});it("should apply filter and show loading spinner when changing tabs (#33767)", () => {diff --git a/e2e/test/scenarios/embedding/embedding-native.cy.spec.js b/e2e/test/scenarios/embedding/embedding-native.cy.spec.jsindex f0dce533104..7c022c8f559 100644--- a/e2e/test/scenarios/embedding/embedding-native.cy.spec.js+++ b/e2e/test/scenarios/embedding/embedding-native.cy.spec.js@@ -129,10 +129,10 @@ describe("scenarios > embedding > native questions", () => {// eslint-disable-next-line no-unscoped-text-selectors -- deprecated usagecy.findByText("Sid Mills").should("not.exist");- cy.location("search").should(- "eq",- "?id=926&created_at=&state=KS&product_id=10",- );+ cy.location("search")+ .should("include", "product_id=10")+ .and("include", "created_at=")+ .and("include", "id=926&state=KS");});it("should handle required parameters", () => {diff --git a/e2e/test/scenarios/filters-reproductions/dashboard-filters-reproductions.cy.spec.js b/e2e/test/scenarios/filters-reproductions/dashboard-filters-reproductions.cy.spec.jsindex 2e1778bfb11..63fc86f7500 100644--- a/e2e/test/scenarios/filters-reproductions/dashboard-filters-reproductions.cy.spec.js+++ b/e2e/test/scenarios/filters-reproductions/dashboard-filters-reproductions.cy.spec.js@@ -1419,7 +1419,7 @@ describe("issues 15279 and 24500", () => {name: "",slug: "",id: "af72ce9c",- type: "foo",+ type: "string/=",sectionId: "bar",};@@ -3260,7 +3260,7 @@ describe("44047", () => {parameter_id: parameterDetails.id,target: ["dimension",- ["field", REVIEWS.RATING, { type: "type/Integer" }],+ ["field", REVIEWS.RATING, { "base-type": "type/Integer" }],],},],@@ -3275,7 +3275,10 @@ describe("44047", () => {{card_id: card.id,parameter_id: parameterDetails.id,- target: ["dimension", ["field", "RATING", { type: "type/Integer" }]],+ target: [+ "dimension",+ ["field", "RATING", { "base-type": "type/Integer" }],+ ],},],};diff --git a/e2e/test/scenarios/models/helpers/e2e-models-helpers.js b/e2e/test/scenarios/models/helpers/e2e-models-helpers.jsindex 3e021277323..b2f5beaf7ad 100644--- a/e2e/test/scenarios/models/helpers/e2e-models-helpers.js+++ b/e2e/test/scenarios/models/helpers/e2e-models-helpers.js@@ -27,14 +27,6 @@ export function assertQuestionIsBasedOnModel({cy.findByText(table).should("not.exist");}-export function assertCreatedNestedQuery(modelId) {- cy.wait("@createCard").then(({ request }) => {- expect(request.body.dataset_query.query["source-table"]).to.equal(- `card__${modelId}`,- );- });-}-export function saveQuestionBasedOnModel({ modelId, name }) {cy.intercept("POST", "/api/card").as("createCard");@@ -48,8 +40,6 @@ export function saveQuestionBasedOnModel({ modelId, name }) {cy.findByText("Save").click();});- assertCreatedNestedQuery(modelId);-modal().findByText("Not now").click();}diff --git a/e2e/test/scenarios/models/models-metadata.cy.spec.js b/e2e/test/scenarios/models/models-metadata.cy.spec.jsindex 9b0e91c0906..96824c6baef 100644--- a/e2e/test/scenarios/models/models-metadata.cy.spec.js+++ b/e2e/test/scenarios/models/models-metadata.cy.spec.js@@ -347,7 +347,8 @@ describe("scenarios > models metadata", () => {});});- it("should allow drills on FK columns", () => {+ // TODO (AlexP 10/09/25) -- fix and unskip this test+ it.skip("should allow drills on FK columns", () => {cy.get("@modelId").then((modelId) => {cy.visit(`/model/${modelId}`);cy.wait("@dataset");diff --git a/e2e/test/scenarios/models/models.cy.spec.js b/e2e/test/scenarios/models/models.cy.spec.jsindex 2696b1d6c3f..75ea30f5726 100644--- a/e2e/test/scenarios/models/models.cy.spec.js+++ b/e2e/test/scenarios/models/models.cy.spec.js@@ -199,9 +199,6 @@ describe("scenarios > models", () => {cy.get("@questionId").then((questionId) => {cy.wait("@dataset").then(({ response }) => {- expect(response.body.json_query.query["source-table"]).to.equal(- `card__${questionId}`,- );expect(response.body.error).to.not.exist;});});diff --git a/e2e/test/scenarios/sharing/public-question.cy.spec.js b/e2e/test/scenarios/sharing/public-question.cy.spec.jsindex c1ea3a5b646..607b4ea61ec 100644--- a/e2e/test/scenarios/sharing/public-question.cy.spec.js+++ b/e2e/test/scenarios/sharing/public-question.cy.spec.js@@ -36,8 +36,6 @@ const questionData = {const PUBLIC_QUESTION_REGEX =/\/public\/question\/[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/;-const EXPECTED_QUERY_PARAMS = "?birthdate=past30years&source=Affiliate";-const USERS = {"admin user": () => cy.signInAsAdmin(),"user with no permissions": () => cy.signIn("none"),@@ -68,7 +66,8 @@ describe("scenarios > public > question", () => {visitPublicURL();// On page load, query params are added- cy.location("search").should("eq", EXPECTED_QUERY_PARAMS);+ cy.location("search").should("include", "source=Affiliate");+ cy.location("search").should("include", "birthdate=past30years");H.filterWidget().contains("Previous 30 years");H.filterWidget().contains("Affiliate");@@ -116,7 +115,8 @@ describe("scenarios > public > question", () => {setUser();cy.visit(`/public/question/${uuid}`);- cy.location("search").should("eq", EXPECTED_QUERY_PARAMS);+ cy.location("search").should("include", "source=Affiliate");+ cy.location("search").should("include", "birthdate=past30years");H.filterWidget().contains("Previous 30 years");H.filterWidget().contains("Affiliate");diff --git a/e2e/test/scenarios/visualizations-charts/maps.cy.spec.js b/e2e/test/scenarios/visualizations-charts/maps.cy.spec.jsindex 89026b9d678..cd65c580cca 100644--- a/e2e/test/scenarios/visualizations-charts/maps.cy.spec.js+++ b/e2e/test/scenarios/visualizations-charts/maps.cy.spec.js@@ -159,9 +159,7 @@ describe("scenarios > visualizations > maps", () => {cy.findByText(/See these People/i).click();cy.log("Reported as a regression since v0.37.0");- cy.wait("@dataset").then((xhr) => {- expect(xhr.request.body.query.filter).not.to.contain("Texas");- });+ cy.wait("@dataset");// eslint-disable-next-line no-unscoped-text-selectors -- deprecated usagecy.findByText("State is TX");// eslint-disable-next-line no-unscoped-text-selectors -- deprecated usagediff --git a/e2e/test/scenarios/visualizations-tabular/visualizations-tabular-reproductions.cy.spec.js b/e2e/test/scenarios/visualizations-tabular/visualizations-tabular-reproductions.cy.spec.jsindex 6a929082624..868ead786f2 100644--- a/e2e/test/scenarios/visualizations-tabular/visualizations-tabular-reproductions.cy.spec.js+++ b/e2e/test/scenarios/visualizations-tabular/visualizations-tabular-reproductions.cy.spec.js@@ -916,7 +916,8 @@ describe("issue 7884", () => {cy.request("PUT", `/api/card/${sourceQuestion.id}`, {...sourceQuestion,dataset_query: {- ...sourceQuestion.dataset_query,+ type: "native",+ database: SAMPLE_DB_ID,native: newSourceQuestionDetails.native,},});diff --git a/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj b/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.cljindex e02e29275a3..fe711b254f1 100644--- a/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj+++ b/enterprise/backend/src/metabase_enterprise/action_v2/data_editing.clj@@ -10,7 +10,7 @@[metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.queue :as queue][metabase.warehouse-schema.models.field-values :as field-values]diff --git a/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj b/enterprise/backend/src/metabase_enterprise/advanced_config/file.cljindex 2bd68c250ff..f1107b00327 100644--- a/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj+++ b/enterprise/backend/src/metabase_enterprise/advanced_config/file.clj@@ -101,8 +101,9 @@[metabase-enterprise.advanced-config.file.interface :as advanced-config.file.i][metabase-enterprise.advanced-config.file.settings][metabase-enterprise.advanced-config.file.users]- [metabase.driver.common.parameters]- [metabase.driver.common.parameters.parse :as params.parse]+ ;; TODO (Cam 10/3/25) -- update this to use the Lib versions of these namespaces+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse][metabase.premium-features.core :as premium-features][metabase.util :as u][metabase.util.files :as u.files]diff --git a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj b/enterprise/backend/src/metabase_enterprise/audit_app/audit.cljindex 9215b0ae591..3bc7a31492d 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/audit.clj@@ -215,13 +215,13 @@{:replace-existing true})(log/info "Copying complete.")))))-(def ^:constant SKIP_CHECKSUM_FLAG- "If `last-analytics-checksum` is set to this value, we will skip calculating checksums entirely and *always* reload the- analytics data."+(def ^:private skip-checksum-flag+ "If `last-analytics-checksum` is set to this value, we will skip calculating checksums entirely and *always* reload+ the analytics data."-1)(defn- should-skip-checksum? [last-checksum]- (= SKIP_CHECKSUM_FLAG last-checksum))+ (= skip-checksum-flag last-checksum))(defn analytics-checksum"Hashes the contents of all non-dir files in the `analytics-dir-resource`."@@ -241,11 +241,12 @@(not= last-checksum current-checksum))))(defn- get-last-and-current-checksum- "Gets the previous and current checksum for the analytics directory, respecting the `-1` flag for skipping checksums entirely."+ "Gets the previous and current checksum for the analytics directory, respecting the `-1` flag for skipping checksums+ entirely."[](let [last-checksum (audit/last-analytics-checksum)](if (should-skip-checksum? last-checksum)- [SKIP_CHECKSUM_FLAG SKIP_CHECKSUM_FLAG]+ [skip-checksum-flag skip-checksum-flag][last-checksum (analytics-checksum)])))(defn- maybe-load-analytics-content!@@ -284,7 +285,7 @@;; Tests need the sync to complete before they run@sync-future))))))))))-(defn- maybe-install-audit-db+(defn- maybe-install-audit-db](cond@@ -310,7 +311,7 @@content if it is available.":feature :none[]- (u/prog1 (maybe-install-audit-db)+ (u/prog1 (maybe-install-audit-db!)(when-let [audit-db (t2/select-one :model/Database :is_audit true)];; prevent sync while loading((sync-util/with-duplicate-ops-preventeddiff --git a/enterprise/backend/src/metabase_enterprise/audit_app/permissions.clj b/enterprise/backend/src/metabase_enterprise/audit_app/permissions.cljindex 0d24aa662ac..386740f815c 100644--- a/enterprise/backend/src/metabase_enterprise/audit_app/permissions.clj+++ b/enterprise/backend/src/metabase_enterprise/audit_app/permissions.clj@@ -6,8 +6,8 @@[metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.query-permissions.core :as query-perms]- ;; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ;; legacy usage -- don't do things like this going forward+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.i18n :refer [tru]][toucan2.core :as t2]))diff --git a/enterprise/backend/src/metabase_enterprise/cache/strategies.clj b/enterprise/backend/src/metabase_enterprise/cache/strategies.cljindex 46c3f4af2ae..0ea13f37525 100644--- a/enterprise/backend/src/metabase_enterprise/cache/strategies.clj+++ b/enterprise/backend/src/metabase_enterprise/cache/strategies.clj@@ -15,10 +15,12 @@;; Data shape+;;; This is basically the same schema as `:metabase.cache.api/cache-strategy.ee` except it adds optional+;;; `:invalidated-at` keys(mr/def ::cache-strategy"Schema for a caching strategy used internally"[:and- :metabase.cache.api/cache-strategy.base+ :metabase.cache.api/cache-strategy.base.ee[:multi {:dispatch :type}[:nocache :metabase.cache.api/cache-strategy.nocache][:ttl [:merge@@ -39,7 +41,8 @@(defenterprise-schema cache-strategy :- [:maybe ::cache-strategy]"Returns the granular cache strategy for a card.":feature :cache-granular-controls- [card dashboard-id]+ [card :- :metabase.queries.schema/card+ dashboard-id :- [:maybe :metabase.lib.schema.id/dashboard]](let [qs (for [[i model model-id] [[1 "question" (:id card)][2 "dashboard" dashboard-id][3 "database" (:database_id card)]diff --git a/enterprise/backend/src/metabase_enterprise/database_routing/api.clj b/enterprise/backend/src/metabase_enterprise/database_routing/api.cljindex a8eb1562224..aa2623a2d6b 100644--- a/enterprise/backend/src/metabase_enterprise/database_routing/api.clj+++ b/enterprise/backend/src/metabase_enterprise/database_routing/api.clj@@ -75,7 +75,6 @@(let [db (t2/select-one :model/Database db-id)](when-not (driver.u/supports? (:engine db) :database-routing db)(throw (ex-info "This database does not support DB routing" {:status-code 400})))-(events/publish-event! :event/database-update {:object db:previous-object db:user-id api/*current-user-id*diff --git a/enterprise/backend/src/metabase_enterprise/database_routing/middleware.clj b/enterprise/backend/src/metabase_enterprise/database_routing/middleware.cljindex c0fc819acb8..14f66fd2a56 100644--- a/enterprise/backend/src/metabase_enterprise/database_routing/middleware.clj+++ b/enterprise/backend/src/metabase_enterprise/database_routing/middleware.clj@@ -9,7 +9,7 @@[metabase.lib.metadata :as lib.metadata][metabase.premium-features.core :refer [defenterprise]];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u]))(defenterprise swap-destination-dbdiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 4ddf2874033..29396ae49b8 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -7,7 +7,7 @@[metabase.api.routes.common :refer [+auth]][metabase.api.util.handlers :as handlers][metabase.collections.models.collection.root :as collection.root]- [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.metadata :as lib.metadata][metabase.models.interface :as mi][metabase.native-query-snippets.core :as native-query-snippets]@@ -19,11 +19,11 @@[toucan2.core :as t2]))(mr/def ::card-body- [:map- [:id {:optional false} ms/PositiveInt]- [:dataset_query {:optional true} [:maybe ms/Map]]- [:type {:optional true} [:maybe ::queries.schema/card-type]]- [:result_metadata {:optional true} [:maybe analyze/ResultsMetadata]]])+ [:merge+ ::queries.schema/card+ ;; TODO (Cam 10/1/25) -- merge this into `::queries.schema/card` itself+ [:map+ [:result_metadata {:optional true} [:maybe analyze/ResultsMetadata]]]])(defn- broken-cards-response[{:keys [card transform]}]@@ -54,7 +54,7 @@_query-paramsbody :- ::card-body](let [database-id (-> body :dataset_query :database)- base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)+ base-provider (lib-be/application-database-metadata-provider database-id)original (lib.metadata/card base-provider (:id body))card (-> original(assoc :dataset-query (:dataset_query body)@@ -73,7 +73,11 @@[:map[:id {:optional false} ms/PositiveInt][:name {:optional true} :string]- [:source {:optional true} [:maybe ms/Map]]+ ;; TODO (Cam 10/8/25) -- no idea what the correct schema for these is supposed to be -- it was just `map` before --+ ;; this is my attempt to guess it+ [:source {:optional true} [:maybe [:map+ [:type {:optional true} :keyword]+ [:query {:optional true} ::queries.schema/query]]]][:target {:optional true} [:maybe ms/Map]]])(api.macros/defendpoint :post "/check_transform"@@ -84,7 +88,7 @@(if (= (keyword (:type source)):query)(let [database-id (-> source :query :database)- base-provider (lib-be.metadata.jvm/application-database-metadata-provider database-id)+ base-provider (lib-be/application-database-metadata-provider database-id)original (lib.metadata/transform base-provider id)transform (-> original(cond-> #_transform source (assoc :source source))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 085f9936c03..a31ec9cb13f 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -1,59 +1,49 @@(ns metabase-enterprise.dependencies.calculation(:require[metabase-enterprise.dependencies.native-validation :as deps.native]- [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase-enterprise.dependencies.schema :as deps.schema]+ [metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.schema.metadata :as lib.schema.metadata]- [metabase.lib.util :as lib.util]+ [metabase.lib.schema :as lib.schema]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli :as mu]))-(defn- upstream-deps:mbql-query [legacy-query]- (lib.util/source-tables-and-cards [legacy-query]))+(mu/defn- upstream-deps:mbql-query :- ::deps.schema/upstream-deps+ [query :- ::lib.schema/query]+ {:card (or (lib/all-source-card-ids query) #{})+ :table (or (lib/all-source-table-ids query) #{})})-(defn- upstream-deps:native-query [metadata-provider query]- (let [engine (:engine (lib.metadata/database metadata-provider))- deps (deps.native/native-query-deps engine metadata-provider query)]- ;; The deps are in #{{:table 7} ...} form and need conversion to ::upstream-deps form.+(mu/defn- upstream-deps:native-query :- ::deps.schema/upstream-deps+ [query :- ::lib.schema/native-only-query]+ (let [driver (:engine (lib.metadata/database query))+ deps (deps.native/native-query-deps driver query)]+ ;; The deps are in #{{:table 7} ...} form and need conversion to ::deps.schema/upstream-deps form.(u/group-by ffirst (comp second first) conj #{} deps)))-(mr/def ::upstream-deps- [:map- [:card {:optional true} [:set ::lib.schema.id/card]]- [:table {:optional true} [:set ::lib.schema.id/table]]- [:snippet {:optional true} [:set ::lib.schema.id/snippet]]- [:transform {:optional true} [:set ::lib.schema.id/transform]]])+(mu/defn- upstream-deps:query :- ::deps.schema/upstream-deps+ [query :- ::lib.schema/query]+ (if (lib/native-only-query? query)+ (upstream-deps:native-query query)+ (upstream-deps:mbql-query query)))-(defn- upstream-deps:query [metadata-provider legacy-query]- (case (:type legacy-query)- (:query "query") (upstream-deps:mbql-query legacy-query)- (:native "native") (upstream-deps:native-query metadata-provider legacy-query)- (throw (ex-info "Unhandled kind of query" {:query legacy-query}))))--(mu/defn upstream-deps:card :- ::upstream-deps+(mu/defn upstream-deps:card :- ::deps.schema/upstream-deps"Given a Toucan `:model/Card`, return its upstream dependencies as a map from the kind to a set of IDs."- ([card] (upstream-deps:card (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))- card))- ([metadata-provider :- ::lib.schema.metadata/metadata-provider- {query :dataset_query :as _toucan-card}]- (upstream-deps:query metadata-provider query)))+ [{query :dataset_query :as _toucan-card} :- ::queries.schema/card]+ (upstream-deps:query query))-(mu/defn upstream-deps:transform :- ::upstream-deps+(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps"Given a Transform (in Toucan form), return its upstream dependencies."- ([transform] (upstream-deps:transform nil transform))- ([metadata-provider :- [:maybe ::lib.schema.metadata/metadata-provider]- {{:keys [query type]} :source :as transform}]- (if (#{"query" :query} type)- (upstream-deps:query (or metadata-provider- (lib-be.metadata.jvm/application-database-metadata-provider (:database query)))- query)- (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) type)- {}))))+ [{{:keys [query], source-type :type} :source :as transform} :- [:map+ [:source [:map+ [:query ::lib.schema/query]]]]]+ (if (= (keyword source-type) :query)+ (upstream-deps:query query)+ (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)+ {})))-(mu/defn upstream-deps:snippet :- ::upstream-deps+(mu/defn upstream-deps:snippet :- ::deps.schema/upstream-deps"Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."[{:keys [template_tags] :as _snippet}](let [type->id-key {:card :card-id, :snippet :snippet-id}diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj b/enterprise/backend/src/metabase_enterprise/dependencies/core.cljindex de034ba06fd..b5308e117f5 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/core.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/core.clj@@ -7,9 +7,10 @@[metabase-enterprise.dependencies.models.dependency :as deps.graph][metabase-enterprise.dependencies.native-validation :as deps.native][metabase.graph.core :as graph]- [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util :as u][metabase.util.log :as log]@@ -39,27 +40,15 @@(let [dependents (or dependents (deps.graph/transitive-dependents graph updated-entities))](deps.provider/override-metadata-provider base-provider updated-entities dependents))))-(defmulti check-query-inner- "Given a `MetadataProvider` and a lib query, find any bad refs in that query."- {:arglists '([metadata-provider driver query])}- (fn [_mp _driver query]- (-> (lib/query-stage query 0)- :lib/type)))+(mu/defn- check-query+ "Find any bad refs in a `query`."+ [driver :- :keyword+ query :- ::lib.schema/query]+ (if (lib/any-native-stage? query)+ (deps.native/validate-native-query driver query)+ (lib/find-bad-refs query)))-(defmethod check-query-inner :mbql.stage/mbql- [_mp _driver query]- (lib/find-bad-refs query))--(defmethod check-query-inner :mbql.stage/native- [mp driver query]- (deps.native/validate-native-query driver mp query))--(defn- check-query- "Given a `MetadataProvider` and a query, find any bad refs in that query."- [mp driver query]- (check-query-inner mp driver (lib/query mp query)))--(defmulti check-entity+(defmulti ^:private check-entity"Given a `MetadataProvider`, and entity type, and an entity id, find any bad refs in that entity."{:arglists '([metadata-provider entity-type entity-id])}(fn [_mp entity-type _entity-id]@@ -69,18 +58,21 @@[_mp _entity-type _entity-id]nil)+;; TODO (Cam 10/2/25) -- we already have keywords for these things, `:model/Card` and `:model/Transform`, can use+;; those for consistency and discoverability?(defmethod check-entity :card- [mp _entity-type entity-id]- (let [query (:dataset-query (lib.metadata/card mp entity-id))- driver (:engine (lib.metadata/database mp))]- (check-query mp driver query)))+ [mp _entity-type card-id]+ (let [query (lib/query mp (:dataset-query (lib.metadata/card mp card-id)))+ driver (:engine (lib.metadata/database query))]+ (check-query driver query)))(defmethod check-entity :transform[mp _entity-type entity-id](let [{{target-schema :schema target-name :name} :target- {:keys [query]} :source+ {query :query} :source:as _transform} (lib.metadata/transform mp entity-id)driver (:engine (lib.metadata/database mp))+ query (lib/query mp query)output-table (some #(when (and (= (:schema %) target-schema)(= (:name %) target-name))%)@@ -95,7 +87,7 @@:duplicate-fields duplicate-fields})){:seen #{}}output-fields)]- (into duplicate-fields (check-query mp driver query))))+ (into duplicate-fields (check-query driver query))))(mu/defn- check-query-soundness ;; :- [:map-of ::lib.schema.id/card [:sequential ::lib.schema.mbql-clause/clause]]"Given a `MetadataProvider` as returned by [[metadata-provider]], scan all its updated entities and their dependents@@ -156,7 +148,7 @@(let [all-deps (deps.graph/transitive-dependents edits)by-db (group-by-db all-deps)](reduce (fn [errors [db-id deps]]- (-> (lib-be.metadata.jvm/application-database-metadata-provider db-id)+ (-> (lib-be/application-database-metadata-provider db-id)(metadata-provider edits :dependents deps)check-query-soundness(merge errors)))@@ -176,9 +168,10 @@#_{:clj-kondo/ignore [:unresolved-namespace]}(comment;; This should work on any fresh-ish Metabase instance; these are the built-in example questions.- (let [base-mp (lib-be.metadata.jvm/application-database-metadata-provider 39)+ (let [base-mp (lib-be/application-database-metadata-provider 39)transform (lib.metadata/transform base-mp 1)transform (-> transform+ ;; NOTE: NO RAW MANIPULATION OUTSIDE OF LIB! DO NOT EVER DO THIS PLEASE <3(update-in [:source :query :query :expressions];; Replacing the expression Age with Duration - this breaks downstream uses!update-keys (constantly "Fair Die"))@@ -201,5 +194,5 @@(metabase.premium-features.core/token-features)(let [card (t2/select-one :model/Card :id 121)- mp (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))]+ mp (lib-be/application-database-metadata-provider (:database_id card))](upstream-deps:card mp card)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.cljnew file mode 100644index 00000000000..7337a3e9c81--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj@@ -0,0 +1,122 @@+(ns metabase-enterprise.dependencies.events+ (:require+ [metabase-enterprise.dependencies.calculation :as deps.calculation]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase.events.core :as events]+ [metabase.premium-features.core :as premium-features]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++;; ## Maintaining the dependency graph+;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the+;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created+;; or changed at that point.++;; ### Cards+(derive ::card-deps :metabase/event)+(derive :event/card-create ::card-deps)+(derive :event/card-update ::card-deps)++(methodical/defmethod events/publish-event! ::card-deps+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/with-transaction [_conn]+ (models.dependency/replace-dependencies! :card (:id object) (deps.calculation/upstream-deps:card object))+ (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)+ (t2/update! :model/Card (:id object)+ {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))++(derive ::card-delete :metabase/event)+(derive :event/card-delete ::card-delete)++(methodical/defmethod events/publish-event! ::card-delete+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))++;; ### Snippets+(derive ::snippet-deps :metabase/event)+(derive :event/snippet-create ::snippet-deps)+(derive :event/snippet-update ::snippet-deps)++(methodical/defmethod events/publish-event! ::snippet-deps+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/with-transaction [_conn]+ (models.dependency/replace-dependencies! :snippet (:id object) (deps.calculation/upstream-deps:snippet object))+ (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)+ (t2/update! :model/NativeQuerySnippet (:id object)+ {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))++(derive ::snippet-delete :metabase/event)+(derive :event/snippet-delete ::snippet-delete)++(methodical/defmethod events/publish-event! ::snippet-delete+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))++;; ### Transforms+(derive ::transform-deps :metabase/event)+(derive :event/create-transform ::transform-deps)+(derive :event/update-transform ::transform-deps)++;; On *saving* a transform, the upstream deps of its query are computed and saved.+(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]+ (let [db-id (some-> source :query :database)+ downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency+ :from_entity_type :table+ :to_entity_type :transform+ :to_entity_id id)+ downstream-tables (when (seq downstream-table-ids)+ (t2/select :model/Table :id [:in downstream-table-ids]))+ outdated-tables (remove (fn [table]+ (and (= (:schema table) (:schema target))+ (= (:name table) (:name target))+ (or (not db-id)+ (= db-id (:db_id table)))))+ downstream-tables)+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)+ downstream-table-ids)]+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)+ (map :id) outdated-tables))]+ (t2/delete! :model/Dependency+ :from_entity_type :table+ :from_entity_id [:in outdated-downstream-table-ids]+ :to_entity_type :transform+ :to_entity_id id))))++(methodical/defmethod events/publish-event! ::transform-deps+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (t2/with-transaction [_conn]+ (models.dependency/replace-dependencies! :transform (:id object) (deps.calculation/upstream-deps:transform object))+ (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)+ (t2/update! :model/Transform (:id object) {:dependency_analysis_version models.dependency/current-dependency-analysis-version}))+ (drop-outdated-target-dep! object))))++(derive ::transform-delete :metabase/event)+(derive :event/delete-transform ::transform-delete)++(methodical/defmethod events/publish-event! ::transform-delete+ [_ {:keys [id]}]+ (when (premium-features/has-feature? :dependencies)+ ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?+ (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))++;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.+;; (And if the target has changed, the old table's dep on the transform is dropped.)+;; The upstream deps of the transform are not touched - those change only when the transform is edited.+(derive ::transform-run :metabase/event)+(derive :event/transform-run-complete ::transform-run)++(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]+ (let [;; output-table is a keyword like :my_schema/my_table+ table-name (name output-table)]+ (when-let [table-id (t2/select-one-fn :id :model/Table :db_id db-id :schema output-schema :name table-name)]+ (models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))))++(methodical/defmethod events/publish-event! ::transform-run+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (transform-table-deps! object)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.cljindex 51eb03b6482..3903f741651 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj@@ -1,4 +1,6 @@(ns metabase-enterprise.dependencies.init(:require- [metabase-enterprise.dependencies.models.dependency]+ [metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.schema]+ [metabase-enterprise.dependencies.settings][metabase-enterprise.dependencies.task.backfill]))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.cljindex 4454ade9055..05f18d6719d 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj@@ -112,11 +112,10 @@(defn- get-returned-columns [mp queryable](let [query (lib/query mp queryable)]- (if (= (:lib/type (lib/query-stage query 0))- :mbql.stage/mbql)- (lib/returned-columns query)+ (if (lib/native-only-query? query)(deps.native/native-result-metadata (:engine (lib.metadata/database mp))- mp query))))+ query)+ (lib/returned-columns query))))(defmethod add-override :card [^OverridingMetadataProvider mp _entity-type id updates](with-overrides mpdiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex d8320e6b6eb..043ec6c7738 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -1,11 +1,8 @@(ns metabase-enterprise.dependencies.models.dependency(:require[clojure.set :as set]- [metabase-enterprise.dependencies.calculation :as deps.calculation]- [metabase.events.core :as events][metabase.graph.core :as graph][metabase.models.interface :as mi]- [metabase.premium-features.core :as premium-features][metabase.util :as u][methodical.core :as methodical][potemkin :as p]@@ -72,7 +69,7 @@(->> (graph/transitive graph starters) ; This returns a flat list.(u/group-by first second conj #{})))))-(defn replace-dependencies+(defn replace-dependencies!"Replace the dependencies of the entity of type `entity-type` with id `entity-id` withthe ones specified in `dependencies-by-type`. "[entity-type entity-id dependencies-by-type]@@ -96,117 +93,3 @@(t2/delete! :model/Dependency :id [:in to-remove]))(when (seq to-add)(t2/insert! :model/Dependency to-add)))))--;; ## Maintaining the dependency graph-;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the-;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created-;; or changed at that point.--;; ### Cards-(derive ::card-deps :metabase/event)-(derive :event/card-create ::card-deps)-(derive :event/card-update ::card-deps)--(methodical/defmethod events/publish-event! ::card-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (replace-dependencies :card (:id object) (deps.calculation/upstream-deps:card object))- (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)- (t2/update! :model/Card (:id object)- {:dependency_analysis_version current-dependency-analysis-version})))))--(derive ::card-delete :metabase/event)-(derive :event/card-delete ::card-delete)--(methodical/defmethod events/publish-event! ::card-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))--;; ### Snippets-(derive ::snippet-deps :metabase/event)-(derive :event/snippet-create ::snippet-deps)-(derive :event/snippet-update ::snippet-deps)--(methodical/defmethod events/publish-event! ::snippet-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (replace-dependencies :snippet (:id object) (deps.calculation/upstream-deps:snippet object))- (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)- (t2/update! :model/NativeQuerySnippet (:id object)- {:dependency_analysis_version current-dependency-analysis-version})))))--(derive ::snippet-delete :metabase/event)-(derive :event/snippet-delete ::snippet-delete)--(methodical/defmethod events/publish-event! ::snippet-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))--;; ### Transforms-(derive ::transform-deps :metabase/event)-(derive :event/create-transform ::transform-deps)-(derive :event/update-transform ::transform-deps)--;; On *saving* a transform, the upstream deps of its query are computed and saved.-(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]- (let [db-id (some-> source :query :database)- downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency- :from_entity_type :table- :to_entity_type :transform- :to_entity_id id)- downstream-tables (when (seq downstream-table-ids)- (t2/select :model/Table :id [:in downstream-table-ids]))- outdated-tables (remove (fn [table]- (and (= (:schema table) (:schema target))- (= (:name table) (:name target))- (or (not db-id)- (= db-id (:db_id table)))))- downstream-tables)- not-found-table-ids (remove (into #{} (map :id) downstream-tables)- downstream-table-ids)]- (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)- (map :id) outdated-tables))]- (t2/delete! :model/Dependency- :from_entity_type :table- :from_entity_id [:in outdated-downstream-table-ids]- :to_entity_type :transform- :to_entity_id id))))--(methodical/defmethod events/publish-event! ::transform-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (replace-dependencies :transform (:id object) (deps.calculation/upstream-deps:transform object))- (when (not= (:dependency_analysis_version object) current-dependency-analysis-version)- (t2/update! :model/Transform (:id object) {:dependency_analysis_version current-dependency-analysis-version}))- (drop-outdated-target-dep! object))))--(derive ::transform-delete :metabase/event)-(derive :event/delete-transform ::transform-delete)--(methodical/defmethod events/publish-event! ::transform-delete- [_ {:keys [id]}]- (when (premium-features/has-feature? :dependencies)- ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?- (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))--;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.-;; (And if the target has changed, the old table's dep on the transform is dropped.)-;; The upstream deps of the transform are not touched - those change only when the transform is edited.-(derive ::transform-run :metabase/event)-(derive :event/transform-run-complete ::transform-run)--(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]- (let [;; output-table is a keyword like :my_schema/my_table- table-name (name output-table)]- (when-let [table-id (t2/select-one-fn :id :model/Table :db_id db-id :schema output-schema :name table-name)]- (replace-dependencies :table table-id {:transform #{transform-id}}))))--(methodical/defmethod events/publish-event! ::transform-run- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (transform-table-deps! object)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.cljindex 51291a74390..3db805cb2d4 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj@@ -2,37 +2,55 @@(:require[metabase.driver :as driver][metabase.lib.core :as lib]- [metabase.query-processor :as qp]))+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.util.malli :as mu]))-(defn- compile-query [metadata-provider query]- (->> (lib/query metadata-provider query)- lib/add-parameters-for-template-tags- (qp/compile-query-with-metadata-provider metadata-provider)- :query))+(mu/defn- compile-query :- ::lib.schema/native-only-query+ [query :- ::lib.schema/query]+ (-> query+ lib/add-parameters-for-template-tags+ ;; to expand template tag parameters+ qp.preprocess/preprocess+ (as-> $query (if (lib/native-only-query? $query)+ $query+ (lib/native-query $query (:query (qp.compile/compile $query)))))))-(defn validate-native-query+(mu/defn validate-native-query"Compiles a (native) query and validates that the fields and tables it refers to really exist.Returns either nil or a list of errors."- [driver metadata-provider query]- (->> (compile-query metadata-provider query)- (driver/validate-native-query-fields driver metadata-provider)))+ [driver :- :keyword+ query :- ::lib.schema/query]+ (->> query+ compile-query+ (driver/validate-native-query-fields driver)))-(defn native-result-metadata+(mu/defn native-result-metadata"Compiles a (native) query and calculates its result metadata"- [driver metadata-provider query]- (->> (compile-query metadata-provider query)- (driver/native-result-metadata driver metadata-provider)))+ [driver :- :keyword+ query :- ::lib.schema/query]+ (->> query+ compile-query+ (driver/native-result-metadata driver)))-(defn native-query-deps+(mu/defn native-query-deps :- [:set+ [:or+ ::driver/native-query-deps.table-dep+ ::driver/native-query-deps.transform-dep+ [:map {:closed true} [:snippet ::lib.schema.id/snippet]]+ [:map {:closed true} [:card ::lib.schema.id/card]]]]"Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs."- [driver metadata-provider query]- (let [compiled (compile-query metadata-provider query)]- (into (driver/native-query-deps driver compiled metadata-provider)+ [driver :- :keyword+ query :- ::lib.schema/native-only-query]+ (let [compiled (compile-query query)]+ (into (driver/native-query-deps driver compiled)+ ;; TODO (Cam 10/1/25) -- Even this much MBQL manipulation outside of Lib is illegal. Move this sort of stuff+ ;; into Lib.(keep #(case (:type %)- "snippet" {:snippet (:snippet-id %)}- :snippet {:snippet (:snippet-id %)}- "card" {:card (:card-id %)}- :card {:card (:card-id %)}+ :snippet {:snippet (:snippet-id %)}+ :card {:card (:card-id %)}nil))- (-> query :native :template-tags vals))))+ (lib/all-template-tags query))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj b/enterprise/backend/src/metabase_enterprise/dependencies/schema.cljnew file mode 100644index 00000000000..4e1d7fd43dc--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj@@ -0,0 +1,11 @@+(ns metabase-enterprise.dependencies.schema+ (:require+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.util.malli.registry :as mr]))++(mr/def ::upstream-deps+ [:map+ [:card {:optional true} [:set ::lib.schema.id/card]]+ [:table {:optional true} [:set ::lib.schema.id/table]]+ [:snippet {:optional true} [:set ::lib.schema.id/snippet]]+ [:transform {:optional true} [:set ::lib.schema.id/transform]]])diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/settings.cljnew file mode 100644index 00000000000..995e6b1e12b--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/settings.clj@@ -0,0 +1,27 @@+(ns metabase-enterprise.dependencies.settings+ (:require+ [metabase.settings.core :refer [defsetting]]))++(defsetting dependency-backfill-batch-size+ "Batch size."+ :visibility :internal+ :type :integer+ :default 20+ :setter :none+ :export? false)++(defsetting dependency-backfill-delay-minutes+ "Backfill delay in minutes."+ :visibility :internal+ :type :integer+ :default 60+ :setter :none+ :export? false)++(defsetting dependency-backfill-variance-minutes+ "Backfill variance (?) whatever that means."+ :visibility :internal+ :type :integer+ :default 10+ :setter :none+ :export? false)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljindex 09a19f3c038..1621f79f8c3 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj@@ -12,9 +12,9 @@[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.scheduler :as qs][clojurewerkz.quartzite.triggers :as triggers]- [environ.core :as env][java-time.api :as t][metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.settings :as deps.settings][metabase.config.core :as config][metabase.events.core :as events][metabase.premium-features.core :as premium-features]@@ -43,22 +43,9 @@(def ^:private terminally-broken(zipmap entities (repeatedly #(ConcurrentHashMap/newKeySet))))-(def ^:private MAX_RETRIES 5)+(def ^:private max-retries 5)-(defn- long-env-value- [env-var-kw default]- (or (parse-long (env/env env-var-kw "")) default))--(defn- get-batch-size []- (long-env-value :mb-dependency-backfill-batch-size 20))--(defn- get-delay-minutes []- (long-env-value :mb-dependency-backfill-delay-minutes 60))--(defn- get-variance-minutes []- (long-env-value :mb-dependency-backfill-variance-minutes 10))--(defn- get-processable-ids [model-kw batch-size]+(defn- processable-ids [model-kw batch-size](let [target-version models.dependency/current-dependency-analysis-versionterminally-broken-ids ^Set (terminally-broken model-kw)retry-state-map ^Map (retry-state model-kw)@@ -99,7 +86,7 @@target-version models.dependency/current-dependency-analysis-versionretry-state-map ^Map (retry-state model-kw)terminally-broken-set ^Set (terminally-broken model-kw)- ids (get-processable-ids model-kw batch-size)] ; Use the new get-processable-ids+ ids (processable-ids model-kw batch-size)] ; Use the new get-processable-ids(when (seq ids)(log/infof "Processing a batch of %s %s(s)..." (count ids) model-name))(reduce (fn [total id]@@ -113,9 +100,9 @@(let [current-time (current-millis)entity-retry-info (.get retry-state-map id)failure-count (inc (:fail-count entity-retry-info 0))- retry-minutes (* failure-count (get-delay-minutes))+ retry-minutes (* failure-count (deps.settings/dependency-backfill-delay-minutes))new-next-retry-timestamp (+ current-time (* retry-minutes 60 1000))]- (if (> failure-count MAX_RETRIES)+ (if (> failure-count max-retries)(do (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."model-name id failure-count)(.add terminally-broken-set id)@@ -128,7 +115,7 @@0ids)))-(defn- backfill-dependencies+(defn- backfill-dependencies!"Job to backfill dependencies for all entities.Returns true if a full batch has been selected, nil or false otherwise."[]@@ -140,7 +127,7 @@(when (pos? processed)(log/info "Updated" processed "entities."))(- batch-size processed))))- (get-batch-size)+ (deps.settings/dependency-backfill-batch-size)entities)(< 1))))@@ -156,12 +143,12 @@org.quartz.DisallowConcurrentExecution true}BackfillDependencies [ctx](log/info "Executing BackfillDependencies job...")- (let [full-batch-selected? (backfill-dependencies)+ (let [full-batch-selected? (backfill-dependencies!)retries? (has-pending-retries?)](if (or full-batch-selected?retries?)- (let [delay-seconds (* (get-delay-minutes) 60)- variance-seconds (* (get-variance-minutes) 60)+ (let [delay-seconds (* (deps.settings/dependency-backfill-delay-minutes) 60)+ variance-seconds (* (deps.settings/dependency-backfill-variance-minutes) 60)delay-in-seconds (max 0 (+ (- delay-seconds variance-seconds) (rand-int (* 2 variance-seconds))))](schedule-next-run! delay-in-seconds (.getScheduler ctx)))(log/info "No more entities to backfill for, stopping."))))@@ -186,6 +173,6 @@(task/schedule-task! job trigger))))))(defmethod task/init! ::DependencyBackfill [_]- (if (pos? (get-batch-size))- (schedule-next-run! (rand-int (* (get-variance-minutes) 60)))+ (if (pos? (deps.settings/dependency-backfill-batch-size))+ (schedule-next-run! (rand-int (* (deps.settings/dependency-backfill-variance-minutes) 60)))(log/info "Not starting dependency backfill job because the batch size is not positive")))diff --git a/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj b/enterprise/backend/src/metabase_enterprise/impersonation/middleware.cljindex 70dede01ab8..88e96b8236f 100644--- a/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj+++ b/enterprise/backend/src/metabase_enterprise/impersonation/middleware.clj@@ -4,7 +4,7 @@[metabase.lib.metadata :as lib.metadata][metabase.premium-features.core :refer [defenterprise]];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]))+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store]))(defenterprise apply-impersonation"Pre-processing middleware. Adds a key to the query. Currently used solely for caching."diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.cljindex 1ef36343f10..29de408b4ae 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/dummy_tools.clj@@ -5,7 +5,7 @@[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u][metabase.api.common :as api][metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.types.isa :as lib.types.isa]@@ -68,7 +68,7 @@([id] (metric-details id nil))([id options](when-let [card (metabot-v3.tools.u/get-card id)]- (metric-details card (lib.metadata.jvm/application-database-metadata-provider (:database_id card)) options)))+ (metric-details card (lib-be/application-database-metadata-provider (:database_id card)) options)))([card metadata-provider {:keys [field-values-fn with-default-temporal-breakout? with-queryable-dimensions?]:or {field-values-fn add-field-valueswith-default-temporal-breakout? true@@ -143,7 +143,7 @@db-id (if metadata-provider (:db-id base) (:db_id base))mp (when query-needed?(or metadata-provider- (lib.metadata.jvm/application-database-metadata-provider db-id)))+ (lib-be/application-database-metadata-provider db-id)))table-query (when query-needed?(lib/query mp (lib.metadata/table mp id)))cols (when with-fields?@@ -190,7 +190,7 @@([id] (card-details id nil))([id options](when-let [card (metabot-v3.tools.u/get-card id)]- (card-details card (lib.metadata.jvm/application-database-metadata-provider (:database_id card)) options)))+ (card-details card (lib-be/application-database-metadata-provider (:database_id card)) options)))([base metadata-provider {:keys [field-values-fn with-fields? with-related-tables? with-metrics?]:or {field-values-fn add-field-valueswith-fields? true@@ -234,7 +234,7 @@"Get the details of metrics or models as specified by `card-type` and `cards`from the database with ID `database-id` respecting `options`."[card-type database-id cards options]- (let [mp (lib.metadata.jvm/application-database-metadata-provider database-id)+ (let [mp (lib-be/application-database-metadata-provider database-id)detail-fn (case card-type:metric metric-details:model card-details)]@@ -248,7 +248,7 @@([metabot-id](answer-sources metabot-id nil))([metabot-id options]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(let [metrics-and-models (metabot-v3.tools.u/get-metrics-and-models metabot-id){metrics :metric, models :model}(->> (for [[[card-type database-id] cards] (group-by (juxt :type :database_id) metrics-and-models)@@ -282,7 +282,7 @@`model-id` is an integer ID of a model (card). Exactly one of `table-id` or `model-id`should be supplied."[{:keys [model-id table-id] :as arguments}]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(let [options (cond-> arguments(= (:with-field-values? arguments) false) (assoc :field-values-fn identity))details (cond@@ -312,7 +312,7 @@(defn get-metric-details"Get information about the metric with ID `metric-id`."[{:keys [metric-id] :as arguments}]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(let [options (cond-> arguments(= (:with-field-values? arguments) false) (assoc :field-values-fn identity))details (if (int? metric-id)@@ -325,7 +325,7 @@(defn get-report-details"Get information about the report (card) with ID `report-id`."[{:keys [report-id] :as arguments}]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(let [options (cond-> arguments(= (:with-field-values? arguments) false) (assoc :field-values-fn identity))details (if (int? report-id)@@ -358,7 +358,7 @@field-id-prefix (metabot-v3.tools.u/query-field-id-prefix query-id)database-id (:database legacy-query)_ (api/read-check :model/Database database-id)- mp (lib.metadata.jvm/application-database-metadata-provider database-id)+ mp (lib-be/application-database-metadata-provider database-id)query (lib/query mp legacy-query)returned-cols (lib/returned-columns query)]{:type :query@@ -371,5 +371,5 @@(defn get-query-details"Get the details of a (legacy) query."[{:keys [query]}]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache{:structured-output (execute-query (u/generate-nano-id) query)}))diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/suggested_prompts.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/suggested_prompts.cljindex 1c29491f905..9ac7e28bdf5 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/suggested_prompts.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/suggested_prompts.clj@@ -4,7 +4,7 @@[metabase-enterprise.metabot-v3.client :as metabot-v3.client][metabase-enterprise.metabot-v3.dummy-tools :as metabot-v3.dummy-tools][metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][toucan2.core :as t2]))(defn- column-input@@ -35,7 +35,7 @@"Generate suggested propmpts for instance of Metabot."[metabot-id & {:as opts}](let [opts (merge default-opts opts)]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(let [{metrics :metric models :model} (->> (metabot-v3.tools.u/get-metrics-and-models metabot-id opts)(sort-by :view_count >)(group-by :type))diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.cljindex 475e0da0fc3..562b09a5ddd 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/table_utils.clj@@ -5,7 +5,7 @@[metabase-enterprise.metabot-v3.query-analyzer :as query-analyzer][metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u][metabase.api.common :as api]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.interface :as mi]@@ -100,8 +100,8 @@(exclude-table-ids (:id %))) fill-tables)all-tables (concat priority-tables fill-tables)all-tables (take all-tables-limit all-tables)]- (lib.metadata.jvm/with-metadata-provider-cache- (let [mp (lib.metadata.jvm/application-database-metadata-provider database-id)+ (lib-be/with-metadata-provider-cache+ (let [mp (lib-be/application-database-metadata-provider database-id)table-ids (map :id all-tables)_ (lib.metadata/bulk-metadata mp :metadata/table table-ids)](mapv (fn [{:keys [id name schema description]}]diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.cljindex 1d5869a1e9a..fb8d29a5f1b 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/filters.clj@@ -2,7 +2,7 @@(:require[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u][metabase.api.common :as api]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options]@@ -148,7 +148,7 @@(defn- query-metric*[{:keys [metric-id filters group-by] :as _arguments}](let [card (metabot-v3.tools.u/get-card metric-id)- mp (lib.metadata.jvm/application-database-metadata-provider (:database_id card))+ mp (lib-be/application-database-metadata-provider (:database_id card))base-query (->> (lib/query mp (lib.metadata/card mp metric-id))lib/remove-all-breakouts)visible-cols (lib/visible-columns base-query)@@ -232,7 +232,7 @@(defn- query-model*[{:keys [model-id fields filters aggregations group-by order-by limit] :as _arguments}](let [card (metabot-v3.tools.u/get-card model-id)- mp (lib.metadata.jvm/application-database-metadata-provider (:database_id card))+ mp (lib-be/application-database-metadata-provider (:database_id card))base-query (lib/query mp (lib.metadata/card mp model-id))visible-cols (lib/visible-columns base-query)filter-field-id-prefix (metabot-v3.tools.u/card-field-id-prefix model-id)@@ -352,7 +352,7 @@handle-query (fn [query query-id](let [database-id (:database query)_ (api/read-check :model/Database database-id)- mp (lib.metadata.jvm/application-database-metadata-provider database-id)]+ mp (lib-be/application-database-metadata-provider database-id)][(if query-id(metabot-v3.tools.u/query-field-id-prefix query-id)metabot-v3.tools.u/any-prefix-pattern)diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/util.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/util.cljindex d2dff5d8d64..52db438ce5a 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/util.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/tools/util.clj@@ -5,7 +5,7 @@[metabase.api.common :as api][metabase.audit-app.core :as audit-app][metabase.collections.models.collection :as collection]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.types.isa :as lib.types.isa]@@ -117,7 +117,7 @@"Return a query based on the model with ID `model-id`."[card-id](when-let [card (get-card card-id)]- (let [mp (lib.metadata.jvm/application-database-metadata-provider (:database_id card))]+ (let [mp (lib-be/application-database-metadata-provider (:database_id card))](lib/query mp (cond-> (lib.metadata/card mp card-id);; pivot questions have strange result-columns so we work with the dataset-query(#{:question} (:type card)) (get :dataset-query))))))@@ -126,14 +126,14 @@"Return a query based on the model with ID `model-id`."[metric-id](when-let [card (get-card metric-id)]- (let [mp (lib.metadata.jvm/application-database-metadata-provider (:database_id card))]+ (let [mp (lib-be/application-database-metadata-provider (:database_id card))](lib/query mp (lib.metadata/metric mp metric-id)))))(defn table-query"Return a query based on the table with ID `table-id`."[table-id](when-let [table (get-table table-id :db_id)]- (let [mp (lib.metadata.jvm/application-database-metadata-provider (:db_id table))]+ (let [mp (lib-be/application-database-metadata-provider (:db_id table))](lib/query mp (lib.metadata/table mp table-id)))))(defn metabot-metrics-and-models-querydiff --git a/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj b/enterprise/backend/src/metabase_enterprise/permission_debug/impl.cljindex 332e4a3a466..2aa8796a0e3 100644--- a/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/permission_debug/impl.clj@@ -14,7 +14,6 @@[clojure.set :as set][metabase.api.common :as api][metabase.lib.core :as lib]- [metabase.lib.util :as lib.util][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi][metabase.permissions.core :as perms]@@ -176,9 +175,8 @@Returns a map of blocked tables by group in the format:{[db-name schema table-name] #{group-name-1 group-name-2}}"[user-id card permissions-blocking permissions-granting]- ;; legacy usage -- don't do things like this going forward- (let [query (-> card :dataset_query qp.preprocess/preprocess #_{:clj-kondo/ignore [:discouraged-var]} lib/->legacy-MBQL)- query-tables (-> query :query #_{:clj-kondo/ignore [:deprecated-var]} lib.util/collect-source-tables)+ (let [query (-> card :dataset_query qp.preprocess/preprocess)+ query-tables (lib/all-source-table-ids query)native? (boolean (lib.util.match/match-one query (m :guard (every-pred map? :native))))](->>(conddiff --git a/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj b/enterprise/backend/src/metabase_enterprise/sandbox/api/table.cljindex d120ee8d5b4..1f93e49f065 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/api/table.clj@@ -3,6 +3,7 @@[metabase-enterprise.sandbox.api.util :as sandbox.api.util][metabase.api.common :as api][metabase.api.macros :as api.macros]+ [metabase.lib.core :as lib][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.util :as u]@@ -41,14 +42,13 @@(defn- filter-fields-for-sandboxing [table query-metadata-response];; If we have sandboxed permissions and the associated sandbox limits the fields returned, we need to make sure;; the query_metadata endpoint also excludes any fields the sandbox query would exclude.- (if-let [sandbox-source-card (find-sandbox-source-card table api/*current-user-id*)]- (case (get-in sandbox-source-card [:dataset_query :type])- :native (update query-metadata-response :fields- (partial filter-fields-by-name- (->> sandbox-source-card :result_metadata (map :name) set)))- :query (update query-metadata-response :fields- (partial filter-fields-by-id- (->> sandbox-source-card :result_metadata (map u/id) set))))+ (if-let [{sandbox-query :dataset_query, :as sandbox-source-card} (find-sandbox-source-card table api/*current-user-id*)]+ (update query-metadata-response :fields+ (if (lib/native-only-query? sandbox-query)+ (partial filter-fields-by-name+ (->> sandbox-source-card :result_metadata (map :name) set))+ (partial filter-fields-by-id+ (->> sandbox-source-card :result_metadata (map u/id) set))));; Sandboxed via user attribute, not a source question, so no column-level sandboxing is in placequery-metadata-response))diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.cljindex 6bc20037f9d..b3c644846a6 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/query_processor/middleware/sandboxing.clj@@ -19,6 +19,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.lib.schema.util :as lib.schema.util][metabase.lib.util :as lib.util][metabase.lib.util.match :as lib.util.match][metabase.lib.walk :as lib.walk]@@ -285,7 +286,9 @@(merge native-col table-col))))original-table-cols))-(mu/defn- apply-sandbox-to-stage :- [:sequential {:min 2} ::lib.schema/stage]+(mu/defn- apply-sandbox-to-stage :- [:and+ [:sequential {:min 2} ::lib.schema/stage]+ ::lib.schema.util/unique-uuids]"Apply a Sandbox to a `stage`, returning a vector of replacement stages."[query :- ::lib.schema/query{:keys [source-table] :as stage} :- ::lib.schema/stage@@ -293,7 +296,9 @@(let [sandbox-query (sandbox->query query sandbox)sandbox-query (project-only-columns-from-original-table query sandbox-query source-table)new-source-stages (mapv (fn [stage]- (assoc stage :query-permissions/sandboxed-table source-table))+ (-> stage+ (assoc :query-permissions/sandboxed-table source-table)+ lib/fresh-uuids))(:stages sandbox-query));; merge stage metadata in the last source stage if needednew-source-stages (m/update-existing-in new-source-stagesdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj b/enterprise/backend/src/metabase_enterprise/serialization/cmd.cljindex 4fe3d810501..218cea64ced 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj@@ -3,9 +3,6 @@(:require[clojure.java.io :as io][clojure.string :as str]- [metabase-enterprise.serialization.dump :as dump]- [metabase-enterprise.serialization.load :as load]- [metabase-enterprise.serialization.serialize :as serialize][metabase-enterprise.serialization.v2.entity-ids :as v2.entity-ids][metabase-enterprise.serialization.v2.extract :as v2.extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest]@@ -18,61 +15,18 @@[metabase.premium-features.core :as premium-features][metabase.setup.core :as setup][metabase.util :as u]- [metabase.util.i18n :refer [deferred-trs trs]]+ [metabase.util.i18n :refer [trs]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.warehouse-schema.models.field :as field][toucan2.core :as t2])(:import(clojure.lang ExceptionInfo)))(set! *warn-on-reflection* true)-(def ^:private Mode- (mu/with-api-error-message [:enum :skip :update]- (deferred-trs "invalid --mode value")))--(def ^:private OnError- (mu/with-api-error-message [:enum :continue :abort]- (deferred-trs "invalid --on-error value")))--(def ^:private Context- (mu/with-api-error-message- [:map {:closed true}- [:on-error {:optional true} OnError]- [:mode {:optional true} Mode]]- (deferred-trs "invalid context seed value")))-(defn- check-premium-token! [](premium-features/assert-has-feature :serialization (trs "Serialization")))-(mu/defn v1-load!- "Load serialized metabase instance as created by [[dump]] command from directory `path`."- [path context :- Context]- (plugins/load-plugins!)- (mdb/setup-db! :create-sample-content? false)- (check-premium-token!)- (when-not (load/compatible? path)- (log/warn "Dump was produced using a different version of Metabase. Things may break!"))- (let [context (merge {:mode :skip- :on-error :continue}- context)]- (try- (log/infof "BEGIN LOAD from %s with context %s" path context)- (let [all-res [(load/load! (str path "/users") context)- (load/load! (str path "/databases") context)- (load/load! (str path "/collections") context)- (load/load-settings! path context)]- reload-fns (filter fn? all-res)]- (when (seq reload-fns)- (log/info "Finished first pass of load; now performing second pass")- (doseq [reload-fn reload-fns]- (reload-fn)))- (log/infof "END LOAD from %s with context %s" path context))- (catch Throwable e- (log/errorf e "ERROR LOAD from %s: %s" path (.getMessage e))- (throw e)))))-(mu/defn v2-load-internal!"SerDes v2 load entry point for internal users.@@ -138,97 +92,6 @@(throw (ex-info (ex-message @err) {:cmd/exit true})))imported))-(defn- select-entities-in-collections- ([model collections]- (select-entities-in-collections model collections :all))- ([model collections state]- (let [state-filter (case state- :all nil- :active [:= :archived false])]- (t2/select model {:where [:and- [:or [:= :collection_id nil]- (if (not-empty collections)- [:in :collection_id (map u/the-id collections)]- false)]- state-filter]}))))--(defn- select-segments-in-tables- ([tables]- (select-segments-in-tables tables :all))- ([tables state]- (case state- :all- (mapcat #(t2/select :model/Segment :table_id (u/the-id %)) tables)- :active- (filter- #(not (:archived %))- (mapcat #(t2/select :model/Segment :table_id (u/the-id %)) tables)))))--(defn- select-collections- "Selects the collections for a given user-id, or all collections without a personal ID if the passed user-id is nil.- If `state` is passed (by default, `:active`), then that will be used to filter for collections that are archived (if- the value is passed as `:all`)."- ([users]- (select-collections users :active))- ([users state]- (let [state-filter (case state- :all nil- :active [:= :archived false])- base-collections (t2/select :model/Collection {:where [:and [:= :location "/"]- [:or [:= :personal_owner_id nil]- [:= :personal_owner_id- (some-> users first u/the-id)]]- state-filter]})]- (if (empty? base-collections)- []- (-> (t2/select :model/Collection- {:where [:and- (reduce (fn [acc coll]- (conj acc [:like :location (format "/%d/%%" (:id coll))]))- [:or] base-collections)- state-filter]})- (into base-collections))))))--(defn v1-dump!- "Legacy Metabase app data dump"- [path {:keys [state user include-entity-id] :or {state :active} :as opts}]- (log/infof "BEGIN DUMP to %s via user %s" path user)- (mdb/setup-db! :create-sample-content? false)- (check-premium-token!)- (t2/select :model/User) ;; TODO -- why??? [editor's note: this comment originally from Cam]- (let [users (if user- (let [user (t2/select-one :model/User- :email user- :is_superuser true)]- (assert user (trs "{0} is not a valid user" user))- [user])- [])- databases (if (contains? opts :only-db-ids)- (t2/select :model/Database :id [:in (:only-db-ids opts)] {:order-by [[:id :asc]]})- (t2/select :model/Database))- tables (if (contains? opts :only-db-ids)- (t2/select :model/Table :db_id [:in (:only-db-ids opts)] {:order-by [[:id :asc]]})- (t2/select :model/Table))- fields (if (contains? opts :only-db-ids)- (t2/select :model/Field :table_id [:in (map :id tables)] {:order-by [[:id :asc]]})- (t2/select :model/Field))- collections (select-collections users state)]- (binding [serialize/*include-entity-id* (boolean include-entity-id)]- (dump/dump! path- databases- tables- (mapcat field/with-values (u/batches-of 32000 fields))- (select-segments-in-tables tables state)- collections- (select-entities-in-collections :model/NativeQuerySnippet collections state)- (select-entities-in-collections :model/Card collections state)- (select-entities-in-collections :model/Dashboard collections state)- (select-entities-in-collections :model/Pulse collections state)- users)))- (dump/dump-settings! path)- (dump/dump-dimensions! path)- (log/infof "END DUMP to %s via user %s" path user))-(defn v2-dump!"Exports Metabase app data to directory at path"[path {:keys [collection-ids] :as opts}]diff --git a/enterprise/backend/src/metabase_enterprise/serialization/dump.clj b/enterprise/backend/src/metabase_enterprise/serialization/dump.cljindex 48a5acb6a04..da945889d37 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/dump.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/dump.clj@@ -3,14 +3,7 @@(:require[clojure.edn :as edn][clojure.java.io :as io]- [metabase-enterprise.serialization.names :refer [fully-qualified-name name-for-logging safe-name]]- [metabase-enterprise.serialization.serialize :as serialize]- [metabase.config.core :as config]- [metabase.models.interface :as mi]- [metabase.settings.core :as setting]- [metabase.util.log :as log]- [metabase.util.yaml :as yaml]- [toucan2.core :as t2]))+ [metabase.util.yaml :as yaml]))(set! *warn-on-reflection* true)@@ -58,57 +51,3 @@(if-not (.canWrite (.getParentFile (io/file filename)))(throw (ex-info (format "Destination path is not writeable: %s" filename) {:filename filename}))(throw e)))))--(defn- as-file?- [instance]- (some (fn [model]- (mi/instance-of? model instance))- [:model/Pulse :model/Dashboard :model/Segment :model/Field :model/User]))--(defn- spit-entity!- [path entity]- (let [filename (if (as-file? entity)- (format "%s%s.yaml" path (fully-qualified-name entity))- (format "%s%s/%s.yaml" path (fully-qualified-name entity) (safe-name entity)))]- (when (.exists (io/as-file filename))- (log/warn (str filename " is about to be overwritten."))- (log/debug (str "With object: " (pr-str entity))))-- (spit-yaml! filename (serialize/serialize entity))))--(defn dump!- "Serialize entities into a directory structure of YAMLs at `path`."- [path & entities]- (doseq [entity (flatten entities)]- (try- (spit-entity! path entity)- (catch Throwable e- (log/errorf e "Error dumping %s" (name-for-logging entity)))))- (spit-yaml! (str path "/manifest.yaml")- {:serialization-version serialize/serialization-protocol-version- :metabase-version config/mb-version-info}))--(defn dump-settings!- "Combine all settings into a map and dump it into YAML at `path`."- [path]- (spit-yaml! (str path "/settings.yaml")- (into {} (for [{:keys [key value]} (setting/admin-writable-site-wide-settings- :getter (partial setting/get-value-of-type :string))]- [key value]))))--(defn dump-dimensions!- "Combine all dimensions into a vector and dump it into YAML at in the directory for the- corresponding schema starting at `path`."- [path]- (doseq [[table-id dimensions] (group-by (comp :table_id #(t2/select-one :model/Field :id %) :field_id)- (t2/select :model/Dimension))- :let [table (t2/select-one :model/Table :id table-id)]]- (spit-yaml! (if (:schema table)- (format "%s%s/schemas/%s/dimensions.yaml"- path- (->> table :db_id (fully-qualified-name :model/Database))- (:schema table))- (format "%s%s/dimensions.yaml"- path- (->> table :db_id (fully-qualified-name :model/Database))))- (mapv serialize/serialize dimensions))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/load.cljdeleted file mode 100644index 455adab4c43..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/load.clj+++ /dev/null@@ -1,802 +0,0 @@-(ns metabase-enterprise.serialization.load- "Load entities serialized by `metabase-enterprise.serialization.dump`."- (:refer-clojure :exclude [load])- (:require- [clojure.java.io :as io]- [clojure.string :as str]- [medley.core :as m]- [metabase-enterprise.serialization.names- :as names- :refer [fully-qualified-name->context]]- [metabase-enterprise.serialization.upsert :refer [maybe-upsert-many!]]- [metabase.app-db.core :as mdb]- [metabase.config.core :as config]- [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.legacy-mbql.util :as mbql.u]- [metabase.lib.util.match :as lib.util.match]- [metabase.models.visualization-settings :as mb.viz]- [metabase.settings.core :as setting]- [metabase.util.date-2 :as u.date]- [metabase.util.i18n :refer [trs]]- [metabase.util.log :as log]- [metabase.util.yaml :as yaml]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(defn- slurp-dir- [path]- (doall- (for [^java.io.File file (.listFiles ^java.io.File (io/file path))- :when (-> file (.getName) (str/ends-with? ".yaml"))]- (yaml/from-file file))))--(defn- slurp-many- [paths]- (apply concat (map slurp-dir paths)))--(defn- list-dirs- [path]- (for [^java.io.File file (.listFiles ^java.io.File (io/file path))- :when (.isDirectory file)]- (.getPath file)))--(defn- source-table- [source-table]- (if (and (string? source-table) (str/starts-with? source-table "card__"))- source-table- (let [{:keys [card table]} (fully-qualified-name->context source-table)]- (if card- (str "card__" card)- table))))--(defn- fq-table-or-card?- "Returns true if the given `nm` is either a fully qualified table name OR fully- qualified card name."- [nm]- (or (names/fully-qualified-table-name? nm) (names/fully-qualified-card-name? nm)))--(defn- update-capture-missing*- [m ks resolve-fn get-fn update-fn]- (let [orig-v (get-fn m ks)- res (update-fn m ks resolve-fn)- new-v (get-fn res ks)]- (if (and (some? orig-v) (nil? new-v))- (update res ::unresolved-names #(assoc % orig-v ks))- res)))--(defn- update-in-capture-missing- [m ks resolve-fn]- (update-capture-missing* m ks resolve-fn get-in update-in))--(defn- update-existing-in-capture-missing- [m ks resolve-fn]- (update-capture-missing* m ks resolve-fn get-in m/update-existing-in))--(defn- update-existing-capture-missing- [m k resolve-fn]- (update-capture-missing* m [k] resolve-fn get-in m/update-existing-in))--(defn- pull-unresolved-names-up- "Assocs the given value `v` to the given key sequence `ks` in the given map `m`. If the given `v` contains any- ::unresolved-names, these are \"pulled into\" `m` directly by prepending `ks` to their existing paths and dissocing- them from `v`."- ([m ks]- (pull-unresolved-names-up m ks (get-in m ks)))- ([m ks v]- (if-let [unresolved-names (::unresolved-names v)]- (-> (update m ::unresolved-names (fn [nms] (merge nms (m/map-vals #(vec (concat ks %)) unresolved-names))))- (assoc-in ks (dissoc v ::unresolved-names)))- (assoc-in m ks v))))--(defn- paths-to-key-in- "Finds all paths to a particular key anywhere in the structure `m` (recursively). `m` must be a map, but values- inside can also be vectors (in which case, the index will be used as the key).-- Adapted from: https://dnaeon.github.io/clojure-map-ks-paths/"- [m match-key]- (letfn [(children [node]- (let [v (get-in m node)]- (cond- (map? v)- (map (fn [x] (conj node x)) (keys v))-- (vector? v)- (map (fn [x] (conj node x)) (range (count v)))-- :else- [])))- (branch? [node] (-> (children node) seq boolean))]- (->> (keys m)- (map vector)- (mapcat #(tree-seq branch? children %))- (filter #(= match-key (last %))))))--(defn- gather-all-unresolved-names- "This is less efficient than calling `pull-unresolved-names-up` because it walks the entire tree, but is- necessary when dealing with the full MBQL query tree (which can have arbitrary nesting of maps and- vectors)."- [m]- (let [paths (paths-to-key-in m ::unresolved-names)]- (if-not (empty? paths)- (reduce (fn [acc ks]- (let [ks* (drop-last ks)]- (if-not (empty? ks*)- (pull-unresolved-names-up acc ks*)- acc)))- m- paths)- m)))--(defn- mbql-fully-qualified-names->ids*- [entity]- (lib.util.match/replace entity- ;; handle legacy `:field-id` forms encoded prior to 0.39.0- ;; and also *current* expression forms used in parameter mapping dimensions- ;; example relevant clause - [:dimension [:fk-> [:field-id 1] [:field-id 2]]]- [:field-id (fully-qualified-name :guard string?)]- (mbql-fully-qualified-names->ids* [:field fully-qualified-name nil])-- [:field (fully-qualified-name :guard names/fully-qualified-field-name?) opts]- [:field (:field (fully-qualified-name->context fully-qualified-name)) (mbql-fully-qualified-names->ids* opts)]-- ;; source-field is also used within parameter mapping dimensions- ;; example relevant clause - [:field 2 {:source-field 1}]- {:source-field (fully-qualified-name :guard string?)}- (assoc &match :source-field (:field (fully-qualified-name->context fully-qualified-name)))-- [:metric (fully-qualified-name :guard string?)]- [:metric (:metric (fully-qualified-name->context fully-qualified-name))]-- [:segment (fully-qualified-name :guard string?)]- [:segment (:segment (fully-qualified-name->context fully-qualified-name))]-- (_ :guard (every-pred map? #(fq-table-or-card? (:source-table %))))- (-> (mbql-fully-qualified-names->ids* (dissoc &match :source-table)) ;; process other keys- (assoc :source-table (:source-table &match)) ;; add :source-table back in for lookup- (update-existing-capture-missing :source-table source-table)))) ;; look up :source-table and capture missing--(defn- mbql-fully-qualified-names->ids- [entity]- (mbql-fully-qualified-names->ids* (mbql.normalize/normalize-tokens entity)))--(def ^:private ^{:arglists '([])} default-user-id- (mdb/memoize-for-application-db- (fn []- (let [user (t2/select-one-pk :model/User :is_superuser true)]- (assert user (trs "No admin users found! At least one admin user is needed to act as the owner for all the loaded entities."))- user))))--(defn- terminal-dir- "Return the last path component (presumably a dir)"- [path]- (.getName (io/file path)))--(defn- unresolved-names->string- ([model]- (unresolved-names->string model nil))- ([model insert-id]- (str- (when-let [nm (:name model)] (str "\"" nm "\""))- (when insert-id (format " (inserted as ID %d) " insert-id))- "missing:\n "- (str/join- "\n "- (map- (fn [[k v]]- (format "at %s -> %s" (str/join "/" v) k))- (::unresolved-names model))))))--(defmulti load!- "Load an entity of type `model` stored at `path` in the context `context`.-- Passing in parent entities as context instead of decoding them from the path each time,- saves a lot of queriying."- {:arglists '([path context])}- (fn [path _context]- (terminal-dir path)))--(defn- load-dimensions!- [path context]- (maybe-upsert-many!- context :model/Dimension- (for [dimension (yaml/from-file (str path "/dimensions.yaml"))]- (-> dimension- (update :human_readable_field_id (comp :field fully-qualified-name->context))- (update :field_id (comp :field fully-qualified-name->context))))))--(defmethod load! "databases"- [path context]- (doseq [path (list-dirs path)]- ;; If we failed to load the DB no use in trying to load its tables- (when-let [db (first (maybe-upsert-many! context :model/Database (slurp-dir path)))]- (doseq [inner-path (conj (list-dirs (str path "/schemas")) path)- :let [context (merge context {:database db- :schema (when (not= inner-path path)- (terminal-dir path))})]]- (load! (str inner-path "/tables") context)- (load-dimensions! inner-path context)))))--(defmethod load! "tables"- [path context]- (let [paths (list-dirs path)- table-ids (maybe-upsert-many!- context :model/Table- (for [table (slurp-many paths)]- (assoc table :db_id (:database context))))]- ;; First load fields ...- (doseq [[path table-id] (map vector paths table-ids)- :when table-id]- (let [context (assoc context :table table-id)]- (load! (str path "/fields") context)))- ;; ... then everything else so we don't have issues with cross-table referencess- (doseq [[path table-id] (map vector paths table-ids)- :when table-id]- (let [context (assoc context :table table-id)]- (load! (str path "/fks") context)- (load! (str path "/segments") context)))))--(def ^:private fully-qualified-name->card-id- (comp :card fully-qualified-name->context))--(defn- load-fields!- [path context]- (let [fields (slurp-dir path)- field-values (map :values fields)- field-ids (maybe-upsert-many!- context :model/Field- (for [field fields]- (-> field- (update :parent_id (comp :field fully-qualified-name->context))- (update :last_analyzed u.date/parse)- (update :fk_target_field_id (comp :field fully-qualified-name->context))- (dissoc :values)- (assoc :table_id (:table context)))))]- (maybe-upsert-many!- context :model/FieldValues- (for [[field-value field-id] (map vector field-values field-ids)- :when field-id]- (assoc field-value :field_id field-id)))))--(defmethod load! "fields"- [path context]- (load-fields! path context))--(defmethod load! "fks"- [path context]- (load-fields! path context))--(defmethod load! "segments"- [path context]- (maybe-upsert-many!- context :model/Segment- (for [metric (slurp-dir path)]- (-> metric- (assoc :table_id (:table context)- :creator_id (default-user-id))- (assoc-in [:definition :source-table] (:table context))- (update :definition mbql-fully-qualified-names->ids)))))--(defn- update-card-parameter-mappings- [parameter-mappings]- (for [parameter-mapping parameter-mappings]- (-> parameter-mapping- (update-existing-capture-missing :card_id fully-qualified-name->card-id)- (update-existing-capture-missing :target mbql-fully-qualified-names->ids))))--(defn- resolve-column-settings-key- [col-key]- (if-let [field-name (::mb.viz/field-str col-key)]- (let [field-id ((comp :field fully-qualified-name->context) field-name)]- (if (nil? field-id)- {::unresolved-names {field-name [::column-settings-key]}}- {::mb.viz/field-id field-id}))- col-key))--(defn- resolve-param-mapping-key [k]- (mbql-fully-qualified-names->ids k))--(defn- resolve-dimension [dimension]- (mbql-fully-qualified-names->ids dimension))--(defn- resolve-param-ref [param-ref]- (cond-> param-ref- (= "dimension" (::mb.viz/param-ref-type param-ref))- ;; from outer cond->- (-> (m/update-existing ::mb.viz/param-ref-id mbql-fully-qualified-names->ids)- (m/update-existing ::mb.viz/param-dimension resolve-dimension))))--(defn- resolve-param-mapping-val [v]- (-> v- (m/update-existing ::mb.viz/param-mapping-id mbql-fully-qualified-names->ids)- (m/update-existing ::mb.viz/param-mapping-source resolve-param-ref)- (m/update-existing ::mb.viz/param-mapping-target resolve-param-ref)))--(defn- resolve-click-behavior-parameter-mapping [parameter-mapping]- (->> parameter-mapping- mb.viz/db->norm-param-mapping- (reduce-kv (fn [acc k v]- (assoc acc (resolve-param-mapping-key k)- (resolve-param-mapping-val v))) {})- mb.viz/norm->db-param-mapping))--(defn- resolve-click-behavior- [click-behavior]- (-> (if-let [link-type (::mb.viz/link-type click-behavior)]- (case link-type- ::mb.viz/card (let [card-id (::mb.viz/link-target-id click-behavior)]- (when (string? card-id)- (update-existing-in-capture-missing- click-behavior- [::mb.viz/link-target-id]- (comp :card fully-qualified-name->context))))- ::mb.viz/dashboard (let [dashboard-id (::mb.viz/link-target-id click-behavior)]- (when (string? dashboard-id)- (update-existing-in-capture-missing- click-behavior- [::mb.viz/link-target-id]- (comp :dashboard fully-qualified-name->context))))- click-behavior)- click-behavior)- (m/update-existing ::mb.viz/parameter-mapping resolve-click-behavior-parameter-mapping)))--(defn- update-col-settings-click-behavior [col-settings-value]- (let [new-cb (resolve-click-behavior (::mb.viz/click-behavior col-settings-value))]- (pull-unresolved-names-up col-settings-value [::mb.viz/click-behavior] new-cb)))--(defn- resolve-column-settings-value- [col-value]- (cond-> col-value- (::mb.viz/click-behavior col-value) update-col-settings-click-behavior))--(defn- accumulate-converted-column-settings- [acc col-key v]- (let [new-key (resolve-column-settings-key col-key)- new-val (resolve-column-settings-value v)]- (-> (pull-unresolved-names-up acc [::column-settings-key] new-key)- (dissoc ::column-settings-key)- (pull-unresolved-names-up [new-key] new-val))))--(defn- resolve-top-level-click-behavior [vs-norm]- (if-let [click-behavior (::mb.viz/click-behavior vs-norm)]- (let [resolved-cb (resolve-click-behavior click-behavior)]- (pull-unresolved-names-up vs-norm [::mb.viz/click-behavior] resolved-cb))- vs-norm))--(defn- resolve-column-settings- "Resolve the entries in a :column_settings map (which is under a :visualization_settings map). These map entries- may contain fully qualified field names, or even other cards. In case of an unresolved name (i.e. a card that hasn't- yet been loaded), we will track it under ::unresolved-names and revisit on the next pass."- [vs-norm]- (if-let [col-settings (::mb.viz/column-settings vs-norm)]- (let [resolved-cs (reduce-kv accumulate-converted-column-settings {} col-settings)]- (pull-unresolved-names-up vs-norm [::mb.viz/column-settings] resolved-cs))- vs-norm))--(defn- resolve-table-column-field-ref [[f-type f-str f-md]]- (if (names/fully-qualified-field-name? f-str)- [f-type ((comp :field fully-qualified-name->context) f-str) f-md]- [f-type f-str f-md]))--(defn- resolve-pivot-table-settings- "Resolve the entries in a :pivot_table.column_split map (which is under a :visualization_settings map). Modern map- entries contain column names that should be preserved as is. Legacy map entries may contain fully qualified field- names, or even other cards. In case of an unresolved name (i.e. a card that hasn't yet been loaded), we will track it- under ::unresolved-names and revisit on the next pass."- [vs-norm]- (if (:pivot_table.column_split vs-norm)- (letfn [(resolve-vec [pivot vec-type]- (update-in pivot [:pivot_table.column_split vec-type]- (fn [tbl-vecs]- (if (every? string? tbl-vecs)- tbl-vecs- (mapv resolve-table-column-field-ref tbl-vecs)))))]- (-> vs-norm- (resolve-vec :rows)- (resolve-vec :columns)))- vs-norm))--(defn- resolve-table-columns- "Resolve the :table.columns key from a :visualization_settings map, which may contain fully qualified field names.- Such fully qualified names will be converted to the numeric field ID before being filled into the loaded card. Only- other field names (not cards, or other collection based entity types) should be referenced here, so there is no need- to detect or track ::unresolved-names."- [vs-norm]- (if (::mb.viz/table-columns vs-norm)- (letfn [(resolve-field-id [tbl-col]- (update tbl-col ::mb.viz/table-column-field-ref resolve-table-column-field-ref))]- (update vs-norm ::mb.viz/table-columns (fn [tbl-cols]- (mapv resolve-field-id tbl-cols))))- vs-norm))--(defn- resolve-visualization-settings- "Resolve all references from a :visualization_settings map, the various submaps of which may contain:- - fully qualified field names- - fully qualified card or dashboard names-- Any unresolved entities from this resolution process will be tracked via ::unresolved-named so that the card or- dashboard card holding these visualization settings can be revisited in a future pass."- [entity]- (if-let [viz-settings (:visualization_settings entity)]- (let [resolved-vs (-> (mb.viz/db->norm viz-settings)- resolve-top-level-click-behavior- resolve-column-settings- resolve-table-columns- resolve-pivot-table-settings- mb.viz/norm->db)]- (pull-unresolved-names-up entity [:visualization_settings] resolved-vs))- entity))--(defn- resolve-dashboard-parameters- [parameters]- (for [p parameters]- ;; Note: not using the full ::unresolved-names functionality here because this is a fix- ;; for a deprecated feature- (m/update-existing-in p [:values_source_config :card_id] fully-qualified-name->card-id)))--(defn- parse-timestamp [ts]- (if (string? ts)- (u.date/parse ts)- ts))--(defn load-dashboards!- "Loads `dashboards` (which is a sequence of maps parsed from a YAML dump of dashboards) in a given `context`."- {:added "0.40.0"}- [context dashboards]- (let [dashboard-ids (maybe-upsert-many!- context :model/Dashboard- (for [dashboard dashboards]- (-> dashboard- (update :parameters resolve-dashboard-parameters)- (update :last_viewed_at parse-timestamp)- (dissoc :dashboard_cards)- (assoc :collection_id (:collection context)- :creator_id (default-user-id)))))- ;; MEGA HACK -- if `load` is ran with `--mode update` we should delete any Cards that were removed from a- ;; Dashboard (according to #20786). However there are literally zero facilities for doing this sort of thing in- ;; the current dump/load codebase. So for now we'll just delete ALL DashboardCards for the dumped Dashboard when- ;; running with `--mode update` and recreate them from the serialized definitions. This is definitely a wack way- ;; of doing things but no one actually understands how this code is supposed to work so this will have to do- ;; until we can come in here and clean things up. -- Cam 2022-03-24- _ (when (and (= (:mode context) :update)- (seq dashboard-ids))- (t2/delete! :model/DashboardCard :dashboard_id [:in (set dashboard-ids)]))- dashboard-cards (map :dashboard_cards dashboards)- ;; a function that prepares a dash card for insertion, while also validating to ensure the underlying- ;; card_id could be resolved from the fully qualified name- prepare-card-fn (fn [dash-idx dashboard-id acc card-idx card]- (let [proc-card (-> card- (update-existing-capture-missing :card_id fully-qualified-name->card-id)- (assoc :dashboard_id dashboard-id))- new-pm (update-card-parameter-mappings (:parameter_mappings proc-card))- with-pm (pull-unresolved-names-up proc-card [:parameter_mappings] new-pm)- with-viz (resolve-visualization-settings with-pm)]- (if-let [unresolved (::unresolved-names with-viz)]- ;; prepend the dashboard card index and :visualization_settings to each unresolved- ;; name path for better debugging- (let [add-keys [:dashboard_cards card-idx :visualization_settings]- fixed-names (m/map-vals #(concat add-keys %) unresolved)- with-fixed-names (assoc with-viz ::unresolved-names fixed-names)]- (-> acc- (update ::revisit (fn [revisit-map]- (update revisit-map dash-idx #(cons with-fixed-names %))))- ;; index means something different here than in the Card case (it's actually the index- ;; of the dashboard)- (update ::revisit-index #(conj % dash-idx))))- (update acc ::process #(conj % with-viz)))))- prep-init-acc {::process [] ::revisit-index #{} ::revisit {}}- filtered-cards (reduce-kv- (fn [acc idx [cards dash-id]]- (if dash-id- (let [res (reduce-kv (partial prepare-card-fn idx dash-id) prep-init-acc (vec cards))]- (merge-with concat acc res))- acc))- prep-init-acc- (mapv vector dashboard-cards dashboard-ids))- revisit-indexes (vec (::revisit-index filtered-cards))- proceed-cards (vec (::process filtered-cards))- dashcard-ids (maybe-upsert-many! context :model/DashboardCard (map #(dissoc % :series) proceed-cards))- series-pairs (map vector (map :series proceed-cards) dashcard-ids)]- (maybe-upsert-many!- context :model/DashboardCardSeries- (for [[series dashboard-card-id] series-pairs- dashboard-card-series series- :when (and dashboard-card-series dashboard-card-id)]- (-> dashboard-card-series- (assoc :dashboardcard_id dashboard-card-id)- (update :card_id fully-qualified-name->card-id))))- (let [revisit-dashboards (map (partial nth dashboards) revisit-indexes)]- (when (seq revisit-dashboards)- (let [revisit-map (::revisit filtered-cards)- revisit-inf-fn (fn [[dash-idx dashcards]]- (format- "For dashboard %s:%n%s"- (->> dash-idx (nth dashboards) :name)- (str/join "\n" (map unresolved-names->string dashcards))))]- (log/infof- "Unresolved references found for dashboard cards in collection %d; will reload after first pass%n%s%n"- (:collection context)- (str/join "\n" (map revisit-inf-fn revisit-map)))- (fn []- (log/infof- "Retrying dashboards for collection %s: %s"- (or (:collection context) "root")- (str/join ", " (map :name revisit-dashboards)))- (load-dashboards! (assoc context :mode :update) revisit-dashboards)))))))--(defmethod load! "dashboards"- [path context]- (binding [names/*suppress-log-name-lookup-exception* true]- (load-dashboards! context (slurp-dir path))))--(defn- load-pulses! [pulses context]- (let [cards (map :cards pulses)- channels (map :channels pulses)- pulse-ids (maybe-upsert-many!- context :model/Pulse- (for [pulse pulses]- (-> pulse- (assoc :collection_id (:collection context)- :creator_id (default-user-id))- (dissoc :channels :cards))))- pulse-cards (for [[cards pulse-id pulse-idx] (map vector cards pulse-ids (range 0 (count pulse-ids)))- card cards- :when pulse-id]- (-> card- (assoc :pulse_id pulse-id)- ;; gather the pulse's name and index for easier bookkeeping later- (assoc ::pulse-index pulse-idx)- (assoc ::pulse-name (:name (nth pulses pulse-idx)))- (update-in-capture-missing [:card_id] fully-qualified-name->card-id)))- grouped (group-by #(empty? (::unresolved-names %)) pulse-cards)- process (get grouped true)- revisit (get grouped false)]- (maybe-upsert-many! context :model/PulseCard (map #(dissoc % ::pulse-index ::pulse-name) process))- (maybe-upsert-many!- context :model/PulseChannel- (for [[channels pulse-id] (map vector channels pulse-ids)- channel channels- :when pulse-id]- (assoc channel :pulse_id pulse-id)))- (when (seq revisit)- (let [revisit-info-map (group-by ::pulse-name revisit)]- (log/infof "Unresolved references for pulses in collection %s; will reload after first pass complete:%n%s%n"- (or (:collection context) "root")- (str/join "\n" (map- (fn [[pulse-name revisit-cards]]- (format " for %s:%n%s"- pulse-name- (str/join "\n" (map (comp unresolved-names->string #(into {} %)) revisit-cards))))- revisit-info-map)))- (fn []- (log/infof "Reloading pulses from collection %d" (:collection context))- (let [pulse-indexes (map ::pulse-index revisit)]- (load-pulses! (map (partial nth pulses) pulse-indexes) (assoc context :mode :update))))))))--(defmethod load! "pulses"- [path context]- (binding [names/*suppress-log-name-lookup-exception* true]- (load-pulses! (slurp-dir path) context)))--(defn- resolve-source-query [query]- (if (:source-query query)- (update-in-capture-missing query [:source-query] resolve-source-query)- query))--(defn- source-card- [fully-qualified-name]- (try- (-> (fully-qualified-name->context fully-qualified-name) :card)- (catch Throwable e- (log/warnf e "Could not find context for fully qualified card name %s" fully-qualified-name))))--(defn- resolve-snippet- [fully-qualified-name]- (try- (-> (fully-qualified-name->context fully-qualified-name) :snippet)- (catch Throwable e- (log/debugf e "Could not find context for fully qualified snippet name %s" fully-qualified-name))))--(defn- resolve-native- [card]- (let [ks [:dataset_query :native :template-tags]- template-tags (get-in card ks)- new-template-tags (reduce-kv- (fn [m k v]- (let [new-v (-> (update-existing-capture-missing v :card-id source-card)- (update-existing-capture-missing :snippet-id resolve-snippet))]- (pull-unresolved-names-up m [k] new-v)))- {}- template-tags)]- (pull-unresolved-names-up card ks new-template-tags)))--(defn- resolve-card-dataset-query [card]- (let [ks [:dataset_query :query]- new-q (update-in-capture-missing card ks resolve-source-query)]- (-> (pull-unresolved-names-up card ks (get-in new-q ks))- (gather-all-unresolved-names))))--(defn- resolve-card [card context]- (-> card- (update :table_id (comp :table fully-qualified-name->context))- (update :database_id (comp :database fully-qualified-name->context))- (update :dataset_query mbql-fully-qualified-names->ids)- (assoc :creator_id (default-user-id)- :collection_id (:collection context))- (update-in [:dataset_query :database] (comp :database fully-qualified-name->context))- resolve-visualization-settings- (cond->- (-> card- :dataset_query- :type- mbql.u/normalize-token- (= :query)) resolve-card-dataset-query- (-> card- :dataset_query- :native- :template-tags- not-empty) (resolve-native))))--(defn- make-dummy-card- "Make a dummy card for first pass insertion"- [card]- (-> card- (assoc :dataset_query {:type :native- :native {:query "-- DUMMY QUERY FOR SERIALIZATION FIRST PASS INSERT"}- :database (:database_id card)})- (dissoc ::unresolved-names)))--(defn load-cards!- "Loads cards in a given `context`, from a given sequence of `paths` (strings). If specified, then `only-cards` (maps- having the structure of cards loaded from YAML dumps) will be used instead of loading data from `paths` (to serve as- a retry mechanism)."- {:added "0.40.0"}- [context paths only-cards]- (let [cards (or only-cards (slurp-many paths))- resolved-cards (for [card cards]- (resolve-card card context))- grouped-cards (reduce-kv- (fn [acc idx card]- (if (::unresolved-names card)- (-> acc- (update ::revisit #(conj % card))- (update ::revisit-index #(conj % idx)))- (update acc ::process #(conj % card))))- {::revisit [] ::revisit-index #{} ::process []}- (vec resolved-cards))- dummy-insert-cards (not-empty (::revisit grouped-cards))- process-cards (::process grouped-cards)]- (maybe-upsert-many! context :model/Card process-cards)- (when dummy-insert-cards- (let [dummy-inserted-ids (maybe-upsert-many!- context- :model/Card- (map make-dummy-card dummy-insert-cards))- id-and-cards (map vector dummy-insert-cards dummy-inserted-ids)- retry-info-fn (fn [[card card-id]]- (unresolved-names->string card card-id))]- (log/infof- "Unresolved references found for cards in collection %d; will reload after first pass%n%s%n"- (:collection context)- (str/join "\n" (map retry-info-fn id-and-cards)))- (fn []- (log/infof "Attempting to reload cards in collection %d" (:collection context))- (let [revisit-indexes (::revisit-index grouped-cards)]- (load-cards! (assoc context :mode :update) paths (mapv (partial nth cards) revisit-indexes))))))))--(defmethod load! "cards"- [path context]- (binding [names/*suppress-log-name-lookup-exception* true]- (load-cards! context (list-dirs path) nil)))--(defn- pre-insert-user- "A function called on each User instance before it is inserted (via upsert)."- [user]- (log/infof "User with email %s is new to target DB; setting a random password" (:email user))- (assoc user :password (str (random-uuid))))--;; leaving comment out for now (deliberately), because this will send a password reset email to newly inserted users-;; when enabled in a future release; see `defmethod load "users"` below-#_(defn- post-insert-user- "A function called on the ID of each `User` instance after it is inserted (via upsert)."- [user-id]- (when-let [{email :email, google-auth? :google_auth, is-active? :is_active}- (t2/select-one [User :email :google_auth :is_active] :id user-id)]- (let [reset-token (user/set-password-reset-token! user-id)- site-url (system/site-url)- password-reset-url (str site-url "/auth/reset_password/" reset-token)- ;; in a web server context, the server-name ultimately comes from ServletRequest/getServerName- ;; (i.e. the Java class, via Ring); this is the closest approximation in our batch context- server-name (.getHost (URL. site-url))]- (let [email-res (email/send-password-reset-email! email google-auth? server-name password-reset-url is-active?)]- (if (:error email-res)- (log/infof "Failed to send password reset email generated for user ID %d (%s): %s"- user-id- (:message email-res))- (log/infof "Password reset email generated for user ID %d (%s)" user-id email)))- user-id)))--(defmethod load! "users"- [path context]- ;; Currently we only serialize the new owner user, so it's fine to ignore mode setting- ;; add :post-insert-fn post-insert-user back to start sending password reset emails- (maybe-upsert-many!- (assoc context :pre-insert-fn pre-insert-user) :model/User- (for [user (slurp-dir path)]- (dissoc user :password))))--(defn- derive-location- [context]- (if-let [parent-id (:collection context)]- (str (t2/select-one-fn :location :model/Collection :id parent-id) parent-id "/")- "/"))--(defn- make-reload-fn [all-results]- (let [all-fns (filter fn? all-results)]- (when (seq all-fns)- (let [new-fns (doall all-fns)]- (fn []- (make-reload-fn (for [reload-fn new-fns]- (reload-fn))))))))--(defn- load-collections!- [path context]- (let [subdirs (list-dirs path)- by-ns (group-by #(let [[_ coll-ns] (re-matches #".*/:([^:/]+)" %)]- coll-ns)- subdirs)- grouped (group-by (comp nil? first) by-ns)- ns-paths (get grouped false)- entity-paths (->> (get grouped true)- (map last)- first)- results (for [path entity-paths]- (let [context (assoc context- :collection (->> (slurp-dir path)- (map #(assoc % :location (derive-location context)- :namespace (-> context- :collection-namespace)))- (maybe-upsert-many! context :model/Collection)- first))]- (log/infof "Processing collection at path %s" path)- [(load! (str path "/collections") context)- (load! (str path "/cards") context)- (load! (str path "/pulses") context)- (load! (str path "/dashboards") context)- (load! (str path "/snippets") context)]))- load-ns-fns (for [[coll-ns [coll-ns-path]] ns-paths]- (do (log/infof "Loading %s namespace for collection at path %s" coll-ns coll-ns-path)- (load-collections! coll-ns-path (assoc context :collection-namespace coll-ns))))]- (make-reload-fn (concat (apply concat results) ; these are each sequences, so need to flatten those first- load-ns-fns))))--(defmethod load! "collections"- [path context]- (load-collections! path context))--(defn- prepare-snippet [context snippet]- (assoc snippet :creator_id (default-user-id)- :collection_id (:collection context)))--(defmethod load! "snippets"- [path context]- (let [paths (list-dirs path)- snippets (map (partial prepare-snippet context) (slurp-many paths))]- (maybe-upsert-many! context :model/NativeQuerySnippet snippets)))--(defn load-settings!- "Load a dump of settings."- [path context]- (doseq [[k v] (yaml/from-file (str path "/settings.yaml"))- :when (or (= context :update)- (nil? (setting/get-value-of-type :string k)))]- (setting/set-value-of-type! :string k v)))--(defn compatible?- "Is dump at path `path` compatible with the currently running version of Metabase?"- [path]- (-> (str path "/manifest.yaml")- yaml/from-file- :metabase-version- (= config/mb-version-info)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/names.clj b/enterprise/backend/src/metabase_enterprise/serialization/names.cljdeleted file mode 100644index 44a94c570fb..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/names.clj+++ /dev/null@@ -1,329 +0,0 @@-(ns metabase-enterprise.serialization.names- "Consistent instance-independent naming scheme that replaces IDs with human-readable paths."- (:require- [clojure.string :as str]- [metabase.app-db.core :as mdb]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.models.interface :as mi]- [metabase.util.log :as log]- [metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]- [ring.util.codec :as codec]- [toucan2.core :as t2]- [toucan2.protocols :as t2.protocols]))--(set! *warn-on-reflection* true)--(def ^:private root-collection-path "/collections/root")--(defn safe-name- "Return entity name URL encoded except that spaces are retained."- [entity]- (some-> entity ((some-fn :email :name)) codec/url-encode (str/replace "%20" " ")))--(def unescape-name- "Inverse of `safe-name`."- codec/url-decode)--(defmulti ^:private fully-qualified-name*- {:arglists '([instance])}- mi/model)--(def ^{:arglists '([entity] [model id])} fully-qualified-name- "Get the logical path for entity `entity`."- (mdb/memoize-for-application-db- (fn- ([entity] (fully-qualified-name* entity))- ([model id]- (if (string? id)- id- (fully-qualified-name* (t2/select-one model :id id)))))))--(defmethod fully-qualified-name* :model/Database- [db]- (str "/databases/" (safe-name db)))--(defmethod fully-qualified-name* :model/Table- [table]- (if (:schema table)- (format "%s/schemas/%s/tables/%s"- (->> table :db_id (fully-qualified-name :model/Database))- (:schema table)- (safe-name table))- (format "%s/tables/%s"- (->> table :db_id (fully-qualified-name :model/Database))- (safe-name table))))--(defmethod fully-qualified-name* :model/Field- [field]- (if (:fk_target_field_id field)- (str (->> field :table_id (fully-qualified-name :model/Table)) "/fks/" (safe-name field))- (str (->> field :table_id (fully-qualified-name :model/Table)) "/fields/" (safe-name field))))--(defmethod fully-qualified-name* :model/Segment- [segment]- (str (->> segment :table_id (fully-qualified-name :model/Table)) "/segments/" (safe-name segment)))--(defn- local-collection-name [collection]- (let [ns-part (when-let [coll-ns (:namespace collection)]- (str ":" (if (keyword? coll-ns) (name coll-ns) coll-ns) "/"))]- (str "/collections/" ns-part (safe-name collection))))--(defmethod fully-qualified-name* :model/Collection- [collection]- (let [parents (some->> (str/split (:location collection) #"/")- rest- not-empty- (map #(local-collection-name (t2/select-one :model/Collection :id (Integer/parseInt %))))- (apply str))]- (str root-collection-path parents (local-collection-name collection))))--(defmethod fully-qualified-name* :model/Dashboard- [dashboard]- (format "%s/dashboards/%s"- (or (some->> dashboard :collection_id (fully-qualified-name :model/Collection))- root-collection-path)- (safe-name dashboard)))--(defmethod fully-qualified-name* :model/Pulse- [pulse]- (format "%s/pulses/%s"- (or (some->> pulse :collection_id (fully-qualified-name :model/Collection))- root-collection-path)- (safe-name pulse)))--(defmethod fully-qualified-name* :model/Card- [card]- (format "%s/cards/%s"- (or (some->> card- :collection_id- (fully-qualified-name :model/Collection))- root-collection-path)- (safe-name card)))--(defmethod fully-qualified-name* :model/User- [user]- (str "/users/" (:email user)))--(defmethod fully-qualified-name* :model/NativeQuerySnippet- [snippet]- (format "%s/snippets/%s"- (or (some->> snippet :collection_id (fully-qualified-name :model/Collection))- root-collection-path)- (safe-name snippet)))--(defmethod fully-qualified-name* nil- [_]- nil)--;; All the references in the dumps should resolved to entities already loaded.-(def ^:private Context- [:map {:closed true}- [:database {:optional true} ms/PositiveInt]- [:table {:optional true} ms/PositiveInt]- [:schema {:optional true} [:maybe :string]]- [:field {:optional true} ms/PositiveInt]- [:metric {:optional true} ms/PositiveInt]- [:segment {:optional true} ms/PositiveInt]- [:card {:optional true} ms/PositiveInt]- [:dashboard {:optional true} ms/PositiveInt]- [:collection {:optional true} [:maybe ms/PositiveInt]] ; root collection- [:pulse {:optional true} ms/PositiveInt]- [:user {:optional true} ms/PositiveInt]- [:snippet {:optional true} [:maybe ms/PositiveInt]]])--(defmulti ^:private path->context*- {:arglists '([context model model-attrs entity-name])}- (fn [_ model _ _]- model))--(def ^:private ^{:arglists '([context model model-attrs entity-name])} path->context- "Extract entities from a logical path."- path->context*)--(defmethod path->context* "databases"- [context _ _ db-name]- (assoc context :database (if (= db-name "__virtual")- lib.schema.id/saved-questions-virtual-database-id- (t2/select-one-pk :model/Database :name db-name))))--(defmethod path->context* "schemas"- [context _ _ schema]- (assoc context :schema schema))--(defmethod path->context* "tables"- [context _ _ table-name]- (assoc context :table (t2/select-one-pk :model/Table- :db_id (:database context)- :schema (:schema context)- :name table-name)))--(defmethod path->context* "fields"- [context _ _ field-name]- (assoc context :field (t2/select-one-pk :model/Field- :table_id (:table context)- :name field-name)))--(defmethod path->context* "fks"- [context _ _ field-name]- (path->context* context "fields" nil field-name))--(defmethod path->context* "segments"- [context _ _ segment-name]- (assoc context :segment (t2/select-one-pk :model/Segment- :table_id (:table context)- :name segment-name)))--(defmethod path->context* "collections"- [context _ model-attrs collection-name]- (if (= collection-name "root")- (assoc context :collection nil)- (assoc context :collection (t2/select-one-pk :model/Collection- :name collection-name- :namespace (:namespace model-attrs)- :location (or (letfn [(collection-location [id]- (t2/select-one-fn :location :model/Collection :id id))]- (some-> context- :collection- collection-location- (str (:collection context) "/")))- "/")))))--(defmethod path->context* "dashboards"- [context _ _ dashboard-name]- (assoc context :dashboard (t2/select-one-pk :model/Dashboard- :collection_id (:collection context)- :name dashboard-name)))--(defmethod path->context* "pulses"- [context _ _ pulse-name]- (assoc context :dashboard (t2/select-one-pk :model/Pulse- :collection_id (:collection context)- :name pulse-name)))--(defmethod path->context* "cards"- [context _ _ dashboard-name]- (assoc context :card (t2/select-one-pk :model/Card- :collection_id (:collection context)- :name dashboard-name)))--(defmethod path->context* "users"- [context _ _ email]- (assoc context :user (t2/select-one-pk :model/User- :email email)))--(defmethod path->context* "snippets"- [context _ _ snippet-name]- (assoc context :snippet (t2/select-one-pk :model/NativeQuerySnippet- :collection_id (:collection context)- :name snippet-name)))--(def ^:private separator-pattern #"\/")--(def ^:dynamic *suppress-log-name-lookup-exception*- "Dynamic boolean var that controls whether warning messages will NOT be logged on a failed name lookup (from within- `fully-qualified-name->context`). Intended to be bound differently in first pass (i.e. set to true), where we expect- some name lookups to fail, in order to avoid polluting the log. On subsequent rounds (i.e. reload fns) then it should- be left off because we wouldn't expect to have failed lookups then."- false)--(defn fully-qualified-field-name?- "Returns true if the given `field-name` is a fully-qualified field name for serialization purposes (as opposed to a- reference to an in-query alias or some other form)."- [field-name]- (and (some? field-name)- (str/starts-with? field-name "/databases/")- (or (str/includes? field-name "/fks/") (str/includes? field-name "/fields/"))))--(defn fully-qualified-table-name?- "Returns true if the given `table-name` is a fully-qualified table name for serialization purposes (as opposed to a- reference to a card)."- [table-name]- (and (some? table-name)- (string? table-name)- (str/starts-with? table-name "/databases/")- (not (str/starts-with? table-name "card__"))))--(defn fully-qualified-card-name?- "Returns true if the given `card-name` is a fully-qualified card name for serialization purposes."- [card-name]- (and (some? card-name)- (string? card-name)- (str/starts-with? card-name "/collections/root/")- (str/includes? card-name "/cards/")))--;; WARNING: THIS MUST APPEAR AFTER ALL path->context* IMPLEMENTATIONS-(def ^:private all-entities (-> path->context*- methods- keys- set))--(defn- partition-name-components- "This is more complicated than it needs to be due to potential clashes between an entity name (ex: a table called- \"users\" and a model name (ex: \"users\"). Could fix in a number of ways, including special prefix of model names,- but that would require changing the format and updating all the `defmethod` calls."- ([name-comps]- (partition-name-components {::name-components [] ::current-component []} name-comps))- ([acc [c & more-comps]]- (cond- (nil? more-comps)- (conj (::name-components acc) (conj (::current-component acc) c))-- (::prev-model-name? acc)- (if (= \: (first c))- (partition-name-components (update acc ::current-component conj c) more-comps)- (partition-name-components (-> (assoc acc ::prev-model-name? false)- (update ::current-component- conj- c))- more-comps))-- (contains? all-entities c)- (partition-name-components (cond-> (assoc acc ::prev-model-name? true- ::current-component [c])- (not-empty (::current-component acc))- (update ::name-components conj (::current-component acc)))- more-comps))))--(defn fully-qualified-name->context- "Parse a logical path into a context map."- [fully-qualified-name]- (when fully-qualified-name- (let [components (->> (str/split fully-qualified-name separator-pattern)- rest ; we start with a /- partition-name-components- (map (fn [[model-name & entity-parts]]- (cond-> {::model-name model-name ::entity-name (last entity-parts)}- (and (= "collections" model-name) (> (count entity-parts) 1))- (assoc :namespace (->> entity-parts- first ; ns is first/only item after "collections"- rest ; strip the starting :- (apply str)))))))- context (loop [acc-context {}- [{::keys [model-name entity-name] :as model-map} & more] components]- (let [model-attrs (dissoc model-map ::model-name ::entity-name)- new-context (path->context acc-context model-name model-attrs (unescape-name entity-name))]- (if (empty? more)- new-context- (recur new-context more))))]- (if (and- (not (mr/validate [:maybe Context] context))- (not *suppress-log-name-lookup-exception*))- (log/warn- (ex-info (format "Can't resolve %s in fully qualified name %s"- (str/join ", " (map name (keys context)))- fully-qualified-name)- {:fully-qualified-name fully-qualified-name- :resolve-name-failed? true- :context context}))- context))))--(defn name-for-logging- "Return a string representation of entity suitable for logs"- ([entity] (name-for-logging (t2.protocols/model entity) entity))- ([model {:keys [name id]}]- (cond- (and name id) (format "%s \"%s\" (ID %s)" model name id)- name (format "%s \"%s\"" model name)- id (format "%s %s" model id)- :else model)))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/serialize.clj b/enterprise/backend/src/metabase_enterprise/serialization/serialize.cljdeleted file mode 100644index 90490973147..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/serialize.clj+++ /dev/null@@ -1,228 +0,0 @@-(ns metabase-enterprise.serialization.serialize- "Transform entity into a form suitable for serialization."- (:require- [clojure.string :as str]- [medley.core :as m]- [metabase-enterprise.serialization.names :refer [fully-qualified-name]]- [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.util.match :as lib.util.match]- [metabase.models.interface :as mi]- [metabase.models.visualization-settings :as mb.viz]- [metabase.util :as u]- [metabase.warehouse-schema.models.field :as field]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(def ^:const ^Long serialization-protocol-version- "Current serialization protocol version.-- This gets stored with each dump, so we can correctly recover old dumps."- ;; version 2 - start adding namespace portion to /collections/ paths- 2)--(def ^:dynamic *include-entity-id*- "If entity_id should be included in v1 serialization dump"- false)--(def ^:private ^{:arglists '([form])} mbql-entity-reference?- "Is given form an MBQL entity reference?"- (partial mbql.normalize/is-clause? #{:field :field-id :fk-> :metric :segment}))--(defn- mbql-id->fully-qualified-name- [mbql]- (-> mbql- mbql.normalize/normalize-tokens- (lib.util.match/replace- ;; `integer?` guard is here to make the operation idempotent- [:field (id :guard integer?) opts]- [:field (fully-qualified-name :model/Field id) (mbql-id->fully-qualified-name opts)]-- ;; field-id is still used within parameter mapping dimensions- ;; example relevant clause - [:dimension [:fk-> [:field-id 1] [:field-id 2]]]- [:field-id (id :guard integer?)]- [:field-id (fully-qualified-name :model/Field id)]-- ;; source-field is also used within parameter mapping dimensions- ;; example relevant clause - [:field 2 {:source-field 1}]- {:source-field (id :guard integer?)}- (assoc &match :source-field (fully-qualified-name :model/Field id))-- [:segment (id :guard integer?)]- [:segment (fully-qualified-name :model/Segment id)])))--(defn- ids->fully-qualified-names- [entity]- (lib.util.match/replace entity- mbql-entity-reference?- (mbql-id->fully-qualified-name &match)-- map?- (as-> &match entity- (m/update-existing entity :database (fn [db-id]- (if (= db-id lib.schema.id/saved-questions-virtual-database-id)- "database/__virtual"- (fully-qualified-name :model/Database db-id))))- (m/update-existing entity :card_id (partial fully-qualified-name :model/Card)) ; attibutes that refer to db fields use _- (m/update-existing entity :card-id (partial fully-qualified-name :model/Card)) ; template-tags use dash- (m/update-existing entity :source-table (fn [source-table]- (if (and (string? source-table)- (str/starts-with? source-table "card__"))- (fully-qualified-name :model/Card (-> source-table- (str/split #"__")- second- Integer/parseInt))- (fully-qualified-name :model/Table source-table))))- (m/update-existing entity :breakout (fn [breakout]- (map mbql-id->fully-qualified-name breakout)))- (m/update-existing entity :aggregation (fn [aggregation]- (m/map-vals mbql-id->fully-qualified-name aggregation)))- (m/update-existing entity :filter (fn [filter]- (m/map-vals mbql-id->fully-qualified-name filter)))- (m/update-existing entity ::mb.viz/param-mapping-source (partial fully-qualified-name :model/Field))- (m/update-existing entity :snippet-id (partial fully-qualified-name :model/NativeQuerySnippet))- (m/map-vals ids->fully-qualified-names entity))))--(defn- strip-crud- "Removes unneeded fields that can either be reconstructed from context or are meaningless- (eg. :created_at)."- [entity]- (cond-> (dissoc entity :id :creator_id :created_at :updated_at :db_id :location :last_used_at- :dashboard_id :fields_hash :personal_owner_id :made_public_by_id :collection_id- :pulse_id :result_metadata :action_id)- (not *include-entity-id*) (dissoc :entity_id)- (some #(instance? % entity) (map type [:model/Field :model/Segment])) (dissoc :table_id)))--(defmulti ^:private serialize-one- {:arglists '([instance])}- mi/model)--(def ^{:arglists '([entity])} serialize- "Serialize entity `entity`."- (comp ids->fully-qualified-names strip-crud serialize-one))--(defmethod serialize-one :default- [instance]- instance)--(defmethod serialize-one :model/Database- [db]- (dissoc db :features))--(defmethod serialize-one :model/Field- [field]- (let [field (-> field- (update :parent_id (partial fully-qualified-name :model/Field))- (update :fk_target_field_id (partial fully-qualified-name :model/Field)))]- (if (contains? field :values)- (update field :values u/select-non-nil-keys [:values :human_readable_values])- (assoc field :values (-> field- field/values- (u/select-non-nil-keys [:values :human_readable_values]))))))--(defn- convert-column-settings-key [k]- (if-let [field-id (::mb.viz/field-id k)]- (-> (t2/select-one :model/Field :id field-id)- fully-qualified-name- mb.viz/field-str->column-ref)- k))--(defn- convert-param-mapping-key- "The `k` is something like [:dimension [:fk-> [:field-id <id1>] [:field-id <id2]]]"- [k]- (mbql-id->fully-qualified-name k))--(defn- convert-param-ref [new-id param-ref]- (cond-> param-ref- (= "dimension" (::mb.viz/param-ref-type param-ref)) ids->fully-qualified-names- (some? new-id) (update ::mb.viz/param-ref-id new-id)))--(defn- convert-param-mapping-val [new-id v]- (-> v- (m/update-existing ::mb.viz/param-mapping-source (partial convert-param-ref new-id))- (m/update-existing ::mb.viz/param-mapping-target (partial convert-param-ref new-id))- (m/assoc-some ::mb.viz/param-mapping-id (or new-id (::mb.viz/param-mapping-id v)))))--(defn- convert-parameter-mapping [param-mapping]- (if (nil? param-mapping)- nil- (reduce-kv (fn [acc k v]- (assoc acc (convert-param-mapping-key k)- (convert-param-mapping-val nil v))) {} param-mapping)))--(defn- convert-click-behavior [{:keys [::mb.viz/link-type ::mb.viz/link-target-id] :as click}]- (-> (if-let [new-target-id (case link-type- ::mb.viz/card (-> (t2/select-one :model/Card :id link-target-id)- fully-qualified-name)- ::mb.viz/dashboard (-> (t2/select-one :model/Dashboard :id link-target-id)- fully-qualified-name)- nil)]- (assoc click ::mb.viz/link-target-id new-target-id)- click)- (m/update-existing ::mb.viz/parameter-mapping convert-parameter-mapping)))--(defn- convert-column-settings-value [{:keys [::mb.viz/click-behavior] :as v}]- (cond (not-empty click-behavior) (assoc v ::mb.viz/click-behavior (convert-click-behavior click-behavior))- :else v))--(defn- convert-column-settings [acc k v]- (assoc acc (convert-column-settings-key k) (convert-column-settings-value v)))--(defn- convert-viz-settings [viz-settings]- (-> (mb.viz/db->norm viz-settings)- (m/update-existing ::mb.viz/column-settings (fn [col-settings]- (reduce-kv convert-column-settings {} col-settings)))- (m/update-existing ::mb.viz/click-behavior convert-click-behavior)- mb.viz/norm->db))--(defn- dashboard-cards-for-dashboard- [dashboard]- (let [dashboard-cards (t2/select :model/DashboardCard :dashboard_id (u/the-id dashboard))- series (when (not-empty dashboard-cards)- (t2/select :model/DashboardCardSeries- :dashboardcard_id [:in (map u/the-id dashboard-cards)]))]- (for [dashboard-card dashboard-cards]- (-> dashboard-card- (assoc :series (for [series series- :when (= (:dashboardcard_id series) (u/the-id dashboard-card))]- (-> series- (update :card_id (partial fully-qualified-name :model/Card))- (dissoc :id :dashboardcard_id))))- (assoc :visualization_settings (convert-viz-settings (:visualization_settings dashboard-card)))- strip-crud))))--(defmethod serialize-one :model/Dashboard- [dashboard]- (assoc dashboard :dashboard_cards (dashboard-cards-for-dashboard dashboard)))--(defmethod serialize-one :model/Card- [card]- (-> card- (m/update-existing :table_id (partial fully-qualified-name :model/Table))- (update :database_id (partial fully-qualified-name :model/Database))- (m/update-existing :visualization_settings convert-viz-settings)))--(defmethod serialize-one :model/Pulse- [pulse]- (assoc pulse- :cards (for [card (t2/select :model/PulseCard :pulse_id (u/the-id pulse))]- (-> card- (dissoc :id :pulse_id)- (update :card_id (partial fully-qualified-name :model/Card))))- :channels (for [channel (t2/select :model/PulseChannel :pulse_id (u/the-id pulse))]- (strip-crud channel))))--(defmethod serialize-one :model/User- [user]- (select-keys user [:first_name :last_name :email :is_superuser]))--(defmethod serialize-one :model/Dimension- [dimension]- (-> dimension- (update :field_id (partial fully-qualified-name :model/Field))- (update :human_readable_field_id (partial fully-qualified-name :model/Field))))--(defmethod serialize-one :model/NativeQuerySnippet- [snippet]- (select-keys snippet [:name :description :content]))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/upsert.clj b/enterprise/backend/src/metabase_enterprise/serialization/upsert.cljdeleted file mode 100644index b4235191fea..00000000000--- a/enterprise/backend/src/metabase_enterprise/serialization/upsert.clj+++ /dev/null@@ -1,137 +0,0 @@-(ns metabase-enterprise.serialization.upsert- "Upsert-or-skip functionality for our models."- (:require- [clojure.data :as data]- [medley.core :as m]- [metabase-enterprise.serialization.names :refer [name-for-logging]]- [metabase.util :as u]- [metabase.util.i18n :refer [trs]]- [metabase.util.json :as json]- [metabase.util.log :as log]- [methodical.core :as methodical]- [toucan2.core :as t2]- [toucan2.tools.after :as t2.after]))--(def ^:private identity-condition- {:model/Database [:name :engine]- :model/Table [:schema :name :db_id]- :model/Field [:name :table_id]- :model/Segment [:name :table_id]- :model/Collection [:name :location :namespace]- :model/Dashboard [:name :collection_id]- :model/DashboardCard [:card_id :dashboard_id :visualization_settings]- :model/DashboardCardSeries [:dashboardcard_id :card_id]- :model/FieldValues [:field_id]- :model/Dimension [:field_id :human_readable_field_id]- :model/Setting [:key]- :model/Pulse [:name :collection_id]- :model/PulseCard [:pulse_id :card_id]- :model/PulseChannel [:pulse_id :channel_type :details]- :model/Card [:name :collection_id]- :model/User [:email]- :model/NativeQuerySnippet [:name :collection_id]})--;; This could potentially be unrolled into one giant select-(defn- select-identical- [model entity]- (->> (or (identity-condition model)- (throw (ex-info (trs "Model {0} does not support upsert" model) {:model model})))- (select-keys entity)- (m/map-vals (fn [v]- (if (coll? v)- (json/encode v)- v)))- (m/mapply t2/select-one model)))--(defn- has-post-insert?- [model]- (not (methodical/is-default-primary-method? t2.after/each-row-fn [:toucan.query-type/insert.* model])))--(defmacro with-error-handling- "Execute body and catch and log any exceptions doing so throws."- [message & body]- `(try- (do ~@body)- (catch Throwable e#- (log/error e# (u/format-color 'red "%s: %s" ~message (.getMessage e#)))- nil)))--(defn- insert-many-individually!- [model on-error entities]- (for [entity entities]- (when-let [entity-id (if (= :abort on-error)- (first (t2/insert-returning-pks! model entity))- (with-error-handling- (trs "Error inserting {0}" (name-for-logging model entity))- (first (t2/insert-returning-pks! model entity))))]- entity-id)))--(defn- maybe-insert-many!- [model on-error entities]- (if (has-post-insert? model)- (insert-many-individually! model on-error entities)- (if (= :abort on-error)- (t2/insert-returning-pks! model entities)- (try- (t2/insert-returning-pks! model entities)- ;; Retry each individually so we can do as much as we can- (catch Throwable _- (insert-many-individually! model on-error entities))))))--(defn- group-by-action- "Return `entities` grouped by the action that needs to be done given the `context`."- [{:keys [mode]} model entities]- (let [same? (comp nil? second data/diff)]- (->> entities- (map-indexed (fn [position entity]- [position- entity- (select-identical model entity)]))- (group-by (fn [[_ entity existing]]- (case mode- :update (cond- (same? existing entity) :skip- existing :update- :else :insert)- :skip (if existing- :skip- :insert)))))))--(defn maybe-upsert-many!- "Batch upsert many entities.-- Within the `context` map, the following keys are recognized:- `mode` indicates mode of operation for existing entities (`:upsert` or `:skip`), as per the `identity-condition`- `on-error` indicates what to do in case of upsert error (`:continue` or `:abort`)- `pre-insert-fn` (optional) is a function to call on each entity to be inserted, before it is inserted- `post-insert-fn` (optional) is a function to call on each entity to be inserted, after it is inserted"- [{:keys [mode on-error pre-insert-fn post-insert-fn]- :or {pre-insert-fn identity- post-insert-fn identity}- :as context}- model- entities]- (let [{:keys [update insert skip]} (group-by-action context model entities)]- (doseq [[_ entity _] insert]- (log/infof "Inserting %s" (name-for-logging (name model) entity)))- (doseq [[_ _ existing] skip]- (if (= mode :skip)- (log/infof "%s already exists -- skipping" (name-for-logging (name model) existing))- (log/infof "Skipping %s (nothing to update)" (name-for-logging (name model) existing))))- (doseq [[_ _ existing] update]- (log/infof "Updating %s" (name-for-logging (name model) existing)))- (->> (concat (for [[position _ existing] skip]- [(u/the-id existing) position])- (map vector (map post-insert-fn- (maybe-insert-many! model on-error (map (comp pre-insert-fn second) insert)))- (map first insert))- (for [[position entity existing] update]- (let [id (u/the-id existing)]- (if (= on-error :abort)- (t2/update! model id entity)- (with-error-handling- (trs "Error updating {0}" (name-for-logging (name model) entity))- (t2/update! model id entity)))- [id position])))- (sort-by second)- (map first))))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.cljindex c359d2f9045..21534bcc74c 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/extract.clj@@ -37,11 +37,6 @@(not (:no-transforms opts))(conj "Transform" "TransformTag" "TransformJob")))-(defn targets-of-type- "Returns target seq filtered on given model name"- [targets model-name]- (filter #(= (first %) model-name) targets))-(defn make-targets-of-type"Returns a targets seq with model type and given ids"[model-name ids]diff --git a/enterprise/backend/src/metabase_enterprise/stale/api.clj b/enterprise/backend/src/metabase_enterprise/stale/api.cljindex 38d650c0ad6..0d66fb207ba 100644--- a/enterprise/backend/src/metabase_enterprise/stale/api.clj+++ b/enterprise/backend/src/metabase_enterprise/stale/api.clj@@ -87,7 +87,7 @@(map (fn [card](-> card(assoc :model (if (queries/model? card) "dataset" "card"))- (assoc :fully_parameterized (api.collection/fully-parameterized-query? card))+ (assoc :fully_parameterized (queries/fully-parameterized? card))(dissoc :dataset_query))))))(defn- annotate-dashboard-with-collection-infodiff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.cljindex 7308dceed70..2dadfa7fdde 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform.clj@@ -4,6 +4,8 @@[medley.core :as m][metabase-enterprise.transforms.models.transform-run :as transform-run][metabase.events.core :as events]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.util :as u]@@ -17,50 +19,59 @@(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped?]](derive :model/Transform trait))+(defn- transform-source-out [m]+ (-> m+ mi/json-out-without-keywordization+ (update-keys keyword)+ (m/update-existing :query lib-be/normalize-query)+ (m/update-existing :type keyword)))++(defn- transform-source-in [m]+ (-> m+ (m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))+ mi/json-in))+(t2/deftransforms :model/Transform- {:source mi/transform-transform-source+ {:source {:out transform-source-out, :in transform-source-in}:target mi/transform-json:run_trigger mi/transform-keyword})-(mi/define-batched-hydration-method with-transform- :transform+(methodical/defmethod t2/batched-hydrate [:model/TransformRun :transform]"Add transform to a TransformRun"- [runs]+ [_model _k runs](if-not (seq runs)runs(let [transform-ids (into #{} (map :transform_id) runs)id->transform (t2/select-pk->fn identity [:model/Transform :id :name] :id [:in transform-ids])]- (for [run runs]- (assoc run :transform (get id->transform (:transform_id run)))))))+ (for [run runs] (assoc run :transform (get id->transform (:transform_id run)))))))-(mi/define-batched-hydration-method with-last-run- :last_run+(methodical/defmethod t2/batched-hydrate [:model/Transform :last_run]"Add last_run to a transform"- [transforms]+ [_model _k transforms](if-not (seq transforms)transforms(let [transform-ids (into #{} (map :id) transforms)- last-runs (m/index-by :transform_id (transform-run/latest-runs transform-ids))]- (for [transform transforms]- (assoc transform :last_run (get last-runs (:id transform)))))))+ last-runs (m/index-by :transform_id (transform-run/latest-runs transform-ids))]+ (for [transform transforms] (assoc transform :last_run (get last-runs (:id transform)))))))-(mi/define-batched-hydration-method transform-tag-ids- :transform_tag_ids+(methodical/defmethod t2/batched-hydrate [:model/Transform :transform_tag_ids]"Add tag_ids to a transform, preserving the order defined by position"- [transforms]+ [_model _k transforms](if-not (seq transforms)transforms- (let [transform-ids (into #{} (map :id) transforms)- tag-associations (when (seq transform-ids)- (t2/select [:model/TransformTransformTag :transform_id :tag_id :position]- :transform_id [:in transform-ids]- {:order-by [[:position :asc]]}))- transform-id->tag-ids (reduce (fn [acc {:keys [transform_id tag_id]}]- (update acc transform_id (fnil conj []) tag_id))- {}- tag-associations)]- (for [transform transforms]- (assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))+ (let [transform-ids (into #{} (map :id) transforms)+ tag-associations (when (seq transform-ids)+ (t2/select+ [:model/TransformTransformTag :transform_id :tag_id :position]+ :transform_id+ [:in transform-ids]+ {:order-by [[:position :asc]]}))+ transform-id->tag-ids (reduce+ (fn [acc {:keys [transform_id tag_id]}]+ (update acc transform_id (fnil conj []) tag_id))+ {}+ tag-associations)]+ (for [transform transforms] (assoc transform :tag_ids (vec (get transform-id->tag-ids (:id transform) [])))))))(t2/define-after-insert :model/Transform [transform](events/publish-event! :event/create-transform {:object transform})@@ -75,9 +86,8 @@transform)(defn update-transform-tags!- "Update the tags associated with a transform using smart diff logic.- Only modifies what has changed: deletes removed tags, updates positions for moved tags,- and inserts new tags. Duplicate tag IDs are automatically deduplicated."+ "Update the tags associated with a transform using smart diff logic. Only modifies what has changed: deletes removed+ tags, updates positions for moved tags, and inserts new tags. Duplicate tag IDs are automatically deduplicated."[transform-id tag-ids](when transform-id(t2/with-transaction [_conn]@@ -150,7 +160,8 @@:skip [:dependency_analysis_version]:transform {:created_at (serdes/date):updated_at (serdes/date)- :source {:export serdes/export-mbql :import serdes/import-mbql}+ :source {:export #(update % :query serdes/export-mbql)+ :import #(update % :query serdes/import-mbql)}:target {:export serdes/export-mbql :import serdes/import-mbql}:tags (serdes/nested :model/TransformTransformTag :transform_id opts)}})diff --git a/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj b/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.cljindex e96079de1f7..c169b83f884 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/models/transform_run.clj@@ -3,9 +3,9 @@[medley.core :as m][metabase-enterprise.transforms.models.transform-run-cancelation :as cancel][metabase.app-db.core :as mdb]- [metabase.driver.common.parameters.dates :as params.dates][metabase.driver.sql.query-processor :as sql.qp][metabase.models.interface :as mi]+ [metabase.query-processor.parameters.dates :as params.dates][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.i18n :refer [tru]]diff --git a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj b/enterprise/backend/src/metabase_enterprise/transforms/ordering.cljindex ae184e38de3..0b7c509f7c6 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/ordering.clj@@ -1,21 +1,23 @@(ns metabase-enterprise.transforms.ordering(:require- [clojure.core.match][clojure.set :as set][clojure.string :as str][flatland.ordered.set :refer [ordered-set]][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.preprocess :as qp.preprocess]- ;; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ [metabase.util.malli :as mu][toucan2.core :as t2]))(set! *warn-on-reflection* true)-(defn- transform-deps [transform]+(mu/defn- transform-deps :- ::driver/native-query-deps+ [transform](if (transforms.util/python-transform? transform);; Python transforms have dependencies via source-tables mapping(into #{}@@ -24,32 +26,28 @@;; Query transforms have dependencies via SQL query analysis(let [query (-> (get-in transform [:source :query])transforms.util/massage-sql-query- qp.preprocess/preprocess- ;; legacy usage -- don't do things like this going forward- #_{:clj-kondo/ignore [:discouraged-var]}- lib/->legacy-MBQL)]- (case (:type query)- :native (driver/native-query-deps (-> (qp.store/metadata-provider)- lib.metadata/database- :engine)- (get-in query [:native :query]))- :query (into #{}- (keep #(clojure.core.match/match %- [:source-table source] (when (int? source)- {:table source})- _ nil))- (tree-seq coll? seq query))))))+ qp.preprocess/preprocess)+ driver (-> query+ lib.metadata/database+ :engine)]+ (if (lib/native-only-query? query)+ (driver/native-query-deps driver query)+ (into #{}+ (map (fn [table-id]+ {:table table-id}))+ (lib/all-source-table-ids query))))))(defn- dependency-map [transforms](into {}(map (juxt :id transform-deps))transforms))-(defn- output-table-map [transforms]+(mu/defn- output-table-map+ [mp :- ::lib.schema.metadata/metadata-provider transforms](let [table-map (into {}(map (fn [{:keys [schema name id]}][[schema name] id]))- (lib.metadata/tables (qp.store/metadata-provider)))]+ (lib.metadata/tables mp))](into {}(keep (fn [transform](when-let [output-table (table-map [(get-in transform [:target :schema])@@ -70,13 +68,14 @@(let [db-id (transforms.util/target-database-id transform)]{db-id [transform]})))(apply merge-with into))-transform-ids (into #{} (map :id) transforms){:keys [output-tables dependencies]} (->> transforms-by-db- (map (fn [[db-id db-transforms]]- (qp.store/with-metadata-provider db-id- {:output-tables (output-table-map db-transforms)- :dependencies (dependency-map db-transforms)})))+ (map (mu/fn [[db-id db-transforms] :- [:tuple+ [:maybe ::lib.schema.id/database]+ [:maybe [:sequential :any]]]]+ (let [mp (lib-be/application-database-metadata-provider db-id)]+ {:output-tables (output-table-map mp db-transforms)+ :dependencies (dependency-map db-transforms)})))(apply merge-with merge))](update-vals dependencies #(into #{}(keep (fn [{:keys [table transform]}]@@ -118,27 +117,27 @@```"[{transform-id :id :as to-check}]- (let [transforms (map (fn [{:keys [id] :as transform}]- (if (= id transform-id)- to-check- transform))- (t2/select :model/Transform))+ (let [transforms (map (fn [{:keys [id] :as transform}]+ (if (= id transform-id)+ to-check+ transform))+ (t2/select :model/Transform))transforms-by-id (into {}(map (juxt :id identity))transforms)- db-id (get-in to-check [:source :query :database])]- (qp.store/with-metadata-provider db-id- (let [db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)- output-tables (output-table-map db-transforms)- transform-ids (into #{} (map :id) db-transforms)- node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]- (or (output-tables table)- (transform-ids transform)))))- id->name (comp :name transforms-by-id)- cycle (find-cycle node->children [transform-id])]- (when cycle- {:cycle-str (str/join " -> " (map id->name cycle))- :cycle cycle})))))+ db-id (get-in to-check [:source :query :database])+ mp (lib-be/application-database-metadata-provider db-id)+ db-transforms (filter #(= (get-in % [:source :query :database]) db-id) transforms)+ output-tables (output-table-map mp db-transforms)+ transform-ids (into #{} (map :id) db-transforms)+ node->children #(->> % transforms-by-id transform-deps (keep (fn [{:keys [table transform]}]+ (or (output-tables table)+ (transform-ids transform)))))+ id->name (comp :name transforms-by-id)+ cycle (find-cycle node->children [transform-id])]+ (when cycle+ {:cycle-str (str/join " -> " (map id->name cycle))+ :cycle cycle})))(defn available-transforms"Given an ordering (see transform-ordering), a set of running transform ids, and a set of completed transform ids,diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 301813f16da..6e5b895c1e4 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -7,10 +7,11 @@[metabase-enterprise.transforms.models.transform-run :as transform-run][metabase-enterprise.transforms.settings :as transforms.settings][metabase.driver :as driver]- [metabase.driver.common.parameters.dates :as params.dates][metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.id :as lib.schema.id][metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.parameters.dates :as params.dates][metabase.query-processor.pipeline :as qp.pipeline][metabase.sync.core :as sync][metabase.util :as u]@@ -104,8 +105,7 @@(log/info "Syncing target" (pr-str target) "for transform")(activate-table-and-mark-computed! database target)))-;; TODO this and target-database-id can be transforms multimethods?-(defn target-database-id+(mu/defn target-database-id :- ::lib.schema.id/database"Return the target database id of a transform"[transform](if (python-transform? transform)diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 7de16b82ca3..2320e56cd63 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -91,7 +91,6 @@;; Workaround for LocalStack, which doesn't support zero byte files.(when @none?(.write writer " "))-(doto writer(.flush)(.close)))@@ -104,10 +103,8 @@(map (fn [[n v]](maybe-fixup-value (filtered-col-meta n) v)))(zipmap (filter filtered-col-meta col-names)))]-(when @none? (vreset! none? false))(json/encode-to row-map writer {})-(doto writer(.newLine)))))))@@ -179,11 +176,9 @@[{:keys [server-url code request-id run-id table-name->id shared-storage timeout-secs]}](let [{:keys [objects]} shared-storage{:keys [output output-manifest events]} objects-url-for-path (fn [path] (:url (get objects path)))table-name->url (update-vals table-name->id #(url-for-path [:table % :data]))table-name->manifest-url (update-vals table-name->id #(url-for-path [:table % :manifest]))-payload {:code code:library (t2/select-fn->fn :path :source :model/PythonLibrary):timeout (or timeout-secs (transforms-python.settings/python-runner-timeout-seconds))@@ -193,7 +188,6 @@:events_url (:url events):table_mapping table-name->url:manifest_mapping table-name->manifest-url}-response (transforms.instrumentation/with-python-api-timing [run-id](python-runner-request server-url :post "/execute" {:body (json/encode payload)}))];; when a 500 is returned we observe a string in the body (despite the python returning json)@@ -268,7 +262,6 @@driver (t2/select-one-fn :engine :model/Database db-id)fields-meta (fields-metadata driver table-id)manifest (generate-manifest table-id fields-meta)]-(transforms.instrumentation/with-stage-timing [run-id :dwh-to-file](write-table-data-to-file!{:db-id db-id@@ -278,17 +271,14 @@:temp-file tmp-data-file:cancel-chan cancel-chan:limit limit}))-(with-open [writer (io/writer tmp-meta-file)](json/encode-to manifest writer {}))(let [data-size (.length tmp-data-file)meta-size (.length tmp-meta-file)](transforms.instrumentation/record-data-transfer! run-id :dwh-to-file data-size nil)-(transforms.instrumentation/with-stage-timing [run-id :file-to-s3](s3/upload-file s3-client bucket-name data-path tmp-data-file)(s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))-(transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))(catch InterruptedException ie (throw ie))(catch Throwable tdiff --git a/enterprise/backend/test/metabase_enterprise/action_v2/models/undo_test.clj b/enterprise/backend/test/metabase_enterprise/action_v2/models/undo_test.cljindex cf9529f7f1a..a51fef8e9ec 100644--- a/enterprise/backend/test/metabase_enterprise/action_v2/models/undo_test.clj+++ b/enterprise/backend/test/metabase_enterprise/action_v2/models/undo_test.clj@@ -10,7 +10,7 @@(use-fixtures :once (fixtures/initialize :test-users-personal-collections))-(deftest diff-keys-test+(deftest ^:parallel diff-keys-test(testing "Detect which keys have changes"(is (= [:a :b :c] (#'undo/diff-keysnildiff --git a/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.clj b/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.cljindex 453e7b66676..539c9d0515a 100644--- a/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.clj+++ b/enterprise/backend/test/metabase_enterprise/advanced_permissions/query_processor/middleware/permissions_test.clj@@ -4,8 +4,8 @@[clojure.test :refer :all][metabase-enterprise.advanced-permissions.query-processor.middleware.permissions :as ee.qp.perms][metabase-enterprise.sandbox.query-processor.middleware.sandboxing :as sandboxing]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema][metabase.lib.test-metadata :as meta][metabase.lib.util.match :as lib.util.match][metabase.permissions.core :as perms]@@ -13,10 +13,10 @@[metabase.permissions.test-util :as perms.test-util][metabase.query-processor.api :as api.dataset][metabase.query-processor.reducible :as qp.reducible]- [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming-test :as streaming-test][metabase.test :as mt]- [metabase.util :as u])+ [metabase.util :as u]+ [metabase.util.malli :as mu])(:import(clojure.lang ExceptionInfo)))@@ -36,9 +36,8 @@(_ :guard (every-pred map? :source-metadata))(remove-metadata (dissoc &match :source-metadata))))-(defn- apply-row-level-permissions [query]- (-> (qp.store/with-metadata-provider (mt/id)- (#'sandboxing/apply-sandboxing (mbql.normalize/normalize query)))+(mu/defn- apply-row-level-permissions [query :- ::lib.schema/query]+ (-> (#'sandboxing/apply-sandboxing query)remove-metadata))(defmacro ^:private with-download-perms!diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/api/collection_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/api/collection_test.cljindex 9bd176b817a..456d9f8e434 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/api/collection_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/api/collection_test.clj@@ -13,7 +13,7 @@(deftest list-collections-instance-analytics-test(mt/with-premium-features #{:audit-app}- (audit-test/with-audit-db-restoration+ (audit-test/with-audit-db-restoration!(mt/with-temp [:model/Collection _ {:name "Zippy"}](testing "Instance Analytics Collection should be the last collection."(testing "GET /api/collection"@@ -27,7 +27,7 @@last:type))))))))(mt/with-premium-features #{}- (audit-test/with-audit-db-restoration+ (audit-test/with-audit-db-restoration!(mt/with-temp [:model/Collection _ {:name "Zippy"}](testing "Instance Analytics Collection should not show up when audit-app isn't enabled."(testing "GET /api/collection"@@ -70,7 +70,7 @@(testing "You should only see your collection and public collections";; Set audit-app feature so that we can assert that audit collections are also visible when running EE(mt/with-premium-features #{:audit-app}- (audit-test/with-audit-db-restoration+ (audit-test/with-audit-db-restoration!(let [admin-user-id (u/the-id (test.users/fetch-user :crowberto))crowberto-root (t2/select-one :model/Collection :personal_owner_id admin-user-id)](mt/with-temp [:model/Collection collection {}diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/api/database_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/api/database_test.cljindex 4bf2c4d0bcf..8b36d0d0c92 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/api/database_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/api/database_test.clj@@ -7,7 +7,7 @@(deftest audit-db-unmodifiable-test(mt/with-premium-features #{:audit-app}- (audit-test/with-audit-db-restoration+ (audit-test/with-audit-db-restoration!(testing "Neither admin nor regular users can modify the audit database"(doseq [[verb path] [[:post "persist/database/%d/unpersist"][:put "database/%d"]diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.cljindex a9d83b86901..342b24069fa 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/audit_test.clj@@ -23,17 +23,17 @@(use-fixtures :once (fixtures/initialize :db :plugins))-(defmacro with-audit-db-restoration [& body]+(defn do-with-audit-db-restoration! [thunk]+ (mbc/ensure-audit-db-installed!)+ (try+ (thunk)+ (finally+ (mbc/ensure-audit-db-installed!))))++(defmacro with-audit-db-restoration! [& body]"Calls `ensure-audit-db-installed!` before and after `body` to ensure that the audit DB is installed and thenrestored if necessary. Also disables audit content loading if it is already loaded."- `(let [audit-collection-exists?# (t2/exists? :model/Collection :type "instance-analytics")]- (clojure.pprint/print-table (t2/select [:model/Database :id :name]))- (mt/with-temp-env-var-value! [mb-load-analytics-content (not audit-collection-exists?#)]- (mbc/ensure-audit-db-installed!)- (try- ~@body- (finally- (mbc/ensure-audit-db-installed!))))))+ `(do-with-audit-db-restoration! (fn [] ~@body)))(deftest audit-db-installation-test(mt/test-drivers #{:postgres :h2 :mysql}@@ -116,7 +116,7 @@(t2/delete! :model/Database :id audit/audit-db-id)(mt/with-temp-scheduler!(#'task.sync-databases/job-init)- (with-audit-db-restoration+ (with-audit-db-restoration!(is (= '("metabase.task.update-field-values.trigger.13371337")(get-audit-db-trigger-keys))"no sync scheduled after installation")diff --git a/enterprise/backend/test/metabase_enterprise/audit_app/permissions_test.clj b/enterprise/backend/test/metabase_enterprise/audit_app/permissions_test.cljindex c2e9e024391..5ccb95fa4ba 100644--- a/enterprise/backend/test/metabase_enterprise/audit_app/permissions_test.clj+++ b/enterprise/backend/test/metabase_enterprise/audit_app/permissions_test.clj@@ -31,74 +31,72 @@(map :table_name (t2/query view-query))))))))(deftest audit-db-basic-query-test- (mt/test-drivers #{:postgres :h2 :mysql}- (audit-test/with-audit-db-restoration- (mt/with-premium-features #{:audit-app}- (is (= "complete" (t2/select-one-fn :initial_sync_status :model/Database :id audit/audit-db-id))- "The sync status should be completed by this point. (In test it is synchronous!)")- (mt/with-test-user :crowberto- (testing "A query using a saved audit model as the source table runs succesfully"- (let [audit-card (t2/select-one :model/Card :database_id audit/audit-db-id :type :model)]- (is (partial=- {:status :completed}- (qp/process-query- {:database audit/audit-db-id- :type :query- :query {:source-table (str "card__" (u/the-id audit-card))}})))))+ (audit-test/with-audit-db-restoration!+ (mt/with-premium-features #{:audit-app}+ (is (= "complete" (t2/select-one-fn :initial_sync_status :model/Database :id audit/audit-db-id))+ "The sync status should be completed by this point. (In test it is synchronous!)")+ (mt/with-test-user :crowberto+ (testing "A query using a saved audit model as the source table runs succesfully"+ (let [audit-card (t2/select-one :model/Card :database_id audit/audit-db-id :type :model)]+ (is (partial=+ {:status :completed}+ (qp/process-query+ {:database audit/audit-db-id+ :type :query+ :query {:source-table (str "card__" (u/the-id audit-card))}})))))- (testing "A non-native query can be run on views in the audit DB"- (let [audit-view (t2/select-one :model/Table :db_id audit/audit-db-id {:where [:like [:lower :name] "v_%"]})]- (when-not (str/starts-with? (u/lower-case-en (:name audit-view)) "v_")- (sync/sync-database! (t2/select-one :model/Database audit/audit-db-id)))- (is (partial=- {:status :completed}- (qp/process-query- {:database audit/audit-db-id- :type :query- :query {:source-table (u/the-id audit-view)}}))))))))))+ (testing "A non-native query can be run on views in the audit DB"+ (let [audit-view (t2/select-one :model/Table :db_id audit/audit-db-id {:where [:like [:lower :name] "v_%"]})]+ (when-not (some-> audit-view :name u/lower-case-en (str/starts-with? "v_"))+ (sync/sync-database! (t2/select-one :model/Database audit/audit-db-id)))+ (is (partial=+ {:status :completed}+ (qp/process-query+ {:database audit/audit-db-id+ :type :query+ :query {:source-table (u/the-id audit-view)}})))))))))(deftest audit-db-disallowed-queries-test- (mt/test-drivers #{:postgres :h2 :mysql}- (audit-test/with-audit-db-restoration- (mt/with-premium-features #{:audit-app}- (mt/with-test-user :crowberto- (testing "Native queries are not allowed to be run on audit DB views, even by admins"+ (audit-test/with-audit-db-restoration!+ (mt/with-premium-features #{:audit-app}+ (mt/with-test-user :crowberto+ (testing "Native queries are not allowed to be run on audit DB views, even by admins"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Native queries are not allowed on the audit database"+ (qp/process-query+ {:database audit/audit-db-id+ :type :native+ :native {:query "SELECT * FROM v_audit_log;"}}))))++ (testing "Non-native queries are not allowed to run on tables in the audit DB that are not views"+ ;; Nothing should be synced directly from the audit DB, just loaded via serialization, so only the views+ ;; should have metadata present in the app DB in the first place. But in case this changes, we want to+ ;; explicitly block other tables from being queried.+ (mt/with-temp [:model/Table table {:db_id audit/audit-db-id}+ :model/Field _ {:table_id (u/the-id table)}](is (thrown-with-msg?clojure.lang.ExceptionInfo- #"Native queries are not allowed on the audit database"+ #"Audit queries are only allowed on audit views"(qp/process-query{:database audit/audit-db-id- :type :native- :native {:query "SELECT * FROM v_audit_log;"}}))))-- (testing "Non-native queries are not allowed to run on tables in the audit DB that are not views"- ;; Nothing should be synced directly from the audit DB, just loaded via serialization, so only the views- ;; should have metadata present in the app DB in the first place. But in case this changes, we want to- ;; explicitly block other tables from being queried.- (mt/with-temp [:model/Table table {:db_id audit/audit-db-id}- :model/Field _ {:table_id (u/the-id table)}]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Audit queries are only allowed on audit views"- (qp/process-query- {:database audit/audit-db-id- :type :query- :query {:source-table (u/the-id table)}})))))+ :type :query+ :query {:source-table (u/the-id table)}})))))- (testing "Users without access to the audit collection cannot run any queries on the audit DB, even if they+ (testing "Users without access to the audit collection cannot run any queries on the audit DB, even if theyhave data perms for the audit DB"- (mt/with-full-data-perms-for-all-users!- (mt/with-test-user :rasta- (binding [api/*current-user-permissions-set* (delay #{})]- (let [audit-view (->> (t2/select-one :model/Table :db_id audit/audit-db-id {:where [:like [:lower :name] "v_%"]})- (tu/poll-until 5000))]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"You do not have access to the audit database"- (qp/process-query- {:database audit/audit-db-id- :type :query- :query {:source-table (u/the-id audit-view)}})))))))))))))+ (mt/with-full-data-perms-for-all-users!+ (mt/with-test-user :rasta+ (binding [api/*current-user-permissions-set* (delay #{})]+ (let [audit-view (->> (t2/select-one :model/Table :db_id audit/audit-db-id {:where [:like [:lower :name] "v_%"]})+ (tu/poll-until 5000))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"You do not have access to the audit database"+ (qp/process-query+ {:database audit/audit-db-id+ :type :query+ :query {:source-table (u/the-id audit-view)}}))))))))))))(deftest analytics-permissions-test(mt/with-premium-features #{:audit-app}diff --git a/enterprise/backend/test/metabase_enterprise/cache/cache_test.clj b/enterprise/backend/test/metabase_enterprise/cache/cache_test.cljindex 6a019d0e024..cd6625f5005 100644--- a/enterprise/backend/test/metabase_enterprise/cache/cache_test.clj+++ b/enterprise/backend/test/metabase_enterprise/cache/cache_test.clj@@ -2,6 +2,8 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.query-processor.card :as qp.card][metabase.test :as mt][metabase.util :as u]@@ -22,11 +24,14 @@:collection_id (:id col1)}:model/Card card1 {:name "card1":database_id (:id db)- :collection_id (:id col1)}+ :collection_id (:id col1)+ :dataset_query (lib/native-query (lib-be/application-database-metadata-provider (:id db)) "SELECT 1;")}:model/Card card2 {:name "card2":database_id (:id db)- :collection_id (:id col1)}- :model/Card card3 {:name "card3"}]+ :collection_id (:id col1)+ :dataset_query (lib/native-query (lib-be/application-database-metadata-provider (:id db)) "SELECT 1;")}+ :model/Card card3 {:name "card3"+ :dataset_query (lib/native-query (mt/metadata-provider) "SELECT 1;")}](testing "Can configure root"(is (mt/user-http-request :crowberto :put 200 "cache/"{:model "root"@@ -45,7 +50,6 @@;; no check for old value in case you had something in appdb:new-value {:strategy "nocache" :config {:name "root"}}}}(last-audit-event)))))-(testing "Can configure others"(is (mt/user-http-request :crowberto :put 200 "cache/"{:model "database"@@ -59,7 +63,6 @@{:model "question":model_id (:id card1):strategy {:type "nocache" :name "card1"}})))-(testing "HTTP responds with correct listings"(is (=? {:data [{:model "root" :model_id 0}]}(mt/user-http-request :crowberto :get 200 "cache/")))@@ -73,22 +76,20 @@{:model "question" :model_id (:id card1)}]}(mt/user-http-request :crowberto :get 200 "cache/" {}:collection (:id col1) :model :dashboard :model :question))))-(testing "We select correct config for something from a db"(testing "First card has own config"(is (=? {:type :nocache :name "card1"}- (:cache-strategy (#'qp.card/query-for-card card1 {} {} {} {}))))+ (:cache-strategy (#'qp.card/query-for-card card1 [] {} {} {}))))(is (=? {:type :nocache :name "card1"}- (:cache-strategy (#'qp.card/query-for-card card1 {} {} {} {:dashboard-id (u/the-id dash1)})))))+ (:cache-strategy (#'qp.card/query-for-card card1 [] {} {} {:dashboard-id (u/the-id dash1)})))))(testing "Second card should hit database or dashboard cache"(is (=? {:type :nocache :name "db"}- (:cache-strategy (#'qp.card/query-for-card card2 {} {} {} {}))))+ (:cache-strategy (#'qp.card/query-for-card card2 [] {} {} {}))))(is (=? {:type :nocache :name "dash"}- (:cache-strategy (#'qp.card/query-for-card card2 {} {} {} {:dashboard-id (u/the-id dash1)})))))+ (:cache-strategy (#'qp.card/query-for-card card2 [] {} {} {:dashboard-id (u/the-id dash1)})))))(testing "Third card targets other db and gets root config"(is (=? {:type :nocache :name "root"}- (:cache-strategy (#'qp.card/query-for-card card3 {} {} {} {}))))))-+ (:cache-strategy (#'qp.card/query-for-card card3 [] {} {} {}))))))(testing "It's possible to delete a configuration"(is (nil? (mt/user-http-request :crowberto :delete 204 "cache/"{:model "database"@@ -99,8 +100,7 @@:model :database))))(testing "And card2 gets root config"(is (=? {:type :nocache :name "root"}- (:cache-strategy (#'qp.card/query-for-card card2 {} {} {} {}))))))-+ (:cache-strategy (#'qp.card/query-for-card card2 [] {} {} {}))))))(testing "It's possible to use advanced cache strategies"(is (mt/user-http-request :crowberto :put 200 "cache/"{:model "root"diff --git a/enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj b/enterprise/backend/test/metabase_enterprise/cache/strategies_test.cljindex e84bcb7d6b3..e97453ecbd1 100644--- a/enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj+++ b/enterprise/backend/test/metabase_enterprise/cache/strategies_test.clj@@ -173,7 +173,7 @@(testing "No cache after job ran again"(mt/with-clock (t 121)(is (pos? (:schedule-invalidated (#'task.cache/refresh-cache-configs!))))- (let [q (#'qp.card/query-for-card card3 {} {} {} {})]+ (let [q (#'qp.card/query-for-card card3 [] {} {} {})](is (=? (mkres nil)(-> (qp/process-query q) (dissoc :data)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj b/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.cljindex f9804684564..dfb5c96e75c 100644--- a/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj+++ b/enterprise/backend/test/metabase_enterprise/cache/task/refresh_cache_configs_test.clj@@ -21,7 +21,8 @@[]{:database (mt/id):type :native- :native {:template-tags {"date" {:name "date"+ :native {:template-tags {"date" {:id "04ae45bd-f8fd-4e8d-9bc6-a9737f231a50"+ :name "date":display-name "Check-In Date":type :text}}:query "SELECT count(*) FROM CHECKINS [[WHERE date = {{date}}]]"}})diff --git a/enterprise/backend/test/metabase_enterprise/comments/api_test.clj b/enterprise/backend/test/metabase_enterprise/comments/api_test.cljindex de6985eec46..0c516ca3347 100644--- a/enterprise/backend/test/metabase_enterprise/comments/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/comments/api_test.clj@@ -321,16 +321,16 @@:model/Comment {comment-id :id} {:target_id doc-id}](testing "Comments are not shown if in iframe"(is (=? {:comments [{:id comment-id}]}- (mt/user-http-request :rasta :get 200 (str "ee/comment/")+ (mt/user-http-request :rasta :get 200 "ee/comment/":target_type "document":target_id doc-id)))(is (=? {:comments []:disabled true}- (mt/user-http-request :rasta :get 200 (str "ee/comment/")+ (mt/user-http-request :rasta :get 200 "ee/comment/"{:request-options {:headers {"x-metabase-client" @#'sdk/embedding-iframe-client}}}:target_type "document":target_id doc-id))))(testing "Users mentions are not available either"(is (= "Not found."- (mt/user-http-request :rasta :get 404 (str "ee/comment/mentions")+ (mt/user-http-request :rasta :get 404 "ee/comment/mentions"{:request-options {:headers {"x-metabase-client" @#'sdk/embedding-iframe-client}}})))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.cljindex d761133355a..b35b656231a 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/core_test.clj@@ -27,8 +27,7 @@mp (lib.tu/metadata-provider-with-card-from-query mp 102 query2);; A transform that depends on card1- tf1-query (-> (lib/query mp card1)- lib/->legacy-MBQL)+ tf1-query (lib/query mp card1)transform1 {:id 30:name "MBQL Transform":source {:query tf1-query}@@ -139,8 +138,10 @@{transformed-card-id :id} :mbql-transform-consumer} (testbed)](testing "a column that no longer exists will cause errors when referenced"(let [card' (-> mbql-base- (update-in [:dataset-query :query :expressions]- update-keys (constantly "Sales Taxes"))+ ;; ridiculous+ (update :dataset-query lib/update-query-stage -1+ update-in [:expressions 0]+ lib/update-options assoc :lib/expression-name "Sales Taxes")(dissoc :result-metadata))errors (dependencies/errors-from-proposed-edits provider graph {:card [card']})](is (=? {:card {downstream-card-id [[:field {} "Tax Rate"]]@@ -148,7 +149,6 @@errors))(is (= [:card] (keys errors)))(is (= #{downstream-card-id transformed-card-id} (set (keys (:card errors)))))))-(testing "changing something unrelated will cause no errors"(let [card' (-> mbql-base(assoc-in [:dataset-query :query :filter]diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljindex a6e802b975e..296805823b5 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -3,6 +3,8 @@[clojure.test :refer :all][metabase-enterprise.dependencies.native-validation :as deps.native-validation][metabase-enterprise.dependencies.test-util :as deps.tu]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta]))@@ -10,66 +12,70 @@([mp query](fake-query mp query {}))([mp query template-tags]- {:database (:id (lib.metadata/database mp))- :type :native- :native {:query query- :template-tags template-tags}}))+ (-> (lib/native-query mp query)+ (lib/with-template-tags template-tags))))(defn- validates?[mp driver card-id expected](is (=? expected- (->> (lib.metadata/card mp card-id)- :dataset-query- (deps.native-validation/validate-native-query driver mp)))))+ (-> (lib.metadata/card mp card-id)+ :dataset-query+ (assoc :lib/metadata mp)+ (->> (deps.native-validation/validate-native-query driver))))))-(deftest basic-deps-test+(deftest ^:parallel basic-deps-test(let [mp (deps.tu/default-metadata-provider)driver (:engine (lib.metadata/database mp))]- (is (= #{{:table (meta/id :products)}}+ (is (= #{{:table (meta/id :products)}} ; how is a driver supposed to work out the ID of the table??(->> (lib.metadata/card mp 4):dataset-query- (deps.native-validation/native-query-deps driver mp))))+ (lib-be/normalize-query mp)+ (deps.native-validation/native-query-deps driver))))(is (= #{{:table (meta/id :products)}{:card 1}}(->> (lib.metadata/card mp 5):dataset-query- (deps.native-validation/native-query-deps driver mp))))+ (lib-be/normalize-query mp)+ (deps.native-validation/native-query-deps driver))))(is (= #{{:table (meta/id :products)}{:card 1}{:snippet 1}}(->> (lib.metadata/card mp 6):dataset-query- (deps.native-validation/native-query-deps driver mp))))+ (lib-be/normalize-query mp)+ (deps.native-validation/native-query-deps driver))))(is (= #{{:table (meta/id :products)}{:card 1}{:snippet 1}{:snippet 2}}(->> (lib.metadata/card mp 7):dataset-query- (deps.native-validation/native-query-deps driver mp))))+ (lib-be/normalize-query mp)+ (deps.native-validation/native-query-deps driver))))(is (= #{{:table (meta/id :products)}{:table (meta/id :orders)}{:card 1}{:card 2}}(->> (lib.metadata/card mp 9):dataset-query- (deps.native-validation/native-query-deps driver mp))))))+ (lib-be/normalize-query mp)+ (deps.native-validation/native-query-deps driver))))))-(deftest validate-bad-queries-test+(deftest ^:parallel validate-bad-queries-test(testing "validate-native-query handles nonsense queries"(let [mp (deps.tu/default-metadata-provider)driver (:engine (lib.metadata/database mp))](testing "complete nonsense query"- (is (= [{:error :bad-sql}]+ (is (= [{:error :metabase.driver.sql/bad-sql}](deps.native-validation/validate-native-query- driver mp+ driver(fake-query mp "this is not a query")))))(testing "bad table wildcard"(is (= [{:type :invalid-table-wildcard,:table "products",:metabase.driver.sql/bad-reference true}](deps.native-validation/validate-native-query- driver mp+ driver(fake-query mp "select products.* from orders")))))(testing "bad col reference"(is (= [{:column "BAD",@@ -78,17 +84,16 @@:source-columns [[{:type :all-columns, :table {:table "PRODUCTS"}}]],:metabase.driver.sql/bad-reference true}](deps.native-validation/validate-native-query- driver mp+ driver(fake-query mp "select bad from products"))))))))-(deftest validate-native-query-with-subquery-columns-test+(deftest ^:parallel validate-native-query-with-subquery-columns-test(testing "validate-native-query should detect invalid columns in subqueries"(let [mp (deps.tu/default-metadata-provider)driver (:engine (lib.metadata/database mp))](testing "Valid query - selecting existing columns from subquery"(validates? mp driver 10 empty?))-(testing "Invalid query - selecting non-existent column from subquery"(validates? mp driver 11 [{:column "CATEGORY",:alias nil,@@ -100,7 +105,6 @@:type :single-column,:source-columns [[]],:metabase.driver.sql/bad-reference true}]))-(testing "Nested subqueries"(validates? mp driver 13 empty?)(validates? mp driver 14 [{:column "CATEGORY",@@ -108,7 +112,6 @@:type :single-column,:source-columns [[]],:metabase.driver.sql/bad-reference true}]))-(testing "SELECT * from subquery expands to subquery columns"(validates? mp driver 15 empty?)(validates? mp driver 16 empty?)@@ -118,14 +121,12 @@:source-columns [[]],:metabase.driver.sql/bad-reference true}])))))-(deftest validate-card-reference-after-expansion-test+(deftest ^:parallel validate-card-reference-after-expansion-test(testing "Validation of queries after card references have been expanded"(let [mp (deps.tu/default-metadata-provider)driver (:engine (lib.metadata/database mp))]-(testing "Card reference expanded to subquery - valid columns"(validates? mp driver 18 empty?))-(testing "Card reference expanded to subquery - invalid column"(validates? mp driver 19[{:column "DESCRIPTION",@@ -133,10 +134,8 @@:type :single-column,:source-columns [[]],:metabase.driver.sql/bad-reference true}]))-(testing "Card reference with alias - valid column"(validates? mp driver 20 empty?))-(testing "Card reference with alias - invalid column"(validates? mp driver 21[{:column "PASSWORD",@@ -144,10 +143,8 @@:type :single-column,:source-columns [[]],:metabase.driver.sql/bad-reference true}]))-(testing "Wildcard selection from card reference"(validates? mp driver 22 empty?))-(testing "Invalid column from aliased card"(validates? mp driver 23[{:column "LATITUDE",@@ -160,12 +157,12 @@(is (=? expected(->> query(fake-query mp)- (deps.native-validation/native-result-metadata driver mp)))))+ (deps.native-validation/native-result-metadata driver)))))(defn- add-desired-column-alias [fields](map #(assoc % :lib/desired-column-alias (:name %)) fields))-(deftest result-metadata-test+(deftest ^:parallel result-metadata-test(testing "Calculates result metadata"(let [mp (deps.tu/default-metadata-provider)driver (:engine (lib.metadata/database mp))]diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljindex cd8c1aa223f..5f79a5919ee 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -14,19 +14,19 @@(use-fixtures :once (fixtures/initialize :db))-(defn- backfill-all-existing-entities+(defn- backfill-all-existing-entities)))+ (while (#'dependencies.backfill/backfill-dependencies!))))-(defn- backfill-dependencies-single-trigger+(defn- backfill-dependencies-single-trigger))+ (#'dependencies.backfill/backfill-dependencies!)))(deftest ^:sequential backfill-dependency-analysis-test(testing "Test that the backfill job correctly updates the dependency_analysis_version"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")](let [query (mt/mbql-query orders)](mt/with-premium-features #{}@@ -38,18 +38,18 @@current-version dependencies.model/current-dependency-analysis-version](is (= 3 (card-count)));; first run, should process 2 cards- (is (true? (backfill-dependencies-single-trigger)))+ (is (true? (backfill-dependencies-single-trigger!)))(is (= 2 (card-count :dependency_analysis_version current-version)));; second run, should process the last card- (is (false? (backfill-dependencies-single-trigger)))+ (is (false? (backfill-dependencies-single-trigger!)))(is (= 3 (card-count :dependency_analysis_version current-version)));; third run, should not process anything- (is (false? (backfill-dependencies-single-trigger)))+ (is (false? (backfill-dependencies-single-trigger!)))(is (= 3 (card-count :dependency_analysis_version current-version))))))))))(deftest ^:sequential backfill-transform-test(testing "Test that transforms are correctly backfilled"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")](mt/with-premium-features #{}(mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)@@ -64,7 +64,7 @@(is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))(is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))(is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))- (backfill-dependencies-single-trigger)+ (backfill-dependencies-single-trigger!)(is (t2/exists? :model/Card :id card-id:dependency_analysis_version dependencies.model/current-dependency-analysis-version))(is (t2/exists? :model/Transform :id transform-id@@ -78,7 +78,7 @@(deftest ^:sequential backfill-snippet-test(testing "Test that cards with snippets are correctly backfilled"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")](mt/with-premium-features #{}(mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"@@ -101,7 +101,7 @@(is (false? (t2/exists? :model/Dependency:from_entity_type :card :from_entity_id card-id:to_entity_type :snippet :to_entity_id snippet-id)))- (backfill-dependencies-single-trigger)+ (backfill-dependencies-single-trigger!)(is (t2/exists? :model/Card :id card-id:dependency_analysis_version dependencies.model/current-dependency-analysis-version))(is (t2/exists? :model/NativeQuerySnippet :id snippet-id@@ -112,7 +112,7 @@(deftest ^:sequential backfill-idempotency-test(testing "Running the backfill multiple times should be idempotent"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")](mt/with-premium-features #{}(mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)@@ -121,11 +121,11 @@(is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :orders))));; First run- (backfill-dependencies-single-trigger)+ (backfill-dependencies-single-trigger!)(is (t2/exists? :model/Card :id card-id:dependency_analysis_version dependencies.model/current-dependency-analysis-version));; Second run - should not change anything- (backfill-dependencies-single-trigger)+ (backfill-dependencies-single-trigger!)(is (t2/exists? :model/Card :id card-id:dependency_analysis_version dependencies.model/current-dependency-analysis-version))(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id@@ -142,7 +142,7 @@(deftest ^:sequential backfill-scheduling-test(testing "Test that the backfill job schedules and reschedules itself correctly"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(mt/with-temp-scheduler!(with-redefs [env/env (assoc env/env:mb-dependency-backfill-batch-size "1"@@ -166,13 +166,13 @@(deftest ^:sequential backfill-terminal-failure-test(testing "Entities should be marked as terminally broken after MAX_RETRIES failures"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(mt/with-premium-features #{}(mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}](is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))(let [update-attempts (volatile! 0)- failures (inc @#'dependencies.backfill/MAX_RETRIES)]+ failures (inc @#'dependencies.backfill/max-retries)](with-redefs [env/env (assoc env/env:mb-dependency-backfill-delay-minutes "0":mb-dependency-backfill-variance-minutes "0")@@ -186,18 +186,18 @@(apply t2/update! model-kw id args)))];; fail MAX_RETRIES + 1 times(while (< @update-attempts failures)- (backfill-dependencies-single-trigger))))+ (backfill-dependencies-single-trigger!))));; verify card is not processed and is terminally broken(is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0));; verify subsequent runs don't process it- (backfill-dependencies-single-trigger)+ (backfill-dependencies-single-trigger!)(is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))))))(deftest ^:sequential backfill-delayed-retry-test(testing "Failed entities should be retried after their delay period expires"- (backfill-all-existing-entities)+ (backfill-all-existing-entities!)(let [current-version dependencies.model/current-dependency-analysis-versionupdate-attempts (volatile! 0)](mt/with-premium-features #{}@@ -217,15 +217,15 @@;; first failure - should be put into retry state;; fail MAX_RETRIES + 1 times(while (zero? @update-attempts)- (backfill-dependencies-single-trigger))+ (backfill-dependencies-single-trigger!))(is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0)));; advance time by less than retry delay - should NOT be processed(mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))- (backfill-dependencies-single-trigger))+ (backfill-dependencies-single-trigger!))(is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0));; advance time by more than retry delay - should be processed(mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))- (backfill-dependencies-single-trigger))+ (backfill-dependencies-single-trigger!))(is (t2/exists? :model/Card :id card-id :dependency_analysis_version current-version)))))))diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.cljindex 04af180be73..7cb99b828a6 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/api_test.clj@@ -14,7 +14,7 @@[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.u][metabase-enterprise.metabot-v3.util :as metabot-v3.u][metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.test :as mt]@@ -717,7 +717,7 @@;; Helper function to set up model test fixtures(defn- model-test-fixtures []- (let [mp (lib.metadata.jvm/application-database-metadata-provider (mt/id))+ (let [mp (lib-be/application-database-metadata-provider (mt/id))source-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))model-data {:name "Model Model":description "Model desc"diff --git a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/util_test.clj b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/util_test.cljindex 86b9423bca2..4c0457a4a17 100644--- a/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/metabot_v3/tools/util_test.clj@@ -4,7 +4,7 @@[metabase-enterprise.metabot-v3.tools.util :as metabot-v3.tools.util][metabase.collections.models.collection :as collection][metabase.content-verification.core :as moderation]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.permissions.models.permissions :as perms]@@ -82,7 +82,7 @@(mt/dataset test-data(mt/with-current-user (mt/user->id :crowberto)(let [test-db-id (mt/id)- mp (lib.metadata.jvm/application-database-metadata-provider test-db-id)+ mp (lib-be/application-database-metadata-provider test-db-id)orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))columns (lib/visible-columns orders-query)]diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/models/params/chain_filter_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/models/params/chain_filter_test.cljindex c8d09a72532..063719aeaee 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/models/params/chain_filter_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/models/params/chain_filter_test.clj@@ -10,7 +10,7 @@(deftest chain-filter-sandboxed-field-values-test(testing "When chain-filter would normally return cached FieldValues (#13832), make sure sandboxing is respected"- (mt/with-model-cleanup [:model/FieldValues]+ (t2/with-transaction [_conn nil {:rollback-only true}](met/with-gtaps! {:gtaps {:categories {:query (mt/mbql-query categories {:filter [:< $id 3]})}}}(field-values/clear-advanced-field-values-for-field! (mt/id :categories :name))(testing "values"diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 3bfa6976864..67e67848af1 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -23,7 +23,7 @@[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test][metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming.test-util :as streaming.test-util][metabase.query-processor.util :as qp.util][metabase.query-processor.util.add-alias-info :as add]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.cljindex f5ce12f0f1a..aaf8a1ec741 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/cmd_test.clj@@ -3,7 +3,6 @@[clojure.java.io :as io][clojure.string :as str][clojure.test :refer :all]- [metabase-enterprise.serialization.load :as load][metabase-enterprise.serialization.test-util :as ts][metabase-enterprise.serialization.v2.extract :as v2.extract][metabase-enterprise.serialization.v2.ingest :as v2.ingest]@@ -11,156 +10,12 @@[metabase.analytics.snowplow-test :as snowplow-test][metabase.cmd.core :as cmd][metabase.test :as mt]- [metabase.test.fixtures :as fixtures]- [metabase.test.initialize.test-users :as test-users]- [metabase.util :as u]- [metabase.util.log :as log]- [metabase.util.yaml :as yaml]- [toucan2.core :as t2]))+ [metabase.test.fixtures :as fixtures]))(set! *warn-on-reflection* true)(use-fixtures :once (fixtures/initialize :db :test-users))-(deftest no-collections-test- (testing "Dumping a card when there are no active collection should work properly (#16931)"- ;; we need a blank H2 app db, temporarily, in order to run this test (to ensure we have no collections present,- ;; while also not deleting or messing with any existing user personal collections that the real app DB might have,- ;; since that will interfere with other tests)- ;;- ;; making use of the functionality in the [[metabase.app-db.schema-migrations-test.impl]] namespace for this (since it- ;; already does what we need)- (mt/with-premium-features #{:serialization}- (mt/with-empty-h2-app-db!- ;; create a single dummy User to own a Card and a Database for it to reference- (let [user (t2/insert! (t2/table-name :model/User)- :email "[email protected]"- :first_name (mt/random-name)- :last_name (mt/random-name)- :password (str (random-uuid))- :date_joined :%now- :is_active true- :is_superuser true)- db-id (first (t2/insert-returning-pks!- (t2/table-name :model/Database)- :name "Test Database"- :engine "h2"- :details "{}"- :created_at :%now- :updated_at :%now))]- ;; then the card itself-- (t2/insert! (t2/table-name :model/Card)- :name "Single Card"- :display "Single Card"- :database_id db-id- :dataset_query "{}"- :creator_id (u/the-id user)- :visualization_settings "{}"- :parameters "[]"- :parameter_mappings "[]"- :created_at :%now- :updated_at :%now)- ;; serialize "everything" (which should just be the card and user), which should succeed if #16931 is fixed- (is (nil? (cmd/dump (ts/random-dump-dir "serdes-")))))))))--(deftest blank-target-db-test- (testing "Loading a dump into an empty app DB still works (#16639)"- (mt/with-premium-features #{:serialization}- (ts/with-dbs [source-db dest-db]- (let [dump-dir (ts/random-dump-dir "serdes-")- user-pre-insert-called? (atom false)]- (log/infof "Dumping to %s" dump-dir)- (ts/with-db source-db- (test-users/init!)- (ts/create! :model/Collection :name "My_Collection")- (cmd/dump dump-dir "--user" "[email protected]"))- (ts/with-db dest-db- (with-redefs [load/pre-insert-user (fn [user]- (reset! user-pre-insert-called? true)- (assoc user :password "test-password"))]- (cmd/load dump-dir "--mode" "update"- "--on-error" "abort")- (is (true? @user-pre-insert-called?)))))))))--(deftest mode-update-remove-cards-test- (testing "--mode update should remove Cards in a Dashboard if they're gone from the serialized YAML (#20786)"- (mt/with-premium-features #{:serialization}- (ts/with-random-dump-dir [dump-dir "serialization"]- (let [dashboard-yaml-filename (str dump-dir "/collections/root/dashboards/Dashboard.yaml")]- (ts/with-dbs [source-db dest-db]- (testing "create 2 questions in the source and add them to a dashboard"- (ts/with-db source-db- (let [{db-id :id, :as db} (ts/create! :model/Database :name "My_Database")]- (mt/with-db db- (let [{user-id :id} (ts/create! :model/User, :is_superuser true)- {card-1-id :id} (ts/create! :model/Card- :database_id db-id- :creator_id user-id- :name "Card_1"- :dataset_query {:database db-id, :type :native, :native {:query "SELECT 1;"}})- {card-2-id :id} (ts/create! :model/Card- :database_id db-id- :creator_id user-id- :name "Card_2"- :dataset_query {:database db-id, :type :native, :native {:query "SELECT 1;"}})- {dashboard-id :id} (ts/create! :model/Dashboard, :creator_id user-id, :name "Dashboard")]- (doseq [card-id [card-1-id card-2-id]]- (ts/create! :model/DashboardCard :dashboard_id dashboard-id, :card_id card-id))- (testing "dump in source"- (is (nil? (cmd/dump dump-dir)))))))))- (testing "verify the Dashboard was dumped as expected"- (is (.exists (io/file dashboard-yaml-filename)))- (let [yaml (yaml/from-file dashboard-yaml-filename)]- (is (partial= {:dashboard_cards [{:card_id "/collections/root/cards/Card_1"}- {:card_id "/collections/root/cards/Card_2"}]}- yaml))))- (testing "load into destination"- (ts/with-db dest-db- (testing "Create admin user"- (is (some? (ts/create! :model/User, :is_superuser true)))- (is (t2/exists? :model/User :is_superuser true)))- (is (nil? (cmd/load dump-dir "--on-error" "abort")))- (testing "verify that things were loaded as expected"- (is (= 1 (t2/count :model/Dashboard)) "# Dashboards")- (is (= 2 (t2/count :model/Card)) "# Cards")- (is (= 2 (t2/count :model/DashboardCard)) "# DashboardCards") >)))- (testing "remove one of the questions in the source's dashboard"- (ts/with-db source-db- (t2/delete! :model/Card :name "Card_2")- (is (= 1 (t2/count :model/Card)) "# Cards")- (is (= 1 (t2/count :model/DashboardCard)) "# DashboardCards")))- (testing "dump again"- (ts/with-db source-db- (cmd/dump dump-dir))- (testing "Verify dump only contains one Card"- (is (.exists (io/file dashboard-yaml-filename)))- (when-let [yaml (yaml/from-file dashboard-yaml-filename)]- (is (partial= {:dashboard_cards [{:card_id "/collections/root/cards/Card_1"}]}- yaml)))))- (testing "load again, with --mode update, destination Dashboard should now only have one question."- (ts/with-db dest-db- (is (nil? (cmd/load dump-dir "--mode" "update", "--on-error" "abort")))- (is (= 1 (t2/count :model/Dashboard)) "# Dashboards")- (testing "Don't delete the Card even tho it was deleted. Just delete the DashboardCard"- (is (= 2 (t2/count :model/Card)) "# Cards"))- (is (= 1 (t2/count :model/DashboardCard)) "# DashboardCards")))))))))--(deftest premium-features-test- (testing "without a premium token"- (mt/with-premium-features #{}- (ts/with-random-dump-dir [dump-dir "serdes-"]- (testing "dump should fail"- (is (thrown-with-msg? Exception #"Please upgrade"- (cmd/dump dump-dir "--user" "[email protected]"))))-- (testing "load should fail"- (mt/with-empty-h2-app-db!- (is (thrown-with-msg? Exception #"Please upgrade"- (cmd/load dump-dir- "--mode" "update"- "--on-error" "abort")))))))))-(deftest dump-readonly-dir-test(testing "command exits early when destination is not writable"(mt/with-premium-features #{:serialization}@@ -253,42 +108,3 @@;; t2/with-transactions re-wraps errors with data about toucan connections"error_message" "Failed to read file {:path \"Collection DoesNotExist\"}"}(-> (snowplow-test/pop-event-data-and-user-id!) first :data)))))))))))))--(deftest entity-id-dump&load-test- (let [entity-ids* (atom {})- eid-map #(into {} (map (juxt :name :entity_id) %))]- (testing "--include-entity-id should include entity ids in serialization"- (mt/with-premium-features #{:serialization}- (ts/with-random-dump-dir [dump-dir "serialization"]- (ts/with-dbs [source-db dest-db]- (testing "create 2 questions in a dashboard"- (ts/with-db source-db- (let [db (ts/create! :model/Database)- dash (ts/create! :model/Dashboard)- c1 (ts/create! :model/Card {:name "card1"- :database_id (:id db)- :dataset_query {:database (:id db), :type :native, :native {:query "SELECT 1;"}}})- c2 (ts/create! :model/Card {:name "card2"- :database_id (:id db)- :dataset_query {:database (:id db), :type :native, :native {:query "SELECT 1;"}}})- _ (ts/create! :model/DashboardCard {:dashboard_id (:id dash) :card_id (:id c1)})- _ (ts/create! :model/DashboardCard {:dashboard_id (:id dash) :card_id (:id c2)})]- (testing "initial dump"- (is (nil? (cmd/dump dump-dir))))- (testing "storing original entity ids"- (is (reset! entity-ids* (eid-map [c1 c2])))))))- (testing "initial load"- (ts/with-db dest-db- (is (some? (ts/create! :model/User, :is_superuser true)))- (is (nil? (cmd/load dump-dir "--on-error" "abort")))- (testing "verify that entities got their own entity_id"- (is (not= @entity-ids*- (eid-map (t2/select :model/Card)))))))- (testing "creating dump with entity ids included"- (ts/with-db source-db- (is (nil? (cmd/dump dump-dir "--include-entity-id")))))- (testing "loading dump with entity ids will overwrite new entity ids with original ones"- (ts/with-db dest-db- (is (nil? (cmd/load dump-dir "--on-error" "abort" "--mode" "update")))- (is (= @entity-ids*- (eid-map (t2/select :model/Card))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/load_test.cljdeleted file mode 100644index 3c5bc51eaf7..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/load_test.clj+++ /dev/null@@ -1,410 +0,0 @@-(ns ^:mb/driver-tests metabase-enterprise.serialization.load-test- (:refer-clojure :exclude [load])- (:require- [clojure.data :as data]- [clojure.java.io :as io]- [clojure.test :refer [deftest is testing use-fixtures]]- [metabase-enterprise.serialization.cmd :refer [v1-dump! v1-load!]]- [metabase-enterprise.serialization.load :as load]- [metabase-enterprise.serialization.test-util :as ts]- [metabase.models.interface :as mi]- [metabase.models.visualization-settings :as mb.viz]- [metabase.query-processor :as qp]- [metabase.query-processor.middleware.permissions :as qp.perms]- [metabase.test :as mt]- [metabase.test.data.users :as test.users]- [metabase.test.fixtures :as fixtures]- [metabase.util :as u]- [metabase.util.i18n :refer [trs]]- [metabase.util.log :as log]- [toucan2.core :as t2])- (:import- (org.apache.commons.io FileUtils)))--(set! *warn-on-reflection* true)--(use-fixtures :once- (fixtures/initialize :test-users-personal-collections))--(defn- delete-directory!- [file-or-filename]- (FileUtils/deleteDirectory (io/file file-or-filename)))--(def ^:private dump-dir "test-dump")--(defn- world-snapshot- []- (into {} (for [model [:model/Database :model/Table :model/Field :model/Segment :model/Collection :model/Dashboard :model/DashboardCard :model/Pulse- :model/Card :model/DashboardCardSeries :model/FieldValues :model/Dimension :model/PulseCard :model/PulseChannel :model/User- :model/NativeQuerySnippet]]- [model (t2/select-fn-set :id model)])))--(defmacro with-world-cleanup- [& body]- `(let [snapshot# (world-snapshot)]- (try- ~@body- (finally- (doseq [[model# ids#] (second (data/diff snapshot# (world-snapshot)))]- (some->> ids#- not-empty- (vector :in)- (t2/delete! model# :id)))))))--(defn- card-query-results [card]- (let [query (:dataset_query card)]- (binding [qp.perms/*card-id* nil]- (-> query- (assoc-in [:info :card-id] (:id card))- qp/process-query))))--(defn- query-res-match- "Checks that the queries for a card match between original (pre-dump) and new (after load). For now, just checks the- native form, not actual data, since some of the statements don't have an ordering (thus the rows won't be stable)."- [orig-results loaded-card]- (let [card-name (:name loaded-card)- orig-result (get orig-results card-name)]- (try- (let [new-result (card-query-results loaded-card)]- (is (some? orig-result) (format "No original query result found for card: %s" card-name))- (is (some? new-result) (format "No results produced for loaded card: %s" card-name))- (is- (apply = (map #(get-in % [:data :native_form :query]) [orig-result new-result]))- "Native query form did not match"))- (catch Exception e- (let [msg (trs "Failed to execute query for loaded card \"{0}\": {1}" card-name (.getMessage e))]- (log/error e msg)- ;; TODO: figure out a better way to simply fail with an explicit message- (is (nil? e) msg))))))--(defn- collection-name-for-card [card]- (let [collection-id (:collection_id card)]- (if (nil? collection-id)- "root"- (t2/select-one-fn :name :model/Collection :id (:collection_id card)))))--(defn- collection-names-match- "Checks that the collection name for a card matches between original (pre-dump) and new (after load)."- [collection-names loaded-card]- (let [card-name (:name loaded-card)]- (is (= (get collection-names card-name) (collection-name-for-card loaded-card)) "Collection name did not match")))--(defn- gather-orig-results- "Create a map from card names to their query results"- [card-ids]- (reduce (fn [acc card-id]- (let [card (t2/select-one :model/Card :id card-id)]- (assoc acc (:name card) (card-query-results card))))- {}- card-ids))--(defn- gather-collections- "Create a map from card names to their collection names"- [card-ids]- (reduce (fn [acc card-id]- (let [card (t2/select-one :model/Card :id card-id)]- (assoc acc (:name card) (collection-name-for-card card))))- {}- card-ids))--(defmulti ^:private assert-loaded-entity- {:arglists '([instance fingerprint])}- (fn [instance _fingerprint]- (mi/model instance)))--(defn- test-loaded-card [card card-name]- (when (= "My Nested Card" card-name)- (testing "Visualization settings for a Card were persisted correctly"- (let [vs (:visualization_settings card)- col (-> (:column_settings vs)- first)- [col-key col-val] col- col-ref (mb.viz/parse-db-column-ref col-key)- {:keys [::mb.viz/field-id]} col-ref- [{col-name :name col-field-ref :fieldRef col-enabled :enabled :as _tbl-col} & _] (:table.columns vs)- [_ col-field-id _] col-field-ref]- (is (some? (:table.columns vs)))- (is (some? (:column_settings vs)))- (is (integer? field-id))- (is (= "latitude" (-> (t2/select-one-fn :name :model/Field :id field-id)- u/lower-case-en)))- (is (= {:show_mini_bar true- :column_title "Parallel"} col-val))- (is (= "Venue Category" col-name))- (is (true? col-enabled))- (is (integer? col-field-id) "fieldRef within table.columns was properly serialized and loaded")- (is (= "category_id" (-> (t2/select-one-fn :name :model/Field :id col-field-id)- u/lower-case-en)))))))--(defn- test-pivot-card [card card-name]- (when (= "Pivot Table Card" card-name)- (testing "Visualization settings for a Card were persisted correctly"- (let [vs (:visualization_settings card)- pivot (:pivot_table.column_split vs)- vecs (concat (:columns pivot) (:rows pivot))]- (is (some? vecs))- (doseq [column-name vecs]- (is (string? column-name) "column names within pivot table was properly serialized and loaded"))))))--(defmethod assert-loaded-entity :model/Card- [{card-name :name :as card} {:keys [query-results collections]}]- (testing (format "Card: %s" card-name)- (query-res-match query-results card)- (collection-names-match collections card)- (test-loaded-card card card-name)- (test-pivot-card card card-name)- card))--(defn- collection-parent-name [collection]- (let [[_ ^String parent-id] (re-matches #".*/(\d+)/$" (:location collection))]- (t2/select-one-fn :name :model/Collection :id (Integer. parent-id))))--(defmethod assert-loaded-entity :model/Collection- [collection _]- (case (:name collection)- "My Nested Collection" (is (= "My Collection" (collection-parent-name collection)))- "My Collection" (is (= "/" (:location collection)))- "Snippet Collection" (is (= "/" (:location collection)))- "Nested Snippet Collection" (is (= "Snippet Collection" (collection-parent-name collection)))- "Crowberto's Personal Collection" (is (= "/" (:location collection)))- "Nested Personal Collection" (is (= "Crowberto Corv's Personal Collection"- (collection-parent-name collection)))- "Deeply Nested Personal Collection" (is (= "Nested Personal Collection"- (collection-parent-name collection)))- "Felicia's Personal Collection" (is (nil? (:name collection))- "Should not have loaded different user's PC")- "Felicia's Nested Collection" (is (nil? (:name collection))- "Should not have loaded different user's PC"))- collection)--(defmethod assert-loaded-entity :model/NativeQuerySnippet- [snippet {:keys [entities]}]- (when-let [orig-snippet (first (filter (every-pred #(mi/instance-of? :model/NativeQuerySnippet %)- #(= (:name snippet) (:name %))) (map last entities)))]- (is (some? orig-snippet))- (is (= (select-keys orig-snippet [:name :description :content])- (select-keys snippet [:name :description :content])))- snippet))--(defn- id->name [model id]- (t2/select-one-fn :name model :id id))--(defn- assert-price-click [click target-id]- ;; first, it should be pointing to "My Card"- (is (= "My Card" (id->name :model/Card target-id)))- (let [param-mapping (:parameterMapping click)]- ;; also, the parameter mappings should have been preserved- (is (not-empty param-mapping))- (let [mapping-key (-> param-mapping- keys- first)- mapping-keyname (#'mb.viz/keyname mapping-key)- [_ f1-id f2-id] (re-matches #".*\[\"field(?:-id)?\",(\d+),.*\[\"field(?:-id)?\",(\d+),.*" mapping-keyname)- f1 (t2/select-one :model/Field :id (Integer/parseInt f1-id))- f2 (t2/select-one :model/Field :id (Integer/parseInt f2-id))- dimension (get-in param-mapping [mapping-key :target :dimension])]- ;; the source and target fields should be category_id and price, respectively- ;; for an explanation of why both cases are allowed, see `case field-nm` below- (is (contains? #{"category_id" "CATEGORY_ID"} (:name f1)))- (is (contains? #{"price" "PRICE"} (:name f2)))- (is (= {:dimension [:field (u/the-id f2) {:source-field (u/the-id f1)}]} dimension)))))--(defmethod assert-loaded-entity :model/Dashboard- [dashboard _]- (testing "The dashboard card series were loaded correctly"- (when (= "My Dashboard" (:name dashboard))- (doseq [dashcard (t2/select :model/DashboardCard :dashboard_id (u/the-id dashboard))]- (doseq [series (t2/select :model/DashboardCardSeries :dashboardcard_id (u/the-id dashcard))]- ;; check that the linked :card_id matches the expected name for each in the series- ;; based on the entities declared in test_util.clj- (let [series-pos (:position series)- expected-name (case (int series-pos)- 0 "My Card"- 1 "My Nested Card"- 2 ts/root-card-name)]- (is (= expected-name (t2/select-one-fn :name :model/Card :id (:card_id series))))- (case (int series-pos)- 1- (testing "Top level click action was preserved for dashboard card"- (let [viz-settings (:visualization_settings dashcard)- cb (:click_behavior viz-settings)]- (is (not-empty cb))- (is (int? (:targetId cb)))- (is (= "My Nested Query Card" (t2/select-one-fn :name :model/Card :id (:targetId cb))))))- 2- (testing "Column level click actions were preserved for dashboard card"- (let [viz-settings (:visualization_settings dashcard)- check-click-fn (fn [[col-key col-value]]- (let [col-ref (mb.viz/parse-db-column-ref col-key)- {:keys [::mb.viz/field-id ::mb.viz/column-name]} col-ref- click-bhv (get col-value :click_behavior)- target-id (get click-bhv :targetId)]- (cond- field-id (let [field-nm (id->name :model/Field field-id)]- (case field-nm- ;; consider either case for these fields, since they are all- ;; caps in some (ex: H2) and lowercase in others (ex: Postgres)- ;; this is simpler than the alternative (dynamically loading- ;; the expected name based on driver)- ("price" "PRICE") (assert-price-click click-bhv target-id)- ("name" "NAME") (is (=- "Root Dashboard"- (id->name :model/Dashboard target-id)))- ("latitude" "LATITUDE") (is (= {:parameterMapping {}- :type "crossfilter"}- click-bhv))))- column-name- (case column-name- "Price Known" (is (= {:type "link"- :linkType "url"- :linkTemplate "/price-info"} click-bhv))))))]- (is (not-empty viz-settings))- (doall (map check-click-fn (:column_settings viz-settings)))))- true))) ; nothing to assert for other series numberst- (when-let [virt-card (get-in dashcard [:visualization_settings :virtual_card])]- (is (= ts/virtual-card virt-card))- (is (= "Textbox Card" (get-in dashcard [:visualization_settings :text])))))))- dashboard)--(defmethod assert-loaded-entity :model/Pulse- [pulse _]- (is (some? pulse))- (let [pulse-cards (t2/select :model/PulseCard :pulse_id (u/the-id pulse))]- (is (= 2 (count pulse-cards)))- (is (= #{ts/root-card-name "My Card"}- (into #{} (map (partial t2/select-one-fn :name :model/Card :id) (map :card_id pulse-cards)))))))--(defmethod assert-loaded-entity :default- [entity _]- entity)--;; If this test fails after adding a new column, add the column to the list of columns in `metabase-enterprise.serialization.serialize/strip-crud`-(deftest dump-load-entities-test- (try- ;; in case it already exists- (u/ignore-exceptions- (delete-directory! dump-dir))- ;; TODO: Examine whether the test could work without :metadata/key-constraints.- (mt/test-drivers (-> (mt/normal-drivers-with-feature :basic-aggregations :binning :expressions- :metadata/key-constraints)- ;; We will run this roundtrip test against any database supporting these features ^ except- ;; certain ones for specific reasons, outlined below.- ;;- ;; Being able to support all of these would require some significant amount of effort (either- ;; to come up with "generic" native queries that will work on all of them, or else possibly- ;; using Metabase itself to spit out the native queries, but even then we would need to splice- ;; different strings together (ex: the SELECT clause, but replace the WHERE clause with the- ;; subquery from the linked card parameter).- ;;- ;; Because this feature is just about testing serialization itself, and not yet another test of- ;; query processor, this seems like an acceptable tradeoff (ex: if they dump and load to the- ;; same native form on one database, then it's likely they would on any, since that is- ;; orthogonal to the issues that serialization has when performing this roundtrip).- (disj :oracle ; no bare table names allowed- :databricks ; table name requires schema prefix with current implementation- :redshift ; bare table name doesn't work; it's test_data_venues instead of venues- :snowflake ; bare table name doesn't work; it's test_data_venues instead of venues- :sqlserver ; ORDER BY not allowed not allowed in derived tables (subselects)- :vertica)) ; bare table name doesn't work; it's test_data_venues instead of venues-- (mt/with-premium-features #{:serialization}- (let [fingerprint (ts/with-world- (v1-dump! dump-dir {:user (:email (test.users/fetch-user :crowberto))- :only-db-ids #{db-id}})- {:query-results (gather-orig-results [card-id- card-arch-id- card-id-root- card-id-nested- card-id-nested-query- card-id-native-query- card-id-root-to-collection- card-id-collection-to-root- card-id-template-tags- card-id-filter-agg- card-id-temporal-unit- card-id-with-native-snippet- card-id-temporal-unit- card-join-card-id- card-id-pivot-table])- :collections (gather-collections [card-id- card-arch-id- card-id-root- card-id-nested- card-id-nested-query- card-id-native-query- card-id-root-to-collection- card-id-collection-to-root- card-id-template-tags- card-id-filter-agg- card-id-temporal-unit- card-id-with-native-snippet- card-id-temporal-unit- card-join-card-id- card-id-pivot-table])- :entities [[:model/Database (t2/select-one :model/Database :id db-id)]- [:model/Table (t2/select-one :model/Table :id table-id)]- [:model/Table (t2/select-one :model/Table :id table-id-categories)]- [:model/Table (t2/select-one :model/Table :id table-id-users)]- [:model/Table (t2/select-one :model/Table :id table-id-checkins)]- [:model/Field (t2/select-one :model/Field :id numeric-field-id)]- [:model/Field (t2/select-one :model/Field :id name-field-id)]- [:model/Field (t2/select-one :model/Field :id category-field-id)]- [:model/Field (t2/select-one :model/Field :id latitude-field-id)]- [:model/Field (t2/select-one :model/Field :id longitude-field-id)]- [:model/Field (t2/select-one :model/Field :id category-pk-field-id)]- [:model/Field (t2/select-one :model/Field :id date-field-id)]- [:model/Field (t2/select-one :model/Field :id user-id-field-id)]- [:model/Field (t2/select-one :model/Field :id users-pk-field-id)]- [:model/Field (t2/select-one :model/Field :id last-login-field-id)]- [:model/Collection (t2/select-one :model/Collection :id collection-id)]- [:model/Collection (t2/select-one :model/Collection :id collection-id-nested)]- [:model/Collection (t2/select-one :model/Collection :id personal-collection-id)]- [:model/Collection (t2/select-one :model/Collection :id pc-nested-id)]- [:model/Collection (t2/select-one :model/Collection :id pc-deeply-nested-id)]- [:model/Segment (t2/select-one :model/Segment :id segment-id)]- [:model/Dashboard (t2/select-one :model/Dashboard :id dashboard-id)]- [:model/Dashboard (t2/select-one :model/Dashboard :id root-dashboard-id)]- [:model/Card (t2/select-one :model/Card :id card-id)]- [:model/Card (t2/select-one :model/Card :id card-arch-id)]- [:model/Card (t2/select-one :model/Card :id card-id-root)]- [:model/Card (t2/select-one :model/Card :id card-id-nested)]- [:model/Card (t2/select-one :model/Card :id card-id-nested-query)]- [:model/Card (t2/select-one :model/Card :id card-id-native-query)]- [:model/DashboardCard (t2/select-one :model/DashboardCard :id dashcard-id)]- [:model/DashboardCard (t2/select-one :model/DashboardCard :id dashcard-top-level-click-id)]- [:model/DashboardCard (t2/select-one :model/DashboardCard :id dashcard-with-click-actions)]- [:model/Card (t2/select-one :model/Card :id card-id-root-to-collection)]- [:model/Card (t2/select-one :model/Card :id card-id-collection-to-root)]- [:model/Card (t2/select-one :model/Card :id card-id-template-tags)]- [:model/Card (t2/select-one :model/Card :id card-id-filter-agg)]- [:model/Card (t2/select-one :model/Card :id card-id-temporal-unit)]- [:model/Pulse (t2/select-one :model/Pulse :id pulse-id)]- [:model/DashboardCard (t2/select-one :model/DashboardCard :id dashcard-with-textbox-id)]- [:model/NativeQuerySnippet (t2/select-one :model/NativeQuerySnippet :id snippet-id)]- [:model/Collection (t2/select-one :model/Collection :id snippet-collection-id)]- [:model/Collection (t2/select-one :model/Collection :id snippet-nested-collection-id)]- [:model/NativeQuerySnippet (t2/select-one :model/NativeQuerySnippet :id nested-snippet-id)]- [:model/Card (t2/select-one :model/Card :id card-id-with-native-snippet)]- [:model/Card (t2/select-one :model/Card :id card-join-card-id)]- [:model/Card (t2/select-one :model/Card :id card-id-pivot-table)]]})]- (with-world-cleanup- (v1-load! dump-dir {:on-error :continue :mode :skip})- (mt/with-db (t2/select-one :model/Database :name ts/temp-db-name)- (doseq [[model entity] (:entities fingerprint)]- (testing (format "%s \"%s\"" (type model) (:name entity))- (is (or (-> entity :name nil?)- (when-let [loaded (t2/select-one model :name (:name entity))]- (assert-loaded-entity loaded fingerprint))- (and (-> entity :archived) ; archived card hasn't been dump-loaded- (= (:name entity) "My Arch Card"))- ;; Rasta's Personal Collection was not loaded- (= "Felicia's Personal Collection" (:name entity)))- (str " failed " (pr-str entity)))))- fingerprint)))))- (finally- (delete-directory! dump-dir))))--(deftest resolve-dashboard-parameters-test- (let [parameters [{:values_source_config {:card_id "foo"}}]]- (with-redefs [load/fully-qualified-name->card-id {"foo" 1}]- (is (= [1] (mapv (comp :card_id :values_source_config) (#'load/resolve-dashboard-parameters parameters)))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/names_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/names_test.cljdeleted file mode 100644index 0874727cdda..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/names_test.clj+++ /dev/null@@ -1,105 +0,0 @@-(ns metabase-enterprise.serialization.names-test- (:require- [clojure.test :refer :all]- [metabase-enterprise.serialization.names :as names]- [metabase-enterprise.serialization.test-util :as ts]- [metabase.test :as mt]- [metabase.util :as u]- [toucan2.core :as t2]))--(deftest safe-name-test- (are [s expected] (= expected (names/safe-name {:name s}))- "foo" "foo"- "foo/bar baz" "foo%2Fbar baz"))--(deftest unescape-name-test- (are [s expected] (= expected- (names/unescape-name s))- "foo" "foo"- "foo%2Fbar baz" "foo/bar baz"))--(deftest safe-name-unescape-name-test- (is (= "foo/bar baz"- (-> {:name "foo/bar baz"} names/safe-name names/unescape-name))))--(deftest roundtrip-test- (ts/with-world- (doseq [object [(t2/select-one :model/Card :id card-id-root)- (t2/select-one :model/Card :id card-id)- (t2/select-one :model/Card :id card-id-nested)- (t2/select-one :model/Table :id table-id)- (t2/select-one :model/Field :id category-field-id)- (t2/select-one :model/Segment :id segment-id)- (t2/select-one :model/Collection :id collection-id)- (t2/select-one :model/Collection :id collection-id-nested)- (t2/select-one :model/Dashboard :id dashboard-id)- (t2/select-one :model/Database :id db-id)- (t2/select-one :model/NativeQuerySnippet :id snippet-id)]]- (testing (class object)- (let [context (names/fully-qualified-name->context (names/fully-qualified-name object))- id-fn (some-fn :snippet :field :segment :card :dashboard :collection :table :database)]- (is (= (u/the-id object)- (id-fn context))))))))--(deftest fully-qualified-name->context-test- (testing "fully-qualified-name->context works as expected"- (testing " with cards in root and in a collection"- (mt/with-temp [:model/Collection {collection-id :id :as coll} {:name "A Collection" :location "/"}- :model/Card root-card {:name "Root Card"}- :model/Card collection-card {:name "Collection Card"- :collection_id collection-id}- :model/Collection {sub-collection-id :id :as coll2} {:name "Sub Collection"- :location (format "/%d/" collection-id)}- :model/Collection coll3 {:name "Deep Collection"- :location (format "/%d/%d/"- collection-id- sub-collection-id)}]- (let [card1-name "/collections/root/cards/Root Card"- card2-name "/collections/root/collections/A Collection/cards/Collection Card"- coll-name "/collections/root/collections/A Collection"- coll2-name "/collections/root/collections/A Collection/collections/Sub Collection"- coll3-name (str "/collections/root/collections/A Collection/collections/Sub Collection/collections"- "/Deep Collection")]- (is (= card1-name (names/fully-qualified-name root-card)))- (is (= card2-name (names/fully-qualified-name collection-card)))- (is (= coll-name (names/fully-qualified-name coll)))- (is (= coll2-name (names/fully-qualified-name coll2)))- (is (= coll3-name (names/fully-qualified-name coll3))))))- (testing " with snippets in a collection"- (mt/with-temp [:model/Collection {base-collection-id :id} {:name "Base Collection"- :namespace "snippets"}- :model/Collection {collection-id :id} {:name "Nested Collection"- :location (format "/%s/" base-collection-id)- :namespace "snippets"}- :model/NativeQuerySnippet snippet {:content "price > 2"- :name "Price > 2"- :description "Price more than 2"- :collection_id collection-id}]- (let [fully-qualified-name (str "/collections/root/collections/:snippets/Base Collection/collections"- "/:snippets/Nested Collection/snippets/Price %3E 2")]- (is (= fully-qualified-name- (names/fully-qualified-name snippet)))- (is (= {:collection collection-id- :snippet (u/the-id snippet)}- (names/fully-qualified-name->context fully-qualified-name))))))- (testing " with path elements matching one of our entity names"- ; these drivers keep table names lowercased, causing "users" table to clash with our entity name "users"- (mt/test-drivers #{:postgres :mysql}- (ts/with-world- (let [users-pk-field (t2/select-one :model/Field :id users-pk-field-id)- fq-name (names/fully-qualified-name users-pk-field)- ctx (names/fully-qualified-name->context fq-name)]- ;; MySQL doesn't have schemas, so either one of these could be acceptable- (is (contains? #{"/databases/Fingerprint test-data copy/tables/users/fields/id"- "/databases/Fingerprint test-data copy/schemas/public/tables/users/fields/id"} fq-name))- (is (map? ctx))- (is (some? (:table ctx)))))))))--(deftest name-for-logging-test- (testing "serialization logging name generation from Toucan 2 records (#29322)"- (mt/with-temp [:model/Collection {collection-id :id} {:name "A Collection"}- :model/Card {card-id :id} {:name "A Card"- :collection_id collection-id}]- (are [model s id] (= (format s id) (names/name-for-logging (t2/select-one model :id id)))- 'Collection ":model/Collection \"A Collection\" (ID %d)" collection-id- 'Card ":model/Card \"A Card\" (ID %d)" card-id))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/serialize_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/serialize_test.cljdeleted file mode 100644index a75a4092c8d..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/serialize_test.clj+++ /dev/null@@ -1,41 +0,0 @@-(ns metabase-enterprise.serialization.serialize-test- (:require- [clojure.string :as str]- [clojure.test :refer :all]- [metabase-enterprise.serialization.serialize :as serialize]- [metabase-enterprise.serialization.test-util :as ts]))--(defn- all-ids-are-fully-qualified-names?- [m]- (testing (format "\nm = %s" (pr-str m))- (doseq [[k v] m- :when (and v (-> k name (str/ends-with? "_id")))]- (testing (format "\nk = %s" (pr-str k))- (is (string? v))))))--(defn- all-mbql-ids-are-fully-qualified-names?- [[_ & ids]]- (testing (format "\nids = %s" (pr-str ids))- (doseq [id ids]- (cond (map? id) (all-ids-are-fully-qualified-names? id)- (some? id) (is (string? id))))))--(deftest serialization-test- (ts/with-world- (doseq [[model id] [[:model/Card card-id]- [:model/Segment segment-id]- [:model/Collection collection-id]- [:model/Dashboard dashboard-id]- [:model/Table table-id]- [:model/Field numeric-field-id]- [:model/Database db-id]- [:model/NativeQuerySnippet snippet-id]]]- (testing (name model)- (let [serialization (serialize/serialize (model id))]- (testing (format "\nserialization = %s" (pr-str serialization))- (doseq [x (->> serialization- (tree-seq coll? identity)- (filter (some-fn map? #'serialize/mbql-entity-reference?)))]- (if (map? x)- (all-ids-are-fully-qualified-names? x)- (all-mbql-ids-are-fully-qualified-names? x)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj b/enterprise/backend/test/metabase_enterprise/serialization/test_util.cljindex f991a4410c8..2c1cd414ecf 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj@@ -2,7 +2,6 @@(:require[clojure.java.io :as io][clojure.test :refer :all]- [metabase-enterprise.serialization.names :as names][metabase.app-db.connection :as mdb.connection][metabase.app-db.core :as mdb][metabase.app-db.schema-migrations-test.impl :as schema-migrations-test.impl]@@ -72,7 +71,7 @@(defn- do-with-in-memory-h2-db [f](schema-migrations-test.impl/do-with-temp-empty-app-db*:h2- (fn [data-source]+ (fn [^javax.sql.DataSource data-source];; DB should stay open as long as `conn` is held open.(with-open [_conn (.getConnection data-source)](next.jdbc/execute! data-source ["RUNSCRIPT FROM ?" (str @data/h2-app-db-script)])@@ -526,9 +525,6 @@venues-pk-field-id]}]~@body)))-;; Don't memoize as IDs change in each `with-world` context-(alter-var-root #'names/path->context (fn [_] #'names/path->context*))-(defn extract-one [model-name where](let [where (cond(nil? where) truediff --git a/enterprise/backend/test/metabase_enterprise/serialization/upsert_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/upsert_test.cljdeleted file mode 100644index aaa0daa27b7..00000000000--- a/enterprise/backend/test/metabase_enterprise/serialization/upsert_test.clj+++ /dev/null@@ -1,123 +0,0 @@-(ns metabase-enterprise.serialization.upsert-test- (:require- [clojure.data :as data]- [clojure.test :refer :all]- [metabase-enterprise.serialization.upsert :as upsert]- [metabase.models.interface :as mi]- [metabase.test :as mt]- [metabase.util :as u]- [toucan2.core :as t2]))--(def ^:private same? (comp nil? second data/diff))--(defn- mutate- [model e]- (let [identity-constituent? (set (#'upsert/identity-condition model))]- (into {} (for [[k v] e]- [k (if (or (not (string? v))- (identity-constituent? k))- v- (str (gensym)))]))))--(def ^:private cards- (delay [{:name "My Card 1"- :table_id (mt/id :venues)- :display :line- :creator_id (mt/user->id :rasta)- :visualization_settings {}- :dataset_query {:query {:source-table (mt/id :venues)}- :type :query- :database (mt/id)}}- {:name "My Card 2"- :table_id (mt/id :venues)- :display :line- :creator_id (mt/user->id :rasta)- :visualization_settings {}- :dataset_query {:query {:source-table (mt/id :venues)}- :type :query- :database (mt/id)}}]))--;; TODO -- I'm not really clear on what these are testing, so I wasn't sure what to name them when I converted them to-;; the new style. Feel free to give them better names - Cam-(deftest maybe-upsert-many!-skip-test- (mt/with-model-cleanup [:model/Card]- (let [existing-ids (sort (t2/insert-returning-pks! :model/Card @cards))- inserted-ids (sort (vec (upsert/maybe-upsert-many! {:mode :skip} :model/Card @cards)))]- (is (= existing-ids inserted-ids)))))--(deftest maybe-upsert-many!-same-objects-test- (mt/with-model-cleanup [:model/Card]- (letfn [(test-mode [mode]- (testing (format "Mode = %s" mode)- (let [[e1 e2] @cards- [id1 id2] (upsert/maybe-upsert-many! {:mode mode} :model/Card @cards)]- (is (every? (partial apply same?)- [[(t2/select-one :model/Card :id id1) e1] [(t2/select-one :model/Card :id id2) e2]])))))]- (doseq [mode [:skip :update]]- (test-mode mode)))))--(deftest maybe-upsert-many!-update-test- (mt/with-model-cleanup [:model/Card]- (let [[e1 e2] @cards- id1 (first (t2/insert-returning-pks! :model/Card e1))- e1-mutated (mutate :model/Card e1)- [id1-mutated id2] (upsert/maybe-upsert-many! {:mode :update} :model/Card [e1-mutated e2])]- (testing "Card 1 ID"- (is (= id1 id1-mutated)))- (testing "Card 1"- (is (same? (t2/select-one :model/Card :id id1-mutated) e1-mutated)))- (testing "Card 2"- (is (same? (t2/select-one :model/Card :id id2) e2))))))--(defn- dummy-entity [dummy-dashboard model entity instance-num]- (cond- (isa? model :model/DashboardCard)- ;; hack to make sure that :visualization_settings are slightly different between the two dummy instances- ;; this is necessary because DashboardCards have that as part of their identity-condition- (assoc entity :dashboard_id (u/the-id dummy-dashboard)- :visualization_settings (if (= 1 instance-num) {:column_settings {}}- {:click_behavior {}}))-- :else- entity))--(defn- test-select-identical [model]- (testing (name model)- (let [id-cond (#'upsert/identity-condition model)- [e1 e2] (if (contains? (set id-cond) :name)- [{:name "a"} {:name "b"}]- [{} {}])]- (mt/with-temp [:model/Dashboard dashboard {:name "Dummy Dashboard"}- ;; create an additional entity so we're sure whe get the right one- model _ (dummy-entity dashboard model e1 1)- model {id :id} (dummy-entity dashboard model e2 2)]- (let [e (t2/select-one model (first (t2/primary-keys model)) id)]- ;; make sure that all columns in identity-condition actually exist in the model- (is (= (set id-cond) (-> e- (select-keys id-cond)- keys- set)))- (is (= (#'upsert/select-identical model (cond-> e- ;; engine is a keyword but has to be a string for- ;; HoneySQL to not interpret it as a col name- (mi/instance-of? :model/Database e) (update :engine name)))- e)))))))--(deftest identical-test- (doseq [model [:model/Collection- :model/Card- :model/Table- :model/Field- :model/NativeQuerySnippet- :model/Segment- :model/Dashboard- :model/DashboardCard- :model/Database- :model/Pulse- :model/User]]- (testing model- (test-select-identical model))))--(deftest has-post-insert?-test- (is (true?- (#'upsert/has-post-insert? :model/User))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.cljindex 300d1bc47c5..ef1abf82c15 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/e2e_test.clj@@ -421,7 +421,7 @@:value_field [:field["my-db" nil "CUSTOMERS" (:name field1s)]nil]},- :values_source_type "card"}]+ :values_source_type :card}](:parameters (first (by-model extraction "Dashboard")))));; card1s has no parameters, card2s does.@@ -433,7 +433,7 @@:value_field [:field["my-db" nil "CUSTOMERS" (:name field1s)]nil]},- :values_source_type "card"}]}+ :values_source_type :card}]}(set (map :parameters (by-model extraction "Card")))))(storage/store! (seq extraction) dump-dir)))@@ -456,7 +456,7 @@:parametersfirst:values_source_config)))- (is (some? (t2/select-one 'ParameterCard :parameterized_object_type "dashboard" :parameterized_object_id (:id dash1d)))))+ (is (some? (t2/select-one :model/ParameterCard :parameterized_object_type "dashboard" :parameterized_object_id (:id dash1d)))))(testing "parameter on card is loaded correctly"(is (= {:card_id (:id card1d),@@ -465,7 +465,7 @@:parametersfirst:values_source_config)))- (is (some? (t2/select-one 'ParameterCard :parameterized_object_type "card" :parameterized_object_id (:id card2d)))))))))))))))+ (is (some? (t2/select-one :model/ParameterCard :parameterized_object_type "card" :parameterized_object_id (:id card2d)))))))))))))))(deftest dashcards-with-link-cards-test(ts/with-random-dump-dir [dump-dir "serdesv2-"]@@ -932,15 +932,14 @@(is (int? new-coll-id))(is (=? {:name "Metric Card":collection_id new-coll-id- :dataset_query (mt/mbql-query orders- {:aggregation [[:count]]- :breakout [$product_id->products.category $created_at]})}+ :dataset_query {:stages [{:aggregation [[:count {}]]+ :breakout [[:field {} (mt/id :products :category)]+ [:field {} (mt/id :orders :created_at)]]}]}}new-metric))(is (=? {:name "Metric Consuming Question Card":collection_id new-coll-id- :dataset_query (mt/mbql-query orders- {:aggregation [[:metric (:id new-metric)]]- :breakout [[:field %orders.user_id nil]]})}+ :dataset_query {:stages [{:aggregation [[:metric {} (:id new-metric)]]+ :breakout [[:field {} (mt/id :orders :user_id)]]}]}}(t2/select-one :model/Card :name "Metric Consuming Question Card"))))))))))))(deftest schema-coercion-testdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex 54497f8d7a5..61f71a07239 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -106,8 +106,8 @@(deftest dashboard-and-cards-test(mt/with-empty-h2-app-db!(ts/with-temp-dpc [:model/Collection- {coll-id :id- coll-eid :entity_id}+ {coll-id :id+ coll-eid :entity_id}{:name "Some Collection"}:model/User@@ -124,17 +124,17 @@:model/Collection{mark-coll-eid :entity_id}- {:name "MK Personal"+ {:name "MK Personal":personal_owner_id mark-id}:model/Collection{dave-coll-id :iddave-coll-eid :entity_id}- {:name "DK Personal"+ {:name "DK Personal":personal_owner_id dave-id}:model/Database- {db-id :id}+ {db-id :id}{:name "My Database"}:model/Table@@ -142,17 +142,17 @@{:name "Schemaless Table" :db_id db-id}:model/Field- {field-id :id}+ {field-id :id}{:name "Some Field" :table_id no-schema-id}:model/Table- {schema-id :id}- {:name "Schema'd Table"- :db_id db-id- :schema "PUBLIC"}+ {schema-id :id}+ {:name "Schema'd Table"+ :db_id db-id+ :schema "PUBLIC"}:model/Field- {field2-id :id}+ {field2-id :id}{:name "Other Field" :table_id schema-id}:model/Card@@ -163,24 +163,24 @@:table_id no-schema-id:collection_id coll-id:creator_id mark-id- :dataset_query {:query {:source-table no-schema-id- :filter [:>= [:field field-id nil] 18]- :aggregation [[:count]]}- :type :query+ :dataset_query {:query {:source-table no-schema-id+ :filter [:>= [:field field-id nil] 18]+ :aggregation [[:count]]}+ :type :query:database db-id}}:model/Card- {model-id :id}+ {model-id :id}{:name "Some Model":database_id db-id:table_id no-schema-id:collection_id coll-id:creator_id mark-id:type :model- :dataset_query {:query {:source-table no-schema-id- :filter [:>= [:field field-id nil] 18]- :aggregation [[:count]]}- :type :query+ :dataset_query {:query {:source-table no-schema-id+ :filter [:>= [:field field-id nil] 18]+ :aggregation [[:count]]}+ :type :query:database db-id}}:model/Card@@ -194,8 +194,8 @@:parameter_mappings[{:parameter_id "deadbeef":card_id c1-id- :target [:dimension [:field field-id- {:source-field field2-id}]]}]}+ :target [:dimension [:field field-id+ {:source-field field2-id}]]}]}:model/Card{c3-id :id@@ -207,23 +207,23 @@:creator_id mark-id:visualization_settings{:table.pivot_column "SOURCE"- :table.cell_column "sum"+ :table.cell_column "sum":table.columns- [{:name "SOME_FIELD"+ [{:name "SOME_FIELD":fieldRef [:field field-id nil]- :enabled true}- {:name "OTHER_FIELD"+ :enabled true}+ {:name "OTHER_FIELD":fieldRef [:field field2-id nil]- :enabled true}- {:name "sum"+ :enabled true}+ {:name "sum":fieldRef [:field "sum" {:base-type :type/Float}]- :enabled true}- {:name "count"+ :enabled true}+ {:name "count":fieldRef [:field "count" {:base-type :type/BigInteger}]- :enabled true}- {:name "Average order total"+ :enabled true}+ {:name "Average order total":fieldRef [:field "Average order total" {:base-type :type/Float}]- :enabled true}]+ :enabled true}]:column_settings{(str "[\"ref\",[\"field\"," field2-id ",null]]") {:column_title "Locus"}}}}@@ -234,10 +234,10 @@:collection_id coll-id:creator_id mark-id:dataset_query- {:query {:source-table no-schema-id- :filter [:>= [:field field-id nil] 18]}+ {:query {:source-table no-schema-id+ :filter [:>= [:field field-id nil] 18]}:database db-id- :type :query}}+ :type :query}}:model/Card{c5-id :idc5-eid :entity_id}@@ -247,16 +247,16 @@:collection_id coll-id:creator_id mark-id:dataset_query- {:query {:source-table (str "card__" c4-id)- :aggregation [[:count]]}- :type :query+ {:query {:source-table (str "card__" c4-id)+ :aggregation [[:count]]}+ :type :query:database db-id}}:model/Action- {action-id :id- action-eid :entity_id}- {:name "Some action"- :type :query+ {action-id :id+ action-eid :entity_id}+ {:name "Some action"+ :type :query:model_id model-id}:model/Dashboard@@ -284,21 +284,21 @@:parameters [{:id "abc":type "category":name "CATEGORY"- :values_source_type "card"+ :values_source_type :card;; card_id is in a different collection with dashboard's collection:values_source_config {:card_id c1-id:value_field [:field field-id nil]}}]}:model/DashboardCard_- {:card_id c1-id- :dashboard_id dash-id+ {:card_id c1-id+ :dashboard_id dash-id:inline_parameters ["12345678"]:parameter_mappings[{:parameter_id "12345678":card_id c1-id- :target [:dimension [:field field-id- {:source-field field2-id}]]}]}+ :target [:dimension [:field field-id+ {:source-field field2-id}]]}]}:model/DashboardCard_@@ -306,49 +306,49 @@:dashboard_id other-dash-id:visualization_settings{:table.pivot_column "SOURCE"- :table.cell_column "sum"+ :table.cell_column "sum":table.columns- [{:name "SOME_FIELD"+ [{:name "SOME_FIELD":fieldRef [:field field-id nil]- :enabled true}- {:name "sum"+ :enabled true}+ {:name "sum":fieldRef [:field "sum" {:base-type :type/Float}]- :enabled true}- {:name "count"+ :enabled true}+ {:name "count":fieldRef [:field "count" {:base-type :type/BigInteger}]- :enabled true}- {:name "Average order total"+ :enabled true}+ {:name "Average order total":fieldRef [:field "Average order total" {:base-type :type/Float}]- :enabled true}]+ :enabled true}]:column_settings{(str "[\"ref\",[\"field\"," field2-id ",null]]") {:column_title "Locus"}}}}:model/DashboardCard_- {:action_id action-id+ {:action_id action-id:dashboard_id other-dash-id}](testing "table and database are extracted as [db schema table] triples"(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c1-id))]- (is (=? {:serdes/meta [{:model "Card" :id c1-eid :label "some_question"}]- :table_id ["My Database" nil "Schemaless Table"]- :creator_id "[email protected]"- :collection_id coll-eid- :dataset_query {:query {:source-table ["My Database" nil "Schemaless Table"]- :filter [:>= [:field ["My Database" nil "Schemaless Table" "Some Field"] nil] 18]- :aggregation [[:count]]}- :database "My Database"}- :created_at string?}+ (is (=? {:serdes/meta [{:model "Card" :id c1-eid :label "some_question"}]+ :table_id ["My Database" nil "Schemaless Table"]+ :creator_id "[email protected]"+ :collection_id coll-eid+ :dataset_query {:query {:source-table ["My Database" nil "Schemaless Table"]+ :filter [:>= [:field ["My Database" nil "Schemaless Table" "Some Field"] {}] 18]+ :aggregation [[:count]]}+ :database "My Database"}+ :created_at string?}ser))(is (not (contains? ser :id)))(testing "cards depend on their Table and Collection, and also anything referenced in the query"- (is (= #{[{:model "Database" :id "My Database"}]- [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}]- [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}- {:model "Field" :id "Some Field"}]+ (is (= #{[{:model "Database" :id "My Database"}]+ [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}]+ [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}+ {:model "Field" :id "Some Field"}][{:model "Collection" :id coll-eid}]}(set (serdes/dependencies ser))))))@@ -360,53 +360,53 @@:dataset_query {}:parameter_mappings [{:parameter_id "deadbeef":card_id c1-eid- :target [:dimension [:field ["My Database" nil "Schemaless Table" "Some Field"]- {:source-field ["My Database" "PUBLIC" "Schema'd Table" "Other Field"]}]]}]+ :target [:dimension [:field ["My Database" nil "Schemaless Table" "Some Field"]+ {:source-field ["My Database" "PUBLIC" "Schema'd Table" "Other Field"]}]]}]:created_at string?}ser))(is (not (contains? ser :id)))(testing "cards depend on their Database, Table and Collection, and any fields in their parameter_mappings"- (is (= #{[{:model "Database" :id "My Database"}]- [{:model "Database" :id "My Database"}- {:model "Schema" :id "PUBLIC"}- {:model "Table" :id "Schema'd Table"}]+ (is (= #{[{:model "Database" :id "My Database"}]+ [{:model "Database" :id "My Database"}+ {:model "Schema" :id "PUBLIC"}+ {:model "Table" :id "Schema'd Table"}][{:model "Collection" :id coll-eid}]- [{:model "Card" :id c1-eid}]- [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}- {:model "Field" :id "Some Field"}]- [{:model "Database" :id "My Database"}- {:model "Schema" :id "PUBLIC"}- {:model "Table" :id "Schema'd Table"}- {:model "Field" :id "Other Field"}]}+ [{:model "Card" :id c1-eid}]+ [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}+ {:model "Field" :id "Some Field"}]+ [{:model "Database" :id "My Database"}+ {:model "Schema" :id "PUBLIC"}+ {:model "Table" :id "Schema'd Table"}+ {:model "Field" :id "Other Field"}]}(set (serdes/dependencies ser))))))(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c3-id))]- (is (=? {:serdes/meta [{:model "Card" :id c3-eid :label "third_question"}]- :table_id ["My Database" "PUBLIC" "Schema'd Table"]- :creator_id "[email protected]"- :collection_id coll-eid- :dataset_query {}+ (is (=? {:serdes/meta [{:model "Card" :id c3-eid :label "third_question"}]+ :table_id ["My Database" "PUBLIC" "Schema'd Table"]+ :creator_id "[email protected]"+ :collection_id coll-eid+ :dataset_query {}:visualization_settings{:table.pivot_column "SOURCE"- :table.cell_column "sum"+ :table.cell_column "sum":table.columns- [{:name "SOME_FIELD"+ [{:name "SOME_FIELD":fieldRef [:field ["My Database" nil "Schemaless Table" "Some Field"] nil]- :enabled true}- {:name "OTHER_FIELD"+ :enabled true}+ {:name "OTHER_FIELD":fieldRef [:field ["My Database" "PUBLIC" "Schema'd Table" "Other Field"] nil]- :enabled true}- {:name "sum"+ :enabled true}+ {:name "sum":fieldRef [:field "sum" {:base-type :type/Float}]- :enabled true}- {:name "count"+ :enabled true}+ {:name "count":fieldRef [:field "count" {:base-type :type/BigInteger}]- :enabled true}- {:name "Average order total"+ :enabled true}+ {:name "Average order total":fieldRef [:field "Average order total" {:base-type :type/Float}]- :enabled true}]+ :enabled true}]:column_settings{"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {:column_title "Locus"}}}:created_at string?}@@ -414,79 +414,79 @@(is (not (contains? ser :id)))(testing "cards depend on their Database, Table and Collection, and any fields in their visualization_settings"- (is (= #{[{:model "Database" :id "My Database"}]- [{:model "Database" :id "My Database"}- {:model "Schema" :id "PUBLIC"}- {:model "Table" :id "Schema'd Table"}]+ (is (= #{[{:model "Database" :id "My Database"}]+ [{:model "Database" :id "My Database"}+ {:model "Schema" :id "PUBLIC"}+ {:model "Table" :id "Schema'd Table"}][{:model "Collection" :id coll-eid}]- [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}- {:model "Field" :id "Some Field"}]- [{:model "Database" :id "My Database"}- {:model "Schema" :id "PUBLIC"}- {:model "Table" :id "Schema'd Table"}- {:model "Field" :id "Other Field"}]}+ [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}+ {:model "Field" :id "Some Field"}]+ [{:model "Database" :id "My Database"}+ {:model "Schema" :id "PUBLIC"}+ {:model "Table" :id "Schema'd Table"}+ {:model "Field" :id "Other Field"}]}(set (serdes/dependencies ser)))))))(testing "Cards can be based on other cards"(let [ser (serdes/extract-one "Card" {} (t2/select-one :model/Card :id c5-id))]- (is (=? {:serdes/meta [{:model "Card" :id c5-eid :label "dependent_question"}]- :table_id ["My Database" nil "Schemaless Table"]- :creator_id "[email protected]"- :collection_id coll-eid- :dataset_query {:query {:source-table c4-eid- :aggregation [[:count]]}- :database "My Database"}- :created_at string?}+ (is (=? {:serdes/meta [{:model "Card" :id c5-eid :label "dependent_question"}]+ :table_id ["My Database" nil "Schemaless Table"]+ :creator_id "[email protected]"+ :collection_id coll-eid+ :dataset_query {:query {:source-table c4-eid+ :aggregation [[:count]]}+ :database "My Database"}+ :created_at string?}ser))(is (not (contains? ser :id)))(testing "and depend on their Database, Table and Collection, and the upstream Card"- (is (= #{[{:model "Database" :id "My Database"}]- [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}]+ (is (= #{[{:model "Database" :id "My Database"}]+ [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}][{:model "Collection" :id coll-eid}]- [{:model "Card" :id c4-eid}]}+ [{:model "Card" :id c4-eid}]}(set (serdes/dependencies ser)))))))(testing "Dashboards include their Dashcards"(let [ser (ts/extract-one "Dashboard" other-dash-id)]- (is (=? {:serdes/meta [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]- :entity_id other-dash+ (is (=? {:serdes/meta [{:model "Dashboard" :id other-dash :label "dave_s_dash"}]+ :entity_id other-dash:dashcards[{:visualization_settings {:table.pivot_column "SOURCE"- :table.cell_column "sum"+ :table.cell_column "sum":table.columns- [{:name "SOME_FIELD"+ [{:name "SOME_FIELD":fieldRef [:field ["My Database" nil "Schemaless Table" "Some Field"] nil]- :enabled true}- {:name "sum"+ :enabled true}+ {:name "sum":fieldRef [:field "sum" {:base-type :type/Float}]- :enabled true}- {:name "count"+ :enabled true}+ {:name "count":fieldRef [:field "count" {:base-type :type/BigInteger}]- :enabled true}- {:name "Average order total"+ :enabled true}+ {:name "Average order total":fieldRef [:field "Average order total" {:base-type :type/Float}]- :enabled true}]+ :enabled true}]:column_settings{"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {:column_title "Locus"}}}:created_at string?}{:action_id action-eid}]- :created_at string?}+ :created_at string?}ser))(is (not (contains? ser :id)))(testing "and depend on all referenced cards and actions, including those in visualization_settings"- (is (= #{[{:model "Card" :id c2-eid}]- [{:model "Action" :id action-eid}]- [{:model "Database" :id "My Database"}- {:model "Table" :id "Schemaless Table"}- {:model "Field" :id "Some Field"}]- [{:model "Database" :id "My Database"}- {:model "Schema" :id "PUBLIC"}- {:model "Table" :id "Schema'd Table"}- {:model "Field" :id "Other Field"}]+ (is (= #{[{:model "Card" :id c2-eid}]+ [{:model "Action" :id action-eid}]+ [{:model "Database" :id "My Database"}+ {:model "Table" :id "Schemaless Table"}+ {:model "Field" :id "Some Field"}]+ [{:model "Database" :id "My Database"}+ {:model "Schema" :id "PUBLIC"}+ {:model "Table" :id "Schema'd Table"}+ {:model "Field" :id "Other Field"}][{:model "Collection" :id dave-coll-eid}]}(set (serdes/dependencies ser)))))))@@ -500,13 +500,13 @@:value_field [:field["My Database" nil "Schemaless Table" "Some Field"]nil]},- :values_source_type "card"}]}+ :values_source_type :card}]}ser))(is (= #{[{:model "Collection" :id dave-coll-eid}]- [{:model "Card" :id c1-eid}]+ [{:model "Card" :id c1-eid}][{:model "Database", :id "My Database"}- {:model "Table", :id "Schemaless Table"}- {:model "Field", :id "Some Field"}]}+ {:model "Table", :id "Schemaless Table"}+ {:model "Field", :id "Some Field"}]}(set (serdes/dependencies ser))))))(testing "Cards with parameters where the source is a card"@@ -519,13 +519,13 @@:value_field [:field["My Database" nil "Schemaless Table" "Some Field"]nil]},- :values_source_type "card"}]}+ :values_source_type :card}]}ser))(is (= #{[{:model "Collection" :id dave-coll-eid}]- [{:model "Card" :id c1-eid}]+ [{:model "Card" :id c1-eid}][{:model "Database", :id "My Database"}- {:model "Table", :id "Schemaless Table"}- {:model "Field", :id "Some Field"}]}+ {:model "Table", :id "Schemaless Table"}+ {:model "Field", :id "Some Field"}]}(set (serdes/dependencies ser))))))(testing "collection filtering based on :user option"@@ -706,10 +706,10 @@:template-tags {"snippet"{:display-name "Snippet 1",:id s1-eid,- :name "snippet 1",+ :name "snippet",:snippet-id s1-id,:snippet-name "snip",- :type "snippet"}}})}]+ :type :number}}})}](testing "native query snippets"(testing "can belong to :snippets collections"(let [ser (serdes/extract-one "NativeQuerySnippet" {} (t2/select-one :model/NativeQuerySnippet :id s1-id))]@@ -848,7 +848,6 @@:created_at string?}ser))(is (not (contains? ser :id)))-(testing "depend on the Table and any fields from the definition"(is (= #{[{:model "Database" :id "My Database"}{:model "Table" :id "Schemaless Table"}]@@ -887,10 +886,9 @@:type "implicit":created_at string?:model_id card-eid-1- :implicit [{:kind "row/update"}]}+ :implicit [{:kind :row/update}]}ser))(is (not (contains? ser :id)))-(testing "depends on the Model"(is (= #{[{:model "Card" :id card-eid-1}]}(set (serdes/dependencies ser)))))))))))))@@ -914,7 +912,7 @@{:keys [action-id]}{:name "My Action":type :http- :template {}+ :template {:method "GET", :url "https://camsaul.com"}:creator_id ann-id:model_id card-id-1}](let [action (action/select-action :id action-id)]@@ -928,7 +926,6 @@:http [{:template {}}]}ser))(is (not (contains? ser :id)))-(testing "depends on the Model"(is (= #{[{:model "Card" :id card-eid-1}]}(set (serdes/dependencies ser)))))))))))))@@ -1105,7 +1102,7 @@(is (= [{:id "abc",:type :category,:name "CATEGORY",- :values_source_type "card",+ :values_source_type :card:values_source_config {:card_id card-eid-1,:value_field [:field ["My Database" nil "Schemaless Table" "A Field"] nil]}}](:parameters ser)))@@ -2050,9 +2047,9 @@transform-eid :entity_id}{:name "Test Transform":description "A test transform for serialization"- :source {:database db-id- :type "query"- :query {:source-table table-id}}+ :source {:query {:database db-id+ :type "query"+ :query {:source-table table-id}}}:target {:database db-id:type "table":schema "public"@@ -2089,7 +2086,7 @@(is (not (contains? ser :id))))(testing "source and target MBQL export"- (is (=? {:source {:database "My Database" :type "query" :query {:source-table ["My Database" nil "Schemaless Table"]}}+ (is (=? {:source {:query {:database "My Database" :type :query :query {:source-table ["My Database" nil "Schemaless Table"]}}}:target {:database "My Database" :type "table" :schema "public" :name "target_table"}}(select-keys ser [:source :target]))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex cb7aa80a524..937533dde84 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -8,6 +8,7 @@[metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase-enterprise.serialization.v2.load :as serdes.load][metabase.actions.models :as action]+ [metabase.lib.core :as lib][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.test :as mt]@@ -231,10 +232,10 @@(testing "the serialized form is as desired"(let [card (first (by-model @serialized "Card"))]- (is (=? {:type :query- :query {:source-table ["my-db" nil "customers"]- :filter [:>= [:field ["my-db" nil "customers" "age"] nil] 18]- :aggregation [[:count]]}+ (is (=? {:type :query+ :query {:source-table ["my-db" nil "customers"]+ :filter [:>= [:field ["my-db" nil "customers" "age"] {}] 18]+ :aggregation [[:count]]}:database "my-db"}(:dataset_query card)))))@@ -263,10 +264,12 @@(is (not= (:dataset_query @card1s)(:dataset_query @card1d)))(testing "the Card's query is based on the new Database, Table, and Field IDs"- (is (=? {:type :query- :query {:source-table (:id @table1d)- :filter [:>= [:field (:id @field1d) nil] 18]- :aggregation [[:count]]}+ (is (=? {:lib/type :mbql/query+ :stages [{:source-table (:id @table1d)+ :filters [[:>= {}+ [:field {} (:id @field1d)]+ 18]]+ :aggregation [[:count {}]]}]:database (:id @db1d)}(:dataset_query @card1d))))))))))@@ -924,8 +927,7 @@(is (= (:id @snippet1s)(-> (t2/select-one :model/Card :entity_id new-eid):dataset_query- :native- :template-tags+ lib/all-template-tags-map(get "snippet: things"):snippet-id)))))))))@@ -1196,7 +1198,7 @@:name "the action":model_id (:id card):type :query- :dataset_query "wow"+ :dataset_query {}:database_id (:id db)})](reset! serialized(->> (serdes.extract/extract {})diff --git a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/api_test.cljindex 406cebf52c2..bf706bf602d 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/api_test.clj@@ -3,16 +3,11 @@(:require[clojure.test :refer :all][medley.core :as m]- [metabase-enterprise.transforms.models.transform-run]- [metabase-enterprise.transforms.models.transform-tag]- [metabase-enterprise.transforms.models.transform-transform-tag][metabase-enterprise.transforms.query-test-util :as query-test-util][metabase-enterprise.transforms.test-dataset :as transforms-dataset]- [metabase-enterprise.transforms.test-util :refer [parse-instant with-transform-cleanup! utc-timestamp get-test-schema]]+ [metabase-enterprise.transforms.test-util :refer [get-test-schema parse-instant utc-timestamp with-transform-cleanup!]][metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver]- [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp]@@ -43,14 +38,12 @@"Create a query filtering products by category, using shared utility.Returns a legacy MBQL query structure for API compatibility."[category]- (let [table-name (t2/select-one-fn :name :model/Table (mt/id :transforms_products))- query (query-test-util/make-query- {:source-table table-name- :source-column "category"- :filter-fn lib/=- :filter-values [category]})]- ;; Convert to legacy MBQL which the transform API expects- (lib.convert/->legacy-MBQL query)))+ (let [table-name (t2/select-one-fn :name :model/Table (mt/id :transforms_products))]+ (query-test-util/make-query+ {:source-table table-name+ :source-column "category"+ :filter-fn lib/=+ :filter-values [category]})))(deftest create-transform-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -203,10 +196,10 @@:schema (get-test-schema):name table-name}}resp (mt/user-http-request :crowberto :post 200 "ee/transform" body)]- (is (=? body+ (is (=? (m/dissoc-in body [:source :query :lib/metadata])(->(mt/user-http-request :crowberto :get 200 (format "ee/transform/%s" (:id resp)))- (update-in [:source :query] mbql.normalize/normalize))))))))))+ (update-in [:source :query] lib/normalize))))))))))(defn- ->transform [transform-name query]{:source {:type "query",@@ -226,8 +219,8 @@(mt/with-premium-features #{:transforms}(is (=? [{:name "transform1":source {:type "query", :query {:database (mt/id)- :type "query"- :query {:source-table (mt/id :orders)}}}+ :lib/type "mbql/query"+ :stages [{:source-table (mt/id :orders)}]}}:id (:id parent):entity_id (:entity_id parent):target {:schema "public", :name "orders_2", :type "table"}}]@@ -253,11 +246,11 @@:target {:type "table":schema (get-test-schema):name table-name}}]- (is (=? transform+ (is (=? (m/dissoc-in transform [:source :query :lib/metadata])(->(mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp))transform)- (update-in [:source :query] mbql.normalize/normalize))))))))))+ (update-in [:source :query] lib/normalize))))))))))(deftest change-target-table-test(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)@@ -281,10 +274,10 @@:target {:type "table":schema (get-test-schema):name table2-name}}]- (is (=? updated- (->- (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) updated)- (update-in [:source :query] mbql.normalize/normalize))))+ (is (=? (-> updated+ (m/dissoc-in [:source :query :lib/metadata]))+ (-> (mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" (:id resp)) updated)+ (update-in [:source :query] lib/normalize))))(is (false? (transforms.util/target-table-exists? original)))))))))(deftest delete-transforms-test@@ -413,10 +406,11 @@:source {:type "query":query query2}:target target2}]- (is (=? updated+ (is (=? (-> updated+ (m/dissoc-in [:source :query :lib/metadata]))(->(mt/user-http-request :crowberto :put 200 (format "ee/transform/%s" transform-id) updated)- (update-in [:source :query] mbql.normalize/normalize))))+ (update-in [:source :query] lib/normalize))))(test-run transform-id)(wait-for-table table2-name 5000)(is (true? (transforms.util/target-table-exists? original)))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex 71d110792bc..9e8939460d3 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -153,7 +153,7 @@t3 #{t1 t2}}(ordering/transform-ordering (t2/select :model/Transform :id [:in [t1 t2 t3]])))))))))-(deftest basic-dependencies-test+(deftest ^:parallel basic-dependencies-test(mt/with-temp [:model/Transform {t1 :id} (make-transform{:database (mt/id),:type "query",@@ -161,7 +161,7 @@(is (= #{{:table (mt/id :orders)}}(transform-deps-for-db (t2/select-one :model/Transform :id t1))))))-(deftest joined-dependencies-test+(deftest ^:parallel joined-dependencies-test(mt/with-temp [:model/Transform {t1 :id} (make-transform{:database (mt/id),:type "query",diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.cljindex a27368b879a..e3df5230390 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/api_test.clj@@ -16,11 +16,9 @@(testing "requires superuser permissions"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 "ee/transforms-python/library/common"))))-(testing "returns nil when no library exists"(t2/delete! :model/PythonLibrary)(is (= "Not found." (mt/user-http-request :crowberto :get 404 "ee/transforms-python/library/common"))))-(testing "returns existing library"(t2/delete! :model/PythonLibrary)(python-library/update-python-library-source! "common" "def my_function():\n return 42")@@ -29,7 +27,6 @@:created_at some?:updated_at some?}(mt/user-http-request :crowberto :get 200 "ee/transforms-python/library/common"))))-(testing "rejects invalid paths"(is (= "Invalid library path. Only 'common' is currently supported."(:message (mt/user-http-request :crowberto :get 400 "ee/transforms-python/library/invalid-path"))))))))@@ -41,14 +38,12 @@(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :put 403 "ee/transforms-python/library/common"{:source "def test(): pass"}))))-(testing "validates request body"(testing "requires source field"(is (=? {:errors{:source "string"}}(mt/user-http-request :crowberto :put 400 "ee/transforms-python/library/common"{}))))-(testing "allows empty string"(t2/delete! :model/PythonLibrary)(is (=? {:source ""@@ -58,7 +53,6 @@:updated_at some?}(mt/user-http-request :crowberto :put 200 "ee/transforms-python/library/common"{:source ""})))))-(testing "creates new library when none exists"(t2/delete! :model/PythonLibrary)(is (=? {:source "def new_function():\n return 100"@@ -70,7 +64,6 @@{:source "def new_function():\n return 100"})))(is (= "def new_function():\n return 100"(t2/select-one-fn :source :model/PythonLibrary))))-(testing "updates existing library"(t2/delete! :model/PythonLibrary)(python-library/update-python-library-source! "common" "def old_function():\n return 1")@@ -84,7 +77,6 @@(is (= "def updated_function():\n return 2"(t2/select-one-fn :source :model/PythonLibrary)))(is (= 1 (t2/count :model/PythonLibrary)) "Should not create duplicate records"))-(testing "rejects invalid paths"(is (= "Invalid library path. Only 'common' is currently supported."(:message (mt/user-http-request :crowberto :put 400 "ee/transforms-python/library/invalid-path"diff --git a/enterprise/frontend/src/embedding-sdk-bundle/test/auth-flow/setup.tsx b/enterprise/frontend/src/embedding-sdk-bundle/test/auth-flow/setup.tsxindex 2afd1b21a0c..22814634cfc 100644--- a/enterprise/frontend/src/embedding-sdk-bundle/test/auth-flow/setup.tsx+++ b/enterprise/frontend/src/embedding-sdk-bundle/test/auth-flow/setup.tsx@@ -16,11 +16,14 @@ import {createMockDataset,createMockDatasetData,} from "metabase-types/api/mocks";+import { createSampleDatabase } from "metabase-types/api/mocks/presets";import { MOCK_INSTANCE_URL } from "../mocks/sso";import { createMockLoginStatusState, createMockSdkState } from "../mocks/state";import { setupSdkState } from "../server-mocks/sdk-init";+const MOCK_DB = createSampleDatabase();+const MOCK_CARD = createMockCard({ id: 1 });const MOCK_COLUMN = createMockColumn({@@ -41,7 +44,10 @@ export const setup = ({}: Pick<MetabaseProviderProps, "authConfig" | "locale">) => {setupCardEndpoints(MOCK_CARD);setupCardQueryEndpoints(MOCK_CARD, MOCK_DATASET);- setupCardQueryMetadataEndpoint(MOCK_CARD, createMockCardQueryMetadata());+ setupCardQueryMetadataEndpoint(+ MOCK_CARD,+ createMockCardQueryMetadata({ databases: [MOCK_DB] }),+ );const { state } = setupSdkState({sdkState: createMockSdkState({diff --git a/enterprise/frontend/src/metabase-enterprise/rich_text_editing/tiptap/extensions/CardEmbed/CardEmbedNode.tsx b/enterprise/frontend/src/metabase-enterprise/rich_text_editing/tiptap/extensions/CardEmbed/CardEmbedNode.tsxindex 2214957a5eb..c89e85d5dc0 100644--- a/enterprise/frontend/src/metabase-enterprise/rich_text_editing/tiptap/extensions/CardEmbed/CardEmbedNode.tsx+++ b/enterprise/frontend/src/metabase-enterprise/rich_text_editing/tiptap/extensions/CardEmbed/CardEmbedNode.tsx@@ -252,7 +252,8 @@ export const CardEmbedComponent = memo(const isBeingDragged = editor.view.draggingNode === node;const displayName = name || card?.name;- const isNativeQuestion = card?.dataset_query?.type === "native";+ const question = card != null ? new Question(card) : undefined;+ const isNativeQuestion = question?.isNative();useEffect(() => {if (isEditingTitle && titleInputRef.current) {diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsxindex 43d04860217..c5369e72971 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/NewTransformPage.tsx@@ -9,7 +9,7 @@ import * as Urls from "metabase/lib/urls";import { PLUGIN_TRANSFORMS_PYTHON } from "metabase/plugins";import type {CardId,- DatasetQuery,+ LegacyDatasetQuery,Transform,TransformSource,} from "metabase-types/api";@@ -29,7 +29,7 @@ type NewTransformPageParams = {};type NewTransformPageParsedParams = {- type: DatasetQuery["type"] | "python";+ type: LegacyDatasetQuery["type"] | "python";cardId?: CardId;};@@ -78,7 +78,7 @@ export function NewTransformPage({ params }: NewTransformPageProps) {}function NewTransformEditorBody(props: {- type: DatasetQuery["type"] | "python";+ type: LegacyDatasetQuery["type"] | "python";cardId?: CardId;onSave: (source: TransformSource) => void;onCancel: () => void;@@ -104,7 +104,7 @@ function NewQueryTransformEditorBody({onSave,onCancel,}: {- type: DatasetQuery["type"];+ type: LegacyDatasetQuery["type"];cardId?: CardId;onSave: (source: TransformSource) => void;onCancel: () => void;diff --git a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.tsindex 7c71e2bdd2e..e21e05790e9 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/pages/NewTransformPage/utils.ts@@ -1,9 +1,9 @@import Question from "metabase-lib/v1/Question";-import type { Card, DatasetQuery } from "metabase-types/api";+import type { Card, LegacyDatasetQuery } from "metabase-types/api";export function getInitialQueryTransformSource(card: Card | undefined,- type: DatasetQuery["type"] | undefined,+ type: LegacyDatasetQuery["type"] | undefined,) {const query =card != nulldiff --git a/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts b/enterprise/frontend/src/metabase-enterprise/transforms/urls.tsindex 8aba3596fe0..18718968e9a 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms/urls.ts@@ -1,7 +1,7 @@import type {CardId,DatabaseId,- DatasetQuery,+ LegacyDatasetQuery,TableId,TransformId,TransformJobId,@@ -39,7 +39,7 @@ export function getTransformListUrl({}export function getNewTransformFromTypeUrl(- type: DatasetQuery["type"] | "python",+ type: LegacyDatasetQuery["type"] | "python",) {return `${ROOT_URL}/new/${type}`;}diff --git a/frontend/src/metabase-lib/breakout.unit.spec.ts b/frontend/src/metabase-lib/breakout.unit.spec.tsindex f929821fd4d..9d3297ac789 100644--- a/frontend/src/metabase-lib/breakout.unit.spec.ts+++ b/frontend/src/metabase-lib/breakout.unit.spec.ts@@ -39,7 +39,7 @@ describe("breakout", () => {});const roundtripQuery = createQuery({- query: Lib.toLegacyQuery(nextQuery),+ query: Lib.toJsQuery(nextQuery),});const roundtripQueryColumns = Lib.breakoutableColumns(roundtripQuery, 0);const roundtripTaxColumn = columnFinder(diff --git a/frontend/src/metabase-lib/order_by.unit.spec.ts b/frontend/src/metabase-lib/order_by.unit.spec.tsindex fb445e329a8..dcf15a18f64 100644--- a/frontend/src/metabase-lib/order_by.unit.spec.ts+++ b/frontend/src/metabase-lib/order_by.unit.spec.ts@@ -129,7 +129,7 @@ describe("order by", () => {);const roundtripQuery = createQuery({- query: Lib.toLegacyQuery(nextQuery),+ query: Lib.toJsQuery(nextQuery),});const roundtripQueryColumns = Lib.orderableColumns(roundtripQuery, 0);const roundtripTaxColumn = columnFinder(diff --git a/frontend/src/metabase-lib/query.ts b/frontend/src/metabase-lib/query.tsindex 9482580aa96..383ecadd09e 100644--- a/frontend/src/metabase-lib/query.ts+++ b/frontend/src/metabase-lib/query.ts@@ -2,8 +2,9 @@ import * as ML from "cljs/metabase.lib.js";import type {CardId,CardType,- DatabaseId,DatasetQuery,+ LegacyDatasetQuery,+ OpaqueDatasetQuery,TableId,} from "metabase-types/api";@@ -20,14 +21,6 @@ import type {TableMetadata,} from "./types";-export function fromLegacyQuery(- databaseId: DatabaseId | null,- metadataProvider: MetadataProvider,- datasetQuery: DatasetQuery,-): Query {- return ML.query(databaseId, metadataProvider, datasetQuery);-}-/*** Use this in combination with Lib.metadataProvider(databaseId, legacyMetadata) andLib.tableOrCardMetadata(metadataProvider, tableOrCardId);@@ -39,7 +32,7 @@ export function queryFromTableOrCardMetadata(return ML.query(metadataProvider, tableOrCardMetadata);}-export function toLegacyQuery(query: Query): DatasetQuery {+export function toLegacyQuery(query: Query): LegacyDatasetQuery {return ML.legacy_query(query);}@@ -136,3 +129,14 @@ export function previewQuery(): Query | null {return ML.preview_query(query, stageIndex, clauseType, clauseIndex);}++export function fromJsQuery(+ metadataProvider: MetadataProvider,+ jsQuery: OpaqueDatasetQuery | DatasetQuery,+): Query {+ return ML.from_js_query(metadataProvider, jsQuery);+}++export function toJsQuery(query: Query): OpaqueDatasetQuery {+ return ML.to_js_query(query);+}diff --git a/frontend/src/metabase-lib/query.unit.spec.ts b/frontend/src/metabase-lib/query.unit.spec.tsindex 048804287e5..b0f330009f1 100644--- a/frontend/src/metabase-lib/query.unit.spec.ts+++ b/frontend/src/metabase-lib/query.unit.spec.ts@@ -7,24 +7,16 @@ import {createQuery,} from "./test-helpers";-describe("fromLegacyQuery", () => {+describe("fromJsQuery", () => {// this is a very important optimization that the FE heavily relies uponit("should return the same object for the same database id, query, and metadata", () => {const metadataProvider = Lib.metadataProvider(SAMPLE_DATABASE.id,SAMPLE_METADATA,);- const query1 = Lib.fromLegacyQuery(- SAMPLE_DATABASE.id,- metadataProvider,- DEFAULT_QUERY,- );- const query2 = Lib.fromLegacyQuery(- SAMPLE_DATABASE.id,- metadataProvider,- DEFAULT_QUERY,- );- expect(query1).toBe(query2);+ const query1 = Lib.fromJsQuery(metadataProvider, DEFAULT_QUERY);+ const query2 = Lib.fromJsQuery(metadataProvider, DEFAULT_QUERY);+ expect(query1 === query2).toBe(true);});});diff --git a/frontend/src/metabase-lib/test-helpers.ts b/frontend/src/metabase-lib/test-helpers.tsindex c3b70727927..6427af28133 100644--- a/frontend/src/metabase-lib/test-helpers.ts+++ b/frontend/src/metabase-lib/test-helpers.ts@@ -52,7 +52,7 @@ export function createQuery({query = DEFAULT_QUERY,}: QueryOpts = {}) {const metadataProvider = createMetadataProvider({ databaseId, metadata });- return Lib.fromLegacyQuery(databaseId, metadataProvider, query);+ return Lib.fromJsQuery(metadataProvider, query);}export const columnFinder =diff --git a/frontend/src/metabase-lib/v1/Question.ts b/frontend/src/metabase-lib/v1/Question.tsindex d23b084394a..7e6a101c37f 100644--- a/frontend/src/metabase-lib/v1/Question.ts+++ b/frontend/src/metabase-lib/v1/Question.ts@@ -177,7 +177,7 @@ class Question {_legacyNativeQuery = _.once((): NativeQuery | undefined => {const datasetQuery = this._card.dataset_query;- if (NativeQuery.isDatasetQueryType(datasetQuery)) {+ if (this.isNative()) {return new NativeQuery(this, datasetQuery);}@@ -213,6 +213,18 @@ class Question {return this.setCard(assoc(this.card(), "dataset_query", newDatasetQuery));}+ isNative() {+ if (+ this.datasetQuery() == null ||+ InternalQuery.isDatasetQueryType(this.datasetQuery())+ ) {+ return false;+ }++ const queryInfo = Lib.queryDisplayInfo(this.query());+ return queryInfo.isNative;+ }+/*** The visualization type of the question*/@@ -749,7 +761,7 @@ class Question {dashboard_id: card.dashboard_id,...(includeEntityId ? { entity_id: card.entity_id } : {}),...(includeDatasetQuery- ? { dataset_query: Lib.toLegacyQuery(this.query()) }+ ? { dataset_query: Lib.toJsQuery(this.query()) }: {}),display: card.display,...(_.isEmpty(card.parameters)@@ -833,8 +845,7 @@ class Question {throw new Error("Internal query is not supported by MLv2");}- this.__mlv2Query ??= Lib.fromLegacyQuery(- this.datasetQuery()?.database,+ this.__mlv2Query ??= Lib.fromJsQuery(this.metadataProvider(),this.datasetQuery(),);@@ -858,7 +869,7 @@ class Question {}setQuery(query: Query): Question {- return this.setDatasetQuery(Lib.toLegacyQuery(query));+ return this.setDatasetQuery(Lib.toJsQuery(query));}generateQueryDescription() {diff --git a/frontend/src/metabase-lib/v1/queries/NativeQuery.ts b/frontend/src/metabase-lib/v1/queries/NativeQuery.tsindex a128ff77f4b..4e19a95493a 100644--- a/frontend/src/metabase-lib/v1/queries/NativeQuery.ts+++ b/frontend/src/metabase-lib/v1/queries/NativeQuery.ts@@ -1,4 +1,3 @@-import { updateIn } from "icepick";import slugg from "slugg";import { t } from "ttag";import _ from "underscore";@@ -94,10 +93,6 @@ export default class NativeQuery {this._datasetQuery = datasetQuery;}- static isDatasetQueryType(datasetQuery: DatasetQuery) {- return datasetQuery?.type === NATIVE_QUERY_TEMPLATE.type;- }-private _query(): Lib.Query {return this.question().query();}@@ -239,23 +234,17 @@ export default class NativeQuery {setParameterIndex(id: string, newIndex: number) {// NOTE: currently all NativeQuery parameters are implicitly generated from// template tags, and the order is determined by the key order- // NOTE 2: currently cannot be ported to MBQL lib because maps with keys in- // different order are considered equal.- return new NativeQuery(- this._question,- updateIn(- this._datasetQuery,- ["native", "template-tags"],- (templateTags: TemplateTags) => {- const entries = Array.from(Object.entries(templateTags));-- const oldIndex = _.findIndex(entries, (entry) => entry[1].id === id);-- entries.splice(newIndex, 0, entries.splice(oldIndex, 1)[0]);- return _.object(entries);- },- ),+ const query = this._query();+ const tags = this.templateTags();+ const oldIndex = tags.findIndex((tag) => tag.id === id);++ const newTags = [...tags];+ newTags.splice(newIndex, 0, newTags.splice(oldIndex, 1)[0]);+ const newTagsMap = Object.fromEntries(+ newTags.map((tag) => [tag.name, tag]),);++ return this._setQuery(Lib.withTemplateTags(query, newTagsMap));}lineCount(): number {diff --git a/frontend/src/metabase-lib/v1/queries/utils/card.js b/frontend/src/metabase-lib/v1/queries/utils/card.jsindex 6ed814876a5..b332bd7bb59 100644--- a/frontend/src/metabase-lib/v1/queries/utils/card.js+++ b/frontend/src/metabase-lib/v1/queries/utils/card.js@@ -3,11 +3,16 @@ import _ from "underscore";import { copy } from "metabase/lib/utils";import * as Lib from "metabase-lib";+import Question from "metabase-lib/v1/Question";import { deriveFieldOperatorFromParameter } from "metabase-lib/v1/parameters/utils/operators";import { normalizeParameterValue } from "metabase-lib/v1/parameters/utils/parameter-values";export function isNative(card) {- return card?.dataset_query?.type === "native";+ if (!card) {+ return false;+ }+ const question = Question.create({ dataset_query: card.dataset_query });+ return question.isNative();}function cardVisualizationIsEquivalent(cardA, cardB) {diff --git a/frontend/src/metabase-lib/v1/urls.ts b/frontend/src/metabase-lib/v1/urls.tsindex 5e6633adc32..f594870af47 100644--- a/frontend/src/metabase-lib/v1/urls.ts+++ b/frontend/src/metabase-lib/v1/urls.ts@@ -104,7 +104,7 @@ export function getAutomaticDashboardUrl(questionWithFilters: Question,) {const questionId = question.id();- const filterQuery = questionWithFilters.datasetQuery();+ const filterQuery = Lib.toLegacyQuery(questionWithFilters.query());const filter = filterQuery.type === "query" ? filterQuery.query.filter : null;const cellQuery = filter? `/cell/${utf8_to_b64url(JSON.stringify(filter))}`@@ -125,7 +125,7 @@ export function getComparisonDashboardUrl() {const questionId = question.id();const tableId = question.legacyQueryTableId();- const filterQuery = questionWithFilters.datasetQuery();+ const filterQuery = Lib.toLegacyQuery(questionWithFilters.query());const filter = filterQuery.type === "query" ? filterQuery.query.filter : null;const cellQuery = filter? `/cell/${utf8_to_b64url(JSON.stringify(filter))}`diff --git a/frontend/src/metabase-types/api/dataset.ts b/frontend/src/metabase-types/api/dataset.tsindex 2431511ac9d..498c1e23fed 100644--- a/frontend/src/metabase-types/api/dataset.ts+++ b/frontend/src/metabase-types/api/dataset.ts@@ -98,7 +98,7 @@ export interface DatasetData {}export type JsonQuery = DatasetQuery & {- parameters?: unknown[];+ parameters?: Parameter[];"cache-strategy"?: CacheStrategy & {/** An ISO 8601 date */"invalidated-at"?: string;diff --git a/frontend/src/metabase-types/api/query.ts b/frontend/src/metabase-types/api/query.tsindex 6fca2c3e254..a6d7af16ba8 100644--- a/frontend/src/metabase-types/api/query.ts+++ b/frontend/src/metabase-types/api/query.ts@@ -32,7 +32,16 @@ export interface NativeDatasetQuery {parameters?: UiParameter[];}-export type DatasetQuery = StructuredDatasetQuery | NativeDatasetQuery;+export type DatasetQuery = OpaqueDatasetQuery | LegacyDatasetQuery;++export type LegacyDatasetQuery = StructuredDatasetQuery | NativeDatasetQuery;++declare const OpaqueDatasetQuerySymbol: unique symbol;+export type OpaqueDatasetQuery = unknown & {+ // TODO (AlexP 10/09/25) -- replace usages of this field with Lib.databaseID and drop it from here+ database: DatabaseId | null;+ _opaque: typeof OpaqueDatasetQuerySymbol;+};interface PublicStructuredDatasetQuery {type: "query";@@ -45,10 +54,12 @@ interface PublicNativeDatasetQuery {};}-export type PublicDatasetQuery =+export type LegacyPublicDatasetQuery =| PublicStructuredDatasetQuery| PublicNativeDatasetQuery;+export type PublicDatasetQuery = OpaqueDatasetQuery | LegacyPublicDatasetQuery;+export const dateTimeAbsoluteUnits = ["minute","hour",diff --git a/frontend/src/metabase/actions/components/ActionViz/ActionParameterMapping.tsx b/frontend/src/metabase/actions/components/ActionViz/ActionParameterMapping.tsxindex b4c61efaddb..221ea3d44ab 100644--- a/frontend/src/metabase/actions/components/ActionViz/ActionParameterMapping.tsx+++ b/frontend/src/metabase/actions/components/ActionViz/ActionParameterMapping.tsx@@ -66,7 +66,7 @@ export const ActionParameterMappingForm = ({setParameterMapping(dashboardParameterId,dashcard.id,- -1, // this is irrelevant for action parameters+ null, // this is irrelevant for action parameterstarget,),);diff --git a/frontend/src/metabase/actions/containers/ActionCreator/ActionContext/QueryActionContextProvider/utils.unit.spec.ts b/frontend/src/metabase/actions/containers/ActionCreator/ActionContext/QueryActionContextProvider/utils.unit.spec.tsindex fa21459b318..d5db3e7f830 100644--- a/frontend/src/metabase/actions/containers/ActionCreator/ActionContext/QueryActionContextProvider/utils.unit.spec.ts+++ b/frontend/src/metabase/actions/containers/ActionCreator/ActionContext/QueryActionContextProvider/utils.unit.spec.ts@@ -3,12 +3,9 @@ import {getDefaultFieldSettings,getDefaultFormSettings,} from "metabase/actions/utils";+import * as Lib from "metabase-lib";import Question from "metabase-lib/v1/Question";-import type {- NativeDatasetQuery,- Parameter,- TemplateTagType,-} from "metabase-types/api";+import type { Parameter, TemplateTagType } from "metabase-types/api";import {SAMPLE_DB_ID,createAdHocNativeCard,@@ -145,9 +142,7 @@ describe("actions > containers > ActionCreator > QueryActionContextProvider > utformSettings,);- const tags = (newQuestion.card().dataset_query as NativeDatasetQuery)- .native["template-tags"];-+ const tags = Lib.templateTags(newQuestion.query());expect(tags?.name.type).toEqual("text");expect(tags?.price.type).toEqual("number");});@@ -168,9 +163,7 @@ describe("actions > containers > ActionCreator > QueryActionContextProvider > utformSettings,);- const tags = (newQuestion.card().dataset_query as NativeDatasetQuery)- .native["template-tags"];-+ const tags = Lib.templateTags(newQuestion.query());expect(tags?.name.type).toEqual("date");expect(tags?.price.type).toEqual("date");});diff --git a/frontend/src/metabase/actions/containers/ActionParametersInputForm/ActionParametersInputForm.unit.spec.tsx b/frontend/src/metabase/actions/containers/ActionParametersInputForm/ActionParametersInputForm.unit.spec.tsxindex 4ef4359535b..dd7de6f1a5c 100644--- a/frontend/src/metabase/actions/containers/ActionParametersInputForm/ActionParametersInputForm.unit.spec.tsx+++ b/frontend/src/metabase/actions/containers/ActionParametersInputForm/ActionParametersInputForm.unit.spec.tsx@@ -18,13 +18,13 @@ import ActionParametersInputModal from "./ActionParametersInputModal";const parameter1 = createMockActionParameter({id: "parameter_1",name: "Parameter 1",- type: "type/Text",+ type: "string/=",});const parameter2 = createMockActionParameter({id: "parameter_2",name: "Parameter 2",- type: "type/Text",+ type: "string/=",});const mockAction = createMockQueryAction({@@ -122,12 +122,12 @@ describe("Actions > ActionParametersInputForm", () => {createMockActionParameter({id: "parameter_1","display-name": "Parameter 1",- type: "type/Text",+ type: "string/=",}),createMockActionParameter({id: "parameter_2","display-name": "Parameter 2",- type: "type/Integer",+ type: "number/=",}),],});diff --git a/frontend/src/metabase/actions/hooks/use-action-form/utils.ts b/frontend/src/metabase/actions/hooks/use-action-form/utils.tsindex 47b00cfba4f..b2464f6c1fa 100644--- a/frontend/src/metabase/actions/hooks/use-action-form/utils.ts+++ b/frontend/src/metabase/actions/hooks/use-action-form/utils.ts@@ -3,8 +3,7 @@ import dayjs from "dayjs";import type { FieldSettings as LocalFieldSettings } from "metabase/actions/types";import { getDefaultFieldSettings } from "metabase/actions/utils";import { isEmpty } from "metabase/lib/validate";-import Field from "metabase-lib/v1/metadata/Field";-import { TYPE } from "metabase-lib/v1/types/constants";+import { getParameterType } from "metabase-lib/v1/parameters/utils/parameter-type";import type {FieldSettings,FieldSettingsMap,@@ -59,48 +58,24 @@ export const formatSubmitValues = (return values;};-const isNumericParameter = (param: Parameter): boolean =>- /integer|float/gi.test(param.type);-const getFieldType = (param: Parameter): "number" | "string" => {- return isNumericParameter(param) ? "number" : "string";+ return getParameterType(param) === "number" ? "number" : "string";};-export const getInputType = (param: Parameter, field?: Field) => {- if (!field) {- return isNumericParameter(param) ? "number" : "string";- }-- if (field.isFK()) {- return field.isNumeric() ? "number" : "string";- }- if (field.isNumeric()) {- return "number";- }- if (field.isBoolean()) {- return "boolean";- }- if (field.isTime()) {- return "time";+export const getInputType = (param: Parameter): InputSettingType => {+ const type = getParameterType(param);+ switch (type) {+ case "string":+ return "string";+ case "number":+ return "number";+ case "date":+ return "datetime";+ case "boolean":+ return "boolean";+ default:+ return "string";}- if (field.isDate()) {- return field.isDateWithoutTime() ? "date" : "datetime";- }- if (- field.semantic_type === TYPE.Description ||- field.semantic_type === TYPE.Comment ||- field.base_type === TYPE.Structured- ) {- return "text";- }- if (- field.semantic_type === TYPE.Title ||- field.semantic_type === TYPE.Email- ) {- return "string";- }-- return "string";};// TODO: @uladzimirdev remove this method once the migration of implicit action fields generating to the BE is complete@@ -139,25 +114,18 @@ export const generateFieldSettingsFromParameters = (params: Parameter[]) => {const fieldSettings: Record<ParameterId, LocalFieldSettings> = {};params.forEach((param, index) => {- const field = new Field({- id: param.id,- name: param.id,- slug: param.id,- display_name: param["display-name"],- base_type: param.type,- semantic_type: param.type,- });+ const title = param["display-name"] ?? param.name;fieldSettings[param.id] = getDefaultFieldSettings({id: param.id,name: param.name,- title: field.displayName(),- placeholder: field.displayName(),+ title,+ placeholder: title,required: !!param.required,order: index,description: "",fieldType: getFieldType(param),- inputType: getInputType(param, field),+ inputType: getInputType(param),});});diff --git a/frontend/src/metabase/actions/hooks/use-action-form/utils.unit.spec.ts b/frontend/src/metabase/actions/hooks/use-action-form/utils.unit.spec.tsindex f1c554cabf1..d360095ad7c 100644--- a/frontend/src/metabase/actions/hooks/use-action-form/utils.unit.spec.ts+++ b/frontend/src/metabase/actions/hooks/use-action-form/utils.unit.spec.ts@@ -1,8 +1,9 @@-import { createMockMetadata } from "__support__/metadata";-import { checkNotNull } from "metabase/lib/types";-import type { FieldSettingsMap } from "metabase-types/api";+import type {+ FieldSettingsMap,+ InputSettingType,+ ParameterType,+} from "metabase-types/api";import {- createMockField,createMockFieldSettings,createMockParameter,} from "metabase-types/api/mocks";@@ -20,74 +21,7 @@ const getFirstEntry = (obj: any): any => {return Object.entries(obj)[0];};-const STRING_FK_FIELD_ID = 1;-const INT_FK_FIELD_ID = 2;-const BOOLEAN_FIELD_ID = 3;-const FLOAT_FIELD_ID = 4;-const CATEGORY_FIELD_ID = 5;-const LONG_TEXT_FIELD_ID = 6;--const DATE_FIELD_ID = 100;-const DATETIME_FIELD_ID = 101;-const DATETIME_LOCAL_TZ_FIELD_ID = 102;-const TIME_FIELD_ID = 103;-const TIME_LOCAL_TZ_FIELD_ID = 104;-describe("actions > containers > ActionParametersInputForm > utils", () => {- const dateTimeFields = [- { id: DATE_FIELD_ID, base_type: "type/Date" },- { id: DATETIME_FIELD_ID, base_type: "type/DateTime" },- { id: DATETIME_LOCAL_TZ_FIELD_ID, base_type: "type/DateTimeWithLocalTZ" },- { id: TIME_FIELD_ID, base_type: "type/Time" },- { id: TIME_LOCAL_TZ_FIELD_ID, base_type: "type/TimeWithLocalTZ" },- ].map(({ id, base_type }) =>- createMockField({- id,- base_type,- effective_type: base_type,- }),- );-- const metadata = createMockMetadata({- fields: [- createMockField({- id: STRING_FK_FIELD_ID,- base_type: "type/Text",- effective_type: "type/Text",- semantic_type: "type/FK",- }),- createMockField({- id: INT_FK_FIELD_ID,- base_type: "type/Integer",- effective_type: "type/Integer",- semantic_type: "type/FK",- }),- createMockField({- id: BOOLEAN_FIELD_ID,- base_type: "type/Boolean",- effective_type: "type/Boolean",- }),- createMockField({- id: FLOAT_FIELD_ID,- base_type: "type/Float",- effective_type: "type/Float",- }),- createMockField({- id: CATEGORY_FIELD_ID,- base_type: "type/Text",- effective_type: "type/Text",- semantic_type: "type/Category",- }),- createMockField({- id: LONG_TEXT_FIELD_ID,- base_type: "type/Text",- effective_type: "type/Text",- semantic_type: "type/Description",- }),- ...dateTimeFields,- ],- });-describe("stripTZInfo", () => {it("should strip timezone info from dateTimes", () => {const result = stripTZInfo("2020-05-01T03:30:00-02:00");@@ -155,86 +89,32 @@ describe("actions > containers > ActionParametersInputForm > utils", () => {});describe("getInputType", () => {- it('should return "number" for numeric parameters', () => {- const intParam = createMockParameter({ type: "type/Integer" });- expect(getInputType(intParam)).toEqual("number");-- const floatParam = createMockParameter({ type: "type/Float" });- expect(getInputType(floatParam)).toEqual("number");- });-- it('should return "string" for non-numeric parameters', () => {- const textParam = createMockParameter({ type: "type/Text" });- expect(getInputType(textParam)).toEqual("string");-- const turtleParam = createMockParameter({ type: "type/Turtle" });- expect(getInputType(turtleParam)).toEqual("string");- });-- it('should return "number" for numeric foreign keys', () => {- const field = checkNotNull(metadata.field(INT_FK_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("number");- });-- it('should return "string" for string foreign keys', () => {- const field = checkNotNull(metadata.field(STRING_FK_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("string");- });-- it('should return "number" for floating point numbers', () => {- const field = checkNotNull(metadata.field(FLOAT_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("number");- });-- it('should return "boolean" for booleans', () => {- const field = checkNotNull(metadata.field(BOOLEAN_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("boolean");- });-- it('should return "date" for dates', () => {- const field = checkNotNull(metadata.field(DATE_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("date");- });-- it('should return "datetime" for datetimes', () => {- const parameter = createMockParameter();-- const dateTimeField = checkNotNull(metadata.field(DATETIME_FIELD_ID));- const dateTimeLocalTZField = checkNotNull(- metadata.field(DATETIME_LOCAL_TZ_FIELD_ID),- );-- expect(getInputType(parameter, dateTimeField)).toEqual("datetime");- expect(getInputType(parameter, dateTimeLocalTZField)).toEqual("datetime");- });-- it('should return "time" for times', () => {- const parameter = createMockParameter();-- const timeField = checkNotNull(metadata.field(TIME_FIELD_ID));- const timeLocalTZField = checkNotNull(- metadata.field(TIME_LOCAL_TZ_FIELD_ID),- );-- expect(getInputType(parameter, timeField)).toEqual("time");- expect(getInputType(parameter, timeLocalTZField)).toEqual("time");- });-- it('should return "string" for categories', () => {- const field = checkNotNull(metadata.field(CATEGORY_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("string");- });-- it('should return "text" for description', () => {- const field = checkNotNull(metadata.field(LONG_TEXT_FIELD_ID));- expect(getInputType(createMockParameter(), field)).toEqual("text");- });+ it.each<{+ parameterType: ParameterType;+ expectedInputType: InputSettingType;+ }>([+ { parameterType: "string", expectedInputType: "string" },+ { parameterType: "string/=", expectedInputType: "string" },+ { parameterType: "number", expectedInputType: "number" },+ { parameterType: "number/=", expectedInputType: "number" },+ { parameterType: "boolean", expectedInputType: "boolean" },+ { parameterType: "boolean/=", expectedInputType: "boolean" },+ { parameterType: "date", expectedInputType: "datetime" },+ { parameterType: "date/all-options", expectedInputType: "datetime" },+ { parameterType: "category", expectedInputType: "string" },+ ])(+ "should get the input type for $parameterType parameterType",+ ({ parameterType, expectedInputType }) => {+ const parameter = createMockParameter({ type: parameterType });+ expect(getInputType(parameter)).toBe(expectedInputType);+ },+ );});describe("generateFieldSettingsFromParameters", () => {it("should generate settings for a string field", () => {const params = [- createMockParameter({ id: "test-field", type: "type/Text" }),+ createMockParameter({ id: "test-field", type: "string" }),];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),@@ -246,7 +126,7 @@ describe("actions > containers > ActionParametersInputForm > utils", () => {it("should generate settings for an Integer field", () => {const params = [- createMockParameter({ id: "test-field", type: "type/Integer" }),+ createMockParameter({ id: "test-field", type: "number" }),];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),@@ -258,7 +138,7 @@ describe("actions > containers > ActionParametersInputForm > utils", () => {it("should generate settings for a float field", () => {const params = [- createMockParameter({ id: "test-field", type: "type/Float" }),+ createMockParameter({ id: "test-field", type: "number" }),];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),@@ -270,7 +150,7 @@ describe("actions > containers > ActionParametersInputForm > utils", () => {it("generates field settings for an integer field", () => {const params = [- createMockParameter({ id: "test-field", type: "type/Integer" }),+ createMockParameter({ id: "test-field", type: "number" }),];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),@@ -281,9 +161,7 @@ describe("actions > containers > ActionParametersInputForm > utils", () => {});it("should generate settings for a dateTime field", () => {- const params = [- createMockParameter({ id: "test-field", type: "type/DateTime" }),- ];+ const params = [createMockParameter({ id: "test-field", type: "date" })];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),);@@ -293,27 +171,25 @@ describe("actions > containers > ActionParametersInputForm > utils", () => {});it("should generate settings for a date field", () => {- const params = [- createMockParameter({ id: "test-field", type: "type/Date" }),- ];+ const params = [createMockParameter({ id: "test-field", type: "date" })];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),);expect(settings.fieldType).toBe("string");- expect(settings.inputType).toBe("date");+ expect(settings.inputType).toBe("datetime");});it("generates field settings for a json field", () => {const params = [- createMockParameter({ id: "test-field", type: "type/Structured" }),+ createMockParameter({ id: "test-field", type: "string" }),];const [_id, settings] = getFirstEntry(generateFieldSettingsFromParameters(params),);expect(settings.fieldType).toBe("string");- expect(settings.inputType).toBe("text");+ expect(settings.inputType).toBe("string");});it("should set the parameter id as the object key", () => {diff --git a/frontend/src/metabase/admin/datamodel/utils/segments.ts b/frontend/src/metabase/admin/datamodel/utils/segments.tsindex b1762ef18d4..d6857b57b46 100644--- a/frontend/src/metabase/admin/datamodel/utils/segments.ts+++ b/frontend/src/metabase/admin/datamodel/utils/segments.ts@@ -13,7 +13,7 @@ export function getSegmentQuery(: undefined;return table && query && metadataProvider- ? Lib.fromLegacyQuery(table.db_id, metadataProvider, {+ ? Lib.fromJsQuery(metadataProvider, {type: "query",database: table.db_id,query: query,diff --git a/frontend/src/metabase/common/components/AdHocQuestionLoader.unit.spec.js b/frontend/src/metabase/common/components/AdHocQuestionLoader.unit.spec.jsindex 1b377356551..8a8fa0e6f05 100644--- a/frontend/src/metabase/common/components/AdHocQuestionLoader.unit.spec.js+++ b/frontend/src/metabase/common/components/AdHocQuestionLoader.unit.spec.js@@ -1,5 +1,7 @@import { render } from "__support__/ui";import { delay } from "__support__/utils";+import * as Lib from "metabase-lib";+import { SAMPLE_METADATA, createQuery } from "metabase-lib/test-helpers";import Question from "metabase-lib/v1/Question";import * as ML_Urls from "metabase-lib/v1/urls";@@ -20,7 +22,9 @@ describe("AdHocQuestionLoader", () => {});it("should load a question given a questionHash", async () => {- const q = Question.create({ databaseId: 1, tableId: 2 });+ const q = Question.create({ metadata: SAMPLE_METADATA }).setQuery(+ createQuery(),+ );const questionHash = ML_Urls.getUrl(q).match(/(#.*)/)[1];render(@@ -37,11 +41,11 @@ describe("AdHocQuestionLoader", () => {// stuff happens asynchronouslyawait delay(0);- expect(loadMetadataSpy.mock.calls[0][0]).toEqual(q.card());-const calls = mockChild.mock.calls;const { question, loading, error } = calls[calls.length - 1][0];- expect(question.card()).toEqual(q.card());+ expect(+ Lib.areLegacyQueriesEqual(question.datasetQuery(), q.datasetQuery()),+ ).toBe(true);expect(loading).toEqual(false);expect(error).toEqual(null);});diff --git a/frontend/src/metabase/common/utils/column-groups.unit.spec.ts b/frontend/src/metabase/common/utils/column-groups.unit.spec.tsindex c60befd0388..4619b2cefb6 100644--- a/frontend/src/metabase/common/utils/column-groups.unit.spec.ts+++ b/frontend/src/metabase/common/utils/column-groups.unit.spec.ts@@ -22,7 +22,7 @@ function getQueryWithTableDataSource() {function getQueryWithCardDataSource() {const cardId = 1;const card = Question.create({- dataset_query: Lib.toLegacyQuery(getQueryWithTableDataSource()),+ dataset_query: Lib.toJsQuery(getQueryWithTableDataSource()),}).setId(cardId).setDisplayName("Card")diff --git a/frontend/src/metabase/dashboard/actions/auto-wire-parameters/utils.ts b/frontend/src/metabase/dashboard/actions/auto-wire-parameters/utils.tsindex 96e58fca071..dc1006588d6 100644--- a/frontend/src/metabase/dashboard/actions/auto-wire-parameters/utils.ts+++ b/frontend/src/metabase/dashboard/actions/auto-wire-parameters/utils.ts@@ -135,7 +135,7 @@ export function getAutoWiredMappingsForDashcards(export function getParameterMappings<DC extends DashboardCard>(dashcard: DC,parameter_id: ParameterId,- card_id: CardId,+ card_id: CardId | null,target: ParameterTarget | null,): NonNullable<DC["parameter_mappings"]> {const isVirtual = isVirtualDashCard(dashcard);diff --git a/frontend/src/metabase/dashboard/actions/data-fetching.ts b/frontend/src/metabase/dashboard/actions/data-fetching.tsindex 5149ea788ab..90ec5b3801c 100644--- a/frontend/src/metabase/dashboard/actions/data-fetching.ts+++ b/frontend/src/metabase/dashboard/actions/data-fetching.ts@@ -1,7 +1,6 @@import { createAction } from "@reduxjs/toolkit";import { getIn } from "icepick";import { denormalize, normalize, schema } from "normalizr";-import { match } from "ts-pattern";import { t } from "ttag";import { automagicDashboardsApi, dashboardApi } from "metabase/api";@@ -56,7 +55,7 @@ import type {DashboardCard,DashboardId,Dataset,- DatasetQuery,+ JsonQuery,ParameterValuesMap,QuestionDashboardCard,} from "metabase-types/api";@@ -288,8 +287,8 @@ export const fetchCardDataAction = createAsyncThunk<const hasParametersChanged =!lastResult ||!equals(- getDatasetQueryParams(lastResult.json_query).parameters,- getDatasetQueryParams(datasetQuery).parameters,+ getDatasetQueryParams(lastResult.json_query),+ getDatasetQueryParams(datasetQuery),);if (clearCache || hasParametersChanged) {@@ -600,34 +599,14 @@ export const clearCardData = createAction((cardId, dashcardId) => ({ payload: { cardId, dashcardId } }),);-function getDatasetQueryParams(datasetQuery: Partial<DatasetQuery> = {}) {- const parameters =- datasetQuery?.parameters- ?.map((parameter) => ({- ...parameter,- value: parameter.value ?? null,- }))- .sort(sortById) ?? [];-- return match(datasetQuery)- .with({ type: "native" }, ({ native }) => ({- type: "native",- query: undefined,- native,- parameters,+function getDatasetQueryParams(datasetQuery?: JsonQuery) {+ const parameters = datasetQuery?.parameters ?? [];+ return parameters+ .map((parameter) => ({+ ...parameter,+ value: parameter.value ?? null,}))- .with({ type: "query" }, ({ query }) => ({- type: "query",- query,- native: undefined,- parameters,- }))- .otherwise(() => ({- type: undefined,- native: undefined,- query: undefined,- parameters: [],- }));+ .sort(sortById);}function sortById(a: UiParameter, b: UiParameter) {diff --git a/frontend/src/metabase/dashboard/actions/parameters.tsx b/frontend/src/metabase/dashboard/actions/parameters.tsxindex 90ede4263a3..680c8b613e2 100644--- a/frontend/src/metabase/dashboard/actions/parameters.tsx+++ b/frontend/src/metabase/dashboard/actions/parameters.tsx@@ -409,7 +409,7 @@ export const setParameterMapping = createThunkAction((parameterId: ParameterId,dashcardId: DashCardId,- cardId: CardId,+ cardId: CardId | null,target: ParameterTarget | null,) => {return (dispatch, getState) => {diff --git a/frontend/src/metabase/dashboard/utils.ts b/frontend/src/metabase/dashboard/utils.tsindex b1308c1bd30..7563a4a4fb8 100644--- a/frontend/src/metabase/dashboard/utils.ts+++ b/frontend/src/metabase/dashboard/utils.ts@@ -9,6 +9,7 @@ import {getPermissionErrorMessage,} from "metabase/visualizations/lib/errors";import { isVisualizerDashboardCard } from "metabase/visualizer/utils";+import Question from "metabase-lib/v1/Question";import type { UiParameter } from "metabase-lib/v1/parameters/types";import {areParameterValuesIdentical,@@ -198,7 +199,11 @@ export function getInlineParameterTabMap(dashboard: Dashboard) {export function isNativeDashCard(dashcard: QuestionDashboardCard) {// The `dataset_query` is null for questions on a dashboard the user doesn't have access to- return dashcard.card.dataset_query?.type === "native";+ if (dashcard.card.dataset_query == null) {+ return false;+ }+ const question = new Question(dashcard.card);+ return question.isNative();}// For a virtual (text) dashcard without any parameters, returns a boolean indicating whether we should display thediff --git a/frontend/src/metabase/detail-view/components/DetailViewSidesheet/DetailViewSidesheet.tsx b/frontend/src/metabase/detail-view/components/DetailViewSidesheet/DetailViewSidesheet.tsxindex e1fe96d3043..d8d6c6020f3 100644--- a/frontend/src/metabase/detail-view/components/DetailViewSidesheet/DetailViewSidesheet.tsx+++ b/frontend/src/metabase/detail-view/components/DetailViewSidesheet/DetailViewSidesheet.tsx@@ -91,9 +91,7 @@ export function DetailViewSidesheet({error,isLoading,} = useGetAdhocQueryQuery(- query != null && rowFromProps == null- ? Lib.toLegacyQuery(query)- : skipToken,+ query != null && rowFromProps == null ? Lib.toJsQuery(query) : skipToken,);const { columns, row } = useMemo(() => extractData(dataset, columnsFromProp, columnSettings, rowFromProps),diff --git a/frontend/src/metabase/detail-view/pages/ModelDetailPage/ModelDetailPage.tsx b/frontend/src/metabase/detail-view/pages/ModelDetailPage/ModelDetailPage.tsxindex 2f0c026bfbd..b40899ffe88 100644--- a/frontend/src/metabase/detail-view/pages/ModelDetailPage/ModelDetailPage.tsx+++ b/frontend/src/metabase/detail-view/pages/ModelDetailPage/ModelDetailPage.tsx@@ -59,7 +59,7 @@ export function ModelDetailPage({ params }: Props) {error: queryError,isLoading: isQueryLoading,} = useGetAdhocQueryQuery(- objectQuery ? Lib.toLegacyQuery(objectQuery) : skipToken,+ objectQuery ? Lib.toJsQuery(objectQuery) : skipToken,);const error = metadataError ?? queryError ?? cardError;diff --git a/frontend/src/metabase/detail-view/pages/TableDetailPage/TableDetailPage.tsx b/frontend/src/metabase/detail-view/pages/TableDetailPage/TableDetailPage.tsxindex 0872f9af915..384458549ac 100644--- a/frontend/src/metabase/detail-view/pages/TableDetailPage/TableDetailPage.tsx+++ b/frontend/src/metabase/detail-view/pages/TableDetailPage/TableDetailPage.tsx@@ -51,7 +51,7 @@ export function TableDetailPage({ params }: Props) {error: queryError,isLoading: isQueryLoading,} = useGetAdhocQueryQuery(- objectQuery ? Lib.toLegacyQuery(objectQuery) : skipToken,+ objectQuery ? Lib.toJsQuery(objectQuery) : skipToken,);const error = tableError ?? queryError;diff --git a/frontend/src/metabase/detail-view/utils.ts b/frontend/src/metabase/detail-view/utils.tsindex ce95216665d..d106ebe936f 100644--- a/frontend/src/metabase/detail-view/utils.ts+++ b/frontend/src/metabase/detail-view/utils.ts@@ -182,16 +182,13 @@ export function getTableQuery(table: Table | undefined): Lib.Query | undefined {const metadata = createMockMetadata({tables: [table],});-const metadataProvider = Lib.metadataProvider(table.db_id, metadata);+ const tableMetadata = Lib.tableOrCardMetadata(metadataProvider, table.id);+ if (tableMetadata == null) {+ return undefined;+ }- return Lib.fromLegacyQuery(table.db_id, metadataProvider, {- type: "query",- database: table.db_id,- query: {- "source-table": table.id,- },- });+ return Lib.queryFromTableOrCardMetadata(metadataProvider, tableMetadata);}export function filterByPk(diff --git a/frontend/src/metabase/lib/click-behavior.js b/frontend/src/metabase/lib/click-behavior.jsindex 4043f7336e2..396b4eb0849 100644--- a/frontend/src/metabase/lib/click-behavior.js+++ b/frontend/src/metabase/lib/click-behavior.js@@ -3,6 +3,7 @@ import { msgid, ngettext, t } from "ttag";import Dashboards from "metabase/entities/dashboards";import Questions from "metabase/entities/questions";+import Question from "metabase-lib/v1/Question";export function getClickBehaviorDescription(dashcard) {const noBehaviorMessage = hasActionsMenu(dashcard)@@ -40,9 +41,15 @@ export function getClickBehaviorDescription(dashcard) {}export function hasActionsMenu(dashcard) {+ if (!dashcard.card.dataset_query) {+ return false;+ }// This seems to work, but it isn't the right logic.// The right thing to do would be to check for any drills. However, we'd need a "clicked" object for that.- return dashcard.card.dataset_query?.type === "query";+ const question = Question.create({+ dataset_query: dashcard.card.dataset_query,+ });+ return !question.isNative();}export function isTableDisplay(dashcard) {diff --git a/frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/FilteringPreview.tsx b/frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/FilteringPreview.tsxindex a100c81a972..e5dbe4fcc6f 100644--- a/frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/FilteringPreview.tsx+++ b/frontend/src/metabase/metadata/pages/DataModel/components/PreviewSection/FilteringPreview.tsx@@ -30,7 +30,7 @@ const FilteringPreviewBase = ({ databaseId, field, fieldId, table }: Props) => {[databaseId, table],);const column = useMemo(- () => getPreviewColumn(query, fieldId),+ () => query && getPreviewColumn(query, fieldId),[fieldId, query],);@@ -38,6 +38,10 @@ const FilteringPreviewBase = ({ databaseId, field, fieldId, table }: Props) => {return <HiddenFieldEmptyStateBlock />;}+ if (query == null || column == null) {+ return null;+ }+return (<FilterPickerBodyautoFocus={false}@@ -50,7 +54,10 @@ const FilteringPreviewBase = ({ databaseId, field, fieldId, table }: Props) => {);};-function getPreviewQuery(table: Table, databaseId: number): Lib.Query {+function getPreviewQuery(+ table: Table,+ databaseId: number,+): Lib.Query | undefined {const metadata = createMockMetadata({tables: table? [@@ -64,14 +71,11 @@ function getPreviewQuery(table: Table, databaseId: number): Lib.Query {: [],});const metadataProvider = Lib.metadataProvider(databaseId, metadata);-- return Lib.fromLegacyQuery(databaseId, metadataProvider, {- type: "query",- database: databaseId,- query: {- "source-table": table.id,- },- });+ const tableMetadata = Lib.tableOrCardMetadata(metadataProvider, table.id);+ if (tableMetadata == null) {+ return undefined;+ }+ return Lib.queryFromTableOrCardMetadata(metadataProvider, tableMetadata);}function getPreviewColumn(diff --git a/frontend/src/metabase/models/components/ModelActions/ModelActionDetails/ModelActionListItem.tsx b/frontend/src/metabase/models/components/ModelActions/ModelActionDetails/ModelActionListItem.tsxindex 87480c6a2f1..8292614b5d7 100644--- a/frontend/src/metabase/models/components/ModelActions/ModelActionDetails/ModelActionListItem.tsx+++ b/frontend/src/metabase/models/components/ModelActions/ModelActionDetails/ModelActionListItem.tsx@@ -7,6 +7,8 @@ import Link from "metabase/common/components/Link";import ModalWithTrigger from "metabase/common/components/ModalWithTrigger";import { useConfirmation } from "metabase/common/hooks/use-confirmation";import { Icon } from "metabase/ui";+import * as Lib from "metabase-lib";+import Question from "metabase-lib/v1/Question";import type { WritebackAction, WritebackQueryAction } from "metabase-types/api";import {@@ -36,7 +38,8 @@ interface ModalProps {}function QueryActionCardContent({ action }: { action: WritebackQueryAction }) {- if (!action.dataset_query?.native?.query) {+ const question = Question.create({ dataset_query: action.dataset_query });+ if (!question.isNative()) {return (<CodeBlock><Icon name="warning" size={16} tooltip={t`No query found`} />@@ -44,7 +47,10 @@ function QueryActionCardContent({ action }: { action: WritebackQueryAction }) {);}- return <CodeBlock>{action.dataset_query.native.query}</CodeBlock>;+ const query = question.query();+ const queryText = Lib.rawNativeQuery(query);++ return <CodeBlock>{queryText}</CodeBlock>;}function ImplicitActionCardContent() {diff --git a/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestionView.tsx b/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestionView.tsxindex ea5c4d20a73..e9b692697ed 100644--- a/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestionView.tsx+++ b/frontend/src/metabase/public/containers/PublicOrEmbeddedQuestion/PublicOrEmbeddedQuestionView.tsx@@ -19,7 +19,6 @@ import type { UiParameter } from "metabase-lib/v1/parameters/types";import type {Card,Dataset,- DatasetQuery,ParameterId,ParameterValuesMap,RawSeries,@@ -28,7 +27,7 @@ import type {export interface PublicOrEmbeddedQuestionViewProps {initialized: boolean;- card: Card<DatasetQuery> | null;+ card: Card | null;metadata: Metadata;result: Dataset | null;uuid: string;@@ -41,7 +40,7 @@ export interface PublicOrEmbeddedQuestionViewProps {hide_parameters: string | null;theme: DisplayTheme | undefined;titled: boolean;- setCard: Dispatch<SetStateAction<Card<DatasetQuery> | null>>;+ setCard: Dispatch<SetStateAction<Card | null>>;downloadsEnabled: EmbedResourceDownloadOptions;}diff --git a/frontend/src/metabase/query_builder/actions/core/updateQuestion.unit.spec.ts b/frontend/src/metabase/query_builder/actions/core/updateQuestion.unit.spec.tsindex 67e16de2183..378faeb1f06 100644--- a/frontend/src/metabase/query_builder/actions/core/updateQuestion.unit.spec.ts+++ b/frontend/src/metabase/query_builder/actions/core/updateQuestion.unit.spec.ts@@ -9,6 +9,7 @@ import type {Card,ConcreteFieldReference,Join,+ LegacyDatasetQuery,NativeDatasetQuery,StructuredDatasetQuery,TemplateTag,@@ -58,7 +59,7 @@ import { UPDATE_QUESTION, updateQuestion } from "./updateQuestion";registerVisualizations();-type TestCard = Card | UnsavedCard;+type TestCard = Card<LegacyDatasetQuery> | UnsavedCard<LegacyDatasetQuery>;type SetupOpts = {card: TestCard;diff --git a/frontend/src/metabase/query_builder/actions/query-updates.ts b/frontend/src/metabase/query_builder/actions/query-updates.tsindex 0bc416090a0..28368473a0b 100644--- a/frontend/src/metabase/query_builder/actions/query-updates.ts+++ b/frontend/src/metabase/query_builder/actions/query-updates.ts@@ -15,7 +15,6 @@ export const setLimit =}const query = question.query();const nextQuery = Lib.limit(query, -1, limit);- const nextLegacyQuery = Lib.toLegacyQuery(nextQuery);- const nextQuestion = question.setDatasetQuery(nextLegacyQuery);+ const nextQuestion = question.setQuery(nextQuery);dispatch(updateQuestion(nextQuestion, { run: true }));};diff --git a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditorTopBar/NativeQueryEditorTopBar.tsx b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditorTopBar/NativeQueryEditorTopBar.tsxindex 9e65b3288ad..c8a937e8cf9 100644--- a/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditorTopBar/NativeQueryEditorTopBar.tsx+++ b/frontend/src/metabase/query_builder/components/NativeQueryEditor/NativeQueryEditorTopBar/NativeQueryEditorTopBar.tsx@@ -99,7 +99,6 @@ const NativeQueryEditorTopBar = (props: NativeQueryEditorTopBarProps) => {parameterIndex: number,) => {const newQuery = query.setParameterIndex(parameterId, parameterIndex);-dispatch(updateQuestion(question.setDatasetQuery(newQuery.datasetQuery())));};diff --git a/frontend/src/metabase/query_builder/components/view/QuestionRowCount/QuestionRowCount.unit.spec.tsx b/frontend/src/metabase/query_builder/components/view/QuestionRowCount/QuestionRowCount.unit.spec.tsxindex a5b57306a97..4170ca6dfd4 100644--- a/frontend/src/metabase/query_builder/components/view/QuestionRowCount/QuestionRowCount.unit.spec.tsx+++ b/frontend/src/metabase/query_builder/components/view/QuestionRowCount/QuestionRowCount.unit.spec.tsx@@ -5,6 +5,7 @@ import {setupDatabasesEndpoints,setupUnauthorizedDatabasesEndpoints,} from "__support__/server-mocks";+import { createMockEntitiesState } from "__support__/store";import {fireEvent,renderWithProviders,@@ -48,7 +49,7 @@ function patchQuestion(question: Question) {if (!isNative) {const [sampleColumn] = Lib.orderableColumns(query, 0);const nextQuery = Lib.orderBy(query, 0, sampleColumn);- return question.setDatasetQuery(Lib.toLegacyQuery(nextQuery));+ return question.setQuery(nextQuery);} else {const query = question.legacyNativeQuery() as NativeQuery;return query.setQueryText("SELECT * FROM __ORDERS__").question();@@ -89,7 +90,13 @@ async function setup({});renderWithProviders(<QuestionRowCount />, {- storeInitialState: { qb: state },+ storeInitialState: {+ qb: state,+ entities: createMockEntitiesState({+ databases: isReadOnly ? [] : databases,+ questions: "id" in card ? [card] : [],+ }),+ },});const rowCount = await screen.findByLabelText("Row count");diff --git a/frontend/src/metabase/query_builder/components/view/View/StructuredQueryRightSidebar/StructuredQueryRightSidebar.jsx b/frontend/src/metabase/query_builder/components/view/View/StructuredQueryRightSidebar/StructuredQueryRightSidebar.jsxindex c3377ac2501..30df17e361d 100644--- a/frontend/src/metabase/query_builder/components/view/View/StructuredQueryRightSidebar/StructuredQueryRightSidebar.jsx+++ b/frontend/src/metabase/query_builder/components/view/View/StructuredQueryRightSidebar/StructuredQueryRightSidebar.jsx@@ -5,7 +5,6 @@ import { QuestionInfoSidebar } from "metabase/query_builder/components/view/sideimport { QuestionSettingsSidebar } from "metabase/query_builder/components/view/sidebars/QuestionSettingsSidebar";import { SummarizeSidebar } from "metabase/query_builder/components/view/sidebars/SummarizeSidebar";import TimelineSidebar from "metabase/query_builder/components/view/sidebars/TimelineSidebar";-import * as Lib from "metabase-lib";export const StructuredQueryRightSidebar = ({deselectTimelineEvents,@@ -60,8 +59,7 @@ export const StructuredQueryRightSidebar = ({<SummarizeSidebarquery={question.query()}onQueryChange={(nextQuery) => {- const datesetQuery = Lib.toLegacyQuery(nextQuery);- const nextQuestion = question.setDatasetQuery(datesetQuery);+ const nextQuestion = question.setQuery(nextQuery);updateQuestion(nextQuestion.setDefaultDisplay(), {run: true,});diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionFiltersHeader/QuestionFiltersHeader.unit.spec.tsx b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionFiltersHeader/QuestionFiltersHeader.unit.spec.tsxindex 9fe3bc8dff8..e8dff8eb08b 100644--- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionFiltersHeader/QuestionFiltersHeader.unit.spec.tsx+++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/QuestionFiltersHeader/QuestionFiltersHeader.unit.spec.tsx@@ -47,7 +47,7 @@ function setup({const question = Question.create({metadata,type: "query",- dataset_query: Lib.toLegacyQuery(query),+ dataset_query: Lib.toJsQuery(query),});const handleQueryChange = (question: Question) => {diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/ViewOnly.unit.spec.tsx b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/ViewOnly.unit.spec.tsxindex 486afd20244..50e0bd6f57b 100644--- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/ViewOnly.unit.spec.tsx+++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/ViewOnly.unit.spec.tsx@@ -121,7 +121,7 @@ function createCardFromQuery({}: Partial<Card> & { query: Lib.Query }): Card {return createMockCard({...rest,- dataset_query: Lib.toLegacyQuery(query),+ dataset_query: Lib.toJsQuery(query),});}diff --git a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.tsindex 1f12e41721c..e8ac997d5fd 100644--- a/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts+++ b/frontend/src/metabase/query_builder/components/view/ViewHeader/components/SavedQuestionLeftSide/hooks.ts@@ -12,12 +12,7 @@ export function useHiddenSourceTables(datasetQuery.database,metadata,);- const query = Lib.fromLegacyQuery(- datasetQuery.database,- metadataProvider,- datasetQuery,- );-+ const query = Lib.fromJsQuery(metadataProvider, datasetQuery);const sourceTableId = Lib.sourceTableOrCardId(query);const joinTablesInfo = Lib.stageIndexes(query).flatMap((stageIndex) =>diff --git a/frontend/src/metabase/query_builder/components/view/sidebars/QuestionInfoSidebar/tests/common.unit.spec.tsx b/frontend/src/metabase/query_builder/components/view/sidebars/QuestionInfoSidebar/tests/common.unit.spec.tsxindex 99705cba44e..dacdb2d75c4 100644--- a/frontend/src/metabase/query_builder/components/view/sidebars/QuestionInfoSidebar/tests/common.unit.spec.tsx+++ b/frontend/src/metabase/query_builder/components/view/sidebars/QuestionInfoSidebar/tests/common.unit.spec.tsx@@ -267,7 +267,7 @@ describe("QuestionInfoSidebar", () => {const query = getJoinedQuery();const card = createMockCard({type: "model",- dataset_query: Lib.toLegacyQuery(query),+ dataset_query: Lib.toJsQuery(query),});await setup({ card });await userEvent.click(screen.getByRole("tab", { name: "Relationships" }));diff --git a/frontend/src/metabase/query_builder/reducers-typed.ts b/frontend/src/metabase/query_builder/reducers-typed.tsindex fd81dc49878..545a697a6ed 100644--- a/frontend/src/metabase/query_builder/reducers-typed.ts+++ b/frontend/src/metabase/query_builder/reducers-typed.ts@@ -6,7 +6,7 @@ import {updateCardEmbeddingParams,updateCardEnableEmbedding,} from "metabase/api";-import type { Card, DatasetQuery } from "metabase-types/api";+import type { Card } from "metabase-types/api";import {API_CREATE_QUESTION,@@ -23,135 +23,132 @@ import {} from "./actions";// the card that is actively being worked on-export const card = createReducer<Card<DatasetQuery> | null>(- null,- (builder) => {- builder- .addCase(RESET_QB, () => null)- .addCase(CLOSE_QB, () => null)- .addCase<- string,- {- type: string;- payload: {- card: Card;- };- }- >(INITIALIZE_QB, (state, action) =>- action.payload ? action.payload.card : null,- )- .addCase<- string,- {- type: string;- payload: Card;- }- >(SOFT_RELOAD_CARD, (state, action) => action.payload)- .addCase<- string,- {- type: string;- payload: Card;- }- >(RELOAD_CARD, (state, action) => action.payload)- .addCase<- string,- {- type: string;- payload: {- card: Card;- };- }- >(SET_CARD_AND_RUN, (state, action) => action.payload.card)- .addCase<- string,- {- type: string;- payload: Card;- }- >(API_CREATE_QUESTION, (state, action) => action.payload)- .addCase<- string,- {- type: string;- payload: Card;- }- >(API_UPDATE_QUESTION, (state, action) => action.payload)- .addCase<- string,- {- type: string;- payload: {- card: Card;- };- }- >(CANCEL_QUESTION_CHANGES, (state, action) => action.payload.card)- .addCase<- string,- {- type: string;- payload: {- card: Card;- };- }- >(UPDATE_QUESTION, (state, action) => action.payload.card)- .addCase<- string,- {- type: string;- payload: {- card: Card;- };- }- >(QUERY_COMPLETED, (state, action) => {- if (!state) {- return state;- }- return {- ...state,- display: action.payload.card.display,- result_metadata: action.payload.card.result_metadata,- visualization_settings: action.payload.card.visualization_settings,+export const card = createReducer<Card | null>(null, (builder) => {+ builder+ .addCase(RESET_QB, () => null)+ .addCase(CLOSE_QB, () => null)+ .addCase<+ string,+ {+ type: string;+ payload: {+ card: Card;};- })- .addMatcher(createCardPublicLink.matchFulfilled, (state, action) => {- if (!state) {- return state;- }- return {- ...state,- public_uuid: action.payload.uuid,+ }+ >(INITIALIZE_QB, (state, action) =>+ action.payload ? action.payload.card : null,+ )+ .addCase<+ string,+ {+ type: string;+ payload: Card;+ }+ >(SOFT_RELOAD_CARD, (state, action) => action.payload)+ .addCase<+ string,+ {+ type: string;+ payload: Card;+ }+ >(RELOAD_CARD, (state, action) => action.payload)+ .addCase<+ string,+ {+ type: string;+ payload: {+ card: Card;};- })- .addMatcher(deleteCardPublicLink.matchFulfilled, (state) => {- if (!state) {- return state;- }-- return {- ...state,- public_uuid: null,+ }+ >(SET_CARD_AND_RUN, (state, action) => action.payload.card)+ .addCase<+ string,+ {+ type: string;+ payload: Card;+ }+ >(API_CREATE_QUESTION, (state, action) => action.payload)+ .addCase<+ string,+ {+ type: string;+ payload: Card;+ }+ >(API_UPDATE_QUESTION, (state, action) => action.payload)+ .addCase<+ string,+ {+ type: string;+ payload: {+ card: Card;};- })- .addMatcher(updateCardEnableEmbedding.matchFulfilled, (state, action) => {- if (!state) {- return state;- }- return {- ...state,- enable_embedding: action.payload.enable_embedding,+ }+ >(CANCEL_QUESTION_CHANGES, (state, action) => action.payload.card)+ .addCase<+ string,+ {+ type: string;+ payload: {+ card: Card;};- })- .addMatcher(updateCardEmbeddingParams.matchFulfilled, (state, action) => {- if (!state) {- return state;- }-- return {- ...state,- embedding_params: action.payload.embedding_params,- initially_published_at: action.payload.initially_published_at,+ }+ >(UPDATE_QUESTION, (state, action) => action.payload.card)+ .addCase<+ string,+ {+ type: string;+ payload: {+ card: Card;};- });- },-);+ }+ >(QUERY_COMPLETED, (state, action) => {+ if (!state) {+ return state;+ }+ return {+ ...state,+ display: action.payload.card.display,+ result_metadata: action.payload.card.result_metadata,+ visualization_settings: action.payload.card.visualization_settings,+ };+ })+ .addMatcher(createCardPublicLink.matchFulfilled, (state, action) => {+ if (!state) {+ return state;+ }+ return {+ ...state,+ public_uuid: action.payload.uuid,+ };+ })+ .addMatcher(deleteCardPublicLink.matchFulfilled, (state) => {+ if (!state) {+ return state;+ }++ return {+ ...state,+ public_uuid: null,+ };+ })+ .addMatcher(updateCardEnableEmbedding.matchFulfilled, (state, action) => {+ if (!state) {+ return state;+ }+ return {+ ...state,+ enable_embedding: action.payload.enable_embedding,+ };+ })+ .addMatcher(updateCardEmbeddingParams.matchFulfilled, (state, action) => {+ if (!state) {+ return state;+ }++ return {+ ...state,+ embedding_params: action.payload.embedding_params,+ initially_published_at: action.payload.initially_published_at,+ };+ });+});diff --git a/frontend/src/metabase/query_builder/selectors.js b/frontend/src/metabase/query_builder/selectors.jsindex 570c8d120c4..f6a6e0d6e26 100644--- a/frontend/src/metabase/query_builder/selectors.js+++ b/frontend/src/metabase/query_builder/selectors.js@@ -219,10 +219,6 @@ const getLastRunDatasetQuery = createSelector([getLastRunCard],(card) => card && card.dataset_query,);-export const getNextRunDatasetQuery = createSelector(- [getCard],- (card) => card && card.dataset_query,-);const getLastRunParameters = createSelector([getFirstQueryResult],@@ -879,22 +875,25 @@ export function getOffsetForQueryAndPosition(queryText, { row, column }) {}export const getNativeEditorCursorOffset = createSelector(- [getNativeEditorSelectedRange, getNextRunDatasetQuery],- (selectedRange, query) => {- if (selectedRange == null || query == null || query.native == null) {+ [getNativeEditorSelectedRange, getQuestionWithoutComposing],+ (selectedRange, question) => {+ if (selectedRange == null || question == null || !question.isNative()) {return null;}- return getOffsetForQueryAndPosition(query.native.query, selectedRange.end);+ const query = question.query();+ const queryText = Lib.rawNativeQuery(query);+ return getOffsetForQueryAndPosition(queryText, selectedRange.end);},);export const getNativeEditorSelectedText = createSelector(- [getNativeEditorSelectedRange, getNextRunDatasetQuery],- (selectedRange, query) => {- if (selectedRange == null || query == null || query.native == null) {+ [getNativeEditorSelectedRange, getQuestionWithoutComposing],+ (selectedRange, question) => {+ if (selectedRange == null || question == null || !question.isNative()) {return null;}- const queryText = query.native.query;+ const query = question.query();+ const queryText = Lib.rawNativeQuery(query);const start = getOffsetForQueryAndPosition(queryText, selectedRange.start);const end = getOffsetForQueryAndPosition(queryText, selectedRange.end);return queryText.slice(start, end);@@ -902,17 +901,18 @@ export const getNativeEditorSelectedText = createSelector();export const getAllNativeEditorSelectedText = createSelector(- [getNativeEditorSelectedRanges, getNextRunDatasetQuery],- (selectedRanges, query) => {+ [getNativeEditorSelectedRanges, getQuestionWithoutComposing],+ (selectedRanges, question) => {if (selectedRanges == null ||selectedRanges.length === 0 ||- query == null ||- query.native == null+ question == null ||+ !question.isNative()) {return null;}- const queryText = query.native.query;+ const query = question.query();+ const queryText = Lib.rawNativeQuery(query);const selectedText = selectedRanges.map((range) =>queryText.slice(getOffsetForQueryAndPosition(queryText, range.start),diff --git a/frontend/src/metabase/query_builder/selectors.unit.spec.js b/frontend/src/metabase/query_builder/selectors.unit.spec.jsindex cb48438aec4..1bf674d1ccc 100644--- a/frontend/src/metabase/query_builder/selectors.unit.spec.js+++ b/frontend/src/metabase/query_builder/selectors.unit.spec.js@@ -1,4 +1,4 @@-import { assoc, assocIn } from "icepick";+import { assoc } from "icepick";import { createMockEntitiesState } from "__support__/store";import {@@ -10,6 +10,7 @@ import {getQuestionDetailsTimelineDrawerState,} from "metabase/query_builder/selectors";import registerVisualizations from "metabase/visualizations/register";+import * as Lib from "metabase-lib";import Question from "metabase-lib/v1/Question";import {createMockCard,@@ -103,9 +104,7 @@ describe("getQuestion", () => {const question = getQuestion(getBaseState({ card }));- expect(question.card()).toEqual(- assocIn(card, ["dataset_query", "query", "source-table"], "card__1"),- );+ expect(Lib.sourceTableOrCardId(question.query())).toBe("card__1");});it("should return real dataset when dataset is open in 'dataset' QB mode", () => {diff --git a/frontend/src/metabase/querying/notebook/components/NativeQueryPreview/NativeQueryPreview.tsx b/frontend/src/metabase/querying/notebook/components/NativeQueryPreview/NativeQueryPreview.tsxindex 176b72c4b77..3451c8f6734 100644--- a/frontend/src/metabase/querying/notebook/components/NativeQueryPreview/NativeQueryPreview.tsx+++ b/frontend/src/metabase/querying/notebook/components/NativeQueryPreview/NativeQueryPreview.tsx@@ -25,7 +25,7 @@ export function NativeQueryPreview({parameters?: UiParameter[];}) {const { data, error, isFetching } = useGetNativeDatasetQuery({- ...Lib.toLegacyQuery(query),+ ...Lib.toJsQuery(query),parameters,});diff --git a/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx b/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsxindex 2380a725f5b..42471146fa0 100644--- a/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx+++ b/frontend/src/metabase/querying/notebook/components/NotebookNativePreview/NotebookNativePreview.tsx@@ -44,7 +44,7 @@ export const NotebookNativePreview = ({const sourceQuery = question.query();const canRun = Lib.canRun(sourceQuery, question.type());- const payload = Lib.toLegacyQuery(sourceQuery);+ const payload = Lib.toJsQuery(sourceQuery);const { data, error, isFetching } = useGetNativeDatasetQuery(payload);const showLoader = isFetching;diff --git a/frontend/src/metabase/querying/segments/components/SegmentEditor/PreviewStep/PreviewStep.tsx b/frontend/src/metabase/querying/segments/components/SegmentEditor/PreviewStep/PreviewStep.tsxindex 67a1c802021..3ef542bfada 100644--- a/frontend/src/metabase/querying/segments/components/SegmentEditor/PreviewStep/PreviewStep.tsx+++ b/frontend/src/metabase/querying/segments/components/SegmentEditor/PreviewStep/PreviewStep.tsx@@ -33,13 +33,11 @@ function PreviewQuery({ query, stageIndex }: PreviewQueryProps) {() => Lib.aggregateByCount(query, stageIndex),[query, stageIndex],);- const { data, isFetching } = useGetAdhocQueryQuery(- Lib.toLegacyQuery(countQuery),- );+ const { data, isFetching } = useGetAdhocQueryQuery(Lib.toJsQuery(countQuery));const count = data?.data?.rows?.[0]?.[0];const previewUrl = Urls.newQuestion({- dataset_query: Lib.toLegacyQuery(query),+ dataset_query: Lib.toJsQuery(query),});return (diff --git a/frontend/src/metabase/reference/utils.unit.spec.js b/frontend/src/metabase/reference/utils.unit.spec.jsindex 3c409e410ce..52105462534 100644--- a/frontend/src/metabase/reference/utils.unit.spec.js+++ b/frontend/src/metabase/reference/utils.unit.spec.js@@ -1,6 +1,8 @@import { createMockMetadata } from "__support__/metadata";import { separateTablesBySchema } from "metabase/reference/databases/TableList";import { getQuestion } from "metabase/reference/utils";+import * as Lib from "metabase-lib";+import Question from "metabase-lib/v1/Question";import {createMockDatabase,createMockField,@@ -9,6 +11,8 @@ import {} from "metabase-types/api/mocks";describe("Reference utils.js", () => {+ const stageIndex = 0;+describe("tablesToSchemaSeparatedTables()", () => {it("should add schema separator to appropriate locations and sort tables by name", () => {const tables = {@@ -56,79 +60,43 @@ describe("Reference utils.js", () => {const database = createMockDatabase({ id: dbId, tables: [table] });const metadata = createMockMetadata({ databases: [database] });- const getNewQuestion = ({- display = "table",- aggregation,- breakout,- filter,- } = {}) => {- const card = {- name: undefined,- collection_id: undefined,- display,- visualization_settings: {},- dataset_query: {- database: database.id,- type: "query",- query: {- "source-table": tableId,- },- },- };- if (aggregation != null) {- card.dataset_query.query.aggregation = aggregation;- }- if (breakout != null) {- card.dataset_query.query.breakout = breakout;- }- if (filter != null) {- card.dataset_query.query.filter = filter;- }- return card;- };-it("should generate correct question for table raw data", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,metadata,});- expect(question).toMatchObject(getNewQuestion());+ const query = new Question(card).query();+ expect(Lib.sourceTableOrCardId(query)).toBe(tableId);});it("should generate correct question for table counts", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,getCount: true,metadata,});- expect(question).toMatchObject(- getNewQuestion({- aggregation: [["count"]],- }),- );+ const query = new Question(card).query();+ expect(Lib.aggregations(query, stageIndex)).toHaveLength(1);});it("should generate correct question for field raw data", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,fieldId,metadata,});- expect(question).toMatchObject(- getNewQuestion({- breakout: [["field", fieldId, { "base-type": "type/Text" }]],- }),- );+ const query = new Question(card).query();+ expect(Lib.breakouts(query, stageIndex)).toHaveLength(1);});it("should generate correct question for field group by bar chart", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,fieldId,@@ -137,17 +105,14 @@ describe("Reference utils.js", () => {metadata,});- expect(question).toMatchObject(- getNewQuestion({- display: "bar",- breakout: [["field", fieldId, { "base-type": "type/Text" }]],- aggregation: [["count"]],- }),- );+ const query = new Question(card).query();+ expect(card.display).toBe("bar");+ expect(Lib.aggregations(query, stageIndex)).toHaveLength(1);+ expect(Lib.breakouts(query, stageIndex)).toHaveLength(1);});it("should generate correct question for field group by pie chart", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,fieldId,@@ -156,32 +121,26 @@ describe("Reference utils.js", () => {metadata,});- expect(question).toMatchObject(- getNewQuestion({- display: "pie",- breakout: [["field", fieldId, { "base-type": "type/Text" }]],- aggregation: [["count"]],- }),- );+ const query = new Question(card).query();+ expect(card.display).toBe("pie");+ expect(Lib.aggregations(query, stageIndex)).toHaveLength(1);+ expect(Lib.breakouts(query, stageIndex)).toHaveLength(1);});it("should generate correct question for segment raw data", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,segmentId,metadata,});- expect(question).toMatchObject(- getNewQuestion({- filter: ["segment", segmentId],- }),- );+ const query = new Question(card).query();+ expect(Lib.filters(query, stageIndex)).toHaveLength(1);});it("should generate correct question for segment counts", () => {- const question = getQuestion({+ const card = getQuestion({dbId,tableId,segmentId,@@ -189,12 +148,9 @@ describe("Reference utils.js", () => {metadata,});- expect(question).toMatchObject(- getNewQuestion({- aggregation: [["count"]],- filter: ["segment", segmentId],- }),- );+ const query = new Question(card).query();+ expect(Lib.filters(query, stageIndex)).toHaveLength(1);+ expect(Lib.aggregations(query, stageIndex)).toHaveLength(1);});});});diff --git a/frontend/src/metabase/visualizations/components/ObjectDetail/ObjectDetailView.tsx b/frontend/src/metabase/visualizations/components/ObjectDetail/ObjectDetailView.tsxindex 30207bd3f0a..7dfdfffd74e 100644--- a/frontend/src/metabase/visualizations/components/ObjectDetail/ObjectDetailView.tsx+++ b/frontend/src/metabase/visualizations/components/ObjectDetail/ObjectDetailView.tsx@@ -198,7 +198,7 @@ export function ObjectDetailView({const pkField = passedData.cols[pkIndex];const query = question?.query();const datasetQuery = query- ? Lib.toLegacyQuery(filterByPk(query, pkField, zoomedRowID))+ ? Lib.toJsQuery(filterByPk(query, pkField, zoomedRowID)): undefined;MetabaseApi.dataset(datasetQuery)diff --git a/frontend/src/metabase/visualizations/components/Visualization/Visualization.tsx b/frontend/src/metabase/visualizations/components/Visualization/Visualization.tsxindex 1af09a0c695..8d79e5e25bf 100644--- a/frontend/src/metabase/visualizations/components/Visualization/Visualization.tsx+++ b/frontend/src/metabase/visualizations/components/Visualization/Visualization.tsx@@ -69,7 +69,6 @@ import type {CardId,Dashboard,DashboardCard,- DatasetQuery,RawSeries,Series,SingleSeries,@@ -152,7 +151,7 @@ type VisualizationOwnProps = {rawSeries?: (| SingleSeries| {- card: Card<DatasetQuery>;+ card: Card;})[];visualizerRawSeries?: RawSeries;@@ -226,8 +225,9 @@ const isLoading = (series: Series | null) => {const deriveStateFromProps = (props: VisualizationProps) => {const rawSeriesArray = props.rawSeries || [];const firstCard = rawSeriesArray[0]?.card;- const isNative = firstCard?.dataset_query?.type === "native";- const isNativeView = isNative && props.queryBuilderMode === "view";+ const firstQuestion = firstCard != null ? new Question(firstCard) : undefined;+ const isNativeView =+ props.queryBuilderMode === "view" && firstQuestion?.isNative();const transformed = props.rawSeries? getVisualizationTransformed(diff --git a/frontend/src/metabase/visualizations/visualizations/Heading/Heading.unit.spec.tsx b/frontend/src/metabase/visualizations/visualizations/Heading/Heading.unit.spec.tsxindex 4fc416bfe99..5b997218552 100644--- a/frontend/src/metabase/visualizations/visualizations/Heading/Heading.unit.spec.tsx+++ b/frontend/src/metabase/visualizations/visualizations/Heading/Heading.unit.spec.tsx@@ -93,7 +93,7 @@ describe("Text", () => {});it("should replace mapped variables with parameter values", () => {- const variableName = "variable";+ const variableName = "foo_bar";const text = `Variable: {{${variableName}}}`;const parameterValue = 15;@@ -322,7 +322,7 @@ function mapParameterToVariable({}) {const parameter: Parameter = {id: "e7f8ca",- name: "foo bar",+ name: "foo_bar",slug: "foo_bar",type: "text",value: parameterValue,diff --git a/frontend/src/metabase/visualizations/visualizations/SankeyChart/events.ts b/frontend/src/metabase/visualizations/visualizations/SankeyChart/events.tsindex 41276c4323f..d1cf43b053e 100644--- a/frontend/src/metabase/visualizations/visualizations/SankeyChart/events.ts+++ b/frontend/src/metabase/visualizations/visualizations/SankeyChart/events.ts@@ -15,6 +15,7 @@ import type {VisualizationProps,} from "metabase/visualizations/types";import type { EChartsEventHandler } from "metabase/visualizations/types/echarts";+import Question from "metabase-lib/v1/Question";import { getColumnKey } from "metabase-lib/v1/queries/utils/column-key";import type {Card,@@ -49,7 +50,10 @@ const isSankeyNodeEvent = (event: EChartsSeriesMouseEvent<SankeyLink | SankeyNode>,): event is EChartsSeriesMouseEvent<SankeyNode> => event.dataType === "node";-const isNativeQuery = (card: Card) => card.dataset_query?.type === "native";+const isNativeQuery = (card: Card) => {+ const question = new Question(card);+ return question.isNative();+};export const createSankeyClickData = (event: EChartsSeriesMouseEvent<SankeyLink | SankeyNode>,diff --git a/frontend/src/metabase/visualizations/visualizations/SmartScalar/compute.ts b/frontend/src/metabase/visualizations/visualizations/SmartScalar/compute.tsindex 0497736ff39..43bbcf12e41 100644--- a/frontend/src/metabase/visualizations/visualizations/SmartScalar/compute.ts+++ b/frontend/src/metabase/visualizations/visualizations/SmartScalar/compute.ts@@ -17,11 +17,12 @@ import {formatPreviousPeriodOptionName,} from "metabase/visualizations/visualizations/SmartScalar/utils";import type { ClickObject } from "metabase-lib";+import Question from "metabase-lib/v1/Question";import { isDate } from "metabase-lib/v1/types/utils/isa";import type {DatasetColumn,- DatasetQuery,DateTimeAbsoluteUnit,+ LegacyDatasetQuery,RowValue,RowValues,Series,@@ -52,7 +53,7 @@ interface DateUnitSettings {dateColumn: DatasetColumn;dateColumnSettings: ColumnSettings;dateUnit?: DateTimeAbsoluteUnit;- queryType: DatasetQuery["type"];+ queryType: LegacyDatasetQuery["type"];}interface MetricData {@@ -229,9 +230,7 @@ function getCurrentMetricData({}): MetricData {const [{- card: {- dataset_query: { type: queryType },- },+ card,data: { rows, cols },},] = series;@@ -278,11 +277,12 @@ function getCurrentMetricData({dateColumnWithUnit.unit ??= dateUnit;const dateColumnSettings = settings?.column?.(dateColumnWithUnit) ?? {};+ const question = new Question(card);const dateUnitSettings: DateUnitSettings = {dateColumn: dateColumnWithUnit,dateColumnSettings,dateUnit,- queryType,+ queryType: question.isNative() ? "native" : "query",};const formatOptions = {diff --git a/frontend/test/metabase-lib/lib/Question.unit.spec.js b/frontend/test/metabase-lib/lib/Question.unit.spec.jsindex 0200adec9f6..7c2fbe37db3 100644--- a/frontend/test/metabase-lib/lib/Question.unit.spec.js+++ b/frontend/test/metabase-lib/lib/Question.unit.spec.js@@ -1,4 +1,4 @@-import { assoc, assocIn, dissoc } from "icepick";+import { assoc, dissoc } from "icepick";import { parse } from "url";import { createMockMetadata } from "__support__/metadata";@@ -37,6 +37,8 @@ const metadata = createMockMetadata({databases: [createSampleDatabase()],});+const metadataProvider = Lib.metadataProvider(SAMPLE_DB_ID, metadata);+const metadata_without_order_pk = createMockMetadata({databases: [createSampleDatabase({@@ -587,7 +589,7 @@ describe("Question", () => {describe("URLs", () => {const adhocUrl =- "/question#eyJkYXRhc2V0X3F1ZXJ5Ijp7ImRhdGFiYXNlIjoxLCJxdWVyeSI6eyJzb3VyY2UtdGFibGUiOjJ9LCJ0eXBlIjoicXVlcnkifSwiZGlzcGxheSI6InRhYmxlIiwibmFtZSI6IlJhdyBvcmRlcnMgZGF0YSIsInZpc3VhbGl6YXRpb25fc2V0dGluZ3MiOnt9fQ==";+ "/question#eyJkYXRhc2V0X3F1ZXJ5Ijp7ImRhdGFiYXNlIjoxLCJsaWIvdHlwZSI6Im1icWwvcXVlcnkiLCJzdGFnZXMiOlt7ImxpYi90eXBlIjoibWJxbC5zdGFnZS9tYnFsIiwic291cmNlLXRhYmxlIjoyfV19LCJkaXNwbGF5IjoidGFibGUiLCJuYW1lIjoiUmF3IG9yZGVycyBkYXRhIiwidmlzdWFsaXphdGlvbl9zZXR0aW5ncyI6e319";// Covered a lot in query_builder/actions.spec.js, just very basic cases here// (currently getUrl has logic that is strongly tied to the logic query builder Redux actions)@@ -815,19 +817,7 @@ describe("Question", () => {});expect(Lib.stageCount(questionWithFilters.query())).toBe(1);- expect(questionWithFilters.datasetQuery().query.filter).toEqual([- "starts-with",- [- "field",- PRODUCTS.CATEGORY,- {- "base-type": "type/Text",- "source-field": ORDERS.PRODUCT_ID,- },- ],- "abc",- { "case-sensitive": false },- ]);+ expect(Lib.filters(questionWithFilters.query())).toHaveLength(1);});});@@ -867,13 +857,15 @@ describe("Question", () => {const card = {id: 1,- dataset_query: {- type: "query",- query: {- "source-table": PRODUCTS_ID,- },- database: SAMPLE_DB_ID,- },+ dataset_query: Lib.toJsQuery(+ Lib.fromJsQuery(metadataProvider, {+ type: "query",+ query: {+ "source-table": PRODUCTS_ID,+ },+ database: SAMPLE_DB_ID,+ }),+ ),};describe("with structured card", () => {@@ -908,20 +900,14 @@ describe("Question", () => {},);- const deserializedCard = {- ...assocIn(- dissoc(card, "id"),- ["dataset_query", "query", "filter"],- ["=", ["field", 1, { "base-type": "type/Text" }], "bar"],- ),- original_card_id: card.id,- };-- expect(parseUrl(url)).toEqual({- pathname: "/question",- query: {},- card: deserializedCard,- });+ const parsedUrl = parseUrl(url);+ const parsedQuestion = new Question(+ parsedUrl.card,+ question.metadata(),+ );+ expect(parsedUrl.pathname).toEqual("/question");+ expect(parsedUrl.query).toEqual({});+ expect(Lib.filters(parsedQuestion.query(), -1)).toHaveLength(1);});it("should return question URL with number MBQL filter added", () => {@@ -934,18 +920,12 @@ describe("Question", () => {},);- expect(parseUrl(url)).toEqual({- pathname: "/question",- query: {},- card: {- ...assocIn(- dissoc(card, "id"),- ["dataset_query", "query", "filter"],- ["=", ["field", 2, { "base-type": "type/Float" }], 123],- ),- original_card_id: card.id,- },- });+ const parsedUrl = parseUrl(url);+ const parsedQuestion = new Question(+ parsedUrl.card,+ question.metadata(),+ );+ expect(Lib.filters(parsedQuestion.query(), -1)).toHaveLength(1);});it("should return question URL with date MBQL filter added", () => {@@ -958,27 +938,12 @@ describe("Question", () => {},);- expect(parseUrl(url)).toEqual({- pathname: "/question",- query: {},- card: {- ...assocIn(- dissoc(card, "id"),- ["dataset_query", "query", "filter"],- [- "between",- [- "field",- PRODUCTS.CREATED_AT,- { "base-type": "type/DateTime" },- ],- "2017-05-01",- "2017-05-31",- ],- ),- original_card_id: card.id,- },- });+ const parsedUrl = parseUrl(url);+ const parsedQuestion = new Question(+ parsedUrl.card,+ question.metadata(),+ );+ expect(Lib.filters(parsedQuestion.query(), -1)).toHaveLength(1);});it("should include objectId in a URL", () => {@@ -1256,6 +1221,7 @@ describe("Question", () => {describe("Question.generateQueryDescription", () => {it("should work with multiple aggregations", () => {const question = base_question.setDatasetQuery({+ type: "query",database: SAMPLE_DB_ID,query: {"source-table": ORDERS_ID,@@ -1269,6 +1235,7 @@ describe("Question", () => {it("should work with named aggregations", () => {const question = base_question.setDatasetQuery({+ type: "query",database: SAMPLE_DB_ID,query: {"source-table": ORDERS_ID,diff --git a/frontend/test/metabase-lib/lib/queries/NativeQuery.unit.spec.js b/frontend/test/metabase-lib/lib/queries/NativeQuery.unit.spec.jsindex 6b2e9c1adcc..7bf11ea3c32 100644--- a/frontend/test/metabase-lib/lib/queries/NativeQuery.unit.spec.js+++ b/frontend/test/metabase-lib/lib/queries/NativeQuery.unit.spec.js@@ -1,5 +1,3 @@-import { assocIn } from "icepick";-import { createMockMetadata } from "__support__/metadata";import Question from "metabase-lib/v1/Question";import NativeQuery, {@@ -218,47 +216,32 @@ describe("NativeQuery", () => {});it("requires a display name", () => {- q = q.setDatasetQuery(- assocIn(q.datasetQuery(), ["native", "template-tags", "foo"], {- name: "foo",- type: "text",- }),- );-+ q = q.setTemplateTag("foo", { name: "foo", type: "text" });expect(q.canRun()).toBe(false);- q = q.setDatasetQuery(- assocIn(q.datasetQuery(), ["native", "template-tags", "foo"], {- name: "foo",- type: "text",- "display-name": "Foo",- }),- );-+ q = q.setTemplateTag("foo", {+ name: "foo",+ type: "text",+ "display-name": "Foo",+ });expect(q.canRun()).toBe(true);});it("dimension type without a dimension", () => {- q = q.setDatasetQuery(- assocIn(q.datasetQuery(), ["native", "template-tags", "foo"], {- type: "dimension",- "widget-type": "category",- "display-name": "bar",- }),- );-+ q = q.setTemplateTag("foo", {+ type: "dimension",+ "widget-type": "category",+ "display-name": "bar",+ });expect(q.canRun()).toBe(false);- q = q.setDatasetQuery(- assocIn(q.datasetQuery(), ["native", "template-tags", "foo"], {- name: "foo",- type: "dimension",- "widget-type": "category",- dimension: ["field", 123, null],- "display-name": "bar",- }),- );-+ q = q.setTemplateTag("foo", {+ name: "foo",+ type: "dimension",+ "widget-type": "category",+ dimension: ["field", 123, null],+ "display-name": "bar",+ });expect(q.canRun()).toBe(true);});});diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 01585a6e792..05d695c9971 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -993,9 +993,10 @@[_]nil)-(defmethod driver/native-query-deps :bigquery-cloud-sdk- [driver query]- (let [db-tables (driver-api/tables (driver-api/metadata-provider))+(mu/defmethod driver/native-query-deps :bigquery-cloud-sdk :- ::driver/native-query-deps+ [driver :- :keyword+ query :- :metabase.lib.schema/native-only-query]+ (let [db-tables (driver-api/tables query)transforms (t2/select [:model/Transform :id :target])](into #{} (comp(map :component)@@ -1004,6 +1005,7 @@{:schema (first parts) :table (second parts)}))(keep #(driver.sql/find-table-or-transform driver db-tables transforms %)))(-> query+ driver-api/raw-native-querymacaw/parsed-querymacaw/query->components:tables))))diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.cljindex 11a520a9bb6..3b2ea883a58 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk/query_processor.clj@@ -9,7 +9,7 @@[metabase.driver-api.core :as driver-api][metabase.driver.bigquery-cloud-sdk.common :as bigquery.common][metabase.driver.common :as driver.common]- [metabase.driver.common.parameters]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.util :as sql.u]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.cljindex 56730c1037d..4dbd8757d9e 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test.clj@@ -16,7 +16,7 @@[metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as add][metabase.sync.core :as sync][metabase.test :as mt]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.cljindex 8250b3534f6..bb4eb831b78 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk/query_processor_test/reconciliation_test_util.clj@@ -8,7 +8,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.util.match :as lib.util.match]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.query-processor.util.add-alias-info :as add][metabase.test :as mt]diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.cljindex e9a381131ab..3e5131efe1a 100644--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj@@ -12,7 +12,7 @@[metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile][metabase.query-processor.pipeline :as qp.pipeline]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.core :as sync][metabase.test :as mt][metabase.test.data.bigquery-cloud-sdk :as bigquery.tx]diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.cljindex 7d386a1f30d..c3aa895c636 100644--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_impersonation_test.clj@@ -9,7 +9,7 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.test-util :as driver.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.core :as sync.core][metabase.test :as mt][metabase.test.data.clickhouse :as ctd]diff --git a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj b/modules/drivers/databricks/test/metabase/driver/databricks_test.cljindex 1863610aa61..7b5dcceae81 100644--- a/modules/drivers/databricks/test/metabase/driver/databricks_test.clj+++ b/modules/drivers/databricks/test/metabase/driver/databricks_test.clj@@ -10,7 +10,7 @@[metabase.driver :as driver][metabase.driver.databricks :as databricks][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp]@@ -431,7 +431,7 @@(mt/with-temp [:model/Database {db-id :id :as db} {:engine :databricks :details details}](mt/with-db db(sync/sync-database! (mt/db) {:scan :schema})- (let [mp (lib.metadata.jvm/application-database-metadata-provider db-id)+ (let [mp (lib-be/application-database-metadata-provider db-id)[t1-id t2-id] (t2/select-fn-vec:id:model/Tablediff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.cljindex 263572020b4..ed2fb6290de 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj@@ -514,8 +514,9 @@(defmethod driver/table-rows-sample :mongo[_driver table fields rff opts](driver-api/with-metadata-provider (:db_id table)- (let [mongo-opts {:limit table-rows-sample/nested-field-sample-limit- :order-by [[:desc [:field (get-id-field-id table) nil]]]}]+ (let [id-column (driver-api/field (driver-api/metadata-provider) (get-id-field-id table))+ mongo-opts {:limit table-rows-sample/nested-field-sample-limit+ :order-by [(driver-api/order-by-clause id-column :desc)]}](table-rows-sample/table-rows-sample table fields rff (merge mongo-opts opts)))))(defn- encode-mongodiff --git a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.cljindex 5ae37fa6a6e..cd5ea534be6 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/parameters.clj@@ -3,11 +3,11 @@[clojure.string :as str][java-time.api :as t][metabase.driver-api.core :as driver-api]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.dates :as params.dates]- [metabase.driver.common.parameters.operators :as params.ops]- [metabase.driver.common.parameters.parse :as params.parse]- [metabase.driver.common.parameters.values :as params.values]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.dates :as params.dates]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.operators :as params.ops]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.values :as params.values][metabase.driver.mongo.query-processor :as mongo.qp][metabase.util :as u][metabase.util.date-2 :as u.date]diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.cljindex dfd39cb2e71..09264a5c11a 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/parameters_test.clj@@ -4,7 +4,7 @@[clojure.string :as str][clojure.test :refer :all][java-time.api :as t]- [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params][metabase.driver.mongo.parameters :as mongo.params][metabase.query-processor :as qp][metabase.test :as mt]diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.cljindex e2271ca2f58..0cce0c1f636 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj@@ -117,7 +117,7 @@(mt/with-clock #t "2021-02-15T17:33:00-08:00[US/Pacific]"(mt/dataset attempted-murders(testing "should still work even with bucketing bucketing"- (let [tz (qp.timezone/results-timezone-id :mongo mt/db)+ (let [tz (qp.timezone/results-timezone-id :mongo (mt/db))query (mt/with-metadata-provider (mt/id)(qp.compile/compile(mt/mbql-query attempts@@ -406,7 +406,7 @@{"$group"{"_id"{"date"- (let [tz (qp.timezone/results-timezone-id :mongo mt/db)]+ (let [tz (qp.timezone/results-timezone-id :mongo (mt/db))](if (date-arithmetic-supported?){:$dateTrunc {:date "$date":startOfWeek "sunday"diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.cljindex 43bfe4831c9..de735f608f8 100644--- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj+++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj@@ -22,7 +22,7 @@[metabase.query-processor-test.order-by-test :as qp-test.order-by-test][metabase.query-processor.compile :as qp.compile][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.core :as sync][metabase.sync.util :as sync-util][metabase.test :as mt]diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.cljindex 8691be2d24a..37334e091b8 100644--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj@@ -9,7 +9,7 @@[java-time.api :as t][medley.core :as m][metabase.driver :as driver]- [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params][metabase.driver.snowflake :as driver.snowflake][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]@@ -26,7 +26,7 @@[metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.secrets.core :as secret][metabase.sync.core :as sync][metabase.sync.fetch-metadata :as fetch-metadata]diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex ca8d7b07ff4..da7f473c54b 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -22,10 +22,12 @@[metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison][metabase.driver.sql.util :as sql.u]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.json :as json][metabase.util.log :as log]+ [metabase.util.malli :as mu][metabase.util.performance :as perf :refer [mapv]])(:import(java.sql Connection DatabaseMetaData PreparedStatement ResultSet Time)@@ -58,9 +60,9 @@:jdbc/statements false}](defmethod driver/database-supports? [:sqlserver feature] [_driver _feature _db] supported?))-(defmethod driver/database-supports? [:sqlserver :percentile-aggregations]- [_ _ db]- (let [major-version (get-in db [:dbms_version :semantic-version 0] 0)]+(mu/defmethod driver/database-supports? [:sqlserver :percentile-aggregations]+ [_ _ db :- ::lib.schema.metadata/database]+ (let [major-version (get-in db [:dbms-version :semantic-version 0] 0)](when (zero? major-version)(log/warn "Unable to determine sqlserver's dbms major version. Fallback to 0."))(>= major-version 16)))diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.cljindex 2e5e6ff1cb7..e25915fcc8f 100644--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj@@ -19,7 +19,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.limit :as limit][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.query-processor.timezone :as qp.timezone][metabase.test :as mt]diff --git a/package.json b/package.jsonindex 5d1b76498e8..a17996bfade 100644--- a/package.json+++ b/package.json@@ -427,7 +427,7 @@"clean-dev:js": "rm -rf resources/frontend_client/app/dist target/classes/frontend_client/app/dist","clean-release:cljs": "rm -rf target/cljs_release/*","clean:cljs": "yarn && yarn clean-dev:cljs && yarn clean-release:cljs",- "cljs-nrepl": "yarn && shadow-cljs node-repl -d cider/cider-nrepl:0.45.0 -d cider/piggieback:0.5.3 -d refactor-nrepl/refactor-nrepl:3.9.1",+ "cljs-nrepl": "yarn && shadow-cljs node-repl -d cider/cider-nrepl:0.57.0 -d cider/piggieback:0.6.1 -d refactor-nrepl/refactor-nrepl:3.11.0","cljs-server-restart": "yarn && shadow-cljs restart","cljs-server-start": "yarn && shadow-cljs start","cljs-server-stop": "yarn && shadow-cljs stop",diff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yamlindex eed31a70b25..f33276ac33e 100644--- a/resources/migrations/057_update_migrations.yaml+++ b/resources/migrations/057_update_migrations.yaml@@ -1902,6 +1902,71 @@ databaseChangeLog:path: instance_analytics_views/view_log/v2/mysql-view_log.sqlrelativeToChangelogFile: true+ - changeSet:+ id: v57.2025-10-06T18:26:00+ author: camsaul+ comment: >-+ In v57, we started persisting MBQL 5 in the app DB; to support rollbacks to v56 and older (which don't work+ 100% correctly with MBQL 5 in the app DB) I'm copying the old legacy query to a new column. Note that new+ Cards created in 57 won't populate this column.+ changes:+ - addColumn:+ tableName: report_card+ columns:+ - column:+ name: legacy_query+ type: ${text.type}+ remarks: >-+ Legacy MBQL version of the query (serialized as JSON) for existing Cards created before v57, to+ support rollbacks to v56. This column should be removed in v58.++ - changeSet:+ id: v57.2025-10-06T18:26:01+ author: camsaul+ comment: >-+ Copy legacy MBQL queries from report_card.dataset_query to report_card.legacy_query to support rollbacks+ changes:+ - sql: >-+ UPDATE report_card+ SET legacy_query = dataset_query;+ rollback:+ - sql: >-+ UPDATE report_card+ SET dataset_query = legacy_query+ WHERE legacy_query IS NOT NULL;++ - changeSet:+ id: v57.2025-10-06T18:26:02+ author: camsaul+ comment: >-+ Copy legacy MBQL queries from Actions v1 query_action.dataset_query to query_action.legacy_query+ changes:+ - addColumn:+ tableName: query_action+ columns:+ - column:+ name: legacy_query+ type: ${text.type}+ remarks: >-+ Legacy MBQL version of the query (serialized as JSON) for existing Query Actions created before+ v57, to support rollbacks to v56. This column should be removed in v58.++ - changeSet:+ id: v57.2025-10-06T18:26:03+ author: camsaul+ comment: >-+ Copy legacy MBQL queries from Actions v1 query_action.dataset_query to query_action.legacy_query to support rollbacks+ changes:+ - sql: >-+ UPDATE query_action+ SET legacy_query = dataset_query;+ rollback:+ - sql: >-+ UPDATE query_action+ SET dataset_query = legacy_query+ WHERE legacy_query IS NOT NULL;++# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################@@ -1931,7 +1996,7 @@ databaseChangeLog:## 2) Migrations should go in the ###_update_migrations.yaml file for the target version.#-# 3) Run mage lint-migrations-file to run core.spec checks against any changes you make here. Liquibase is pretty+# 3) Run mage lint-migrations to run core.spec checks against any changes you make here. Liquibase is pretty# forgiving and won't complain if you accidentally mix up things like deleteCascade and onDelete: CASCADE. CI runs# this check but it's nicer to know now instead of waiting for CI.#diff --git a/src/metabase/actions/actions.clj b/src/metabase/actions/actions.cljindex 3b8a9b652b8..a71c90d4ce0 100644--- a/src/metabase/actions/actions.clj+++ b/src/metabase/actions/actions.clj@@ -9,11 +9,12 @@[metabase.api.common :as api][metabase.driver :as driver][metabase.driver.util :as driver.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.query-processor.middleware.permissions :as qp.perms];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.settings.core :as setting][metabase.util :as u][metabase.util.i18n :as i18n :refer [tru]]@@ -40,10 +41,10 @@(assoc (select-keys scope [:table-id]) :row :metabase-enterprise.action-v2.api/root)))(methodical/defmulti perform-action!*- "Multimethod for doing an Action. The specific `action` is a keyword like `:model.row/create` or `:table.row/create`; the shape- of each input depends on the action being performed. [[action-arg-map-schema]] returns the appropriate spec to use to- validate the inputs for a given action. When implementing a new action type, be sure to implement both this method- and [[action-arg-map-schema]].+ "Multimethod for doing an Action. The specific `action` is a keyword like `:model.row/create` or `:table.row/create`;+ the shape of each input depends on the action being performed. [[metabase.actions.args/action-arg-map-schema]]+ returns the appropriate spec to use to validate the inputs for a given action. When implementing a new action type,+ be sure to implement both this method and [[metabase.actions.args/action-arg-map-schema]].DON'T CALL THIS METHOD DIRECTLY TO PERFORM ACTIONS -- use [[perform-action!]] instead which does normalization,validation, and binds Database-local values."@@ -178,7 +179,7 @@;; Since the inner map shape will depend on action-kw, we will need to dynamically validate it.inputs :- [:sequential :map]& {:as _opts}]- (lib.metadata.jvm/with-metadata-provider-cache+ (lib-be/with-metadata-provider-cache(let [invocation-id (nano-id/nano-id)context-before (-> (assoc ctx :invocation-id invocation-id)(update :invocation-stack u/conjv [action-kw invocation-id]))]@@ -243,6 +244,20 @@(log/logf level "%s %s => %s" context before after)after)+(mu/defn- check-permissions+ [policy :- :keyword+ arg-maps :- [:sequential [:or+ ::actions.args/common+ [:= {:description "empty map"} {}]]]]+ (when (#{:model-action :ad-hoc-invocation} policy)+ (doseq [arg-map arg-maps+ :when (seq arg-map)+ :let [mp (lib-be/application-database-metadata-provider (:database arg-map))+ query (if (:table-id arg-map)+ (lib/query mp (lib.metadata/table mp (:table-id arg-map)))+ (lib-be/normalize-query arg-map))]]+ (qp.perms/check-query-action-permissions* query))))+;; TODO rename this to just perform-action! and rename the legacy entry point to clearly deprecate it.(mu/defn perform-action-v2!"Perform an *implicit* `action`. This is the main entry point that handles validation, permissions, and more.@@ -257,7 +272,7 @@(log/with-context {:action action}(let [action-kw (keyword action)scope (actions.scope/hydrate-scope scope)- arg-maps (if (map? arg-map-or-maps) [arg-map-or-maps] arg-map-or-maps)+ arg-maps (u/one-or-many arg-map-or-maps)policy (or policy(cond(:model-id scope) :model-action@@ -285,7 +300,6 @@:database-ids (map :id dbs)})))db (first dbs)driver (:engine db)]-;; -- * Authorization* --;; The action might not be database-centric (e.g., call a webhook)(when db@@ -300,21 +314,9 @@(when-not (api/check-superuser)(throw (ex-info (i18n/tru "You don''t have permissions to do that.") {:status-code 403})))(check-data-editing-enabled-for-database! db))))-(log/with-context {:db-id (:id db)}(binding [*misc-value-cache* (atom {:databases (zipmap (map :id dbs) dbs)})]- (when (= :model-action policy)- (doseq [arg-map arg-maps]- (qp.perms/check-query-action-permissions* arg-map)))- (when (= :ad-hoc-invocation policy)- (doseq [arg-map arg-maps]- (cond- (:query arg-map) (qp.perms/check-query-action-permissions* arg-map)- (:table-id arg-map) (qp.perms/check-query-action-permissions*- {:type :query,- :database (:database arg-map)- :query {:source-table (:table-id arg-map)}}))))-+ (check-permissions policy arg-maps)(let [result (let [context (-> existing-context;; TODO fix tons of tests which execute without user scope(u/assoc-default :user-id (identity #_api/check-500diff --git a/src/metabase/actions/api.clj b/src/metabase/actions/api.cljindex 0497cc0a3b1..83e47fa2e08 100644--- a/src/metabase/actions/api.clj+++ b/src/metabase/actions/api.clj@@ -3,62 +3,30 @@(:require[metabase.actions.actions :as actions][metabase.actions.execution :as actions.execution]- [metabase.actions.http-action :as actions.http-action][metabase.actions.models :as actions.models]+ [metabase.actions.schema :as actions.schema][metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.collections.models.collection :as collection]+ [metabase.lib.schema.id :as lib.schema.id][metabase.permissions.core :as perms][metabase.public-sharing.validation :as public-sharing.validation][metabase.queries.core :as queries][metabase.util :as u]- [metabase.util.i18n :refer [deferred-tru tru]]+ [metabase.util.i18n :refer [tru]][metabase.util.json :as json]- [metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2]))(set! *warn-on-reflection* true)-;;; TODO -- by convention these should either have class-like names e.g. `JSONQuery` and `SupportedActionType` or we-;;; should use [[metabase.util.malli.registry/def]] and register them and make them namespaced keywords-(def ^:private JSONQuery- [:and- string?- (mu/with-api-error-message- [:fn #(actions.http-action/apply-json-query {} %)]- (deferred-tru "must be a valid json-query, something like ''.item.title''"))])--(def ^:private SupportedActionType- (mu/with-api-error-message- ;; TODO -- make this a keyword and let coercion convert it automatically- [:enum "http" "query" "implicit"]- (deferred-tru "Unsupported action type")))--(def ^:private ImplicitActionKind- (mu/with-api-error-message- (into [:enum]- (for [ns ["row" "bulk"]- action ["create" "update" "delete"]]- (str ns "/" action)))- (deferred-tru "Unsupported implicit action kind")))--(def ^:private HTTPActionTemplate- [:map {:closed true}- [:method [:enum "GET" "POST" "PUT" "DELETE" "PATCH"]]- [:url [string? {:min 1}]]- [:body {:optional true} [:maybe string?]]- [:headers {:optional true} [:maybe string?]]- [:parameters {:optional true} [:maybe [:sequential map?]]]- [:parameter_mappings {:optional true} [:maybe map?]]])--(api.macros/defendpoint :get "/"+(api.macros/defendpoint :get "/" :- [:sequential ::actions.schema/action]"Returns actions that can be used for QueryActions. By default lists all viewable actions. Pass optional`?model-id=<model-id>` to limit to actions on a particular model."[_route-params{:keys [model-id]} :- [:map- [:model-id {:optional true} [:maybe ms/PositiveInt]]]]+ [:model-id {:optional true} [:maybe ::lib.schema.id/card]]]](letfn [(actions-for [models](if (seq models)(t2/hydrate (actions.models/select-actions models@@ -77,14 +45,14 @@(collection/visible-collection-filter-clause)]}))](actions-for models))))-(api.macros/defendpoint :get "/public"+(api.macros/defendpoint :get "/public" :- [:sequential ::actions.schema/action]"Fetch a list of Actions with public UUIDs. These actions are publicly-accessible *if* public sharing is enabled."[](perms/check-has-application-permission :setting)(public-sharing.validation/check-public-sharing-enabled)(t2/select [:model/Action :name :id :public_uuid :model_id], :public_uuid [:not= nil], :archived false))-(api.macros/defendpoint :get "/:action-id"+(api.macros/defendpoint :get "/:action-id" :- ::actions.schema/action"Fetch an Action."[{:keys [action-id]} :- [:map[:action-id ms/PositiveInt]]]@@ -104,33 +72,20 @@(t2/delete! :model/Action :id action-id)api/generic-204-no-content)-(api.macros/defendpoint :post "/"+(api.macros/defendpoint :post "/" :- ::actions.schema/action"Create a new action."[_route-params_query-params{:keys [model_id parameters database_id]action-type :type- :as action} :- [:map- [:name :string]- [:model_id ms/PositiveInt]- [:type {:optional true} [:maybe SupportedActionType]]- [:description {:optional true} [:maybe :string]]- [:parameters {:optional true} [:maybe [:sequential map?]]]- [:parameter_mappings {:optional true} [:maybe map?]]- [:visualization_settings {:optional true} [:maybe map?]]- [:kind {:optional true} [:maybe ImplicitActionKind]]- [:database_id {:optional true} [:maybe ms/PositiveInt]]- [:dataset_query {:optional true} [:maybe map?]]- [:template {:optional true} [:maybe HTTPActionTemplate]]- [:response_handle {:optional true} [:maybe JSONQuery]]- [:error_handle {:optional true} [:maybe JSONQuery]]]]+ :as action} :- ::actions.schema/action.for-insert](when (and (nil? database_id)- (= "query" action-type))+ (= action-type :query))(throw (ex-info (tru "Must provide a database_id for query actions"){:type action-type:status-code 400})))(let [model (api/write-check :model/Card model_id)]- (when (and (= "implicit" action-type)+ (when (and (= action-type :implicit)(not (queries/model-supports-implicit-actions? model)))(throw (ex-info (tru "Implicit actions are not supported for models with clauses."){:status-code 400})))@@ -152,23 +107,9 @@(api.macros/defendpoint :put "/:id""Update an Action."[{:keys [id]} :- [:map- [:id ms/PositiveInt]]+ [:id ::actions.schema/id]]_query-params- action :- [:map- [:archived {:optional true} [:maybe :boolean]]- [:database_id {:optional true} [:maybe ms/PositiveInt]]- [:dataset_query {:optional true} [:maybe :map]]- [:description {:optional true} [:maybe :string]]- [:error_handle {:optional true} [:maybe JSONQuery]]- [:kind {:optional true} [:maybe ImplicitActionKind]]- [:model_id {:optional true} [:maybe ms/PositiveInt]]- [:name {:optional true} [:maybe :string]]- [:parameter_mappings {:optional true} [:maybe :map]]- [:parameters {:optional true} [:maybe [:sequential :map]]]- [:response_handle {:optional true} [:maybe JSONQuery]]- [:template {:optional true} [:maybe HTTPActionTemplate]]- [:type {:optional true} [:maybe SupportedActionType]]- [:visualization_settings {:optional true} [:maybe :map]]]]+ action :- ::actions.schema/action.for-update](actions/check-actions-enabled! id)(let [existing-action (api/write-check :model/Action id)](actions.models/update! (assoc action :id id) existing-action))@@ -185,7 +126,7 @@Action has already been shared, it will return the existing public link rather than creating a new one.) Publicsharing must be enabled."[{:keys [id]} :- [:map- [:id ms/PositiveInt]]]+ [:id ::actions.schema/id]]](api/check-superuser)(public-sharing.validation/check-public-sharing-enabled)(let [action (api/read-check :model/Action id :archived false)]@@ -199,7 +140,7 @@(api.macros/defendpoint :delete "/:id/public_link""Delete the publicly-accessible link to this Dashboard."[{:keys [id]} :- [:map- [:id ms/PositiveInt]]]+ [:id ::actions.schema/id]]];; check the /application/setting permission, not superuser because removing a public link is possible from /admin/settings(perms/check-has-application-permission :setting)(public-sharing.validation/check-public-sharing-enabled)@@ -224,7 +165,7 @@`parameters` should be the mapped dashboard parameters with values."[{:keys [id]} :- [:map- [:id ms/PositiveInt]]+ [:id ::actions.schema/id]]_query-params{:keys [parameters], :as _body} :- [:maybe [:map[:parameters {:optional true} [:maybe [:map-of :keyword any?]]]]]]diff --git a/src/metabase/actions/args.clj b/src/metabase/actions/args.cljindex 0c79c2c401a..1593d0dc19a 100644--- a/src/metabase/actions/args.clj+++ b/src/metabase/actions/args.clj@@ -1,6 +1,7 @@-(ns ^{:instrument/always true} metabase.actions.args+(ns ^:instrument/always metabase.actions.args(:require[metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.schema.id :as lib.schema.id][metabase.util :as u][metabase.util.log :as log][metabase.util.malli.registry :as mr]@@ -45,7 +46,7 @@;;; Anything else required depends on the action type.(mr/def ::common- [:map [:database pos-int?]])+ [:map [:database ::lib.schema.id/database]])(mr/def ::row [:map-of :string :any])@@ -54,7 +55,7 @@;;; {:database <id>, :query {:source-table <id>}}(mr/def ::query- [:map [:source-table pos-int?]])+ [:map [:source-table ::lib.schema.id/table]])(mr/def ::crud.row.common[:merge@@ -165,6 +166,6 @@(when (seq row-arg)(log/warn ":arg is deprecated, use :row instead"));; TODO it would be nice to use cached-database-via-table-id here, but need to solve circular dependency.- {:database (or database (when table-id (t2/select-one-fn :db_id :model/Table table-id)))+ {:database (or database (when table-id (t2/select-one-fn :db_id [:model/Table :db_id] table-id))):table-id table-id:row (update-keys (or row row-arg) u/qualified-name)})diff --git a/src/metabase/actions/execution.clj b/src/metabase/actions/execution.cljindex cd047c46041..46fc6322c93 100644--- a/src/metabase/actions/execution.clj+++ b/src/metabase/actions/execution.clj@@ -9,6 +9,7 @@[metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.model-persistence.core :as model-persistence][metabase.queries.models.query :as query]@@ -30,17 +31,13 @@`action` should already be hydrated with its `:card`."[{query :dataset_query, model-id :model_id, :as action} :- [:map[:model_id ::lib.schema.id/card]- [:dataset_query [:map- [:type [:= :native]]- [:native [:map- [:query :string]]]]]]+ [:dataset_query ::lib.schema/native-only-query]]request-parameters](log/tracef "Executing action\n\n%s" (u/pprint-to-str action))(try(let [parameters (for [parameter (:parameters action)](assoc parameter :value (get request-parameters (:id parameter))))query (-> query- (update :type keyword)(assoc :parameters parameters))](log/debugf "Query (before preprocessing):\n\n%s" (u/pprint-to-str query))(binding [qp.perms/*card-id* model-id]@@ -54,9 +51,9 @@e))))))(mu/defn- implicit-action-table- [card-id :- pos-int?]- (let [card (t2/select-one :model/Card :id card-id)- {:keys [table-id]} (query/query->database-and-table-ids (:dataset_query card))]+ [card-id :- ::lib.schema.id/card]+ (let [query (t2/select-one-fn :dataset_query :model/Card :id card-id)+ {:keys [table-id]} (query/query->database-and-table-ids query)](t2/hydrate (t2/select-one :model/Table :id table-id) :fields)))(defn- execute-custom-action! [action request-parameters]diff --git a/src/metabase/actions/http_action.clj b/src/metabase/actions/http_action.cljindex dbfa481998c..5b28b3c1c65 100644--- a/src/metabase/actions/http_action.clj+++ b/src/metabase/actions/http_action.clj@@ -2,8 +2,8 @@(:require[clj-http.client :as http][clojure.string :as str]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.parse :as params.parse]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse][metabase.query-processor.error-type :as qp.error-type][metabase.util :as u][metabase.util.i18n :refer [tru]]diff --git a/src/metabase/actions/models.clj b/src/metabase/actions/models.cljindex 87b984a115e..17874263dad 100644--- a/src/metabase/actions/models.clj+++ b/src/metabase/actions/models.clj@@ -1,13 +1,16 @@(ns metabase.actions.models(:require[medley.core :as m]+ [metabase.actions.schema :as actions.schema]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.models.interface :as mi][metabase.models.serialization :as serdes]+ [metabase.parameters.core :as parameters][metabase.queries.models.query :as query][metabase.search.core :as search][metabase.util :as u][metabase.util.i18n :refer [tru]]- [metabase.util.json :as json][metabase.util.malli :as mu][methodical.core :as methodical][toucan2.core :as t2]@@ -45,24 +48,27 @@{:in mi/json-in:out (comp (fn [viz-settings];; the keys of :fields should be strings, not keywords- (m/update-existing viz-settings :fields update-keys name))+ (m/update-existing viz-settings :fields update-keys u/qualified-name))mi/json-out-with-keywordization)})(t2/deftransforms :model/Action{:type mi/transform-keyword- :parameter_mappings mi/transform-parameters-list- :parameters mi/transform-card-parameters-list+ :parameter_mappings parameters/transform-parameter-mappings+ :parameters parameters/transform-parameters:visualization_settings transform-action-visualization-settings})(t2/deftransforms :model/QueryAction- {:dataset_query mi/transform-metabase-query})+ {:dataset_query lib-be/transform-query})++(t2/deftransforms :model/ImplicitAction+ {:kind mi/transform-keyword})(def ^:private transform-json-with-nested-parameters{:in (comp mi/json-in(fn [template]- (u/update-if-exists template :parameters mi/normalize-parameters-list)))+ (u/update-if-exists template :parameters parameters/normalize-parameters))):out (comp (fn [template]- (u/update-if-exists template :parameters (mi/catch-normalization-exceptions mi/normalize-parameters-list)))+ (u/update-if-exists template :parameters (mi/catch-normalization-exceptions parameters/normalize-parameters)))mi/json-out-with-keywordization)})(t2/deftransforms :model/HTTPAction@@ -99,15 +105,15 @@read-or-write :- [:enum :read :write]](mi/perms-objects-set (t2/select-one :model/Card :id (:model_id instance)) read-or-write))-(def action-columns+(def ^:private action-columns"The columns that are common to all Action types."[:archived :created_at :creator_id :description :entity_id :made_public_by_id :model_id :name :parameter_mappings:parameters :public_uuid :type :updated_at :visualization_settings])-(defn type->model+(mu/defn- type->model"Returns the model from an action type.`action-type` can be a string or a keyword."- [action-type]+ [action-type :- ::actions.schema/type](case action-type:http :model/HTTPAction:implicit :model/ImplicitAction@@ -115,40 +121,49 @@;;; ------------------------------------------------ CRUD fns ------------------------------------------------------(defn insert!- "Inserts an Action and related type table. Returns the action id."- [action-data]+;;; TODO (Cam 10/2/25) -- this should just be the default Toucan 2 insert behavior for an action+(mu/defn- insert*! :- ::actions.schema/id+ [action-data :- ::actions.schema/action.for-insert](t2/with-transaction [_conn](let [action (first (t2/insert-returning-instances! :model/Action (select-keys action-data action-columns)))- model (type->model (:type action))]- (t2/query-one {:insert-into (t2/table-name model)- :values [(-> (apply dissoc action-data action-columns)- (assoc :action_id (:id action))- (cond-> (= (:type action) :implicit)- (dissoc :database_id)- (= (:type action) :http)- (update :template json/encode)- (= (:type action) :query)- (update :dataset_query json/encode)))]})+ model (type->model (:type action))+ row (-> (apply dissoc action-data action-columns)+ (assoc :action_id (:id action))+ (cond-> (= (:type action) :implicit) (dissoc :database_id)))]+ (t2/insert! model row)(:id action))))-(defn update!+(mu/defn insert! :- ::actions.schema/id+ "Inserts an Action and related type table. Returns the action id."+ [action-data :- :map]+ (insert*! (lib/normalize ::actions.schema/action.for-insert action-data)))++(mu/defn- update*!+ [{:keys [id] :as updates} :- ::actions.schema/action.for-update+ existing-action :- ::actions.schema/action]+ (t2/with-transaction [_conn]+ (when-let [action-row (not-empty (select-keys updates action-columns))]+ (t2/update! :model/Action id action-row))+ (when-let [type-row (not-empty (cond-> (apply dissoc updates :id action-columns)+ (= (or (:type updates) (:type existing-action))+ :implicit)+ (dissoc :database_id)))]+ (let [type-row (assoc type-row :action_id id)+ existing-model (type->model (:type existing-action))]+ (if (and (:type updates) (not= (:type updates) (:type existing-action)))+ (let [new-model (type->model (:type updates))]+ (t2/delete! existing-model :action_id id)+ (t2/insert! new-model (assoc type-row :action_id id)))+ (t2/update! existing-model id type-row))))))++(mu/defn update!"Updates an Action and the related type table.Deletes the old type table row if the type has changed."- [{:keys [id] :as action} existing-action]- (when-let [action-row (not-empty (select-keys action action-columns))]- (t2/update! :model/Action id action-row))- (when-let [type-row (not-empty (cond-> (apply dissoc action :id action-columns)- (= (or (:type action) (:type existing-action))- :implicit)- (dissoc :database_id)))]- (let [type-row (assoc type-row :action_id id)- existing-model (type->model (:type existing-action))]- (if (and (:type action) (not= (:type action) (:type existing-action)))- (let [new-model (type->model (:type action))]- (t2/delete! existing-model :action_id id)- (t2/insert! new-model (assoc type-row :action_id id)))- (t2/update! existing-model id type-row)))))+ [updates :- [:map+ [:id ::actions.schema/id]]+ existing-action :- ::actions.schema/action]+ (let [updates (merge (select-keys existing-action [:type]) updates)] ; in case the updates do not include it.+ (update*! (lib/normalize ::actions.schema/action.for-update updates) existing-action)))(defn- normalize-query-actions [actions](when (seq actions)@@ -232,14 +247,105 @@:target [:variable [:template-tag (u/slugify (:name field))]];; TODO (Cam 8/12/25) -- Field base type is NOT a valid parameter types!;; See [[metabase.lib.schema.parameter/types]].- :type (:base_type field)+ :type (let [base-type (:base_type field)]+ (condp #(isa? %2 %1) base-type+ :type/Number :number+ :type/Temporal :date+ :type/Boolean :boolean+ :text)):required (:database_required field):is-auto-increment (:database_is_auto_increment field)::field-id (:id field)::pk? (isa? (:semantic_type field) :type/PK)})))]][(:id card) parameters]))))-(defn select-actions+(defn- select-actions-implicit-params [action model-id->implicit-parameters]+ (let [model-id (:model_id action)+ saved-params (m/index-by :id (:parameters action))+ action-kind (keyword (:kind action))]+ (cond->> (get model-id->implicit-parameters model-id)+ :always+ (map (fn [param]+ (let [saved-param (saved-params (:id param))+ ;; we ignore the saved type, to allow schema changes (type changes) to be+ ;; reflected in the field presentation+ ;; this also fixes #39101 and avoids us making awkward changes to+ ;; :parameter transforms for QueryActions.+ saved-param' (dissoc saved-param :type)]+ (merge param saved-param'))))++ (= action-kind :row/delete)+ (filter ::pk?)++ (= action-kind :row/create)+ (remove #(or (:is-auto-increment %)+ ;; non-required PKs like column with default is uuid_generate_v4()+ (and (::pk? %) (not (:required %)))))++ (contains? #{:row/update :row/delete} action-kind)+ (map (fn [param] (cond-> param (::pk? param) (assoc :required true)))))))++(defn- implicit-parameters->viz-fields [implicit-parameters]+ (let [field-ids (into #{}+ (comp cat+ (keep ::field-id))+ implicit-parameters)]+ (when (seq field-ids)+ (t2/select-pk->fn (fn [field]+ (merge+ (select-keys field [:base_type :display_name :description])+ {:title (:display_name field)+ :placeholder (:display_name field)+ ;; these "illegal" camelCase keys are for viz+ ;; settings purposes, and that's what the FE uses.+ ;; See+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1759981400217489?thread_ts=1759289751.539169&cid=C0645JP1W81+ :fieldType (if (isa? (:base_type field) :type/Number) :number :string)+ :inputType (condp #(isa? %2 %1) (:base_type field)+ :type/Number :number+ :type/DateTime :datetime+ :type/Time :time+ :type/Temporal :date+ :type/Boolean :boolean+ :string)}))+ [:model/Field :id :base_type :display_name :description]+ :id [:in field-ids]))))++(defn- enrich-viz-settings-fields [viz-fields implicit-params field-id->viz-field]+ (let [param-ids (map :id implicit-params)+ param-id->order (zipmap param-ids (range))+ param-id->required (into {} (map (juxt :id :required)) implicit-params)+ viz-fields (->> (or viz-fields {})+ ;; remove entries that don't match params (in case of deleted+ ;; columns)+ (m/filter-keys (set param-ids)))+ ;; add default entries for params that don't have an entry+ viz-fields (reduce (fn [acc param-id]+ (if (contains? acc param-id)+ acc+ (assoc acc param-id {:id param-id, :hidden false})))+ viz-fields+ param-ids)+ param-id->field-id (into {} (map (juxt :id ::field-id)) implicit-params)]+ (update-vals viz-fields (fn [{param-id :id, :as viz-field}]+ (let [field-id (get param-id->field-id param-id)]+ (merge viz-field+ (get field-id->viz-field field-id)+ {:order (get param-id->order param-id)+ :required (get param-id->required param-id)}))))))++(defn- enrich-action [action model-id->db-id model-id->implicit-parameters field-id->viz-field]+ (case (:type action)+ :implicit+ (let [implicit-params (select-actions-implicit-params action model-id->implicit-parameters)]+ (cond-> (assoc action :database_id (model-id->db-id (:model_id action)))+ (seq implicit-params)+ (-> (assoc :parameters implicit-params)+ (update-in [:visualization_settings :fields] enrich-viz-settings-fields implicit-params field-id->viz-field))))+ (:query :http)+ action))++(mu/defn select-actions :- [:maybe [:sequential ::actions.schema/action]]"Find actions with given options and generate implicit parameters for execution. Also adds the `:database_id` of themodel for implicit actions.@@ -256,57 +362,12 @@model-id->db-id (into {} (for [card implicit-action-models][(:id card) (:database_id card)]))model-id->implicit-parameters (when (seq implicit-action-models)- (implicit-action-parameters implicit-action-models))]+ (implicit-action-parameters implicit-action-models))+ field-id->viz-field (implicit-parameters->viz-fields (vals model-id->implicit-parameters))](for [action actions]- (case (:type action)- :implicit- (let [model-id (:model_id action)- saved-params (m/index-by :id (:parameters action))- action-kind (:kind action)- implicit-params (cond->> (get model-id->implicit-parameters model-id)- :always- (map (fn [param]- (let [saved-param (saved-params (:id param))- ;; we ignore the saved type, to allow schema changes (type changes) to be- ;; reflected in the field presentation- ;; this also fixes #39101 and avoids us making awkward changes to- ;; :parameter transforms for QueryActions.- saved-param' (dissoc saved-param :type)]- (merge param saved-param'))))-- (= "row/delete" action-kind)- (filter ::pk?)-- (= "row/create" action-kind)- (remove #(or (:is-auto-increment %)- ;; non-required PKs like column with default is uuid_generate_v4()- (and (::pk? %) (not (:required %)))))-- (contains? #{"row/update" "row/delete"} action-kind)- (map (fn [param] (cond-> param (::pk? param) (assoc :required true))))-- :always- (map #(dissoc % ::pk? ::field-id)))]- (cond-> (assoc action :database_id (model-id->db-id (:model_id action)))- (seq implicit-params)- (-> (assoc :parameters implicit-params)- (update-in [:visualization_settings :fields]- (fn [fields]- (let [param-ids (map :id implicit-params)- fields (->> (or fields {})- ;; remove entries that don't match params (in case of deleted columns)- (m/filter-keys (set param-ids)))]- ;; add default entries for params that don't have an entry- (reduce (fn [acc param-id]- (if (contains? acc param-id)- acc- (assoc acc param-id {:id param-id, :hidden false})))- fields- param-ids)))))))- (:query :http)- action))))--(defn select-action+ (enrich-action action model-id->db-id model-id->implicit-parameters field-id->viz-field))))++(mu/defn select-action :- [:maybe ::actions.schema/action]"Selects an Action and fills in the subtype data and implicit parameters.`options` is [[apply]]ed to [[t2/select]]."[& options]@@ -362,6 +423,8 @@(defmethod serdes/generate-path "QueryAction" [_ _] nil)(defmethod serdes/make-spec "QueryAction" [_model-name _opts]{:copy []+ :skip [;; this is a temporary column to power v57 => v56 rollbacks, and we can remove it in v58.+ :legacy_query]:transform {:action_id (serdes/parent-ref):database_id (serdes/fk :model/Database :name):dataset_query {:export serdes/export-mbql :import serdes/import-mbql}}})diff --git a/src/metabase/actions/schema.clj b/src/metabase/actions/schema.cljnew file mode 100644index 00000000000..dd7783183d1--- /dev/null+++ b/src/metabase/actions/schema.clj@@ -0,0 +1,116 @@+(ns metabase.actions.schema+ (:require+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.parameters.schema :as parameters.schema]+ [metabase.queries.schema :as queries.schema]+ [metabase.util.i18n :refer [deferred-tru]]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]))++(mr/def ::id+ "Valid Action ID"+ pos-int?)++(mr/def ::type+ [:enum+ {:description (deferred-tru "Unsupported action type")}+ :http+ :implicit+ :query])++(mr/def ::http-action.json-query+ [:and+ {:description (deferred-tru "must be a valid json-query, something like ''.item.title''")}+ string?+ [:fn+ {:error/fn (fn [_ _]+ (deferred-tru "must be a valid json-query, something like ''.item.title''"))}+ #((requiring-resolve 'metabase.actions.http-action/apply-json-query) {} %)]])++(mr/def ::http-action.template+ [:map {:closed true}+ [:method [:enum "GET" "POST" "PUT" "DELETE" "PATCH"]]+ [:url [string? {:min 1}]]+ [:body {:optional true} [:maybe string?]]+ [:headers {:optional true} [:maybe string?]]+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters]]])++(mr/def ::http-action+ [:map+ [:template {:optional true} [:maybe ::http-action.template]]+ [:response_handle {:optional true} [:maybe ::http-action.json-query]]+ [:error_handle {:optional true} [:maybe ::http-action.json-query]]])++(mr/def ::implicit-action.kind+ [:enum+ {:decode/normalize keyword+ :description (deferred-tru "Unsupported implicit action kind")}+ :row/create+ :row/update+ :row/delete+ :bulk/create+ :bulk/update+ :bulk/delete])++(mr/def ::implicit-action+ [:map+ [:kind {:optional true} [:maybe ::implicit-action.kind]]])++(mr/def ::query-action+ [:map+ [:database_id {:optional true} [:maybe ::lib.schema.id/database]]+ [:dataset_query {:optional true} [:maybe ::queries.schema/query]]])++(mu/defn- action-schema [schema-type :- [:enum :select :update :insert]]+ ;; `required-for-insert` = you have to specify this when you insert a row+ ;;+ ;; `not-null-in-app-db` = this is `NOT NULL` in the app DB, and will always come back when you `SELECT` something,+ ;; but its value is populated automatically on `INSERT` or `UPDATE`.+ (let [required-for-insert (case schema-type+ (:select :update) {:optional true}+ :insert {})]+ [:and+ (into+ [:map]+ cat+ [(case schema-type+ :select [[:id ::id]]+ :update [[:id {:optional true} ::id]]+ :insert nil)+ [[:name required-for-insert :string]+ [:type required-for-insert ::type]+ [:model_id required-for-insert ::lib.schema.id/card]+ [:archived {:optional true} :boolean]+ [:description {:optional true} [:maybe :string]]+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters]]+ [:parameter_mappings {:optional true} [:maybe ::parameters.schema/parameter-mappings]]+ [:visualization_settings {:optional true} [:maybe map?]]]+ (when (= schema-type :select)+ ;; technically these are always required, but they are not always selected.+ [[:created_at {:optional true} (ms/InstanceOfClass java.time.temporal.Temporal)]+ [:updated_at {:optional true} (ms/InstanceOfClass java.time.temporal.Temporal)]+ ;; TODO (Cam 10/2/25) -- these are things you can set in updates or inserts but aren't things you can pass in+ ;; via the API... Maybe we need even more versions of this schema e.g. `::action.for-update.api` versus+ ;; `::action.for-update.internal`. or something. Idk.+ [:public_uuid {:optional true} [:maybe ms/UUIDString]]+ [:made_public_by_id {:optional true} [:maybe ::lib.schema.id/user]]+ [:creator_id {:optional true} [:maybe ::lib.schema.id/user]]])])+ [:multi+ {:dispatch :type}+ [:http ::http-action]+ [:implicit ::implicit-action]+ [:query ::query-action]+ [nil :map]]]))++(mr/def ::action+ "An Action as it should appear when we `SELECT` it from the app DB."+ (action-schema :select))++(mr/def ::action.for-insert+ "Schema for inserting a new Action (REST API or internally)."+ (action-schema :insert))++(mr/def ::action.for-update+ "Schema for updating an Action (REST API or internally)."+ (action-schema :update))diff --git a/src/metabase/actions/scope.clj b/src/metabase/actions/scope.cljindex b71f591fd8c..b1f7afaa85b 100644--- a/src/metabase/actions/scope.clj+++ b/src/metabase/actions/scope.clj@@ -2,6 +2,7 @@(:require[macaw.util :as u][metabase.actions.types :as types]+ [metabase.lib.core :as lib][metabase.util.malli :as mu][toucan2.core :as t2]))@@ -29,9 +30,7 @@(if (and (contains? scope :collection-id) (contains? scope :table-id) (contains? scope :database-id))scope(let [card (t2/select-one [:model/Card :dataset_query :collection_id :database_id :display] card-id)- source-table (-> card :dataset_query :query :source-table)- table-id (when (pos-int? source-table)- source-table)]+ table-id (lib/source-table-id (:dataset_query card))](merge {:table-id table-id:collection-id (:collection_id card missing-id):database-id (:database_id card missing-id)}diff --git a/src/metabase/api/macros.clj b/src/metabase/api/macros.cljindex 815dbbca017..2a10cf4ec6c 100644--- a/src/metabase/api/macros.clj+++ b/src/metabase/api/macros.clj@@ -23,6 +23,7 @@[malli.core :as mc][malli.error :as me][malli.transform :as mtx]+ [malli.util][medley.core :as m][metabase.api.common.internal][metabase.api.macros.defendpoint.open-api]@@ -351,16 +352,26 @@me/with-spell-checking(me/humanize {:wrap mu/humanize-include-value})))-(defn- invalid-params-errors [schema explanation specific-errors]- (or (when (and (map? specific-errors)- (= (mc/type schema) :map))- (into {}- (let [specific-error-keys (set (keys specific-errors))]- (keep (fn [child]- (when (contains? specific-error-keys (first child))- [(first child) (umd/describe (last child))]))))- (mc/children schema)))- (me/humanize explanation {:wrap #(umd/describe (:schema %))})))+(defn- invalid-params-errors [{:keys [schema], :as explanation}]+ (reduce+ (fn [m {:keys [path in], :as _explanation}]+ (let [error-path (remove integer? in)]+ ;; if there is already an error here keep the existing one, this is usually something like an `[:and x y]`+ ;; where `x` has already failed so it's preferable to return the error for that than the `y` one, which+ ;; probably won't make any sense (for some weird reason Malli `:and` schemas don't short-circut)+ (if (get-in m error-path)+ m+ (let [nice-path (loop [path (vec path)]+ (if (integer? (last path))+ (recur (pop path))+ path))+ nested-schema (loop [path nice-path]+ (when (seq path)+ (or (malli.util/get-in schema path)+ (recur (pop path)))))]+ (assoc-in m error-path (umd/describe nested-schema))))))+ {}+ (:errors explanation)))(mu/defn decode-and-validate-params"Impl for [[defendpoint]]."@@ -376,7 +387,7 @@:body "body"))(let [explanation (mr/explain schema decoded)specific-errors (invalid-params-specific-errors explanation)- errors (invalid-params-errors schema explanation specific-errors)]+ errors (invalid-params-errors explanation)]{:status-code 400#_:api/debug #_{:params-type params-type:schema (mc/form schema)@@ -725,7 +736,12 @@motivation behind it.REPL Tip: use [[call-core-fn]] to call the core-fn directly."- {:added "0.53.0"}+ {:added "0.53.0", :arglists '([method+ route+ docstring?+ metadata?+ [route-params? query-params? body-params? request? respond? raise?]+ & body])}[& args](let [parsed (parse-args args)]`(let [core-fn# (endpoint-core-fn ~parsed)diff --git a/src/metabase/api/routes/common.clj b/src/metabase/api/routes/common.cljindex ff41cb6dce2..858961cc6d3 100644--- a/src/metabase/api/routes/common.clj+++ b/src/metabase/api/routes/common.clj@@ -7,7 +7,8 @@[metabase.api.response :as api.response][metabase.api.settings :as api.settings][metabase.util.i18n :refer [deferred-trs deferred-tru]]- [metabase.util.log :as log]))+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]));;; these use vars rather than plain functions so changes to the underlying functions get propagated during REPL usage.@@ -42,11 +43,11 @@[handler](fn [request respond _raise](let [raise (fn [^Throwable e]+ (log/error e "Exception in API call")(respond {:status 400, :body (ex-message e)}))](try(handler request respond raise)(catch Throwable e- (log/error e "Exception in API call")(raise e))))))(def ^:private mb-api-key-doc-url@@ -83,9 +84,9 @@:else(respond api.response/response-forbidden))))-(defn- enforce-authentication+(mu/defn- enforce-authentication :- ifn?"Middleware that returns a 401 response if `request` has no associated `:metabase-user-id`."- [handler]+ [handler :- ifn?](fn [{:keys [metabase-user-id] :as request} respond raise](if metabase-user-id(handler request respond raise)diff --git a/src/metabase/cache/api.clj b/src/metabase/cache/api.cljindex 46d52d2a45e..539147edbfb 100644--- a/src/metabase/cache/api.clj+++ b/src/metabase/cache/api.clj@@ -14,7 +14,13 @@;; Data shape-(mr/def ::cache-strategy.base+;;; TODO (Cam 10/3/25) -- move these schemas into a `.schemas` namespace to follow module shape guidelines++(mr/def ::cache-strategy.base.oss+ [:map+ [:type [:enum :nocache :ttl]]])++(mr/def ::cache-strategy.base.ee[:map[:type [:enum :nocache :ttl :duration :schedule]]])@@ -31,7 +37,7 @@(mr/def ::cache-strategy.oss"Schema for a caching strategy (OSS)"[:and- ::cache-strategy.base+ ::cache-strategy.base.oss[:multi {:dispatch :type}[:nocache ::cache-strategy.nocache][:ttl ::cache-strategy.ttl]]])@@ -40,6 +46,7 @@[:map {:closed true}[:type [:= :duration]][:duration ms/PositiveInt]+ ;; TODO (Cam 10/3/25) -- change these to keywords and let API coercion convert them for us automatically.[:unit [:enum "hours" "minutes" "seconds" "days"]][:refresh_automatically {:optional true} [:maybe :boolean]]])@@ -49,15 +56,17 @@[:schedule u.cron/CronScheduleString][:refresh_automatically {:optional true} [:maybe :boolean]]])+;;; This is basically the same schema as `:metabase-enterprise.cache.strategies/cache-strategy` except it doesn't have+;;; the optional `:invalidated-at` keys(mr/def ::cache-strategy.ee"Schema for a caching strategy in EE when we have an premium token with `:cache-granular-controls`."[:and- ::cache-strategy.base+ ::cache-strategy.base.ee[:multi {:dispatch :type}- [:nocache ::cache-strategy.nocache]- [:ttl ::cache-strategy.ttl]- [:duration ::cache-strategy.ee.duration]- [:schedule ::cache-strategy.ee.schedule]]])+ [:nocache ::cache-strategy.nocache]+ [:ttl ::cache-strategy.ttl]+ [:duration ::cache-strategy.ee.duration]+ [:schedule ::cache-strategy.ee.schedule]]])(mr/def ::cache-strategy(if config/ee-available?@@ -162,11 +171,9 @@(ms/QueryVectorOf ms/IntGreaterThanOrEqualToZero)]]]](when-not (premium-features/enable-cache-granular-controls?)(throw (premium-features/ee-feature-error (tru "Granular Caching"))))-(doseq [db-id database] (api/write-check :model/Database db-id))(doseq [dashboard-id dashboard] (api/write-check :model/Dashboard dashboard-id))(doseq [question-id question] (api/write-check :model/Card question-id))-(let [cnt (cache-config/invalidate! {:databases database:dashboards dashboard:questions questiondiff --git a/src/metabase/cache/models/cache_config.clj b/src/metabase/cache/models/cache_config.cljindex e99e4d05e43..e327eef2a97 100644--- a/src/metabase/cache/models/cache_config.clj+++ b/src/metabase/cache/models/cache_config.clj@@ -10,6 +10,7 @@[methodical.core :as methodical][toucan2.core :as t2]))+;; TODO (Cam 10/3/25) -- change these to keywords and let API coercion convert them for us automatically(def CachingModel "Caching is configurable for those models" [:enum "root" "database" "dashboard" "question"])(doto :model/CacheConfigdiff --git a/src/metabase/channel/email/result_attachment.clj b/src/metabase/channel/email/result_attachment.cljindex 4ae95d580d2..f5343408fe1 100644--- a/src/metabase/channel/email/result_attachment.clj+++ b/src/metabase/channel/email/result_attachment.clj@@ -5,7 +5,7 @@[metabase.driver.util :as driver.u][metabase.lib.schema.id :as lib.schema.id];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming :as qp.streaming][metabase.query-processor.streaming.common :as streaming.common][metabase.query-processor.streaming.interface :as qp.si]diff --git a/src/metabase/channel/params.clj b/src/metabase/channel/params.cljindex 5da1d669654..305f7ba84c8 100644--- a/src/metabase/channel/params.clj+++ b/src/metabase/channel/params.clj@@ -1,8 +1,9 @@(ns metabase.channel.params(:require[clojure.string :as str]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.parse :as params.parse]))+ ;; TODO (Cam 10/3/25) -- migrate this namespace to the Lib version of these namespaces+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse]))(defn- param-name->path[param-name]diff --git a/src/metabase/cmd/core.clj b/src/metabase/cmd/core.cljindex 6da6900d105..ab991034323 100644--- a/src/metabase/cmd/core.clj+++ b/src/metabase/cmd/core.clj@@ -20,7 +20,6 @@[clojure.tools.cli :as cli][metabase.classloader.core :as classloader][metabase.config.core :as config]- [metabase.legacy-mbql.util :as mbql.u][metabase.util :as u][metabase.util.encryption :as encryption][metabase.util.i18n :refer [trs]]@@ -170,23 +169,6 @@(classloader/require 'metabase.cmd.driver-methods)((resolve 'metabase.cmd.driver-methods/print-available-multimethods) true)))-(defn ^:command load- {:doc "Note: this command is deprecated. Use `import` instead.- Load serialized Metabase instance as created by [[dump]] command from directory `path`."- :arg-spec [["-m" "--mode (skip|update)" "Update or skip on conflicts."- :default :skip- :default-desc "skip"- :parse-fn mbql.u/normalize-token- :validate [#{:skip :update} "Must be 'skip' or 'update'"]]- ["-e" "--on-error (continue|abort)" "Abort or continue on error."- :default :continue- :default-desc "continue"- :parse-fn mbql.u/normalize-token- :validate [#{:continue :abort} "Must be 'continue' or 'abort'"]]]}- [path & options]- (log/warn (u/colorize :red "'load' is deprecated and will be removed in a future release. Please migrate to 'import'."))- (call-enterprise 'metabase-enterprise.serialization.cmd/v1-load! path (get-parsed-options #'load options)))-(defn ^:command ^:requires-init import{:doc "Load serialized Metabase instance as created by the [[export]] command from directory `path`.":arg-spec [["-e" "--continue-on-error" "Do not break execution on errors."]@@ -194,20 +176,6 @@[path & options](call-enterprise 'metabase-enterprise.serialization.cmd/v2-load! path (get-parsed-options #'import options)))-(defn ^:command dump- {:doc "Note: this command is deprecated. Use `export` instead.- Serializes Metabase instance into directory `path`."- :arg-spec [["-u" "--user EMAIL" "Export collections owned by the specified user"]- ["-s" "--state (active|all)" "When set to `active`, do not dump archived entities. Default behavior is `all`."- :default :all- :default-desc "all"- :parse-fn mbql.u/normalize-token- :validate [#{:active :all} "Must be 'active' or 'all'"]]- [nil "--include-entity-id" "Include entity_id property in all dumped entities. Default: false."]]}- [path & options]- (log/warn (u/colorize :red "'dump' is deprecated and will be removed in a future release. Please migrate to 'export'."))- (call-enterprise 'metabase-enterprise.serialization.cmd/v1-dump! path (get-parsed-options #'dump options)))-(defn ^:command export{:doc "Serialize Metabase instance into directory at `path`.":arg-spec [["-c" "--collection ID" "Export only specified ID(s). Use commas to separate multiple IDs. Pass either PKs or entity IDs."diff --git a/src/metabase/collections/api.clj b/src/metabase/collections/api.cljindex d244fa4cf66..5d48150ea01 100644--- a/src/metabase/collections/api.clj+++ b/src/metabase/collections/api.clj@@ -14,11 +14,9 @@[metabase.app-db.core :as mdb][metabase.collections.models.collection :as collection][metabase.collections.models.collection.root :as collection.root]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.parse :as params.parse][metabase.eid-translation.core :as eid-translation][metabase.events.core :as events]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.models.interface :as mi][metabase.notification.core :as notification][metabase.permissions.core :as perms]@@ -499,52 +497,9 @@[_ collection options](card-query :question collection options))-(defn- fully-parameterized-text?- "Decide if `text`, usually (a part of) a query, is fully parameterized given the parameter types- described by `template-tags` (usually the template tags of a native query).-- The rules to consider a piece of text fully parameterized is as follows:-- 1. All parameters not in an optional block are field-filters or snippets or have a default value.- 2. All required parameters have a default value.-- The first rule is absolutely necessary, as queries violating it cannot be executed without- externally supplied parameter values. The second rule is more controversial, as field-filters- outside of optional blocks ([[ ... ]]) don't prevent the query from being executed without- external parameter values (neither do parameters in optional blocks). The rule has been added- nonetheless, because marking a parameter as required is something the user does intentionally- and queries that are technically executable without parameters can be unacceptably slow- without the necessary constraints. (Marking parameters in optional blocks as required doesn't- seem to be useful any way, but if the user said it is required, we honor this flag.)"- [text template-tags]- (try- (let [obligatory-params (into #{}- (comp (filter params/Param?)- (map :k))- (params.parse/parse text))]- (and (every? #(or (#{:dimension :snippet :card} (:type %))- (:default %))- (map template-tags obligatory-params))- (every? #(or (not (:required %))- (:default %))- (vals template-tags))))- (catch clojure.lang.ExceptionInfo _- ;; An exception might be thrown during parameter parsing if the syntax is invalid. In this case we return- ;; true so that we still can try to generate a preview for the query and display an error.- false)))--(defn fully-parameterized-query?- "Given a Card, returns `true` if its query is fully parameterized."- [row]-;; TODO TB handle pMBQL native queries- (let [native-query (-> row :dataset_query :native)]- (if-let [template-tags (:template-tags native-query)]- (fully-parameterized-text? (:query native-query) template-tags)- true)))-(defn- post-process-card-row [row](-> (t2/instance :model/Card row)- (update :dataset_query (:out mi/transform-metabase-query))+ (update :dataset_query (:out lib-be/transform-query))(update :collection_preview api/bit->boolean)(update :archived api/bit->boolean)(update :archived_directly api/bit->boolean)))@@ -552,7 +507,7 @@(defn- post-process-card-row-after-hydrate [row](-> (dissoc row :authority_level :icon :personal_owner_id :dataset_query :table_id :query_type :is_upload)(update :dashboard #(when % (select-keys % [:id :name :moderation_status])))- (assoc :fully_parameterized (fully-parameterized-query? row))))+ (assoc :fully_parameterized (queries/fully-parameterized? row))))(defn- post-process-card-like[{:keys [include-can-run-adhoc-query hydrate-based-on-upload]} rows]@@ -562,12 +517,11 @@:dashboard_count[:dashboard :moderation_status]]include-can-run-adhoc-query (conj :can_run_adhoc_query))]- (lib.metadata.jvm/with-metadata-provider-cache- (as-> (map post-process-card-row rows) $- (apply t2/hydrate $ hydration)- (cond-> $- hydrate-based-on-upload upload/model-hydrate-based-on-upload)- (map post-process-card-row-after-hydrate $)))))+ (as-> (map post-process-card-row rows) $+ (apply t2/hydrate $ hydration)+ (cond-> $+ hydrate-based-on-upload upload/model-hydrate-based-on-upload)+ (map post-process-card-row-after-hydrate $))))(defmethod post-process-collection-children :card[_ options _ rows]diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.cljindex d60fd5350cd..f7f6ad7dc7d 100644--- a/src/metabase/core/core.clj+++ b/src/metabase/core/core.clj@@ -113,7 +113,9 @@(defenterprise ensure-audit-db-installed!"OSS implementation of `audit-db/ensure-db-installed!`, which is an enterprise feature, so does nothing in the OSSversion."- metabase-enterprise.audit-app.audit [] ::noop)+ metabase-enterprise.audit-app.audit []+ (log/info "Not installing EE audit app DB")+ ::noop)(defn- signal-handler"Create a signal handler that logs the received signal and then delegates to the original handler."diff --git a/src/metabase/dashboards/api.clj b/src/metabase/dashboards/api.cljindex 239b0d4c198..2571d86cfca 100644--- a/src/metabase/dashboards/api.clj+++ b/src/metabase/dashboards/api.clj@@ -21,16 +21,16 @@[metabase.eid-translation.core :as eid-translation][metabase.embedding.validation :as embedding.validation][metabase.events.core :as events]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]- [metabase.lib.util.match :as lib.util.match]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.parameters.chain-filter :as chain-filter]+ [metabase.parameters.core :as parameters][metabase.parameters.dashboard :as parameters.dashboard][metabase.parameters.params :as params][metabase.parameters.schema :as parameters.schema][metabase.permissions.core :as perms][metabase.public-sharing.validation :as public-sharing.validation]- ^{:clj-kondo/ignore [:deprecated-namespace]}[metabase.pulse.core :as pulse][metabase.queries.core :as queries][metabase.query-permissions.core :as query-perms]@@ -121,7 +121,7 @@{:keys [name description parameters cache_ttl collection_id collection_position], :as _dashboard}:- [:map[:name ms/NonBlankString]- [:parameters {:optional true} [:maybe [:sequential ::parameters.schema/parameter]]]+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters]][:description {:optional true} [:maybe :string]][:cache_ttl {:optional true} [:maybe ms/PositiveInt]][:collection_id {:optional true} [:maybe ms/PositiveInt]]@@ -201,11 +201,14 @@run.Returns nil if `:dataset_query` isn't set, eg. for a markdown card."- [{:keys [dataset_query]}]- (when dataset_query- (u/ignore-exceptions- [(qp.util/query-hash dataset_query)- (qp.util/query-hash (assoc dataset_query :constraints (qp.constraints/default-query-constraints)))])))+ [{query :dataset_query, :as _card}]+ (when query+ (try+ [(qp.util/query-hash query)+ (qp.util/query-hash (assoc query :constraints (qp.constraints/default-query-constraints)))]+ (catch Throwable e+ (log/errorf e "Error hashing query %s: %s" (pr-str query) (ex-message e))+ nil))))(defn- dashcard->query-hashes"Return a sequence of all the query hashes for this `dashcard`, including the top-level Card and any Series."@@ -313,10 +316,10 @@(defn- do-with-dashboard-load-id [dashboard-load-id body-fn](if dashboard-load-id- (binding [*dashboard-load-id* dashboard-load-id- lib.metadata.jvm/*metadata-provider-cache* (dashboard-load-metadata-provider-cache dashboard-load-id)]- (log/debugf "Using dashboard_load_id %s" dashboard-load-id)- (body-fn))+ (binding [*dashboard-load-id* dashboard-load-id]+ (lib-be/with-existing-metadata-provider-cache (dashboard-load-metadata-provider-cache dashboard-load-id)+ (log/debugf "Using dashboard_load_id %s" dashboard-load-id)+ (body-fn)))(do(log/debug "No dashboard_load_id provided")(body-fn))))@@ -632,15 +635,15 @@(events/publish-event! :event/dashboard-delete {:object dashboard :user-id api/*current-user-id*}))api/generic-204-no-content)-(defn- param-target->field-id [target query]- (when-let [field-clause (params/param-target->field-clause target {:dataset_query query})]- (lib.util.match/match-one field-clause [:field (id :guard integer?) _] id)))+(mu/defn- param-target->field-id :- [:maybe ::lib.schema.id/field]+ [target query]+ (params/param-target->field-id target {:dataset_query query}));; TODO -- should we only check *new* or *modified* mappings?(mu/defn- check-parameter-mapping-permissions"Starting in 0.41.0, you must have *data* permissions in order to add or modify a DashboardCard parameter mapping."{:added "0.41.0"}- [parameter-mappings :- [:sequential dashboard-card/ParamMapping]]+ [parameter-mappings :- [:sequential ::parameters.schema/parameter-mapping]](when (seq parameter-mappings);; calculate a set of all Field IDs referenced by parameter mappings; then from those Field IDs calculate a set of;; all Table IDs to which those Fields belong. This is done in a batched fashion so we can avoid N+1 query issues@@ -693,7 +696,7 @@[dashboard-id dashcards](let [dashcard-id->existing-mappings (existing-parameter-mappings dashboard-id)existing-mapping? (fn [dashcard-id mapping]- (let [[mapping] (mi/normalize-parameters-list [mapping])+ (let [mapping (parameters/normalize-parameter-mapping mapping)existing-mappings (get dashcard-id->existing-mappings dashcard-id)](contains? existing-mappings (select-keys mapping [:target :parameter_id]))))new-mappings (for [{mappings :parameter_mappings, dashcard-id :id} dashcards@@ -763,9 +766,7 @@[:size_y ms/PositiveInt][:row ms/IntGreaterThanOrEqualToZero][:col ms/IntGreaterThanOrEqualToZero]- [:parameter_mappings {:optional true} [:maybe [:sequential [:map- [:parameter_id ms/NonBlankString]- [:target :any]]]]]+ [:parameter_mappings {:optional true} [:maybe [:ref ::parameters.schema/parameter-mappings]]][:inline_parameters {:optional true} [:maybe [:sequential ms/NonBlankString]]][:series {:optional true} [:maybe [:sequential map?]]]])@@ -892,7 +893,7 @@;;;;;;;;;;;;;;;;;;;;; End functions to handle broken subscriptions-(defn- update-dashboard+(defn- update-dashboard!"Updates a Dashboard. Designed to be reused by PUT /api/dashboard/:id and PUT /api/dashboard/:id/cards"[id {:keys [dashcards tabs parameters] :as dash-updates}](span/with-span!@@ -996,7 +997,7 @@[:show_in_getting_started {:optional true} [:maybe :boolean]][:enable_embedding {:optional true} [:maybe :boolean]][:embedding_params {:optional true} [:maybe ms/EmbeddingParams]]- [:parameters {:optional true} [:maybe [:sequential ::parameters.schema/parameter]]]+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters]][:position {:optional true} [:maybe ms/PositiveInt]][:width {:optional true} [:enum "fixed" "full"]][:archived {:optional true} [:maybe :boolean]]@@ -1013,7 +1014,7 @@[:id ms/PositiveInt]]_query-paramsdash-updates :- DashUpdates]- (update-dashboard id dash-updates))+ (update-dashboard! id dash-updates))(api.macros/defendpoint :put "/:id/cards""(DEPRECATED -- Use the `PUT /api/dashboard/:id` endpoint instead.)@@ -1038,7 +1039,7 @@[:tabs {:optional true} [:maybe (ms/maps-with-unique-key [:sequential UpdatedDashboardTab] :id)]]]](log/warn"DELETE /api/dashboard/:id/cards is deprecated. Use PUT /api/dashboard/:id instead.")- (let [dashboard (update-dashboard id {:dashcards cards :tabs tabs})]+ (let [dashboard (update-dashboard! id {:dashcards cards :tabs tabs})]{:cards (:dashcards dashboard):tabs (:tabs dashboard)}))@@ -1186,8 +1187,8 @@`filtered` Field ID -> subset of `filtering` Field IDs that would be used in chain filter query"[_route-params{:keys [filtered filtering]} :- [:map- [:filtered (ms/QueryVectorOf ms/PositiveInt)]- [:filtering {:optional true} [:maybe (ms/QueryVectorOf ms/PositiveInt)]]]]+ [:filtered (ms/QueryVectorOf ::lib.schema.id/field)]+ [:filtering {:optional true} [:maybe (ms/QueryVectorOf ::lib.schema.id/field)]]]](let [filtered-field-ids (if (sequential? filtered) (set filtered) #{filtered})filtering-field-ids (if (sequential? filtering) (set filtering) #{filtering})](doseq [field-id (set/union filtered-field-ids filtering-field-ids)]diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex f7a423096ce..5cf744d76e4 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -9,9 +9,12 @@[metabase.config.core :as config][metabase.dashboards.models.dashboard-card :as dashboard-card][metabase.dashboards.models.dashboard-tab :as dashboard-tab]+ [metabase.dashboards.schema :as dashboards.schema][metabase.events.core :as events]+ [metabase.lib.core :as lib][metabase.models.interface :as mi][metabase.models.serialization :as serdes]+ [metabase.parameters.core :as parameters][metabase.parameters.params :as params][metabase.parameters.schema :as parameters.schema][metabase.permissions.core :as perms]@@ -66,7 +69,7 @@#{:last_viewed_at})(t2/deftransforms :model/Dashboard- {:parameters mi/transform-card-parameters-list+ {:parameters parameters/transform-parameters:embedding_params mi/transform-json})(t2/define-before-delete :model/Dashboard@@ -78,7 +81,7 @@(t2/define-before-insert :model/Dashboard[dashboard](let [defaults {:parameters []}- dashboard (merge defaults dashboard)]+ dashboard (lib/normalize ::dashboards.schema/dashboard (merge defaults dashboard))](u/prog1 dashboard(params/assert-valid-parameters dashboard)(collection/check-collection-namespace :model/Dashboard (:collection_id dashboard)))))@@ -90,7 +93,9 @@(t2/define-before-update :model/Dashboard[dashboard]- (let [changes (t2/changes dashboard)]+ (let [changes (t2/changes dashboard)+ dashboard (lib/normalize ::dashboards.schema/dashboard dashboard)+ changes (lib/normalize ::dashboards.schema/dashboard changes)](u/prog1 (maybe-populate-initially-published-at dashboard)(params/assert-valid-parameters dashboard)(when (:parameters changes)@@ -99,7 +104,7 @@(when (:archived changes)(t2/delete! :model/Pulse :dashboard_id (u/the-id dashboard))))))-(defn- migrate-parameter [p]+(mu/defn- migrate-parameter [p :- ::parameters.schema/parameter](cond-> p;; It was previously possible for parameters to have empty strings for :name and;; :slug, but these are now required to be non-blank strings. (metabase#24500)@@ -111,7 +116,7 @@;; but it was previously possible to set :values_source_type to "static-list" or "card" and still;; have linked filters. (metabase#33892)(some? (:values_source_type p))- (= (:values_query_type p) "none"))+ (= (:values_query_type p) :none));; linked filters don't do anything when parameters have values_query_type="none" (aka "Input box"),;; but it was previously possible to set :values_query_type to "none" and still have linked filters.;; (metabase#34657)@@ -341,7 +346,7 @@[:map[:id ms/NonBlankString][:name ms/NonBlankString]- [:mappings [:maybe [:set dashboard-card/ParamMapping]]]])+ [:mappings [:maybe [:set ::parameters.schema/parameter-mapping]]]])(mu/defn- dashboard->resolved-params :- [:map-of ms/NonBlankString ParamWithMapping][dashboard :- [:map [:parameters [:maybe [:sequential :map]]]]]diff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.cljindex 4545c13a5fb..4e0d3e67cf0 100644--- a/src/metabase/dashboards/models/dashboard_card.clj+++ b/src/metabase/dashboards/models/dashboard_card.clj@@ -5,6 +5,7 @@[metabase.app-db.core :as mdb][metabase.models.interface :as mi][metabase.models.serialization :as serdes]+ [metabase.parameters.core :as parameters][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu]@@ -24,7 +25,7 @@#_(derive :hook/search-index))(t2/deftransforms :model/DashboardCard- {:parameter_mappings mi/transform-parameters-list+ {:parameter_mappings parameters/transform-parameter-mappings:visualization_settings mi/transform-visualization-settings:inline_parameters mi/transform-json})@@ -73,7 +74,7 @@[dashboard-card](t2/instance :model/DashboardCard(-> dashboard-card- (m/update-existing :parameter_mappings mi/normalize-parameters-list)+ (m/update-existing :parameter_mappings parameters/normalize-parameter-mappings)(m/update-existing :visualization_settings mi/normalize-visualization-settings))))(defmethod serdes/hash-fields :model/DashboardCard@@ -190,15 +191,6 @@(update-dashboard-cards-series! {dashcard-id series}))nil)))-(def ParamMapping- "Schema for a parameter mapping as it would appear in the DashboardCard `:parameter_mappings` column."- [:and- [:map-of :keyword :any]- [:map- ;; TODO -- validate `:target` as well... breaks a few tests tho so those will have to be fixed (#40021)- [:parameter_id ms/NonBlankString]- #_[:target :any]]])-(def ^:private NewDashboardCard;; TODO - make the rest of the options explicit instead of just allowing whatever for other keys (#40021)[:mapdiff --git a/src/metabase/dashboards/schema.clj b/src/metabase/dashboards/schema.cljnew file mode 100644index 00000000000..126cbf4ce83--- /dev/null+++ b/src/metabase/dashboards/schema.clj@@ -0,0 +1,26 @@+(ns metabase.dashboards.schema+ (:require+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.parameters.schema :as parameters.schema]+ [metabase.queries.schema :as queries.schema]+ [metabase.util.malli.registry :as mr]))++(mr/def ::dashcard+ [:map+ [:id {:optional true} ::lib.schema.id/dashcard]+ [:card {:optional true} [:ref ::queries.schema/card]]])++(mr/def ::parameters+ [:sequential+ ;; the same as the normal parameters schema, but type is optional here.+ [:merge+ ::parameters.schema/parameter+ [:map+ [:type {:optional true} [:ref ::lib.schema.parameter/type]]]]])++(mr/def ::dashboard+ [:map+ [:id {:optional true} ::lib.schema.id/dashboard]+ [:parameters {:optional true} [:maybe ::parameters]]+ [:dashcards {:optional true} [:maybe [:sequential ::dashcard]]]])diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 6a955cf82b5..ff689abe59e 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -22,6 +22,7 @@[metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.performance :refer [mapv]][potemkin :as p]))@@ -785,7 +786,7 @@Note that it's the same set of `driver-features` with respect toboth database-supports? and [[supports?]])- Database is guaranteed to be a Database instance.+ Database is guaranteed to be a `:metabase.lib.schema.metadata/database` instance.Most drivers can always return true or always return false for a given feature(e.g., :left-join is not supported by any version of Mongo DB).@@ -1194,24 +1195,53 @@[(dispatch-on-initialized-driver driver) (:type transform-details)]):hierarchy #'hierarchy)+(mr/def ::native-query-deps.table-dep+ [:map+ {:closed true}+ [:schema {:optional true} :string]+ [:table [:or+ ;; not really clear how a driver would be able to work out the ID of a Table+ ;; but [[metabase-enterprise.dependencies.native-validation-test/basic-deps-test]] says they can.+ :metabase.lib.schema.id/table+ :string]]])++(mr/def ::native-query-deps.transform-dep+ [:map+ {:closed true}+ [:transform :metabase.lib.schema.id/transform]])++(mr/def ::native-query-deps+ [:set [:or+ ::native-query-deps.table-dep+ ::native-query-deps.transform-dep]])+(defmulti native-query-deps"Gets the table dependencies of a given sql string (or equivalent).- Drivers that support any of the `:transforms/...` features must implement this method."- {:added "0.57.0" :arglists '([driver query] [driver query metadata-provider transforms])}+ Drivers that support any of the `:transforms/...` features must implement this method.++ `query` is a Lib `:metabase.lib.schema/native-only-query`; you can use [[metabase.driver-api.core/raw-native-query]]+ to get the raw native query as needed.++ The return value should match the `:metabase.driver/native-query-deps` schema."+ {:added "0.57.0" :arglists '([driver query])}dispatch-on-initialized-driver:hierarchy #'hierarchy)(defmulti native-result-metadata"Gets the result-metadata for a native query using static analysis (i.e., without actually- going to the database)."- {:added "0.57.0" :arglists '([driver metadata-provider native-query])}+ going to the database).++ `native-query` is a `:metabase.lib.schema/native-only-query`."+ {:added "0.57.0" :arglists '([driver native-query])}dispatch-on-initialized-driver:hierarchy #'hierarchy)(defmulti validate-native-query-fields- "Validates a native query and returns a list of all 'bad' field references."- {:added "0.57.0" :arglists '([driver metadata-provider native-query])}+ "Validates a native query and returns a list of all 'bad' field references.++ `native-query` is a Lib `:metabase.lib.schema/native-only-query`."+ {:added "0.57.0" :arglists '([driver native-query])}dispatch-on-initialized-driver:hierarchy #'hierarchy)diff --git a/src/metabase/driver/common/parameters.clj b/src/metabase/driver/common/parameters.cljindex 5756dc87ae0..c46b433a832 100644--- a/src/metabase/driver/common/parameters.clj+++ b/src/metabase/driver/common/parameters.clj@@ -1,5 +1,9 @@(ns metabase.driver.common.parameters- "Various record types below are used as a convenience for differentiating the different param types."+ "Various record types below are used as a convenience for differentiating the different param types.++ DEPRECATED: `driver.common.parameters.*` namespaces deal with legacy MBQL queries. Migrate to MBQL-5-friendly+ replacement namespaces. The replacement for this namespace is [[metabase.lib.parameters.parse.types]]."+ {:deprecated "0.57.0"}(:require[potemkin.types :as p.types][pretty.core :as pretty]))diff --git a/src/metabase/driver/common/parameters/dates.clj b/src/metabase/driver/common/parameters/dates.cljindex d9b64c76efe..b02f43fa634 100644--- a/src/metabase/driver/common/parameters/dates.clj+++ b/src/metabase/driver/common/parameters/dates.clj@@ -1,55 +1,29 @@(ns metabase.driver.common.parameters.dates- "Shared code for handling datetime parameters, used by both MBQL and native params implementations."+ "Shared code for handling datetime parameters, used by both MBQL and native params implementations.++ DEPRECATED: This namespace will be removed in the near future, in favor+ of [[metabase.query-processor.parameters.dates]] (which uses Lib/MBQL 5).++ TODO (Cam 9/30/25) -- a ton of stuff in this namespace is an exact duplicate of the version+ in [[metabase.query-processor.parameters.dates]], we should remove the version here to encourage migration to that+ namespace."+ {:deprecated "0.57.0"}(:refer-clojure :exclude [every? some])- #_{:clj-kondo/ignore [:metabase/modules]}(:require[clojure.string :as str][java-time.api :as t]- [medley.core :as m][metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.schema.parameter :as lib.schema.parameter][metabase.query-processor.error-type :as qp.error-type]- [metabase.query-processor.timezone :as qp.timezone]- [metabase.util.date-2 :as u.date]+ [metabase.query-processor.parameters.dates :as qp.parameters.dates][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]- [metabase.util.performance :refer [every? some]]- [metabase.util.time :as u.time])- (:import- (java.time.temporal Temporal)))+ [metabase.util.performance :refer [every?]]+ [potemkin :as p]))(set! *warn-on-reflection* true)-(def ^:private temporal-units-regex #"(millisecond|second|minute|hour|day|week|month|quarter|year)")--(def date-exclude-regex- "Regex to match date exclusion values, e.g. exclude-days-Mon, exclude-months-Jan, etc."- (re-pattern (str "exclude-" temporal-units-regex #"s-([-\p{Alnum}]+)")))--(mu/defn date-type?- "Is param type `:date` or some subtype like `:date/month-year`?"- [param-type :- :keyword]- (= (get-in lib.schema.parameter/types [param-type :type]) :date))--(defn not-single-date-type?- "Does date `param-type` represent a range of dates, rather than a single absolute date? (The value may be relative,- such as `past30days`, or absolute, such as `2020-01`.)"- [param-type]- (and (date-type? param-type)- (not (#{:date/single :date} param-type))))--(defn exclusion-date-type- "When date `param-type` represent an exclusion of dates returns the temporal unit that's excluded."- [param-type value]- (when (and (date-type? param-type)- (string? value))- (some->> (re-matches date-exclude-regex value)- second- keyword)))-;; Both in MBQL and SQL parameter substitution a field value is compared to a date range, either relative or absolute.;; Currently the field value is casted to a day (ignoring the time of day), so the ranges should have the same;; granularity level.@@ -57,125 +31,13 @@;; See https://github.com/metabase/metabase/pull/4607#issuecomment-290884313 how we could support;; hour/minute granularity in field parameter queries.-(defn- day-range- [start end]- {:start start :end end :unit :day})--(defn- comparison-range- ([t unit]- (comparison-range t t unit :day))-- ([start end unit]- (comparison-range start end unit :day))-- ([start end unit resolution]- (merge- (u.date/comparison-range start unit :>= {:resolution resolution})- (u.date/comparison-range end unit :<= {:resolution resolution, :end :inclusive})- {:unit unit})))--(defn- second-range- [start end]- (comparison-range start end :second :second))--(defn- minute-range- [start end]- (comparison-range start end :minute :minute))--(defn- hour-range- [start end]- (comparison-range start end :hour :hour))--(defn- week-range [start end]- (comparison-range start end :week))--(defn- month-range [start end]- (comparison-range start end :month))--(defn- year-range [start end]- (comparison-range start end :year))--(defn- relative-quarter-range- [start end]- (comparison-range start end :quarter))--(defn- absolute-quarter-range- [quarter year]- (let [year-quarter (t/year-quarter year (case quarter- "Q1" 1- "Q2" 2- "Q3" 3- "Q4" 4))]- {:start (.atDay year-quarter 1)- :end (.atEndOfQuarter year-quarter)- :unit :quarter}))--(def ^:private operations-by-date-unit- {"second" {:unit-range second-range- :to-period t/seconds}- "minute" {:unit-range minute-range- :to-period t/minutes}- "hour" {:unit-range hour-range- :to-period t/hours}- "day" {:unit-range day-range- :to-period t/days}- "week" {:unit-range week-range- :to-period t/weeks}- "month" {:unit-range month-range- :to-period t/months}- "quarter" {:unit-range relative-quarter-range- :to-period (comp t/months (partial * 3))}- "year" {:unit-range year-range- :to-period t/years}})--(defn- maybe-reduce-resolution [unit dt]- (if (contains? #{"second" "minute" "hour"} unit)- dt- ; for units that are a day or longer, convert back to LocalDate- (t/local-date dt)))--;;; +----------------------------------------------------------------------------------------------------------------+-;;; | DATE STRING DECODERS |-;;; +----------------------------------------------------------------------------------------------------------------+-;; For parsing date strings and producing either a date range (for raw SQL parameter substitution) or a MBQL clause-(defn- expand-parser-groups- [group-label group-value]- (when group-value- (case group-label- :unit (conj (seq (get operations-by-date-unit group-value))- [group-label group-value])- (:int-value :int-value-1) [[group-label (Integer/parseInt group-value)]]- (:date :date-1 :date-2) [[group-label (u.date/parse group-value)]]- [[group-label group-value]])))--(mu/defn- regex->parser :- fn?- "Takes a regex and labels matching the regex capturing groups. Returns a parser which takes a parameter value,- validates the value against regex and gives a map of labels and group values. Respects the following special label- names:-- :unit – finds a matching date unit and merges date unit operations to the result- :int-value, :int-value-1 – converts the group value to integer- :date, :date1, date2 – converts the group value to absolute date"- [regex :- [:fn {:error/message "regular expression"} m/regexp?] group-labels]- (fn [param-value]- (when-let [regex-result (re-matches regex param-value)]- (into {} (mapcat expand-parser-groups group-labels (rest regex-result))))))-;; Decorders consist of:;; 1) Parser which tries to parse the date parameter string;; 2) Range decoder which takes the parser output and produces a date range relative to the given datetime;; 3) Filter decoder which takes the parser output and produces a mbql clause for a given mbql field reference-(def ^:private relative-suffix-regex (re-pattern (format "(|~|-from-([0-9]+)%ss)" temporal-units-regex)))--(defn- include-current?- "Adding a tilde (~) at the end of a past<n><unit>s filter means we should include the current time-unit (e.g. year, day,- week, or month)."- [relative-suffix]- (= "~" relative-suffix))-(defn- with-temporal-unit-if-field[clause unit];; legacy usages -- use Lib in new code going forward@@ -209,14 +71,14 @@;; case of past filters into the past, in the case of next filters into the future.;; The implementation below uses the fact that if the relative suffix is not empty, then the;; include-current flag is true.- {:parser (regex->parser (re-pattern (str #"past([0-9]+)" temporal-units-regex #"s" relative-suffix-regex))- [:int-value :unit :relative-suffix :int-value-1 :unit-1])+ {:parser (#'qp.parameters.dates/regex->parser (re-pattern (str #"past([0-9]+)" @#'qp.parameters.dates/temporal-units-regex #"s" @#'qp.parameters.dates/relative-suffix-regex))+ [:int-value :unit :relative-suffix :int-value-1 :unit-1]):range (fn [{:keys [unit int-value unit-range to-period relative-suffix unit-1 int-value-1]} dt](let [dt-offset (cond-> dt- unit-1 (t/minus ((get-in operations-by-date-unit [unit-1 :to-period]) int-value-1)))- dt-resolution (maybe-reduce-resolution unit dt-offset)]+ unit-1 (t/minus ((get-in @#'qp.parameters.dates/operations-by-date-unit [unit-1 :to-period]) int-value-1)))+ dt-resolution (#'qp.parameters.dates/maybe-reduce-resolution unit dt-offset)](unit-range (t/minus dt-resolution (to-period int-value))- (t/minus dt-resolution (to-period (if (include-current? relative-suffix) 0 1))))))+ (t/minus dt-resolution (to-period (if (#'qp.parameters.dates/include-current? relative-suffix) 0 1)))))):filter (fn [{:keys [unit int-value relative-suffix unit-1 int-value-1]} field-clause](if unit-1@@ -226,15 +88,19 @@(keyword unit)(- int-value-1)(keyword unit-1)]- [:time-interval field-clause (- int-value) (keyword unit) {:include-current (include-current? relative-suffix)}]))}+ [:time-interval+ field-clause+ (- int-value)+ (keyword unit)+ {:include-current (#'qp.parameters.dates/include-current? relative-suffix)}]))}- {:parser (regex->parser (re-pattern (str #"next([0-9]+)" temporal-units-regex #"s" relative-suffix-regex))- [:int-value :unit :relative-suffix :int-value-1 :unit-1])+ {:parser (#'qp.parameters.dates/regex->parser (re-pattern (str #"next([0-9]+)" @#'qp.parameters.dates/temporal-units-regex #"s" @#'qp.parameters.dates/relative-suffix-regex))+ [:int-value :unit :relative-suffix :int-value-1 :unit-1]):range (fn [{:keys [unit int-value unit-range to-period relative-suffix unit-1 int-value-1]} dt](let [dt-offset (cond-> dt- unit-1 (t/plus ((get-in operations-by-date-unit [unit-1 :to-period]) int-value-1)))- dt-resolution (maybe-reduce-resolution unit dt-offset)]- (unit-range (t/plus dt-resolution (to-period (if (include-current? relative-suffix) 0 1)))+ unit-1 (t/plus ((get-in @#'qp.parameters.dates/operations-by-date-unit [unit-1 :to-period]) int-value-1)))+ dt-resolution (#'qp.parameters.dates/maybe-reduce-resolution unit dt-offset)]+ (unit-range (t/plus dt-resolution (to-period (if (#'qp.parameters.dates/include-current? relative-suffix) 0 1)))(t/plus dt-resolution (to-period int-value))))):filter (fn [{:keys [unit int-value relative-suffix unit-1 int-value-1]} field-clause](if unit-1@@ -244,330 +110,118 @@(keyword unit)int-value-1(keyword unit-1)]- [:time-interval field-clause int-value (keyword unit) {:include-current (include-current? relative-suffix)}]))}+ [:time-interval+ field-clause+ int-value+ (keyword unit)+ {:include-current (#'qp.parameters.dates/include-current? relative-suffix)}]))}- {:parser (regex->parser (re-pattern (str #"last" temporal-units-regex))- [:unit])+ {:parser (#'qp.parameters.dates/regex->parser (re-pattern (str #"last" @#'qp.parameters.dates/temporal-units-regex))+ [:unit]):range (fn [{:keys [unit unit-range to-period]} dt]- (let [last-unit (t/minus (maybe-reduce-resolution unit dt) (to-period 1))]+ (let [last-unit (t/minus (#'qp.parameters.dates/maybe-reduce-resolution unit dt) (to-period 1))](unit-range last-unit last-unit))):filter (fn [{:keys [unit]} field-clause][:time-interval field-clause :last (keyword unit)])}- {:parser (regex->parser (re-pattern (str #"this" temporal-units-regex))- [:unit])+ {:parser (#'qp.parameters.dates/regex->parser (re-pattern (str #"this" @#'qp.parameters.dates/temporal-units-regex))+ [:unit]):range (fn [{:keys [unit unit-range]} dt]- (let [dt-adj (maybe-reduce-resolution unit dt)]+ (let [dt-adj (#'qp.parameters.dates/maybe-reduce-resolution unit dt)](unit-range dt-adj dt-adj))):filter (fn [{:keys [unit]} field-clause][:time-interval field-clause :current (keyword unit)])}])-(defn- ->iso-8601-date [t]- (t/format :iso-local-date t))--(defn- ->iso-8601-date-time [t]- (t/format :iso-local-date-time t))-;; TODO - using `range->filter` so much below seems silly. Why can't we just bucket the field and use `:=` clauses?(defn- range->filter[{:keys [start end]} field-clause]- [:between (with-temporal-unit-if-field field-clause :day) (->iso-8601-date start) (->iso-8601-date end)])--(def ^:private short-day->day- {"Mon" :monday- "Tue" :tuesday- "Wed" :wednesday- "Thu" :thursday- "Fri" :friday- "Sat" :saturday- "Sun" :sunday})--(def ^:private short-month->month- (into {}- (map-indexed (fn [i m] [m (inc i)]))- ["Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec"]))--(defn- parse-int-in-range [s min-val max-val]- (try- (let [i (Integer/parseInt s)]- (when (<= min-val i max-val)- i))- (catch NumberFormatException _)))--(defn- excluded-datetime [unit date exclusion]- (let [year (t/year date)]- (case unit- :hour (when-let [hour (parse-int-in-range exclusion 0 23)]- (format "%sT%02d:00:00" date hour))- :day (when-let [day (short-day->day exclusion)]- (str (t/adjust date :next-or-same-day-of-week day)))- :month (when-let [month (short-month->month exclusion)]- (format "%s-%02d-01" year month))- :quarter (when-let [quarter (parse-int-in-range exclusion 1 4)]- (format "%s-%02d-01" year (inc (* 3 (dec quarter)))))- nil)))--(def ^:private excluded-temporal-unit- {:hour :hour-of-day- :day :day-of-week- :month :month-of-year- :quarter :quarter-of-year})--(defn- absolute-date->unit- [date-string]- (if (str/includes? date-string "T")- ;; on the UI you can specify the time up to the minute, so we use minute here- :minute- :day))+ [:between+ (with-temporal-unit-if-field field-clause :day)+ (#'qp.parameters.dates/->iso-8601-date start)+ (#'qp.parameters.dates/->iso-8601-date end)])(def ^:private absolute-date-string-decoders;; year and month- [{:parser (regex->parser #"([0-9]{4}-[0-9]{2})" [:date])+ [{:parser (#'qp.parameters.dates/regex->parser #"([0-9]{4}-[0-9]{2})" [:date]):range (fn [{:keys [date]} _]- (month-range date date))+ (#'qp.parameters.dates/month-range date date)):filter (fn [{:keys [date]} field-clause]- (range->filter (month-range date date) field-clause))}+ (range->filter (#'qp.parameters.dates/month-range date date) field-clause))};; quarter year- {:parser (regex->parser #"(Q[1-4]{1})-([0-9]{4})" [:quarter :year])+ {:parser (#'qp.parameters.dates/regex->parser #"(Q[1-4]{1})-([0-9]{4})" [:quarter :year]):range (fn [{:keys [quarter year]} _]- (absolute-quarter-range quarter (Integer/parseInt year)))+ (#'qp.parameters.dates/absolute-quarter-range quarter (Integer/parseInt year))):filter (fn [{:keys [quarter year]} field-clause]- (range->filter (absolute-quarter-range quarter (Integer/parseInt year))+ (range->filter (#'qp.parameters.dates/absolute-quarter-range quarter (Integer/parseInt year))field-clause))};; single day- {:parser (regex->parser #"([0-9-T:]+)" [:date])+ {:parser (#'qp.parameters.dates/regex->parser #"([0-9-T:]+)" [:date]):range (fn [{:keys [date]} _]- {:start date :end date :unit (absolute-date->unit date)})+ {:start date :end date :unit (#'qp.parameters.dates/absolute-date->unit date)}):filter (fn [{:keys [date]} field-clause]- (let [unit (absolute-date->unit date)+ (let [unit (#'qp.parameters.dates/absolute-date->unit date)iso8601str (case unit- :day (->iso-8601-date date)- :minute (->iso-8601-date-time date))]+ :day (#'qp.parameters.dates/->iso-8601-date date)+ :minute (#'qp.parameters.dates/->iso-8601-date-time date))][:= (with-temporal-unit-if-field field-clause unit) iso8601str]))};; day range- {:parser (regex->parser #"([0-9-T]+)~([0-9-T]+)" [:date-1 :date-2])+ {:parser (#'qp.parameters.dates/regex->parser #"([0-9-T]+)~([0-9-T]+)" [:date-1 :date-2]):range (fn [{:keys [date-1 date-2]} _]- {:start date-1 :end date-2 :unit (absolute-date->unit date-1)})+ {:start date-1 :end date-2 :unit (#'qp.parameters.dates/absolute-date->unit date-1)}):filter (fn [{:keys [date-1 date-2]} field-clause]- [:between (with-temporal-unit-if-field field-clause :day) (->iso-8601-date date-1) (->iso-8601-date date-2)])}+ [:between+ (with-temporal-unit-if-field field-clause :day)+ (#'qp.parameters.dates/->iso-8601-date date-1)+ (#'qp.parameters.dates/->iso-8601-date date-2)])};; datetime range- {:parser (regex->parser #"([0-9-T:]+)~([0-9-T:]+)" [:date-1 :date-2])+ {:parser (#'qp.parameters.dates/regex->parser #"([0-9-T:]+)~([0-9-T:]+)" [:date-1 :date-2]):range (fn [{:keys [date-1 date-2]} _]- {:start date-1, :end date-2 :unit (absolute-date->unit date-1)})+ {:start date-1, :end date-2 :unit (#'qp.parameters.dates/absolute-date->unit date-1)}):filter (fn [{:keys [date-1 date-2]} field-clause]- [:between (with-temporal-unit-if-field field-clause :default)- (->iso-8601-date-time date-1)- (->iso-8601-date-time date-2)])}+ [:between+ (with-temporal-unit-if-field field-clause :default)+ (#'qp.parameters.dates/->iso-8601-date-time date-1)+ (#'qp.parameters.dates/->iso-8601-date-time date-2)])};; before day- {:parser (regex->parser #"~([0-9-T:]+)" [:date])+ {:parser (#'qp.parameters.dates/regex->parser #"~([0-9-T:]+)" [:date]):range (fn [{:keys [date]} _]- {:end date :unit (absolute-date->unit date)})+ {:end date :unit (#'qp.parameters.dates/absolute-date->unit date)}):filter (fn [{:keys [date]} field-clause]- [:< (with-temporal-unit-if-field field-clause :day) (->iso-8601-date date)])}+ [:< (with-temporal-unit-if-field field-clause :day) (#'qp.parameters.dates/->iso-8601-date date)])};; after day- {:parser (regex->parser #"([0-9-T:]+)~" [:date])+ {:parser (#'qp.parameters.dates/regex->parser #"([0-9-T:]+)~" [:date]):range (fn [{:keys [date]} _]- {:start date :unit (absolute-date->unit date)})+ {:start date :unit (#'qp.parameters.dates/absolute-date->unit date)}):filter (fn [{:keys [date]} field-clause]- [:> (with-temporal-unit-if-field field-clause :day) (->iso-8601-date date)])}+ [:> (with-temporal-unit-if-field field-clause :day) (#'qp.parameters.dates/->iso-8601-date date)])};; exclusions- {:parser (regex->parser date-exclude-regex [:unit :exclusions])+ {:parser (#'qp.parameters.dates/regex->parser qp.parameters.dates/date-exclude-regex [:unit :exclusions]):filter (fn [{:keys [unit exclusions]} field-clause](let [unit (keyword unit)- exclusions (map (partial excluded-datetime unit (t/local-date))+ exclusions (map (partial #'qp.parameters.dates/excluded-datetime unit (t/local-date))(str/split exclusions #"-"))](when (and (seq exclusions) (every? some? exclusions))- (into [:!= (with-temporal-unit-if-field field-clause (excluded-temporal-unit unit))] exclusions))))}])+ (into [:!= (with-temporal-unit-if-field field-clause (#'qp.parameters.dates/excluded-temporal-unit unit))]+ exclusions))))}])(def ^:private all-date-string-decoders(concat relative-date-string-decoders absolute-date-string-decoders))-(mu/defn- execute-decoders- "Returns the first successfully decoded value, run through both parser and a range/filter decoder depending on- `decoder-type`. This generates an *inclusive* range by default. The range is adjusted to be exclusive as needed: see- dox for [[date-string->range]] for more details."- [decoders- decoder-type :- [:enum :range :filter]- decoder-param- date-string :- :string]- (some (fn [{parser :parser, parser-result-decoder decoder-type}]- (when-let [parser-result (and parser-result-decoder (parser date-string))]- (parser-result-decoder parser-result decoder-param)))- decoders))--(def ^:private TemporalUnit- (into [:enum] u.date/add-units))--(def ^:private TemporalRange- [:map- [:start {:optional true} (lib.schema.common/instance-of-class Temporal)]- [:end {:optional true} (lib.schema.common/instance-of-class Temporal)]- [:unit TemporalUnit]])--(mu/defn- adjust-inclusive-range-if-needed :- [:maybe TemporalRange]- "Make an inclusive date range exclusive as needed."- [temporal-range :- [:maybe TemporalRange] {:keys [inclusive-start? inclusive-end?]}]- (-> temporal-range- (m/update-existing :start #(if inclusive-start?- %- (u.date/add % (case (:unit temporal-range)- (:year :quarter :month :week :day)- :day- (:unit temporal-range)) -1)))- (m/update-existing :end #(if inclusive-end?- %- (u.date/add % (case (:unit temporal-range)- (:year :quarter :month :week :day)- :day- (:unit temporal-range)) 1)))))--(def ^:private DateStringRange- "Schema for a valid date range returned by `date-string->range`."- [:and [:map {:closed true}- [:start {:optional true} ::lib.schema.common/non-blank-string]- [:end {:optional true} ::lib.schema.common/non-blank-string]]- [:fn {:error/message "must have either :start or :end"}- (fn [{:keys [start end]}]- (or start end))]- [:fn {:error/message ":start must come before :end"}- (fn [{:keys [start end]}]- (or (not start)- (not end)- (not (pos? (compare start end)))))]])--(defn- format-date-range- [date-range]- (-> date-range- (m/update-existing :start u.date/format)- (m/update-existing :end u.date/format)- (dissoc :unit)))--(defn- date-string->raw-range- [date-string]- (let [now (t/local-date-time)]- ;; Relative dates respect the given time zone because a notion like "last 7 days" might mean a different range of- ;; days depending on the user timezone- (or (execute-decoders relative-date-string-decoders :range now date-string)- ;; Absolute date ranges don't need the time zone conversion because in SQL the date ranges are compared- ;; against the db field value that is casted granularity level of a day in the db time zone- (execute-decoders absolute-date-string-decoders :range nil date-string)- ;; if both of the decoders above fail, then the date string is invalid- (throw (ex-info (tru "Don''t know how to parse date param ''{0}'' — invalid format" date-string)- {:param date-string- :type qp.error-type/invalid-parameter})))))--(mu/defn date-string->range :- DateStringRange- "Takes a string description of a date range such as `lastmonth` or `2016-07-15~2016-08-6` and returns a map with- `:start` and/or `:end` keys, as ISO-8601 *date* strings. By default, `:start` and `:end` are inclusive,-- e.g:- (date-string->range \"past2days\") ; -> {:start \"2020-01-20\", :end \"2020-01-21\"}-- intended for use with SQL like-- WHERE date(some_column) BETWEEN date '2020-01-20' AND date '2020-01-21'-- which is *INCLUSIVE*. If the filter clause you're generating is not inclusive, pass the `:inclusive-start?` or- `:inclusive-end?` options as needed to generate an appropriate range.-- Note that some ranges are open-ended on one side, and will have only a `:start` or an `:end`."- ;; 1-arg version returns inclusive start/end; 2-arg version can adjust as needed- ([date-string]- (date-string->range date-string nil))-- ([date-string :- ::lib.schema.common/non-blank-string- {:keys [inclusive-start? inclusive-end?]- :or {inclusive-start? true inclusive-end? true}}]- (let [options {:inclusive-start? inclusive-start?, :inclusive-end? inclusive-end?}]- (-> (date-string->raw-range date-string)- (adjust-inclusive-range-if-needed options)- format-date-range))))--(defn- date-str->qp-aware-offset-dt- "Generate offset datetime from `date-str` with respect to qp's `results-timezone`."- [date-str]- (when date-str- (let [[y M d h m s] (u.time/yyyyMMddhhmmss->parts date-str)]- (try (.toOffsetDateTime (t/zoned-date-time y M d h m s 0 (t/zone-id (qp.timezone/results-timezone-id))))- (catch Throwable _- (t/offset-date-time y M d h m s 0 (t/zone-offset (qp.timezone/results-timezone-id))))))))--(defn- date-str->local-dt- "Generate local datetime from `date-str`"- [date-str]- (when date-str- (let [[y M d h m s] (u.time/yyyyMMddhhmmss->parts date-str)]- (t/local-date-time y M d h m s 0))))--(defn- date-str->unit-fn- "Return appropriate function for interval end adjustments in [[exclusive-datetime-range-end]]."- [date-str]- (when date-str- (if (re-matches u.time/local-date-regex date-str)- t/days- t/minutes)))--(defn- exclusive-datetime-range-end- "Transform `end-dt` OffsetDateTime to appropriate range end.-- Context. Datetime range is required for `FieldFilter`s on `:type/DateTime` fields (see the- [[metabase.driver.sql.parameters.substitution/field-filter->replacement-snippet-info]]) instead of _Date Range_- available from [[date-string->range]].-- [[date-string->range]] returns interval of dates. [[date-str->datetime-range]] modifies the interval to consist- of datetimes. By adding 0 temporal padding the end interval has to be adjusted."- [end-dt unit-fn]- (when (and end-dt unit-fn)- (t/+ end-dt (unit-fn 1))))--(defn- fallback-raw-range- "Try to extract date time value if [[date-string->range]] fails."- [date-str]- (let [date-str (first (re-find #"\d+-\d+-\d+T?(\d?+)?(:\d+)?(:\d+)?" date-str))]- {:start date-str- :end date-str}))--(defn- maybe-adjust-open-range- [{:keys [start end] :as range} unit-fn]- (assert (some some? [start end]))- (cond (and start end) range- start (update range :start t/+ (unit-fn 1))- end (update range :end t/- (unit-fn 1))))--(mu/defn date-str->datetime-range :- DateStringRange- "Generate range from `date-range-str`.-- First [[date-string->range]] generates range for dates (inclusive by default). Operating on that range,- this function:- 1. converts dates to LocalDateTime or OffsetDateTime respecting qp's timezone, adding zero temporal padding,- 2. updates range to correct _end-exclusive datetime_*- 3. formats the range.-- This function is meant to be used for generating inclusive intervals for `:type/DateTime` field filters.-- * End-exclusive gte lt filters are generated for `:type/DateTime` fields."- [raw-date-str field-type]- (let [;; `raw-date-str` is sanitized in case it contains millis and timezone which are incompatible- ;; with [[date-string->range]]. `substitute-field-filter-test` expects that to happen.- [range-raw unit] (try (let [r (date-string->raw-range raw-date-str)]- [(format-date-range r) (:unit r)])- (catch Throwable _- [(fallback-raw-range raw-date-str) :day]))- date-str-conversion (if (isa? field-type :type/DateTimeWithTZ)- date-str->qp-aware-offset-dt- date-str->local-dt)]- (-> (update-vals range-raw date-str-conversion)- (m/update-existing :end exclusive-datetime-range-end (or ({:second t/seconds, :hour t/hours} unit)- (date-str->unit-fn (:end range-raw))))- (maybe-adjust-open-range (date-str->unit-fn ((some-fn :start :end) range-raw)))- format-date-range)))-(mu/defn date-string->filter :- mbql.s/Filter"Takes a string description of a *date* (not datetime) range such as 'lastmonth' or '2016-07-15~2016-08-6', oran absolute date *or datetime* string, and returns a corresponding MBQL filter clause for a given field reference."[date-string :- :stringfield :- [:or ::lib.schema.id/field mbql.s/Field]]- (or (execute-decoders all-date-string-decoders :filter (mbql.u/wrap-field-id-if-needed field) date-string)+ (or (#'qp.parameters.dates/execute-decoders all-date-string-decoders :filter (mbql.u/wrap-field-id-if-needed field) date-string)(throw (ex-info (tru "Don''t know how to parse date string {0}" (pr-str date-string)){:type qp.error-type/invalid-parameter:date-string date-string}))))++(p/import-vars+ [qp.parameters.dates+ date-str->datetime-range+ not-single-date-type?+ date-exclude-regex+ date-type?+ date-string->range+ exclusion-date-type])diff --git a/src/metabase/driver/common/parameters/operators.clj b/src/metabase/driver/common/parameters/operators.cljindex ef554d6c1ba..f55e95f7fd4 100644--- a/src/metabase/driver/common/parameters/operators.clj+++ b/src/metabase/driver/common/parameters/operators.clj@@ -6,19 +6,23 @@[:field26{:source-field 5}]]- :value [3 5]}"- #_{:clj-kondo/ignore [:metabase/modules]}+ :value [3 5]}++ DEPRECATED: use [[metabase.query-processor.parameters.operators]] going forward."+ {:deprecated "0.57.0"}(:require[metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.query-processor.error-type :as qp.error-type]+ [metabase.query-processor.parameters.operators :as qp.params.ops][metabase.util.i18n :refer [tru]]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [potemkin :as p]))-(mu/defn- operator-arity :- [:maybe [:enum :unary :binary :variadic]]- [param-type]- (get-in lib.schema.parameter/types [param-type :operator]))+(p/import-vars+ [qp.params.ops+ operator?])(defn- operator-options-fn[param-type]@@ -28,11 +32,6 @@(cond-> clauseoptions (conj options)))))-(defn operator?- "Returns whether param-type is an \"operator\" type."- [param-type]- (boolean (operator-arity param-type)))-(mu/defn- verify-type-and-arity[field param-type param-value](letfn [(maybe-arity-error [n]@@ -43,7 +42,7 @@:param-value param-value:field-id (second field):type qp.error-type/invalid-parameter}))))]- (condp = (operator-arity param-type)+ (condp = (#'qp.params.ops/operator-arity param-type):unary(maybe-arity-error 1)@@ -64,25 +63,15 @@:field-id (second field):type qp.error-type/invalid-parameter})))))-(defn- normalize-param- [param]- (case (:type param)- :number/between- (let [[l u] (:value param)]- (cond-> param- (nil? u) (assoc :type :number/>=, :value [l])- (nil? l) (assoc :type :number/<=, :value [u])))- param))-(mu/defn to-clause :- ::mbql.s/Filter"Convert an operator style parameter into an mbql clause. Will also do arity checks and throws an ex-info with`:type qp.error-type/invalid-parameter` if arity is incorrect."[param :- ::lib.schema.parameter/parameter]- (let [{param-type :type, [a b :as param-value] :value, [_ field] :target, options :options} (normalize-param param)]+ (let [{param-type :type, [a b :as param-value] :value, [_ field] :target, options :options} (#'qp.params.ops/normalize-param param)](verify-type-and-arity field param-type param-value)(let [field' (mbql.u/wrap-field-id-if-needed field)opts-fn (operator-options-fn param-type)]- (case (operator-arity param-type)+ (case (#'qp.params.ops/operator-arity param-type):binary (opts-fn [(keyword (name param-type)) field' a b] options):unary (opts-fn [(keyword (name param-type)) field' a] options):variadic (opts-fn (into [(keyword (name param-type)) field'] param-value) options)diff --git a/src/metabase/driver/common/parameters/parse.clj b/src/metabase/driver/common/parameters/parse.cljindex 69ba2b4151b..2cd21ac4d6e 100644--- a/src/metabase/driver/common/parameters/parse.clj+++ b/src/metabase/driver/common/parameters/parse.clj@@ -1,9 +1,11 @@(ns metabase.driver.common.parameters.parse- #_{:clj-kondo/ignore [:metabase/modules]}+ "DEPRECATED: `driver.common.parameters.*` namespaces deal with legacy MBQL queries. Migrate to MBQL-5-friendly+ replacement namespaces. The replacement for this namespace is [[metabase.lib.parameters.parse]]."+ {:deprecated "0.57.0"}(:require[clojure.core.match :refer [match]][clojure.string :as str]- [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params][metabase.lib.core :as lib][metabase.lib.schema.common :as lib.schema.common][metabase.query-processor.error-type :as qp.error-type]diff --git a/src/metabase/driver/common/parameters/values.clj b/src/metabase/driver/common/parameters/values.cljindex a1639e4aff7..52bef73a30a 100644--- a/src/metabase/driver/common/parameters/values.clj+++ b/src/metabase/driver/common/parameters/values.clj@@ -7,12 +7,20 @@;; -> {\"checkin_date\" {:field {:name \"date\", :parent_id nil, :table_id 1375}:param {:type \"date/range\":target [\"dimension\" [\"template-tag\" \"checkin_date\"]]- :value \"2015-01-01~2016-09-01\"}}}"+ :value \"2015-01-01~2016-09-01\"}}}+++ DEPRECATED: `driver.common.parameters.*` namespaces deal with legacy MBQL queries. Migrate to MBQL-5-friendly+ replacement namespaces. The replacement for this namespace is [[metabase.query-processor.parameters.values]].++ TODO (Cam 10/3/25) -- that namespace was introduced in #61158, but then I removed it in a subsequent PR to prune+ unused namespaces. We can't migrate to it if it's gone... please restore it when we start migrating usages of it+ over."+ {:deprecated "0.57.0"}(:refer-clojure :exclude [every? some mapv])- #_{:clj-kondo/ignore [:metabase/modules]}(:require[clojure.string :as str]- [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params][metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -24,13 +32,13 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.limit :as limit]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.persisted-cache :as qp.persistence][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [every? some mapv]])+ [metabase.util.performance :refer [every? mapv some]])(:import(clojure.lang ExceptionInfo)(java.util UUID)))@@ -452,7 +460,11 @@(query->params-map some-inner-query)->{:checkin_date #t \"2019-09-19T23:30:42.233-07:00\"}"- [{tags :template-tags, params :parameters} :- :map]+ [{tags :template-tags, params :parameters, :as _inner-query} :- [:and+ [:map]+ [:fn+ {:error/message "should be a legacy inner query"}+ (complement :lib/type)]]](log/tracef "Building params map out of tags\n%s\nand params\n%s\n" (u/pprint-to-str tags) (u/pprint-to-str params))(try(into {} (for [[k tag] tagsdiff --git a/src/metabase/driver/common/table_rows_sample.clj b/src/metabase/driver/common/table_rows_sample.cljindex dd7348e8816..b3f9353141c 100644--- a/src/metabase/driver/common/table_rows_sample.clj+++ b/src/metabase/driver/common/table_rows_sample.clj@@ -1,20 +1,23 @@(ns metabase.driver.common.table-rows-sample- #_{:clj-kondo/ignore [:metabase/modules]}(:require[metabase.driver :as driver][metabase.driver.util :as driver.u]- [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor :as qp]- [metabase.util :as u]+ [metabase.query-processor.schema :as qp.schema][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr];; TODO -- for historical reasons this stuff uses the Toucan models instead of the QP Metadata Store and;; `:metadata/*` models -- at some point we should fix this. [[driver/table-rows-sample]] is called by sync however;; so we need to go in and update the sync code as well.^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.util.malli.schema :as ms]- [metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]- ^{:clj-kondo/ignore [:discouraged-namespace]}- [toucan2.core :as t2]))+ [metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]))(def max-sample-rows"The maximum number of values we should return when using `table-rows-sample`. This many is probably fine for@@ -25,45 +28,56 @@"Number of rows to sample for tables with nested (e.g., JSON) columns."500)-(def ^:private TableRowsSampleOptions+(mr/def ::table-rows-sample.options"Schema for `table-rows-sample` options"[:maybe[:map[:truncation-size {:optional true} :int][:limit {:optional true} :int]- [:order-by {:optional true} [:ref ::mbql.s/OrderBys]]+ [:order-by {:optional true} [:maybe [:sequential :some]]] ; something that can be passed to [[metabase.lib.core/order-by]][:rff {:optional true} fn?]]])-(defn- table-rows-sample-query+(mu/defn- table-rows-sample-query :- ::lib.schema/query"Returns the mbql query to query a table for sample rows"- [table- fields- {:keys [truncation-size limit order-by] :or {limit max-sample-rows} :as _opts}]- (let [database (t2/select-one :model/Database (:db_id table))- driver (driver.u/database->driver database)- text-fields (filter (comp #{:type/Text} (some-fn :effective_type :base_type)) fields)- field->expressions (when (and truncation-size (driver.u/supports? driver :expressions database))- (into {} (for [field text-fields]- [field [(str (gensym "substring"))- [:substring [:field (u/the-id field) nil]- 1 truncation-size]]])))- expressions (into {} (vals field->expressions))]- {:database (:db_id table)- :type :query- :query (cond-> {:source-table (u/the-id table)- :expressions expressions- :fields (vec (for [field fields]- (if-let [[expression-name _] (get field->expressions field)]- [:expression expression-name]- [:field (u/the-id field) nil])))- :limit limit}- order-by- (assoc :order-by order-by)+ [mp :- ::lib.schema.metadata/metadata-provider+ table :- ::lib.schema.metadata/table+ cols :- [:maybe [:sequential ::lib.schema.metadata/column]]+ {:keys [truncation-size limit order-by] :or {limit max-sample-rows} :as _opts} :- ::table-rows-sample.options]+ (let [database (lib.metadata/database mp)+ driver (driver.u/database->driver database)+ col->expression-name (if (and truncation-size (driver.u/supports? driver :expressions database))+ (into {}+ (comp (filter (fn [field]+ ;; only do this for columns that are `:type/Text`, not for anything derived from+ ;; `:type/Text`+ (= ((some-fn :effective-type :base-type) field) :type/Text)))+ (map (juxt identity (fn [_field]+ (str (gensym "substring"))))))+ cols)+ {})]+ (-> (lib/query mp table)+ (as-> $query (transduce+ (filter col->expression-name)+ (completing+ (fn [query col]+ (let [expression-name (get col->expression-name col)+ expression (lib/substring col 1 truncation-size)]+ (lib/expression query expression-name expression))))+ $query+ cols))+ (as-> $query (lib/with-fields $query (for [col cols]+ (if-let [expression-name (get col->expression-name col)]+ (lib/expression-ref $query expression-name)+ col))))+ (lib/limit limit)+ (cond-> (seq order-by)+ (as-> $query (reduce lib/order-by $query order-by)))+ (assoc :middleware {:format-rows? false+ :skip-results-metadata? true})+ schema.metadata-queries/add-required-filters-if-needed)))- true- schema.metadata-queries/add-required-filters-if-needed)- :middleware {:format-rows? false- :skip-results-metadata? true}}))+;;; TODO (Cam 9/30/25) -- at some point we should update this stuff to use Lib-style metadata instead of Toucan+;;; instances(mu/defn table-rows-sample"Run a basic MBQL query to fetch a sample of rows of `fields` belonging to a `table`.@@ -77,11 +91,19 @@rff](table-rows-sample table fields rff nil))- ([table :- (ms/InstanceOf :model/Table)+ ([table :- [:and+ (ms/InstanceOf :model/Table)+ [:map+ [:id ::lib.schema.id/table]+ [:db_id ::lib.schema.id/database]]]fields :- [:sequential (ms/InstanceOf :model/Field)]- rff :- fn?- opts :- TableRowsSampleOptions]- (let [query (table-rows-sample-query table fields opts)]+ rff :- ::qp.schema/rff+ opts :- ::table-rows-sample.options]+ (let [database-id (:db_id table)+ mp (lib-be/application-database-metadata-provider database-id)+ table (lib-be/instance->metadata table :metadata/table)+ fields (map #(lib-be/instance->metadata % :metadata/column) fields)+ query (table-rows-sample-query mp table fields opts)](qp/process-query query rff))))(defmethod driver/table-rows-sample :defaultdiff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.cljindex b9e418a4129..abc23aae653 100644--- a/src/metabase/driver/h2.clj+++ b/src/metabase/driver/h2.clj@@ -17,18 +17,16 @@[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql.normalize :as sql.normalize][metabase.driver.sql.query-processor :as sql.qp]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [some every?]])+ [metabase.util.performance :refer [every? some]])(:import- (java.sql- Clob- ResultSet- ResultSetMetaData- SQLException)+ (java.sql Clob ResultSet ResultSetMetaData SQLException)(java.time OffsetTime)(org.h2.command CommandInterface Parser)(org.h2.engine SessionLocal)))@@ -157,7 +155,8 @@(let [[_ {:strs [USER]}] (connection-string->file+options db)]USER)))-(defn- check-native-query-not-using-default-user [{query-type :type, :as query}]+(mu/defn- check-native-query-not-using-default-user [{query-type :type, :as query} :- [:map+ [:type [:enum :native :query]]]](u/prog1 query;; For :native queries check to make sure the DB in question has a (non-default) NAME property specified in the;; connection string. We don't allow SQL execution on H2 databases for the default admin account for security@@ -173,10 +172,18 @@(defn- make-h2-parser"Returns an H2 Parser object for the given (H2) database ID"- ^Parser [h2-db-id]- (with-open [conn (.getConnection (sql-jdbc.execute/datasource-with-diagnostic-info! :h2 h2-db-id))]+ ^Parser [h2-database-or-id]+ (with-open [conn (.getConnection (sql-jdbc.execute/datasource-with-diagnostic-info! :h2 h2-database-or-id))];; The H2 Parser class is created from the H2 JDBC session, but these fields are not public- (let [inner (.unwrap conn java.sql.Connection) ;; May be a wrapper, get the innermost object that has session field+ (let [^org.h2.jdbc.JdbcConnection inner (try+ ;; May be a wrapper, get the innermost object that has session field+ (u/prog1 (.unwrap conn org.h2.jdbc.JdbcConnection)+ (assert (instance? org.h2.jdbc.JdbcConnection <>)))+ (catch java.sql.SQLException e+ (throw (ex-info "Not an H2 connection. Are we sure this is an H2 database?"+ {:database h2-database-or-id+ :conn conn}+ e))))session (get-field inner "session")];; Only SessionLocal represents a connection we can create a parser with. Remote sessions and other;; session types are ignored.@@ -196,8 +203,11 @@- Each `command-type` corresponds to a value in org.h2.command.CommandInterface, and match the commands from `query` in order.- `remaining-sql` is a nillable sql string that is unable to be classified without running preceding queries first.Usually if `remaining-sql` exists we will deny the query."- [database query]- (when-let [h2-parser (make-h2-parser database)]+ [database-or-id :- [:or+ ::lib.schema.id/database+ ::lib.schema.metadata/database]+ ^String query :- :string]+ (when-let [h2-parser (make-h2-parser database-or-id)](try(let [command (.prepareCommand h2-parser query)first-command-type (.getCommandType command)@@ -257,10 +267,14 @@CommandInterface/CALL} cmd-type-nums)(nil? remaining-sql)))))-(defn- check-read-only-statements [{{:keys [query]} :native}]- (when query+(mu/defn- check-read-only-statements [{{sql :query} :native, :as _query} :- [:map+ [:type [:enum :query :native]]+ [:native+ [:map+ [:query string?]]]]]+ (when sql(let [query-classification (classify-query (driver-api/database (driver-api/metadata-provider))- query)]+ sql)](when-not (read-only-statements? query-classification)(throw (ex-info "Only SELECT statements are allowed in a native query."{:classification query-classification}))))))diff --git a/src/metabase/driver/impl.clj b/src/metabase/driver/impl.cljindex 2452708ed8e..b99b32dda91 100644--- a/src/metabase/driver/impl.clj+++ b/src/metabase/driver/impl.clj@@ -1,9 +1,8 @@(ns metabase.driver.impl"Internal implementation functions for [[metabase.driver]]. These functions live in a separate namespace to reduce theclutter in [[metabase.driver]] itself."- #_{:clj-kondo/ignore [:metabase/modules]}(:require- [metabase.classloader.impl :as classloader]+ [metabase.classloader.core :as classloader][metabase.lib.util :as lib.util][metabase.util :as u][metabase.util.i18n :refer [trs tru]]diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex a735196d350..f6ae81e4aba 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -3,11 +3,13 @@(:refer-clojure :exclude [some])(:require[clojure.set :as set]+ ;; TODO (Cam 10/1/25) -- Isn't having drivers use Macaw directly against the spirt of all the work we did to make a+ ;; Driver API namespace?[macaw.core :as macaw][metabase.driver :as driver][metabase.driver-api.core :as driver-api]- [metabase.driver.common.parameters.parse :as params.parse]- [metabase.driver.common.parameters.values :as params.values]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.values :as params.values][metabase.driver.sql.normalize :as sql.normalize][metabase.driver.sql.parameters.substitute :as sql.params.substitute][metabase.driver.sql.parameters.substitution :as sql.params.substitution]@@ -173,26 +175,25 @@{:table (sql.normalize/normalize-name driver table):schema (some->> schema (sql.normalize/normalize-name driver))})-(defmethod driver/native-query-deps :sql- ([driver query]- (driver/native-query-deps driver query- (driver-api/metadata-provider)))- ([driver query metadata-provider]- (let [db-tables (driver-api/tables metadata-provider)- db-transforms (driver-api/transforms metadata-provider)]- (->> query- macaw/parsed-query- macaw/query->components- :tables- (map :component)- (into #{} (keep #(->> (normalize-table-spec driver %)- (find-table-or-transform driver db-tables db-transforms))))))))+(mu/defmethod driver/native-query-deps :sql :- ::driver/native-query-deps+ [driver :- :keyword+ query :- :metabase.lib.schema/native-only-query]+ (let [db-tables (driver-api/tables query)+ db-transforms (driver-api/transforms query)]+ (->> query+ driver-api/raw-native-query+ macaw/parsed-query+ macaw/query->components+ :tables+ (map :component)+ (into #{} (keep #(->> (normalize-table-spec driver %)+ (find-table-or-transform driver db-tables db-transforms)))))));;; +----------------------------------------------------------------------------------------------------------------+;;; | Dependencies |;;; +----------------------------------------------------------------------------------------------------------------+-(defmulti resolve-field+(defmulti ^:private resolve-field"Resolves a field reference to one or more actual database fields.This uses a supplied metadata provider instead of hitting the db directly. 'Field reference' refers to the field@@ -271,25 +272,31 @@:effective-type (apply lca :type/* (map :effective-type member-fields)):semantic-type (apply lca :Semantic/* (map :semantic-type member-fields))}]))-(defmethod driver/native-result-metadata :sql- [driver metadata-provider native-query]- (let [{:keys [returned-fields]} (->> (macaw/parsed-query native-query)+(mu/defmethod driver/native-result-metadata :sql+ [driver :- :keyword+ native-query :- :metabase.lib.schema/native-only-query]+ (let [{:keys [returned-fields]} (->> native-query+ driver-api/raw-native-query+ macaw/parsed-querymacaw/->ast(sql.references/field-references driver))]- (->> (mapcat (partial resolve-field driver metadata-provider) returned-fields)+ (->> (mapcat (partial resolve-field driver native-query) returned-fields)(remove ::bad-reference))))-(defmethod driver/validate-native-query-fields :sql- [driver metadata-provider native-query]- (let [{:keys [used-fields returned-fields bad-sql]} (->> (macaw/parsed-query native-query)+(mu/defmethod driver/validate-native-query-fields :sql+ [driver :- :keyword+ native-query :- :metabase.lib.schema/native-only-query]+ (let [{:keys [used-fields returned-fields bad-sql]} (->> native-query+ driver-api/raw-native-query+ macaw/parsed-querymacaw/->ast(sql.references/field-references driver))check-fields #(mapcat (fn [col-spec]- (->> (resolve-field driver metadata-provider col-spec)+ (->> (resolve-field driver (driver-api/->metadata-provider native-query) col-spec)(filter ::bad-reference)))%)](-> (concat (when bad-sql- [{:error :bad-sql}])+ [{:error ::bad-sql}])(check-fields used-fields)(check-fields returned-fields))distinct)))diff --git a/src/metabase/driver/sql/parameters/substitute.clj b/src/metabase/driver/sql/parameters/substitute.cljindex 12843aca42f..e78c356a8f9 100644--- a/src/metabase/driver/sql/parameters/substitute.clj+++ b/src/metabase/driver/sql/parameters/substitute.clj@@ -3,8 +3,8 @@[clojure.string :as str][metabase.driver :as driver][metabase.driver-api.core :as driver-api]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.parse :as params.parse]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.util :as u][metabase.util.i18n :refer [tru]]diff --git a/src/metabase/driver/sql/parameters/substitution.clj b/src/metabase/driver/sql/parameters/substitution.cljindex 2f050af0053..af5855b2a2d 100644--- a/src/metabase/driver/sql/parameters/substitution.clj+++ b/src/metabase/driver/sql/parameters/substitution.clj@@ -10,9 +10,9 @@[clojure.string :as str][metabase.driver :as driver][metabase.driver-api.core :as driver-api]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.dates :as params.dates]- [metabase.driver.common.parameters.operators :as params.ops]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.dates :as params.dates]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.operators :as params.ops][metabase.driver.sql.query-processor :as sql.qp][metabase.util :as u][metabase.util.date-2 :as u.date]diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex be352f69d23..b0fd6545129 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -1,7 +1,6 @@(ns metabase.driver.util"Utility functions for common operations on drivers."(:refer-clojure :exclude [mapv])- #_{:clj-kondo/ignore [:metabase/modules]}(:require[clojure.core.memoize :as memoize][clojure.set :as set]@@ -11,19 +10,20 @@[metabase.config.core :as config][metabase.driver :as driver][metabase.driver.settings :as driver.settings]+ [metabase.lib-be.core :as lib-be][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.premium-features.core :as premium-features][metabase.query-processor.error-type :as qp.error-type]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.i18n :refer [deferred-tru trs]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :as perf :refer [mapv]]- [metabase.util.snake-hating-map :refer [snake-hating-map?]])+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.util.malli.schema :as ms]+ [metabase.util.performance :as perf :refer [mapv]])(:import(java.io ByteArrayInputStream)(java.security KeyFactory KeyStore PrivateKey)@@ -219,18 +219,38 @@(-> supports?*(vary-meta assoc ::memoize/args-fn(fn [[driver feature database]]- [driver feature (mdb/unique-identifier) (:id database)- (if (snake-hating-map? database)- (:updated-at database)- (:updated_at database))])))))--(defn supports?- "A defensive wrapper around [[database-supports?]]. It adds logging, caching, and error handling to avoid crashing the app- if this method takes a long time to execute or throws an exception. This is useful because `supports?` is used in so many- critical places in the app, and we don't want a single driver to crash the app if it throws an exception, or delay the user- if it takes a long time to execute."- [driver feature database]- (let [f (if *memoize-supports?* memoized-supports?* supports?*)]+ [driver feature (mdb/unique-identifier) (:id database) (:updated-at database)])))))++;;; this can get called in post-select which doesn't always have ID+(mu/defn- ensure-lib-database :- [:map+ [:lib/type [:= :metadata/database]]]+ [database :- [:or+ [:map+ [:lib/type [:= :metadata/database]]]+ (ms/InstanceOf :model/Database)]]+ (if-not (:lib/type database)+ (lib-be/instance->metadata database :metadata/database)+ database))++(mu/defn supports?+ "A defensive wrapper around [[metabase.driver/database-supports?]]. It adds logging, caching, and error handling to+ avoid crashing the app if this method takes a long time to execute or throws an exception. This is useful because+ `supports?` is used in so many critical places in the app, and we don't want a single driver to crash the app if it+ throws an exception, or delay the user if it takes a long time to execute.++ TODO -- this is only supposed to be called with a Lib-style Database metadata; if this is called with a Toucan+ instance that is incorrect. I don't have the time to fix every single incorrect usage, so we'll just log an error+ and move on for now."+ [driver :- :keyword+ feature :- :keyword+ database :- [:maybe+ [:or+ ;; this can get called with an incomplete object in post-select+ [:map+ [:lib/type [:= :metadata/database]]]+ (ms/InstanceOf :model/Database)]]]+ (let [database (some-> database ensure-lib-database)+ f (if *memoize-supports?* memoized-supports?* supports?*)](f driver feature database)))(def ^:private skip-internal-features@@ -247,21 +267,24 @@(-> features*(vary-meta assoc ::memoize/args-fn(fn [[driver database]]- [driver (mdb/unique-identifier) (:id database)- (if (snake-hating-map? database)- (:updated-at database)- (:updated_at database))])))))+ [driver (mdb/unique-identifier) (:id database) (:updated-at database)])))))-(defn features+(mu/defn features"Return a set of all features supported by `driver` with respect to `database`."- [driver database]- (let [f (if *memoize-supports?* memoized-features* features*)]+ [driver :- :keyword+ database :- [:or+ ;; this can get called in post-select which doesn't always have ID+ [:map+ [:lib/type [:= :metadata/database]]]+ (ms/InstanceOf :model/Database)]]+ (let [database (ensure-lib-database database)+ f (if *memoize-supports?* memoized-features* features*)](f driver database)))-(defn- supported-in-environment?+(mu/defn- supported-in-environment?"Returns true if a driver is supported in the the current metabase environment. As implemented this just disallows thesqlite driver on hosted metabase because hosted metabase does not support uploading a SQLite file for use."- [driver]+ [driver :- :keyword](or (not (premium-features/is-hosted?))(not= :sqlite (keyword driver))))diff --git a/src/metabase/driver_api/core.clj b/src/metabase/driver_api/core.cljindex 2ec9e1d06b4..360ed392f33 100644--- a/src/metabase/driver_api/core.clj+++ b/src/metabase/driver_api/core.clj@@ -3,8 +3,9 @@;; this is actually ok here since this is a drivers namespace{:discouraged-namespace {metabase.query-processor.store {:level :off}};; this is also ok here since this is a drivers namespace- :discouraged-var {metabase.lib.core/->legacy-MBQL {:level :off}}}}}- (:refer-clojure :exclude [replace compile require])+ :discouraged-var {metabase.lib.core/->legacy-MBQL {:level :off}}+ :missing-docstring {:level :off}}}}+ (:refer-clojure :exclude [replace compile])(:require[metabase.actions.core :as actions][metabase.api.common :as api]@@ -19,7 +20,6 @@[metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u][metabase.lib-be.core :as lib-be]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm][metabase.lib.core :as lib][metabase.lib.field :as lib.field][metabase.lib.metadata :as lib.metadata]@@ -49,7 +49,7 @@[metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.setup :as qp.setup]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.query-processor.util :as qp.util][metabase.query-processor.util.add-alias-info :as add]@@ -62,7 +62,6 @@[metabase.sync.util :as sync-util][metabase.system.core :as system][metabase.upload.core :as upload]- [metabase.warehouse-schema.metadata-queries :as schema.metadata-queries][metabase.warehouse-schema.models.table :as table][potemkin :as p]))@@ -100,7 +99,7 @@events/publish-event!lib-be/start-of-weeklib.field/json-field?- lib.metadata.jvm/instance->metadata+ lib-be/instance->metadatalib.metadata/databaselib.metadata/fieldlib.metadata/fields@@ -116,7 +115,10 @@lib.util.match/replacelib.util/truncate-aliaslib/->legacy-MBQL+ lib/->metadata-provider+ lib/order-by-clauselib/query-from-legacy-inner-query+ lib/raw-native-querylimit/absolute-max-resultslimit/determine-query-max-rowslogger/level-enabled?@@ -171,7 +173,6 @@qp.wrap-value-literals/wrap-value-literals-in-mbqlqp.writeback/execute-write-sql!qp/process-query- schema.metadata-queries/add-required-filters-if-neededsecrets/clean-secret-properties-from-detailssecrets/uploaded-base-64-prefix-patternsetting/defsetting@@ -189,22 +190,19 @@to see if the query has been canceled."[]qp.pipeline/*canceled-chan*)--#_{:clj-kondo/ignore [:missing-docstring]};; should use import-vars :rename once https://github.com/clj-kondo/clj-kondo/issues/2498 is fixed-(do- (p/import-fn setting/get-value-of-type setting-get-value-of-type)- (p/import-fn secrets/value-as-string secret-value-as-string)- (p/import-fn secrets/value-as-file! secret-value-as-file!)- (p/import-fn table/database table->database)-- (p/import-def qp.error-type/db qp.error-type.db)- (p/import-def qp.error-type/driver qp.error-type.driver)- (p/import-def qp.error-type/invalid-parameter qp.error-type.invalid-parameter)- (p/import-def qp.error-type/invalid-query qp.error-type.invalid-query)- (p/import-def qp.error-type/missing-required-parameter qp.error-type.missing-required-parameter)- (p/import-def qp.error-type/qp qp.error-type.qp)- (p/import-def qp.error-type/unsupported-feature qp.error-type.unsupported-feature))+(p/import-fn setting/get-value-of-type setting-get-value-of-type)+(p/import-fn secrets/value-as-string secret-value-as-string)+(p/import-fn secrets/value-as-file! secret-value-as-file!)+(p/import-fn table/database table->database)++(p/import-def qp.error-type/db qp.error-type.db)+(p/import-def qp.error-type/driver qp.error-type.driver)+(p/import-def qp.error-type/invalid-parameter qp.error-type.invalid-parameter)+(p/import-def qp.error-type/invalid-query qp.error-type.invalid-query)+(p/import-def qp.error-type/missing-required-parameter qp.error-type.missing-required-parameter)+(p/import-def qp.error-type/qp qp.error-type.qp)+(p/import-def qp.error-type/unsupported-feature qp.error-type.unsupported-feature)(def schema.common.non-blank-string"::lib.schema.common/non-blank-string"@@ -313,3 +311,7 @@(def qp.util.transformations.nest-breakouts.externally-remapped-field":metabase.query-processor.util.transformations.nest-breakouts/externally-remapped-field"::qp.util.transformations.nest-breakouts/externally-remapped-field)++(def qp.compile.query-with-compiled-query+ "Schema for the output of [[compile]]: `:metabase.query-processor.compile/query-with-compiled-query`"+ ::qp.compile/query-with-compiled-query)diff --git a/src/metabase/driver_api/impl.clj b/src/metabase/driver_api/impl.cljindex 98795d7df1c..35a31e3f9f7 100644--- a/src/metabase/driver_api/impl.clj+++ b/src/metabase/driver_api/impl.clj@@ -4,7 +4,7 @@{:discouraged-namespace {metabase.query-processor.store {:level :off}}}}}(:require[metabase.lib.metadata :as lib.metadata]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u]));;; replacement for the old [[metabase.query-processor.store/cached]] macro, deprecated in 57diff --git a/src/metabase/embedding/api/common.clj b/src/metabase/embedding/api/common.cljindex 09a7b5f5b6b..45fddada4a3 100644--- a/src/metabase/embedding/api/common.clj+++ b/src/metabase/embedding/api/common.clj@@ -4,6 +4,7 @@[clojure.string :as str][medley.core :as m][metabase.api.common :as api]+ [metabase.dashboards.schema :as dashboards.schema][metabase.eid-translation.core :as eid-translation][metabase.embedding.jwt :as embed][metabase.embedding.validation :as embedding.validation]@@ -322,7 +323,8 @@;;; -------------------------- Dashboard Fns used by both /api/embed and /api/preview_embed ---------------------------(defn- remove-locked-parameters [dashboard embedding-params]+(defn- remove-locked-parameters+ [dashboard embedding-params](let [params (:parameters dashboard){params-to-remove :removeparams-to-keep :keep} (classify-params-as-keep-or-remove params embedding-params)@@ -350,7 +352,7 @@;; TODO cleanup(update :param_fields update-vals (fn [fields] (into [] (filter #(not (field-ids-to-remove (:id %)))) fields))))))-(defn dashboard-for-unsigned-token+(mu/defn dashboard-for-unsigned-token :- ::dashboards.schema/dashboard"Return the info needed for embedding about Dashboard specified in `token`. Additional `constraints` can be passed tothe `public-dashboard` function that fetches the Dashboard."[unsigned-token & {:keys [embedding-params constraints]}]@@ -360,7 +362,7 @@embedding-params (or embedding-params(t2/select-one-fn :embedding_params :model/Dashboard, :id dashboard-id))token-params (embed/get-in-unsigned-token-or-throw unsigned-token [:params])]- (-> (apply api.public/public-dashboard :id dashboard-id, constraints)+ (-> (apply api.public/public-dashboard :id dashboard-id constraints)(substitute-token-parameters-in-text token-params)(remove-locked-parameters embedding-params)(remove-token-parameters token-params)diff --git a/src/metabase/indexed_entities/models/model_index.clj b/src/metabase/indexed_entities/models/model_index.cljindex 721b4b796cd..693eee04f21 100644--- a/src/metabase/indexed_entities/models/model_index.clj+++ b/src/metabase/indexed_entities/models/model_index.clj@@ -29,8 +29,11 @@#_(derive :model/ModelIndex :hook/search-index)(t2/deftransforms :model/ModelIndex- {:pk_ref mi/transform-field-ref- :value_ref mi/transform-field-ref})+ ;; TODO (Cam 10/1/25) -- update these to normalize to MBQL 5 Field refs (or stop storing field refs like this in the+ ;; first place!) on the way out+ #_{:clj-kondo/ignore [:deprecated-var]}+ {:pk_ref mi/transform-legacy-field-ref+ :value_ref mi/transform-legacy-field-ref})(t2/define-before-delete :model/ModelIndex[model-index]@@ -83,6 +86,7 @@pk-ref (-> model-index :pk_ref (fix :type/Integer))](try[nil (->> (qp/process-query+ ;; TODO (Cam 10/1/25) -- update this to generate the query using Lib{:database (:database_id model):type :query:query {:source-table (format "card__%d" (:id model))diff --git a/src/metabase/legacy_mbql/normalize.cljc b/src/metabase/legacy_mbql/normalize.cljcindex 8984756df39..6bb06730082 100644--- a/src/metabase/legacy_mbql/normalize.cljc+++ b/src/metabase/legacy_mbql/normalize.cljc@@ -31,6 +31,7 @@(:refer-clojure :exclude [mapv every? some select-keys])(:require[clojure.set :as set]+ [clojure.string :as str][medley.core :as m][metabase.legacy-mbql.predicates :as mbql.preds][metabase.legacy-mbql.schema :as mbql.s]@@ -1071,12 +1072,13 @@(declare remove-empty-clauses path->special-remove-empty-clauses-fn)(defn- remove-empty-clauses-in-map [m special-fns]- (not-empty (reduce-kv (fn [m k v]- (let [v' (remove-empty-clauses v (get special-fns k))]- (cond (nil? v') (dissoc m k)- (identical? v v') m- :else (assoc m k v'))))- m m)))+ (when (map? m)+ (not-empty (reduce-kv (fn [m k v]+ (let [v' (remove-empty-clauses v (get special-fns k))]+ (cond (nil? v') (dissoc m k)+ (identical? v v') m+ :else (assoc m k v'))))+ m m))))(defn- remove-empty-clauses-in-sequence* [xs special-fns](let [special-fns (::sequence special-fns)@@ -1111,11 +1113,12 @@(remove-empty-clauses source-query (:query path->special-remove-empty-clauses-fn))))(defn- remove-empty-clauses-in-parameter [parameter]- (merge- ;; don't remove `value: nil` from a parameter, the FE code (`haveParametersChanged`) is extremely dumb and will- ;; consider the parameter to have changed and thus the query to be 'dirty' if we do this.- (select-keys parameter [:value])- (remove-empty-clauses-in-map parameter (-> path->special-remove-empty-clauses-fn :parameters ::sequence))))+ (when (map? parameter)+ (merge+ ;; don't remove `value: nil` from a parameter, the FE code (`haveParametersChanged`) is extremely dumb and will+ ;; consider the parameter to have changed and thus the query to be 'dirty' if we do this.+ (select-keys parameter [:value])+ (remove-empty-clauses-in-map parameter (-> path->special-remove-empty-clauses-fn :parameters ::sequence)))))(def ^:private path->special-remove-empty-clauses-fn{:native identity@@ -1142,7 +1145,10 @@(sequential? x) (remove-empty-clauses-in-sequence x special-fns):else x))(catch #?(:clj Throwable :cljs js/Error) e- (throw (ex-info "Error removing empty clauses from form."+ (throw (ex-info (let [msg (ex-message e)]+ (if (str/starts-with? msg "Error removing empty clauses")+ msg+ (str "Error removing empty clauses from form: " msg))){:form x}e))))))@@ -1154,6 +1160,8 @@"Normalize the tokens in a Metabase query (i.e., make them all `lisp-case` keywords), rewrite deprecated clauses asup-to-date MBQL 2000, and remove empty clauses."[query]+ (when (:lib/type query)+ (throw (ex-info "Legacy MBQL normalization code cannot normalize MBQL >= 5" {:query query})))(try(-> querynormalize-tokens@@ -1177,9 +1185,12 @@where this fragment would normally live in a full query.(normalize-fragment [:query :filter] [\"=\" 100 200])- ;;-> [:= [:field-id 100] 200]"+ ;;-> [:= [:field-id 100] 200]++ DEPRECATED -- this is notoriously unreliable, convert to MBQL 5 and use [[metabase.lib.normalize]] instead."+ {:deprecated "0.57.0"}[path :- [:maybe [:sequential :keyword]]x]- (if-not (seq path)+ (if (empty? path)(normalize x)(get (normalize-fragment (butlast path) {(last path) x}) (last path))))diff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex cd57da6134d..76e95efbb86 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -1369,13 +1369,7 @@Map of template tag name -> template tag definition"[:and[:map-of ::lib.schema.common/non-blank-string TemplateTag]- ;; make sure people don't try to pass in a `:name` that's different from the actual key in the map.- [:fn- {:error/message "keys in template tag map must match the :name of their values"}- (fn [m]- (every? (fn [[tag-name tag-definition]]- (core/= tag-name (:name tag-definition)))- m))]])+ [:ref ::lib.schema.template-tag/template-tag-map.validate-names]])(def ^:private NativeQuery:Common[:and@@ -1384,7 +1378,8 @@;; collection (table) this query should run against. Needed for MongoDB[:collection {:optional true} [:maybe ::lib.schema.common/non-blank-string]]](lib.schema.common/disallowed-keys- {:type "An inner query must not include :type, this will cause us to mix it up with an outer query"+ {:lib/type "Legacy MBQL inner queries must not have :lib/type"+ :type "An inner query must not include :type, this will cause us to mix it up with an outer query":source-table ":source-table is only allowed in MBQL inner queries.":fields ":fields is only allowed in MBQL inner queries."})])@@ -1651,7 +1646,8 @@(fn [{:keys [breakout fields]}](empty? (set/intersection (set breakout) (set fields))))](lib.schema.common/disallowed-keys- {:type "An inner query must not include :type, this will cause us to mix it up with an outer query"})])+ {:lib/type "Legacy MBQL inner queries must not have :lib/type"+ :type "An inner query must not include :type, this will cause us to mix it up with an outer query"})]);;; ----------------------------------------------------- Params -----------------------------------------------------@@ -1707,7 +1703,8 @@[:and[:fn{:error/message "Query must specify at most one of `:native` or `:query`, but not both."}- (complement (every-pred :native :query))]+ (every-pred (some-fn :native :query)+ (complement (every-pred :native :query)))][:fn{:error/message "Native queries must not specify `:query`; MBQL queries must not specify `:native`."}(fn [{native :native, mbql :query, query-type :type}]@@ -1773,5 +1770,7 @@[:ref ::check-keys-for-query-type][:ref ::check-query-does-not-have-source-metadata](lib.schema.common/disallowed-keys- {:source-table "An outer query must not include inner-query keys like :source-table; this might cause us to confuse it with an inner query"- :source-query "An outer query must not include inner-query keys like :source-query; this might cause us to confuse it with an inner query"})])+ {:lib/type "Legacy MBQL queries must not have :lib/type"+ :source-table "An outer query must not include inner-query keys like :source-table; this might cause us to confuse it with an inner query"+ :source-query "An outer query must not include inner-query keys like :source-query; this might cause us to confuse it with an inner query"+ :stages "Legacy MBQL queries cannot have :stages; use :query or :native instead"})])diff --git a/src/metabase/legacy_mbql/util.cljc b/src/metabase/legacy_mbql/util.cljcindex 8e6c04f80d5..dcaaa7516e5 100644--- a/src/metabase/legacy_mbql/util.cljc+++ b/src/metabase/legacy_mbql/util.cljc@@ -122,17 +122,6 @@#_{:clj-kondo/ignore [:deprecated-var]}(simplify-compound-filter (cons :and (cons filter-clause more-filter-clauses))))-;;; TODO (Cam 7/16/25) -- why does the LEGACY MBQL UTILS have STAGE STUFF IN IT!-(defn legacy-last-stage-number- "Returns the canonical stage number of the last stage of the legacy `inner-query`."- {:deprecated "0.57.0"}- [inner-query]- (loop [{:keys [source-query qp/stage-had-source-card]} inner-query, n 0]- (if (or (nil? source-query)- stage-had-source-card)- n- (recur source-query (inc n)))))-(defn desugar-inside"Rewrite `:inside` filter clauses as a pair of `:between` clauses."{:deprecated "0.57.0"}@@ -859,20 +848,3 @@:elsefield-id-or-form))--(mu/defn unwrap-field-clause :- [:maybe mbql.s/field]- "Unwrap something that contains a `:field` clause, such as a template tag.- Also handles unwrapped integers for legacy compatibility.-- (unwrap-field-clause [:field 100 nil]) ; -> [:field 100 nil]"- [field-form]- (if (integer? field-form)- [:field field-form nil]- (lib.util.match/match-lite-recursive field-form :field field-form)))--(mu/defn unwrap-field-or-expression-clause :- mbql.s/Field- "Unwrap a `:field` clause or expression clause, such as a template tag. Also handles unwrapped integers for- legacy compatibility."- [field-or-ref-form]- (or (unwrap-field-clause field-or-ref-form)- (lib.util.match/match-lite-recursive field-or-ref-form :expression field-or-ref-form)))diff --git a/src/metabase/lib/card.cljc b/src/metabase/lib/card.cljcindex 70bef645874..43b9130032e 100644--- a/src/metabase/lib/card.cljc+++ b/src/metabase/lib/card.cljc@@ -2,9 +2,7 @@(:refer-clojure :exclude [mapv select-keys])(:require[medley.core :as m]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.binning :as lib.binning]- [metabase.lib.convert :as lib.convert][metabase.lib.field.util :as lib.field.util][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.calculation :as lib.metadata.calculation]@@ -255,7 +253,8 @@(mapv (fn [col](assoc col :lib/source :source/card, :lib/card-id (:id card)))(if (= (:type card) :metric)- (let [metric-query (-> card :dataset-query mbql.normalize/normalize lib.convert/->pMBQL+ (let [metric-query (-> card+ :dataset-query(lib.util/update-query-stage -1 dissoc :aggregation :breakout))](lib.metadata.calculation/returned-columns(assoc metric-query :lib/metadata (:lib/metadata query))diff --git a/src/metabase/lib/convert.cljc b/src/metabase/lib/convert.cljcindex 0acf6252aed..8d2c55d0119 100644--- a/src/metabase/lib/convert.cljc+++ b/src/metabase/lib/convert.cljc@@ -616,11 +616,15 @@(and top-level? (:native inner-query)) (set/rename-keys {:native :query})))))(defmethod ->legacy-MBQL :dispatch-type/map [m]- (into {}- (comp (disqualify)- (map (fn [[k v]]- [k (->legacy-MBQL v)])))- m))+ (if (and (:database m)+ (#{:query :native} (:type m)))+ ;; already a legacy query+ m+ (into {}+ (comp (disqualify)+ (map (fn [[k v]]+ [k (->legacy-MBQL v)])))+ m)))(defmethod ->legacy-MBQL :aggregation [[_ opts agg-uuid :as ag]](if (map? opts)@@ -763,7 +767,7 @@stage-number :- :intlegacy-ref :- some?](let [legacy-ref (->> #?(:clj legacy-ref :cljs (js->clj legacy-ref :keywordize-keys true))- (mbql.normalize/normalize-fragment nil))+ mbql.normalize/normalize-field-ref){aggregations :aggregation} (lib.util/query-stage query stage-number)](with-aggregation-list aggregations(trydiff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex 0f290f2f049..a2b2127f6db 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -364,6 +364,7 @@add-parameters-for-template-tagsengineextract-template-tags+ fully-parameterized-query?has-template-tag-variables?has-write-permissionnative-extras@@ -421,7 +422,8 @@replace-join][metabase.lib.schemanative-only-query?]- [metabase.lib.schema.util]+ [metabase.lib.schema.util+ remove-lib-uuids][lib.segmentavailable-segments][metabase.lib.serializediff --git a/src/metabase/lib/drill_thru/fk_details.cljc b/src/metabase/lib/drill_thru/fk_details.cljcindex 703d09df03f..9601bcce4ee 100644--- a/src/metabase/lib/drill_thru/fk_details.cljc+++ b/src/metabase/lib/drill_thru/fk_details.cljc@@ -40,6 +40,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.drill-thru :as lib.schema.drill-thru][metabase.lib.types.isa :as lib.types.isa]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.performance :refer [select-keys]]))@@ -70,21 +71,26 @@;; - Not for this same column, but;; - Relevant to OTHER FKs which point to PKs on the target table;;; then preserve those filters.- (let [fk-column-id (:fk-target-field-id column)- fk-column (some->> fk-column-id (lib.metadata/field query))- fk-column-table (some->> (:table-id fk-column) (lib.metadata/table query))- other-fk-filters (for [filter-clause (lib.filter/filters query stage-number)- :let [parts (lib.filter/filter-parts query stage-number filter-clause)]- :when (= (:short (:operator parts)) :=)- :let [other-fk-target (some->> parts- :column- :fk-target-field-id- (lib.metadata/field query))]- :when (and other-fk-target- (= (:table-id other-fk-target) (:id fk-column-table)) ; FK to this table- (not= (:id other-fk-target) fk-column-id))] ; But not this column- (lib.filter/= other-fk-target (first (:args parts))))]- (reduce #(lib.filter/filter %1 stage-number %2)- (lib.query/query query fk-column-table)- (concat other-fk-filters- [(lib.filter/= fk-column object-id)]))))+ (when-let [fk-column-id (or (:fk-target-field-id column)+ (do+ (log/warnf "Cannot apply :fk-details drill because column is missing :fk-target-field-id: %s"+ (pr-str column))+ nil))]+ (when-let [fk-column (lib.metadata/field query fk-column-id)]+ (when-let [fk-column-table (lib.metadata/table query (:table-id fk-column))]+ (let [other-fk-filters (for [filter-clause (lib.filter/filters query stage-number)+ :let [parts (lib.filter/filter-parts query stage-number filter-clause)]+ :when (= (:short (:operator parts)) :=)+ :let [other-fk-target (some->> parts+ :column+ :fk-target-field-id+ (lib.metadata/field query))]+ :when (and other-fk-target+ (= (:table-id other-fk-target) (:id fk-column-table)) ; FK to this table+ (not= (:id other-fk-target) fk-column-id))] ; But not this column+ (lib.filter/= other-fk-target (first (:args parts))))]+ (reduce+ #(lib.filter/filter %1 stage-number %2)+ (lib.query/query query fk-column-table)+ (concat other-fk-filters+ [(lib.filter/= fk-column object-id)])))))))diff --git a/src/metabase/lib/drill_thru/underlying_records.cljc b/src/metabase/lib/drill_thru/underlying_records.cljcindex 0f8e686bdcf..f462e1fa939 100644--- a/src/metabase/lib/drill_thru/underlying_records.cljc+++ b/src/metabase/lib/drill_thru/underlying_records.cljc@@ -126,7 +126,9 @@value :- :any](let [filter-column (lib.drill-thru.common/breakout->filterable-column query stage-number column-ref column)filter-clauses (or (when (lib.binning/binning column)- (let [unbinned-column (lib.binning/with-binning filter-column nil)]+ (let [unbinned-column (-> filter-column+ (lib.binning/with-binning nil)+ (dissoc :lib/original-binning))](if (some? value)(when-let [{:keys [min-value max-value]} (lib.binning/resolve-bin-width query column value)][(lib.filter/>= unbinned-column min-value)diff --git a/src/metabase/lib/drill_thru/zoom_in_bins.cljc b/src/metabase/lib/drill_thru/zoom_in_bins.cljcindex f9a2f77c6bb..97d6b0cab32 100644--- a/src/metabase/lib/drill_thru/zoom_in_bins.cljc+++ b/src/metabase/lib/drill_thru/zoom_in_bins.cljc@@ -68,6 +68,7 @@In other words, this bin adds a filter for the selected bin and then divides the bin width in the breakout binningoptions by 10."+ (:refer-clojure :exclude [some every?])(:require[metabase.lib.binning :as lib.binning][metabase.lib.breakout :as lib.breakout]diff --git a/src/metabase/lib/fe_util.cljc b/src/metabase/lib/fe_util.cljcindex 7a682de4164..a097fa1e2ee 100644--- a/src/metabase/lib/fe_util.cljc+++ b/src/metabase/lib/fe_util.cljc@@ -3,11 +3,9 @@(:require[inflections.core :as inflections][medley.core :as m]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.aggregation :as lib.aggregation][metabase.lib.card :as lib.card][metabase.lib.common :as lib.common]- [metabase.lib.convert :as lib.convert][metabase.lib.dispatch :as lib.dispatch][metabase.lib.expression :as lib.expression][metabase.lib.filter :as lib.filter]@@ -867,7 +865,7 @@(query-dependents-foreign-keys metadata-providerable card-columns))(when (and (= (:type card) :metric) definition)(query-dependents metadata-providerable- (-> definition mbql.normalize/normalize lib.convert/->pMBQL))))))+ definition)))))(when-let [table-id (:source-table base-stage)](cons {:type :table, :id table-id}(query-dependents-foreign-keys metadata-providerablediff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljcindex 3888008a9d0..4a54d4efeed 100644--- a/src/metabase/lib/field/util.cljc+++ b/src/metabase/lib/field/util.cljc@@ -10,11 +10,10 @@[metabase.lib.util :as lib.util][metabase.util.malli :as mu]))-;;; TODO (Cam 6/24/25) -- this is fundamentally broken -- see QUE-1375(mu/defn inherited-column? :- :boolean"Is the `column` coming directly from a card, a native query, or a previous query stage?"[column :- [:map- [:lib/source {:optional true} ::lib.schema.metadata/column.source]]]+ [:lib/source {:optional true} [:maybe ::lib.schema.metadata/column.source]]]](some? (#{:source/card :source/native :source/previous-stage} (:lib/source column))))(mu/defn inherited-column-name :- [:maybe :string]diff --git a/src/metabase/lib/filter.cljc b/src/metabase/lib/filter.cljcindex 93cd8501f85..63b9ba4cc29 100644--- a/src/metabase/lib/filter.cljc+++ b/src/metabase/lib/filter.cljc@@ -531,11 +531,12 @@([query :- ::lib.schema/querystage-number :- :intlegacy-filter :- some?]- (let [legacy-filter (mbql.normalize/normalize-fragment [:query :filter] legacy-filter)+ (let [legacy-filter #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query :filter] legacy-filter)query-filters (vec (filters query stage-number))- matching-filters (clojure.core/filter #(clojure.core/= (mbql.normalize/normalize-fragment- [:query :filter]- (lib.convert/->legacy-MBQL %))+ matching-filters (clojure.core/filter #(clojure.core/= #_{:clj-kondo/ignore [:deprecated-var]}+ (mbql.normalize/normalize-fragment+ [:query :filter]+ (lib.convert/->legacy-MBQL %))legacy-filter)query-filters)](when (seq matching-filters)diff --git a/src/metabase/lib/js.cljs b/src/metabase/lib/js.cljsindex da75e06e602..77587a840a1 100644--- a/src/metabase/lib/js.cljs+++ b/src/metabase/lib/js.cljs@@ -78,7 +78,6 @@[metabase.lib.order-by :as lib.order-by][metabase.lib.query :as lib.query][metabase.lib.schema.ref :as lib.schema.ref]- [metabase.lib.schema.util :as lib.schema.util][metabase.lib.types.isa :as lib.types.isa][metabase.lib.util :as lib.util][metabase.util :as u]@@ -643,15 +642,14 @@(defn- normalize-to-clj[a-query]- (let [normalize-fn (fn [q]- (if (= (lib.util/normalized-query-type q) :mbql/query)- (lib.normalize/normalize q)- (mbql.normalize/normalize q)))]- (-> a-query (js->clj :keywordize-keys true) unwrap normalize-fn)))+ (letfn [(normalize* [q]+ (if (= (lib.util/normalized-query-type q) :mbql/query)+ (lib.normalize/normalize q)+ (mbql.normalize/normalize q)))]+ (-> a-query (js->clj :keywordize-keys true) unwrap normalize*)))(defn ^:export normalize"Normalize the MBQL or pMBQL query `a-query`.-Returns the JS form of the normalized query."[a-query](-> a-query normalize-to-clj (clj->js :keyword-fn u/qualified-name)))@@ -683,21 +681,20 @@;; Ignore :info since it contains the randomized :card-entity-id. (This is no longer populated either.)(dissoc :info)))-(defn- prep-query-for-equals-pMBQL+;;; TODO (Cam 10/8/25) -- we're currently converting MBQL 5 to legacy to do the equality check (icky). It would be+;;; better to convert both to 5 and use [[metabase.lib.equality/=]]... but that's a task for another day.+(defn- prep-query-for-equals-mbql5[a-query field-ids]- (let [fields (or (some->> (lib.core/fields a-query)- (map #(assoc % 1 {})))- (mapv (fn [id] [:field {} id]) field-ids))]- (lib.util/update-query-stage a-query -1- #(-> %- (assoc :fields (frequencies fields))- lib.schema.util/remove-lib-uuids))))+ (-> a-query+ lib.core/->legacy-MBQL+ (prep-query-for-equals-legacy field-ids)))(defn- prep-query-for-equals [a-query field-ids](when-let [normalized-query (some-> a-query normalize-to-clj)]- (if (contains? normalized-query :lib/type)- (prep-query-for-equals-pMBQL normalized-query field-ids)- (prep-query-for-equals-legacy normalized-query field-ids))))+ (let [f (if (= (lib.core/normalized-mbql-version normalized-query) :mbql-version/mbql5)+ prep-query-for-equals-mbql5+ prep-query-for-equals-legacy)]+ (f normalized-query field-ids))))(defn- compare-field-refs[[key1 id1 opts1]@@ -744,7 +741,7 @@If `field-ids` is specified, an input MBQL query without `:fields` set defaults to the `field-ids`.- Currently this works only for legacy queries in JS form!+ Works on either MBQL 4 or MBQL 5 in either JS or Cljs form.It duplicates the logic formerly found in `query_builder/selectors.js`.> **Code health:** Legacy. New calls are acceptable if necessary. Eventually this will be replaced with an equivalent@@ -753,9 +750,9 @@([query1 query2] (query= query1 query2 nil))([query1 query2 field-ids](let [ids (mapv js->clj field-ids)- n1 (prep-query-for-equals query1 ids)- n2 (prep-query-for-equals query2 ids)]- (query=* n1 n2))))+ q1 (prep-query-for-equals query1 ids)+ q2 (prep-query-for-equals query2 ids)]+ (query=* q1 q2))));; # Column Groups;; In many places in the FE we show a list of columns which might be used to filter, aggregate, etc. These are shown in@@ -1476,7 +1473,7 @@(-> a-legacy-ref(js->clj :keywordize-keys true)(update 0 keyword)- (->> (mbql.normalize/normalize-fragment nil))+ mbql.normalize/normalize-field-reflib.convert/->pMBQL(->> (lib.normalize/normalize ::lib.schema.ref/ref))))@@ -2616,3 +2613,67 @@> **Code health:** Healthy"[a-query](lib.core/ensure-filter-stage a-query))++(defn ^:export to-js-query+ "Serialize a query to a plain JS object."+ [cljs-query]+ (letfn [(->js [x]+ (cond+ (map? x)+ (-> (reduce-kv (fn [m k v]+ (assoc m (->js k) (->js v)))+ {}+ x)+ clj->js)++ (sequential? x)+ (-> (mapv ->js x)+ clj->js)++ (qualified-keyword? x)+ (str (namespace x) "/" (name x))++ (simple-keyword? x)+ (name x)++ :else+ (clj->js x)))]+ (-> cljs-query+ lib.core/prepare-for-serialization+ ->js)))++;;; TODO (Cam 10/8/25) -- I'm starting to think that we should just have this be a Cljs-specific implementation+;;; of [[metabase.lib.query/query-method]], then you can just use the normal `lib/query` with a JS map and things will+;;; work as expected.+(defn- from-js-query*+ [mp js-query]+ (when (zero? (alength (js/Object.keys js-query)))+ (throw (ex-info "Invalid query: query cannot be empty" {:query js-query})))+ (-> js-query+ js->clj+ (->> (lib.core/query mp))+ ;; TODO (Cam 10/7/25) -- no idea why but Alex P reported that this function is returning queries without attached+ ;; metadata providers -- force them to have them. I can't work out why it is happening, so this is a temporary+ ;; HACK to keep things moving.+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1759846641359159?thread_ts=1759289751.539169&cid=C0645JP1W81+ (cond-> mp+ (assoc :lib/metadata mp))))++(defn ^:export from-js-query+ "Deserialize a query from a plain JS object. Works with either MBQL 4 or MBQL 5.++ Attaches a cache to `metadata-provider` so that subsequent calls with the same `js-query` return the same query+ object. If the metadata gets updated, the `metadata-provider` will be discarded and replaced, destroying the cache."+ [^js mp js-query]+ ;; even in JS, this will never work right without a valid MetadataProvider, so error if we don't get one.+ (assert (satisfies? lib.metadata.protocols/MetadataProvider mp))+ ;; TODO (Cam 10/8/25) -- I attempted to use the side-channel cache stuff here and it didn't work correctly, so I+ ;; made a different cache instead... this seems to work correctly now. Maybe we can fix it and use the existing+ ;; stuff.+ (let [^js/WeakMap cache (if-let [cache (.-__fromJsQueryCache mp)]+ cache+ (u/prog1 (js/WeakMap.)+ (set! (.-__fromJsQueryCache mp) <>)))]+ (or (.get cache js-query)+ (u/prog1 (from-js-query* mp js-query)+ (.set cache js-query <>)))))diff --git a/src/metabase/lib/js/metadata.cljs b/src/metabase/lib/js/metadata.cljsindex 1bf0d127235..2831174ce8d 100644--- a/src/metabase/lib/js/metadata.cljs+++ b/src/metabase/lib/js/metadata.cljs@@ -311,6 +311,7 @@:visibility-type (keyword v):id (parse-field-id v):metabase.lib.field/binning (parse-binning-info v)+ :lib/original-binning (parse-binning-info v)::field-values (parse-field-values v)::dimension (parse-dimension v)v)))diff --git a/src/metabase/lib/metadata.cljc b/src/metabase/lib/metadata.cljcindex 7fdb4631c7c..db9aef7612c 100644--- a/src/metabase/lib/metadata.cljc+++ b/src/metabase/lib/metadata.cljc@@ -1,6 +1,7 @@(ns metabase.lib.metadata(:refer-clojure :exclude [every?])(:require+ [medley.core :as m][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]@@ -90,7 +91,7 @@::lib.schema.metadata/metric::lib.schema.metadata/segment]]]"Return active (non-archived) metadatas associated with a particular Table, either Fields, Metrics, or- Segments -- `metadata-type` must be one of either `:metadata/column`, `:metadata/metric`, `:metadata/segment`."+ Segments -- `metadata-type` must be one of either `:metadata/column`, `:metadata/metric`, `:metadata/segment`."[metadata-providerable :- ::lib.schema.metadata/metadata-providerablemetadata-type :- [:enum :metadata/column :metadata/metric :metadata/segment]table-id :- ::lib.schema.id/table]@@ -120,14 +121,23 @@(when-let [remap-field-id (get-in column [:lib/external-remap :field-id])](field metadata-providerable remap-field-id))))-;; TODO: Better schemas for transforms coming out of the metadata provider.-(mu/defn transform :- [:maybe [:map]]+(defn- normalize-query [query metadata-providerable]+ (if (or (and (:lib/type query)+ (lib.metadata.protocols/metadata-provider? (:lib/metadata query)))+ (empty? query))+ query+ ((#?(:clj requiring-resolve :cljs resolve) 'metabase.lib.query/query)+ metadata-providerable+ query)))++(mu/defn transform :- [:maybe ::lib.schema.metadata/transform]"Gets a Transform by ID, or nil if it does not exist."[metadata-providerable :- ::lib.schema.metadata/metadata-providerabletransform-id :- :int]- (lib.metadata.protocols/transform (->metadata-provider metadata-providerable) transform-id))+ (some-> (lib.metadata.protocols/transform (->metadata-provider metadata-providerable) transform-id)+ (m/update-existing-in [:source :query] normalize-query metadata-providerable)))-(mu/defn transforms :- [:maybe [:sequential [:map]]]+(mu/defn transforms :- [:maybe [:sequential ::lib.schema.metadata/transform]]"Gets all Transforms"[metadata-providerable :- ::lib.schema.metadata/metadata-providerable](lib.metadata.protocols/transforms (->metadata-provider metadata-providerable)))@@ -142,7 +152,8 @@"Get metadata for a Card, aka Saved Question, with `card-id`, if it can be found."[metadata-providerable :- ::lib.schema.metadata/metadata-providerablecard-id :- ::lib.schema.id/card]- (lib.metadata.protocols/card (->metadata-provider metadata-providerable) card-id))+ (some-> (lib.metadata.protocols/card (->metadata-provider metadata-providerable) card-id)+ (m/update-existing :dataset-query normalize-query metadata-providerable)))(mu/defn native-query-snippet :- [:maybe ::lib.schema.metadata/native-query-snippet]"Get metadata for a NativeQuerySnippet with `snippet-id` if it can be found."diff --git a/src/metabase/lib/metadata/invocation_tracker.cljc b/src/metabase/lib/metadata/invocation_tracker.cljcindex ca664f6e9a6..8698d7cdc98 100644--- a/src/metabase/lib/metadata/invocation_tracker.cljc+++ b/src/metabase/lib/metadata/invocation_tracker.cljc@@ -1,7 +1,8 @@(ns metabase.lib.metadata.invocation-tracker(:require#?@(:clj- ([metabase.util.json :as json]+ (^{:clj-kondo/ignore [:discouraged-namespace]} [clj-yaml.core]+ [metabase.util.json :as json][pretty.core :as pretty]))[metabase.lib.metadata.protocols :as lib.metadata.protocols]))@@ -98,3 +99,11 @@InvocationTracker(fn [_mp json-generator](json/generate-nil nil json-generator))))++;;; this is here as a sanity check to make sure we're not trying to write MBQL 5 to SerDes yet until we update the+;;; SerDes code to handle it+#?(:clj+ (extend-protocol clj-yaml.core/YAMLCodec+ InvocationTracker+ (encode [_this]+ (throw (Exception. "Encoding MBQL 5 queries to YAML for SerDes is not yet supported; please convert the query to legacy first.")))))diff --git a/src/metabase/lib/metric.cljc b/src/metabase/lib/metric.cljcindex 7c028e9ed20..cf1fa3c534e 100644--- a/src/metabase/lib/metric.cljc+++ b/src/metabase/lib/metric.cljc@@ -9,6 +9,7 @@[metabase.lib.join :as lib.join][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.calculation :as lib.metadata.calculation]+ [metabase.lib.normalize :as lib.normalize][metabase.lib.options :as lib.options][metabase.lib.query :as lib.query][metabase.lib.ref :as lib.ref]@@ -27,10 +28,11 @@(mu/defn- metric-definition :- [:maybe ::lib.schema/stage.mbql][{:keys [dataset-query], :as _metric-metadata} :- ::lib.schema.metadata/metric](when dataset-query- (let [normalized-definition (cond-> dataset-query- (not (contains? dataset-query :lib/type))- ;; legacy; needs conversion- (-> mbql.normalize/normalize lib.convert/->pMBQL))]+ (let [normalized-definition (case (lib.util/normalized-mbql-version dataset-query)+ ;; TODO (Cam 10/7/25) -- not sure we'll ever see legacy queries here anymore since+ ;; they get normalized to MBQL 5 coming out of the app DB+ :mbql-version/legacy (-> dataset-query mbql.normalize/normalize lib.convert/->pMBQL)+ :mbql-version/mbql5 (lib.normalize/normalize ::lib.schema/query dataset-query))](lib.util/query-stage normalized-definition -1))))(defmethod lib.ref/ref-method :metadata/metricdiff --git a/src/metabase/lib/native.cljc b/src/metabase/lib/native.cljcindex 017579dd241..05da6e4598f 100644--- a/src/metabase/lib/native.cljc+++ b/src/metabase/lib/native.cljc@@ -8,6 +8,8 @@[medley.core :as m][metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options]+ [metabase.lib.parameters.parse :as lib.params.parse]+ [metabase.lib.parameters.parse.types :as lib.params.parse.types][metabase.lib.parse :as lib.parse][metabase.lib.query :as lib.query][metabase.lib.schema :as lib.schema]@@ -17,11 +19,12 @@[metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.lib.template-tags :as lib.template-tags][metabase.lib.util :as lib.util]+ [metabase.lib.walk.util :as lib.walk.util][metabase.util.humanization :as u.humanization][metabase.util.i18n :as i18n][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [some select-keys mapv every?]]))+ [metabase.util.performance :refer [every? mapv select-keys some]]));; Template Tags: Variables@@ -290,21 +293,38 @@:template-tags (extract-template-tags query inner-query existing-tags)))));;; TODO (Cam 7/16/25) -- this really doesn't seem to do what I'd expect, maybe we should rename it something like-;;; `with-replaced-template-tags`+;;; `with-replaced-template-tags`. It only replaces tags you specify rather then completely setting a new list(mu/defn with-template-tags :- ::lib.schema/query"Updates the native query's template tags."- [query :- ::lib.schema/query- tags :- ::lib.schema.template-tag/template-tag-map]- (lib.util/update-query-stage- query 0- (fn [{existing-tags :template-tags :as stage}]- (assert-native-query stage)- (let [valid-tags (keys existing-tags)]- (assoc stage :template-tags- (merge existing-tags (select-keys tags valid-tags)))))))--(mu/defn raw-native-query :- ::common/non-blank-string- "Returns the native query string"+ [query :- ::lib.schema/query+ updated-tags :- ::lib.schema.template-tag/template-tag-map]+ (letfn [(update-template-tags [existing-tags]+ ;; the way we do this is really weird, but it's important that we use the order of the keys in+ ;; `updated-tags` See+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1759975007383889?thread_ts=1759289751.539169&cid=C0645JP1W81+ ;;+ ;; first, filter out the tags in `updated-tags` not in existing tags, preserving the original order.+ (let [tags (reduce-kv+ (fn [m k v]+ (cond-> m+ (contains? existing-tags k) (assoc k v)))+ {}+ updated-tags)]+ ;; merge in old values that weren't in the `updated-tags` map+ (reduce-kv+ (fn [m k v]+ (cond-> m+ (not (contains? m k)) (assoc k v)))+ tags+ existing-tags)))+ (update-stage [stage]+ (assert-native-query stage)+ (update stage :template-tags update-template-tags))]+ (lib.util/update-query-stage query 0 update-stage)))++(mu/defn raw-native-query :- some?+ "Returns the native query. This is a SQL string for SQL-based drivers; for other drivers like MongoDB it might be a+ Clojure map."[query :- ::lib.schema/query](:native (lib.util/query-stage query 0)))@@ -416,3 +436,52 @@ttags)](cond-> query(seq new-parameters) (assoc :parameters new-parameters))))++(mu/defn- fully-parameterized-text?+ "Decide if `text`, usually (a part of) a query, is fully parameterized given the parameter types+ described by `template-tags` (usually the template tags of a native query).++ The rules to consider a piece of text fully parameterized is as follows:++ 1. All parameters not in an optional block are field-filters or snippets or have a default value.+ 2. All required parameters have a default value.++ The first rule is absolutely necessary, as queries violating it cannot be executed without+ externally supplied parameter values. The second rule is more controversial, as field-filters+ outside of optional blocks ([[ ... ]]) don't prevent the query from being executed without+ external parameter values (neither do parameters in optional blocks). The rule has been added+ nonetheless, because marking a parameter as required is something the user does intentionally+ and queries that are technically executable without parameters can be unacceptably slow+ without the necessary constraints. (Marking parameters in optional blocks as required doesn't+ seem to be useful any way, but if the user said it is required, we honor this flag.)"+ [text :- :string+ template-tags-map :- ::lib.schema.template-tag/template-tag-map]+ (try+ (let [obligatory-params (into #{}+ (comp (filter lib.params.parse.types/param?)+ (map :k))+ (lib.params.parse/parse text))]+ (and (every? #(or (#{:dimension :snippet :card} (:type %))+ (:default %))+ (map template-tags-map obligatory-params))+ (every? #(or (not (:required %))+ (:default %))+ (vals template-tags-map))))+ (catch #?(:clj clojure.lang.ExceptionInfo :cljs :default) _+ ;; An exception might be thrown during parameter parsing if the syntax is invalid. In this case we return+ ;; true so that we still can try to generate a preview for the query and display an error.+ false)))++;;; TODO (Cam 10/3/25) -- this needs a much better docstring+(mu/defn fully-parameterized-query? :- boolean?+ "Given a query, returns `true` if its query is fully parameterized."+ [query :- ::lib.schema/query]+ (let [raw-native-query-string (when (lib.schema/native-only-query? query)+ (let [query (raw-native-query query)]+ (when (string? query)+ query)))+ template-tags-map (when raw-native-query-string+ (not-empty (lib.walk.util/all-template-tags-map query)))]+ (if (and template-tags-map raw-native-query-string)+ (boolean (fully-parameterized-text? raw-native-query-string template-tags-map))+ true)))diff --git a/src/metabase/lib/normalize.cljc b/src/metabase/lib/normalize.cljcindex 3ac4f77799c..03aa9571d25 100644--- a/src/metabase/lib/normalize.cljc+++ b/src/metabase/lib/normalize.cljc@@ -35,7 +35,7 @@(or (-> x lib-type lib-type->schema):map)- (and (vector? x)+ (and (sequential? x)((some-fn simple-keyword? string?) (first x)))(lib.schema.mbql-clause/tag->registered-schema-name (first x))diff --git a/src/metabase/lib/parameters/parse.cljc b/src/metabase/lib/parameters/parse.cljcnew file mode 100644index 00000000000..c0287b5c0c7--- /dev/null+++ b/src/metabase/lib/parameters/parse.cljc@@ -0,0 +1,50 @@+(ns metabase.lib.parameters.parse+ (:refer-clojure :exclude [mapv])+ (:require+ [clojure.core.match :refer [match]]+ [clojure.string :as str]+ [metabase.lib.parameters.parse.types :as lib.params.parse.types]+ [metabase.lib.parse :as lib.parse]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.performance :refer [mapv]]))++(mr/def ::parsed-token+ [:or+ :string+ ::lib.params.parse.types/param+ ::lib.params.parse.types/function-param+ ::lib.params.parse.types/optional])++(defn- ->param [value]+ (match [value]+ [s :guard string?]+ s++ [{:type :metabase.lib.parse/param+ :name param-name}]+ (lib.params.parse.types/param {:k (str/trim param-name)})++ [{:type :metabase.lib.parse/function-param+ :name param-name+ :args args}]+ (lib.params.parse.types/function-param {:function-name param-name, :args (mapv ->param args)})++ [{:type :metabase.lib.parse/optional+ :contents contents}]+ (lib.params.parse.types/optional {:args (mapv ->param contents)})))++(mu/defn parse :- [:sequential ::parsed-token]+ "Attempts to parse parameters in string `s`. Parses any optional clauses or parameters found, and returns a sequence+ of non-parameter string fragments (possibly) interposed with `Param` or `Optional` instances.++ If `handle-sql-comments` is true (default) then we make a best effort to ignore params in SQL comments."+ ([s :- :string]+ (parse s true))++ ([s :- :string+ handle-sql-comments :- :boolean]+ (->> (lib.parse/parse {:parse-error-type :invalid-query}+ s+ handle-sql-comments)+ (mapv ->param))))diff --git a/src/metabase/lib/parameters/parse/types.cljc b/src/metabase/lib/parameters/parse/types.cljcnew file mode 100644index 00000000000..28f4e4bd076--- /dev/null+++ b/src/metabase/lib/parameters/parse/types.cljc@@ -0,0 +1,222 @@+(ns metabase.lib.parameters.parse.types+ "Schemas and helper functions for various PARSED parameter maps. These are created from the `:parameters` passed in+ with a query."+ (:require+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.literal :as lib.schema.literal]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(def no-value+ "Convenience for representing an *optional* parameter present in a query but whose value is unspecified in the param+ values."+ ::no-value)++(mr/def ::no-value+ [:= ::no-value])++(mr/def ::field-filter.value.map+ [:map+ [:type ::lib.schema.parameter/type]+ [:value :any]])++(mr/def ::field-filter.value+ [:or+ ::no-value+ ::field-filter.value.map+ [:sequential [:ref ::field-filter.value]]])++(mr/def ::field-filter+ "'FieldFilter' is something that expands to a clause like 'some_field BETWEEN 1 AND 10'++ `field` is a `:metadata/column`++ `value` is either:++ * [[no-value]]++ * A map contianing the value and type info for the value, e.g.++ {:type :date/single+ :value #t \"2019-09-20T19:52:00.000-07:00\"}++ * A vector of maps like the one above (for multiple values)"+ [:map+ [:lib/type [:= ::field-filter]]+ [:field ::lib.schema.metadata/column]+ [:value ::field-filter.value]])++(mu/defn field-filter :- ::field-filter+ "Create a parsed `field-filter` parameter from map `m`."+ [m]+ (assoc m :lib/type ::field-filter))++(defn field-filter?+ "Whether `x` is a map representing a parsed [[field-filter]] parameter."+ [x]+ (= (:lib/type x) ::field-filter))++(mr/def ::temporal-unit+ [:map+ [:lib/type [:= ::temporal-unit]]+ [:name ::lib.schema.common/non-blank-string]+ ;; TODO (Cam 7/16/25) -- constrain `:value`+ [:value some?]])++(mu/defn temporal-unit :- ::temporal-unit+ "Create a parsed `temporal-unit` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::temporal-unit))++(defn temporal-unit?+ "Whether `x` is a map representing a parsed [[temporal-unit]] parameter."+ [x]+ (= (:lib/type x) ::temporal-unit))++(mr/def ::referenced-card-query+ "A ReferencedCardQuery parameter expands to the native query of the referenced card.++ `card-id` is the ID of the Card instance whose query is the value for this parameter.++ `query` is the native query as stored in the Card++ `parameters` are positional parameters for a parameterized native query e.g. the JDBC parameters corresponding to+ `?` placeholders"+ [:map+ [:lib/type [:= ::referenced-card-query]]+ [:card-id ::lib.schema.id/card]+ [:query :string]+ [:parameters {:optional true} [:maybe [:sequential :any]]]])++(mu/defn referenced-card-query :- ::referenced-card-query+ "Create a parsed `referenced-card-query` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::referenced-card-query))++(defn referenced-card-query?+ "Whether `x` is a map representing a parsed [[referenced-card-query]] parameter."+ [x]+ (= (:lib/type x) ::referenced-card-query))++(mr/def ::referenced-query-snippet+ "A `ReferencedQuerySnippet` expands to the partial query snippet stored in the `NativeQuerySnippet` table in the+ application DB.++ `snippet-id` is the integer ID of the row in the application DB from where the snippet content is loaded.++ `content` is the raw query snippet which will be replaced, verbatim, for this template tag."+ [:map+ [:lib/type [:= ::referenced-query-snippet]]+ [:snippet-id ::lib.schema.id/snippet]+ [:content :string]])++(mu/defn referenced-query-snippet :- ::referenced-query-snippet+ "Create a parsed `referenced-query-snippet` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::referenced-query-snippet))++(defn referenced-query-snippet?+ "Whether `x` is a map representing a parsed [[referenced-query-snippet]]."+ [x]+ (= (:lib/type x) ::referenced-query-snippet))++(mr/def ::date+ "As in a literal date, defined by date-string `s`."+ [:map+ [:lib/type [:= ::date]]+ [:s ::lib.schema.literal/string.date]])++(mu/defn date :- ::date+ "Create a parsed `date` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::date))++(defn date?+ "Whether `x` is a map representing a parsed [[date]] parameter."+ [x]+ (= (:lib/type x) ::date))++(mr/def ::date-range+ [:map+ [:lib/type [:= ::date-range]]+ ;; TODO (Cam 7/16/25) -- constrain `:start` and `:end` values+ [:start :any]+ [:end :any]])++(mu/defn date-range :- ::date-range+ "Create a new parsed `date-range` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::date-range))++(defn date-range?+ "Whether `x` is a map representing a parsed [[date-range]] parameter."+ [x]+ (= (:lib/type x) ::date-range))++(mr/def ::date-time-range+ [:map+ [:lib/type [:= ::date-time-range]]+ ;; TODO (Cam 7/16/25) -- constrain `:start` and `:end` values+ [:start :any]+ [:end :any]])++(mu/defn date-time-range :- ::date-time-range+ "Create a new parsed `date-time-range` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::date-time-range))++(defn date-time-range?+ "Whether `x` is a map representing a parsed [[date-time-range]] parameter."+ [x]+ (= (:lib/type x) ::date-time-range))++(mr/def ::param+ [:map+ [:lib/type [:= ::param]]+ [:k :string]])++(mu/defn param :- ::param+ "Create a new parsed `param` parameter from map `m`."+ [m :- :map]+ (assoc m :lib/type ::param))++(defn param?+ "Whether `x` is a map that represents a parsed [[param]] parameter."+ [x]+ (= (:lib/type x) ::param))++(mr/def ::function-param+ [:map+ [:lib/type [:= ::function-param]]+ [:function-name :string]+ ;; TODO (Cam 7/16/25) -- constrain further; I think these have to be valid parameters+ [:args [:sequential :any]]])++(mu/defn function-param :- ::function-param+ "Create a new parsed `function-param` from map `m`."+ [m :- :map]+ (assoc m :lib/type ::function-param))++(defn function-param?+ "Whether `x` is a map that represents a parsed [[function-param]]."+ [x]+ (= (:lib/type x) ::function-param))++(mr/def ::optional+ [:map+ [:lib/type [:= ::optional]]+ ;; TODO (Cam 7/16/25) -- constrain further+ [:args [:sequential :any]]])++(mu/defn optional :- ::optional+ "Create a new parsed `optional` param from map `m`."+ [m :- :map]+ (assoc m :lib/type ::optional))++(defn optional?+ "Whether `x` is a map that represents a parsed [[optional]] parameter."+ [x]+ (= (:lib/type x) ::optional))diff --git a/src/metabase/lib/parse.cljc b/src/metabase/lib/parse.cljcindex a448e899afd..bf8d1f2f670 100644--- a/src/metabase/lib/parse.cljc+++ b/src/metabase/lib/parse.cljc@@ -1,4 +1,5 @@(ns metabase.lib.parse+ "Code for parsing parameters in raw SQL strings."(:refer-clojure :exclude [some])(:require[clojure.string :as str]diff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljcindex 1ed90bb094c..fb2578b45a4 100644--- a/src/metabase/lib/query.cljc+++ b/src/metabase/lib/query.cljc@@ -165,10 +165,15 @@([database-id :- ::lib.schema.id/databasemetadata-providerable :- ::lib.schema.metadata/metadata-providerablestages]- {:lib/type :mbql/query- :lib/metadata (lib.metadata/->metadata-provider metadata-providerable)- :database database-id- :stages stages}))+ (->> (merge+ {:lib/type :mbql/query+ :lib/metadata (lib.metadata/->metadata-provider metadata-providerable)+ :stages stages}+ ;; this can be nil in the FE with empty metadata providers, don't stomp on existing DB IDs that are+ ;; not nil.+ (when database-id+ {:database database-id}))+ (lib.normalize/normalize ::lib.schema/query))))(defn- query-from-legacy-query[metadata-providerable legacy-query]@@ -189,8 +194,7 @@"Implementation for [[query]]."{:arglists '([metadata-providerable x])}(fn [_metadata-providerable x]- (or (lib.util/normalized-query-type x)- (lib.dispatch/dispatch-value x)))+ ((some-fn lib.util/normalized-query-type lib.dispatch/dispatch-value) x)):hierarchy lib.hierarchy/hierarchy)(defmethod query-method :query ; legacy MBQL query@@ -218,29 +222,30 @@(dissoc :lib.convert/converted?)lib.normalize/normalize)stages (:stages query)]- (cond-> query- converted?- (assoc- :stages- (mapv (fn [[stage-number stage]]- (-> stage- (add-types-to-fields metadata-provider)- (lib.util.match/replace- [:expression- (opts :guard (every-pred map? (complement (every-pred :base-type :effective-type))))- expression-name]- (let [found-ref (try- (m/remove-vals- #(= :type/* %)- (-> (lib.expression/expression-ref query stage-number expression-name)- second- (select-keys [:base-type :effective-type])))- (catch #?(:clj Exception :cljs :default) _- ;; This currently does not find expressions defined in join stages- nil))]- ;; Fallback if metadata is missing- [:expression (merge found-ref opts) expression-name]))))- (m/indexed stages))))))+ (-> query+ (cond-> converted?+ (assoc+ :stages+ (mapv (fn [[stage-number stage]]+ (-> stage+ (add-types-to-fields metadata-provider)+ (lib.util.match/replace+ [:expression+ (opts :guard (every-pred map? (complement (every-pred :base-type :effective-type))))+ expression-name]+ (let [found-ref (try+ (m/remove-vals+ #(= :type/* %)+ (-> (lib.expression/expression-ref query stage-number expression-name)+ second+ (select-keys [:base-type :effective-type])))+ (catch #?(:clj Exception :cljs :default) _+ ;; This currently does not find expressions defined in join stages+ nil))]+ ;; Fallback if metadata is missing+ [:expression (merge found-ref opts) expression-name]))))+ (m/indexed stages))))+ (->> (lib.normalize/normalize ::lib.schema/query)))))(defmethod query-method :metadata/table[metadata-providerable table-metadata]@@ -301,7 +306,7 @@existing MBQL query or saved question or whatever. If the thing in question does not already include metadata, passit in separately -- metadata is needed for most query manipulation operations."[metadata-providerable :- ::lib.schema.metadata/metadata-providerable- x]+ x :- some?](ensure-cached-metadata-provider (query-method metadata-providerable x)))(mu/defn ->query :- ::lib.schema/querydiff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex d05759fbde6..c65ed669ff5 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -6,7 +6,7 @@Some primitives below are duplicated from [[metabase.util.malli.schema]] since that's not `.cljc`. Other stuff iscopied from [[metabase.legacy-mbql.schema]] so this can exist completely independently; hopefully at some point in thefuture we can deprecate that namespace and eventually do away with it entirely."- (:refer-clojure :exclude [ref every? some])+ (:refer-clojure :exclude [ref every? some select-keys])(:require[medley.core :as m][metabase.legacy-mbql.util :as mbql.u]@@ -35,7 +35,7 @@[metabase.lib.schema.util :as lib.schema.util][metabase.lib.util.match :as lib.util.match][metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [every? some]]))+ [metabase.util.performance :refer [every? select-keys some]]))(comment metabase.lib.schema.expression.arithmetic/keep-memetabase.lib.schema.expression.conditional/keep-me@@ -44,23 +44,37 @@metabase.lib.schema.expression.window/keep-memetabase.lib.schema.filter/keep-me)+(defn- normalize-stage-common [m]+ (when-let [m (common/normalize-map m)]+ (reduce+ (fn [m k]+ (cond-> m+ (and (contains? m k)+ (empty? (m k)))+ (dissoc m k)))+ m+ [:parameters+ :lib/stage-metadata])))+(mr/def ::stage.common[:map- [:parameters {:optional true} [:ref ::lib.schema.parameter/parameters]]])+ {:decode/normalize normalize-stage-common}+ [:parameters {:optional true} [:ref ::lib.schema.parameter/parameters]]+ [:lib/stage-metadata {:optional true} [:ref ::lib.schema.metadata/stage]]])(mr/def ::stage.native[:and[:merge::stage.common[:map- {:decode/normalize #(->> %- common/normalize-map- ;; filter out null :collection keys -- see #59675- (m/filter-kv (fn [k v]- (not (and (= k :collection)- (nil? v))))))}+ {:decode/normalize #(->> %+ normalize-stage-common+ ;; filter out null :collection keys -- see #59675+ (m/filter-kv (fn [k v]+ (not (and (= k :collection)+ (nil? v))))))+ :encode/for-hashing #'common/encode-map-for-hashing}[:lib/type [:= {:decode/normalize common/normalize-keyword} :mbql.stage/native]]- [:lib/stage-metadata {:optional true} [:maybe [:ref ::lib.schema.metadata/stage]]];; the actual native query, depends on the underlying database. Could be a raw SQL string or something like that.;; Only restriction is that, if present, it is non-nil.;; It is valid to have a blank query like `{:type :native}` in legacy.@@ -196,19 +210,25 @@[:items pos-int?]])(defn- normalize-mbql-stage [m]- (when (map? m)- (let [m (common/normalize-map m)]- ;; remove deprecated ident keys if they are present for some reason.- (dissoc m :aggregation-idents :breakout-idents :expression-idents))))+ (normalize-stage-common m))++(defn- encode-mbql-stage-for-hashing [stage]+ (-> stage+ common/encode-map-for-hashing+ lib.schema.util/indexed-order-bys-for-stage+ ;; preserve these keys because we want to hash two identical queries from different source cards+ ;; differently (see [[metabase.query-processor.middleware.cache-test/multiple-models-e2e-test]]) and this is a+ ;; reliable way to differentiate them since it gets populated by the QP.+ (merge (select-keys stage [:qp/stage-is-from-source-card :qp/stage-had-source-card]))))(mr/def ::stage.mbql[:and[:merge::stage.common[:map- {:decode/normalize normalize-mbql-stage}+ {:decode/normalize #'normalize-mbql-stage+ :encode/for-hashing #'encode-mbql-stage-for-hashing}[:lib/type [:= {:decode/normalize common/normalize-keyword} :mbql.stage/mbql]]- [:lib/stage-metadata {:optional true} [:maybe [:ref ::lib.schema.metadata/stage]]][:joins {:optional true} [:ref ::join/joins]][:expressions {:optional true} [:ref ::expression/expressions]][:breakout {:optional true} [:ref ::breakouts]]@@ -247,7 +267,7 @@(let [stage (common/normalize-map stage)];; infer stage type(cond- (:lib/type stage)+ ((some-fn :lib/type #(get % "lib/type")) stage)stage((some-fn :source-table :source-card) stage)@@ -287,7 +307,9 @@[:multi {:dispatch lib-type:error/message "Invalid stage :lib/type: expected :mbql.stage/native or :mbql.stage/mbql"}[:mbql.stage/native :map]- [:mbql.stage/mbql :map]])+ [:mbql.stage/mbql [:fn+ {:error/message "Initial MBQL stage must have either :source-table or :source-card (but not both)"}+ (some-fn :source-table :source-card)]]])(mr/def ::stage.additional[:multi {:dispatch lib-type@@ -359,7 +381,7 @@(defn- normalize-stages [stages](when (sequential? stages)- (if (every? :lib/type stages)+ (if (every? (some-fn :lib/type #(get % "lib/type")) stages)stages(into [(first stages)](comp@@ -384,8 +406,22 @@(defn- normalize-query [query](when-let [query (common/normalize-map query)]- (cond-> query- (not (:lib/metadata query)) (dissoc :lib/metadata))))+ (reduce-kv (fn [query k v]+ (case k+ :lib/metadata (cond-> query+ (nil? v) (dissoc k))+ (:constraints+ :create-row+ :info+ :middleware+ :parameters+ :settings+ :update-row)+ (cond-> query+ (empty? v) (dissoc k))+ #_else query))+ query+ query)))(defn- serialize-query [query];; this stuff all gets added in when you actually run a query with one of the QP entrypoints, and is not considered@@ -399,12 +435,27 @@(= (namespace k) "lib"))))query)))+(defn- encode-query-for-hashing [query]+ (let [keys-for-hashing #{:constraints+ :database+ :destination-database/id+ :impersonation/role+ :lib/type+ :parameters+ :stages}]+ (reduce-kv (fn [m k v]+ (cond-> m+ (contains? keys-for-hashing k) (assoc k v)))+ (common/unfussy-sorted-map)+ query)))+(mr/def ::query[:and[:map- {:description "Valid MBQL 5 query."- :decode/normalize #'normalize-query- :encode/serialize #'serialize-query}+ {:description "Valid MBQL 5 query."+ :decode/normalize #'normalize-query+ :encode/serialize #'serialize-query+ :encode/for-hashing #'encode-query-for-hashing}[:lib/type [:={:decode/normalize common/normalize-keyword, :default :mbql/query}:mbql/query]]@@ -416,37 +467,41 @@[true ::id/saved-questions-virtual-database][false ::id/database]]][:stages [:ref ::stages]]- [:parameters {:optional true} [:maybe [:ref ::lib.schema.parameter/parameters]]]+ [:parameters {:optional true} [:ref ::lib.schema.parameter/parameters]];;;; OPTIONS;;;; These keys are used to tweak behavior of the Query Processor.;;- [:settings {:optional true} [:maybe [:ref ::lib.schema.settings/settings]]]- [:constraints {:optional true} [:maybe [:ref ::lib.schema.constraints/constraints]]]- [:middleware {:optional true} [:maybe [:ref ::lib.schema.middleware-options/middleware-options]]]+ [:settings {:optional true} [:ref ::lib.schema.settings/settings]]+ [:constraints {:optional true} [:ref ::lib.schema.constraints/constraints]]+ [:middleware {:optional true} [:ref ::lib.schema.middleware-options/middleware-options]];; TODO -- `:viz-settings` ?;;;; INFO;;;; Used when recording info about this run in the QueryExecution log; things like context query was ran in and;; User who ran it- [:info {:optional true} [:maybe [:ref ::info/info]]]+ [:info {:optional true} [:ref ::info/info]];;;; ACTIONS;;;; This stuff is only used for Actions.- [:create-row {:optional true} [:maybe [:ref ::actions/row]]]- [:update-row {:optional true} [:maybe [:ref ::actions/row]]]]+ [:create-row {:optional true} [:ref ::actions/row]]+ [:update-row {:optional true} [:ref ::actions/row]]];;;; CONSTRAINTS[:ref ::lib.schema.util/unique-uuids](common/disallowed-keys- {:filter ":filter is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version"- :source-query ":source-query is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version"- :expressions ":expressions is not allowed in the top level of a query, only in MBQL stages"+ {:expressions ":expressions is not allowed in the top level of a query, only in MBQL stages"+ :filter ":filter is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version":filters ":filters is not allowed in the top level of a query, only in MBQL stages"- :source-table ":source-table is not allowed in the top level of a query, only in MBQL stages"})])+ :joins ":joins is not allowed in the top level of a query, only in MBQL stages"+ :native ":native is not allowed in MBQL 5, use :stages instead."+ :query ":query is not allowed in MBQL 5, use :stages instead."+ :source-query ":source-query is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version"+ :source-table ":source-table is not allowed in the top level of a query, only in MBQL stages"+ :type ":type is not allowed in MBQL 5, use :lib/type instead."})])(defn native-only-query?"Whether MBQL 5 `query` only has a single native stage (and is thus pure-native). This is the equivalent of the olddiff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljcindex fa07524b3d0..aae19f9d3c6 100644--- a/src/metabase/lib/schema/common.cljc+++ b/src/metabase/lib/schema/common.cljc@@ -1,5 +1,5 @@(ns metabase.lib.schema.common- (:refer-clojure :exclude [update-keys #?@(:clj [some])])+ (:refer-clojure :exclude [update-keys every? #?@(:clj [some])])(:require[clojure.string :as str][medley.core :as m]@@ -8,7 +8,7 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.memoize :as u.memo]- [metabase.util.performance :refer [update-keys #?@(:clj [some])]]))+ [metabase.util.performance :refer [update-keys every? #?@(:clj [some])]]))(comment metabase.types.core/keep-me)@@ -192,6 +192,41 @@;; remove deprecated `:ident` key(dissoc :ident))))+(defn- unfussy-sorted-map-compare [x y]+ (cond+ (= (type x) (type y))+ (compare x y)++ (keyword? x)+ -1++ (keyword? y)+ 1++ :else+ (compare (str x) (str y))))++(defn unfussy-sorted-map+ "Like [[clojure.core/sorted-map]] but unfussy if you try to [[get]] a keyword in a string-keyed map or [[get]] a+ string in a keyword-keyed map."+ [& kvs]+ (apply sorted-map-by unfussy-sorted-map-compare kvs))++(defn encode-map-for-hashing+ "Base encoding for hashing for all MBQL 5 maps.++ - Convert to a sorted map+ - Remove all namespaced keywords"+ [m]+ (when (map? m)+ (reduce-kv+ (fn [m k v]+ (cond-> m+ (or (= k :lib/type)+ (not (qualified-keyword? k))) (assoc k v)))+ (unfussy-sorted-map)+ m)))+(mu/defn disallowed-keys"Helper for generating a schema to disallow certain keys in a map.@@ -232,7 +267,8 @@[:and{:default {}}[:map- {:decode/normalize normalize-options-map}+ {:decode/normalize #'normalize-options-map+ :encode/for-hashing #'encode-map-for-hashing}[:lib/uuid ::uuid];; these options aren't required for any clause in particular, but if they're present they must follow these schemas.[:base-type {:optional true} [:maybe ::base-type]]@@ -268,3 +304,29 @@(def ^{:arglists '([& classes])} instance-of-class"Convenience for defining a Malli schema for an instance of a particular Class."(memoize instance-of-class*)))++(defn- kebab-cased-key? [k]+ (and (keyword? k)+ (not (str/includes? (str k) "_"))))++(defn- kebab-cased-map? [m]+ (and (map? m)+ (every? kebab-cased-key? (keys m))))++(mr/def ::kebab-cased-map+ [:fn+ {:error/message "map with all kebab-cased keys"+ :error/fn (fn [{:keys [value]} _]+ (if-not (map? value)+ "map with all kebab-cased keys"+ (str "map with all kebab-cased keys, got: " (pr-str (remove kebab-cased-key? (keys value))))))}+ kebab-cased-map?])++(def url-encoded-string-regex+ "Schema for a URL-encoded string++ This matches strings containing:++ - Unreserved characters: letters, digits, hyphens, periods, underscores, tildes+ - Percent-encoded characters: `%` followed by exactly 2 hex digits"+ #"(?:[A-Za-z0-9\-._~]|%[0-9A-Fa-f]{2})+")diff --git a/src/metabase/lib/schema/drill_thru.cljc b/src/metabase/lib/schema/drill_thru.cljcindex 758d93a2244..6ec158f3ad3 100644--- a/src/metabase/lib/schema/drill_thru.cljc+++ b/src/metabase/lib/schema/drill_thru.cljc@@ -195,8 +195,8 @@[:merge::drill-thru.common[:map- [:type [:= :drill-thru/automatic-insights]]- [:lib/type [:= :metabase.lib.drill-thru/drill-thru]]+ [:type [:= :drill-thru/automatic-insights]]+ [:lib/type [:= :metabase.lib.drill-thru/drill-thru]][:column-ref [:maybe [:ref ::lib.schema.ref/ref]]][:dimensions [:ref ::context.row]]]])diff --git a/src/metabase/lib/schema/expression.cljc b/src/metabase/lib/schema/expression.cljcindex 6ad062d2d5f..1fb8270a5b4 100644--- a/src/metabase/lib/schema/expression.cljc+++ b/src/metabase/lib/schema/expression.cljc@@ -108,7 +108,7 @@[:and;; vector = MBQL clause, anything else = not an MBQL clause[:multi- {:dispatch vector?}+ {:dispatch sequential?}[true [:ref :metabase.lib.schema.mbql-clause/clause]][false [:ref :metabase.lib.schema.literal/literal]]][:fndiff --git a/src/metabase/lib/schema/id.cljc b/src/metabase/lib/schema/id.cljcindex 02899dfaa98..b9b83dd283e 100644--- a/src/metabase/lib/schema/id.cljc+++ b/src/metabase/lib/schema/id.cljc@@ -37,7 +37,10 @@(mr/def ::field"Valid Field ID"- pos-int?)+ [:schema+ {:decode/api (fn [id]+ (cond-> id (string? id) parse-long))}+ pos-int?])(mr/def ::card"Valid Card ID"diff --git a/src/metabase/lib/schema/mbql_clause.cljc b/src/metabase/lib/schema/mbql_clause.cljcindex 0bf16d85433..c78a5472050 100644--- a/src/metabase/lib/schema/mbql_clause.cljc+++ b/src/metabase/lib/schema/mbql_clause.cljc@@ -23,6 +23,13 @@(def ^:private invalid-clause-schema[:fn {:error/message "not a known MBQL clause"} (constantly false)])+(mu/defn- mbql-clause-tag :- [:maybe :keyword]+ "If `x` is a (possibly not-yet-normalized) MBQL clause, return its `tag`."+ [x]+ (when (and (sequential? x)+ ((some-fn keyword? string?) (first x)))+ (keyword (first x))))+(defn- clause-schema"Build the schema for `::clause`, a `:multi` schema that maps MBQL clause tag -> the schemain [[clause-schema-registry]]."@@ -30,9 +37,12 @@(into [:multi{:dispatch common/mbql-clause-tag:error/fn (fn [{:keys [value]} _]- (if-let [tag (common/mbql-clause-tag value)]+ (if-let [tag (mbql-clause-tag value)](str "Invalid " tag " clause: " (pr-str value))- "not an MBQL clause"))}+ "not an MBQL clause"))+ :decode/normalize (fn [x]+ (when-let [tag (mbql-clause-tag x)]+ (into [tag] (rest x))))}[::mc/default invalid-clause-schema]](map (fn [tag][tag [:ref (tag->registered-schema-name tag)]]))diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex 13be3d4f39c..76ab72bad25 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -1,9 +1,7 @@(ns metabase.lib.schema.metadata- (:refer-clojure :exclude [every?])(:require#?@(:clj([metabase.util.regex :as u.regex]))- [clojure.string :as str][medley.core :as m][metabase.lib.schema.binning :as lib.schema.binning][metabase.lib.schema.common :as lib.schema.common]@@ -11,25 +9,7 @@[metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]- [metabase.util.malli.registry :as mr]- [metabase.util.performance :refer [every?]]))--(defn- kebab-cased-key? [k]- (and (keyword? k)- (not (str/includes? (str k) "_"))))--(defn- kebab-cased-map? [m]- (and (map? m)- (every? kebab-cased-key? (keys m))))--(mr/def ::kebab-cased-map- [:fn- {:error/message "map with all kebab-cased keys"- :error/fn (fn [{:keys [value]} _]- (if-not (map? value)- "map with all kebab-cased keys"- (str "map with all kebab-cased keys, got: " (pr-str (remove kebab-cased-key? (keys value))))))}- kebab-cased-map?])+ [metabase.util.malli.registry :as mr]));;; Column vs Field?;;;@@ -528,8 +508,7 @@;;;; Additional constraints;;- ;; TODO (Cam 6/13/25) -- go add this to some of the other metadata schemas as well.- ::kebab-cased-map+ ::lib.schema.common/kebab-cased-map[:ref ::column.validate-for-source]])(mr/def ::persisted-info.definition@@ -773,3 +752,10 @@{:decode/normalize lib.schema.common/normalize-map}[:lib/type [:= {:default :metadata/results} :metadata/results]][:columns [:sequential ::column]]])++(mr/def ::transform+ "TODO (Cam 10/1/25) -- I'm putting this here as a placeholder until you guys go fill it out a little more."+ [:map+ [:id ::lib.schema.id/transform]+ [:source {:optional true} [:map+ [:query {:optional true} [:ref :metabase.lib.schema/query]]]]])diff --git a/src/metabase/lib/schema/parameter.cljc b/src/metabase/lib/schema/parameter.cljcindex acffae442a7..8f274e2d0ed 100644--- a/src/metabase/lib/schema/parameter.cljc+++ b/src/metabase/lib/schema/parameter.cljc@@ -26,6 +26,7 @@([flatland.ordered.map :as ordered-map]))[malli.core :as mc][metabase.lib.schema.common :as lib.schema.common]+ [metabase.util :as u][metabase.util.malli.registry :as mr]))(defn- variadic-opts-first@@ -145,8 +146,15 @@(mr/def ::type"Valid parameter :type"- (into [:enum {:error/message "valid parameter type"- :decode/normalize lib.schema.common/normalize-keyword}]+ (into [:enum {:default :text+ :error/message "valid parameter type"+ :decode/normalize (fn [param-type]+ ;; a lot of broken code in Actions was setting param types to invalid things like+ ;; `:type/Text`... fix it+ (when-let [param-type (lib.schema.common/normalize-keyword param-type)]+ (if (= (namespace param-type) "type")+ (keyword (u/lower-case-en (name param-type)))+ param-type)))}](keys types)))(mr/def ::widget-type@@ -183,7 +191,9 @@;;; is not ported to MLv2 yet, so conversion isn't implemented YET.(defn- normalize-legacy-ref [legacy-ref]- ((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) legacy-ref))+ (if (pos-int? legacy-ref)+ [:field legacy-ref nil]+ ((#?(:clj requiring-resolve :cljs resolve) 'metabase.legacy-mbql.normalize/normalize-field-ref) legacy-ref)))(mr/def ::target.legacy-field-ref[:ref@@ -246,7 +256,11 @@#_tag-name [:multi {:dispatch map?}[true [:map[:id ::lib.schema.common/non-blank-string]]]- [false ::lib.schema.common/non-blank-string]]])+ [false [:schema+ {:decode/normalize (fn [x]+ (cond-> x+ (keyword? x) u/qualified-name))}+ ::lib.schema.common/non-blank-string]]]])(mr/def ::variable.target[:multi {:dispatch lib.schema.common/mbql-clause-tag@@ -261,17 +275,36 @@#_tag [:= {:decode/normalize lib.schema.common/normalize-keyword} :variable]#_target [:ref ::variable.target]])+(mr/def ::text-tag+ "A :text-tag parameter :target applies to parameterized text cards in viz settings"+ [:tuple+ [:= {:decode/normalize lib.schema.common/normalize-keyword} :text-tag]+ :string])+(mr/def ::target- [:multi {:dispatch lib.schema.common/mbql-clause-tag+ [:multi {:dispatch (fn [x]+ (if (pos-int? x)+ :field+ (lib.schema.common/mbql-clause-tag x))):error/fn (fn [{:keys [value]} _]- (str "Invalid parameter :target, must be either :field, :dimension, or :variable; got: "- (pr-str value)))}+ (str "Invalid parameter :target, must be either :field, :dimension, :variable, or :text-tag; got: "+ (pr-str value)))+ ;; you're not allowed to have a `:template-tag` here unless it's wrapped in `:variable` or `:dimension`...+ ;; not sure which one is supposed to be correct TBH+ :decode/normalize (fn [x]+ (if (= (lib.schema.common/mbql-clause-tag x) :template-tag)+ [:variable x]+ x))};; TODO (Cam 9/12/25) -- the old legacy MBQL schema also said `:expression` refs where allowed here, but I don't;; know if we actually did allow that in practice.- [:dimension [:ref ::dimension]]- [:variable [:ref ::variable]]+ [:dimension [:ref ::dimension]]+ [:variable [:ref ::variable]]+ [:text-tag [:ref ::text-tag]]+ [:field [:ref ::target.legacy-field-ref]];; MBQL 3 refs like `:field-id` should get normalized to `:field`- [::mc/default [:ref ::target.legacy-field-ref]]])+ [:field-id [:ref ::target.legacy-field-ref]]+ [:field-literal [:ref ::target.legacy-field-ref]]+ [:fk-> [:ref ::target.legacy-field-ref]]])(defn- normalize-parameter[param]@@ -286,14 +319,29 @@param))))(mr/def ::id- [:ref ::lib.schema.common/non-blank-string])+ [:schema+ {:api/regex lib.schema.common/url-encoded-string-regex}+ [:ref ::lib.schema.common/non-blank-string]])++(defn- sort-parameter-values+ "Return the sequence of parameter maps, but with any :value keys sorted if they are a sequence. Parameter values can+ be of mixed types, as bigintegers are passed as strings to avoid precision loss."+ [param-value]+ (if (sequential? param-value)+ (vec (sort-by str param-value))+ param-value))++(mr/def ::parameter.value+ [:schema+ {:encode/for-hashing #'sort-parameter-values}+ :any])(mr/def ::parameter"Schema for the *value* of a parameter (e.g. a Dashboard parameter or a native query template tag) as passed in aspart of the `:parameters` list in a query."[:and[:map- {:decode/normalize normalize-parameter}+ {:decode/normalize #'normalize-parameter}[:type [:ref ::type]];; TODO -- these definitely SHOULD NOT be optional but a ton of tests aren't passing them in like they should be.;; At some point we need to go fix those tests and then make these keys required@@ -301,7 +349,7 @@[:target {:optional true} [:ref ::target]];; not specified if the param has no value. TODO - make this stricter; type of `:value` should be validated based;; on the [[ParameterType]]- [:value {:optional true} :any]+ [:value {:optional true} [:ref ::parameter.value]];; the name of the parameter we're trying to set -- this is actually required now I think, or at least needs to get;; merged in appropriately[:name {:optional true} ::lib.schema.common/non-blank-string]@@ -310,9 +358,15 @@[:slug {:optional true} ::lib.schema.common/non-blank-string][:default {:optional true} :any][:required {:optional true} :any]]+ ::lib.schema.common/kebab-cased-map(lib.schema.common/disallowed-keys{:dimension ":dimension is not allowed in a parameter, you probably meant to use :target [:dimension ...] instead."})])+(defn- encode-parameters-for-hashing [parameters]+ (vec (sort-by (some-fn :id (constantly "")) parameters)))+(mr/def ::parameters"Schema for a list of `:parameters` as passed in to a query."- [:sequential [:ref ::parameter]])+ [:sequential+ {:encode/for-hashing #'encode-parameters-for-hashing}+ [:ref ::parameter]])diff --git a/src/metabase/lib/schema/ref.cljc b/src/metabase/lib/schema/ref.cljcindex f755574dc85..7d6e8bc8f8d 100644--- a/src/metabase/lib/schema/ref.cljc+++ b/src/metabase/lib/schema/ref.cljc@@ -29,6 +29,7 @@[:join-alias {:optional true} [:ref ::lib.schema.join/alias]][:temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]][:binning {:optional true} [:ref ::binning/binning]]+ [:lib/original-binning {:optional true} [:ref ::binning/binning]][:metabase.lib.field/original-effective-type {:optional true} [:ref ::common/base-type]][:metabase.lib.field/original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]];;@@ -73,6 +74,19 @@(mbql-clause/define-mbql-clause :field[:and+ {:decode/normalize (fn [x]+ (when (sequential? x)+ (cond+ (= (count x) 2)+ [:field {} (second x)]++ (and (= (count x) 3)+ ((some-fn pos-int? string?) (second x))+ ((some-fn map? nil?) (last x)))+ [:field (or (last x) {}) (second x)]++ :else+ x)))}[:tuple[:= {:decode/normalize common/normalize-keyword} :field][:ref ::field.options]diff --git a/src/metabase/lib/schema/template_tag.cljc b/src/metabase/lib/schema/template_tag.cljcindex ef00e34d417..18de5350fcf 100644--- a/src/metabase/lib/schema/template_tag.cljc+++ b/src/metabase/lib/schema/template_tag.cljc@@ -69,6 +69,10 @@[:alias {:optional true} :string][:dimension {:optional true} [:ref :mbql.clause/field]]]])+(mr/def ::field-filter.options+ [:map+ {:decode/normalize common/normalize-map-no-kebab-case}])+;; Example:;;;; {:id "c20851c7-8a80-0ffa-8a99-ae636f0e9539"@@ -90,7 +94,7 @@;; are allowed to be specified for it.[:widget-type [:ref ::widget-type]];; optional map to be appended to filter clause- [:options {:optional true} [:maybe :map]]]])+ [:options {:optional true} [:maybe ::field-filter.options]]]])(mr/def ::disallow-dimension(common/disallowed-keys {:dimension ":dimension is only allowed for :type :dimension template tags"}))@@ -170,13 +174,23 @@;; :number, :text, :date[::mc/default [:ref ::raw-value]]]])+;;; make sure people don't try to pass in a `:name` that's different from the actual key in the map.+(mr/def ::template-tag-map.validate-names+ [:fn+ {:error/message "keys in template tag map must match the :name of their values"+ :decode/normalize (fn [m]+ (when (map? m)+ (reduce-kv+ (fn [m k _v]+ (assoc-in m [k :name] k))+ m+ m)))}+ (fn [m]+ (every? (fn [[tag-name tag-definition]]+ (= tag-name (:name tag-definition)))+ m))])+(mr/def ::template-tag-map[:and[:map-of ::name ::template-tag]- ;; make sure people don't try to pass in a `:name` that's different from the actual key in the map.- [:fn- {:error/message "keys in template tag map must match the :name of their values"}- (fn [m]- (every? (fn [[tag-name tag-definition]]- (= tag-name (:name tag-definition)))- m))]])+ [:ref ::template-tag-map.validate-names]])diff --git a/src/metabase/lib/schema/util.cljc b/src/metabase/lib/schema/util.cljcindex d1709fe8576..b16f0914b44 100644--- a/src/metabase/lib/schema/util.cljc+++ b/src/metabase/lib/schema/util.cljc@@ -16,7 +16,6 @@(if (@result our-uuid)(vswap! result vary-meta update :duplicates (fnil conj #{}) our-uuid)(vswap! result conj our-uuid)))-(reduce-kv (fn [_ k v](when (not (qualified-keyword? k))(collect-uuids* v result)))@@ -125,7 +124,7 @@(map? x) (dissoc :lib/uuid)))x))-(defn- indexed-order-bys-for-stage+(defn indexed-order-bys-for-stage"Convert all order-bys in a stage to refer to aggregations by index instead of uuid"[{:keys [aggregation order-by] :as stage}](if (and aggregation order-by)diff --git a/src/metabase/lib/util.cljc b/src/metabase/lib/util.cljcindex 8b328e88957..28798f0567c 100644--- a/src/metabase/lib/util.cljc+++ b/src/metabase/lib/util.cljc@@ -500,11 +500,6 @@[query :- :map](:lib/type (query-stage query 0)))-(mu/defn first-stage-is-native? :- :boolean- "Whether the first stage of the query is a native query stage."- [query :- :map]- (= (first-stage-type query) :mbql.stage/native))-(mu/defn- unique-alias :- :string[original :- :stringsuffix :- :string]@@ -731,45 +726,3 @@(:query :native) :mbql-version/legacy;; otherwise, this is not a valid MBQL query.nil))--(defn collect-source-tables- "Return sequence of source tables from `query`.-- DEPRECATED: This operates on legacy MBQL, so it's really out of place here in Lib.- Use [[metabase.lib.walk.util/all-source-table-ids]] going forward."- {:deprecated "0.57.0"}- [legacy-query]- #_{:clj-kondo/ignore [:deprecated-var]}- (let [from-joins (mapcat collect-source-tables (:joins legacy-query))]- (if-let [source-query (:source-query legacy-query)]- (concat (collect-source-tables source-query) from-joins)- (cond->> from-joins- (:source-table legacy-query) (cons (:source-table legacy-query))))))--(defn- split-tables-and-legacy-card-refs [source-ids]- (-> (reduce (fn [m src]- (if-let [card-id (legacy-string-table-id->card-id src)]- (update m :card conj! card-id)- (update m :table conj! src)))- {:card (transient #{})- :table (transient #{})}- source-ids)- (update-vals persistent!)))--(mu/defn source-tables-and-cards :- [:map- [:card {:optional true} [:set ::lib.schema.id/card]]- [:table {:optional true} [:set ::lib.schema.id/table]]]- "Returns all the table and card IDs that this list of **legacy, MBQL 4 queries** depend on."- [legacy-queries]- (let [metric-ids (-> (lib.util.match/match legacy-queries- [:metric (id :guard pos-int?) & _]- id)- set)- source-ids (-> #{}- (into (map #(str "card__" %)) metric-ids)- (into- ;; existing usage -- do not use in new code- #_{:clj-kondo/ignore [:deprecated-var]}- (mapcat #(collect-source-tables (:query %)))- legacy-queries))]- (split-tables-and-legacy-card-refs source-ids)))diff --git a/src/metabase/lib/validate.cljc b/src/metabase/lib/validate.cljcindex f057b4b0ef5..92d75fe8bbb 100644--- a/src/metabase/lib/validate.cljc+++ b/src/metabase/lib/validate.cljc@@ -14,18 +14,19 @@[query :- ::lib.schema/query](let [bad-fields (volatile! [])](lib.walk/walk-clauses- query (fn [query path-type path clause]- (when (and (= path-type :lib.walk/stage)- (vector? clause)- (= (first clause) :field))- (let [column (lib.walk/apply-f-for-stage-at-path- lib.field.resolution/resolve-field-ref- query path clause)]- (when (or (not column)- (::lib.field.resolution/fallback-metadata? column)- (not (:active column true)))- (vswap! bad-fields conj clause))))- nil))+ query+ (fn [query path-type path clause]+ (when (and (= path-type :lib.walk/stage)+ (vector? clause)+ (= (first clause) :field))+ (let [column (lib.walk/apply-f-for-stage-at-path+ lib.field.resolution/resolve-field-ref+ query path clause)]+ (when (or (not column)+ (::lib.field.resolution/fallback-metadata? column)+ (not (:active column true)))+ (vswap! bad-fields conj clause))))+ nil))(not-empty @bad-fields)))(commentdiff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljcindex cfe8452e452..77359783929 100644--- a/src/metabase/lib/walk/util.cljc+++ b/src/metabase/lib/walk/util.cljc@@ -1,8 +1,10 @@(ns metabase.lib.walk.util"Utility functions built on top of [[metabase.lib.walk]]."(:require+ [clojure.set :as set][metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.lib.util.match :as lib.util.match][metabase.lib.walk :as lib.walk]@@ -61,14 +63,34 @@(assoc template-tag :lib.walk/template-tag-name template-tag-name)))(all-template-tags-map query))))-(mu/defn all-source-card-ids :- [:maybe [:set {:min 1} ::lib.schema.id/card]]- "Return a set of all `:source-card` Card IDs anywhere in the query, as well as all `:card-id`s in template tags."- [query :- ::lib.schema/query]+(defn- all-metric-ids [query]+ (let [metric-ids (volatile! (transient #{}))]+ (lib.walk/walk-clauses+ query+ (fn [_query _path-type _stage-or-join-path clause]+ (lib.util.match/match-lite clause+ [:metric _opts (id :guard pos-int?)]+ (do+ (vswap! metric-ids conj! id)+ nil))))+ (not-empty (persistent! @metric-ids))))++(defn- all-template-tag-card-ids [query](not-empty- (into (set (stage-values-set query (keep :source-card)))+ (into #{}(keep :card-id)(all-template-tags query))))+(mu/defn all-source-card-ids :- [:maybe [:set {:min 1} ::lib.schema.id/card]]+ "Return a set of all `:source-card` Card IDs anywhere in the query, as well as all `:card-id`s in template tags and+ in `:metric` references."+ [query :- ::lib.schema/query]+ (not-empty+ (set/union+ (stage-values-set query (keep :source-card))+ (all-metric-ids query)+ (all-template-tag-card-ids query))))+(mu/defn any-native-stage?"Returns true if any stage of this query is native."[query :- ::lib.schema/query]@@ -86,7 +108,7 @@"Set of all Field IDs referenced in `:field` refs in a query or MBQL clause."[query-or-clause :- [:or::lib.schema/query- vector?]]+ ::lib.schema.mbql-clause/clause]](let [field-ids (volatile! (transient #{}))walk-clause (fn [clause](lib.util.match/match-lite clause@@ -126,6 +148,7 @@(mapcat all-field-ids))(all-template-tags query))))+;;; TODO (Cam 10/1/25) -- overlapping responsibilities with [[metabase.lib.template-tags/template-tags->snippet-ids]](mu/defn all-template-tag-snippet-ids :- [:maybe [:set {:min 1} ::lib.schema.id/snippet]]"Set of all Native Query Snippet IDs used in template tags."[query :- ::lib.schema/query]diff --git a/src/metabase/lib_be/core.clj b/src/metabase/lib_be/core.cljindex f832ad06c77..1d31b2dbde4 100644--- a/src/metabase/lib_be/core.clj+++ b/src/metabase/lib_be/core.clj@@ -1,5 +1,6 @@(ns metabase.lib-be.core(:require+ [metabase.lib-be.hash][metabase.lib-be.metadata.bootstrap][metabase.lib-be.metadata.jvm][metabase.lib-be.models.transforms]@@ -7,17 +8,22 @@[potemkin :as p]))(comment+ metabase.lib-be.hash/keep-memetabase.lib-be.metadata.bootstrap/keep-memetabase.lib-be.metadata.jvm/keep-memetabase.lib-be.models.transforms/keep-memetabase.lib-be.settings/keep-me)(p/import-vars+ [metabase.lib-be.hash+ query-hash][metabase.lib-be.metadata.bootstrapresolve-database][metabase.lib-be.metadata.jvmapplication-database-metadata-provider+ instance->metadatametadata-provider-cache+ with-existing-metadata-provider-cachewith-metadata-provider-cache][metabase.lib-be.models.transformsnormalize-querydiff --git a/src/metabase/lib_be/hash.clj b/src/metabase/lib_be/hash.cljnew file mode 100644index 00000000000..279fb040b24--- /dev/null+++ b/src/metabase/lib_be/hash.clj@@ -0,0 +1,43 @@+(ns metabase.lib-be.hash+ "Code for hashing a query."+ (:require+ [buddy.core.hash :as buddy-hash]+ [malli.core :as mc]+ [malli.transform :as mtx]+ [metabase.lib-be.models.transforms :as lib-be.models.transforms]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.util.json :as json]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(defn- encoder* []+ (mc/encoder+ ::lib.schema/query+ (mtx/transformer+ {:name :for-hashing}+ ;; remove Lib UUID from anywhere that might still have it+ {:encoders {:map (fn [m]+ (cond-> m+ (:lib/uuid m) (dissoc :lib/uuid)))}}+ ;; convert anything that's not a sorted map to one+ {:encoders {:map (fn [m]+ (when (map? m)+ (if (sorted? m)+ m+ (into (lib.schema.common/unfussy-sorted-map) m))))}})))++(defn- encoder []+ (mr/cached ::encoder ::lib.schema/query encoder*))++(defn- encode [query]+ ((encoder) query))++(mu/defn query-hash :- bytes?+ "Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"+ ^bytes [query :- :map]+ (-> query+ (cond-> (not= (keyword (:type query)) :internal) lib-be.models.transforms/normalize-query)+ encode+ json/encode+ buddy-hash/sha3-256))diff --git a/src/metabase/lib_be/metadata/bootstrap.clj b/src/metabase/lib_be/metadata/bootstrap.cljindex b8d8f811e92..a5ea89f5556 100644--- a/src/metabase/lib_be/metadata/bootstrap.clj+++ b/src/metabase/lib_be/metadata/bootstrap.clj@@ -1,20 +1,16 @@(ns metabase.lib-be.metadata.bootstrap(:require[clojure.set :as set]+ [metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.util :as lib.util][metabase.util.i18n :as i18n][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][toucan2.core :as t2]))-(mu/defn- query-type :- [:enum :query :native :internal :mbql/query]- [query :- :map]- (or (some-> ((some-fn :lib/type :type) query) keyword)- (throw (ex-info (i18n/tru "Invalid query: missing or invalid query type (:lib/type or :type)")- {:query query, :type :invalid-query}))))-(mu/defn- source-card-id-for-mbql5-query :- [:maybe ::lib.schema.id/card][query :- :map](-> query :stages first :source-card))@@ -59,7 +55,9 @@Once the *actual* Database ID is resolved, we will create areal [[metabase.lib-be.metadata.jvm/application-database-metadata-provider]]. (The App DB provider needs to be- initialized with an actual Database ID)."+ initialized with an actual Database ID).++ Note that the virtual DB ID is only allowed in legacy MBQL queries, and MBQL 5 queries do not use it."[](->BootstrapMetadataProvider))@@ -73,7 +71,7 @@(mu/defn- source-card-id :- ::lib.schema.id/card[query :- :map]- (case (query-type query)+ (case (lib/normalized-query-type query):mbql/query(source-card-id-for-mbql5-query query)@@ -87,25 +85,31 @@(mu/defn- resolved-database-id :- [:maybe ::lib.schema.id/database][metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]query :- :map]- (when-not (= (query-type query) :internal)- (let [database-id (:database query)]- (cond- (pos-int? database-id)- database-id+ (let [database-id (:database query)]+ (cond+ (pos-int? database-id)+ database-id- (= database-id lib.schema.id/saved-questions-virtual-database-id)- (resolve-database-id-for-source-card metadata-provider (source-card-id query))+ (= database-id lib.schema.id/saved-questions-virtual-database-id)+ (resolve-database-id-for-source-card metadata-provider (source-card-id query))- :else- (throw (ex-info (i18n/tru "Invalid query: missing or invalid Database ID (:database)")- {:query query, :type :invalid-query}))))))+ :else+ (throw (ex-info (i18n/tru "Invalid query: missing or invalid Database ID (:database)")+ {:query query, :type :invalid-query})))))++(mr/def ::empty-map+ [:= {:description "empty map"} {}])++(mr/def ::maybe-unresolved-database-id+ [:or+ ::lib.schema.id/database+ ::lib.schema.id/saved-questions-virtual-database])(mu/defn resolve-database :- [:maybe[:or+ ::empty-map[:map- [:database ::lib.schema.id/database]]- [:map- [:type [:= :internal]]]]]+ [:database ::lib.schema.id/database]]]]"If query has `:database` `-1337` (the legacy database ID for queries using a source Card that had an unknowndatabase), resolve the correct database ID and assoc it into the query."([query]@@ -113,16 +117,16 @@([metadata-provider :- [:maybe ::lib.metadata.protocols/metadata-provider]query :- [:maybe- [:and- :map- [:multi {:dispatch (comp boolean empty?)}- [true [:= {:description "empty map"} {}]]- [false [:map- [:database [:or- ::lib.schema.id/database- ::lib.schema.id/saved-questions-virtual-database]]]]]]]]+ [:or+ ::empty-map+ [:map+ [:database ::maybe-unresolved-database-id]]+ [:map+ ["database" ::maybe-unresolved-database-id]]]]](when (seq query)- (let [query (set/rename-keys query {"database" :database})- database-id (resolved-database-id metadata-provider query)]- (cond-> query- database-id (assoc :database database-id))))))+ (if (pos-int? (:database query))+ query+ (let [query (set/rename-keys query {"database" :database})+ database-id (resolved-database-id metadata-provider query)]+ (cond-> query+ database-id (assoc :database database-id)))))))diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex a5ba12c99d3..26ecf0756a1 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -102,7 +102,7 @@#_resolved-query clojure.lang.IPersistentMap][query-type model parsed-args honeysql](merge (next-method query-type model parsed-args honeysql)- {:select [:id :db_id :name :display_name :schema :active :visibility_type]}))+ {:select [:id :db_id :name :display_name :schema :active :visibility_type :database_require_filter]}))(t2/define-after-select :metadata/table[table]@@ -148,6 +148,7 @@{:select [:field/active:field/base_type:field/coercion_strategy+ :field/database_partitioned:field/database_type:field/description:field/display_name@@ -531,12 +532,22 @@(defmacro with-metadata-provider-cache"Wrapper to create a [[*metadata-provider-cache*]] for the duration of the `body`.- If there is already a [[*metadata-provider-cache*]], this leaves it in place."+ If there is already a [[*metadata-provider-cache*]], this leaves it in place.++ Note that the metadata provider cache is initialized automatically in a REST API request context by+ the [[metabase.server.middleware.metadata-provider-cache]] middleware; if writing a REST API endpoint you do not+ need to manually initialize one."[& body]`(binding [*metadata-provider-cache* (or *metadata-provider-cache*(atom (cache/basic-cache-factory {})))]~@body))+(defmacro with-existing-metadata-provider-cache+ "Wrapper to bind [[*metadata-provider-cache*]] to an existing cache, if you are doing something weird."+ [metadata-provider-cache & body]+ `(binding [*metadata-provider-cache* ~metadata-provider-cache]+ ~@body))+(mu/defn application-database-metadata-provider :- ::lib.schema.metadata/metadata-provider"An implementation of [[metabase.lib.metadata.protocols/MetadataProvider]] for the application database.diff --git a/src/metabase/lib_be/models/transforms.clj b/src/metabase/lib_be/models/transforms.cljindex fbf5935930b..a76ec7cb2b5 100644--- a/src/metabase/lib_be/models/transforms.clj+++ b/src/metabase/lib_be/models/transforms.clj@@ -1,13 +1,30 @@(ns metabase.lib-be.models.transforms+ (:refer-clojure :exclude [some])(:require[metabase.lib-be.metadata.bootstrap :as lib-be.bootstrap][metabase.lib-be.metadata.jvm :as lib.metadata.jvm][metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.lib.schema :as lib.schema][metabase.models.interface :as mi]- [metabase.util.malli :as mu]))+ [metabase.util :as u]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.performance :refer [some]]))-(mu/defn normalize-query+(set! *warn-on-reflection* true)++(defn- has-normalized-key? [m k]+ (or (get m k)+ (get m (u/qualified-name k))))++(defn- has-any-of-these-normalized-keys? [m ks]+ (some (partial has-normalized-key? m) ks))++(mu/defn normalize-query :- [:maybe+ [:multi {:dispatch (comp boolean empty?)}+ [true [:= {:description "empty map"} {}]]+ [false ::lib.schema/query]]]"Normalize an MBQL `query` to MBQL 5 and attach a metadata provider."([query](normalize-query nil query))@@ -23,8 +40,23 @@(empty? query){}+ (not (has-normalized-key? query :database))+ (throw (ex-info "Query must include :database" {:query query}))++ (not (has-any-of-these-normalized-keys? query #{:lib/type :type}))+ (throw (ex-info "Query must include :lib/type or :type" {:query query}))++ (and (has-normalized-key? query :lib/type)+ (has-any-of-these-normalized-keys? query #{:type :query :native}))+ (throw (ex-info "MBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type"+ {:query query}))++ (and (has-normalized-key? query :type)+ (has-normalized-key? query :stages))+ (throw (ex-info "MBQL 5 :stages is not allowed in an MBQL 4 query with :type" {:query query}))+(lib/cached-metadata-provider-with-cache? (:lib/metadata query))- query+ (lib/normalize ::lib.schema/query query):else(->> query@@ -32,7 +64,26 @@metadata-providerable))(lib/query metadata-providerable))))))+(defn- transform-query-in [query]+ (when-not (map? query)+ (throw (ex-info (format "Query must be a map, got ^%s %s" (.getCanonicalName (class query)) (pr-str query))+ {:query query, :status-code 400})))+ (-> query+ normalize-query+ lib/prepare-for-serialization+ mi/json-in))++(defn- transform-query-out [s]+ (when (some? s)+ (try+ (let [query (mi/json-out-without-keywordization s)]+ (assert (map? query)+ (format "Expected deserialized query to be a map, got ^%s %s" (.getCanonicalName (class query)) (pr-str query)))+ (normalize-query query))+ (catch Throwable e+ (log/errorf e "Error deserializing dataset_query from app DB: %s" (ex-message e))+ nil))))+(def transform-query"Toucan 2 transform spec for Card `dataset_query` and other columns that store MBQL."- {:in (comp mi/json-in lib/prepare-for-serialization normalize-query)- :out (comp (mi/catch-normalization-exceptions normalize-query) mi/json-out-without-keywordization)})+ {:in transform-query-in, :out transform-query-out})diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex 15d96a69fc9..63055c20989 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -1,4 +1,11 @@(ns metabase.models.interface+ "Stuff useful to ALL models.++ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+ !!! !!!+ !!! PLEASE DON'T ADD NEW TRANSFORMS HERE, GO PUT THEM IN RELEVANT MODULES THAT USE THEM INSTEAD !!!+ !!! !!!+ !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"(:require[buddy.core.codecs :as codecs][clojure.core.memoize :as memoize]@@ -9,8 +16,6 @@[medley.core :as m][metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.binning :as lib.binning]- [metabase.lib.core :as lib]- [metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.normalize :as lib.normalize][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.temporal-bucket :as lib.temporal-bucket]@@ -193,100 +198,19 @@{:in json-in-with-eliding:out json-out-with-keywordization})-(defn- serialize-mbql5-query- "Saving MBQL 5 queries we can assume MBQL 5 queries are normalized enough already, but remove the metadata provider- before saving it, because it's not something that lends itself well to serialization."- [query]- (dissoc query :lib/metadata))--(defn- deserialize-mbql5-query- "Reading MBQL 5 queries: normalize them, then attach a MetadataProvider based on their Database."- [query]- (let [metadata-provider (if (lib.metadata.protocols/metadata-provider? (:lib/metadata query))- ;; in case someone passes in an already-normalized query to [[maybe-normalize-query]] below,- ;; preserve the existing metadata provider.- (:lib/metadata query)- ((requiring-resolve 'metabase.lib-be.metadata.jvm/application-database-metadata-provider)- (u/the-id (some #(get query %) [:database "database"]))))]- (lib/query metadata-provider query)))--(mu/defn maybe-normalize-query- "For top-level query maps like `Card.dataset_query`. Normalizes them on the way in & out."- [in-or-out :- [:enum :in :out]- query]- (letfn [(normalize [query]- (let [f (if (= (lib/normalized-query-type query) :mbql/query)- ;; MBQL 5 queries- (case in-or-out- :in serialize-mbql5-query- :out deserialize-mbql5-query)- ;; legacy queries: just normalize them with the legacy normalization code for now... in the near- ;; future we'll probably convert to MBQL 5 before saving so everything in the app DB is MBQL 5- (case in-or-out- :in mbql.normalize/normalize- :out mbql.normalize/normalize))]- (f query)))]- (cond-> query- (and (map? query) (seq query))- normalize)))--(defn maybe-normalize-transform-source- "Normalizes the `source` of a transform."- [in-or-out raw-source]- (let [{source-type :type :as source} (m/map-keys keyword raw-source)]- (cond-> source- (= (keyword source-type) :query) (update :query (partial maybe-normalize-query in-or-out)))))-(defn catch-normalization-exceptions"Wraps normalization fn `f` and returns a version that gracefully handles Exceptions during normalization. Wheninvalid queries (etc.) come out of the Database, it's best we handle normalization failures gracefully rather thanletting the Exception cause the entire API call to fail because of one bad object. (See #8914 for more details.)"[f]- (fn [query]+ (fn [x](try- (doall (f query))+ (doall (f x))(catch Throwable e- (log/errorf e "Unable to normalize:\n%s" (u/pprint-to-str 'red query))+ (log/errorf e "Unable to normalize:\n%s" (u/pprint-to-str 'red x))nil))))-(defn normalize-parameters-list- "Normalize `parameters` or `parameter-mappings` when coming out of the application database or in via an API request."- [parameters]- (or (mbql.normalize/normalize-fragment [:parameters] parameters)- []))--(defn- keywordize-temporal_units- [parameter]- (m/update-existing parameter :temporal_units (fn [units] (mapv keyword units))))--(defn normalize-card-parameters-list- "Normalize `parameters` of actions, cards, and dashboards when coming out of the application database."- [parameters]- (->> parameters- normalize-parameters-list- (mapv keywordize-temporal_units)))--(def transform-metabase-query- "Transform for metabase-query."- {:in (comp json-in (partial maybe-normalize-query :in))- :out (comp (catch-normalization-exceptions (partial maybe-normalize-query :out)) json-out-without-keywordization)})--(def transform-transform-source- "Transform for transform source fields."- {:in (comp json-in (partial maybe-normalize-transform-source :in))- :out (comp (catch-normalization-exceptions (partial maybe-normalize-transform-source :out)) json-out-without-keywordization)})--(def transform-parameters-list- "Transform for parameters list."- {:in (comp json-in normalize-parameters-list)- :out (comp (catch-normalization-exceptions normalize-parameters-list) json-out-with-keywordization)})--(def transform-card-parameters-list- "Transform for parameters list."- {:in (comp json-in normalize-card-parameters-list)- :out (comp (catch-normalization-exceptions normalize-card-parameters-list) json-out-with-keywordization)})--(def transform-field-ref+(def ^{:deprecated "0.57.0"} transform-legacy-field-ref"Transform field refs"{:in json-in:out (comp (catch-normalization-exceptions mbql.normalize/normalize-field-ref) json-out-with-keywordization)})@@ -519,6 +443,12 @@{:in identity:out decompress})+;;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+;;; !!! !!!+;;; !!! PLEASE DON'T ADD NEW TRANSFORMS HERE, GO PUT THEM IN RELEVANT MODULES THAT USE THEM INSTEAD !!!+;;; !!! !!!+;;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!+;; --- predefined hooks(defmulti non-timestamped-fieldsdiff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex cd9307dfc4b..23221a2608b 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -1,6 +1,8 @@;; -*- outline-regexp: "[; ]+#+[[:space:]]+" -*-(ns metabase.models.serialization- "Defines core interfaces for serialization.+ "TODO (Cam 10/1/25) -- move this into the serialization module or something like that, not all models need this.++ Defines core interfaces for serialization.Serialization is an enterprise feature, but in the interest of keeping all the code for an entity in one place,these methods are defined here and implemented for all the exported models.@@ -61,6 +63,7 @@[malli.transform :as mtx][medley.core :as m][metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi]@@ -956,7 +959,7 @@"Given a numeric ID, look up a different identifying field for that entity, and return it as a portable ID.Eg. `Database.name`.[[import-fk-keyed]] is the inverse.- Unusual parameter order lets this be called as, for example, `(update x :db_id export-fk-keyed 'Database :name)`.+ Unusual parameter order lets this be called as, for example, `(update x :db_id export-fk-keyed :model/Database :name)`.Note: This assumes the primary key is called `:id`."[id model field]@@ -968,7 +971,7 @@Eg. `Database.name`.Unusual parameter order lets this be called as, for example,- `(update x :creator_id import-fk-keyed 'Database :name)`."+ `(update x :creator_id import-fk-keyed :model/Database :name)`."[portable model field](t2/select-one-pk model field portable))@@ -1000,8 +1003,8 @@[[import-table-fk]] is the inverse."[table-id](when table-id- (let [{:keys [db_id name schema]} (t2/select-one 'Table :id table-id)- db-name (t2/select-one-fn :name 'Database :id db_id)]+ (let [{:keys [db_id name schema]} (t2/select-one :model/Table :id table-id)+ db-name (t2/select-one-fn :name :model/Database :id db_id)][db-name schema name])))(defn ^:dynamic ^::cache *import-table-fk*@@ -1009,13 +1012,13 @@The input might be nil, in which case so is the output. This is legal for a native question."[[db-name schema table-name :as table-id]](when table-id- (if-let [db-id (t2/select-one-fn :id 'Database :name db-name)]- (or (t2/select-one-fn :id 'Table :name table-name :schema schema :db_id db-id)+ (if-let [db-id (t2/select-one-fn :id :model/Database :name db-name)]+ (or (t2/select-one-fn :id :model/Table :name table-name :schema schema :db_id db-id)(throw (ex-info (format "table id present, but no table found: %s" table-id){:table-id table-id})))(throw (ex-info (format "table id present, but database not found: %s" table-id){:table-id table-id- :database-names (sort (t2/select-fn-vec :name 'Table))})))))+ :database-names (sort (t2/select-fn-vec :name :model/Table))})))))(defn table->path"Given a `table_id` as exported by [[export-table-fk]], turn it into a `[{:model ...}]` path for the Table.@@ -1025,7 +1028,7 @@(when schema {:model "Schema" :id schema}){:model "Table" :id table-name}]))-(def ^:private STORAGE-DIRS {"Database" "databases"+(def ^:private storage-dirs {"Database" "databases""Schema" "schemas""Table" "tables""Field" "fields"})@@ -1041,7 +1044,7 @@[path](into [] cat(for [entry path]- [(or (get STORAGE-DIRS (:model entry))+ [(or (get storage-dirs (:model entry))(throw (ex-info "Could not find dir name" {:entry entry})))(:id entry)])))@@ -1163,34 +1166,31 @@(mapv ids->fully-qualified-names &match)map?- (as-> &match entity- (m/update-existing entity :database (fn [db-id]- (if (= db-id lib.schema.id/saved-questions-virtual-database-id)- "database/__virtual"- (t2/select-one-fn :name 'Database :id db-id))))- (m/update-existing entity :card_id #(*export-fk* % 'Card)) ; attibutes that refer to db fields use _- (m/update-existing entity :card-id #(*export-fk* % 'Card)) ; template-tags use dash- (m/update-existing entity :source-table export-source-table)- (m/update-existing entity :source_table export-source-table)- (m/update-existing entity :breakout (fn [breakout]- (mapv mbql-id->fully-qualified-name breakout)))- (m/update-existing entity :aggregation (fn [aggregation]- (mapv mbql-id->fully-qualified-name aggregation)))- (m/update-existing entity :filter ids->fully-qualified-names)- (m/update-existing entity ::mb.viz/param-mapping-source *export-field-fk*)- (m/update-existing entity :segment *export-fk* 'Segment)- (m/update-existing entity :snippet-id *export-fk* 'NativeQuerySnippet)- (merge entity- (m/map-vals ids->fully-qualified-names- (dissoc entity- :database :card_id :card-id :source-table :breakout :aggregation :filter :segment- ::mb.viz/param-mapping-source :snippet-id))))))+ (reduce-kv+ (fn [entity k _v]+ (let [f (case k+ :database (fn [db-id]+ (if (= db-id lib.schema.id/saved-questions-virtual-database-id)+ "database/__virtual"+ (t2/select-one-fn :name :model/Database :id db-id)))+ (:card_id :card-id) #(*export-fk* % :model/Card) ; attibutes that refer to db fields use `_`; template-tags use `-`+ (:source_table :source-table) export-source-table+ ::mb.viz/param-mapping-source *export-field-fk*+ :segment #(*export-fk* % :model/Segment)+ :snippet-id #(*export-fk* % :model/NativeQuerySnippet)+ #_else ids->fully-qualified-names)]+ (update entity k f)))+ &match+ &match)))(defn export-mbql"Given an MBQL expression, convert it to an EDN structure and turn the non-portable Database, Table and Field IDsinside it into portable references."[encoded]- (ids->fully-qualified-names encoded))+ (let [encoded (cond-> encoded+ ;; temporary usage until we port SerDes to Lib / MBQL 5+ (:lib/type encoded) #_{:clj-kondo/ignore [:discouraged-var]} lib/->legacy-MBQL)]+ (ids->fully-qualified-names encoded)))(defn- portable-id?"True if the provided string is either an Entity ID or identity-hash string."@@ -1222,7 +1222,7 @@(-> &match(assoc :database (if (= fully-qualified-name "database/__virtual")lib.schema.id/saved-questions-virtual-database-id- (t2/select-one-pk 'Database :name fully-qualified-name)))+ (t2/select-one-pk :model/Database :name fully-qualified-name)))mbql-fully-qualified-names->ids*) ; Process other keys{:card-id (entity-id :guard portable-id?)}@@ -1294,6 +1294,8 @@entity)))(defn- mbql-deps-map [entity]+ (assert (not (:lib/type entity))+ "SerDes v2 does not currently work on MBQL 5, please convert to legacy first")(->> (for [[k v] entity](cond(and (= k :database)@@ -1359,7 +1361,7 @@[parameters](reduce set/union #{}(for [parameter parameters- :when (= "card" (:values_source_type parameter))+ :when (= (keyword (:values_source_type parameter)) :card):let [config (:values_source_config parameter)]](set/union #{[{:model "Card" :id (:card_id config)}]}(mbql-deps-vector (:value_field config))))))@@ -1389,7 +1391,7 @@(merge entity{:id (case model"table" (*export-table-fk* id)- "database" (*export-fk-keyed* id 'Database :name)+ "database" (*export-fk-keyed* id :model/Database :name)(*export-fk* id (link-card-model->toucan-model model)))}))))(defn- json-ids->fully-qualified-names@@ -1563,7 +1565,7 @@(merge entity{:id (case model"table" (*import-table-fk* id)- "database" (*import-fk-keyed* id 'Database :name)+ "database" (*import-fk-keyed* id :model/Database :name)(*import-fk* id (link-card-model->toucan-model model)))}))))(defn- import-visualizations [entity]diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.cljindex 8bef8d2da48..5aae77763c5 100644--- a/src/metabase/parameters/chain_filter.clj+++ b/src/metabase/parameters/chain_filter.clj@@ -67,9 +67,12 @@[clojure.string :as str][honey.sql :as sql][metabase.app-db.core :as mdb]- [metabase.driver.common.parameters.dates :as params.dates]- [metabase.legacy-mbql.util :as mbql.u]- [metabase.lib.util.match :as lib.util.match]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.types.isa :as lib.types.isa][metabase.parameters.chain-filter.dedupe-joins :as dedupe][metabase.parameters.field-values :as params.field-values][metabase.parameters.params :as params]@@ -77,13 +80,14 @@[metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.permissions :as qp.perms]+ [metabase.query-processor.parameters.dates :as params.dates][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.setup :as qp.setup]- [metabase.types.core :as types][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.warehouse-schema.metadata-queries :as schema.metadata-queries][metabase.warehouse-schema.models.field :as field]@@ -94,17 +98,17 @@;; so the hydration method for name_field is loaded(comment params/keep-me)-(def Constraint+(mr/def ::constraint"Schema for a constraint on a field."[:map- [:field-id ms/PositiveInt]- [:op :keyword]- [:value :any]- [:options {:optional true} [:maybe map?]]])+ [:field-id ::lib.schema.id/field]+ [:op :keyword] ; name of an MBQL filter clause e.g. `:=` or `:starts-with`+ [:value :any]+ [:options {:optional true} [:maybe map?]]])-(def Constraints- "Schema for a list of Constraints."- [:sequential Constraint])+(mr/def ::constraints+ "Schema for a list of constraints."+ [:sequential ::constraint])(def ^:dynamic *enable-reverse-joins*"Whether to chain filter via joins where we must follow relationships in reverse, e.g. child -> parent (e.g.@@ -126,31 +130,32 @@(t2/select-one [:model/Field :base_type :semantic_type] :id field-id)):ttl/threshold (u/minutes->ms 10)))-(mu/defn- filter-clause+(mu/defn- add-filter :- ::lib.schema/query"Generate a single MBQL `:filter` clause for a Field and `value` (or multiple values, if `value` is a collection)."- [source-table-id- {:keys [field-id op value options]} :- Constraint]- (let [{:keys [base_type] :as field-metadata} (memoized-field-types-by-id field-id)- field-clause (let [this-field-table-id (field/field-id->table-id field-id)]- [:field field-id (merge (when base_type- ;; This may be prone to eg. coercion errors. However effective in- ;; _clause options_ is not standard part of options.- {:base-type base_type})- (when-not (= this-field-table-id source-table-id)- {:join-alias (joined-table-alias this-field-table-id)}))])]- (if (and #_{:clj-kondo/ignore [:deprecated-var]} (types/temporal-field? field-metadata) ; legacy usage -- do not use going forward- (string? value))- (u/ignore-exceptions- (params.dates/date-string->filter value field-id))- ;; we don't want to skip our value, even if its nil- (let [values (if (nil? value) [nil] (u/one-or-many value))]- (if (and (#{:starts-with :ends-with :contains :does-not-contain} op)- (next values))- ;; special form: options come after the tag- (into [op options field-clause] values)- ;; standard form: options at the end- (cond-> (into [op field-clause] values)- (seq options) (conj options)))))))+ [query :- ::lib.schema/query+ source-table-id :- ::lib.schema.id/table+ {:keys [field-id op value options]} :- ::constraint]+ (let [field (let [this-field-table-id (field/field-id->table-id field-id)]+ (cond-> (lib.metadata/field query field-id)+ (not= this-field-table-id source-table-id)+ (lib/with-join-alias (joined-table-alias this-field-table-id))))+ filter-op (if (and (lib.types.isa/temporal? field)+ (string? value))+ (try+ (params.dates/date-string->filter value field-id)+ (catch Throwable e+ (log/error e "Error creating filter for date string")+ nil))+ ;; we don't want to skip our value, even if its nil+ (let [values (if (nil? value) [nil] (u/one-or-many value))]+ {:lib/type :lib/external-op+ :operator op+ :options options+ :args (cons field values)}))]+ (when filter-op+ (log/tracef "Adding filter clause: %s" (pr-str filter-op)))+ (cond-> query+ filter-op (lib/filter filter-op))))(defn- name-for-logging [model id](format "%s %d %s" (name model) id (u/format-color 'blue (pr-str (t2/select-one-fn :name model :id id)))))@@ -168,7 +173,11 @@(format "%d. %s" (inc i) (format-join-for-logging join)))joins)))-(defn- add-filters [query source-table-id joined-table-ids constraints]+(mu/defn- add-filters :- ::lib.schema/query+ [query :- ::lib.schema/query+ source-table-id :- ::lib.schema.id/table+ joined-table-ids :- [:set ::lib.schema.id/table]+ constraints :- [:maybe ::constraints]](reduce(fn [query {:keys [field-id] :as constraint}];; only add a where clause for the Field if it's part of the source Table or if we're actually joining against@@ -177,13 +186,12 @@(let [field-table-id (field/field-id->table-id field-id)](if (or (= field-table-id source-table-id)(contains? joined-table-ids field-table-id))- (let [clause (filter-clause source-table-id constraint)]- (log/tracef "Added filter clause for %s %s: %s"+ (do+ (log/tracef "Added filter clause for %s %s with constraint %s"(name-for-logging :model/Table field-table-id)(name-for-logging :model/Field field-id)- clause)- #_{:clj-kondo/ignore [:deprecated-var]}- (update query :filter mbql.u/combine-filter-clauses clause))+ (pr-str constraint))+ (add-filter query source-table-id constraint))(do(log/tracef "Not adding filter clause for %s %s because we did not join against its Table"(name-for-logging :model/Table field-table-id)@@ -341,14 +349,15 @@(log/tracef "Deduplicated:\n%s" (format-joins-for-logging <>))))))):ttl/threshold find-joins-cache-duration-ms))-(defn- find-all-joins+(mu/defn- find-all-joins"Find the complete set of joins we need to do for `source-table-id` to join against Fields in `field-ids`."- [source-table-id field-ids]+ [source-table-id :- ::lib.schema.id/table+ field-ids :- [:set ::lib.schema.id/field]](when-let [other-table-ids (not-empty (disj (set (map field/field-id->table-id (set field-ids)))source-table-id))](find-all-joins* source-table-id other-table-ids *enable-reverse-joins*)))-(defn- add-joins+(mu/defn- add-joins :- ::lib.schema/query"Add joins to the MBQL `query` we're generating. The Field for which we are returning values is the \"source Field\",and the Table it belongs to is the source Table; `field-ids` is a set of Fields belonging to Tables other than thesource Table.@@ -364,91 +373,99 @@Since we're not sure which way the relationship goes, `resolve-fk-id` fetches all possible relationships between thetwo Tables and we generate the appropriate join against the other Table."- [query source-table-id joins]+ [query :- ::lib.schema/query+ source-table-id :- ::lib.schema.id/table+ joins](reduce(fn [query {{lhs-table-id :table, lhs-field-id :field} :lhs, {rhs-table-id :table, rhs-field-id :field} :rhs}]- (let [join {:source-table rhs-table-id- :condition [:=- [:field lhs-field-id (when-not (= lhs-table-id source-table-id)- {:join-alias (joined-table-alias lhs-table-id)})]- [:field rhs-field-id {:join-alias (joined-table-alias rhs-table-id)}]]- :alias (joined-table-alias rhs-table-id)}]+ (let [lhs-field (lib.metadata/field query lhs-field-id)+ rhs-field (lib.metadata/field query rhs-field-id)+ rhs-table (lib.metadata/table query rhs-table-id)+ join (-> (lib/join-clause rhs-table)+ (lib/with-join-alias (joined-table-alias rhs-table-id))+ (lib/with-join-conditions [(lib/=+ (cond-> lhs-field+ (not= lhs-table-id source-table-id)+ (lib/with-join-alias (joined-table-alias lhs-table-id)))+ (-> rhs-field+ (lib/with-join-alias (joined-table-alias rhs-table-id))))]))](log/tracef "Adding join against %s\n%s"- (name-for-logging :model/Table rhs-table-id) (u/pprint-to-str join))- (update query :joins concat [join])))+ (name-for-logging :model/Table rhs-table-id) (u/cprint-to-str join))+ (lib/join query join)))queryjoins))-(def ^:private Options+(mr/def ::options;; if original-field-id is specified, we'll include this in the results. For Field->Field remapping.[:map {:closed true}- [:original-field-id {:optional true} [:maybe ms/PositiveInt]]+ [:original-field-id {:optional true} [:maybe ::lib.schema.id/field]];; return at most the lesser of `limit` (if specified) and `max-results`.[:limit {:optional true} [:maybe ms/PositiveInt]]])(def ^:private max-results 1000)-(mu/defn- chain-filter-mbql-query+(mu/defn- chain-filter-mbql-query :- ::lib.schema/query"Generate the MBQL query powering `chain-filter`."- [field-id :- ms/PositiveInt- constraints :- [:maybe Constraints]- {:keys [original-field-id limit]} :- [:maybe Options]]- {:database (field/field-id->database-id field-id)- :type :query- :query (let [source-table-id (field/field-id->table-id field-id)- joins (find-all-joins source-table-id (cond-> (set (map :field-id constraints))- original-field-id (conj original-field-id)))- joined-table-ids (set (map #(get-in % [:rhs :table]) joins))- original-field-clause (when original-field-id- (let [original-table-id (field/field-id->table-id original-field-id)]- [:field- original-field-id- (when-not (= source-table-id original-table-id)- {:join-alias (joined-table-alias original-table-id)})]))]- (when original-field-id- (log/tracef "Finding values of %s, remapped from %s."- (name-for-logging :model/Field field-id)- (name-for-logging :model/Field original-field-id))- (log/tracef "MBQL clause for %s is %s"- (name-for-logging :model/Field original-field-id) (pr-str original-field-clause)))- (when (seq joins)- (log/tracef "Generating joins and filters for source %s with joins info\n%s"- (name-for-logging :model/Table source-table-id) (pr-str joins)))- (-> (merge {:source-table source-table-id- ;; return the lesser of limit (if set) or max results- :limit ((fnil min Integer/MAX_VALUE) limit max-results)}- (if original-field-clause- {;; don't return rows that don't have values for the original Field. e.g. if- ;; venues.category_id is remapped to categories.name and we do a search with query 's',- ;; we only want to return [category_id name] tuples where [category_id] is not nil- ;;- ;; TODO -- would this be more efficient if we just did an INNER JOIN against the original- ;; Table instead of a LEFT JOIN with this additional filter clause? Would that still- ;; work?- :filter [:not-null original-field-clause]- ;; for Field->Field remapping we want to return pairs of [original-value remapped-value],- ;; but sort by [remapped-value]- :order-by [[:asc [:field field-id nil]]]- ;; original-field-id is used to power Field->Field breakouts.- ;; We include both remapped and original- :breakout [original-field-clause [:field field-id nil]]}- {:breakout [[:field field-id nil]]}))- (add-joins source-table-id joins)- (add-filters source-table-id joined-table-ids constraints)- schema.metadata-queries/add-required-filters-if-needed))- :middleware {:disable-remaps? true}})+ [field-id :- ::lib.schema.id/field+ constraints :- [:maybe ::constraints]+ {:keys [original-field-id limit]} :- [:maybe ::options]]+ (log/tracef "Chain filter %s with constraints %s" (name-for-logging :model/Field field-id) (u/cprint-to-str constraints))+ (let [database-id (field/field-id->database-id field-id)+ mp (lib-be/application-database-metadata-provider database-id)+ source-table-id (field/field-id->table-id field-id)+ joins (find-all-joins source-table-id (cond-> (set (map :field-id constraints))+ original-field-id (conj original-field-id)))+ joined-table-ids (set (map #(get-in % [:rhs :table]) joins))+ field (lib.metadata/field mp field-id)+ original-field (when original-field-id+ (let [original-table-id (field/field-id->table-id original-field-id)]+ (cond-> (lib.metadata/field mp original-field-id)+ (not= source-table-id original-table-id)+ (lib/with-join-alias (joined-table-alias original-table-id)))))]+ (when original-field-id+ (log/tracef "Finding values of %s, remapped from %s."+ (name-for-logging :model/Field field-id)+ (name-for-logging :model/Field original-field-id)))+ (when (seq joins)+ (log/tracef "Generating joins and filters for source %s with joins info\n%s"+ (name-for-logging :model/Table source-table-id) (u/cprint-to-str joins)))+ (-> (lib/query mp (lib.metadata/table mp source-table-id))+ ;; return the lesser of limit (if set) or max results+ (lib/limit ((fnil min Integer/MAX_VALUE) limit max-results))+ (assoc-in [:middleware :disable-remaps?] true)+ (add-joins source-table-id joins)+ (cond-> original-field (->+ ;; don't return rows that don't have values for the original Field. e.g. if+ ;; venues.category_id is remapped to categories.name and we do a search with query+ ;; 's', we only want to return [category_id name] tuples where [category_id] is+ ;; not nil+ ;;+ ;; TODO -- would this be more efficient if we just did an INNER JOIN against the+ ;; original Table instead of a LEFT JOIN with this additional filter clause? Would+ ;; that still work?+ (lib/filter (lib/not-null original-field))+ ;; original-field-id is used to power Field->Field breakouts.+ ;; We include both remapped and original+ (lib/breakout original-field)+ (lib/breakout field)+ ;; for Field->Field remapping we want to return pairs of [original-value+ ;; remapped-value], but sort by [remapped-value]+ (lib/order-by field))+ (not original-field) (lib/breakout field))+ (add-filters source-table-id joined-table-ids constraints)+ schema.metadata-queries/add-required-filters-if-needed)));;; ------------------------ Chain filter (powers GET /api/dashboard/:id/params/:key/values) -------------------------(mu/defn- unremapped-chain-filter :- ms/FieldValuesResult"Chain filtering without all the fancy remapping stuff on top of it."- [field-id :- ms/PositiveInt- constraints :- [:maybe Constraints]- options :- [:maybe Options]]+ [field-id :- ::lib.schema.id/field+ constraints :- [:maybe ::constraints]+ options :- [:maybe ::options]](let [mbql-query (chain-filter-mbql-query field-id constraints options)]- (log/debugf "Chain filter MBQL query:\n%s" (u/pprint-to-str 'magenta mbql-query))+ (log/debugf "Chain filter MBQL query:\n%s" (u/cprint-to-str mbql-query))(try- (let [query-limit (get-in mbql-query [:query :limit])+ (let [query-limit (lib/current-limit mbql-query);; FIXME: this can OOM for text column if each value are too large. See #46411;; Consider using the [[field-values/distinct-text-field-rff] rff]values (qp/process-query mbql-query (constantly conj))]@@ -535,18 +552,18 @@:limit 1});; TODO -- add some caching here?-(mu/defn remapped-field-id :- [:maybe ms/PositiveInt]+(mu/defn remapped-field-id :- [:maybe ::lib.schema.id/field]"Efficient query to find the ID of the Field we're remapping `field-id` to, if it has either type of Field -> Fieldremapping."- [field-id :- [:maybe ms/PositiveInt]]+ [field-id :- [:maybe ::lib.schema.id/field]](:id (t2/query-one (remapped-field-id-query field-id))))(mu/defn remapping :- [:maybe [:map- [:id ms/PositiveInt]+ [:id ::lib.schema.id/field][:mapping-type [:enum :fk->field :fk->pk->name :pk->name]]]]"Efficient query to find the ID of the Field we're remapping `field-id` to, if it has either type of Field -> Fieldremapping."- [field-id :- [:maybe ms/PositiveInt]]+ [field-id :- [:maybe ::lib.schema.id/field]](when-let [raw-mapping (t2/query-one (remapped-field-id-query field-id))](-> raw-mapping(dissoc :mapping_type)@@ -568,9 +585,12 @@qp.preprocess/preprocessqp.perms/check-query-permissions*))))-(defn- cached-field-values [field-id constraints {:keys [limit]}]+(mu/defn- cached-field-values+ [field-id :- ::lib.schema.id/field+ constraints :- [:maybe ::constraints]+ {:keys [limit], :as _options}];; TODO: why don't we remap the human readable values here?- (let [{:keys [values has_more_values]}+ (let [{:keys [values] has-more-values? :has_more_values}(if (empty? constraints)(params.field-values/get-or-create-field-values-for-current-user! (t2/select-one :model/Field :id field-id))(params.field-values/get-or-create-linked-filter-field-values! (t2/select-one :model/Field :id field-id) constraints))]@@ -578,7 +598,7 @@limit (take limit)):has_more_values (or (when limit(< limit (count values)))- has_more_values)}))+ has-more-values?)}))(mu/defn chain-filter :- ms/FieldValuesResult"Fetch a sequence of possible values of Field with `field-id` by restricting the possible values to rows that match@@ -603,8 +623,8 @@For remapped columns (when remapping is detected or when an explicit remapping field-id is provided), this returnsresults as a sequence of `[value remapped-value]` pairs."- [field-id :- ms/PositiveInt- constraints :- [:maybe Constraints]+ [field-id :- ::lib.schema.id/field+ constraints :- [:maybe ::constraints]& options](assert (even? (count options)))(let [{:as options} options@@ -667,10 +687,10 @@:base-type base-type}))))))(mu/defn- unremapped-chain-filter-search- [field-id :- ms/PositiveInt- constraints :- [:maybe Constraints]+ [field-id :- ::lib.schema.id/field+ constraints :- [:maybe ::constraints]query :- ms/NonBlankString- options :- [:maybe Options]]+ options :- [:maybe ::options]](check-valid-search-field field-id)(let [constraints (conj constraints {:field-id field-id:op :contains@@ -689,11 +709,11 @@"Chain filter search, but for Fields that have human-readable values defined (e.g. you've went in and specified thatenum value `1` should be displayed as `BIRD_TYPE_TOUCAN`). `v->human-readable` is a map of actual values in thedatabase (e.g. `1`) to the human-readable version (`BIRD_TYPE_TOUCAN`)."- [field-id :- ms/PositiveInt+ [field-id :- ::lib.schema.id/fieldv->human-readable :- ::parameters.schema/human-readable-remapping-map- constraints :- [:maybe Constraints]+ constraints :- [:maybe ::constraints]query :- ms/NonBlankString- options :- [:maybe Options]]+ options :- [:maybe ::options]](or (when-let [unremapped-values (not-empty (matching-unremapped-values query v->human-readable))](let [constraints (conj constraints {:field-id field-id:op :=@@ -736,9 +756,9 @@(mu/defn chain-filter-search :- ms/FieldValuesResult"Convenience version of `chain-filter` that adds a constraint to only return values of Field with `field-id`containing String `query`. Powers the `search/:query` version of the chain filter endpoint."- [field-id :- ms/PositiveInt- constraints :- [:maybe Constraints]- query :- [:maybe ms/NonBlankString]+ [field-id :- ::lib.schema.id/field+ constraints :- [:maybe ::constraints]+ query-string :- [:maybe ms/NonBlankString]& options](assert (even? (count options)))(let [{:as options} options@@ -751,37 +771,36 @@(not (isa? (or effective_type base_type) :type/UUID)))](remapped-field-id field-id))))](cond- (str/blank? query)+ (str/blank? query-string)(apply chain-filter field-id constraints options)(some? @v->human-readable)- (human-readable-values-remapped-chain-filter-search field-id @v->human-readable constraints query options)+ (human-readable-values-remapped-chain-filter-search field-id @v->human-readable constraints query-string options)(and (search-cached-field-values? field-id constraints) (nil? @the-remapped-field-id))(do(check-field-value-query-permissions field-id constraints options)- (cached-field-values-search field-id query constraints options))+ (cached-field-values-search field-id query-string constraints options))(some? @the-remapped-field-id)- (unremapped-chain-filter-search @the-remapped-field-id constraints query (assoc options :original-field-id field-id))+ (unremapped-chain-filter-search @the-remapped-field-id constraints query-string (assoc options :original-field-id field-id)):else- (unremapped-chain-filter-search field-id constraints query options))))+ (unremapped-chain-filter-search field-id constraints query-string options))));;; ------------------ Filterable Field IDs (powers GET /api/dashboard/params/valid-filter-fields) --------------------(mu/defn filterable-field-ids+(mu/defn filterable-field-ids :- [:maybe [:set :metabase.lib.schema.id/field]]"Return the subset of `filter-ids` we can actually use in a `chain-filter` query to fetch values of Field with`id`.;; maybe we can't filter against Field 2 because there's no FK-> relationship(filterable-field-ids 1 #{2 3 4}) ; -> #{3 4}"- [field-id :- ms/PositiveInt- filter-field-ids :- [:maybe [:set ms/PositiveInt]]]+ [field-id :- ::lib.schema.id/field+ filter-field-ids :- [:maybe [:set ::lib.schema.id/field]]](when (seq filter-field-ids)(let [mbql-query (chain-filter-mbql-query field-id(for [id filter-field-ids]- {:field-id id :op := :value nil})+ {:field-id id, :op :=, :value nil})nil)]- (set (lib.util.match/match (-> mbql-query :query :filter)- [:field (id :guard integer?) _] id)))))+ (into #{} (mapcat lib/all-field-ids) (lib/filters mbql-query)))))diff --git a/src/metabase/parameters/core.clj b/src/metabase/parameters/core.cljnew file mode 100644index 00000000000..9a704621526--- /dev/null+++ b/src/metabase/parameters/core.clj@@ -0,0 +1,15 @@+(ns metabase.parameters.core+ (:require+ [metabase.parameters.schema]+ [potemkin :as p]))++(comment metabase.parameters.schema/keep-me)++(p/import-vars+ [metabase.parameters.schema+ normalize-parameter+ normalize-parameter-mapping+ normalize-parameter-mappings+ normalize-parameters+ transform-parameter-mappings+ transform-parameters])diff --git a/src/metabase/parameters/custom_values.clj b/src/metabase/parameters/custom_values.cljindex a9de105c833..6f503d0dc23 100644--- a/src/metabase/parameters/custom_values.clj+++ b/src/metabase/parameters/custom_values.clj@@ -8,9 +8,11 @@(:require[clojure.string :as str][medley.core :as m]- [metabase.lib-be.metadata.jvm :as lib-be.metadata.jvm]+ [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.lib.types.isa :as lib.types.isa][metabase.models.interface :as mi][metabase.parameters.schema :as parameters.schema]@@ -18,6 +20,7 @@[metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2]))@@ -59,36 +62,49 @@Maybe we should lower it for the sake of displaying a parameter dropdown."1000)-(defn- values-from-card-query- [card value-field-ref {:keys [query-string] :as _opts}]- (let [metadata-provider (lib-be.metadata.jvm/application-database-metadata-provider (:database_id card))- query (lib/query metadata-provider (lib.metadata/card metadata-provider (:id card)))- value-column (lib/find-column-for-legacy-ref query value-field-ref (lib/visible-columns query))- textual? (lib.types.isa/string? value-column)- nonempty ((if textual? lib/not-empty lib/not-null) value-column)- query-filter (when query-string- (if textual?- (lib/contains (lib/lower value-column) (u/lower-case-en query-string))- (lib/= value-column query-string)))]- (-> query- (lib/limit *max-rows*)- (lib/filter nonempty)- (cond-> #_query query-filter (lib/filter query-filter))- (lib/breakout value-column)- ;; TODO(Braden, 07/04/2025): This should probably become a lib helper? I suspect this isn't the only- ;; "internal" query in the BE.- (assoc-in [:middleware :disable-remaps?] true))))+(mu/defn- values-from-card-query :- [:maybe ::lib.schema/query]+ [{query :dataset_query, :keys [id], :as _card} :- [:and+ :metabase.queries.schema/card+ [:map+ [:id ::lib.schema.id/card]]]+ legacy-field-ref :- ::mbql.s/field-or-expression-ref+ {:keys [query-string] :as _opts}]+ (when (seq query)+ ;; start a new query using this Card as a starting point+ (let [query (lib/query query (lib.metadata/card query id))]+ (when-let [visible-columns (or (not-empty (lib/visible-columns query))+ (log/warnf "Cannot get values from Card %d: Card query has no visible columns"+ id))]+ (when-let [value-column (or (lib/find-column-for-legacy-ref query legacy-field-ref visible-columns)+ (log/warnf "Cannot get values from Card %d: failed to find column for legacy ref %s\nFound: %s"+ id+ (pr-str legacy-field-ref)+ (pr-str (map (some-fn :lib/source-column-alias :name) visible-columns))))]+ (let [textual? (lib.types.isa/string? value-column)+ nonempty ((if textual? lib/not-empty lib/not-null) value-column)+ query-filter (when query-string+ (if textual?+ (lib/contains (lib/lower value-column) (u/lower-case-en query-string))+ (lib/= value-column query-string)))]+ (-> query+ (lib/limit *max-rows*)+ (lib/filter nonempty)+ (cond-> #_query query-filter (lib/filter query-filter))+ (lib/breakout value-column)+ ;; TODO(Braden, 07/04/2025): This should probably become a lib helper? I suspect this isn't the only+ ;; "internal" query in the BE.+ (assoc-in [:middleware :disable-remaps?] true))))))))(mu/defn values-from-card"Get distinct values of a field from a card.(values-from-card 1 [:field \"name\" nil] \"red\");; will execute a mbql that looks like- ;; {:source-table (format \"card__%d\" card-id)- ;; :fields [value-field]- ;; :breakout [value-field]- ;; :filter [:contains [:lower value-field] \"red\"]- ;; :limit *max-rows*}+ ;; {:source-card card-id+ ;; :fields [value-field]+ ;; :breakout [value-field]+ ;; :filters [[:contains {} [:lower {} value-field] \"red\"]]+ ;; :limit *max-rows*}=>{:values [[\"Red Medicine\"]]:has_more_values false}@@ -96,23 +112,24 @@([card value-field](values-from-card card value-field nil))- ([card :- (ms/InstanceOf :model/Card)- value-field-ref :- ::parameters.schema/legacy-field-or-expression-reference- opts :- [:maybe :map]]- (let [mbql-query (values-from-card-query card value-field-ref opts)- result (qp/process-query mbql-query)+ ([card :- :metabase.queries.schema/card+ legacy-field-ref :- ::mbql.s/field-or-expression-ref+ opts :- [:maybe :map]]+ (let [mbql-query (values-from-card-query card legacy-field-ref opts)+ result (some-> mbql-query qp/process-query)values (get-in result [:data :rows])]{:values values;; If the row_count returned = the limit we specified, then it's probably has more than that.;; If the query has its own limit smaller than *max-rows*, then there's no more values.:has_more_values (= (:row_count result) *max-rows*)})))-(defn card-values+(mu/defn card-values"Given a param and query returns the values."- [{config :values_source_config :as _param} query]+ [{config :values_source_config :as _param} :- ::parameters.schema/parameter+ query-string :- [:maybe ms/NonBlankString]](let [card-id (:card_id config)card (t2/select-one :model/Card :id card-id)]- (values-from-card card (:value_field config) {:query-string query})))+ (values-from-card card (:value_field config) {:query-string query-string})))(defn- can-get-card-values?[card value-field]@@ -130,19 +147,21 @@`default-case-thunk` is a 0-arity function that returns values list when:- :values_source_type = card but the card is archived or the card no longer contains the value-field.- :values_source_type = nil."- [parameter query default-case-thunk]+ [parameter :- ::parameters.schema/parameter+ query-string :- [:maybe ms/NonBlankString]+ default-case-thunk :- [:=> [:cat :any] ms/FieldValuesResult]](case (:values_source_type parameter)- "static-list" (static-list-values parameter query)- "card" (let [card (t2/select-one :model/Card :id (get-in parameter [:values_source_config :card_id]))]- (when-not (mi/can-read? card)- (throw (ex-info "You don't have permissions to do that." {:status-code 403})))- (if (can-get-card-values? card (get-in parameter [:values_source_config :value_field]))- (card-values parameter query)- (default-case-thunk)))- nil (default-case-thunk)+ :static-list (static-list-values parameter query-string)+ :card (let [card (t2/select-one :model/Card :id (get-in parameter [:values_source_config :card_id]))]+ (when-not (mi/can-read? card)+ (throw (ex-info "You don't have permissions to do that." {:status-code 403})))+ (if (can-get-card-values? card (get-in parameter [:values_source_config :value_field]))+ (card-values parameter query-string)+ (default-case-thunk)))+ nil (default-case-thunk)(throw (ex-info (tru "Invalid parameter source {0}" (:values_source_type parameter)){:status-code 400- :parameter parameter}))))+ :parameter parameter}))))(defn pk-of-fk-pk-field-ids"Check if the collection `field-ids` contains the IDs of FK fields pointing to the same PK and@@ -175,17 +194,19 @@;; more than two groups are always ambiguous, so no matchnil))))))-(defn parameter-remapped-value+(mu/defn parameter-remapped-value"Fetch the remapped value for the given `value` of parameter `param` with default values provided bythe function `default-case-thunk`.`default-case-thunk` is a 0-arity function that returns values list when :values_source_type = nil."- [param value default-case-thunk]+ [param :- ::parameters.schema/parameter+ value+ default-case-thunk :- [:=> [:cat] :any]](case (:values_source_type param)- "static-list" (m/find-first #(and (vector? %) (= (count %) 2) (= (first %) value))- (get-in param [:values_source_config :values]))- "card" nil- nil (default-case-thunk)+ :static-list (m/find-first #(and (vector? %) (= (count %) 2) (= (first %) value))+ (get-in param [:values_source_config :values]))+ :card nil+ nil (default-case-thunk)(throw (ex-info (tru "Invalid parameter source {0}" (:values_source_type param)){:status-code 400- :parameter param}))))+ :parameter param}))))diff --git a/src/metabase/parameters/dashboard.clj b/src/metabase/parameters/dashboard.cljindex 094b9964b21..db825a58c04 100644--- a/src/metabase/parameters/dashboard.clj+++ b/src/metabase/parameters/dashboard.clj@@ -8,6 +8,7 @@[metabase.parameters.chain-filter :as chain-filter][metabase.parameters.custom-values :as custom-values][metabase.parameters.params :as params]+ [metabase.parameters.schema :as parameters.schema][metabase.query-processor.error-type :as qp.error-type][metabase.util :as u][metabase.util.i18n :refer [tru]]@@ -31,7 +32,7 @@{:case-sensitive false}))(mu/defn- param->fields- [param :- ::lib.schema.parameter/parameter]+ [param :- ::parameters.schema/parameter](let [op (param-type->op (:type param))options (or (:options param) (param-type->default-options (:type param)))](for [field-id (params/dashboard-param->field-ids param)]@@ -39,7 +40,7 @@:op op:options options})))-(mu/defn ^:private chain-filter-constraints :- chain-filter/Constraints+(mu/defn ^:private chain-filter-constraints :- ::chain-filter/constraints[dashboard :- :mapconstraint-param-key->value :- [:map-of string? any?]](vec (for [[param-key value] constraint-param-key->value@@ -55,8 +56,8 @@(let [dashboard (t2/hydrate dashboard :resolved-params)param (get-in dashboard [:resolved-params param-key])results (for [{:keys [target] {:keys [card]} :dashcard} (:mappings param)- :let [[_ field-ref opts] (->> (mbql.normalize/normalize-tokens target :ignore-path)- (mbql.u/check-clause :dimension))]+ :let [[_dimension field-ref opts] (->> (mbql.normalize/normalize-tokens target :ignore-path)+ (mbql.u/check-clause :dimension))]:when field-ref](custom-values/values-from-card card field-ref opts))](when-some [values (seq (distinct (mapcat :values results)))]@@ -102,6 +103,7 @@(or (filter-values-from-field-refs dashboard param-key)(throw (ex-info (tru "Parameter {0} does not have any Fields associated with it" (pr-str param-key)){:param (get (:resolved-params dashboard) param-key)+ :param-key param-key:status-code 400})))(try(let [;; results can come back as [[value] ...] *or* as [[value remapped] ...].diff --git a/src/metabase/parameters/field.clj b/src/metabase/parameters/field.cljindex 1677e292065..5a67dd28109 100644--- a/src/metabase/parameters/field.clj+++ b/src/metabase/parameters/field.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][metabase.api.common :as api]+ [metabase.lib.schema.id :as lib.schema.id][metabase.parameters.chain-filter :as chain-filter][metabase.parameters.field-values :as params.field-values][metabase.parameters.field.search-values-query :as search-values-query]@@ -75,12 +76,13 @@(mu/defn search-values-from-field-id :- ms/FieldValuesResult"Search for values of a field given by `field-id` that contain `query`."- [field-id query]+ [field-id :- ::lib.schema.id/field+ query-string :- [:maybe :string]](let [field (api/read-check (t2/select-one :model/Field :id field-id))search-field (or (some->> (chain-filter/remapped-field-id field-id)(t2/select-one :model/Field :id))field)]- {:values (search-values field search-field query)+ {:values (search-values field search-field query-string);; assume there are more if doing a search, otherwise there are no more values- :has_more_values (not (str/blank? query))+ :has_more_values (not (str/blank? query-string)):field_id field-id}))diff --git a/src/metabase/parameters/field/search_values_query.clj b/src/metabase/parameters/field/search_values_query.cljindex c9a883ae720..c14105ccd39 100644--- a/src/metabase/parameters/field/search_values_query.clj+++ b/src/metabase/parameters/field/search_values_query.clj@@ -1,31 +1,49 @@(ns metabase.parameters.field.search-values-query(:require[clojure.string :as str]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.util :as u]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.warehouse-schema.metadata-from-qp :as warehouse-schema.metadata-from-qp]- [metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]))+ [metabase.warehouse-schema.metadata-queries :as schema.metadata-queries]+ [toucan2.core :as t2]))-(defn search-values-query+(mu/defn search-values-query"Generate the MBQL query used to power FieldValues search in [[metabase.parameters.field/search-values]]. The actualquery generated differs slightly based on whether the two Fields are the same Field.Note: the generated MBQL query assume that both `field` and `search-field` are from the same table."- [field search-field value limit]+ [field :- [:or+ (ms/InstanceOf :model/Field)+ ::lib.schema.metadata/column]+ search-field :- [:or+ (ms/InstanceOf :model/Field)+ ::lib.schema.metadata/column]+ value+ limit :- pos-int?](if-let [value->human-readable-value (schema.metadata-queries/human-readable-remapping-map (u/the-id field))]- (let [query (some-> value u/lower-case-en)]+ (let [query-string (some-> value u/lower-case-en)](cond->> value->human-readable-value- value (filter #(str/includes? (-> % val u/lower-case-en) query))+ value (filter #(str/includes? (-> % val u/lower-case-en) query-string))true (sort-by key)))- (-> (warehouse-schema.metadata-from-qp/table-query- (:table_id field)- {:filter (when (some? value)- [:contains [:field (u/the-id search-field) nil] value {:case-sensitive false}])- ;; if both fields are the same then make sure not to refer to it twice in the `:breakout` clause.- ;; Otherwise this will break certain drivers like BigQuery that don't support duplicate- ;; identifiers/aliases- :breakout (if (= (u/the-id field) (u/the-id search-field))- [[:field (u/the-id field) nil]]- [[:field (u/the-id field) nil]- [:field (u/the-id search-field) nil]])- :limit limit})- :data :rows)))+ (let [field (cond-> field+ (t2/model field) (lib-be/instance->metadata :metadata/column))+ search-field (cond-> search-field+ (t2/model search-field) (lib-be/instance->metadata :metadata/column))+ query-xform (fn query-xform [query]+ (-> query+ (cond-> (some? value)+ (lib/filter (-> (lib/contains search-field value)+ (lib/update-options assoc :case-sensitive false))))+ ;; if both fields are the same then make sure not to refer to it twice in the `:breakout` clause.+ ;; Otherwise this will break certain drivers like BigQuery that don't support duplicate+ ;; identifiers/aliases+ (lib/breakout field)+ (cond-> (not= (u/the-id field) (u/the-id search-field))+ (lib/breakout search-field))+ (lib/limit limit)))]+ (-> (warehouse-schema.metadata-from-qp/table-query (:table-id field) query-xform)+ :data :rows))))diff --git a/src/metabase/parameters/params.clj b/src/metabase/parameters/params.cljindex 47465266d2f..abd0e157257 100644--- a/src/metabase/parameters/params.clj+++ b/src/metabase/parameters/params.clj@@ -11,24 +11,25 @@- custom-values: see [metabase.parameters.custom-values]"(:require[clojure.set :as set]+ [malli.error :as me][medley.core :as m][metabase.app-db.core :as app-db][metabase.legacy-mbql.normalize :as mbql.normalize][metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.lib.util.match :as lib.util.match]- [metabase.models.interface :as mi][metabase.parameters.schema :as parameters.schema][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.malli.schema :as ms]+ [methodical.core :as methodical][toucan2.core :as t2]));;; +----------------------------------------------------------------------------------------------------------------+@@ -38,11 +39,12 @@(defn assert-valid-parameters"Receive a Paremeterized Object and check if its parameters is valid."[{:keys [parameters]}]- (let [schema [:maybe [:sequential ::parameters.schema/parameter]]]- (when-not (mr/validate schema parameters)- (throw (ex-info ":parameters must be a sequence of maps with :id and :type keys"+ (let [schema [:maybe ::parameters.schema/parameters]]+ (when-let [error (mr/explain schema parameters)]+ (throw (ex-info (str ":parameters must be a sequence of maps with :id and :type keys; "+ (pr-str (me/humanize error))){:parameters parameters- :errors (:errors (mr/explain schema parameters))})))))+ :errors (:errors error)})))))(defn assert-valid-parameter-mappings"Receive a Paremeterized Object and check if its parameters is valid."@@ -60,31 +62,52 @@checks (since there is no current User) and get *all* values."false)-(defn- template-tag->field-form+(mu/defn- template-tag->field-ref :- [:maybe :mbql.clause/field]"Fetch the `:field` clause from `dashcard` referenced by `template-tag`.(template-tag->field-form [:template-tag :company] some-dashcard) ; -> [:field 100 nil]"- [[_ tag] card]- (get-in card [:dataset_query :native :template-tags (u/qualified-name tag) :dimension]))--(mu/defn param-target->field-clause :- [:maybe mbql.s/Field]+ [[_template-tag tag] :- ::lib.schema.parameter/template-tag+ card :- :metabase.queries.schema/card]+ (some-> card+ :dataset_query+ not-empty+ lib-be/normalize-query+ lib/all-template-tags-map+ (get (u/qualified-name tag))+ :dimension))++(mu/defn param-target->field-id :- [:maybe ::lib.schema.id/field]"Parse a Card parameter `target` form, which looks something like `[:dimension [:field-id 100]]`, and return the FieldID it references (if any)."- [target card]+ [target :- ::lib.schema.parameter/target+ ;; TODO (Cam 9/25/25) -- `card` should actually be required but I don't have all day to fix broken tests from+ ;; before I schematized this.+ card :- [:maybe :metabase.queries.schema/card]](let [target (mbql.normalize/normalize target)](when (mbql.u/is-clause? :dimension target)+ ;; MBQL 5 `:parameters` still use legacy field refs (for now), while `:template-tags` use MBQL 5 field refs. So+ ;; we have to handle either one depending on where the `:field` ref actually lives.(let [[_ dimension] target- field-form (if (mbql.u/is-clause? :template-tag dimension)- (template-tag->field-form dimension card)- dimension)]- ;; Being extra safe here since we've got many reports on this cause loading dashboard to fail- ;; for unknown reasons. See #8917- (if field-form- (try- (mbql.u/unwrap-field-or-expression-clause field-form)- (catch Exception e- (log/error e "Failed unwrap field form" (pr-str field-form))))- (log/error "Could not find matching field clause for target:" target))))))+ field-ref (cond+ (mbql.u/is-clause? :template-tag dimension)+ (when card+ (template-tag->field-ref dimension card))++ (mbql.u/is-clause? :field dimension)+ dimension)]+ (cond+ (not field-ref)+ (log/errorf "Could not find matching field clause for target: %s" (pr-str target))++ ;; Being extra safe here since we've got many reports on this cause loading dashboard to fail+ ;; for unknown reasons. See #8917+ (not (mr/validate [:or :mbql.clause/field ::mbql.s/field] field-ref))+ (log/errorf "Template tag :dimension is not a valid MBQL 5 or legacy field ref, got %s" (pr-str field-ref))++ :else+ (lib.util.match/match-one field-ref+ [:field _opts (id :guard pos-int?)] id+ [:field (id :guard pos-int?) _opts] id))))))(defn- pk-fields"Return the `fields` that are PK Fields."@@ -113,13 +136,18 @@;; `:list` instead of `:auto-list`.)(t2/hydrate :has_field_values)))))-(mi/define-batched-hydration-method add-name-field- :name_field+(methodical/defmethod t2/simple-hydrate [nil :name_field]+ "Not really 100% sure why this is even needed but when we do recursive hydration of `[:target :name_field]` it tries+ to hydrate for `nil`... I guess we have to explicitly tell it to do nothing."+ [_model _k _nil]+ nil)++(methodical/defmethod t2/batched-hydrate [:model/Field :name_field]"For all `fields` that are `:type/PK` Fields, look for a `:type/Name` Field belonging to the same Table. For eachField, if a matching name Field exists, add it under the `:name_field` key. This is so the Fields can be used inpublic/embedded field values search widgets. This only includes the information needed to power those widgets, andno more."- [fields]+ [_model _k fields](let [table-id->name-field (fields->table-id->name-field (pk-fields fields))](for [field fields];; add matching `:name_field` if it's a PK@@ -149,7 +177,7 @@"Get the Fields (as a map of Parameter ID -> Fields) that should be returned for hydrated `:param_fields` for a Cardor Dashboard. These only contain the minimal amount of information necessary needed to power public or embeddedparameter widgets."- [param-id->field-ids :- [:map-of ms/NonBlankString [:set ::lib.schema.id/field]]]+ [param-id->field-ids :- [:maybe [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]]](let [field-ids (into #{} cat (vals param-id->field-ids))field-id->field (when (seq field-ids)(m/index-by :id (-> (t2/select Field:params-columns-only :id [:in field-ids])@@ -159,19 +187,6 @@(->> param-id->field-ids(m/map-vals #(into [] (keep field-id->field) %)))))-(defmulti ^:private ^{:hydrate :param_fields} param-fields- "Add a `:param_fields` map (Field ID -> Field) for all of the Fields referenced by the parameters of a Card or- Dashboard. Implementations are below in respective sections."- {:arglists '([instance])}- t2/model)--#_{:clj-kondo/ignore [:unused-private-var]}-(mi/define-simple-hydration-method ^:private hydrate-param-fields- :param_fields- "Hydration method for `:param_fields`."- [instance]- (param-fields instance))-;;; +----------------------------------------------------------------------------------------------------------------+;;; | DASHBOARD-SPECIFIC |;;; +----------------------------------------------------------------------------------------------------------------+@@ -179,7 +194,7 @@(defn- filterable-columns-for-query"Get filterable columns for query."[database-id card stage-number]- (let [metadata-provider (lib.metadata.jvm/application-database-metadata-provider database-id)+ (let [metadata-provider (lib-be/application-database-metadata-provider database-id);; Regular questions are used directly. If a model or metric has been used directly in this card, wrap it into;; a query against that model or metric.query (lib/query metadata-provider (if (= :question (:type card))@@ -215,13 +230,14 @@filterable-columns (get-in ctx [:card-id->filterable-columns card-id stage-number])[_ dimension] (->> (mbql.normalize/normalize-tokens param-target :ignore-path)(mbql.u/check-clause :dimension))]- (if-some [field-id (lib.util.match/match-one dimension- ;; TODO it's basically a workaround for ignoring non-dimension parameter targets such as SQL variables- ;; TODO code is misleading; let's check for :dimension and drop the match call here- [:field (field-name :guard string?) _]- (->> filterable-columns- (lib/find-matching-column (lib/->pMBQL dimension))- :id))]+ (if-some [field-id (when (seq filterable-columns)+ (lib.util.match/match-one dimension+ ;; TODO it's basically a workaround for ignoring non-dimension parameter targets such as SQL variables+ ;; TODO code is misleading; let's check for :dimension and drop the match call here+ [:field (field-name :guard string?) _]+ (->> filterable-columns+ (lib/find-matching-column (lib/->pMBQL &match))+ :id)))](-> ctx(update :param-id->field-ids #(merge {param-id #{}} %))(update-in [:param-id->field-ids param-id] conj field-id))@@ -259,50 +275,50 @@(swap! *field-id-context* update :card-id->filterable-columnsmerge (:card-id->filterable-columns ctx)))(:param-id->field-ids ctx))- ([ctx {:keys [param-mapping param-target-field] :as param-dashcard-info}]- (if-not param-target-field- ctx- (let [card-id (:card_id param-mapping)- card (if card-id- (m/find-first #(= (:id %) card-id)- (cons (get-in param-dashcard-info [:dashcard :card])- (get-in param-dashcard-info [:dashcard :series])))- (get-in param-dashcard-info [:dashcard :card]))- param-id (:parameter_id param-mapping)- stage-number (get-in param-mapping [:target 2 :stage-number] -1)]- ;; Get the field id from the field-clause if it contains it. This is the common case- ;; for mbql queries.- (if-some [field-id (lib.util.match/match-one param-target-field [:field (id :guard integer?) _] id)]- (update-in ctx [:param-id->field-ids param-id] (fnil conj #{}) field-id)- ;; In case the card doesn't have the same result_metadata columns as filterable columns (a question that- ;; aggregates a native query model with a field that was mapped to a db field), we need to load metadata in- ;; [[ensure-filterable-columns-for-card]] to find the originating field. (#42829)- (-> ctx- (ensure-filterable-columns-for-card card stage-number)- (field-id-from-dashcards-filterable-columns param-dashcard-info stage-number)))))))--(mu/defn dashcards->param-id->field-ids* :- [:map-of ms/NonBlankString [:set ::lib.schema.id/field]]+ ([ctx {:keys [param-mapping param-target-field-id] :as param-dashcard-info}]+ (let [card-id (:card_id param-mapping)+ card (if card-id+ (m/find-first #(= (:id %) card-id)+ (cons (get-in param-dashcard-info [:dashcard :card])+ (get-in param-dashcard-info [:dashcard :series])))+ (get-in param-dashcard-info [:dashcard :card]))+ param-id (:parameter_id param-mapping)+ stage-number (get-in param-mapping [:target 2 :stage-number] -1)]+ ;; Get the field id from the field-clause if it contains it. This is the common case+ ;; for mbql queries.+ (if param-target-field-id+ (update-in ctx [:param-id->field-ids param-id] (fnil conj #{}) param-target-field-id)+ ;; In case the card doesn't have the same result_metadata columns as filterable columns (a question that+ ;; aggregates a native query model with a field that was mapped to a db field), we need to load metadata in+ ;; [[ensure-filterable-columns-for-card]] to find the originating field. (#42829)+ (-> ctx+ (ensure-filterable-columns-for-card card stage-number)+ (field-id-from-dashcards-filterable-columns param-dashcard-info stage-number))))))++(mu/defn dashcards->param-id->field-ids* :- [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]"Return map of parameter ids to mapped field ids."[dashcards](letfn [(dashcard->param-dashcard-info [dashcard](for [mapping (:parameter_mappings dashcard)]- {:dashcard dashcard- :param-mapping mapping- :param-target-field (param-target->field-clause (:target mapping) (:card dashcard))}))]+ {:dashcard dashcard+ :param-mapping mapping+ :param-target-field-id (when (:target mapping)+ (param-target->field-id (:target mapping) (:card dashcard)))}))](transduce (mapcat dashcard->param-dashcard-info)field-id-into-context-rfdashcards)))-(declare card->template-tag-param-id->field-ids)+(declare card->template-tag-id->field-ids)-(mu/defn- dashcards->param-id->field-ids :- [:map-of ms/NonBlankString [:set ::lib.schema.id/field]]+(mu/defn- dashcards->param-id->field-ids :- [:map-of ::lib.schema.parameter/id [:set ::lib.schema.id/field]]"Return a map of Parameter ID to the set of Field IDs referenced by parameters in the Cards on the given `dashcards`,or `nil` if none are referenced. `dashcards` must be hydrated with :card."[dashcards]- (transduce (map card->template-tag-param-id->field-ids)- (completing #(merge-with set/union %1 %2))+ (transduce (comp (map :card)+ (map card->template-tag-id->field-ids))+ (partial merge-with set/union)(dashcards->param-id->field-ids* dashcards)- (map :card dashcards)))+ dashcards))(mu/defn dashcards->param-field-ids :- [:set ::lib.schema.id/field]"Return a set of Field IDs referenced by parameters in Cards in the given `dashcards`, or `nil` if@@ -314,11 +330,11 @@"Return field ids mapped to the parameter. `dashcard` and `card` must be present for each mapping."[{:keys [mappings]} :- ::parameters.schema/parameter](let [param-id->field-ids (transduce (map (fn [mapping]- {:dashcard (:dashcard mapping)- :param-mapping mapping- :param-target-field (param-target->field-clause- (:target mapping)- (get-in mapping [:dashcard :card]))}))+ {:dashcard (:dashcard mapping)+ :param-mapping mapping+ :param-target-field-id (param-target->field-id+ (:target mapping)+ (get-in mapping [:dashcard :card]))}))field-id-into-context-rfmappings)](into #{} cat (vals param-id->field-ids))))@@ -326,62 +342,53 @@(defn get-linked-field-ids"Retrieve a map relating paramater ids to field ids."[dashcards]- (letfn [(targets [params card]+ (letfn [(targets [{params :parameter_mappings card :card, :as _dashcard}](into {}(for [param params- :let [clause (param-target->field-clause (:target param)- card)- ids (lib.util.match/match clause- [:field (id :guard integer?) _]- id)]- :when (seq ids)]- [(:parameter_id param) (set ids)])))]+ :let [target (:target param)]+ :when target+ :let [id (param-target->field-id target card)]+ :when id]+ [(:parameter_id param) #{id}])))](->> dashcards- (mapv (fn [{params :parameter_mappings card :card}] (targets params card)))+ (map targets)(apply merge-with into {}))))-(defmethod param-fields :model/Dashboard [dashboard]- (-> (t2/hydrate dashboard [:dashcards :card])- :dashcards- dashcards->param-id->field-ids- param-field-ids->fields))+(methodical/defmethod t2/batched-hydrate [:model/Dashboard :param_fields]+ "Add a `:param_fields` map (Field ID -> Field) for all of the Fields referenced by the parameters of a Dashboard."+ [_model k dashboards]+ (mapv (fn [dashboard]+ (let [param-fields (-> dashboard+ :dashcards+ dashcards->param-id->field-ids+ param-field-ids->fields)]+ (assoc dashboard k param-fields)))+ (t2/hydrate dashboards [:dashcards :card])));;; +----------------------------------------------------------------------------------------------------------------+;;; | CARD-SPECIFIC |;;; +----------------------------------------------------------------------------------------------------------------+-(mu/defn- card->template-tag-param-id->field-clauses :- [:map-of- ::lib.schema.common/non-blank-string- [:set mbql.s/field]]- "Return a set of `:field` clauses referenced in template tag parameters in `card`."- [card]- (into {} (for [[_ {param-id :id- dimension :dimension}] (get-in card [:dataset_query :native :template-tags])- :when (and dimension param-id)- :let [field (mbql.u/unwrap-field-clause dimension)]- :when field]- [param-id #{field}])))--(mu/defn- card->template-tag-param-id->field-ids :- [:map-of- ::lib.schema.common/non-blank-string- [:set ::lib.schema.id/field]]+(mu/defn- card->template-tag-id->field-ids :- [:maybe [:map-of+ ::lib.schema.template-tag/id+ [:set ::lib.schema.id/field]]]"Return a map of Param IDs to sets of Field IDs referenced by each template tag parameter in this `card`.Mostly used for determining Fields referenced by Cards for purposes other than processing queries. Filters out`:field` clauses which use names."- [card]- (-> card- card->template-tag-param-id->field-clauses- (update-vals #(set (lib.util.match/match (seq %)- [:field (id :guard integer?) _]- id)))))+ [card :- [:maybe :map]]+ (some-> card :dataset_query not-empty lib-be/normalize-query lib/all-template-tags-id->field-ids))-(defmethod param-fields :model/Card [card]- (-> card card->template-tag-param-id->field-ids param-field-ids->fields))+(methodical/defmethod t2/simple-hydrate [:model/Card :param_fields]+ "Add a `:param_fields` map (Field ID -> Field) for all of the Fields referenced by the parameters of a Card."+ [_model k card]+ (let [param-fields (or (some-> card card->template-tag-id->field-ids param-field-ids->fields)+ {})]+ (assoc card k param-fields)))-(mu/defn card->template-tag-field-ids :- [:maybe [:set ::lib.schema.id/field]]+(mu/defn card->template-tag-field-ids :- [:maybe [:set {:min 1} ::lib.schema.id/field]]"Returns a set of all Field IDs referenced by template tags on this card.To get these IDs broken out by the Param ID that references them, use [[card->template-tag-param-id->field-ids]]."- [card]- (not-empty (into #{} cat (vals (card->template-tag-param-id->field-ids card)))))+ [card :- [:maybe :map]]+ (some-> card :dataset_query not-empty lib-be/normalize-query lib/all-template-tag-field-ids not-empty))diff --git a/src/metabase/parameters/schema.clj b/src/metabase/parameters/schema.cljdeleted file mode 100644index aef046ec275..00000000000--- a/src/metabase/parameters/schema.clj+++ /dev/null@@ -1,78 +0,0 @@-(ns metabase.parameters.schema- (:require- [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.legacy-mbql.schema :as mbql.s]- [metabase.lib.schema.common :as lib.schema.common]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]- [metabase.util.malli.registry :as mr]))--(mr/def ::human-readable-remapping-map- "Schema for the map of actual value -> human-readable value. Cannot be empty."- [:map-of {:min 1} :any [:maybe :string]])--(mr/def ::legacy-field-or-expression-reference- "Schema for a valid legacy `:field` or `:expression` reference (possibly not yet normalized)."- [:fn- (fn [k]- ((comp (mr/validator mbql.s/Field)- mbql.normalize/normalize-tokens) k))])--(mr/def ::values-source-config- "Schema for valid source_options within a Parameter"- ;; TODO: This should be tighter- [:map- [:values {:optional true} [:* :any]]- [:card_id {:optional true} ::lib.schema.id/card]- [:value_field {:optional true} ::legacy-field-or-expression-reference]- [:label_field {:optional true} ::legacy-field-or-expression-reference]])--#_(def ParameterSource- (mc/schema- [:multi {:dispatch :values_source_type}- ["card" [:map- [:values_source_type :string]- [:values_source_config- [:map {:closed true}- [:card_id {:optional true} IntGreaterThanZero]- [:value_field {:optional true} Field]- [:label_field {:optional true} Field]]]]]- ["static-list" [:map- [:values_source_type :string]- [:values_source_config- [:map {:closed true}- [:values {:optional true} [:* :any]]]]]]]))--(mr/def ::keyword-or-non-blank-string- [:or- {:json-schema {:type "string" :minLength 1}}- :keyword- ::lib.schema.common/non-blank-string])--(mr/def ::parameter- "Schema for a valid Parameter. We're not using [[metabase.legacy-mbql.schema/Parameter]] here because this Parameter- is meant to be used for Parameters we store on dashboard/card, and it has some difference with Parameter in MBQL."- ;; TODO we could use :multi to dispatch values_source_type to the correct values_source_config- [:map- {:description "parameter must be a map with :id and :type keys"}- [:id ::lib.schema.common/non-blank-string]- [:type ::keyword-or-non-blank-string]- ;; TODO how to merge this with ParameterSource above?- [:values_source_type {:optional true} [:enum "static-list" "card" nil]]- [:values_source_config {:optional true} ::values-source-config]- [:slug {:optional true} :string]- [:name {:optional true} :string]- [:default {:optional true} :any]- [:sectionId {:optional true} ::lib.schema.common/non-blank-string]- [:temporal_units {:optional true} [:sequential ::lib.schema.temporal-bucketing/unit]]- ;; TODO FIXME -- I've seen this key used in [[metabase.parameters.params/dashboard-param->field-ids]] but no idea- ;; what the expected shape is supposed to be. Please fixx- [:mappings {:optional true} :any]])--(mr/def ::parameter-mapping- "Schema for a valid Parameter Mapping"- [:map- {:description "parameter_mapping must be a map with :parameter_id and :target keys"}- [:parameter_id ::lib.schema.common/non-blank-string]- [:target :any]- [:card_id {:optional true} ::lib.schema.id/card]])diff --git a/src/metabase/parameters/schema.cljc b/src/metabase/parameters/schema.cljcnew file mode 100644index 00000000000..12115327435--- /dev/null+++ b/src/metabase/parameters/schema.cljc@@ -0,0 +1,147 @@+(ns metabase.parameters.schema+ (:require+ #?@(:clj+ ([metabase.models.interface :as mi]))+ [metabase.lib.core :as lib]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter]+ [metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++(mr/def ::human-readable-remapping-map+ "Schema for the map of actual value -> human-readable value. Cannot be empty."+ [:map-of {:min 1} :any [:maybe :string]])++(mr/def ::legacy-ref+ [:multi {:dispatch (fn [x]+ (if (and (sequential? x)+ (#{:expression "expression"} (first x)))+ :expression+ :field))}+ [:expression ::lib.schema.parameter/target.legacy-expression-ref]+ [:field ::lib.schema.parameter/target.legacy-field-ref]])++(defn- normalize-values-source-config [m]+ (when (map? m)+ ;; remove empty/nil values+ (reduce-kv+ (fn [m k v]+ (cond-> m+ (or (nil? v)+ (and (sequential? v)+ (empty? v)))+ (dissoc k)))+ m+ m)))++(mr/def ::values-source-config+ "Schema for valid source_options within a Parameter"+ ;; TODO: This should be tighter+ [:map+ {:decode/normalize normalize-values-source-config}+ [:values {:optional true} [:* :any]]+ [:card_id {:optional true} ::lib.schema.id/card]+ [:value_field {:optional true} [:ref ::legacy-ref]]+ [:label_field {:optional true} [:ref ::legacy-ref]]])++#_(def ParameterSource+ (mc/schema+ [:multi {:dispatch :values_source_type}+ ["card" [:map+ [:values_source_type :string]+ [:values_source_config+ [:map {:closed true}+ [:card_id {:optional true} IntGreaterThanZero]+ [:value_field {:optional true} Field]+ [:label_field {:optional true} Field]]]]]+ ["static-list" [:map+ [:values_source_type :string]+ [:values_source_config+ [:map {:closed true}+ [:values {:optional true} [:* :any]]]]]]]))++(mr/def ::values-source-type+ [:enum {:decode/normalize lib.schema.common/normalize-keyword} :static-list :card])++(mr/def ::values-query-type+ [:enum {:decode/normalize lib.schema.common/normalize-keyword} :none :list :search])++(mr/def ::parameter+ "Schema for a valid Parameter. We're not using [[metabase.legacy-mbql.schema/Parameter]] here because this Parameter+ is meant to be used for Parameters we store on dashboard/card, and it has some difference with Parameter in MBQL."+ ;; TODO we could use :multi to dispatch values_source_type to the correct values_source_config+ [:map+ {:description "parameter must be a map with :id and :type keys"+ :decode/normalize lib.schema.common/normalize-map-no-kebab-case}+ [:default {:optional true} :any]+ ;; TODO (Cam 9/18/25) -- why are we mixing `camelCase` and `snake_case` here? Is this to make me sad?+ [:filteringParameters {:optional true} [:maybe [:sequential ::lib.schema.parameter/id]]]+ [:id ::lib.schema.parameter/id]+ [:mappings {:optional true} [:maybe [:or+ [:sequential [:ref ::parameter-mapping]]+ [:set [:ref ::parameter-mapping]]]]]+ [:name {:optional true} :string]+ ;; ok now I know you're trying to mess with me+ [:sectionId {:optional true} ::lib.schema.common/non-blank-string]+ [:slug {:optional true} :string]+ [:target {:optional true} [:ref ::lib.schema.parameter/target]]+ [:temporal_units {:optional true} [:maybe [:sequential ::lib.schema.temporal-bucketing/unit]]]+ [:type [:ref ::lib.schema.parameter/type]]+ [:values_query_type {:optional true} [:maybe ::values-query-type]]+ [:values_source_config {:optional true} [:maybe ::values-source-config]]+ [:values_source_type {:optional true} [:maybe ::values-source-type]]])++(mu/defn normalize-parameter :- ::parameter+ "Normalize `parameter` when coming out of the application database or in via an API request."+ [parameter]+ (lib/normalize ::parameter parameter))++(mr/def ::parameters+ [:sequential [:ref ::parameter]])++(mu/defn normalize-parameters :- ::parameters+ "Normalize `parameters` when coming out of the application database or in via an API request."+ [parameters]+ (lib/normalize ::parameters parameters))++#?(:clj+ (def transform-parameters+ "Toucan 2 transform for columns that are sequences of Card/Dashboard parameters."+ {:in (comp mi/json-in normalize-parameters)+ :out (comp (mi/catch-normalization-exceptions normalize-parameters) mi/json-out-with-keywordization)}))++(mr/def ::parameter-mapping+ "Schema for a valid Parameter Mapping"+ [:map+ {:decode/normalize (fn [mapping]+ (when (map? mapping)+ (let [mapping (lib.schema.common/normalize-map-no-kebab-case mapping)]+ (cond-> mapping+ (not (pos-int? (:card_id mapping))) (dissoc :card_id)))))+ :description "parameter_mapping must be a map with :parameter_id and :target keys"}+ [:parameter_id ::lib.schema.parameter/id]+ [:target ::lib.schema.parameter/target]+ [:card_id {:optional true} [:maybe ::lib.schema.id/card]]+ [:dashcard {:optional true} :map]])++(mu/defn normalize-parameter-mapping :- ::parameter-mapping+ "Normalize `parameter-mappings` when coming out of the application database or in via an API request."+ [parameter-mapping]+ (lib/normalize ::parameter-mapping parameter-mapping))++(mr/def ::parameter-mappings+ [:sequential [:ref ::parameter-mapping]])++(mu/defn normalize-parameter-mappings :- [:maybe ::parameter-mappings]+ "Normalize `parameter-mappings` when coming out of the application database or in via an API request."+ [parameter-mappings :- [:maybe [:sequential :map]]]+ (when parameter-mappings+ (lib/normalize ::parameter-mappings parameter-mappings)))++#?(:clj+ (def transform-parameter-mappings+ "Toucan 2 transform for columns that are sequences of Card/Dashboard parameter mappings."+ {:in (comp mi/json-in normalize-parameter-mappings)+ :out (comp (mi/catch-normalization-exceptions normalize-parameter-mappings) mi/json-out-with-keywordization)}))diff --git a/src/metabase/parameters/shared.cljc b/src/metabase/parameters/shared.cljcindex 769c93fa48a..7c2a0abd4ae 100644--- a/src/metabase/parameters/shared.cljc+++ b/src/metabase/parameters/shared.cljc@@ -9,10 +9,11 @@#?@(:cljs(["moment" :as moment]))[clojure.string :as str]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.core :as lib]+ [metabase.parameters.schema :as parameters.schema][metabase.util :as u][metabase.util.i18n :refer [trs]]+ [metabase.util.log :as log][metabase.util.time :as time])(:import#?@(:clj@@ -303,14 +304,6 @@#?(:clj tag-names:cljs (clj->js tag-names))))-(defn- normalize-parameter- "Normalize a single parameter by calling [[mbql.normalize/normalize-fragment]] on it, and converting all string keys- to keywords."- [parameter]- (-> (mbql.normalize/normalize-fragment [:parameters] [parameter])- first- (update-keys keyword)))-(defn ^:export substitute-tags"Given the context of a text dashboard card, replace all template tags in the text with their corresponding values,formatted and escaped appropriately if escape-markdown is true. Specifically escape-markdown should be false when the@@ -321,7 +314,10 @@(when text(let [tag->param #?(:clj tag->param:cljs (js->clj tag->param))- tag->normalized-param (update-vals tag->param normalize-parameter)]+ tag->normalized-param (try+ (update-vals tag->param parameters.schema/normalize-parameter)+ (catch #?(:clj Throwable :cljs :default) e+ (log/warnf "Unable to substitute tags: invalid parameters: %s" (ex-message e))))];; Most of the functions in this pipeline are relating to handling optional blocks in the text which use;; the [[ ]] syntax.;; For example, given an input "[[a {{b}}]] [[{{c}}]]", where `b` has no value and `c` = 3:diff --git a/src/metabase/public_sharing/api.clj b/src/metabase/public_sharing/api.cljindex 415826f61f0..bb8c4feca40 100644--- a/src/metabase/public_sharing/api.clj+++ b/src/metabase/public_sharing/api.clj@@ -8,8 +8,8 @@[metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.dashboards.api :as api.dashboard]+ [metabase.dashboards.schema :as dashboards.schema][metabase.events.core :as events]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info][metabase.models.interface :as mi]@@ -68,8 +68,9 @@card(mi/instance:model/Card- (u/select-nested-keys card [:id :name :description :display :visualization_settings :parameters :entity_id- [:dataset_query :type [:native :template-tags]]]))))+ (-> card+ (select-keys [:id :name :description :display :visualization_settings :parameters :entity_id :dataset_query])+ (update :dataset_query select-keys [:lib/metadata :lib/type :database :stages])))))(defn public-card"Return a public Card matching key-value `conditions`, removing all columns that should not be visible to the general@@ -222,7 +223,7 @@(m/remove-keys hidden-parameter-ids fields)))(select-keys action-public-keys))))-(defn public-dashboard+(mu/defn public-dashboard :- ::dashboards.schema/dashboard"Return a public Dashboard matching key-value `conditions`, removing all columns that should not be visible to thegeneral public. Throws a 404 if the Dashboard doesn't exist."[& conditions]@@ -248,10 +249,9 @@"Fetch a publicly-accessible Dashboard. Does not require auth credentials. Public sharing must be enabled."[{:keys [uuid]} :- [:map[:uuid ms/UUIDString]]]- (lib.metadata.jvm/with-metadata-provider-cache- (public-sharing.validation/check-public-sharing-enabled)- (u/prog1 (dashboard-with-uuid uuid)- (events/publish-event! :event/dashboard-read {:object-id (:id <>), :user-id api/*current-user-id*}))))+ (public-sharing.validation/check-public-sharing-enabled)+ (u/prog1 (dashboard-with-uuid uuid)+ (events/publish-event! :event/dashboard-read {:object-id (:id <>), :user-id api/*current-user-id*})))(defn process-query-for-dashcard"Return the results of running a query for Card with `card-id` belonging to Dashboard with `dashboard-id` via@@ -463,11 +463,10 @@[:uuid ms/UUIDString][:param-key ms/NonBlankString]]constraint-param-key->value :- [:map-of string? any?]]- (lib.metadata.jvm/with-metadata-provider-cache- (let [dashboard (dashboard-with-uuid uuid)]- (request/as-admin- (binding [qp.perms/*param-values-query* true]- (parameters.dashboard/param-values dashboard param-key constraint-param-key->value))))))+ (let [dashboard (dashboard-with-uuid uuid)]+ (request/as-admin+ (binding [qp.perms/*param-values-query* true]+ (parameters.dashboard/param-values dashboard param-key constraint-param-key->value)))))(api.macros/defendpoint :get "/dashboard/:uuid/params/:param-key/search/:query""Fetch filter values for dashboard parameter `param-key`, containing specified `query`."diff --git a/src/metabase/pulse/core.clj b/src/metabase/pulse/core.cljindex fd2d02d77f2..a6acbc28fae 100644--- a/src/metabase/pulse/core.clj+++ b/src/metabase/pulse/core.clj@@ -1,4 +1,4 @@-(ns ^:deprecated metabase.pulse.core+(ns metabase.pulse.core"API namespace for the `metabase.pulse` module.This namespace is deprecated, soon everything will be migrated to notifications."diff --git a/src/metabase/pulse/update_alerts.clj b/src/metabase/pulse/update_alerts.cljindex 410c7a6db50..d6f90e7ee9b 100644--- a/src/metabase/pulse/update_alerts.clj+++ b/src/metabase/pulse/update_alerts.clj@@ -2,7 +2,9 @@;; TODO this should be moved to notification(:require[metabase.events.core :as events]+ [metabase.lib.core :as lib][metabase.notification.models :as models.notification]+ [metabase.util.malli :as mu][toucan2.core :as t2]))(defn- card-archived? [old-card new-card]@@ -43,14 +45,14 @@(get-in old-card [:visualization_settings :graph.goal_value])(not (get-in new-card [:visualization_settings :graph.goal_value]))))-(defn- multiple-breakouts?+(mu/defn- multiple-breakouts?"If there are multiple breakouts and a goal, we don't know which breakout to compare to the goal, so it invalidatesthe alert"- [{:keys [display] :as new-card}]+ [{:keys [display], query :dataset_query, :as new-card} :- :metabase.queries.schema/card](and (get-in new-card [:visualization_settings :graph.goal_value])(or (line-area-bar? display)(progress? display))- (< 1 (count (get-in new-card [:dataset_query :query :breakout])))))+ (< 1 (count (lib/breakouts query)))))(defn delete-alert-and-notify!"Removes all of the alerts and notifies all of the email recipients of the alerts change."diff --git a/src/metabase/queries/api/card.clj b/src/metabase/queries/api/card.cljindex 7526b0e2978..63d08fb5b30 100644--- a/src/metabase/queries/api/card.clj+++ b/src/metabase/queries/api/card.clj@@ -12,6 +12,8 @@[metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.types.isa :as lib.types.isa][metabase.models.interface :as mi][metabase.parameters.schema :as parameters.schema]@@ -21,7 +23,6 @@[metabase.queries.metadata :as queries.metadata][metabase.queries.models.card :as card][metabase.queries.models.card.metadata :as card.metadata]- [metabase.queries.models.query :as query][metabase.queries.schema :as queries.schema][metabase.query-permissions.core :as query-perms][metabase.query-processor.api :as api.dataset]@@ -99,22 +100,20 @@:where [:and [:= :m.id model-id] [:not :c.archived]]:order-by [[[:lower :c.name] :asc]]});; now check if model-id really occurs as a card ID- ;;- ;; existing usage -- do not use in new code- #_{:clj-kondo/ignore [:deprecated-var]}- (filter (fn [card] (some #{model-id} (-> card :dataset_query query/collect-card-ids))))))--(defn- cards-for-segment-or-metric- [model-type model-id]- (lib-be/with-metadata-provider-cache- (->> (t2/select :model/Card (merge order-by-name- {:where [:like :dataset_query (str "%" (name model-type) "%" model-id "%")]}))- ;; now check if the segment/metric with model-id really occurs in a filter/aggregation expression- (filter (fn [card]- (when-let [query (some-> card :dataset_query not-empty lib-be/normalize-query)]- (case model-type- :segment (lib/uses-segment? query model-id)- :metric (lib/uses-metric? query model-id))))))))+ (filter (fn [card]+ (some-> card :dataset_query not-empty lib/all-source-card-ids (contains? model-id))))))++(mu/defn- cards-for-segment-or-metric+ [model-type :- [:enum :segment :metric]+ model-id :- pos-int?]+ (->> (t2/select :model/Card (merge order-by-name+ {:where [:like :dataset_query (str "%" (name model-type) "%" model-id "%")]}))+ ;; now check if the segment/metric with model-id really occurs in a filter/aggregation expression+ (filter (fn [{query :dataset_query, :as _card}]+ (when (seq query)+ (case model-type+ :segment (lib/uses-segment? query model-id)+ :metric (lib/uses-metric? query model-id)))))))(defmethod cards-for-filter-option* :using_segment[_filter-option model-id]@@ -222,20 +221,32 @@(api/present-in-trash-if-archived-directly (collection/trash-collection-id)))))(api.macros/defendpoint :get "/:id"- "Get `Card` with ID."+ "Get `Card` with ID.++ As of v57, returns the MBQL query (`dataset_query`) as MBQL 5; to return the query as MBQL 4 (aka legacy MBQL)+ instead, you can specify `?legacy-mbql=true`."[{:keys [id]} :- [:map[:id [:or ms/PositiveInt ms/NanoIdString]]]- {ignore-view? :ignore_view, :keys [context]} :- [:map- [:ignore_view {:optional true} [:maybe :boolean]]- [:context {:optional true} [:maybe [:enum :collection]]]]]+ {ignore-view? :ignore_view+ legacy-mbql? :legacy-mbql+ :keys [context]} :- [:map+ [:ignore_view {:optional true} [:maybe :boolean]]+ [:context {:optional true} [:maybe [:enum :collection]]]+ [:legacy-mbql {:optional true, :default false} [:maybe :boolean]]]](let [resolved-id (eid-translation/->id-or-404 :card id)- card (get-card resolved-id)]- (u/prog1 card+ card (get-card resolved-id)+ response (cond-> card+ legacy-mbql?+ (update :dataset_query (fn [query]+ #_{:clj-kondo/ignore [:discouraged-var]}+ (cond-> query+ (seq query) lib/->legacy-MBQL))))]+ (u/prog1 response(when-not ignore-view?(events/publish-event! :event/card-read{:object-id (:id <>)- :user-id api/*current-user-id*- :context (or context :question)})))))+ :user-id api/*current-user-id*+ :context (or context :question)})))))(defn- check-allowed-to-remove-from-existing-dashboards [card](let [dashboards (or (:in_dashboards card)@@ -253,15 +264,6 @@(api/write-check dashboard))(map #(dissoc % :collection_id :description :archived) dashboards)))-(defn- dataset-query->query- "Convert the `dataset_query` column of a Card to a MLv2 pMBQL query."- ([dataset-query]- (some-> (:database dataset-query)- lib-be/application-database-metadata-provider- (dataset-query->query dataset-query)))- ([metadata-provider dataset-query]- (some->> dataset-query card.metadata/normalize-dataset-query (lib/query metadata-provider))))-(defn- card-columns-from-names[card names](when-let [names (not-empty (set names))]@@ -273,24 +275,25 @@(mu/defn- source-cols[card- source :- [:enum ::breakouts ::aggregations]- database-id->metadata-provider]+ source :- [:enum ::breakouts ::aggregations]](if-let [names (get-in card [:visualization_settings (case source::breakouts :graph.dimensions::aggregations :graph.metrics)])](cols->kebab-case (card-columns-from-names card names))- (->> (dataset-query->query (get database-id->metadata-provider (:database_id card)) (:dataset_query card))- lib/returned-columns- (filter (case source- ::breakouts :lib/breakout?- ::aggregations #(= (:lib/source %) :source/aggregations))))))+ (some->> card+ :dataset_query+ not-empty+ lib/returned-columns+ (filter (case source+ ::breakouts :lib/breakout?+ ::aggregations #(= (:lib/source %) :source/aggregations))))))(defn- area-bar-line-series-are-compatible?- [first-card second-card database-id->metadata-provider]+ [first-card second-card](and (#{:area :line :bar} (:display second-card))- (let [initial-dimensions (source-cols first-card ::breakouts database-id->metadata-provider)- new-dimensions (source-cols second-card ::breakouts database-id->metadata-provider)- new-metrics (source-cols second-card ::aggregations database-id->metadata-provider)]+ (let [initial-dimensions (source-cols first-card ::breakouts)+ new-dimensions (source-cols second-card ::breakouts)+ new-metrics (source-cols second-card ::aggregations)](cond;; must have at least one dimension and one metric(or (zero? (count new-dimensions))@@ -319,24 +322,24 @@(defmulti series-are-compatible?"Check if the `second-card` is compatible to be used as series of `card`."- {:arglists '([card second-card database-id->metadata-provider])}- (fn [card _second-card _database-id->metadata-provider]+ {:arglists '([card second-card])}+ (fn [card _second-card](:display card)))(defmethod series-are-compatible? :area- [first-card second-card database-id->metadata-provider]- (area-bar-line-series-are-compatible? first-card second-card database-id->metadata-provider))+ [first-card second-card]+ (area-bar-line-series-are-compatible? first-card second-card))(defmethod series-are-compatible? :line- [first-card second-card database-id->metadata-provider]- (area-bar-line-series-are-compatible? first-card second-card database-id->metadata-provider))+ [first-card second-card]+ (area-bar-line-series-are-compatible? first-card second-card))(defmethod series-are-compatible? :bar- [first-card second-card database-id->metadata-provider]- (area-bar-line-series-are-compatible? first-card second-card database-id->metadata-provider))+ [first-card second-card]+ (area-bar-line-series-are-compatible? first-card second-card))(defmethod series-are-compatible? :scalar- [first-card second-card _database-id->metadata-provider]+ [first-card second-card](and (= :scalar (:display second-card))(= 1(count (:result_metadata first-card))@@ -386,7 +389,7 @@;; so we can't use series-are-compatible? to filter out incompatible native cards.;; => we assume all native queries are compatible and FE will figure it out later(= (:query_type %) :native)- (series-are-compatible? card % database-id->metadata-provider))))]+ (series-are-compatible? card %))))](if page-size[database-id->metadata-provider (take page-size compatible-cards)][database-id->metadata-provider compatible-cards])))@@ -454,10 +457,11 @@;;; ------------------------------------------------- Creating Cards --------------------------------------------------(defn- check-if-card-can-be-saved- [dataset-query card-type]- (when (and dataset-query (= card-type :metric))- (when-not (lib/can-save (dataset-query->query dataset-query) card-type)+(mu/defn- check-if-card-can-be-saved+ [dataset-query :- [:maybe ::queries.schema/query]+ card-type :- [:maybe ::queries.schema/card-type]]+ (when (and (seq dataset-query) (= card-type :metric))+ (when-not (lib/can-save dataset-query card-type)(throw (ex-info (tru "Card of type {0} is invalid, cannot be saved." (name card-type)){:type card-type:status-code 400})))))@@ -489,7 +493,7 @@:else nil)))-(def CardCreateSchema+(def ^:private CardCreateSchema"Schema for creating a new card"[:map[:name ms/NonBlankString]@@ -497,8 +501,8 @@[:dataset_query ms/Map];; TODO: Make entity_id a NanoID regex schema?[:entity_id {:optional true} [:maybe ms/NonBlankString]]- [:parameters {:optional true} [:maybe [:sequential ::parameters.schema/parameter]]]- [:parameter_mappings {:optional true} [:maybe [:sequential ::parameters.schema/parameter-mapping]]]+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters]]+ [:parameter_mappings {:optional true} [:maybe ::parameters.schema/parameter-mappings]][:description {:optional true} [:maybe ms/NonBlankString]][:display ms/NonBlankString][:visualization_settings ms/Map]@@ -507,29 +511,27 @@[:result_metadata {:optional true} [:maybe analyze/ResultsMetadata]][:cache_ttl {:optional true} [:maybe ms/PositiveInt]][:dashboard_id {:optional true} [:maybe ms/PositiveInt]]- [:dashboard_tab_id {:optional true} [:maybe ms/PositiveInt]]])+ [:dashboard_tab_id {:optional true} [:maybe ms/PositiveInt]]])(api.macros/defendpoint :post "/""Create a new `Card`. Card `type` can be `question`, `metric`, or `model`."[_route-params_query-params- {query :dataset_query- card-type :type- :as body} :- CardCreateSchema]- (check-if-card-can-be-saved query card-type)- ;; check that we have permissions to run the query that we're trying to save- (query-perms/check-run-permissions-for-query query)- ;; check that we have permissions for the collection we're trying to save this card to, if applicable.- ;; if a `dashboard-id` is specified, check permissions on the *dashboard's* collection ID.- (collection/check-write-perms-for-collection- (actual-collection-id body))- (try- (lib/check-card-overwrite ::no-id (dataset-query->query query))- (catch clojure.lang.ExceptionInfo e- (throw (ex-info (ex-message e) (assoc (ex-data e) :status-code 400)))))- (let [body (cond-> body- (string? (:type body)) (update :type keyword))]- (-> (card/create-card! body @api/*current-user*)+ {card-type :type, :as card} :- CardCreateSchema]+ (let [card (update card :dataset_query lib-be/normalize-query)+ query (:dataset_query card)]+ (check-if-card-can-be-saved query card-type)+ ;; check that we have permissions to run the query that we're trying to save+ (query-perms/check-run-permissions-for-query query)+ ;; check that we have permissions for the collection we're trying to save this card to, if applicable.+ ;; if a `dashboard-id` is specified, check permissions on the *dashboard's* collection ID.+ (collection/check-write-perms-for-collection+ (actual-collection-id card))+ (try+ (lib/check-card-overwrite ::no-id query)+ (catch clojure.lang.ExceptionInfo e+ (throw (ex-info (ex-message e) (assoc (ex-data e) :status-code 400)))))+ (-> (card/create-card! card @api/*current-user*)hydrate-card-details(assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*)))))@@ -546,12 +548,12 @@;;; ------------------------------------------------- Updating Cards --------------------------------------------------(defn- check-allowed-to-modify-query+(mu/defn- check-allowed-to-modify-query"If the query is being modified, check that we have data permissions to run the query."- [card-before-updates card-updates]- (let [card-updates (m/update-existing card-updates :dataset_query card.metadata/normalize-dataset-query)]- (when (api/column-will-change? :dataset_query card-before-updates card-updates)- (query-perms/check-run-permissions-for-query (:dataset_query card-updates)))))+ [card-before-updates :- ::queries.schema/card+ card-updates :- ::queries.schema/card]+ (when (api/column-will-change? :dataset_query card-before-updates card-updates)+ (query-perms/check-run-permissions-for-query (:dataset_query card-updates))))(defn- check-allowed-to-change-embedding"You must be a superuser to change the value of `enable_embedding` or `embedding_params`. Embedding must be@@ -562,22 +564,27 @@(embedding.validation/check-embedding-enabled)(api/check-superuser)))-(defn- check-allowed-to-move [card-before-update card-updates]+(mu/defn- check-allowed-to-move+ [card-before-update :- ::queries.schema/card+ card-updates :- ::queries.schema/card](when (api/column-will-change? :dashboard_id card-before-update card-updates)(check-allowed-to-remove-from-existing-dashboards card-before-update))(collection/check-allowed-to-change-collection card-before-update card-updates))-(defn- check-update-result-metadata-data-perms- [card-before-updates card-updates]+(mu/defn- check-update-result-metadata-data-perms+ [card-before-updates :- ::queries.schema/card+ card-updates :- ::queries.schema/card](when (api/column-will-change? :result_metadata card-before-updates card-updates)(let [database-id (some :database_id [card-before-updates card-updates])result-metadata (:result_metadata card-updates)](query-perms/check-result-metadata-data-perms database-id result-metadata))))+;;; TODO -- merge this into `:metabase.queries.schema/card`(def ^:private CardUpdateSchema[:map[:name {:optional true} [:maybe ms/NonBlankString]]- [:parameters {:optional true} [:maybe [:sequential ::parameters.schema/parameter]]]+ [:parameters {:optional true} [:maybe ::parameters.schema/parameters]]+ [:parameter_mappings {:optional true} [:maybe ::parameters.schema/parameter-mappings]][:dataset_query {:optional true} [:maybe ms/Map]][:type {:optional true} [:maybe ::queries.schema/card-type]][:display {:optional true} [:maybe ms/NonBlankString]]@@ -609,78 +616,78 @@(mu/defn update-card!"Updates a card - impl"- [id :- ms/PositiveInt- {:keys [dataset_query- result_metadata- type] :as card-updates} :- CardUpdateSchema+ [id :- ::lib.schema.id/card+ {metadata :result_metadata, card-type :type, :as card-updates} :- CardUpdateSchemadelete-old-dashcards? :- :boolean]- (check-if-card-can-be-saved dataset_query type)- (when-some [query (dataset-query->query dataset_query)]- (try- (lib/check-card-overwrite id query)- (catch clojure.lang.ExceptionInfo e- (throw (ex-info (ex-message e) (assoc (ex-data e) :status-code 400))))))- (let [card-before-update (t2/hydrate (api/write-check :model/Card id)- [:moderation_reviews :moderator_details])- card-updates (maybe-populate-collection-id- card-before-update- (api/updates-with-archived-directly card-before-update card-updates))- is-model-after-update? (if (nil? type)- (card/model? card-before-update)- (card/model? card-updates))]- ;; Do various permissions checks- (doseq [f [check-update-result-metadata-data-perms- check-allowed-to-move- check-allowed-to-modify-query- check-allowed-to-change-embedding]]- (f card-before-update card-updates))- (let [{:keys [metadata metadata-future]} (card.metadata/maybe-async-result-metadata- {:original-query (:dataset_query card-before-update)- :query dataset_query- :metadata result_metadata- :original-metadata (:result_metadata card-before-update)- :model? is-model-after-update?- :entity-id (or (:entity_id card-updates)- (:entity_id card-before-update))})- card-updates (merge card-updates- (when (and (some? type)- is-model-after-update?- ;; leave display unchanged if explicitly set to "list"- (not (= :list (keyword (get card-updates :display)))))- {:display :table})- (when (and- (api/column-will-change? :dashboard_id- card-before-update- card-updates)- (:dashboard_id card-updates))- (api/check-400- (not (:archived card-updates)))- {:archived false}))- card-updates (cond-> card-updates- metadata- (assoc :result_metadata metadata- :verified-result-metadata? true))- card (-> (card/update-card! {:card-before-update card-before-update- :card-updates card-updates- :actor @api/*current-user*- :delete-old-dashcards? delete-old-dashcards?})- hydrate-card-details- (assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*)))]- ;; We expose the search results for models and metrics directly in FE grids, from which items can be archived.- ;; The grid is then refreshed synchronously with the latest search results, so we need this change to be- ;; reflected synchronously.- ;; An alternate solution would be to have first class APIs for these views, that don't rely on an- ;; eventually consistent search index.- (when (:archived_directly card-updates)- ;; For now, we hard-code all the possible search-model types, and queue them all as this has no extra overhead.- ;; Ideally this would be DRY with the actual specification some way, but since this is a stop-gap solution, we- ;; decided not to complicate the solution further to accomplish this.- (search/bulk-ingest! (for [search-model ["card" "dataset" "metric"]]- [search-model [:= :this.id id]])))- (when metadata-future- (log/infof "Metadata not available soon enough. Saving card %s and asynchronously updating metadata" id)- (card.metadata/save-metadata-async! metadata-future card))- card)))+ (let [card-updates (m/update-existing card-updates :dataset_query lib-be/normalize-query)+ query (:dataset_query card-updates)]+ (check-if-card-can-be-saved query card-type)+ (when-some [query (:dataset_query card-updates)]+ (try+ (lib/check-card-overwrite id query)+ (catch clojure.lang.ExceptionInfo e+ (throw (ex-info (ex-message e) (assoc (ex-data e) :status-code 400))))))+ (let [card-before-update (t2/hydrate (api/write-check :model/Card id)+ [:moderation_reviews :moderator_details])+ card-updates (maybe-populate-collection-id+ card-before-update+ (api/updates-with-archived-directly card-before-update card-updates))+ is-model-after-update? (if (nil? card-type)+ (card/model? card-before-update)+ (card/model? card-updates))]+ ;; Do various permissions checks+ (doseq [f [check-update-result-metadata-data-perms+ check-allowed-to-move+ check-allowed-to-modify-query+ check-allowed-to-change-embedding]]+ (f card-before-update card-updates))+ (let [{:keys [metadata metadata-future]} (card.metadata/maybe-async-result-metadata+ {:original-query (:dataset_query card-before-update)+ :query query+ :metadata metadata+ :original-metadata (:result_metadata card-before-update)+ :model? is-model-after-update?+ :entity-id (or (:entity_id card-updates)+ (:entity_id card-before-update))})+ card-updates (merge card-updates+ (when (and (some? card-type)+ is-model-after-update?+ ;; leave display unchanged if explicitly set to "list"+ (not (= :list (keyword (get card-updates :display)))))+ {:display :table})+ (when (and+ (api/column-will-change? :dashboard_id+ card-before-update+ card-updates)+ (:dashboard_id card-updates))+ (api/check-400+ (not (:archived card-updates)))+ {:archived false}))+ card-updates (cond-> card-updates+ metadata+ (assoc :result_metadata metadata+ :verified-result-metadata? true))+ card (-> (card/update-card! {:card-before-update card-before-update+ :card-updates card-updates+ :actor @api/*current-user*+ :delete-old-dashcards? delete-old-dashcards?})+ hydrate-card-details+ (assoc :last-edit-info (revisions/edit-information-for-user @api/*current-user*)))]+ ;; We expose the search results for models and metrics directly in FE grids, from which items can be archived.+ ;; The grid is then refreshed synchronously with the latest search results, so we need this change to be+ ;; reflected synchronously.+ ;; An alternate solution would be to have first class APIs for these views, that don't rely on an+ ;; eventually consistent search index.+ (when (:archived_directly card-updates)+ ;; For now, we hard-code all the possible search-model types, and queue them all as this has no extra overhead.+ ;; Ideally this would be DRY with the actual specification some way, but since this is a stop-gap solution, we+ ;; decided not to complicate the solution further to accomplish this.+ (search/bulk-ingest! (for [search-model ["card" "dataset" "metric"]]+ [search-model [:= :this.id id]])))+ (when metadata-future+ (log/infof "Metadata not available soon enough. Saving card %s and asynchronously updating metadata" id)+ (card.metadata/save-metadata-async! metadata-future card))+ card))))(api.macros/defendpoint :put "/:id""Update a `Card`."@@ -908,7 +915,7 @@GET /api/card/1/params/abc/values"[{:keys [card-id param-key]} :- [:map[:card-id ms/PositiveInt]- [:param-key ms/NonBlankString]]]+ [:param-key ::lib.schema.parameter/id]]](queries.card/card-param-values (api/read-check :model/Card card-id) param-key))(api.macros/defendpoint :get "/:card-id/params/:param-key/search/:query"@@ -920,7 +927,7 @@Currently limited to first 1000 results."[{:keys [card-id param-key query]} :- [:map[:card-id ms/PositiveInt]- [:param-key ms/NonBlankString]+ [:param-key ::lib.schema.parameter/id][:query ms/NonBlankString]]](queries.card/card-param-values (api/read-check :model/Card card-id) param-key query))@@ -930,8 +937,8 @@;; fetch the remapped value for Card 1 parameter 'abc' for value 100GET /api/card/1/params/abc/remapping?value=100"[{:keys [id param-key]} :- [:map- [:id ms/PositiveInt]- [:param-key :string]]+ [:id ::lib.schema.id/card]+ [:param-key ::lib.schema.parameter/id]]{:keys [value]} :- [:map [:value :string]]](-> (api/read-check :model/Card id)(queries.card/card-param-remapped-value param-key (codec/url-decode value))))diff --git a/src/metabase/queries/card.clj b/src/metabase/queries/card.cljindex 9f6eb0b6561..e04c98f2a99 100644--- a/src/metabase/queries/card.clj+++ b/src/metabase/queries/card.clj@@ -1,19 +1,23 @@(ns metabase.queries.card(:require[medley.core :as m]- [metabase.lib.util.match :as lib.util.match]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.parameter :as lib.schema.parameter][metabase.parameters.chain-filter :as chain-filter][metabase.parameters.custom-values :as custom-values][metabase.parameters.field :as parameters.field][metabase.parameters.params :as params]+ [metabase.parameters.schema :as parameters.schema][metabase.queries.models.card :as card]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]))-(defn- get-param-or-throw- [card param-key]+(mu/defn- get-param-or-throw :- ::parameters.schema/parameter+ [card :- ::queries.schema/card+ param-key :- ::lib.schema.parameter/id](u/prog1 (m/find-first #(= (:id %) param-key)(or (seq (:parameters card));; some older cards or cards in e2e just use the template tags on native queries@@ -22,17 +26,19 @@(throw (ex-info (tru "Card does not have a parameter with the ID {0}" (pr-str param-key)){:status-code 400})))))-(defn- param->field-id- [card param]- (when-let [field-clause (params/param-target->field-clause (:target param) card)]- (lib.util.match/match-one field-clause [:field (id :guard integer?) _] id)))+(mu/defn- param->field-id :- [:maybe ::lib.schema.id/field]+ [card :- ::queries.schema/card+ param :- ::parameters.schema/parameter]+ (params/param-target->field-id (:target param) card))-(defn mapping->field-values+(mu/defn- mapping->field-values :- ms/FieldValuesResult"Get param values for the \"old style\" parameters. This mimic's the api/dashboard version except we don't havechain-filter issues or dashcards to worry about."- [card param query]+ [card :- ::queries.schema/card+ param :- ::parameters.schema/parameter+ query-string :- [:maybe :string]](when-let [field-id (param->field-id card param)]- (parameters.field/search-values-from-field-id field-id query)))+ (parameters.field/search-values-from-field-id field-id query-string)))(mu/defn card-param-values"Fetch values for a parameter that contain `query`. If `query` is nil or not provided, return all values.@@ -43,15 +49,18 @@([card param-key](card-param-values card param-key nil))- ([card :- ms/Map- param-key :- ms/NonBlankString- query :- [:maybe ms/NonBlankString]]+ ([card :- ms/Map+ param-key :- ::lib.schema.parameter/id+ query-string :- [:maybe ms/NonBlankString]](let [param (get-param-or-throw card param-key)]- (custom-values/parameter->values param query (fn [] (mapping->field-values card param query))))))+ (custom-values/parameter->values param query-string (mu/fn :- ms/FieldValuesResult []+ (mapping->field-values card param query-string))))))-(defn card-param-remapped-value+(mu/defn card-param-remapped-value"Fetch the remapped value for the given `value` of parameter with ID `:param-key` of `card`."- [card param-key value]+ [card :- ::queries.schema/card+ param-key :- ::lib.schema.parameter/id+ value](or (let [param (get-param-or-throw card param-key)](custom-values/parameter-remapped-valueparamdiff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.cljindex 2edf09741ec..8dfef06d6ce 100644--- a/src/metabase/queries/core.clj+++ b/src/metabase/queries/core.clj@@ -26,6 +26,7 @@batch-fetch-dashboard-metadatabatch-fetch-query-metadata][metabase.queries.models.card+ fully-parameterized?model-supports-implicit-actions?model?sole-dashboard-id@@ -41,9 +42,6 @@query->database-and-table-idssave-query-and-update-average-execution-time!])-#_{:clj-kondo/ignore [:missing-docstring]}-(p/import-def metabase.queries.models.card/lib-query card->lib-query)-#_{:clj-kondo/ignore [:missing-docstring]}(p/import-def metabase.queries.models.card/populate-query-fields populate-card-query-fields)diff --git a/src/metabase/queries/metadata.clj b/src/metabase/queries/metadata.cljindex c2caa525a87..a86e1bfbd24 100644--- a/src/metabase/queries/metadata.clj+++ b/src/metabase/queries/metadata.clj@@ -2,11 +2,14 @@(:require[clojure.set :as set][metabase.api.common :as api]- [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.util :as lib.util][metabase.models.interface :as mi][metabase.permissions.core :as perms]+ [metabase.queries.schema :as queries.schema][metabase.util :as u][metabase.util.malli :as mu][metabase.warehouse-schema.field :as schema.field]@@ -49,26 +52,6 @@(t2/select-fn-set :table_id :model/Field :id [:in field-ids])#{}))-(defn- query->template-tags- [query]- (-> query :native :template-tags vals))--(defn- query->template-tag-field-ids [query]- (when-let [template-tags (query->template-tags query)]- (for [{tag-type :type, [dim-tag id _opts] :dimension} template-tags- :when (and (#{:dimension :temporal-unit} tag-type)- (= dim-tag :field)- (integer? id))]- id)))--(defn- query->template-tag-snippet-ids- "Extract snippet IDs from template tags in a native query."- [query]- (some->> (query->template-tags query)- (into #{}- (comp (filter #(= :snippet (:type %)))- (keep :snippet-id)))))-(defn- collect-recursive-snippets([initial-snippet-ids](when (seq initial-snippet-ids)@@ -111,31 +94,35 @@(integer? field-id))]field-id)))-(defn- batch-fetch-query-metadata*+(mu/defn- batch-fetch-query-metadata*"Fetch dependent metadata for ad-hoc queries."- [queries]- (let [{source-table-ids :table- source-card-ids :card} (lib.util/source-tables-and-cards queries)- source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids)- (schema.table/batch-fetch-card-query-metadatas source-card-ids- {:include-database? false}))- fk-target-field-ids (into #{} (comp (mapcat :fields)- (keep :fk_target_field_id))- source-tables)- fk-target-table-ids (into #{} (remove source-table-ids)- (field-ids->table-ids fk-target-field-ids))- fk-target-tables (schema.table/batch-fetch-table-query-metadatas fk-target-table-ids)- tables (concat source-tables fk-target-tables)- template-tag-field-ids (into #{} (mapcat query->template-tag-field-ids) queries)- direct-snippet-ids (into #{} (mapcat query->template-tag-snippet-ids) queries)- snippets (collect-recursive-snippets direct-snippet-ids)- snippet-field-ids (collect-snippet-field-ids snippets)+ [queries :- [:maybe [:sequential ::lib.schema/query]]]+ (let [source-table-ids (into #{}+ (mapcat lib/all-source-table-ids)+ queries)+ source-card-ids (into #{}+ (mapcat lib/all-source-card-ids)+ queries)+ source-tables (concat (schema.table/batch-fetch-table-query-metadatas source-table-ids)+ (schema.table/batch-fetch-card-query-metadatas source-card-ids+ {:include-database? false}))+ fk-target-field-ids (into #{} (comp (mapcat :fields)+ (keep :fk_target_field_id))+ source-tables)+ fk-target-table-ids (into #{} (remove source-table-ids)+ (field-ids->table-ids fk-target-field-ids))+ fk-target-tables (schema.table/batch-fetch-table-query-metadatas fk-target-table-ids)+ tables (concat source-tables fk-target-tables)+ template-tag-field-ids (into #{} (mapcat lib/all-template-tag-field-ids) queries)+ direct-snippet-ids (into #{} (mapcat lib/all-template-tag-snippet-ids) queries)+ snippets (collect-recursive-snippets direct-snippet-ids)+ snippet-field-ids (collect-snippet-field-ids snippets);; Combine all field IDs- all-field-ids (set/union template-tag-field-ids snippet-field-ids)- query-database-ids (into #{} (keep :database) queries)- database-ids (into query-database-ids- (keep :db_id)- tables)]+ all-field-ids (set/union template-tag-field-ids snippet-field-ids)+ query-database-ids (into #{} (keep :database) queries)+ database-ids (into query-database-ids+ (keep :db_id)+ tables)]{;; TODO: This is naive and issues multiple queries currently. That's probably okay for most dashboards,;; since they tend to query only a handful of databases at most.:databases (sort-by :id (get-databases database-ids))@@ -156,20 +143,31 @@(defn batch-fetch-query-metadata"Fetch dependent metadata for ad-hoc queries."[queries]- (batch-fetch-query-metadata* (map mbql.normalize/normalize queries)))+ (batch-fetch-query-metadata*+ (into []+ (comp (filter not-empty)+ (map lib-be/normalize-query))+ queries)))-(defn batch-fetch-card-metadata+(mu/defn batch-fetch-card-metadata"Fetch dependent metadata for cards.Models and native queries need their definitions walked as well as their own, card-level metadata."- [cards]- (let [queries (into (vec (keep :dataset_query cards)) ; All the queries on all the cards- ;; Plus the card-level metadata of each model and native query- (comp (filter (fn [card] (or (= :model (:type card))- (= :native (-> card :dataset_query :type)))))- (map (fn [card] {:query {:source-table (str "card__" (u/the-id card))}})))+ [cards :- [:sequential ::queries.schema/card]]+ (let [;; remove any Cards with empty queries+ cards (remove #(empty? (:dataset_query %)) cards)+ ;; All the queries on all the cards+ card-queries (map :dataset_query cards)+ ;; Plus the card-level metadata of each model and native query+ queries (into []+ (comp (filter (fn [{query :dataset_query, card-type :type, :as _card}]+ (or (= card-type :model)+ (lib/native-only-query? query))))+ (map (fn [{database-id :database_id, card-id :id, :as _card}]+ (let [mp (lib-be/application-database-metadata-provider database-id)]+ (lib/query mp (lib.metadata/card mp card-id))))))cards)]- (batch-fetch-query-metadata queries)))+ (batch-fetch-query-metadata (concat card-queries queries))))(defn- click-behavior->link-details[{:keys [linkType type targetId] :as _click-behavior}]@@ -219,9 +217,15 @@{:cards (sort-by :id link-cards):dashboards (sort-by :id dashboards)}))-(defn batch-fetch-dashboard-metadata+(mu/defn batch-fetch-dashboard-metadata"Fetch dependent metadata for dashboards."- [dashboards]+ [dashboards :- [:sequential+ [:map {:optional true} [:dashcards+ [:sequential+ [:map+ [:card {:optional true} [:maybe ::queries.schema/card]]+ [:series {:optional true} [:maybe [:sequential [:map+ [:dataset_query ::queries.schema/query]]]]]]]]]]](let [dashcards (mapcat :dashcards dashboards)cards (for [{:keys [card series]} dashcards:let [all (conj series card)]@@ -232,6 +236,7 @@(merge(->> (remove (comp card-ids :id) (:cards links))(concat cards)+ (filter some?)batch-fetch-card-metadata){:cards (or (:cards links) []):dashboards (or (:dashboards links) [])})))diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 1f8a46a08ed..de5e8a91ee8 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -4,7 +4,6 @@(:require[clojure.data :as data][clojure.set :as set]- [clojure.string :as str][clojure.walk :as walk][honey.sql.helpers :as sql.helpers][medley.core :as m]@@ -17,27 +16,28 @@[metabase.content-verification.core :as moderation][metabase.dashboards.autoplace :as autoplace][metabase.events.core :as events]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib-be.core :as lib-be]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm][metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.normalize :as lib.normalize]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.template-tag :as lib.schema.template-tag][metabase.lib.types.isa :as lib.types]- [metabase.lib.util :as lib.util][metabase.models.interface :as mi][metabase.models.serialization :as serdes]+ [metabase.parameters.core :as parameters][metabase.parameters.params :as params]+ [metabase.parameters.schema :as parameters.schema][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.public-sharing.core :as public-sharing]- ^{:clj-kondo/ignore [:deprecated-namespace]}[metabase.pulse.core :as pulse][metabase.queries.models.card.metadata :as card.metadata][metabase.queries.models.parameter-card :as parameter-card][metabase.queries.models.query :as query]+ [metabase.queries.schema :as queries.schema][metabase.query-permissions.core :as query-perms][metabase.query-processor.util :as qp.util][metabase.search.core :as search]@@ -99,14 +99,14 @@target-schema-version))(t2/deftransforms :model/Card- {:dataset_query mi/transform-metabase-query+ {:dataset_query lib-be/transform-query:display mi/transform-keyword:embedding_params mi/transform-json:query_type mi/transform-keyword:result_metadata mi/transform-result-metadata:visualization_settings mi/transform-visualization-settings- :parameters mi/transform-card-parameters-list- :parameter_mappings mi/transform-parameters-list+ :parameters parameters/transform-parameters+ :parameter_mappings parameters/transform-parameter-mappings:type mi/transform-keyword})(doto :model/Card@@ -141,19 +141,6 @@[card](= (keyword (:type card)) :model))-(defn lib-query- "Given a card with at least its `:dataset_query` field, this returns the `metabase.lib` form of the query.-- A `metadata-provider` may be passed as an optional first parameter, if the caller has one to hand."- ([{:keys [database_id dataset_query] :as card}]- (when dataset_query- (let [db-id (or database_id (:database dataset_query))- mp (lib.metadata.jvm/application-database-metadata-provider db-id)]- (lib-query mp card))))- ([metadata-providerable {:keys [dataset_query] :as _card}]- (when dataset_query- (lib/query metadata-providerable dataset_query))))-;;; -------------------------------------------------- Hydration --------------------------------------------------(methodical/defmethod t2/batched-hydrate [:model/Card :dashboard_count]@@ -212,44 +199,46 @@:id{:default []}))-(defn- source-card-id- [query]- (some-> query not-empty lib-be/normalize-query lib/source-card-id))+(mu/defn- source-card-id :- [:maybe ::lib.schema.id/card]+ [query :- [:maybe ::queries.schema/query]]+ (some-> query not-empty lib/source-card-id))-(defn- card->integer-table-ids+(mu/defn- card->integer-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]"Return integer source table ids for card's :dataset_query."- [card]- (when-some [query (-> card :dataset_query :query)]- ;; existing usage -- do not use in new code- #_{:clj-kondo/ignore [:deprecated-var]}- (not-empty (filter pos-int? (lib.util/collect-source-tables query)))))+ [card :- [:maybe ::queries.schema/card]]+ (some-> card :dataset_query not-empty lib/all-source-table-ids))-(defn- prefetch-tables-for-cards!+(mu/defn- prefetch-tables-for-cards!"Collect tables from `dataset-cards` and prefetch metadata. Should be used only with metdata provider cachingenabled, as per https://github.com/metabase/metabase/pull/45050. Returns `nil`."- [dataset-cards]- (let [db-id->table-ids (-> (group-by :database_id dataset-cards)+ [cards-with-non-empty-queries :- [:maybe+ [:sequential+ [:merge+ ::queries.schema/card+ [:map+ [:dataset_query ::lib.schema/query]]]]]]+ (let [db-id->table-ids (-> (group-by :database_id cards-with-non-empty-queries)(update-vals (partial into #{} (comp (mapcat card->integer-table-ids)(remove nil?)))))](doseq [[db-id table-ids] db-id->table-ids- :let [mp (lib.metadata.jvm/application-database-metadata-provider db-id)]+ :let [mp (lib-be/application-database-metadata-provider db-id)]:when (seq table-ids)](lib.metadata.protocols/metadatas mp {:lib/type :metadata/table, :id (set table-ids)}))))-(defn with-can-run-adhoc-query+(mu/defn with-can-run-adhoc-query"Adds `:can_run_adhoc_query` to each card."- [cards]+ [cards :- [:maybe [:sequential ::queries.schema/card]]];; TODO: for metrics, we can get (some-fn :source_model_id :source_question_id)- (let [dataset-cards (filter (comp seq :dataset_query) cards)- source-card-ids (into #{}- (keep (comp source-card-id :dataset_query))- dataset-cards)]+ (let [cards-with-non-empty-queries (filter (comp seq :dataset_query) cards)+ source-card-ids (into #{}+ (keep (comp source-card-id :dataset_query))+ cards-with-non-empty-queries)];; Prefetching code should not propagate any exceptions.- (when lib.metadata.jvm/*metadata-provider-cache*+ (when (lib-be/metadata-provider-cache)(try- (prefetch-tables-for-cards! dataset-cards)+ (prefetch-tables-for-cards! cards-with-non-empty-queries)(catch Throwable t- (log/errorf t "Failed prefetching cards `%s`." (pr-str (map :id dataset-cards))))))+ (log/errorf t "Failed prefetching cards `%s`." (pr-str (map :id cards-with-non-empty-queries))))))(query-perms/with-card-instances (when (seq source-card-ids)(t2/select-fn->fn :id identity [:model/Card :id :collection_id :card_schema]:id [:in source-card-ids]))@@ -261,23 +250,21 @@(map(fn [{card-id :id :keys [dataset_query]}][card-id (query-perms/can-run-query? dataset_query)]))- dataset-cards))+ cards-with-non-empty-queries)):id{:default false}))))-(mi/define-batched-hydration-method add-can-run-adhoc-query- :can_run_adhoc_query+(methodical/defmethod t2/batched-hydrate [:model/Card :can_run_adhoc_query]"Hydrate can_run_adhoc_query onto cards"- [cards]+ [_model _k cards](with-can-run-adhoc-query cards));; note: perms lookup here in the course of fetching a card/model should hit a cache pre-warmed by;; the `:can_run_adhoc_query` above-(mi/define-batched-hydration-method add-can-manage-db- :can_manage_db+(methodical/defmethod t2/batched-hydrate [:model/Card :can_manage_db]"Hydrate can_manage_db onto cards. Indicates whether the current user has access to the database admin page for thedatabase powering this card."- [cards]+ [_model _k cards](map(fn [card](assoc card@@ -343,35 +330,33 @@;;; --------------------------------------------------- Lifecycle -----------------------------------------------------(defn populate-query-fields+(mu/defn populate-query-fields"Lift `database_id`, `table_id`, `query_type`, and `source_card_id` fieldsfrom query definition when inserting/updating a Card."- [{query :dataset_query, :as card}]+ [{query :dataset_query, :as card} :- ::queries.schema/card](mergecard- (when-let [source-id (source-card-id query)]- {:source_card_id source-id})- ;; mega HACK FIXME -- don't update this stuff when doing deserialization because it might differ from what's in the- ;; YAML file and break tests like [[metabase-enterprise.serialization.v2.e2e.yaml-test/e2e-storage-ingestion-test]].- ;; The root cause of this issue is that we're generating Cards that have a different Database ID or Table ID from- ;; what's actually in their query -- we need to fix [[metabase.test.generate]], but I'm not sure how to do that- (when (and (map? query)- (not mi/*deserializing?*))- (when-let [{:keys [database-id table-id]} (query/query->database-and-table-ids query)]- ;; TODO -- not sure `query_type` is actually used for anything important anyway- (let [query-type (if (query/query-is-native? query)- :native- :query)]- (merge- {:query_type (keyword query-type)}- (when database-id- {:database_id database-id})- (when table-id- {:table_id table-id})))))))--(defn- maybe-normalize-query [card]- (cond-> card- (seq (:dataset_query card)) (update :dataset_query #(mi/maybe-normalize-query :in %))))+ (when (seq query)+ (merge+ (when-let [source-id (source-card-id query)]+ {:source_card_id source-id})+ ;; mega HACK FIXME -- don't update this stuff when doing deserialization because it might differ from what's in the+ ;; YAML file and break tests like [[metabase-enterprise.serialization.v2.e2e.yaml-test/e2e-storage-ingestion-test]].+ ;; The root cause of this issue is that we're generating Cards that have a different Database ID or Table ID from+ ;; what's actually in their query -- we need to fix [[metabase.test.generate]], but I'm not sure how to do that+ (when (and (map? query)+ (not mi/*deserializing?*))+ (when-let [{:keys [database-id table-id]} (query/query->database-and-table-ids query)]+ ;; TODO -- not sure `query_type` is actually used for anything important anyway+ (let [query-type (if (query/query-is-native? query)+ :native+ :query)]+ (merge+ {:query_type (keyword query-type)}+ (when database-id+ {:database_id database-id})+ (when table-id+ {:table_id table-id})))))))));;; TODO -- move this to [[metabase.query-processor.card]] or MLv2 so the logic can be shared between the backend and;;; frontend (?)@@ -380,12 +365,12 @@;;; If this function moves you should update the comment that links to this one (#40013);;;;;; TODO -- does this belong HERE or in the `parameters` module?-(defn template-tag-parameters+(mu/defn template-tag-parameters :- ::parameters.schema/parameters"Transforms native query's `template-tags` into `parameters`.An older style was to not include `:template-tags` onto cards as parameters. I think this is a mistake and theyshould always be there. Apparently lots of e2e tests are sloppy about this so this is included as a convenience."- [card]- (for [[_ {tag-type :type, widget-type :widget-type, :as tag}] (get-in card [:dataset_query :native :template-tags])+ [card :- [:maybe ::queries.schema/card]]+ (for [{tag-type :type, widget-type :widget-type, :as tag} (some-> card :dataset_query not-empty lib/all-template-tags):when (and tag-type(or (contains? lib.schema.template-tag/raw-value-template-tag-types tag-type)(and (= tag-type :dimension) widget-type (not= widget-type :none))))]@@ -434,14 +419,14 @@:query-database query-db-id:field-filter-database field-db-id})))))))-(defn- assert-valid-type+(mu/defn- assert-valid-type"Check that the card is a valid model if being saved as one. Throw an exception if not."- [{query :dataset_query, card-type :type, :as _card}]+ [{query :dataset_query, card-type :type, :as _card} :- [:maybe ::queries.schema/card]](when (= (keyword card-type) :model)- (let [template-tag-types (->> (get-in query [:native :template-tags])- vals- (map (comp keyword :type)))]- (when (some (complement #{:card :snippet}) template-tag-types)+ (let [template-tag-types (into #{}+ (map :type)+ (some-> query not-empty lib/all-template-tags))]+ (when (seq (set/difference template-tag-types #{:card :snippet}))(throw (ex-info (tru "A model made from a native SQL question cannot have a variable or field filter."){:status-code 400})))))nil)@@ -510,7 +495,8 @@;; TODO (Cam 7/18/25) -- weird/offputting to have half of the before-insert logic live here and then the other half live;; in `define-before-insert`... we should consolidate it so it all lives in one or the other.(defn- pre-insert [card]- (let [defaults {:parameters []+ (let [card (lib/normalize ::queries.schema/card card)+ defaults {:parameters []:parameter_mappings []:card_schema current-schema-version}card (maybe-check-dashboard-internal-card@@ -519,7 +505,6 @@(check-field-filter-fields-are-from-correct-database card);; TODO: add a check to see if all id in :parameter_mappings are in :parameters (#40013)(assert-valid-type card)-(params/assert-valid-parameters card)(params/assert-valid-parameter-mappings card)(collection/check-collection-namespace :model/Card (:collection_id card)))))@@ -573,15 +558,17 @@(when-not (= parameters new-parameters)(t2/update! model po-id {:parameters new-parameters})))))))-(defn model-supports-implicit-actions?+(mu/defn model-supports-implicit-actions?"A model with implicit action supported iff they are a raw table,meaning there are no clauses such as filter, limit, breakout...It should be the opposite of [[metabase.lib.stage/has-clauses]] but for all stages."- [{dataset-query :dataset_query :as _card}]- (and (= :query (:type dataset-query))- (every? #(nil? (get-in dataset-query [:query %]))- [:expressions :filter :limit :breakout :aggregation :joins :order-by :fields])))+ [{query :dataset_query :as _card} :- ::queries.schema/card]+ (and (seq query)+ (every? (fn [stage-number]+ (and (lib/mbql-stage? query stage-number)+ (not (lib/has-clauses? query stage-number))))+ (range 0 (count (:stages query))))))(defn- disable-implicit-action-for-model!"Delete all implicit actions of a model if exists."@@ -608,7 +595,7 @@:id (u/the-id id)))];; if the template tag params for this Card have changed in any way we need to update the FieldValues for;; On-Demand DB Fields- (when (get-in changes [:dataset_query :native])+ (when (some-> changes :dataset_query lib/native-only-query?)(let [old-param-field-ids (params/card->template-tag-field-ids old-card-info)new-param-field-ids (params/card->template-tag-field-ids changes)](when (and (seq new-param-field-ids)@@ -656,7 +643,7 @@(-> card :dataset_query not-empty)(-> card :database_id))card- (m/assoc-some card :query_description (some-> (lib.metadata.jvm/application-database-metadata-provider+ (m/assoc-some card :query_description (some-> (lib-be/application-database-metadata-provider(:database_id card))(lib/query (:dataset_query card))lib/suggested-name))))@@ -728,7 +715,7 @@(-> card(assoc :metabase_version config/mb-version-string:card_schema current-schema-version)- maybe-normalize-query+ queries.schema/normalize-card;; Must have an entity_id before populating the metadata. TODO (Cam 7/11/25) -- actually, this is no longer true,;; since we're removing `:ident`s; we can probably remove this now.(u/assoc-default :entity_id (u/generate-nano-id))@@ -766,12 +753,13 @@(t2/define-before-update :model/Card[{:keys [verified-result-metadata?] :as card}]- (let [changes (t2/changes card)]+ (let [changes (some-> card t2/changes queries.schema/normalize-card)+ card (queries.schema/normalize-card card)](-> card(dissoc :verified-result-metadata?)(assoc :card_schema current-schema-version)(apply-dashboard-question-updates changes)- maybe-normalize-query+ (m/update-existing :dataset_query lib-be/normalize-query)(populate-result-metadata changes verified-result-metadata?)(pre-update changes)populate-query-fields@@ -922,7 +910,7 @@:entity_id (u/generate-nano-id))(cond-> (nil? type)(assoc :type :question))- maybe-normalize-query)+ (m/update-existing :dataset_query lib-be/normalize-query)){:keys [metadata metadata-future]} (card.metadata/maybe-async-result-metadata{:query (:dataset_query card-data):metadata result_metadata@@ -986,12 +974,21 @@:query_type ;; these first three may not even be changeable:dataset_query})-(defn- breakout-->identifier->refs+(mu/defn- mbql-clause->identifier-for-parameter-updates+ "Get the unique key used to power the parameter mapping update stuff added in #49308. Key is an MBQL-5-style clause+ but with an empty options clause."+ [mbql-clause :- vector?]+ ;; this can currently get called with either MBQL 5 (from query breakouts) or with legacy refs (in `parameter_mappings`)+ (if (map? (second mbql-clause))+ (into [(first mbql-clause)] (drop 2) mbql-clause)+ (subvec mbql-clause 0 2)))++(defn- breakout->identifier->refs"Generate mapping of of _ref identifier_ -> #{_ref..._}._ref identifier_ is a vector of first 2 elements of ref, eg. [:expression \"xix\"] or [:field 10]"[breakouts]- (-> (group-by #(subvec % 0 2) breakouts)+ (-> (group-by mbql-clause->identifier-for-parameter-updates breakouts)(update-vals set)))(defn- action-for-identifier+refs@@ -1001,27 +998,27 @@the [[update-associated-parameters!]]'s docstring._Action_ has a form of [<action> & args]."- [after--identifier->refs identifier before--refs]- (let [after--refs (get after--identifier->refs identifier #{})]- (when (and (= 1 (count before--refs) (count after--refs))- (not= before--refs after--refs))- [:update (first after--refs)])))+ [after-identifier->refs identifier before-refs]+ (let [after-refs (get after-identifier->refs identifier #{})]+ (when (and (= 1 (count before-refs) (count after-refs))+ (not= (lib/remove-lib-uuids before-refs) (lib/remove-lib-uuids after-refs)))+ [:update (first after-refs)])))-(defn- breakouts-->identifier->action+(defn- breakouts->identifier->action"Generate mapping of _identifier_ -> _action_._identifier_ is is a vector of first 2 elements of ref, eg. [:expression \"xix\"] or [:field 10]. Action is generatedin [[action-for-identifier+refs]] and performed later in [[update-mapping]]."[breakout-before-update breakout-after-update]- (let [before--identifier->refs (breakout-->identifier->refs breakout-before-update)- after--identifier->refs (breakout-->identifier->refs breakout-after-update)]+ (let [before-identifier->refs (breakout->identifier->refs breakout-before-update)+ after-identifier->refs (breakout->identifier->refs breakout-after-update)];; Remove no-ops to avoid redundant db calls in [[update-associated-parameters!]].- (->> before--identifier->refs- (m/map-kv-vals #(action-for-identifier+refs after--identifier->refs %1 %2))+ (->> before-identifier->refs+ (m/map-kv-vals #(action-for-identifier+refs after-identifier->refs %1 %2))(m/filter-vals some?)not-empty)))-(defn eligible-mapping?+(defn- eligible-mapping?"Decide whether parameter mapping has strucuture so it can be updated presumably using [[update-mapping]]."[{[dim [ref-kind]] :target :as _mapping}](and (= dim :dimension)@@ -1029,11 +1026,12 @@(defn- update-mapping"Return modifed mapping according to action."- [identifier->action {[_dim ref] :target :as mapping}]- (let [identifier (subvec ref 0 2)+ [identifier->action {[_dim field-ref] :target :as mapping}]+ (let [identifier (mbql-clause->identifier-for-parameter-updates field-ref)[action arg] (get identifier->action identifier)](case action- :update (assoc-in mapping [:target 1] arg)+ ;; allowed for now since parameter mappings still use Legacy MBQL fields refs (for now)+ :update (assoc-in mapping [:target 1] #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL arg))mapping)))(defn- updates-for-dashcards@@ -1046,7 +1044,7 @@:when (not= parameter_mappings updated)][id {:parameter_mappings updated}])))-(defn- update-associated-parameters!+(mu/defn- update-associated-parameters!"Update _parameter mappings_ of _dashcards_ that target modified _card_, to reflect the modification.This function handles only modifications to breakout.@@ -1056,13 +1054,14 @@_parameter mappings_ have to be updated to target new, modified refs. This function takes care of that.First mappings of _identifier_ -> _action_ are generated. _identifier_ is described- eg. in [[breakouts-->identifier->action]] docstring. Then, dashcards are fetched and updates are generated+ eg. in [[breakouts->identifier->action]] docstring. Then, dashcards are fetched and updates are generatedby [[updates-for-dashcards]]. Updates are then executed."- [card-before card-after]- (let [card->breakout #(-> % :dataset_query mbql.normalize/normalize :query :breakout)- breakout-before (card->breakout card-before)- breakout-after (card->breakout card-after)]- (when-some [identifier->action (breakouts-->identifier->action breakout-before breakout-after)]+ [card-before :- ::queries.schema/card+ card-after :- ::queries.schema/card]+ (let [card->breakouts #(some-> % :dataset_query not-empty lib/breakouts)+ breakouts-before (card->breakouts card-before)+ breakouts-after (card->breakouts card-after)]+ (when-some [identifier->action (breakouts->identifier->action breakouts-before breakouts-after)](let [dashcards (t2/select :model/DashboardCard :card_id (some :id [card-after card-before]))updates (updates-for-dashcards identifier->action dashcards)];; Beware. This can have negative impact on card update performance as queries are fired in sequence. I'm not@@ -1108,7 +1107,7 @@(try(update-associated-parameters! card-before-update card-updates)(catch Throwable e- (log/error "Update of dependent card parameters failed!")+ (log/error e "Update of dependent card parameters failed!")(log/debug e"`card-before-update`:" (pr-str card-before-update)"`card-updates`:" (pr-str card-updates)))))@@ -1134,6 +1133,14 @@(not (:archived card)))(:id dashboard))))+(mu/defn fully-parameterized?+ "Given a Card, returns `true` if its query is fully parameterized."+ [{query :dataset_query, :as _card} :- [:map+ [:dataset_query [:maybe ::queries.schema/query]]]]+ (if (empty? query)+ true+ (lib/fully-parameterized-query? query)))+(methodical/defmethod mi/to-json :model/Card[card json-generator];; we're doing update + dissoc instead of [[medley.core/dissoc-in]] because we don't want it to remove the@@ -1191,7 +1198,9 @@;; this column is not used anymore:cache_ttl;; dependencies aren't serialized, so the version of dependency analysis done shouldn't be serialized- :dependency_analysis_version]+ :dependency_analysis_version+ ;; temporary column to power rollback from v57 to v56; we can remove it in v58+ :legacy_query]:transform{:created_at (serdes/date):database_id (serdes/fk :model/Database :name)@@ -1228,15 +1237,16 @@(defmethod serdes/descendants "Card" [_model-name id](let [card (t2/select-one :model/Card :id id)- source-table (some-> card :dataset_query :query :source-table)- template-tags (some->> card :dataset_query :native :template-tags vals (keep :card-id))+ query (not-empty (:dataset_query card))+ source-card (some-> query lib/source-card-id)+ template-tags (some-> query lib/all-template-tags)parameters-card-id (some->> card :parameters (keep (comp :card_id :values_source_config)))- snippets (some->> card :dataset_query :native :template-tags vals (keep :snippet-id))]+ snippets (some->> template-tags (keep :snippet-id))](into {} (concat- (when (and (string? source-table)- (str/starts-with? source-table "card__"))- {["Card" (parse-long (subs source-table 6))] {"Card" id}})- (for [card-id template-tags]+ (when source-card+ {["Card" source-card] {"Card" id}})+ (for [{:keys [card-id]} template-tags+ :when card-id]{["Card" card-id] {"Card" id}})(for [card-id parameters-card-id]{["Card" card-id] {"Card" id}})@@ -1245,21 +1255,17 @@;;;; ------------------------------------------------- Search -----------------------------------------------------------(defn- dataset-query->dimensions+(mu/defn- dataset-query->dimensions :- [:maybe [:sequential ::lib.schema.metadata/column]]"Extract dimensions (non-aggregation columns) from a dataset query."- [dataset-query-str]+ [dataset-query-str :- [:maybe :string]](when dataset-query-str;; In production the :database should be always present and correct. That is not the case for some test mocks.;; As e.g. in [[metabase-enterprise.semantic-search.test-util/do-with-indexable-documents!]]. Hence the thorough;; checking.- (when-some [dataset-query (not-empty ((:out mi/transform-metabase-query) dataset-query-str))]- (when (pos-int? (:database dataset-query))- (lib.metadata.jvm/with-metadata-provider-cache- (let [metadata-provider (lib.metadata.jvm/application-database-metadata-provider (:database dataset-query))- lib-query (lib/query metadata-provider dataset-query)- columns (lib/returned-columns lib-query)]- ;; Dimensions are columns that are not aggregations- (remove (comp #{:source/aggregations} :lib/source) columns)))))))+ (when-some [query (not-empty ((:out lib-be/transform-query) dataset-query-str))]+ (let [columns (lib/returned-columns query)]+ ;; Dimensions are columns that are not aggregations+ (remove (comp #{:source/aggregations} :lib/source) columns)))))(defn- extract-non-temporal-dimension-ids"Extract list of nontemporal dimension field IDs, stored as JSON string. See PR 60912"@@ -1279,10 +1285,10 @@(defn- maybe-extract-native-query"Return the native SQL text (truncated to `max-searchable-value-length`) if `dataset_query` is native; else nil."- [{:keys [dataset_query]}]- (let [query ((:out mi/transform-metabase-query) dataset_query)- query-text (when (= :native (:type query))- (get-in query [:native :query]))]+ [{query :dataset_query, :as _card}]+ (let [query ((:out lib-be/transform-query) query)+ query-text (when (lib/native-only-query? query)+ (:native (lib/query-stage query 0)))](when query-text(subs query-text 0 (min (count query-text) search/max-searchable-value-length)))))diff --git a/src/metabase/queries/models/card/metadata.clj b/src/metabase/queries/models/card/metadata.cljindex af625358829..5ee01377b7c 100644--- a/src/metabase/queries/models/card/metadata.clj+++ b/src/metabase/queries/models/card/metadata.clj@@ -5,7 +5,7 @@[metabase.analyze.core :as analyze][metabase.api.common :as api][metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib.core :as lib]+ [metabase.lib-be.core :as lib-be][metabase.lib.normalize :as lib.normalize][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -81,10 +81,9 @@ saved later when it is ready."(defn normalize-dataset-query"Normalize the query `dataset-query` received via an HTTP call.Handles both (legacy) MBQL and pMBQL queries."+ {:deprecated "0.57.0"}[dataset-query]- (if (= (lib/normalized-query-type dataset-query) :mbql/query)- (lib/normalize dataset-query)- (mbql.normalize/normalize dataset-query)))+ (lib-be/normalize-query dataset-query))(mu/defn maybe-async-result-metadata :- ::maybe-async-result-metadata"Return result metadata for the passed in `query`. If metadata needs to be recalculated, waits up to@@ -105,8 +104,8 @@ saved later when it is ready."(cond(or;; query didn't change, preserve existing metadata- (and (= (normalize-dataset-query original-query)- (normalize-dataset-query query))+ (and (= (lib-be/normalize-query original-query)+ (lib-be/normalize-query query))valid-metadata?);; only sent valid metadata in the edit. Metadata might be the same, might be different. We save in either case(and (nil? query)diff --git a/src/metabase/queries/models/parameter_card.clj b/src/metabase/queries/models/parameter_card.cljindex 522416457b7..a6c017003cf 100644--- a/src/metabase/queries/models/parameter_card.clj+++ b/src/metabase/queries/models/parameter_card.clj@@ -20,8 +20,9 @@(t2/deftransforms :model/ParameterCard{:parameterized_object_type mi/transform-keyword})-(defonce ^{:doc "Set of valid parameterized_object_type for a ParameterCard"}- valid-parameterized-object-type #{"dashboard" "card"})+(def valid-parameterized-object-type+ "Set of valid parameterized_object_type for a ParameterCard"+ #{"dashboard" "card"})(defn- validate-parameterized-object-type[{:keys [parameterized_object_type] :as _parameter-card}]@@ -69,12 +70,12 @@(mu/defn upsert-or-delete-from-parameters!"From a parameters list on card or dashboard, create, update,or delete appropriate ParameterCards for each parameter in the dashboard"- [parameterized-object-type :- ms/NonBlankString+ [parameterized-object-type :- ms/NonBlankString ; TODO (Cam 9/25/25) -- change this to take `:model/Dashboard` or `:model/Card` instead of ANY STRINGparameterized-object-id :- ms/PositiveIntparameters :- [:maybe [:sequential ::parameters.schema/parameter]]](let [upsertable? (fn [{:keys [values_source_type values_source_config id]}](and values_source_type id (:card_id values_source_config)- (= values_source_type "card")))+ (= values_source_type :card)))upsertable-parameters (filter upsertable? parameters)](upsert-from-parameters! parameterized-object-type parameterized-object-id upsertable-parameters)(delete-all-for-parameterized-object! parameterized-object-type parameterized-object-id (map :id upsertable-parameters))))diff --git a/src/metabase/queries/models/query.clj b/src/metabase/queries/models/query.cljindex 0a50a6d7233..d061d92ccc8 100644--- a/src/metabase/queries/models/query.clj+++ b/src/metabase/queries/models/query.clj@@ -1,15 +1,13 @@(ns metabase.queries.models.query"Functions related to the 'Query' model, which records stuff such as average query execution time."(:require- [clojure.walk :as walk][metabase.app-db.core :as mdb][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id][metabase.lib.util :as lib.util][metabase.models.interface :as mi]- ;; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ [metabase.queries.schema :as queries.schema][metabase.util.honey-sql-2 :as h2x][metabase.util.json :as json][metabase.util.malli :as mu]@@ -78,8 +76,8 @@(defn save-query-and-update-average-execution-time!"Update the recorded average execution time (or insert a new record if needed) for `query` with `query-hash`."- [query, ^bytes query-hash, ^Integer execution-time-ms]- {:pre [(instance? (Class/forName "[B") query-hash)]}+ [query ^bytes query-hash ^Integer execution-time-ms]+ {:pre [(bytes? query-hash)]}(or;; if there's already a matching Query update the rolling average(update-rolling-average-execution-time! query query-hash execution-time-ms)@@ -96,89 +94,23 @@[:database-id ::lib.schema.id/database][:table-id [:maybe ::lib.schema.id/table]]])-(mu/defn- pmbql-query->database-and-table-ids :- ::database-and-table-ids- [{database-id :database, :as query} :- [:map- [:lib/type [:= :mbql/query]]]]- (if-let [source-card-id (lib.util/source-card-id query)]- (let [card (lib.metadata/card query source-card-id)]- (merge {:table-id nil} (select-keys card [:database-id :table-id])))- (let [table-id (lib.util/source-table-id query)]- {:database-id database-id- :table-id table-id})))--(mu/defn- legacy-query->database-and-table-ids :- ::database-and-table-ids- [{database-id :database, query-type :type, {:keys [source-table source-query]} :query} :- [:map- [:type [:enum :query :native]]]]- (cond- (= :native query-type) {:database-id database-id, :table-id nil}- (integer? source-table) {:database-id database-id, :table-id source-table}- (string? source-table) (let [card-id (lib.util/legacy-string-table-id->card-id source-table)]- (if (qp.store/initialized?)- (lib.metadata/card (qp.store/metadata-provider) card-id)- (-> (t2/select-one [:model/Card- ;; `card_schema` is only needed for post-processing.- :card_schema- [:table_id :table-id]- [:database_id :database-id]]- :id card-id)- ;; remove `card_schema` so people don't try to use a key that would be `kebab-case`- ;; if we got it from the metadata provider.- (dissoc :card_schema))))- (map? source-query) (legacy-query->database-and-table-ids {:database database-id- :type query-type- :query source-query})))-(mu/defn query->database-and-table-ids :- [:maybe ::database-and-table-ids]"Return a map with `:database-id` and source `:table-id` that should be saved for a Card.- Handles either pMBQL (MLv2) queries or legacy MBQL queries. Handles source Cards by fetching them as needed."- [query :- [:maybe :map]]- (when query- (when-let [f (case (lib/normalized-query-type query)- :mbql/query pmbql-query->database-and-table-ids- (:native :query) legacy-query->database-and-table-ids- nil)]- (f (mi/maybe-normalize-query :out query)))))--(defn- parse-source-query-id- "Return the ID of the card used as source table, if applicable; otherwise return `nil`."- [source-table]- (when (string? source-table)- (when-let [[_ card-id-str] (re-matches #"card__(\d+)" source-table)]- (parse-long card-id-str))))--(defn collect-card-ids- "Return a sequence of model ids referenced in the MBQL `query`.-- DEPRECATED: Use [[metabase.lib.core/all-source-card-ids]] going forward."- {:deprecated "0.57.0"}- [query]- (let [ids (java.util.HashSet.)- walker (fn [form]- (when (map? form)- ;; model references in native queries- (when-let [card-id (:card-id form)]- (when (int? card-id)- (.add ids card-id)))- ;; source tables (possibly in joins)- ;;- ;; MLv2 `:source-card`- (when-let [card-id (:source-card form)]- (.add ids card-id))- ;; legacy MBQL card__<id> `:source-table`- (when-let [card-id (parse-source-query-id (:source-table form))]- (.add ids card-id)))- form)]- (walk/prewalk walker query)- (seq ids)))+ Expects MBQL 5 queries."+ [{database-id :database, :as query} :- ::queries.schema/query]+ (when (seq query)+ (if-let [source-card-id (lib.util/source-card-id query)]+ (let [card (or (lib.metadata/card query source-card-id)+ ;; Card may belong to a different Database; fetch from the app DB+ (t2/select-one [:model/Card [:database_id :database-id] [:table_id :table-id]] :id source-card-id))]+ (merge {:table-id nil, :database-id (:database query)} (select-keys card [:database-id :table-id])))+ (let [table-id (lib.util/source-table-id query)]+ {:database-id database-id+ :table-id table-id}))))(mu/defn query-is-native? :- :boolean"Whether this query (pMBQL or legacy) has a `:native` first stage. Queries with source Cards are considered to be MBQLregardless of whether the Card has a native query or not."- [query :- :map]- (case (lib/normalized-query-type query)- :query false- :native true- :mbql/query (let [query (mi/maybe-normalize-query :out query)]- (lib.util/first-stage-is-native? query))- false))+ [query :- [:maybe ::queries.schema/query]]+ (boolean (some-> query not-empty lib/native-only-query?)))diff --git a/src/metabase/queries/schema.clj b/src/metabase/queries/schema.cljindex 642b9fc693e..24bfd096510 100644--- a/src/metabase/queries/schema.clj+++ b/src/metabase/queries/schema.clj@@ -1,20 +1,57 @@(ns metabase.queries.schema(:require+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.parameters.schema :as parameters.schema]+ [metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.regex :as u.regex]))+ [potemkin :as p]))-(def card-types- "All acceptable card types.+(p/import-vars+ [lib.schema.metadata+ card-types])- Previously (< 49), we only had 2 card types: question and model, which were differentiated using the boolean- `dataset` column. Soon we'll have more card types (e.g: metric) and we will longer be able to use a boolean column- to differentiate between all types. So we've added a new `type` column for this purpose.+(mr/def ::card-type+ ::lib.schema.metadata/card.type)- Migrating all the code to use `report_card.type` will be quite an effort, we decided that we'll migrate it- gradually."- #{:model :question :metric})+(mr/def ::empty-map+ "An empty map, allowed for Card.dataset_query for historic purposes."+ [:= {} {}])-(mr/def ::card-type- (into [:enum {:decode/json keyword- :api/regex (u.regex/re-or (map name card-types))}]- card-types))+(mr/def ::query+ "Schema for Card.dataset_query. Cards are for some wacko reason allowed to be saved with empty queries (`{}`), but not+ `NULL` ones, because the column is non-null. This sorta seems like an oversight but fixing all the tests that save+ Cards with empty queries is too much to attempt at this point. So a Card either has an empty query or a valid MBQL 5+ query."+ [:multi {:dispatch (comp boolean empty?)}+ [true ::empty-map]+ [false [:schema+ {:decode/normalize lib-be/normalize-query}+ [:merge+ [:ref ::lib.schema/query]+ ;; Card query is guaranteed to have a metadata provider+ [:map+ [:lib/metadata ::lib.schema.metadata/metadata-provider]]]]]])++;;; TODO (Cam 9/29/25) -- fill this out more, `:metabase.lib.schema.metadata/card` has a lot of stuff and there's also+;;; stuff sprinkled thruout this module. For example [[metabase.queries.api.card/CardUpdateSchema]] should get merged+;;; into this+;;;+;;; TODO (Cam 9/30/25) -- consider renaming this to `:model/Card` so it can serve as the "official" schema of a Card+;;; instance+(mr/def ::card+ "Schema for an instance of a `:model/Card` (everything is optional to support updates)."+ [:map+ [:id {:optional true} [:maybe ::lib.schema.id/card]]+ [:dataset_query {:optional true} [:maybe ::query]]+ [:parameters {:optional true} [:maybe [:ref ::parameters.schema/parameters]]]+ [:parameter_mappings {:optional true} [:maybe [:ref ::parameters.schema/parameter-mappings]]]+ [:type {:optional true} [:maybe ::lib.schema.metadata/card.type]]])++(mu/defn normalize-card :- [:maybe ::card]+ "Normalize a `card` so it satisfies the `::card` schema."+ [card :- [:maybe :map]]+ (lib/normalize ::card card))diff --git a/src/metabase/query_permissions/impl.clj b/src/metabase/query_permissions/impl.cljindex d2fd8a95383..fb40b80dacb 100644--- a/src/metabase/query_permissions/impl.clj+++ b/src/metabase/query_permissions/impl.clj@@ -19,7 +19,7 @@[metabase.permissions.core :as perms][metabase.query-processor.error-type :as qp.error-type];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log]@@ -247,7 +247,7 @@;; if for some reason we can't expand the Card (i.e. it's an invalid legacy card) just return a set of permissions;; that means no one will ever get to see it(catch Throwable e- (let [e (ex-info "Error calculating permissions for query"+ (let [e (ex-info (format "Error calculating permissions for query: %s" (ex-message e)){:query (or (u/ignore-exceptions (mbql.normalize/normalize query))query)}e)]diff --git a/src/metabase/query_processor.clj b/src/metabase/query_processor.cljindex db160ec0279..b505b5d7298 100644--- a/src/metabase/query_processor.clj+++ b/src/metabase/query_processor.clj@@ -7,7 +7,6 @@Various REST API endpoints, such as `POST /api/dataset`, return the results of queries; they usuallyuse [[userland-query]] or [[userland-query-with-default-constraints]] (see below)."(:require- [metabase.lib.core :as lib][metabase.lib.schema.info :as lib.schema.info][metabase.query-processor.compile :as qp.compile][metabase.query-processor.debug :as qp.debug]@@ -81,13 +80,6 @@(let [rff (or rff qp.reducible/default-rff)](process-query* query rff)))))-(mu/defn compile-query-with-metadata-provider- "Compiles a given query with the provided metadata provider."- [metadata-providerable :- :metabase.lib.schema.metadata/metadata-providerable- query :- ::qp.schema/any-query]- (-> (lib/query metadata-providerable query)- qp.compile/compile))-(mu/defn userland-query :- ::qp.schema/any-query"Add middleware options and `:info` to a `query` so it is ran as a 'userland' query, which slightly changes the QPbehavior:diff --git a/src/metabase/query_processor/card.clj b/src/metabase/query_processor/card.cljindex f4b4aa9c050..20e14e08053 100644--- a/src/metabase/query_processor/card.clj+++ b/src/metabase/query_processor/card.clj@@ -9,9 +9,9 @@;; legacy usages -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use;; Lib soon^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.normalize :as mbql.normalize]- ^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info][metabase.lib.schema.parameter :as lib.schema.parameter]@@ -19,6 +19,7 @@[metabase.lib.util.match :as lib.util.match][metabase.premium-features.core :refer [defenterprise]][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.query-processor :as qp][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.constraints :as qp.constraints]@@ -26,6 +27,7 @@[metabase.query-processor.middleware.results-metadata :as qp.results-metadata][metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.schema :as qp.schema]+ ^{:clj-kondo/ignore [:deprecated-namespace]}[metabase.query-processor.store :as qp.store][metabase.query-processor.streaming :as qp.streaming][metabase.query-processor.util :as qp.util]@@ -74,18 +76,13 @@(assoc-in [:target 2 :stage-number] -1)))parameters))-(defn- last-stage-number- [outer-query]- #_{:clj-kondo/ignore [:deprecated-var]}- (mbql.u/legacy-last-stage-number (:query outer-query)))--(defn- nest-query- [query]- (assoc query :query {:source-query (:query query)}))+(mu/defn- last-stage-number+ [query :- ::lib.schema/query]+ (dec (count (:stages query))))-(defn- add-stage-to-temporal-unit-parameters+(mu/defn- add-stage-to-temporal-unit-parameters :- ::lib.schema.parameter/parameters"Points temporal-unit parameters to the penultimate stage unless the stage is specified."- [parameters]+ [parameters :- ::lib.schema.parameter/parameters](mapv (fn [{param-type :type, :keys [target], :as parameter}](cond-> parameter(and (= param-type :temporal-unit)@@ -94,41 +91,51 @@(assoc-in [:target 2 :stage-number] -2)))parameters))-(defn query-for-card+(mu/defn query-for-card :- [:maybe ::lib.schema/query]"Generate a query for a saved Card"[{dataset-query :dataset_querycard-type :type- :as card} parameters constraints middleware & [ids]]- (let [stage-numbers (explict-stage-references parameters)- explicit-stage-numbers? (boolean (seq stage-numbers))- parameters (cond-> parameters- ;; models are not transparent (questions and metrics are)- (and explicit-stage-numbers? (= card-type :model))- point-parameters-to-last-stage)- ;; The FE might have "added" a stage so that a question with breakouts- ;; at the last stage can be filtered on the summary results. We know- ;; this happened if we get a reference to one above the last stage.- filter-stage-added? (and explicit-stage-numbers?- (= (inc (last-stage-number dataset-query))- (apply max stage-numbers)))- query (cond-> dataset-query- (and explicit-stage-numbers?- (or- ;; stage-number 0 means filtering the results of models and metrics- (not= card-type :question)- ;; the FE assumed an extra stage, so we add it- filter-stage-added?))- nest-query)- query (-> query- ;; don't want default constraints overridding anything that's already there- (m/dissoc-in [:middleware :add-default-userland-constraints?])- (assoc :constraints constraints- :parameters (cond-> parameters- filter-stage-added? add-stage-to-temporal-unit-parameters)- :middleware middleware))- cs (-> (cache-strategy card (:dashboard-id ids))- (enrich-strategy query))]- (assoc query :cache-strategy cs)))+ :as card} :- [:map+ [:dataset_query [:or+ [:= {:description "empty map"} {}]+ ::lib.schema/query]]]+ parameters :- [:maybe [:sequential :map]]+ constraints :- [:maybe :map]+ middleware :- [:maybe :map]+ & [ids]]+ (when (seq dataset-query)+ (let [stage-numbers (explict-stage-references parameters)+ explicit-stage-numbers? (boolean (seq stage-numbers))+ parameters (cond-> parameters+ ;; models are not transparent (questions and metrics are)+ (and explicit-stage-numbers? (= card-type :model))+ point-parameters-to-last-stage)+ ;; The FE might have "added" a stage so that a question with breakouts+ ;; at the last stage can be filtered on the summary results. We know+ ;; this happened if we get a reference to one above the last stage.+ filter-stage-added? (and explicit-stage-numbers?+ (= (inc (last-stage-number dataset-query))+ (apply max stage-numbers)))+ query (cond-> dataset-query+ (and explicit-stage-numbers?+ (or+ ;; stage-number 0 means filtering the results of models and metrics+ (not= card-type :question)+ ;; the FE assumed an extra stage, so we add it+ filter-stage-added?))+ lib/append-stage)+ query (-> query+ ;; don't want default constraints overridding anything that's already there+ (m/dissoc-in [:middleware :add-default-userland-constraints?])+ (m/assoc-some :constraints (not-empty constraints)+ :parameters (not-empty (cond-> parameters+ filter-stage-added? add-stage-to-temporal-unit-parameters))+ :middleware (not-empty middleware)))+ cs (-> (cache-strategy card (:dashboard-id ids))+ (enrich-strategy query))]+ (-> query+ (assoc :cache-strategy cs)+ (->> (lib/normalize ::lib.schema/query))))))(def ^:dynamic *allow-arbitrary-mbql-parameters*"In 0.41.0+ you can no longer add arbitrary `:parameters` to a query for a saved question -- only parameters for@@ -142,7 +149,7 @@here."false)-(defn- card-template-tag-parameters+(mu/defn- card-template-tag-parameters"Template tag parameters that have been specified for the query for Card with `card-id`, if any, returned as a map inthe format@@ -154,12 +161,11 @@Parameter type in this case is something like `:string` or `:number` or `:date/month-year`; parameters passed in asparameters to the API request must be allowed for this type (i.e. `:string/=` is allowed for a `:string` parameter,but `:number/=` is not)."- [card-id]+ [card-id :- ::lib.schema.id/card](let [query (api/check-404 (t2/select-one-fn :dataset_query :model/Card :id card-id))](into{}- (comp- (map (fn [[param-name {widget-type :widget-type, tag-type :type}]]+ (keep (fn [[param-name {widget-type :widget-type, tag-type :type}]];; Field Filter parameters have a `:type` of `:dimension` and the widget type that should be used is;; specified by `:widget-type`. Non-Field-filter parameters just have `:type`. So prefer;; `:widget-type` if available but fall back to `:type` if not.@@ -171,8 +177,7 @@(or (contains? lib.schema.template-tag/raw-value-template-tag-types tag-type)(= tag-type :temporal-unit))[param-name tag-type])))- (filter some?))- (get-in query [:native :template-tags]))))+ (lib/all-template-tags-map query))))(defn- allowed-parameter-type-for-template-tag-widget-type? [parameter-type widget-type](when-let [allowed-template-tag-types (get-in lib.schema.parameter/types [parameter-type :allowed-for])]@@ -250,7 +255,7 @@(qp.streaming/streaming-response [rff export-format (qp.streaming/safe-filename-prefix (:card-name info))](qp (update query :info merge info) rff))))-(defn combined-parameters-and-template-tags+(mu/defn combined-parameters-and-template-tags"Enrich `card.parameters` to include parameters from template-tags.On native queries parameters exists in 2 forms:@@ -261,7 +266,7 @@However, since card.parameters is a recently added feature, there may be instances where a template-tagis not present in the parameters.This function ensures that all template-tags are converted to parameters and added to card.parameters."- [{:keys [parameters] :as card}]+ [{:keys [parameters] :as card} :- ::queries.schema/card](let [template-tag-parameters (queries/card-template-tag-parameters card)id->template-tags-parameter (m/index-by :id template-tag-parameters)id->parameter (m/index-by :id parameters)]@@ -295,8 +300,8 @@`StreamingResponse`.`context` is a keyword describing the situation in which this query is being ran, e.g. `:question` (from a Saved- Question) or `:dashboard` (from a Saved Question in a Dashboard). See [[metabase.legacy-mbql.schema/Context]] for all valid- options."+ Question) or `:dashboard` (from a Saved Question in a Dashboard). See [[metabase.legacy-mbql.schema/Context]] for+ all valid options."[card-id :- ::lib.schema.id/cardexport-format& {:keys [parameters constraints context dashboard-id dashcard-id middleware qp make-run ignore-cache]@@ -305,7 +310,7 @@;; param `make-run` can be used to control how the query is ran, e.g. if you need to customize the `context`;; passed to the QPmake-run process-query-for-card-default-run-fn}}]- {:pre [(int? card-id) (u/maybe? sequential? parameters)]}+ {:pre [(pos-int? card-id) (u/maybe? sequential? parameters)]}(let [card (api/read-check (t2/select-one [:model/Card :id :name :dataset_query :database_id :collection_id:type :result_metadata :visualization_settings :display:cache_invalidated_at :entity_id :created_at :card_schema@@ -337,7 +342,7 @@(and (= (:type card) :model) (seq (:result_metadata card)))(assoc :metadata/model-metadata (:result_metadata card)))](when (seq parameters)- (validate-card-parameters card-id (mbql.normalize/normalize-fragment [:parameters] parameters)))+ (validate-card-parameters card-id (lib/normalize ::lib.schema.parameter/parameters parameters)))(log/tracef "Running query for Card %d:\n%s" card-id(u/pprint-to-str query))(binding [qp.perms/*card-id* card-id]diff --git a/src/metabase/query_processor/compile.clj b/src/metabase/query_processor/compile.cljindex 7c69aec1c1b..6dbbb67dc42 100644--- a/src/metabase/query_processor/compile.clj+++ b/src/metabase/query_processor/compile.clj@@ -28,7 +28,7 @@[:params {:optional true} [:maybe [:sequential {:max 0} :any]]]])(mr/def ::query-with-compiled-query- "A legacy query that also has a compiled native query attached (unless it was already a native-only query in the first+ "An MBQL 5 query that also has a compiled native query attached (unless it was already a native-only query in the firstplace)."[:merge[:ref ::lib.schema/query]@@ -38,17 +38,10 @@;; into the existing query. Otherwise we should include this.[:qp/compiled-inline {:optional true} ::compiled-with-inlined-parameters]]])-(defn- native-only?- "Whether the query is native-only and thus does not need further compiling; true if the only stage is native. Native- stages can only be the first stage, so if the last stage is native it means we have only one stage and this query is- native-only."- [query]- (= (:lib/type (lib/query-stage query -1)) :mbql.stage/native))-(mu/defn- compile* :- ::compiled[query :- ::lib.schema/query](assert (not (:qp/compiled query)) "This query has already been compiled!")- (if (native-only? query)+ (if (lib/native-only-query? query)(set/rename-keys (lib/query-stage query -1) {:native :query})(driver/mbql->native driver/*driver* (lib/->legacy-MBQL query))))diff --git a/src/metabase/query_processor/dashboard.clj b/src/metabase/query_processor/dashboard.cljindex 367937f1014..96183f83c86 100644--- a/src/metabase/query_processor/dashboard.clj+++ b/src/metabase/query_processor/dashboard.clj@@ -5,11 +5,9 @@[clojure.string :as str][medley.core :as m][metabase.api.common :as api]+ [metabase.dashboards.schema :as dashboards.schema][metabase.events.core :as events]- ;; legacy usage -- don't use Legacy MBQL utils in QP code going forward, prefer Lib. This will be updated to use- ;; Lib soon- ^{:clj-kondo/ignore [:discouraged-namespace]}- [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.card :as qp.card][metabase.query-processor.error-type :as qp.error-type]@@ -20,7 +18,7 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [some select-keys]]+ [metabase.util.performance :refer [select-keys some]][steffan-westcott.clj-otel.api.trace.span :as span]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2]))@@ -127,7 +125,7 @@dashcard-id :- ::lib.schema.id/dashcardrequest-params :- [:maybe [:sequential :map]]](log/tracef "Resolving Dashboard %d Card %d query request parameters" dashboard-id card-id)- (let [request-params (mbql.normalize/normalize-fragment [:parameters] request-params)+ (let [request-params (some-> request-params not-empty (->> (lib/normalize ::dashboards.schema/parameters)))dashboard (-> (t2/select-one :model/Dashboard :id dashboard-id)(t2/hydrate :resolved-params)(api/check-404))diff --git a/src/metabase/query_processor/metadata.clj b/src/metabase/query_processor/metadata.cljindex b546c7e7d9b..f23db67639b 100644--- a/src/metabase/query_processor/metadata.clj+++ b/src/metabase/query_processor/metadata.clj@@ -14,7 +14,7 @@[metabase.legacy-mbql.normalize :as mbql.normalize]^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.schema :as mbql.s]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -132,7 +132,7 @@(mapv(fn [col](-> col- (lib.metadata.jvm/instance->metadata :metadata/column)+ (lib-be/instance->metadata :metadata/column)(add-extra-column-metadata ::mlv2)))(result-metadata* query current-user-id))))diff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.cljindex e2da2db1229..09a1db8a964 100644--- a/src/metabase/query_processor/middleware/add_remaps.clj+++ b/src/metabase/query_processor/middleware/add_remaps.clj@@ -29,7 +29,7 @@(:require[clojure.data :as data][medley.core :as m]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]@@ -43,7 +43,7 @@[metabase.lib.walk :as lib.walk][metabase.query-processor.middleware.large-int :as large-int][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]@@ -608,7 +608,7 @@rff(fn remap-results-rff* [metadata](let [mlv2-cols (map- #(lib.metadata.jvm/instance->metadata % :metadata/column)+ #(lib-be/instance->metadata % :metadata/column)(:cols metadata))internal-cols-info (internal-columns-info mlv2-cols)metadata (add-remapped-to-and-from-metadata metadata external-remaps internal-cols-info)]diff --git a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.cljindex fdc657c7522..5ead9ac39cd 100644--- a/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj+++ b/src/metabase/query_processor/middleware/annotate/legacy_helper_fns.clj@@ -13,6 +13,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.query-processor.error-type :as qp.error-type]+ ^{:clj-kondo/ignore [:deprecated-namespace]}[metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as-alias add][metabase.util.i18n :refer [tru]]@@ -20,6 +21,7 @@(mu/defn legacy-inner-query->mlv2-query :- ::lib.schema/query"Convert a legacy `inner-query` to an MLv2 query. Requires bound QP store."+ {:deprecated "0.57.0"}[inner-query :- [:and:map[:fn@@ -49,6 +51,7 @@(or (::add/desired-alias (lib/options ag-clause))(:name (lib/options ag-clause))(lib/column-name+ #_{:clj-kondo/ignore [:deprecated-var]}(legacy-inner-query->mlv2-query legacy-inner-query)(lib/->pMBQL legacy-ag-clause)))))diff --git a/src/metabase/query_processor/middleware/large_int.clj b/src/metabase/query_processor/middleware/large_int.cljindex 35fbd6829c7..eac03084ed4 100644--- a/src/metabase/query_processor/middleware/large_int.clj+++ b/src/metabase/query_processor/middleware/large_int.clj@@ -2,7 +2,7 @@"Middleware for handling conversion of integers to strings for proper display of large numbers"(:refer-clojure :exclude [mapv])(:require- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util.performance :as perf :refer [mapv]])(:import(clojure.lang BigInt)diff --git a/src/metabase/query_processor/middleware/limit.clj b/src/metabase/query_processor/middleware/limit.cljindex 9cb0017257f..cf89c11d558 100644--- a/src/metabase/query_processor/middleware/limit.clj+++ b/src/metabase/query_processor/middleware/limit.clj@@ -5,7 +5,7 @@[metabase.lib.schema :as lib.schema][metabase.query-processor.schema :as qp.schema][metabase.query-processor.settings :as qp.settings]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.malli :as mu][potemkin :as p]))diff --git a/src/metabase/query_processor/middleware/normalize_query.clj b/src/metabase/query_processor/middleware/normalize_query.cljindex 75e8707ada2..cbccb9d498d 100644--- a/src/metabase/query_processor/middleware/normalize_query.clj+++ b/src/metabase/query_processor/middleware/normalize_query.clj@@ -6,7 +6,7 @@[metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.error-type :as qp.error-type]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]))diff --git a/src/metabase/query_processor/middleware/parameters.clj b/src/metabase/query_processor/middleware/parameters.cljindex 2503bc8f1a1..4fc677c9026 100644--- a/src/metabase/query_processor/middleware/parameters.clj+++ b/src/metabase/query_processor/middleware/parameters.clj@@ -73,10 +73,16 @@(reduce(fn [query parameter](let [param-stage-number (parameter->stage-number query parameter)]- (lib/update-query-stage query param-stage-number- update :parameters- (fn [parameters]- (conj (vec parameters) parameter)))))+ (if (>= param-stage-number (count (:stages query)))+ (do+ (log/errorf "Query does not have a stage %d, ignoring parameter %s"+ param-stage-number+ (pr-str parameter))+ query)+ (lib/update-query-stage query param-stage-number+ update :parameters+ (fn [parameters]+ (conj (vec parameters) parameter))))))queryparameters))diff --git a/src/metabase/query_processor/middleware/parameters/native.clj b/src/metabase/query_processor/middleware/parameters/native.cljindex 58654928f41..4ce565ea151 100644--- a/src/metabase/query_processor/middleware/parameters/native.clj+++ b/src/metabase/query_processor/middleware/parameters/native.clj@@ -35,7 +35,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.metadata :as lib.schema.metadata]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util.malli :as mu][metabase.util.performance :refer [mapv]]))@@ -45,8 +45,10 @@(m/update-existing :parameters (fn [parameters](mapv lib/->legacy-MBQL parameters)))(m/update-existing :template-tags update-vals lib/->legacy-MBQL)+ (dissoc :lib/type)(->> (driver/substitute-native-parameters driver/*driver*))- (set/rename-keys {:query :native})))+ (set/rename-keys {:query :native})+ (assoc :lib/type (:lib/type stage))))(mu/defn expand-stage :- ::lib.schema/stage.native"Expand parameters inside an *inner* native `query`. Not recursive -- recursive transformations are handled in@@ -55,8 +57,10 @@stage :- ::lib.schema/stage.native](if-not (driver.u/supports? driver/*driver* :native-parameters (lib.metadata/database metadata-providerable))stage- (let [substituted-stage (qp.store/with-metadata-provider (lib.metadata/->metadata-provider metadata-providerable)- (substitute-native-parameters* stage))]+ (let [substituted-stage (if (qp.store/initialized?)+ (substitute-native-parameters* stage)+ (qp.store/with-metadata-provider (lib.metadata/->metadata-provider metadata-providerable)+ (substitute-native-parameters* stage)))](->substituted-stage(dissoc :parameters :template-tags)))))diff --git a/src/metabase/query_processor/middleware/permissions.clj b/src/metabase/query_processor/middleware/permissions.cljindex b559dca7e13..d27216b3294 100644--- a/src/metabase/query_processor/middleware/permissions.clj+++ b/src/metabase/query_processor/middleware/permissions.clj@@ -6,13 +6,13 @@[metabase.audit-app.core :as audit][metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols]- [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema :as lib.schema][metabase.lib.walk :as lib.walk][metabase.permissions.core :as perms][metabase.premium-features.core :refer [defenterprise]][metabase.query-permissions.core :as query-perms][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]))@@ -157,23 +157,23 @@;;; +----------------------------------------------------------------------------------------------------------------+(mu/defn check-query-action-permissions*- "Check that User with `user-id` has permissions to run query action `query`, or throw an exception."- [{database-id :database, :as outer-query} :- [:map- [:database ::lib.schema.id/database]- [:type [:enum :query :native]]]]+ "Check that User with `user-id` has permissions to run query action `query`, or throw an exception. Takes+ as [[metabase.actions.args/action-arg-map-schema]]."+ [{database-id :database, :as query} :- ::lib.schema/query](log/tracef "Checking query permissions. Current user perms set = %s" (pr-str @*current-user-permissions-set*))(when *card-id*(query-perms/check-card-read-perms database-id *card-id*))(when-not (query-perms/check-data-perms- outer-query- (query-perms/required-perms-for-query outer-query :already-preprocessed? true)+ query+ (query-perms/required-perms-for-query query :already-preprocessed? true):throw-exceptions? false)- (check-block-permissions outer-query)))+ (check-block-permissions query)))-(defn check-query-action-permissions+(mu/defn check-query-action-permissions :- ::qp.schema/qp"Middleware that check that the current user has permissions to run the current query action."- [qp]- (fn [query rff]+ [qp :- ::qp.schema/qp]+ (mu/fn [query :- ::lib.schema/native-only-query+ rff :- ::qp.schema/rff](check-query-action-permissions* query)(qp query rff)))diff --git a/src/metabase/query_processor/middleware/resolve_joins.clj b/src/metabase/query_processor/middleware/resolve_joins.cljindex 9d26ca70e49..adf39c3e39d 100644--- a/src/metabase/query_processor/middleware/resolve_joins.clj+++ b/src/metabase/query_processor/middleware/resolve_joins.clj@@ -34,8 +34,6 @@;; bucketed in different ways in previous stages; joins thus ought to be using field name refs; but this ends up;; breaking a ton of stuff, especially `lib.equality`... #63109 was my attempt to make this stuff work when using;; field name refs for joins but it's a long way off from landing.- ;;- ;; NOCOMMIT(for [{field-id :id, :as col} cols:let [[_tag opts id-or-name, :as field-ref] (lib/ref col)force-id-ref? (and (string? id-or-name)diff --git a/src/metabase/query_processor/middleware/results_metadata.clj b/src/metabase/query_processor/middleware/results_metadata.cljindex 5c454ae0628..0540a1dfdf4 100644--- a/src/metabase/query_processor/middleware/results_metadata.clj+++ b/src/metabase/query_processor/middleware/results_metadata.clj@@ -14,7 +14,7 @@[metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]diff --git a/src/metabase/query_processor/middleware/update_used_cards.clj b/src/metabase/query_processor/middleware/update_used_cards.cljindex 9d95347ffb0..c2db6a30658 100644--- a/src/metabase/query_processor/middleware/update_used_cards.clj+++ b/src/metabase/query_processor/middleware/update_used_cards.clj@@ -7,7 +7,7 @@[metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util.log :as log][metabase.util.malli :as mu]^{:clj-kondo/ignore [:discouraged-namespace]}diff --git a/src/metabase/query_processor/middleware/wrap_value_literals.clj b/src/metabase/query_processor/middleware/wrap_value_literals.cljindex 91a088cca2b..2327b7060a1 100644--- a/src/metabase/query_processor/middleware/wrap_value_literals.clj+++ b/src/metabase/query_processor/middleware/wrap_value_literals.clj@@ -14,7 +14,7 @@[metabase.lib.util.match :as lib.util.match][metabase.lib.walk :as lib.walk][metabase.query-processor.error-type :as qp.error-type]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.util.date-2 :as u.date][metabase.util.i18n :as i18n]diff --git a/src/metabase/query_processor/parameters/dates.clj b/src/metabase/query_processor/parameters/dates.cljindex 829abcdf843..b00b1bb63d8 100644--- a/src/metabase/query_processor/parameters/dates.clj+++ b/src/metabase/query_processor/parameters/dates.clj@@ -1,5 +1,8 @@(ns metabase.query-processor.parameters.dates- "Shared code for handling datetime parameters, used by both MBQL and native params implementations."+ "Shared code for handling datetime parameters, used by both MBQL and native params implementations.++ TODO -- move this into the `lib-be` module since it's not really QP-specific, it's something that would live in Lib+ if it didn't have dependencies on [[metabase.util.date-2]]."(:refer-clojure :exclude [every? some])(:require[clojure.string :as str]diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex 1ddf4bd2c12..e72406dfc62 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -6,7 +6,7 @@(:refer-clojure :exclude [every? mapv some select-keys update-keys])(:require[medley.core :as m]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.equality :as lib.equality][metabase.lib.schema :as lib.schema]@@ -24,7 +24,7 @@[metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema][metabase.query-processor.setup :as qp.setup]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]@@ -383,7 +383,7 @@show-row-totals (get viz-settings :pivot.show_row_totals true)show-column-totals (get viz-settings :pivot.show_column_totals true)metadata-provider (or (:lib/metadata query)- (lib.metadata.jvm/application-database-metadata-provider (:database query)))+ (lib-be/application-database-metadata-provider (:database query)))query (lib/query metadata-provider query)unique-name-fn (lib.util/unique-name-generator)returned-columns (->> (lib/returned-columns query)@@ -412,7 +412,7 @@[:database ::lib.schema.id/database]]viz-settings :- [:maybe :map]](let [metadata-provider (or (:lib/metadata query)- (lib.metadata.jvm/application-database-metadata-provider (:database query)))+ (lib-be/application-database-metadata-provider (:database query)))query (lib/query metadata-provider query)index-in-breakouts (into {}(comp (filter (some-fn :lib/breakout? #(= (:lib/source %) :source/aggregations)))@@ -436,7 +436,7 @@show-row-totals (get viz-settings "pivot.show_row_totals" true)show-column-totals (get viz-settings "pivot.show_column_totals" true)metadata-provider (or (:lib/metadata query)- (lib.metadata.jvm/application-database-metadata-provider (:database query)))+ (lib-be/application-database-metadata-provider (:database query)))mlv2-query (lib/query metadata-provider query)breakouts (into [](map-indexed (fn [i col]diff --git a/src/metabase/query_processor/setup.clj b/src/metabase/query_processor/setup.cljindex e600f8b1566..f8ddeb34e54 100644--- a/src/metabase/query_processor/setup.clj+++ b/src/metabase/query_processor/setup.clj@@ -3,6 +3,7 @@[clojure.core.async :as a][clojure.core.async.impl.dispatch :as a.impl.dispatch][clojure.set :as set]+ [metabase.config.core :as config][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.lib.metadata :as lib.metadata]@@ -13,9 +14,10 @@[metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.settings.core :as setting][metabase.util.i18n :as i18n]+ [metabase.util.log :as log][metabase.util.malli :as mu]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2]))@@ -26,7 +28,7 @@(throw (ex-info (i18n/tru "Invalid query: missing or invalid query type (:lib/type or :type)"){:query query, :type qp.error-type/invalid-query}))))-(mu/defn- source-card-id-for-pmbql-query :- [:maybe ::lib.schema.id/card]+(mu/defn- source-card-id-for-mbql5-query :- [:maybe ::lib.schema.id/card][query :- ::qp.schema/any-query](-> query :stages first :source-card))@@ -87,7 +89,7 @@[query :- ::qp.schema/any-query](case (query-type query):mbql/query- (source-card-id-for-pmbql-query query)+ (source-card-id-for-mbql5-query query)(:query :native)(source-card-id-for-legacy-query query)@@ -149,7 +151,17 @@(fn [query](conddriver/*driver*- (f query)+ (do+ ;; dev mode: check that we're not doing something crazy like binding `driver/*driver*` to `:h2` and the+ ;; running tests against the Audit App DB with a Postgres app DB. (This was a real test I had to fix.)+ (when (or config/is-dev? config/is-test?)+ (let [expected-driver (driver.u/database->driver (:database query))]+ (when-not (= driver/*driver* expected-driver)+ (log/warnf "driver/*driver* is bound to %s, but Database %d has engine %s. Query may not work as expected."+ driver/*driver*+ (:database query)+ expected-driver))))+ (f query))(= (query-type query) :internal)(f query)diff --git a/src/metabase/query_processor/store.clj b/src/metabase/query_processor/store.cljindex 9ed07b2eece..b058becfe68 100644--- a/src/metabase/query_processor/store.clj+++ b/src/metabase/query_processor/store.clj@@ -11,11 +11,14 @@Of course, it would be entirely possible to call `(t2/select-one Field :id 10)` every time you needed informationabout that Field, but fetching all Fields in a single pass and storing them for reuse is dramatically more efficient- than fetching those Fields potentially dozens of times in a single query execution."+ than fetching those Fields potentially dozens of times in a single query execution.++ THE QP STORE IS DEPRECATED! It's only needed for legacy queries, and we're moving to an MBQL 5 world. Don't use it+ in new code going forward.";; This whole namespace is in the process of deprecation so ignore deprecated vars in this namespace.- {:clj-kondo/config '{:linters {:deprecated-var {:level :off}}}}+ {:clj-kondo/config '{:linters {:deprecated-var {:level :off}}}, :deprecated "0.57.0"}(:require- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.cached-provider :as lib.metadata.cached-provider]@@ -136,7 +139,7 @@(mu/defn- ->metadata-provider :- ::lib.schema.metadata/metadata-provider[database-id-or-metadata-providerable :- ::database-id-or-metadata-providerable](let [mp (if (pos-int? database-id-or-metadata-providerable)- (lib.metadata.jvm/application-database-metadata-provider database-id-or-metadata-providerable)+ (lib-be/application-database-metadata-provider database-id-or-metadata-providerable)(lib.metadata/->metadata-provider database-id-or-metadata-providerable))](ensure-cached-metadata-provider mp)))diff --git a/src/metabase/query_processor/streaming/common.clj b/src/metabase/query_processor/streaming/common.cljindex 13d31ea3df3..63acf11072a 100644--- a/src/metabase/query_processor/streaming/common.clj+++ b/src/metabase/query_processor/streaming/common.clj@@ -7,7 +7,7 @@[metabase.appearance.core :as appearance][metabase.driver :as driver][metabase.models.visualization-settings :as mb.viz]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.util.currency :as currency][metabase.util.date-2 :as u.date]diff --git a/src/metabase/query_processor/timezone.clj b/src/metabase/query_processor/timezone.cljindex dadecdc9fc4..b7964959c1b 100644--- a/src/metabase/query_processor/timezone.clj+++ b/src/metabase/query_processor/timezone.clj@@ -6,8 +6,11 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.lib.metadata :as lib.metadata]- [metabase.query-processor.store :as qp.store]- [metabase.util.log :as log])+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.util.malli.schema :as ms])(:import(java.time ZonedDateTime)))@@ -38,15 +41,23 @@;;; | Public Interface |;;; +----------------------------------------------------------------------------------------------------------------+-(defn report-timezone-id-if-supported+(mu/defn report-timezone-id-if-supported"Timezone ID for the report timezone, if the current driver and database supports it. (If the current driver supports it, this isbound by the `bind-effective-timezone` middleware.)"(^String [](report-timezone-id-if-supported driver/*driver* (lib.metadata/database (qp.store/metadata-provider))))- (^String [driver database]- (when (driver.u/supports? driver :set-timezone database)- (valid-timezone-id (report-timezone-id*)))))+ (^String [driver :- :keyword+ database :- [:or+ ::lib.schema.metadata/database+ (ms/InstanceOf :model/Database)+ [:= ::db-from-store]]]+ (when-let [database (if (= database ::db-from-store)+ (when (qp.store/initialized?)+ (lib.metadata/database (qp.store/metadata-provider)))+ database)]+ (when (driver.u/supports? driver :set-timezone database)+ (valid-timezone-id (report-timezone-id*))))))(defn database-timezone-id"The timezone that the current database is in, as determined by the most recent sync."@@ -57,7 +68,8 @@(valid-timezone-id(or *database-timezone-id-override*(let [database (if (= database ::db-from-store)- (lib.metadata/database (qp.store/metadata-provider))+ (when (qp.store/initialized?)+ (lib.metadata/database (qp.store/metadata-provider)))database)](:timezone database))))))diff --git a/src/metabase/query_processor/util.clj b/src/metabase/query_processor/util.cljindex 4c5a59e34d0..87a562cff05 100644--- a/src/metabase/query_processor/util.clj+++ b/src/metabase/query_processor/util.clj@@ -3,7 +3,6 @@(:refer-clojure :exclude [select-keys])(:require[buddy.core.codecs :as codecs]- [buddy.core.hash :as buddy-hash][clojure.string :as str][medley.core :as m][metabase.driver :as driver]@@ -11,19 +10,22 @@;; Lib only soon^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core][metabase.lib.core :as lib]- [metabase.lib.schema.expression :as lib.schema.expression]- [metabase.lib.schema.util :as lib.schema.util][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store][metabase.util :as u]- [metabase.util.json :as json][metabase.util.malli :as mu]- [metabase.util.performance :as perf :refer [select-keys]]))+ [metabase.util.performance :refer [select-keys]]+ [potemkin :as p]))(set! *warn-on-reflection* true)+(comment metabase.lib-be.core/keep-me)++(p/import-vars+ [metabase.lib-be.core+ query-hash])+;; TODO - I think most of the functions in this namespace that we don't remove could be moved;; to [[metabase.legacy-mbql.util]]@@ -65,8 +67,6 @@[_ query](default-query->remark query))-;;; ------------------------------------------------- Normalization ---------------------------------------------------;; TODO - this has been moved to `metabase.legacy-mbql.util`; use that implementation instead.(mu/defn ^:deprecated normalize-token :- :keyword"Convert a string or keyword in various cases (`lisp-case`, `snake_case`, or `SCREAMING_SNAKE_CASE`) to a lisp-cased@@ -77,83 +77,6 @@(str/replace #"_" "-")keyword))-;;; ---------------------------------------------------- Hashing -------------------------------------------------------(defn- walk-query-sort-maps- "We don't want two queries to have different hashes because their map keys are in different orders, now do we? Convert- all the maps to sorted maps so queries are serialized to JSON in an identical order."- [x]- (perf/postwalk- (fn [x]- (if (and (map? x)- (not (sorted? x)))- (into (sorted-map) x)- x))- x))--(mu/defn- sort-parameter-values- "Return the sequence of parameter maps, but with any :value keys sorted if they are a sequence. Parameter values can- be of mixed types, as bigintegers are passed as strings to avoid precision loss."- [params :- [:or :nil [:sequential :any]]]- (map #(if (sequential? (:value %)) (update % :value (partial sort-by str)) %) params))--(mu/defn- select-keys-for-hashing- "Return `query` with only the keys relevant to hashing kept.- (This is done so irrelevant info or options that don't affect query results doesn't result in the same query- producing different hashes.)"- [query :- [:maybe :map]]- (let [{:keys [constraints parameters], :as query} (select-keys query [:database- :lib/type- :stages- :parameters- :constraints- :destination-database/id- :impersonation/role])]- (-> query- (cond-> (empty? constraints) (dissoc :constraints))- (update :parameters sort-parameter-values)- (cond-> (empty? parameters) (dissoc :parameters))- lib.schema.util/indexed-order-bys- lib.schema.util/remove-lib-uuids- walk-query-sort-maps)))--(defn- ->metadata-provider [legacy-query]- (if (qp.store/initialized?)- (qp.store/metadata-provider)- (lib.metadata.jvm/application-database-metadata-provider (:database legacy-query))))--(mu/defn query-hash :- bytes?- "Return a 256-bit SHA3 hash of `query` as a key for the cache. (This is returned as a byte array.)"- ^bytes [query :- [:maybe :map]]- ;; convert to pMBQL first if this is a legacy query.- (let [query (try- ;; Expression type check supression is necessary because coerced fields in `query` may not have- ;; `:effective-type` populated. That's the case during call to this function in- ;; `process-userland-query-middleware` that occurs before normalization.- ;; TODO: This is an unfortunate leak of lib internals but I can't see a clean way to fix it.- (binding [lib.schema.expression/*suppress-expression-type-check?* true]- (case (:type query)- ;; TODO (Cam 8/20/25) -- [[lib/query]] is supposed to call [[mbql.normalize/normalize]] on legacy- ;; queries automatically anyway but it seems like some tests break if I change this... investigate- ("query" "native") (lib/query (->metadata-provider query) (mbql.normalize/normalize query))- (:query :native) (lib/query (->metadata-provider query) query)- query))- (catch Throwable e- (throw (ex-info "Error hashing query. Is this a valid query?"- {:query query}- e))))]- (buddy-hash/sha3-256 (json/encode (select-keys-for-hashing query)))))--;;; ------------------------------------------- Metadata Combination Utils ----------------------------------------------(defn field-ref->key- "A standard and repeatable way to address a column. Names can collide and sometimes are not unique. Field refs should- be stable, except we have to exclude the last part as extra information can be tucked in there. Names can be- non-unique at times, numeric ids are not guaranteed."- {:deprecated "0.57.0"}- [[tyype identifier]]- [tyype identifier])-(def ^:private ^{:deprecated "0.57.0"} field-options-for-identification"Set of FieldOptions that only mattered for identification purposes." ;; base-type is required for field that use name instead of id#{:source-field :join-alias})diff --git a/src/metabase/query_processor/util/add_alias_info.clj b/src/metabase/query_processor/util/add_alias_info.cljindex ab0359c8226..8df578546c5 100644--- a/src/metabase/query_processor/util/add_alias_info.clj+++ b/src/metabase/query_processor/util/add_alias_info.clj@@ -56,7 +56,7 @@[metabase.lib.walk :as lib.walk][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]@@ -529,6 +529,7 @@;; MBQL 4 inner MBQL query((some-fn :source-table :source-query) query)(-> query+ #_{:clj-kondo/ignore [:deprecated-var]}annotate.legacy-helper-fns/legacy-inner-query->mlv2-query(add-alias-info options)lib/->legacy-MBQLdiff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.cljindex fd9856b9c99..8ecfe892be1 100644--- a/src/metabase/query_processor/util/nest_query.clj+++ b/src/metabase/query_processor/util/nest_query.clj@@ -20,7 +20,7 @@[metabase.lib.schema.common :as lib.schema.common][metabase.lib.util.match :as lib.util.match][metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as add][metabase.util :as u][metabase.util.log :as log]@@ -231,6 +231,7 @@[inner-query :- :mapexpression :- [:maybe ::mbql.s/FieldOrExpressionDef]](when expression+ #_{:clj-kondo/ignore [:deprecated-var]}(let [mlv2-query (annotate.legacy-helper-fns/legacy-inner-query->mlv2-query inner-query)](lib/type-of mlv2-query (lib/->pMBQL expression)))))diff --git a/src/metabase/query_processor/writeback.clj b/src/metabase/query_processor/writeback.cljindex c7853f22bf5..ee34aee2a2a 100644--- a/src/metabase/query_processor/writeback.clj+++ b/src/metabase/query_processor/writeback.clj@@ -2,7 +2,10 @@"Code for executing writeback queries."(:require[metabase.driver :as driver]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.query-processor :as qp][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.enterprise :as qp.enterprise]@@ -10,12 +13,11 @@[metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.setup :as qp.setup]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli :as mu]))(def ^:private execution-middleware"Middleware that happens after compilation, AROUND query execution itself. Has the form@@ -33,18 +35,11 @@qpmiddleware-fns))-(mr/def ::native-mbql-4-query- [:map- [:type [:= :native]]- [:native [:map- [:query :string]]]])--(mu/defn- substitute-params :- ::native-mbql-4-query- [query :- ::native-mbql-4-query]+(mu/defn- substitute-params :- ::lib.schema/native-only-query+ [query :- ::lib.schema/native-only-query](->> query(lib/query (qp.store/metadata-provider))- parameters/substitute-parameters- lib/->legacy-MBQL))+ parameters/substitute-parameters))(defn- writeback-qp [];; `rff` and `context` are not currently used by the writeback QP stuff, so these parameters can be ignored; we pass@@ -53,31 +48,32 @@(let [query (substitute-params query)];; ok, now execute the query.(log/debugf "Executing query\n\n%s" (u/pprint-to-str query))- (driver/execute-write-query! driver/*driver* query)))]+ (driver/execute-write-query! driver/*driver* (lib/->legacy-MBQL query))))](apply-middleware qp* (concat execution-middleware qp/around-middleware))))(mu/defn execute-write-query!"Execute an writeback query (which currently has to be an MBQL 4 native query) from an action."- [query :- ::native-mbql-4-query]+ [query :- ::lib.schema/native-only-query](qp.setup/with-qp-setup [query query]- (let [{query-type :type, :as query} (-> (qp.preprocess/preprocess query)- ;; TODO (Cam 8/19/25) -- update this code to use MBQL 5 / Lib- lib/->legacy-MBQL)]+ (let [query (qp.preprocess/preprocess query)];; make sure this is a native query.- (when-not (= query-type :native)+ (when-not (lib/native-only-query? query)(throw (ex-info (tru "Only native queries can be executed as write queries."){:type qp.error-type/invalid-query, :status-code 400, :query query})))((writeback-qp) query (constantly conj)))))-(defn execute-write-sql!+(mu/defn execute-write-sql!"Execute a write query in SQL against a database given by `db-id`."- [db-id sql-or-sql+params]- (if (sequential? sql-or-sql+params)- (let [[sql & params] sql-or-sql+params]- (execute-write-query! {:type :native- :database db-id- :native {:query sql- :params params}}))- (execute-write-query! {:type :native- :database db-id- :native {:query sql-or-sql+params}})))+ [db-id :- ::lib.schema.id/database+ sql-or-sql+params :- [:or+ :string+ [:cat+ :string+ [:* :any]]]]+ (let [mp (lib-be/application-database-metadata-provider db-id)+ [sql & params] (if (string? sql-or-sql+params)+ (cons sql-or-sql+params nil)+ sql-or-sql+params)+ query (cond-> (lib/native-query mp sql)+ (seq params) (lib/update-query-stage 0 assoc :params params))]+ (execute-write-query! query)))diff --git a/src/metabase/revisions/impl/card.clj b/src/metabase/revisions/impl/card.cljindex 1a8c7222041..cf8312772ad 100644--- a/src/metabase/revisions/impl/card.clj+++ b/src/metabase/revisions/impl/card.clj@@ -12,6 +12,7 @@:id:initially_published_at:last_used_at+ :legacy_query:made_public_by_id:metabase_version:public_uuiddiff --git a/src/metabase/search/config.clj b/src/metabase/search/config.cljindex 687ee3b9a3a..154e5a996b6 100644--- a/src/metabase/search/config.clj+++ b/src/metabase/search/config.clj@@ -2,10 +2,11 @@(:require[metabase.api.common :as api][metabase.config.core :as config]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.permissions.core :as perms][metabase.search.settings :as search.settings][metabase.util :as u]- [metabase.util.json :as json][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]))@@ -280,7 +281,7 @@(defmethod column->string [:card :dataset_query][value _ _]- (let [query (json/decode+kw value)]- (if (= "native" (:type query))- (-> query :native :query)- "")))+ (or (when-let [query (not-empty ((lib-be/transform-query :out) value))]+ (when (lib/native-only-query? query)+ (lib/raw-native-query query)))+ ""))diff --git a/src/metabase/search/filter.clj b/src/metabase/search/filter.cljindex cf4986c1665..4adc28c089e 100644--- a/src/metabase/search/filter.clj+++ b/src/metabase/search/filter.clj@@ -1,8 +1,8 @@(ns metabase.search.filter(:require[honey.sql.helpers :as sql.helpers]- [metabase.driver.common.parameters.dates :as params.dates][metabase.premium-features.core :as premium-features]+ [metabase.query-processor.parameters.dates :as qp.parameters.dates][metabase.search.config :as search.config][metabase.search.permissions :as search.permissions][metabase.search.spec :as search.spec]@@ -54,7 +54,7 @@(defn- date-range-filter-clause[dt-col dt-val](let [date-range (try- (params.dates/date-string->range dt-val {:inclusive-end? false})+ (qp.parameters.dates/date-string->range dt-val {:inclusive-end? false})(catch Exception _e(throw (ex-info (tru "Failed to parse datetime value: {0}" dt-val) {:status-code 400}))))start (some-> (:start date-range) u.date/parse)diff --git a/src/metabase/search/in_place/filter.clj b/src/metabase/search/in_place/filter.cljindex 90586ba19d6..7907e8dee1c 100644--- a/src/metabase/search/in_place/filter.clj+++ b/src/metabase/search/in_place/filter.clj@@ -14,11 +14,9 @@[clojure.string :as str][honey.sql.helpers :as sql.helpers][metabase.audit-app.core :as audit]- [metabase.driver.common.parameters.dates :as params.dates][metabase.premium-features.core :as premium-features]- [metabase.search.config- :as search.config- :refer [SearchableModel SearchContext]]+ [metabase.query-processor.parameters.dates :as params.dates]+ [metabase.search.config :as search.config :refer [SearchableModel SearchContext]][metabase.search.in-place.util :as search.util][metabase.search.permissions :as search.permissions][metabase.util.date-2 :as u.date]@@ -68,7 +66,7 @@(mu/defn- search-string-clause-for-model[model :- SearchableModelsearch-context :- SearchContext- search-native-query :- [:maybe true?]]+ search-native-query :- [:maybe :boolean]](when-let [query (:search-string search-context)](into[:or]diff --git a/src/metabase/segments/api.clj b/src/metabase/segments/api.cljindex 5a0f5ea2921..7a208259b93 100644--- a/src/metabase/segments/api.clj+++ b/src/metabase/segments/api.clj@@ -59,7 +59,8 @@clean-body (u/select-keys-when body:present #{:description :caveats :points_of_interest}:non-nil #{:archived :definition :name :show_in_getting_started})- new-def (->> clean-body :definition (mbql.normalize/normalize-fragment []))+ ;; TODO (Cam 10/1/25) -- update segments to persist MBQL 5+ new-def (->> clean-body :definition #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query]))new-body (merge(dissoc clean-body :revision_message)(when new-def {:definition new-def}))diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 4aced0ea630..7f3d9ba63d1 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -8,7 +8,7 @@[metabase.legacy-mbql.normalize :as mbql.normalize][metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.query :as lib.query]@@ -50,7 +50,8 @@"Segment `definition`s are just the inner MBQL query."[definition](when (seq definition)- (u/prog1 (mbql.normalize/normalize-fragment [:query] definition)+ ;; TODO (Cam 10/1/25) -- update segments to persist MBQL 5+ (u/prog1 #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query] definition)(validate-segment-definition <>))))(def ^:private transform-segment-definition@@ -110,9 +111,9 @@(mu/defn- warmed-metadata-provider :- ::lib.schema.metadata/metadata-provider[database-id :- ::lib.schema.id/databasesegments :- [:maybe [:sequential (ms/InstanceOf :model/Segment)]]]- (let [metadata-provider (doto (lib.metadata.jvm/application-database-metadata-provider database-id)+ (let [metadata-provider (doto (lib-be/application-database-metadata-provider database-id)(lib.metadata.protocols/store-metadatas!- (map #(lib.metadata.jvm/instance->metadata % :metadata/segment)+ (map #(lib-be/instance->metadata % :metadata/segment)segments)));; existing usage, do not use going forwardfield-ids #_{:clj-kondo/ignore [:deprecated-var]} (mbql.u/referenced-field-ids (map :definition segments))diff --git a/src/metabase/server/handler.clj b/src/metabase/server/handler.cljindex df568af7ae7..2e696d1be92 100644--- a/src/metabase/server/handler.clj+++ b/src/metabase/server/handler.clj@@ -9,6 +9,7 @@[metabase.server.middleware.exceptions :as mw.exceptions][metabase.server.middleware.json :as mw.json][metabase.server.middleware.log :as mw.log]+ [metabase.server.middleware.metadata-provider-cache :as mw.mp-cache][metabase.server.middleware.misc :as mw.misc][metabase.server.middleware.offset-paging :as mw.offset-paging][metabase.server.middleware.request-id :as mw.request-id]@@ -60,7 +61,12 @@(catch Exception _ nil)))(def ^:private middleware- ;; ▼▼▼ POST-PROCESSING ▼▼▼ happens from TOP-TO-BOTTOM+ "Ring async middleware has the form++ (defn middleware-fn [handler]+ (fn handler' [request respond raise]+ (handler request respond raise)))"+ ;; ▼▼▼ Middleware is APPLIED from TOP-TO-BOTTOM, but the returned `handlers` will see the requests in order from BOTTOM-TO-TOP. ▼▼▼(->> [#'mw.exceptions/catch-uncaught-exceptions ; catch any Exceptions that weren't passed to `raise`#'mw.exceptions/catch-api-exceptions ; catch exceptions and return them in our expected format#'mw.log/log-api-call ; log info about the request, db call counts etc.@@ -69,6 +75,7 @@#'mw.json/wrap-json-body ; extracts json POST/PUT body and makes it available on request#'mw.offset-paging/handle-paging ; binds per-request parameters to handle paging#'mw.json/wrap-streamed-json-response ; middleware to automatically serialize suitable objects as JSON in responses+ #'mw.mp-cache/wrap-metadata-provider-cache ; initializes the Lib-BE metadata provider cache#'wrap-keyword-params ; converts string keys in :params to keyword keys#'wrap-params ; parses GET and POST params as :query-params/:form-params and both as :params#'mw.misc/maybe-set-site-url ; set the value of `site-url` if it hasn't been set yet@@ -89,7 +96,6 @@wrap-reload-dev-mw ; reloads outdated clojure code when --hot flag is passed with the :dev-start alias](remove nil?)))-;; ▲▲▲ PRE-PROCESSING ▲▲▲ happens from BOTTOM-TO-TOP(mu/defn- apply-middleware :- ::api.macros/handler[handler :- ::api.macros/handler]@@ -104,7 +110,7 @@(mu/defn- dev-handler :- ::api.macros/handler[server-routes :- ::api.macros/handler](let [handler (atom (apply-middleware server-routes))]- (doseq [varr middleware+ (doseq [varr (cons #'middleware middleware):when (instance? clojure.lang.IRef varr)](add-watch varr ::reload (fn [_key _ref _old-state _new-state](log/infof "%s changed, rebuilding handler" varr)diff --git a/src/metabase/server/middleware/metadata_provider_cache.clj b/src/metabase/server/middleware/metadata_provider_cache.cljnew file mode 100644index 00000000000..ec44a7a2217--- /dev/null+++ b/src/metabase/server/middleware/metadata_provider_cache.clj@@ -0,0 +1,11 @@+(ns metabase.server.middleware.metadata-provider-cache+ (:require+ [metabase.lib-be.core :as lib-be]))++(defn wrap-metadata-provider-cache+ "Async Ring middleware that initializes a Lib-BE metadata provider cache for the duration of the request, so we can+ re-use Lib/QP metadata providers for the same database."+ [handler]+ (fn [request respond raise]+ (lib-be/with-metadata-provider-cache+ (handler request respond raise))))diff --git a/src/metabase/sync/analyze/classify.clj b/src/metabase/sync/analyze/classify.cljindex 2e5dcb07de6..7c9dce6a834 100644--- a/src/metabase/sync/analyze/classify.clj+++ b/src/metabase/sync/analyze/classify.clj@@ -23,7 +23,7 @@[metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.models.interface :as mi];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.analyze.fingerprint :as sync.fingerprint][metabase.sync.interface :as i][metabase.sync.util :as sync-util]diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.cljindex f6a21f0b67c..9f5aeedc276 100644--- a/src/metabase/sync/analyze/fingerprint.clj+++ b/src/metabase/sync/analyze/fingerprint.clj@@ -10,7 +10,7 @@[metabase.driver.util :as driver.u][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.interface :as i][metabase.sync.util :as sync-util][metabase.util :as u]diff --git a/src/metabase/tiles/api.clj b/src/metabase/tiles/api.cljindex 880ce2ef0ea..913a780c682 100644--- a/src/metabase/tiles/api.clj+++ b/src/metabase/tiles/api.clj@@ -1,18 +1,24 @@(ns metabase.tiles.api"`/api/tiles` endpoints."(:require- [clojure.set :as set][metabase.api.common :as api][metabase.api.macros :as api.macros][metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.legacy-mbql.util :as mbql.u]+ [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata]+ [metabase.parameters.schema :as parameters.schema][metabase.query-processor :as qp][metabase.query-processor.card :as qp.card][metabase.query-processor.dashboard :as qp.dashboard]- [metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms])(:import(java.awt Color)@@ -57,21 +63,23 @@(/ Math/PI 2)))]{:lat lat, :lon lon}))-(defn- query-with-inside-filter+(mu/defn- add-inside-filter :- ::lib.schema/query"Add an `INSIDE` filter to the given query to restrict results to a bounding box. The fields passed in can be eitherinteger field ids or string field names. When a field name, the `base-type` will be set to `:type/Float`."- [details lat-field lon-field x y zoom]+ [query :- ::lib.schema/query+ lat-field :- ::lib.schema.metadata/column+ lon-field :- ::lib.schema.metadata/column+ x y zoom](let [top-left (x+y+zoom->lat-lon x y zoom)bottom-right (x+y+zoom->lat-lon (inc x) (inc y) zoom)- inside-filter [:inside+ inside-filter (lib/insidelat-fieldlon-field(top-left :lat)(top-left :lon)(bottom-right :lat)- (bottom-right :lon)]]- #_{:clj-kondo/ignore [:deprecated-var]}- (update details :filter mbql.u/combine-filter-clauses inside-filter)))+ (bottom-right :lon))]+ (lib/filter query inside-filter)));;; --------------------------------------------------- RENDERING ----------------------------------------------------@@ -125,54 +133,73 @@(u/ignore-exceptions(.close output-stream))))))-(defn- native->source-query- "Adjust native queries to be an mbql from a source query so we can add the filter clause."- [query]- (if (contains? query :native)- (let [source-query (-> (:native query)- (set/rename-keys {:query :native})- (cond-> (:parameters query) (assoc :parameters (:parameters query))))]- {:database (:database query)- :type :query- :query {:source-query source-query}})- query))-;;; ---------------------------------------------------- ENDPOINTS -----------------------------------------------------(defn- tiles-query+;; TODO (Cam 9/30/25) -- we should update these endpoints to accept Field IDs and/or desired column aliases instead of+;; (or preferentially to) legacy refs+(mr/def ::legacy-ref+ "Form-encoded JSON-encoded legacy MBQL :field ref."+ [:schema+ {:decode/api (fn [field]+ (when (string? field)+ (let [deserialized (json/decode+kw field)]+ (when (sequential? deserialized)+ (mbql.normalize/normalize deserialized)))))}+ [:ref ::mbql.s/field]])++(mu/defn- resolve-field :- ::lib.schema.metadata/column+ [query :- ::lib.schema/query+ legacy-ref :- ::legacy-ref]+ (lib/metadata query (lib/->pMBQL legacy-ref)))++(mu/defn- tiles-query :- ::lib.schema/query"Transform a card's query into a query finding coordinates in a particular region.- transform native queries into nested mbql queries from that native query- add [:inside lat lon bounding-region coordings] filter- limit query results to `tile-coordinate-limit` number of results- only select lat and lon fields rather than entire query's fields"- [query zoom x y lat-field-ref lon-field-ref]- (let [query (mbql.normalize/normalize query)]+ [query :- :map+ zoom+ x+ y+ lat-field-legacy-ref :- ::legacy-ref+ lon-field-legacy-ref :- ::legacy-ref]+ (let [query (-> query+ lib-be/normalize-query+ lib/append-stage)+ lat-field (resolve-field query lat-field-legacy-ref)+ lon-field (resolve-field query lon-field-legacy-ref)](-> query- native->source-query- (update :query query-with-inside-filter- lat-field-ref lon-field-ref- x y zoom)- (assoc-in [:query :fields] [lat-field-ref lon-field-ref])- (assoc-in [:query :limit] tile-coordinate-limit))))+ (add-inside-filter lat-field lon-field x y zoom)+ (lib/with-fields [lat-field lon-field])+ (lib/limit tile-coordinate-limit))))(defn- result->points- [{{:keys [rows cols]} :data} lat-field-ref lon-field-ref]- ;; existing usages, don't use this going forward.- #_{:clj-kondo/ignore [:deprecated-var]}- (let [lat-key (qp.util/field-ref->key lat-field-ref)- lon-key (qp.util/field-ref->key lon-field-ref)- find-fn (fn [lat-or-lon-key]- (first (keep-indexed- (fn [idx col] (when (= (qp.util/field-ref->key (:field_ref col)) lat-or-lon-key) idx))- cols)))- lat-idx (find-fn lat-key)- lon-idx (find-fn lon-key)]+ [{{:keys [rows cols]} :data, :as qp-response} lat-field-ref lon-field-ref]+ (when-not (= (:status qp-response) :completed)+ (throw (ex-info (format "Error running tiles query: %s" (:error qp-response))+ (assoc qp-response :status-code 400))))+ (let [lat-id-or-name (second lat-field-ref)+ lon-id-or-name (second lon-field-ref)++ find-fn (fn [id-or-name]+ (or (first (keep-indexed+ (fn [idx col]+ (when (if (pos-int? id-or-name)+ (= (:id col) id-or-name)+ (= (:lib/deduplicated-name col) id-or-name))+ idx))+ cols))+ (throw (ex-info (format "Failed to find matching column in query results for column %s" (pr-str id-or-name))+ {:id-or-name id-or-name, :cols cols, :status-code 500}))))+ lat-idx (find-fn lat-id-or-name)+ lon-idx (find-fn lon-id-or-name)](for [row rows][(nth row lat-idx) (nth row lon-idx)])))-;; TODO - this should be async and stream results from the QP instead of requiring them all to be in memory at the same-;; time+;; TODO - this should be async and stream results from the QP instead of requiring them all to be in memory at the+;; same time(defn- tiles-response[result zoom points](if (= (:status result) :completed)@@ -183,6 +210,16 @@;; `result` might be a `core.async` channel or something we're not expecting(assoc (when (map? result) result) :status-code 400)))))+(mr/def ::query+ "Form-encoded JSON-encoded MBQL query."+ [:schema+ {:decode/api (fn [s]+ (when (string? s)+ (let [deserialized (json/decode+kw s)]+ (when (map? deserialized)+ (lib-be/normalize-query deserialized)))))}+ [:ref ::lib.schema/query]])+;; These endpoints provides an image with the appropriate pins rendered given a MBQL `query` (passed as a GET query;; string param). We evaluate the query and find the set of lat/lon pairs which are relevant and then render the;; appropriate ones. It's expected that to render a full map view several calls will be made to this endpoint in@@ -193,14 +230,13 @@[:zoom ms/Int][:x ms/Int][:y ms/Int]]- {:keys [query latField lonField]} :- [:map- [:query ms/JSONString]- [:latField string?]- [:lonField string?]]]- (let [query (json/decode+kw query)- lat-field (mbql.normalize/normalize (json/decode+kw latField))- lon-field (mbql.normalize/normalize (json/decode+kw lonField))- updated-query (tiles-query query zoom x y lat-field lon-field)+ {:keys [query]+ lat-field :latField+ lon-field :lonField} :- [:map+ [:query ::query]+ [:latField ::legacy-ref]+ [:lonField ::legacy-ref]]]+ (let [updated-query (tiles-query query zoom x y lat-field lon-field)result (qp/process-query(qp/userland-query updated-query {:executed-by api/*current-user-id*:context :map-tiles}))@@ -251,40 +287,42 @@points (result->points result lat-field-ref lon-field-ref)](tiles-response result zoom points)))+(mr/def ::parameters+ "Form-encoded JSON-encoded array of parameter maps."+ [:schema+ {:decode/api (fn [s]+ (when (string? s)+ (json/decode+kw s)))}+ ::parameters.schema/parameters])+(api.macros/defendpoint :get "/:card-id/:zoom/:x/:y""Generates a single tile image for a saved Card."[{:keys [card-id zoom x y]}:- [:map- [:card-id ms/PositiveInt]+ [:card-id ::lib.schema.id/card][:zoom ms/Int][:x ms/Int][:y ms/Int]]- {:keys [parameters latField lonField]}+ {:keys [parameters], lat-field :latField lon-field :lonField}:- [:map- [:parameters {:optional true} ms/JSONString]- [:latField string?]- [:lonField string?]]]- (let [parameters (when parameters (json/decode+kw parameters))- lat-field (json/decode+kw latField)- lon-field (json/decode+kw lonField)]- (process-tiles-query-for-card card-id parameters zoom x y lat-field lon-field)))+ [:parameters {:optional true} ::parameters]+ [:latField ::legacy-ref]+ [:lonField ::legacy-ref]]]+ (process-tiles-query-for-card card-id parameters zoom x y lat-field lon-field))(api.macros/defendpoint :get "/:dashboard-id/dashcard/:dashcard-id/card/:card-id/:zoom/:x/:y""Generates a single tile image for a dashcard."- [{:keys [dashboard-id dashcard-id card-id zoom x y]}+ [{:keys [dashboard-id dashcard-id card-id zoom x y], :as _route-params}:- [:map- [:dashboard-id ms/PositiveInt]- [:dashcard-id ms/PositiveInt]- [:card-id ms/PositiveInt]+ [:dashboard-id ::lib.schema.id/dashboard]+ [:dashcard-id ::lib.schema.id/dashcard]+ [:card-id ::lib.schema.id/card][:zoom ms/Int][:x ms/Int][:y ms/Int]]- {:keys [parameters latField lonField]}+ {:keys [parameters] lat-field :latField, lon-field :lonField, :as _query-params}:- [:map- [:parameters {:optional true} ms/JSONString]- [:latField string?]- [:lonField string?]]]- (let [parameters (when parameters (json/decode+kw parameters))- lat-field (json/decode+kw latField)- lon-field (json/decode+kw lonField)]- (process-tiles-query-for-dashcard dashboard-id dashcard-id card-id parameters zoom x y lat-field lon-field)))+ [:parameters {:optional true} ::parameters]+ [:latField ::legacy-ref]+ [:lonField ::legacy-ref]]]+ (process-tiles-query-for-dashcard dashboard-id dashcard-id card-id parameters zoom x y lat-field lon-field))diff --git a/src/metabase/upload/impl.clj b/src/metabase/upload/impl.cljindex ddd2e4c1c8f..e44c64b6e52 100644--- a/src/metabase/upload/impl.clj+++ b/src/metabase/upload/impl.clj@@ -744,9 +744,9 @@(defn- only-table-id"For models that depend on only one table, return its id, otherwise return nil. Doesn't support native queries."- [model]+ [{query :dataset_query, :as model}]; dataset_query can be empty in tests- (when-let [query (queries/card->lib-query model)]+ (when-let [query (not-empty query)](when (and (mbql? model) (no-joins? query))(lib/source-table-id query))))diff --git a/src/metabase/util/i18n.clj b/src/metabase/util/i18n.cljindex b2df9076978..f822a54e43e 100644--- a/src/metabase/util/i18n.clj+++ b/src/metabase/util/i18n.clj@@ -181,7 +181,8 @@"Ensures that `trs`/`tru` isn't called prematurely, during compilation."(if *compile-files*(fn [& _]- (throw (Exception. "Premature i18n string lookup. Is there a top-level call to `trs` or `tru`?")))+ (throw (Exception. (format "Premature i18n string lookup. Is there a top-level call to `trs` or `tru`? (In: %s)"+ (pr-str *file*)))))str))(defmacro tru-cljdiff --git a/src/metabase/util/json.clj b/src/metabase/util/json.cljindex 9ffc93c4334..04a998cddab 100644--- a/src/metabase/util/json.clj+++ b/src/metabase/util/json.clj@@ -78,8 +78,11 @@([source key-fn](cond (string? source) (cheshire/parse-string source key-fn)(instance? Reader source) (cheshire/parse-stream source key-fn)- (instance? InputStream source) (cheshire/parse-stream (io/reader source) key-fn)+ (instance? InputStream source) (with-open [r (io/reader source)]+ (cheshire/parse-stream r key-fn))(nil? source) nil+ (bytes? source) (with-open [r (io/reader (java.io.ByteArrayInputStream. ^bytes source))]+ (cheshire/parse-stream r)):else (throw (ex-info (str "Unsupported source type: " (type source)) {})))))(defn decode+kwdiff --git a/src/metabase/util/malli/defn.clj b/src/metabase/util/malli/defn.cljindex 40c86a5d9e1..25969666883 100644--- a/src/metabase/util/malli/defn.clj+++ b/src/metabase/util/malli/defn.clj@@ -86,6 +86,7 @@Known issue: does not currently generate automatic type hints the way [[schema.core/defn]] does, nor does it attemptto preserve them if you specify them manually. We can fix this in the future."+ {:style/indent [:defn]}[& [fn-name :as fn-tail]](let [parsed (mu.fn/parse-fn-tail fn-tail)cosmetic-name (gensym (munge (str fn-name)))diff --git a/src/metabase/util/malli/schema.clj b/src/metabase/util/malli/schema.cljindex d963ae4d914..81c3f63e162 100644--- a/src/metabase/util/malli/schema.clj+++ b/src/metabase/util/malli/schema.clj@@ -17,19 +17,36 @@;;; -------------------------------------------------- Utils --------------------------------------------------;;; TODO -- consider renaming this to `InstanceOfModel` to differentiate it from [[InstanceOfClass]]-(def ^{:arglists '([model])} InstanceOf+;;;+;;; TODO (Cam 9/29/25) -- maybe we should just automatically generate schemas for all the known models+;;; in [[metabase.models.resolution]] e.g. `:model/Card` that way you can just do+;;;+;;; [card :- :model/Card]+;;;+;;; instead of+;;;+;;; [card :- (ms/InstanceOf :model/Card)]+(def ^{:arglists '([model-or-models])} InstanceOf"Helper for creating a schema to check whether something is an instance of `model`.(ms/defn my-fn[user :- (ms/InstanceOf User)]...)"(memoize- (fn [model]+ (mu/fn [model-or-models :- [:or+ :keyword+ [:sequential {:min 1} :keyword]+ [:set {:min 1} :keyword]]](mu/with-api-error-message[:fn- {:error/message (format "value must be an instance of %s" (name model))}- #(t2/instance-of? model %)]- (deferred-tru "value must be an instance of {0}" (name model))))))+ {:error/message (format "value must be an instance of %s" (pr-str model-or-models))}+ (if (keyword? model-or-models)+ #(t2/instance-of? model-or-models %)+ (fn [instance]+ (some (fn [model]+ (t2/instance-of? model instance))+ model-or-models)))]+ (deferred-tru "value must be an instance of {0}" (pr-str model-or-models))))))(def ^{:arglists '([^Class klass])} InstanceOfClass"Helper for creating schemas to check whether something is an instance of a given class."diff --git a/src/metabase/warehouse_schema/api/table.clj b/src/metabase/warehouse_schema/api/table.cljindex b16402b7a22..22d041ebd8f 100644--- a/src/metabase/warehouse_schema/api/table.clj+++ b/src/metabase/warehouse_schema/api/table.clj@@ -14,7 +14,7 @@[metabase.models.interface :as mi][metabase.query-processor :as qp];; legacy usage -- don't do things like this going forward- ^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace :discouraged-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming :as qp.streaming][metabase.request.core :as request][metabase.sync.core :as sync]diff --git a/src/metabase/warehouse_schema/metadata_from_qp.clj b/src/metabase/warehouse_schema/metadata_from_qp.cljindex 39d9b660a2e..2012c28c226 100644--- a/src/metabase/warehouse_schema/metadata_from_qp.clj+++ b/src/metabase/warehouse_schema/metadata_from_qp.clj@@ -1,40 +1,66 @@(ns metabase.warehouse-schema.metadata-from-qp"Get metadata by running queries with the query processor."(:require+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor :as qp][metabase.query-processor.interface :as qp.i]- [metabase.util :as u]+ [metabase.query-processor.schema :as qp.schema]+ [metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.warehouse-schema.metadata-queries :as schema.metadata-queries][toucan2.core :as t2]))-(defn table-query+(mu/defn table-query"Runs the `mbql-query` where the source table is `table-id` and returns the result.Add the required filters if the table requires it,see [[metabase.warehouse-schema.metadata-queries/add-required-filters-if-needed]] for more details. Also takes anoptional `rff`, use the default rff if not provided."- ([table-id mbql-query]- (table-query table-id mbql-query nil))- ([table-id mbql-query rff]- {:pre [(integer? table-id)]}- (binding [qp.i/*disable-qp-logging* true]- (qp/process-query- {:type :query- :database (t2/select-one-fn :db_id :model/Table table-id)- :query (-> mbql-query- (assoc :source-table table-id)- schema.metadata-queries/add-required-filters-if-needed)- :middleware {:disable-remaps? true}}- rff))))+ ([table-id query-xform]+ (table-query table-id query-xform nil))-(defn field-distinct-count+ ([table-id :- ::lib.schema.id/table+ query-xform :- [:=> [:cat ::lib.schema/query] ::lib.schema/query]+ rff :- [:maybe ::qp.schema/rff]]+ {:pre [(pos-int? table-id)]}+ (let [database-id (t2/select-one-fn :db_id :model/Table table-id)+ mp (lib-be/application-database-metadata-provider database-id)+ query (-> (lib/query mp (lib.metadata/table mp table-id))+ query-xform+ (assoc-in [:middleware :disable-remaps?] true)+ schema.metadata-queries/add-required-filters-if-needed)]+ (binding [qp.i/*disable-qp-logging* true]+ (qp/process-query query rff)))))++(mu/defn field-distinct-count"Return the distinct count of `field`."- [field & [limit]]- (-> (table-query (:table_id field) {:aggregation [[:distinct [:field (u/the-id field) nil]]]- :limit limit})- :data :rows first first int))+ ([field]+ (field-distinct-count field nil))++ ([field :- [:or+ ::lib.schema.metadata/column+ (ms/InstanceOf :model/Field)]+ limit :- [:maybe pos-int?]]+ (let [field (cond-> field+ (t2/model field) (lib-be/instance->metadata :metadata/column))]+ (-> (table-query (:table-id field)+ (fn [query]+ (-> query+ (lib/aggregate (lib/distinct field))+ (cond-> limit+ (lib/limit limit)))))+ :data :rows first first int))))-(defn field-count+(mu/defn field-count"Return the count of `field`."- [field]- (-> (table-query (:table_id field) {:aggregation [[:count [:field (u/the-id field) nil]]]})- :data :rows first first int))+ [field :- [:or+ ::lib.schema.metadata/column+ (ms/InstanceOf :model/Field)]]+ (let [field (cond-> field+ (t2/model field) (lib-be/instance->metadata :metadata/column))]+ (-> (table-query (:table-id field) #(lib/aggregate % (lib/count field)))+ :data :rows first first int)))diff --git a/src/metabase/warehouse_schema/metadata_queries.clj b/src/metabase/warehouse_schema/metadata_queries.cljindex ea98fb56845..f218912c9e3 100644--- a/src/metabase/warehouse_schema/metadata_queries.clj+++ b/src/metabase/warehouse_schema/metadata_queries.clj@@ -2,48 +2,50 @@"Functions for constructing queries that can be used to get metadata about an attached data warehouse. TODO -- dothese belong here? Or in `warehouses`? Or in `sync`?"(:require+ [medley.core :as m]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.parameters.schema :as parameters.schema][metabase.util.malli :as mu][toucan2.core :as t2]))-(defn- partition-field->filter-form+(mu/defn- partition-field->filter-form :- :mbql.clause/>"Given a partition field, returns the default value can be used to query."- [field]- (let [field-form [:field (:id field) {:base-type (:base_type field)}]]- (condp #(isa? %2 %1) (:base_type field)- :type/Number [:> field-form -9223372036854775808]- :type/Date [:> field-form "0001-01-01"]- :type/DateTime [:> field-form "0001-01-01T00:00:00"])))+ [field :- ::lib.schema.metadata/column]+ (condp #(isa? %2 %1) (:base-type field)+ :type/Number (lib/> field -9223372036854775808)+ :type/Date (lib/> field "0001-01-01")+ :type/DateTime (lib/> field "0001-01-01T00:00:00")))-(defn add-required-filters-if-needed+(mu/defn add-required-filters-if-needed :- ::lib.schema/query"Add a dummy filter for tables that require filters.Look into tables from source tables and all the joins.Currently this only apply to partitioned tables on bigquery that requires a partition filter.In the future we probably want this to be dispatched by database engine or handled by QP."- [query]- (let [table-ids (->> (conj (keep :source-table (:joins query)) (:source-table query))- (filter pos-int?))- required-filter-fields (when (seq table-ids)- (t2/select :model/Field {:select [:f.*]- :from [[:metabase_field :f]]- :left-join [[:metabase_table :t] [:= :t.id :f.table_id]]- :where [:and- [:= :f.active true]- [:= :f.database_partitioned true]- [:= :t.active true]- [:= :t.database_require_filter true]- [:in :t.id table-ids]]}))- update-query-filter-fn (fn [existing-filter new-filter]- (if (some? existing-filter)- [:and existing-filter new-filter]- new-filter))]- (case (count required-filter-fields)- 0- query- 1- (update query :filter update-query-filter-fn (partition-field->filter-form (first required-filter-fields)))- ;; > 1- (update query :filter update-query-filter-fn (into [:and] (map partition-field->filter-form required-filter-fields))))))+ [query :- ::lib.schema/query]+ (transduce+ (comp (map (fn [table-id]+ (lib.metadata/table query table-id)))+ (filter :active)+ (filter :database-require-filter)+ (mapcat (fn [{table-id :id, :as _table}]+ (lib.metadata/active-fields query table-id)))+ (filter :database-partitioned)+ ;; resolve the Field ID to correct visible metadata, e.g. if this column+ ;; comes from a join or something we need to make sure we have metadata+ ;; with a join alias.+ ;;+ ;; don't fetch visible columns unless we actually need to+ (keep (let [visible-columns (delay (lib/visible-columns query))]+ (fn [field]+ (m/find-first #(= (:id %) (:id field))+ @visible-columns))))+ (map partition-field->filter-form))+ (completing lib/filter)+ query+ (lib/all-source-table-ids query)))(mu/defn human-readable-remapping-map :- [:maybe ::parameters.schema/human-readable-remapping-map]"Get the human readable (internally mapped) values of the field specified by `field-id`."diff --git a/src/metabase/warehouse_schema/models/field.clj b/src/metabase/warehouse_schema/models/field.cljindex 164cbff74e4..35ea1c377bf 100644--- a/src/metabase/warehouse_schema/models/field.clj+++ b/src/metabase/warehouse_schema/models/field.clj@@ -6,7 +6,7 @@[medley.core :as m][metabase.api.common :as api][metabase.app-db.core :as mdb]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.field :as lib.field][metabase.lib.schema.metadata][metabase.models.humanization :as humanization]@@ -297,7 +297,7 @@See [[lib.field/infer-has-field-values]] for more info."[_model k field](when field- (let [has-field-values (lib.field/infer-has-field-values (lib.metadata.jvm/instance->metadata field :metadata/column))]+ (let [has-field-values (lib.field/infer-has-field-values (lib-be/instance->metadata field :metadata/column))](assoc field k has-field-values))))(methodical/defmethod t2.hydrate/needs-hydration? [#_model :default #_k :has_field_values]diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 637a4543bf6..3396862cbaf 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -28,6 +28,9 @@[medley.core :as m][metabase.analyze.core :as analyze][metabase.app-db.core :as app-db]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.premium-features.core :refer [defenterprise]]@@ -365,7 +368,7 @@(rf result row)))))));;; TODO -- move into [[metabase.warehouse-schema.metadata-from-qp]] ??-(defn distinct-values+(mu/defn distinct-values"Fetch a sequence of distinct values for `field` that are below the [[*total-max-length*]] threshold. If the valuesare past the threshold, this returns a subset of possible values values where the total length of all items is lessthan [[*total-max-length*]]. It also returns a `has_more_values` flag, `has_more_values` = `true` when the returned@@ -378,12 +381,18 @@(This function provides the values that normally get saved as a Field's FieldValues. You most likely should not beusing this directly in code outside of this namespace, unless it's for a very specific reason, such as certain caseswhere we fetch ad-hoc FieldValues for GTAP-filtered Fields.)"- [field]+ [field :- [:or+ (ms/InstanceOf :model/Field)+ ::lib.schema.metadata/column]](try- (let [result ((requiring-resolve 'metabase.warehouse-schema.metadata-from-qp/table-query)- (:table_id field)- {:breakout [[:field (u/the-id field) nil]]- :limit *absolute-max-distinct-values-limit*}+ (let [field (cond-> field+ (t2/model field) (lib-be/instance->metadata :metadata/column))+ result ((requiring-resolve 'metabase.warehouse-schema.metadata-from-qp/table-query)+ (:table-id field)+ (fn [query]+ (-> query+ (lib/breakout field)+ (lib/limit *absolute-max-distinct-values-limit*)))(limit-max-char-len-rff qp.reducible/default-rff *total-max-length*))distinct-values (-> result :data :rows)]{:values distinct-valuesdiff --git a/src/metabase/warehouses/api.clj b/src/metabase/warehouses/api.cljindex df3b6d6b49e..4320bd1231e 100644--- a/src/metabase/warehouses/api.clj+++ b/src/metabase/warehouses/api.clj@@ -16,6 +16,8 @@[metabase.driver.util :as driver.u][metabase.events.core :as events][metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi]@@ -93,10 +95,11 @@:write:none))))-(defn- card-database-supports-nested-queries? [{{database-id :database, :as database} :dataset_query, :as _card}]+(defn- card-database-supports-nested-queries? [{{database-id :database, :as query} :dataset_query, :as _card}](when database-id(when-let [driver (driver.u/database->driver database-id)]- (driver.u/supports? driver :nested-queries database))))+ (when-let [database (some-> query not-empty lib.metadata/database)]+ (driver.u/supports? driver :nested-queries database)))))(defn- card-has-ambiguous-columns?"We know a card has ambiguous columns if any of the columns that come back end in `_2` (etc.) because that's what@@ -113,17 +116,18 @@would be ambiguous. Too many things break when attempting to use a query like this. In the future, this may besupported, but it will likely require rewriting the source SQL query to add appropriate aliases (this is eventrickier if the source query uses `SELECT *`)."- [{result-metadata :result_metadata, dataset-query :dataset_query}]- (and (= (:type dataset-query) :native)+ [{result-metadata :result_metadata, query :dataset_query, :as _card}]+ (and (lib/native-only-query? query)(some (partial re-find #"_2$")(map (comp name :name) result-metadata))))-(defn- card-uses-unnestable-aggregation?+(mu/defn- card-uses-unnestable-aggregation?"Since cumulative count and cumulative sum aggregations are done in Clojure-land we can't use Cards that use querieswith those aggregations as source queries. This function determines whether `card` is using one of those queries sowe can filter it out in Clojure-land."- [{{{aggregations :aggregation} :query} :dataset_query}]- (lib.util.match/match aggregations #{:cum-count :cum-sum}))+ [{query :dataset_query, :as _card} :- [:map+ [:dataset_query ::queries.schema/query]]]+ (lib.util.match/match (lib/aggregations query) #{:cum-count :cum-sum}))(defn card-can-be-used-as-source-query?"Does `card`'s query meet the conditions required for it to be used as a source query for another query?"diff --git a/src/metabase/xrays/api/automagic_dashboards.clj b/src/metabase/xrays/api/automagic_dashboards.cljindex 57cd3a7b209..2d2cb3e5259 100644--- a/src/metabase/xrays/api/automagic_dashboards.clj+++ b/src/metabase/xrays/api/automagic_dashboards.clj@@ -1,19 +1,24 @@(ns metabase.xrays.api.automagic-dashboards(:require[buddy.core.codecs :as codecs]+ [medley.core :as m][metabase.api.common :as api][metabase.api.macros :as api.macros]+ [metabase.lib-be.core :as lib-be][metabase.models.interface :as mi][metabase.queries.core :as queries][metabase.query-permissions.core :as query-perms]+ [metabase.util :as u][metabase.util.i18n :as i18n :refer [deferred-tru]][metabase.util.json :as json][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.regex :as u.regex][metabase.xrays.automagic-dashboards.comparison :as automagic-dashboards.comparison][metabase.xrays.automagic-dashboards.core :as automagic-dashboards.core][metabase.xrays.automagic-dashboards.dashboard-templates :as automagic-dashboards.dashboard-templates]+ [metabase.xrays.automagic-dashboards.schema :as ads][metabase.xrays.transforms.dashboard :as transforms.dashboard][metabase.xrays.transforms.materialize :as transforms.materialize][ring.util.codec :as codec]@@ -49,11 +54,17 @@(def ^:private ^{:arglists '([s])} decode-base64-json(comp json/decode+kw codecs/bytes->str codec/base64-decode))-(def ^:private Base64EncodedJSON+(mr/def ::base-64-encoded-json+ "form-encoded base-64-encoded JSON"[:fn- {:description "form-encoded base-64-encoded JSON"- :error/fn (fn [_ _]- (i18n/tru "value couldn''t be parsed as base64 encoded JSON"))}+ ;; TODO (Cam 10/7/25) -- you would expect `=` to get form-encoded here but apparently the FE is having trouble+ ;; doing it... allow it for now I guess.+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1759892123044939?thread_ts=1759289751.539169&cid=C0645JP1W81+ ;;+ ;; TODO (Cam 10/7/25) -- not clear whether we expect `-` and `_` as per RFC 4648 here or if we expect form-encoded+ ;; `+` and `/`... accept either for now I guess+ {:api/regex #"(?:[A-Za-z0-9\-_]|(?:%2B)|(?:%2F))+(?:(?:%3D)|=){0,2}"+ :error/fn (fn [_ _] (i18n/tru "value couldn''t be parsed as base64 encoded JSON"))}decode-base64-json])(api.macros/defendpoint :get "/database/:id/candidates"@@ -110,12 +121,16 @@[_entity-type card-id-str](api/read-check (t2/select-one :model/Card :id (ensure-int card-id-str))))-(mu/defn adhoc-query-instance :- (ms/InstanceOf :model/Query)+(mu/defn adhoc-query-instance :- [:and+ (ms/InstanceOf :model/Query)+ [:map+ [:dataset_query ::ads/query]]]"Wrap query map into a Query object (mostly to facilitate type dispatch)."[query :- :map]- (mi/instance :model/Query- (merge (queries/query->database-and-table-ids query)- {:dataset_query (mi/maybe-normalize-query :out query)})))+ (let [query (lib-be/normalize-query query)]+ (mi/instance :model/Query+ (merge (queries/query->database-and-table-ids query)+ {:dataset_query query}))))(defmethod ->entity :adhoc[_entity-type encoded-query]@@ -131,28 +146,48 @@transform-name)(def ^:private entities- (map name (keys (methods ->entity))))+ (sort (keys (methods ->entity))))(def ^:private Entity- (into [:enum {:api/regex (u.regex/re-or entities)- :error/fn (fn [_ _]- (i18n/tru "Invalid entity type"))}]+ (into [:enum {:api/regex (u.regex/re-or (map name entities))+ :error/fn (fn [_ _] (i18n/tru "Invalid entity type"))}]entities))+(mr/def ::entity-id-or-query+ "One of these:++ * A non-empty string with an Entity ID (including `card__<id>`-encoded Card IDs)++ * a form-encoded base-64-encoded JSON-encoded MBQL query++ * The name of a transform++ (Effectively since the names of transforms are unconstrained this parameter is allowed to be any form-encoded+ string.)"+ ;; TODO (Cam 10/7/25) -- you would expect `=` to get form-encoded here but apparently the FE is having trouble doing+ ;; it... allow it for now I guess.+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1759892123044939?thread_ts=1759289751.539169&cid=C0645JP1W81+ [:string+ {:min 1+ :api/regex #"(?:[A-Za-z0-9\-._~=]|%[0-9A-Fa-f]{2})+"}])+(def ^:private ComparisonEntity- (mu/with-api-error-message- [:enum "segment" "adhoc" "table"]- (deferred-tru "Invalid comparison entity type. Can only be one of \"table\", \"segment\", or \"adhoc\"")))+ (let [entity-types [:adhoc :segment :table]]+ (mu/with-api-error-message+ (into [:enum+ {:api/regex (u.regex/re-or (map name entity-types))}]+ entity-types)+ (deferred-tru "Invalid comparison entity type. Can only be one of \"table\", \"segment\", or \"adhoc\""))))(defn- coerce-show"Show is either nil, \"all\", or a number. If it's a string it needs to be converted into a keyword."[show](cond-> show (= "all" show) keyword))-(defn get-automagic-dashboard+(mu/defn get-automagic-dashboard"Return an automagic dashboard for entity `entity` with id `id`."- [entity entity-id-or-query show]- (if (= entity "transform")+ [entity :- Entity entity-id-or-query show]+ (if (= entity :transform)(transforms.dashboard/dashboard (->entity entity entity-id-or-query))(-> (->entity entity entity-id-or-query)(automagic-dashboards.core/automagic-analysis {:show (coerce-show show)}))))@@ -160,17 +195,35 @@(api.macros/defendpoint :get "/:entity/:entity-id-or-query""Return an automagic dashboard for entity `entity` with id `id`."[{:keys [entity entity-id-or-query]} :- [:map- [:entity Entity]]+ [:entity Entity]+ [:entity-id-or-query ::entity-id-or-query]]{:keys [show]} :- [:map[:show {:optional true} [:maybe [:or [:= "all"] nat-int?]]]]](get-automagic-dashboard entity entity-id-or-query show))+(defn- dashboard-metadata [dashboard]+ (letfn [(normalize-card [card]+ (-> card+ (m/update-existing :dataset_query lib-be/normalize-query)+ (dissoc :id)))+ (normalize-series [series]+ (map normalize-card series))+ (normalize-dashcard [{:keys [card], :as dashcard}]+ (-> dashcard+ (u/assoc-dissoc :card (some-> card not-empty normalize-card))+ (m/update-existing :series normalize-series)))+ (normalize-dashcards [dashcards]+ (mapv normalize-dashcard dashcards))+ (normalize-dashboard [dashboard]+ (update dashboard :dashcards normalize-dashcards))]+ (queries/batch-fetch-dashboard-metadata [(normalize-dashboard dashboard)])))+(api.macros/defendpoint :get "/:entity/:entity-id-or-query/query_metadata""Return all metadata for an automagic dashboard for entity `entity` with id `id`."[{:keys [entity entity-id-or-query]} :- [:map- [:entity Entity]]]- (queries/batch-fetch-dashboard-metadata- [(get-automagic-dashboard entity entity-id-or-query nil)]))+ [:entity Entity]+ [:entity-id-or-query ::entity-id-or-query]]]+ (dashboard-metadata (get-automagic-dashboard entity entity-id-or-query nil)))(defn linked-entities"Identify the pk field of the model with `pk_ref`, and then find any fks that have that pk as a target."@@ -180,7 +233,7 @@(fn [{:keys [table_id id]}]{:linked-table-id table_id:linked-field-id id})- (t2/select 'Field :fk_target_field_id field-id))))+ (t2/select :model/Field :fk_target_field_id field-id))))(defn- add-source-model-link"Insert a source model link card into the sequence of passed in cards."@@ -283,7 +336,7 @@"Return an automagic dashboard for entity `entity` with id `id` using dashboard-template `dashboard-template`."[{:keys [entity entity-id-or-query prefix dashboard-template]} :- [:map[:entity Entity]- [:entity-id-or-query ms/NonBlankString]+ [:entity-id-or-query ::entity-id-or-query][:prefix Prefix][:dashboard-template DashboardTemplate]]{:keys [show]} :- [:map@@ -297,8 +350,8 @@`cell-query`."[{:keys [entity entity-id-or-query cell-query]} :- [:map[:entity Entity]- [:entity-id-or-query ms/NonBlankString]- [:cell-query Base64EncodedJSON]]+ [:entity-id-or-query ::entity-id-or-query]+ [:cell-query ::base-64-encoded-json]]{:keys [show]} :- [:map[:show {:optional true} Show]]](-> (->entity entity entity-id-or-query)@@ -310,10 +363,10 @@dashboard-template `dashboard-template`."[{:keys [entity entity-id-or-query cell-query prefix dashboard-template]} :- [:map[:entity Entity]- [:entity-id-or-query ms/NonBlankString]+ [:entity-id-or-query ::entity-id-or-query][:prefix Prefix][:dashboard-template DashboardTemplate]- [:cell-query Base64EncodedJSON]]+ [:cell-query ::base-64-encoded-json]]{:keys [show]} :- [:map[:show {:optional true} Show]]](-> (->entity entity entity-id-or-query)@@ -324,11 +377,14 @@(api.macros/defendpoint :get "/:entity/:entity-id-or-query/compare/:comparison-entity/:comparison-entity-id-or-query""Return an automagic comparison dashboard for entity `entity` with id `id` compared with entity `comparison-entity`with id `comparison-entity-id-or-query.`"- [{:keys [entity entity-id-or-query comparison-entity+ [{:keys [entity+ entity-id-or-query+ comparison-entitycomparison-entity-id-or-query]} :- [:map- [:entity-id-or-query ms/NonBlankString]- [:entity Entity]- [:comparison-entity ComparisonEntity]]+ [:entity-id-or-query ::entity-id-or-query]+ [:entity Entity]+ [:comparison-entity ComparisonEntity]+ [:comparison-entity-id-or-query ::entity-id-or-query]]{:keys [show]} :- [:map[:show {:optional true} Show]]](let [left (->entity entity entity-id-or-query)@@ -341,13 +397,18 @@(api.macros/defendpoint :get "/:entity/:entity-id-or-query/rule/:prefix/:dashboard-template/compare/:comparison-entity/:comparison-entity-id-or-query""Return an automagic comparison dashboard for entity `entity` with id `id` using dashboard-template`dashboard-template`; compared with entity `comparison-entity` with id `comparison-entity-id-or-query.`."- [{:keys [entity entity-id-or-query prefix dashboard-template- comparison-entity comparison-entity-id-or-query]} :- [:map- [:entity Entity]- [:entity-id-or-query ms/NonBlankString]- [:prefix Prefix]- [:dashboard-template DashboardTemplate]- [:comparison-entity ComparisonEntity]]+ [{:keys [entity+ entity-id-or-query+ prefix+ dashboard-template+ comparison-entity+ comparison-entity-id-or-query]} :- [:map+ [:entity Entity]+ [:entity-id-or-query ::entity-id-or-query]+ [:prefix Prefix]+ [:dashboard-template DashboardTemplate]+ [:comparison-entity ComparisonEntity]+ [:comparison-entity-id-or-query ::entity-id-or-query]]{:keys [show]} :- [:map[:show {:optional true} Show]]](let [left (->entity entity entity-id-or-query)@@ -362,12 +423,16 @@"Return an automagic comparison dashboard for cell in automagic dashboard for entity `entity`with id `id` defined by query `cell-query`; compared with entity `comparison-entity` with id`comparison-entity-id-or-query.`."- [{:keys [entity entity-id-or-query cell-query- comparison-entity comparison-entity-id-or-query]} :- [:map- [:entity Entity]- [:entity-id-or-query ms/NonBlankString]- [:cell-query Base64EncodedJSON]- [:comparison-entity ComparisonEntity]]+ [{:keys [entity+ entity-id-or-query+ cell-query+ comparison-entity+ comparison-entity-id-or-query]} :- [:map+ [:entity Entity]+ [:entity-id-or-query ::entity-id-or-query]+ [:cell-query ::base-64-encoded-json]+ [:comparison-entity ComparisonEntity]+ [:comparison-entity-id-or-query ::entity-id-or-query]]{:keys [show]} :- [:map[:show {:optional true} Show]]](let [left (->entity entity entity-id-or-query)@@ -381,14 +446,20 @@"Return an automagic comparison dashboard for cell in automagic dashboard for entity `entity`with id `id` defined by query `cell-query` using dashboard-template `dashboard-template`; compared with entity`comparison-entity` with id `comparison-entity-id-or-query.`."- [{:keys [entity entity-id-or-query cell-query prefix dashboard-template- comparison-entity comparison-entity-id-or-query]} :- [:map- [:entity Entity]- [:entity-id-or-query ms/NonBlankString]- [:prefix Prefix]- [:dashboard-template DashboardTemplate]- [:cell-query Base64EncodedJSON]- [:comparison-entity ComparisonEntity]]+ [{:keys [entity+ entity-id-or-query+ cell-query+ prefix+ dashboard-template+ comparison-entity+ comparison-entity-id-or-query]} :- [:map+ [:entity Entity]+ [:entity-id-or-query ::entity-id-or-query]+ [:prefix Prefix]+ [:dashboard-template DashboardTemplate]+ [:cell-query ::base-64-encoded-json]+ [:comparison-entity ComparisonEntity]+ [:comparison-entity-id-or-query ::entity-id-or-query]]{:keys [show]} :- [:map[:show {:optional true} Show]]](let [left (->entity entity entity-id-or-query)diff --git a/src/metabase/xrays/automagic_dashboards/combination.clj b/src/metabase/xrays/automagic_dashboards/combination.cljindex ebbfac02b50..baaaaed9bde 100644--- a/src/metabase/xrays/automagic_dashboards/combination.clj+++ b/src/metabase/xrays/automagic_dashboards/combination.clj@@ -260,17 +260,3 @@(vals merged-dims)(mapv (comp :filter simple-grounded-filters) card-filters))(add-dataset-query base-context))))))--(defn items->str- "Convert a seq of items to a string. If more than two items are present, they are separated by commas, including the- oxford comma on the final pairing."- [[f s :as items]]- (condp = (count items)- 0 ""- 1 (str f)- 2 (format "%s and %s" f s)- (format "%s, and %s" (str/join ", " (butlast items)) (last items))))--(def dim-name- "Name of the dimension. Trying for `:display_name` and falling back to `:name`"- (some-fn :display_name :name))diff --git a/src/metabase/xrays/automagic_dashboards/comparison.clj b/src/metabase/xrays/automagic_dashboards/comparison.cljindex e0bec2b801a..e775d20d446 100644--- a/src/metabase/xrays/automagic_dashboards/comparison.clj+++ b/src/metabase/xrays/automagic_dashboards/comparison.clj@@ -5,7 +5,6 @@[metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi]- [metabase.query-processor.util :as qp.util][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]@@ -13,14 +12,15 @@[metabase.xrays.automagic-dashboards.filters :as filters][metabase.xrays.automagic-dashboards.names :as names][metabase.xrays.automagic-dashboards.populate :as populate]+ [metabase.xrays.automagic-dashboards.schema :as ads][metabase.xrays.automagic-dashboards.util :as magic.util][metabase.xrays.related :as related]))(def ^:private ^{:arglists '([root])} comparison-name(comp capitalize-first (some-fn :comparison-name :full-name)))-(defn- dashboard->cards- [dashboard]+(mu/defn- dashboard->cards :- [:sequential ::ads/card]+ [dashboard :- ::ads/dashboard](->> dashboard:dashcards(map (fn [{:keys [size_y card col row series] :as dashcard}]@@ -29,7 +29,10 @@:series series:height size_y:position (+ (* row populate/grid-width) col))))- (sort-by :position)))+ (sort-by :position)+ (map (fn [dashcard]+ (cond-> dashcard+ (nil? (:dataset_query dashcard)) (dissoc :dataset_query))))))(defn- clone-card[card]@@ -40,19 +43,26 @@:collection_id nil:id (gensym))))-(def ^:private ^{:arglists '([card])} display-type- (comp qp.util/normalize-token :display))+(mu/defn- display-type :- [:maybe :keyword]+ [card :- :map]+ (keyword (:display card)))-(defn- add-filter-clauses+(mu/defn- add-filter-clauses :- ::ads/query"Add `new-filter-clauses` to a query. There is actually an `mbql.u/add-filter-clause` function we should be usinginstead, but that validates its input and output, and the queries that come in here aren't always valid (forexample, missing `:database`). If we can, it would be nice to use that instead of reinventing the wheel here."- [{{existing-filter-clause :filter} :query, :as query}, new-filter-clauses]- (let [clauses (filter identity (cons existing-filter-clause new-filter-clauses))- new-filter-clause (when (seq clauses)- (mbql.normalize/normalize-fragment [:query :filter] (cons :and clauses)))]- (cond-> query- (seq new-filter-clause) (assoc-in [:query :filter] new-filter-clause))))+ [query :- ::ads/query+ new-filter-clauses :- [:maybe [:sequential :any]]]+ (magic.util/do-with-legacy-query+ query+ (fn [{{existing-filter-clause :filter} :query, :as query}]+ (let [clauses (filter identity (cons existing-filter-clause new-filter-clauses))+ new-filter-clause (when (seq clauses)+ ;; existing usage, this namespace will be rewritten to use Lib soon.+ #_{:clj-kondo/ignore [:deprecated-var]}+ (mbql.normalize/normalize-fragment [:query :filter] (cons :and clauses)))]+ (cond-> query+ (seq new-filter-clause) (magic.util/do-with-legacy-query assoc-in [:query :filter] new-filter-clause))))))(defn- inject-filter"Inject filter clause into card."@@ -61,14 +71,21 @@(update :dataset_query #(add-filter-clauses % [query-filter cell-query]))(update :series (partial map (partial inject-filter root)))))-(defn- multiseries?- [card]+(mu/defn- multiseries?+ [card :- [:map+ [:dataset_query {:optional true} ::ads/query]]](or (-> card :series not-empty)- (-> card (get-in [:dataset_query :query :aggregation]) count (> 1))- (-> card (get-in [:dataset_query :query :breakout]) count (> 1))))+ (some-> card+ :dataset_query+ not-empty+ (magic.util/do-with-legacy-query+ (fn [query]+ (or (-> query (get-in [:query :aggregation]) count (> 1))+ (-> query (get-in [:query :breakout]) count (> 1))))))))-(defn- overlay-comparison?- [card]+(mu/defn- overlay-comparison?+ [card :- [:map+ [:dataset_query {:optional true} ::ads/query]]](and (-> card display-type (#{:bar :line}))(not (multiseries? card))))@@ -79,7 +96,7 @@card-left (->> card (inject-filter left) clone-card)card-right (->> card (inject-filter right) clone-card)[color-left color-right] (->> [left right]- (map #(get-in % [:dataset_query :query :filter]))+ (map #(magic.util/do-with-legacy-query (:dataset_query %) get-in [:query :filter]))populate/map-to-colors)](if (overlay-comparison? card)(let [card (-> card-left@@ -88,7 +105,7 @@series (-> card-right(update :name #(format "%s (%s)" % (comparison-name right)))vector)]- (update dashboard :dashcards conj (merge (populate/card-defaults)+ (update dashboard :dashcards conj (merge (populate/dashcard-defaults){:col 0:row row:size_x populate/grid-width@@ -108,7 +125,7 @@(not (multiseries? card-right))(assoc-in [:visualization_settings :graph.colors] [color-right]))](-> dashboard- (update :dashcards conj (merge (populate/card-defaults)+ (update :dashcards conj (merge (populate/dashcard-defaults){:col 0:row row:size_x width@@ -117,7 +134,7 @@:card_id (:id card-left):series series-left:visualization_settings {}}))- (update :dashcards conj (merge (populate/card-defaults)+ (update :dashcards conj (merge (populate/dashcard-defaults){:col width:row row:size_x width@@ -162,21 +179,23 @@(/ populate/grid-width 2))][dashboard (max height-left height-right)]))-(defn- series-labels- [card]+(mu/defn- series-labels+ [card :- [:map+ [:dataset_query {:optional true} ::ads/query]]](get-in card [:visualization_settings :graph.series_labels](map (comp capitalize-first names/metric-name)- (get-in card [:dataset_query :query :aggregation]))))+ (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :aggregation]))))-(defn- unroll-multiseries- [card]+(mu/defn- unroll-multiseries+ [card :- [:map+ [:dataset_query {:optional true} ::ads/query]]](if (and (multiseries? card)(-> card :display (= :line)))(for [[aggregation label] (map vector- (get-in card [:dataset_query :query :aggregation])+ (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :aggregation])(series-labels card))](-> card- (assoc-in [:dataset_query :query :aggregation] [aggregation])+ (update :dataset_query magic.util/do-with-legacy-query assoc-in [:query :aggregation] [aggregation])(assoc :name label)(m/dissoc-in [:visualization_settings :graph.series_labels])))[card]))@@ -258,8 +277,11 @@distinct(map #(automagic-analysis % {:source (:source left):rules-prefix ["comparison"]})))]- (assert (or (= (:source left) (:source right))- (= (-> left :source :table_id) (-> right :source u/the-id))))+ (assert (or (let [left-source (m/update-existing (:source left) :dataset_query magic.util/do-with-legacy-query identity)+ right-source (m/update-existing (:source right) :dataset_query magic.util/do-with-legacy-query identity)]+ (= left-source right-source))+ (= (-> left :source :table_id)+ (-> right :source u/the-id))))(->> (concat segment-dashboards [dashboard])(reduce (fn [dashboard-1 dashboard-2](if dashboard-1diff --git a/src/metabase/xrays/automagic_dashboards/core.clj b/src/metabase/xrays/automagic_dashboards/core.cljindex 69eb954ad54..28ebc84e053 100644--- a/src/metabase/xrays/automagic_dashboards/core.clj+++ b/src/metabase/xrays/automagic_dashboards/core.clj@@ -149,9 +149,8 @@[medley.core :as m][metabase.analyze.core :as analyze][metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.lib-be.core :as lib-be]+ [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.core :as lib]- [metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.query-processor.util :as qp.util]@@ -236,29 +235,37 @@:url (format "%sfield/%s" public-endpoint (:id field)):dashboard-templates-prefix ["field"]}))-(defn- source-card-id [card-or-question]- (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]- (some-> card-or-question :dataset_query not-empty lib-be/normalize-query lib/source-card-id)))+(mu/defn- source-card-id [card-or-question :- [:map+ [:dataset_query ::ads/query]]]+ (magic.util/do-with-mbql5-query (:dataset_query card-or-question) lib/source-card-id))-(defn- nested-query?+(mu/defn- nested-query?"Is this card or question derived from another model or question?"- [card-or-question]+ [card-or-question :- [:map+ [:dataset_query ::ads/query]]](some? (source-card-id card-or-question)))-(def ^:private ^{:arglists '([card-or-question])} native-query?+(mu/defn- native-query?"Is this card or question native (SQL)?"- (comp some? #{:native} qp.util/normalize-token #(get-in % [:dataset_query :type])))+ [{query :dataset_query, :as _card-or-question} :- [:map+ [:dataset_query ::ads/query]]]+ (magic.util/do-with-legacy-query query #(= (:type %) :native)))-(defn- source-question- [card-or-question]+(mu/defn- source-question :- [:maybe (ms/InstanceOf :model/Card)]+ [card-or-question :- [:map+ [:dataset_query ::ads/query]]](when-let [source-card-id (source-card-id card-or-question)](t2/select-one :model/Card :id source-card-id)))-(defn- table-like?- [card-or-question]- (and- (nil? (get-in card-or-question [:dataset_query :query :aggregation]))- (nil? (get-in card-or-question [:dataset_query :query :breakout]))))+(mu/defn- table-like?+ [card-or-question :- [:map+ [:dataset_query ::ads/query]]]+ (magic.util/do-with-legacy-query+ (:dataset_query card-or-question)+ (fn [query]+ (and+ (nil? (get-in query [:query :aggregation]))+ (nil? (get-in query [:query :breakout]))))))(defn- table-id"Get the Table ID from `card-or-question`, which can be either a Card from the DB (which has a `:table_id` property)@@ -270,8 +277,9 @@(or (:table_id card-or-question)(:table-id card-or-question)))-(defn- source- [card]+(mu/defn- source+ [card :- [:map+ [:dataset_query ::ads/query]]](cond;; This is a model(= (:type card) :model) (assoc card :entity_type :entity/GenericTable)@@ -285,11 +293,16 @@(mu/defmethod ->root :model/Card :- ::ads/root[card]- (let [source (source card)]+ (let [card (update card+ :dataset_query+ ;; temporary until we rework X-rays to use Lib/MBQL 5+ #_{:clj-kondo/ignore [:discouraged-var]}+ lib/->legacy-MBQL)+ source (source card)]{:entity card:source source:database (:database_id card)- :query-filter (get-in card [:dataset_query :query :filter])+ :query-filter (magic.util/do-with-legacy-query (:dataset_query card) get-in [:query :filter]):full-name (tru "\"{0}\"" (:name card)):short-name (names/source-name {:source source}):url (format "%s%s/%s" public-endpoint (name (:type source :question)) (u/the-id card))@@ -300,12 +313,14 @@(mu/defmethod ->root :model/Query :- ::ads/root[query :- [:map[:database-id ::lib.schema.id/database]]]- (let [source (source query)+ ;; use of [[lib/->legacy-MBQL]] here is temporary until we update X-Rays to use Lib+ (let [query (update query :dataset_query #_{:clj-kondo/ignore [:discouraged-var]} lib/->legacy-MBQL)+ source (source query)root {:database (:database-id query), :source source}]{:entity query:source source:database (:database-id query)- :query-filter (get-in query [:dataset_query :query :filter])+ :query-filter (magic.util/do-with-legacy-query (:dataset_query query) get-in [:query :filter]):full-name (cond(native-query? query) (tru "Native query")(table-like? query) (-> source ->root :full-name)@@ -466,7 +481,7 @@(assoc group :title (or comparison_title title))))))(instantiate-metadata context available-metrics {}))))-(mu/defn- generate-base-dashboard+(mu/defn- generate-base-dashboard :- ::ads/dashboard-template"Produce the \"base\" dashboard from the base context for an item and a dashboard template.This includes dashcards and global filters, but does not include related items and is not yet populated.Repeated calls of this might be generated (e.g. the main dashboard and related) then combined once using@@ -741,8 +756,10 @@(automagic-dashboard (merge (->root metric) opts)))(mu/defn- collect-metrics :- [:maybe [:sequential (ms/InstanceOf :xrays/Metric)]]- [root question]- (map (fn [aggregation-clause]+ [root :- ::ads/root+ question :- [:map+ [:dataset_query ::ads/query]]]+ (map (mu/fn [aggregation-clause :- ::mbql.s/Aggregation](if (-> aggregation-clausefirstqp.util/normalize-token@@ -754,11 +771,13 @@:source-table table-id}:name (names/metric->description root aggregation-clause):table_id table-id}))))- (get-in question [:dataset_query :query :aggregation])))+ (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :aggregation])))(mu/defn- collect-breakout-fields :- [:maybe [:sequential (ms/InstanceOf :model/Field)]]- [root question]- (for [breakout (get-in question [:dataset_query :query :breakout])+ [root :- ::ads/root+ question :- [:map+ [:dataset_query ::ads/query]]]+ (for [breakout (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :breakout])field-clause (take 1 (magic.util/collect-field-references breakout)):let [field (magic.util/->field root field-clause)]:when (and field@@ -766,8 +785,9 @@field))(mu/defn- decompose-question- [root :- ::ads/root- question+ [root :- ::ads/root+ question :- [:map+ [:dataset_query ::ads/query]]opts](letfn [(analyze [x](try@@ -788,14 +808,16 @@(defn- preserve-entity-element"Ensure that elements of an original dataset query are preserved in dashcard queries."- [dashboard entity entity-element]- (if-let [element-value (get-in entity [:dataset_query :query entity-element])]+ [dashboard+ entity+ entity-element]+ (if-let [element-value (magic.util/do-with-legacy-query (:dataset_query entity) get-in [:query entity-element])](letfn [(splice-element [dashcard]- (cond-> dashcard- (get-in dashcard [:card :dataset_query :query])- (update-in [:card :dataset_query :query entity-element]- (fnil into (empty element-value))- element-value)))]+ (m/update-existing-in dashcard [:card :dataset_query]+ magic.util/do-with-legacy-query+ update-in [:query entity-element]+ (fnil into (empty element-value))+ element-value))](update dashboard :dashcards (partial map splice-element)))dashboard))@@ -841,7 +863,9 @@(defmethod automagic-analysis :model/Query[query {:keys [cell-query] :as opts}](let [root (->root query)- cell-query (when cell-query (mbql.normalize/normalize-fragment [:query :filter] cell-query))+ cell-query (when cell-query+ #_{:clj-kondo/ignore [:deprecated-var]}+ (mbql.normalize/normalize-fragment [:query :filter] cell-query))opts (cond-> optscell-query (assoc :cell-query cell-query))cell-url (format "%sadhoc/%s/cell/%s" public-endpointdiff --git a/src/metabase/xrays/automagic_dashboards/filters.clj b/src/metabase/xrays/automagic_dashboards/filters.cljindex ec104e35ed0..a01cddb4d90 100644--- a/src/metabase/xrays/automagic_dashboards/filters.clj+++ b/src/metabase/xrays/automagic_dashboards/filters.clj@@ -4,7 +4,9 @@[metabase.legacy-mbql.util :as mbql.u][metabase.util :as u][metabase.util.date-2 :as u.date]+ [metabase.util.malli :as mu][metabase.warehouse-schema.models.field :as field]+ [metabase.xrays.automagic-dashboards.schema :as ads][metabase.xrays.automagic-dashboards.util :as magic.util][toucan2.core :as t2]))@@ -75,7 +77,7 @@(defn- add-filter[dashcard filter-id field](let [mappings (->> (conj (:series dashcard) (:card dashcard))- (keep (fn [card]+ (keep (mu/fn [card :- [:maybe ::ads/card]](when-let [target (filter-for-card card field)]{:parameter_id filter-id:target target@@ -113,10 +115,11 @@(partial remove (fn [{:keys [fingerprint]}](some-> fingerprint :global :distinct-count (< 2)))))-(defn add-filters+(mu/defn add-filters"Add up to `max-filters` filters to dashboard `dashboard`. The `dimensions` argument is a list of fields for which tocreate filters."- [dashboard dimensions max-filters]+ [dashboard :- ::ads/dashboard+ dimensions max-filters](let [fks (when-let [table-ids (not-empty (set (keep (comp :table_id :card)(:dashcards dashboard))))](field/with-targets (t2/select :model/Fielddiff --git a/src/metabase/xrays/automagic_dashboards/interesting.clj b/src/metabase/xrays/automagic_dashboards/interesting.cljindex 89d3f413c75..e792df9b836 100644--- a/src/metabase/xrays/automagic_dashboards/interesting.clj+++ b/src/metabase/xrays/automagic_dashboards/interesting.clj@@ -51,6 +51,7 @@[metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.malli :as mu]+ [metabase.util.malli.schema :as ms][metabase.warehouse-schema.models.field :as field][metabase.xrays.automagic-dashboards.dashboard-templates :as dashboard-templates][metabase.xrays.automagic-dashboards.schema :as ads]@@ -60,37 +61,6 @@;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Code for creation of instantiated affinities-(defn find-field-ids- "A utility function for pulling field definitions from mbql queries and return their IDs.- Does something like this already exist in our utils? I was unable to find anything like it."- [m]- (let [fields (atom #{})]- (walk/prewalk- (fn [v]- (when (vector? v)- (let [[f id] v]- (when (and id (= :field f))- (swap! fields conj id))))- v)- m)- @fields))--(defn semantic-groups- "From a :xrays/Metric, construct a mapping of semantic types of linked fields to- sets of fields that can satisfy that type. A linked field is one that is in the- source table for the metric contribute to the metric itself, is not a PK, and- has a semantic_type (we assume nil semantic_type fields are boring)."- [{:keys [table_id definition]}]- (let [field-ids (find-field-ids definition)- potential-dimensions (t2/select :model/Field- :id [:not-in field-ids]- :table_id table_id- :semantic_type [:not-in [:type/PK]])]- (update-vals- (->> potential-dimensions- (group-by :semantic_type))- set)))-(defmulti ->reference"Get a reference for a given model to be injected into a template (either MBQL, native query, or string)."{:arglists '([template-type model])}@@ -119,7 +89,7 @@(can-use? :hour) :hour))(if (can-use? :day) :day :hour))))-(defmethod ->reference [:mbql :model/Field]+(mu/defmethod ->reference [:mbql :model/Field] :- ::mbql.s/field[_ {:keys [fk_target_field_id id link aggregation name base_type] :as field}](let [reference (mbql.normalize/normalize(cond@@ -304,10 +274,11 @@(def ^:private ^{:arglists '([field])} id-or-name(some-fn :id :name))-(defn- candidate-bindings+(mu/defn- candidate-bindings"For every field in a given context determine all potential dimensions each field may map to.This will return a map of field id (or name) to collection of potential matching dimensions."- [context dimension-specs]+ [context :- ::ads/context+ dimension-specs];; TODO - Fix this so that the intermediate representations aren't so crazy.;; all-bindings a map of binding dim identifier to binding def which contains;; field matches which are all the same field except they are merged with the binding.@@ -390,8 +361,9 @@;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; TODO - Deduplicate from core-(def ^:private ^{:arglists '([source])} source->db- (comp (partial t2/select-one :model/Database :id) (some-fn :db_id :database_id)))+(mu/defn- source->db :- (ms/InstanceOf :model/Database)+ [source :- (ms/InstanceOf #{:model/Table :model/Card})]+ (t2/select-one :model/Database :id ((some-fn :db_id :database_id) source)))(defn- enriched-field-with-sources [{:keys [tables source]} field](assoc fielddiff --git a/src/metabase/xrays/automagic_dashboards/names.clj b/src/metabase/xrays/automagic_dashboards/names.cljindex 772c907e489..0973a863945 100644--- a/src/metabase/xrays/automagic_dashboards/names.clj+++ b/src/metabase/xrays/automagic_dashboards/names.clj@@ -63,9 +63,9 @@[root :- ::ads/rootquestion :- [:map[:dataset_query ::ads/query]]]- (let [aggregations (->> (get-in question [:dataset_query :query :aggregation])+ (let [aggregations (->> (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :aggregation])(metric->description root))- dimensions (->> (get-in question [:dataset_query :query :breakout])+ dimensions (->> (magic.util/do-with-legacy-query (:dataset_query question) get-in [:query :breakout])(mapcat magic.util/collect-field-references)(map (partial magic.util/->field root))(map :display_name)@@ -231,7 +231,8 @@(defn cell-title"Return a cell title given a root object and a cell query."[root cell-query]- (str/join " " [(if-let [aggregation (get-in root [:entity :dataset_query :query :aggregation])]+ (str/join " " [(if-let [aggregation (-> (get-in root [:entity :dataset_query])+ (magic.util/do-with-legacy-query get-in [:query :aggregation]))](metric->description root aggregation)(:full-name root))(tru "where {0}" (humanize-filter-value root cell-query))]))diff --git a/src/metabase/xrays/automagic_dashboards/populate.clj b/src/metabase/xrays/automagic_dashboards/populate.cljindex e5a1e862285..ffe69de6767 100644--- a/src/metabase/xrays/automagic_dashboards/populate.clj+++ b/src/metabase/xrays/automagic_dashboards/populate.clj@@ -1,17 +1,21 @@(ns metabase.xrays.automagic-dashboards.populate"Create and save models that make up automagic dashboards."(:require+ [clojure.set :as set][clojure.string :as str][medley.core :as m][metabase.api.common :as api][metabase.appearance.core :as appearance]+ [metabase.lib-be.core :as lib-be][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.id :as lib.schema.id][metabase.queries.core :as queries][metabase.query-processor.util :as qp.util]+ [metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.xrays.automagic-dashboards.filters :as filters]+ [metabase.xrays.automagic-dashboards.schema :as ads][metabase.xrays.automagic-dashboards.util :as magic.util][toucan2.core :as t2]))@@ -85,7 +89,7 @@Colors are then determined by using the hashs of color keys to index into the vectorof available colors."- [{:keys [visualization dataset_query]}]+ [{:keys [visualization dataset_query], :as _dashcard}](let [display (first visualization)breakout (-> dataset_query :query :breakout)aggregation (-> dataset_query :query :aggregation)]@@ -106,8 +110,8 @@(defn- visualization-settings[{:keys [metrics x_label y_label series_labels visualizationdimensions dimension-name->field metric-definition]- :as card}]- (let [{:keys [aggregation]} metric-definition+ :as dashcard}]+ (let [{:keys [aggregation]} metric-definition[display visualization-settings] visualizationviz-dims (mapv(comp :name dimension-name->field ffirst)@@ -117,40 +121,40 @@(assoc :graph.series_labels (map :name metrics):graph.metrics (mapv first aggregation):graph.dimensions (seq viz-dims))- (merge (colorize card))+ (merge (colorize dashcard))(cond->series_labels (assoc :graph.series_labels series_labels)-x_label (assoc :graph.x_axis.title_text x_label)-y_label (assoc :graph.y_axis.title_text y_label)))}))-(defn card-defaults+(defn dashcard-defaults"Default properties for a dashcard on magic dashboard."[]{:id (gensym):dashboard_tab_id nil:visualization_settings {}})-(mu/defn- add-card+(mu/defn- add-normal-dashcard :- ::ads/dashboard"Add a card to dashboard `dashboard` at position [`x`, `y`]."- [dashboard- {:keys [title description dataset_query width height id] :as card} :- [:map- [:dataset_query- [:map- [:database ::lib.schema.id/database]]]]+ [dashboard :- ::ads/dashboard+ {query :dataset_query, :keys [title description width height id] :as dashcard} :- ::ads/card-template[x y]]- (let [card (-> {:creator_id api/*current-user-id*- :dataset_query dataset_query- :description description- :name title- :collection_id nil- :id (or id (gensym))}- (merge (visualization-settings card))- (as-> $card (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]- (queries/populate-card-query-fields $card))))]+ (let [query-fields (when query+ (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (-> {:dataset_query (lib-be/normalize-query query)}+ queries/populate-card-query-fields+ (select-keys [:query_type :database_id :table_id]))))+ card (merge+ {:creator_id api/*current-user-id*+ :dataset_query query+ :description description+ :name title+ :collection_id nil+ :id (or id (gensym))}+ (visualization-settings dashcard)+ query-fields)](update dashboard :dashcards conj- (merge (card-defaults)+ (merge (dashcard-defaults){:col y:row x:size_x width@@ -159,11 +163,13 @@:card_id (:id card):visualization_settings {}}))))-(defn add-text-card+(mu/defn add-text-card :- ::ads/dashboard"Add a text card to dashboard `dashboard` at position [`x`, `y`]."- [dashboard {:keys [text width height visualization-settings]} [x y]]+ [dashboard :- ::ads/dashboard+ {:keys [text width height visualization-settings]} :- ::ads/card-template+ [x y] :- [:tuple nat-int? nat-int?]](update dashboard :dashcards conj- (merge (card-defaults)+ (merge (dashcard-defaults){:creator_id api/*current-user-id*:visualization_settings (merge{:text text@@ -175,17 +181,24 @@:col y:row x:size_x width- :size_y height- :card nil})))+ :size_y height})))-(defn- make-grid- [width height]+(mr/def ::grid+ [:sequential [:sequential :boolean]])++(mu/defn- make-grid :- ::grid+ [width :- nat-int?+ height :- nat-int?](vec (repeat height (vec (repeat width false)))))-(defn- fill-grid+(mu/defn- fill-grid :- ::grid"Mark a rectangular area starting at [`x`, `y`] of size [`width`, `height`] asoccupied."- [grid [x y] {:keys [width height]}]+ [grid :- ::grid+ [x y] :- [:tuple nat-int? nat-int?]+ {:keys [width height], :as _dashcard} :- [:map+ [:width nat-int?]+ [:height nat-int?]]](reduce (fn [grid xy](assoc-in grid xy true))grid@@ -193,24 +206,30 @@y (range y (+ y width))][x y])))-(defn- accomodates?+(mu/defn- accomodates?"Can we place card on grid starting at [x y] (top left corner)?Since we are filling the grid top to bottom and the cards are rectangulard,it suffices to check just the first (top) row."- [grid [x y] {:keys [width height]}]+ [grid :- ::grid+ [x y] :- [:tuple nat-int? nat-int?]+ {:keys [width height]} :- [:map+ [:width nat-int?]+ [:height nat-int?]]](and (<= (+ x height) (count grid))(<= (+ y width) (-> grid first count))(every? false? (subvec (grid x) y (+ y width)))))-(defn- card-position+(mu/defn- dashcard-position"Find position on the grid where to put the card.We use the dumbest possible algorithm (the grid size is relatively small, sowe should be fine): startting at top left move along the grid from left toright, row by row and try to place the card at each position until we find anunoccupied area. Mark the area as occupied."- [grid start-row card]+ [grid :- ::grid+ start-row+ dashcard](reduce (fn [grid xy]- (if (accomodates? grid xy card)+ (if (accomodates? grid xy dashcard)(reduced xy)grid))grid@@ -218,14 +237,14 @@y (range (count (first grid)))][x y])))-(defn- bottom-row+(mu/defn- bottom-row"Find the bottom of the grid. Bottom is the first completely empty row withanother empty row below it."- [grid]+ [grid :- ::grid](let [row {:height 0, :width grid-width}](loop [bottom (long 0)]- (let [[bottom _] (card-position grid bottom row)- [next-bottom _] (card-position grid (inc bottom) row)]+ (let [[bottom _] (dashcard-position grid bottom row)+ [next-bottom _] (dashcard-position grid (inc bottom) row)](if (= (inc bottom) next-bottom)bottom(recur (long next-bottom)))))))@@ -235,21 +254,20 @@(def ^:private ^Long ^:const group-heading-height 2)-(mu/defn- add-group- [dashboard grid group cards :- [:sequential- [:map- [:dataset_query- [:map- [:database ::lib.schema.id/database]]]]]]+(mu/defn- add-group :- [:tuple ::ads/dashboard ::grid]+ [dashboard :- ::ads/dashboard+ grid :- ::grid+ group+ cards :- [:sequential ::ads/card-template]](let [start-row (bottom-row grid)start-row (cond-> start-rowgroup (+ group-heading-height))]- (reduce (fn [[dashboard grid] card]- (let [xy (card-position grid start-row card)]- [(if (text-card? card)- (add-text-card dashboard card xy)- (add-card dashboard card xy))- (fill-grid grid xy card)]))+ (reduce (fn [[dashboard grid] dashcard]+ (let [xy (dashcard-position grid start-row dashcard)]+ [(if (text-card? dashcard)+ (add-text-card dashboard dashcard xy)+ (add-normal-dashcard dashboard dashcard xy))+ (fill-grid grid xy dashcard)]))(if group(let [xy [(- start-row 2) 0]card {:text (format "# %s" (:title group))@@ -262,7 +280,7 @@[dashboard grid])cards)))-(defn- shown-cards+(mu/defn- shown-cards :- [:sequential ::ads/card-template]"Pick up to `max-cards` with the highest `:card-score`.Keep groups together if possible by pulling all the cards within together andusing the same (highest) card-score for all.@@ -270,7 +288,8 @@favourized, but it is still possible that not all cards in a group make it(consider a group of 4 cards which starts as 7/9; in that case only 2 cardsfrom the group will be picked)."- [max-cards cards]+ [max-cards :- nat-int?+ cards :- [:maybe [:sequential ::ads/card-template]]](->> cards(sort-by :card-score >)(take max-cards)@@ -299,17 +318,29 @@(let [g (group-by f coll)](access key-order g))))-(mu/defn create-dashboard+(mu/defn- create-dashboard-populate-dashcards :- [:sequential ::ads/dashcard]+ [dashcards :- [:maybe [:sequential ::ads/card-template]]]+ (let [card-id->can-run-adhoc-query (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (into {}+ (map (juxt ::id :can_run_adhoc_query))+ (queries/with-can-run-adhoc-query+ (for [{:keys [card]} dashcards+ :when (and (:id card)+ (:dataset_query card))]+ (-> card+ (update :dataset_query lib-be/normalize-query)+ (set/rename-keys {:id ::id}))))))]+ (for [dashcard dashcards+ :let [card (:card dashcard)+ card (when (seq card)+ (assoc card :can_run_adhoc_query (get card-id->can-run-adhoc-query (:id card))))]]+ (u/assoc-dissoc dashcard :card card))))++(mu/defn create-dashboard :- ::ads/dashboard"Create dashboard and populate it with cards."([dashboard] (create-dashboard dashboard :all))- ([{:keys [title transient_title description groups filters cards]} :- [:map- [:cards- [:sequential- [:map- [:dataset_query- [:map- [:database ::lib.schema.id/database]]]]]]]- n]+ ([{:keys [title transient_title description groups filters cards]} :- ::ads/dashboard-template+ n :- [:or pos-int? :keyword]](let [n (cond(= n :all) (count cards)(keyword? n) (Integer/parseInt (name n))@@ -332,15 +363,7 @@;; Height doesn't need to be precise, just some;; safe upper bound.(make-grid grid-width (* n grid-width))]))- dashboard (update dashboard :dashcards (fn [dashcards]- (let [cards (map :card dashcards)]- (mapv- (fn [dashcard card]- (m/assoc-some dashcard :card card))- dashcards- (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]- (queries/with-can-run-adhoc-query cards))))))]-+ dashboard (update dashboard :dashcards create-dashboard-populate-dashcards)](log/debugf "Adding %s cards to dashboard %s:\n%s"(count cards)title@@ -381,8 +404,7 @@offset (->> target:dashcards(map #(+ (:row %) (:size_y %)))- (apply max -1) ; -1 so it neturalizes +1 for spacing- ; if the target dashboard is empty.+ (apply max -1) ; -1 so it neturalizes +1 for spacing if the target dashboard is empty.inc)cards (->> dashboard:dashcards@@ -398,8 +420,7 @@(assoc mapping :card_id card-id)))))))](-> target(assoc :parameters parameters)- (cond->- (not skip-titles?)+ (cond-> (not skip-titles?)(add-text-card {:width grid-width:height group-heading-height:text (format "# %s" (:name dashboard))diff --git a/src/metabase/xrays/automagic_dashboards/schema.clj b/src/metabase/xrays/automagic_dashboards/schema.cljindex 38e389ca80c..bbaaf3f3930 100644--- a/src/metabase/xrays/automagic_dashboards/schema.clj+++ b/src/metabase/xrays/automagic_dashboards/schema.clj@@ -3,24 +3,47 @@[malli.core :as mc][malli.util :as mut][metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms]))(mr/def ::root[:map[:database ::lib.schema.id/database]])+(mr/def ::source+ [:or+ (ms/InstanceOf #{:model/Table :model/Card})+ [:map+ [:entity_type [:and+ qualified-keyword?+ [:fn+ {:error/message ":entity/ keyword"}+ #(= (namespace %) "entity")]]]]])+(mr/def ::context"The big ball of mud data object from which we generate x-rays"(mc/schema[:map- [:source any?]- [:root [:ref ::root]]+ [:source {:optional true} ::source]+ [:root {:optional true} [:ref ::root]][:tables {:optional true} any?][:query-filter {:optional true} any?]]))(mr/def ::query- [:ref ::mbql.s/Query])+ "Schema for the type of MBQL queries handled by X-Rays. Currently, either legacy or MBQL 5 is supported, and functions+ use [[metabase.xrays.automagic-dashboards.util/do-with-legacy-query]]+ or [[metabase.xrays.automagic-dashboards.util/do-with-mbql5-query]] as needed; in the future we can change this to+ only accept MBQL 5 and update any code working with legacy MBQL."+ [:and+ [:map+ [:database ::lib.schema.id/database]]+ [:multi {:dispatch lib/normalized-mbql-version}+ [:mbql-version/mbql5 [:ref ::lib.schema/query]]+ [:mbql-version/legacy [:ref ::mbql.s/Query]]]])(mr/def ::table-id-or-database-id[:and@@ -287,3 +310,45 @@(mg/sample affinities)(mg/sample affinity-matches)(mg/sample grounded-metric))++(mr/def ::field+ [:and+ [:map+ ;; as mentioned elsewhere X-Rays does some kind of insane nonsense and creates fields with types like+ ;; `:type/GenericNumber` when instantiating templates+ [:base_type {:optional true} ::lib.schema.common/base-type]]+ [:fn+ {:error/message "Should be a field with snake_case keys"}+ (complement :base-type)]])++(mr/def ::card+ [:map+ [:id {:optional true} [:or symbol? ::lib.schema.id/card]]+ [:dataset_query {:optional true} ::query]])++(mr/def ::dashcard+ [:map+ [:id {:optional true} [:or symbol? ::lib.schema.id/dashcard]]+ [:card {:optional true} ::card]+ [:card_id {:optional true} [:or symbol? ::lib.schema.id/card]]+ [:col {:optional true} nat-int?]+ [:row {:optional true} nat-int?]+ [:size_x {:optional true} pos-int?]+ [:size_y {:optional true} pos-int?]+ [:visualization_settings {:optional true} map?]+ [:title {:optional true} string?]+ [:card-score {:optional true} number?]])++(mr/def ::dashboard+ [:map+ [:dashcards {:optional true} [:sequential ::dashcard]]+ [:filters {:optional true} [:sequential :any]]])++(mr/def ::card-template+ :map)++(mr/def ::dashboard-template+ "This is somewhat different [[metabase.xrays.automagic-dashboards.schema/DashboardTemplate]], I haven't exactly worked+ out what the schema is supposed to be yet."+ [:map+ [:cards {:optional true} [:maybe [:sequential ::card-template]]]])diff --git a/src/metabase/xrays/automagic_dashboards/util.clj b/src/metabase/xrays/automagic_dashboards/util.cljindex c103f15ab00..74bfa0e2832 100644--- a/src/metabase/xrays/automagic_dashboards/util.clj+++ b/src/metabase/xrays/automagic_dashboards/util.clj@@ -4,9 +4,12 @@[clojure.string :as str][medley.core :as m][metabase.analyze.core :as analyze]- [metabase.legacy-mbql.predicates :as mbql.preds][metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.id :as lib.schema.id][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi][metabase.util :as u]@@ -18,15 +21,21 @@[ring.util.codec :as codec][toucan2.core :as t2]))-(defn field-isa?+(mu/defn field-isa?"`isa?` on a field, checking semantic_type and then base_type"- [{:keys [base_type semantic_type]} t]+ [{:keys [base_type semantic_type]} :- ::ads/field+ ;; for some insane reason this is called with totally imaginary types like `:type/GenericNumber`+ t :- [:and+ qualified-keyword?+ [:fn+ {:error/message "should be a keyword starting with :type/ or :entity/ (not necessarily an actual metabase.types.core type)"}+ #(#{"type" "entity"} (namespace %))]]](or (isa? (keyword semantic_type) t)(isa? (keyword base_type) t)))-(defn key-col?+(mu/defn key-col?"Workaround for our leaky type system which conflates types with properties."- [{:keys [base_type semantic_type name]}]+ [{:keys [base_type semantic_type name]} :- ::ads/field](and (isa? base_type :type/Number)(or (#{:type/PK :type/FK} semantic_type)(let [name (u/lower-case-en name)]@@ -34,9 +43,9 @@(str/starts-with? name "id_")(str/ends-with? name "_id"))))))-(defn filter-tables+(mu/defn filter-tables :- [:sequential ::ads/source]"filter `tables` by `tablespec`, which is just an entity type (eg. :entity/GenericTable)"- [tablespec tables]+ [tablespec tables :- [:maybe [:sequential ::ads/source]]](filter #(-> % :entity_type (isa? tablespec)) tables))(def ^{:arglists '([metric])} saved-metric?@@ -51,9 +60,9 @@"Is this an adhoc metric?"(complement (some-fn saved-metric? custom-expression?)))-(def ^{:arglists '([x]) :doc "Base64 encode"} encode-base64-json- "Encode given object as base-64 encoded JSON."- (comp codec/base64-encode codecs/str->bytes json/encode))+(def ^{:arglists '([x])} encode-base64-json+ "Encode given object as form-encoded base-64-encoded JSON."+ (comp codec/form-encode codec/base64-encode codecs/str->bytes json/encode))(mu/defn field-reference->id :- [:maybe [:or ms/NonBlankString ms/PositiveInt]]"Extract field ID from a given field reference form."@@ -70,7 +79,10 @@::ads/field]]"Return `Field` instance for a given ID or name in the context of root."[{{result-metadata :result_metadata} :source, :as root} :- ::ads/root- field-id-or-name-or-clause :- [:or ms/PositiveInt ms/NonBlankString [:fn mbql.preds/Field?]]]+ field-id-or-name-or-clause :- [:or+ ::lib.schema.id/field+ ms/NonBlankString+ ::mbql.s/field-or-expression-ref]](let [id-or-name (if (sequential? field-id-or-name-or-clause)(field-reference->id field-id-or-name-or-clause)field-id-or-name-or-clause)]@@ -97,3 +109,28 @@"Generate a parameter ID for the given field. In X-ray dashboards a parameter is mapped to a single field only."[field](-> field ((juxt :id :name :unit)) hash str))++(defn do-with-legacy-query+ "Call++ (apply f query args)++ with `query` converted to a legacy MBQL query if needed."+ [query f & args]+ (when (seq query)+ (case (lib/normalized-mbql-version query)+ :mbql-version/legacy (apply f query args)+ :mbql-version/mbql5 (apply f #_{:clj-kondo/ignore [:discouraged-var]} (lib/->legacy-MBQL query) args))))++(defn do-with-mbql5-query+ "Call++ (apply f query args)++ with `query` converted to an MBQL 5 query if needed."+ [query f & args]+ (when (seq query)+ (case (lib/normalized-mbql-version query)+ :mbql-version/legacy (binding [lib.schema/*HACK-disable-join-alias-in-field-ref-validation* true]+ (apply f (lib-be/normalize-query query) args))+ :mbql-version/mbql5 (apply f query args))))diff --git a/src/metabase/xrays/domain_entities/core.clj b/src/metabase/xrays/domain_entities/core.cljindex a37747e0e28..961970c2288 100644--- a/src/metabase/xrays/domain_entities/core.clj+++ b/src/metabase/xrays/domain_entities/core.clj@@ -3,16 +3,21 @@[clojure.string :as str][medley.core :as m][metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib.schema.common :as lib.schema.common][metabase.lib.util.match :as lib.util.match][metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]- [metabase.xrays.domain-entities.specs :refer [domain-entity-specs MBQL]]+ [metabase.xrays.domain-entities.specs :as domain-entities.specs :refer [*domain-entity-specs* MBQL]][toucan2.core :as t2]))-(def ^:private ^{:arglists '([field])} field-type+(mu/defn field-type :- [:or+ ::lib.schema.common/base-type+ ::lib.schema.common/semantic-or-relation-type]"Return the most specific type of a given field."- (some-fn :semantic_type :base_type))+ [field :- [:map+ [:base_type ::lib.schema.common/base-type]]]+ ((some-fn :semantic_type :base_type) field))(def SourceName"A reference to a `SourceEntity`."@@ -26,7 +31,7 @@(def SourceEntity"A source for a card. Can be either a table or another card."- [:or (ms/InstanceOf :model/Table) (ms/InstanceOf :model/Card)])+ (ms/InstanceOf #{:model/Table :model/Card}))(def Bindings"Top-level lexical context mapping source names to their corresponding entity and constituent dimensions. See also@@ -60,13 +65,15 @@(mu/defn mbql-reference :- MBQL"Return MBQL clause for a given field-like object."- [{:keys [id name base_type]}]+ [{:keys [id name base_type]} :- [:map+ [:base_type ::lib.schema.common/base-type]]](if id[:field id nil][:field name {:base-type base_type}]))-(defn- has-attribute?- [entity {:keys [field _domain_entity _has_many]}]+(mu/defn- has-attribute?+ [entity :- SourceEntity+ {:keys [field]} :- ::domain-entities.specs/attribute](condfield (some (fn [col](when (or (isa? (field-type col) field)@@ -74,9 +81,10 @@col))((some-fn :fields :result_metadata) entity))))-(defn satisfies-requierments?+(mu/defn satisfies-requirements?"Does source entity satisfies requirements of given spec?"- [entity {:keys [required_attributes]}]+ [entity :- [:or SourceEntity]+ {:keys [required_attributes]} :- domain-entities.specs/DomainEntitySpec](every? (partial has-attribute? entity) required_attributes))(defn- best-match@@ -93,8 +101,9 @@(lib.util.match/match entity [:dimension dimension] dimension))](resolve-dimension-clauses bindings source entity))))-(defn- instantiate-domain-entity- [table {:keys [name description metrics segments breakout_dimensions type]}]+(mu/defn- instantiate-domain-entity :- ::domain-entities.specs/instantiated-domain-entity+ [table :- (ms/InstanceOf :model/Table)+ {:keys [name description metrics segments breakout_dimensions type]} :- domain-entities.specs/DomainEntitySpec](let [dimensions (into {} (for [field (:fields table)][(-> field field-type clojure.core/name) field]))bindings {name {:entity table@@ -108,13 +117,13 @@:source_table (u/the-id table):name name}))-(defn domain-entity-for-table+(mu/defn domain-entity-for-table :- [:maybe ::domain-entities.specs/instantiated-domain-entity]"Find the best fitting domain entity for given table."- [table]+ [table :- (ms/InstanceOf :model/Table)](let [table (t2/hydrate table :fields)]- (some->> @domain-entity-specs+ (some->> @*domain-entity-specs*vals- (filter (partial satisfies-requierments? table))+ (filter (partial satisfies-requirements? table))best-match(instantiate-domain-entity table))))@@ -122,4 +131,5 @@"Fake hydration function."[tables](for [table tables]+ ;; TODO (Cam 9/29/25) -- this key is only used in Clojure-land, there's no reason we should use snake-case for it(assoc table :domain_entity (domain-entity-for-table table))))diff --git a/src/metabase/xrays/domain_entities/malli.clj b/src/metabase/xrays/domain_entities/malli.cljdeleted file mode 100644index 0f6a245fe7d..00000000000--- a/src/metabase/xrays/domain_entities/malli.clj+++ /dev/null@@ -1,127 +0,0 @@-(ns metabase.xrays.domain-entities.malli- (:refer-clojure :exclude [defn])- (:require- [malli.instrument]- [net.cgrand.macrovich :as macros]))--(defmacro -define-getter- "Generates an accessor, given the symbol and path to the value."- [sym path]- `(clojure.core/defn ~(vary-meta sym assoc :export true)- ~(str "Accessor for `" path "`.")- [obj#]- (get-in obj# ~path)))--(defmacro -define-converter- "Incoming converter for the replacement value. `identity` in CLJ."- [schema path in-sym]- `(def ~in-sym- ~(macros/case- :cljs `(-> ~schema- (metabase.xrays.domain-entities.malli/schema-for-path ~path)- metabase.xrays.domain-entities.converters/incoming)- :clj `identity)))--(defmacro -define-setter- "Generates a setter. Prefixes the symbol with `with-`, ie. `with-foo-bar`.- `in-sym` is the name of the incoming converter defined by [[-define-converter]].- Calls that converter on the new value before `assoc`ing it in."- [sym path in-sym]- `(clojure.core/defn ~(vary-meta (symbol (str "with-" (name sym)))- assoc :export true)- ~(str "Updater for `" path "`.")- [obj# new-value#]- (assoc-in obj# ~path (~in-sym new-value#))))--(defmacro -define-js-converter- "Generates the outgoing converter from CLJS data structures to vanilla JS objects.- Generates nothing in CLJ mode."- [schema path out-sym]- (macros/case- :cljs `(def ~out-sym- (metabase.xrays.domain-entities.converters/outgoing- (metabase.xrays.domain-entities.malli/schema-for-path ~schema ~path)))))--(defmacro -define-js-returning-getter- "Generates a getter that converts back to a JS object, in CLJS.- Generates nothing in CLJ.- `sym` is the main symbol, eg. `foo-bar`. `out-sym` is the outgoing converter defined by- [[-define-js-converter]], eg. `foo-bar->`."- [sym path out-sym]- (macros/case- :cljs `(clojure.core/defn ~(vary-meta (symbol (str (name sym) "-js"))- assoc :export true)- ~(str "Fetches `" path "` and converts it to plain JS.")- [obj#]- (~out-sym (~sym obj#)))))--(defmacro -define-getter-and-setter- "Generates the getter, setter and necessary JS<->CLJS converters for a single `sym` and `path` pair.-- In CLJ, this generates the getter, setter and a dummy incoming converter that is just `identity`.-- In CLJS, generates the getter and setter, real converters in both directions, and a getter that returns- vanilla JS objects instead of CLJS data."- [schema sym path]- (let [in-sym (vary-meta (symbol (str "->" (name sym)))- assoc :private true)- out-sym (vary-meta (symbol (str (name sym) "->"))- assoc :private true)]- `(do- (-define-getter ~sym ~path)- (-define-converter ~schema ~path ~in-sym)- (-define-setter ~sym ~path ~in-sym)- (-define-js-converter ~schema ~path ~out-sym)- (-define-js-returning-getter ~sym ~path ~out-sym))))--(defmacro define-getters-and-setters- "Generates an accessor (`get-in`) and updater (`assoc-in`) for each specified path.-- For example:- ```- (define-getters-and-setters Question- dataset-query [:card :dataset-query]- cache-ttl [:card :cache-ttl])- ```- will generate:- ```- (mu/defn ^:export dataset-query :- DatasetQuery- \"Accessor for [:card :dataset-query].\"- [obj :- Question]- (get-in obj [:card :dataset-query]))-- ;; This converter is always defined, but it's `identity` in CLJ.- ;; Note that it's safe to call these converters even if the incoming data is already CLJS.- ;; This is what's generated in CLJS:- (def ^:private ->dataset-query- (converters/incoming DatasetQuery))-- (mu/defn ^:export with-dataset-query :- Question- \"Updater for [:card :dataset-query].\"- [obj :- Question new-value :- DatasetQuery]- (assoc-in obj [:card :dataset-query] (->dataset-query new-value)))-- ;; This converter is only generated in CLJS.- (def ^:private dataset-query->- (converters/outgoing Question))-- ;; This function is also only generated in CLJS.- (mu/defn ^:export dataset-query-js :- :any- \"Fetches `[:card :dataset-query]` and converts it to plain JS.\"- [obj :- Question]- (dataset-query-> (dataset-query obj)))-- ;; ... and the same five things generated for cache-ttl and any other args.- ```-- You provide the schema for the parent object; the macro will examine that schema to determine the- schema for the field being fetched or updated. The updater's name gets prefixed with `with-`, and the- JS-returning getter suffixed with `-js`.-- The converters are private and intended to be internal to the macros. Since they only depend on the- schema it's more efficient to compute them once and reuse them."- [schema sym path & more]- `(do- (-define-getter-and-setter ~schema ~sym ~path)- ~(when (seq more)- `(define-getters-and-setters ~schema ~@more))))diff --git a/src/metabase/xrays/domain_entities/malli.cljs b/src/metabase/xrays/domain_entities/malli.cljsdeleted file mode 100644index 6609e12e06a..00000000000--- a/src/metabase/xrays/domain_entities/malli.cljs+++ /dev/null@@ -1,22 +0,0 @@-(ns metabase.xrays.domain-entities.malli- (:require- [malli.core :as mc]- [malli.util :as mut]- [metabase.xrays.domain-entities.converters])- (:require-macros [metabase.xrays.domain-entities.malli]))--(clojure.core/defn schema-for-path- "Given a schema and a *value path* (as opposed to a *schema path*), finds the schema for that- path. Throws if there are multiple such paths and those paths have different schemas."- [schema path]- (let [paths (-> schema mc/schema (mut/in->paths path))]- (cond- (empty? paths) (throw (ex-info "Path does not match schema" {:schema schema :path path}))- (= (count paths) 1) (mut/get-in schema (first paths))- :else (let [child-schemas (map #(mut/get-in schema %) paths)]- (if (apply = child-schemas)- (first child-schemas)- (throw (ex-info "Value path has multiple schema paths, with different schemas"- {:schema schema- :paths paths- :child-schemas child-schemas})))))))diff --git a/src/metabase/xrays/domain_entities/specs.clj b/src/metabase/xrays/domain_entities/specs.cljindex 76a9e03f07d..236d72fcb6f 100644--- a/src/metabase/xrays/domain_entities/specs.clj+++ b/src/metabase/xrays/domain_entities/specs.clj@@ -4,7 +4,11 @@[malli.transform :as mtx][medley.core :as m][metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.legacy-mbql.schema :as mbql.s][metabase.legacy-mbql.util :as mbql.u]+ [metabase.lib.schema.id :as lib.schema.id]+ [metabase.util.malli.registry :as mr]+ [metabase.util.malli.schema :as ms][metabase.util.yaml :as yaml]))(def MBQL@@ -15,7 +19,9 @@:error/message "valid MBQL clause"}mbql.u/mbql-clause?])-(def FieldType+;;; TODO (Cam 9/29/25) -- these decoding rules are BUSTED because it ends up creating totally nonsensical types like+;;; `:type/AvgPrice` which is not a real type at all. We should probably just leave the field names as strings.+(def ^:private BrokenFieldNameTypeKeyword"Field type designator -- a keyword derived from `type/*`"[:keyword(letfn [(decoder [k]@@ -36,13 +42,16 @@(def ^:private Description :string)-(def ^:private Attributes- [:sequential- [:map- [:field {:optional true} FieldType]- [:domain_entity {:optional true} DomainEntityReference]- [:has_many {:optional true} [:map- [:domain_entity DomainEntityReference]]]]])+(mr/def ::attribute+ [:map+ [:field {:optional true} BrokenFieldNameTypeKeyword]+ [:domain_entity {:optional true} DomainEntityReference]+ [:has_many {:optional true} [:map+ [:domain_entity DomainEntityReference]]]])++(mr/def ::attributes+ "Schema for `required_attributes` and `optional_attributes` in a [[DomainEntitySpec]]."+ [:sequential ::attribute])(def ^:private BreakoutDimensions[:sequential@@ -50,7 +59,7 @@(for [dimension breakout-dimensions](if (string? dimension)(do- (mc/assert FieldType (keyword "type" dimension))+ (mc/assert BrokenFieldNameTypeKeyword (keyword "type" dimension))[:dimension dimension])dimension)))}MBQL])@@ -84,9 +93,9 @@[:map[:name DomainEntityReference][:type DomainEntityType]- [:required_attributes Attributes]+ [:required_attributes ::attributes][:description {:optional true} Description]- [:optional_attributes {:optional true} Attributes]+ [:optional_attributes {:optional true} ::attributes][:metrics {:optional true} Metrics][:segments {:optional true} Segments][:breakout_dimensions {:optional true} BreakoutDimensions]])@@ -110,8 +119,47 @@(def ^:private domain-entities-dir "domain_entity_specs/")-(def domain-entity-specs- "List of registered domain entities."+(def ^:dynamic *domain-entity-specs*+ "Delay with registered domain entities specs, loaded from YAML and coerced to match the [[DomainEntitySpec]] schema."(delay (into {} (for [spec (yaml/load-dir domain-entities-dir (comp coerce-to-domain-entity-specadd-to-hiearchy!))][(:name spec) spec]))))++(mr/def ::field-name :string)++(mr/def ::instantiated-metric+ [:map+ {:closed true}+ [:name :string]+ [:aggregation ::mbql.s/Aggregation]+ [:filter {:optional true} ::mbql.s/Filter]])++(mr/def ::instantiated-metrics+ [:map-of ::field-name ::instantiated-metric])++(mr/def ::instantiated-segment+ [:map+ {:closed true}+ [:name :string]+ [:filter ::mbql.s/Filter]])++(mr/def ::instantiated-segments+ [:map-of ::field-name ::instantiated-segment])++(mr/def ::instantiated-breakouts+ [:sequential ::mbql.s/field])++(mr/def ::instantiated-domain-entity+ "Schema for a [[DomainEntitySpec]] that has been instantiated+ by [[metabase.xrays.domain-entities.core/instantiate-domain-entity]], i.e. the `[:dimension <field-name>]`+ placeholder clauses have been replaced with real Field refs."+ [:map+ {:closed true}+ [:breakout_dimensions [:maybe ::instantiated-breakouts]]+ [:description [:maybe Description]]+ [:dimensions [:map-of ::field-name (ms/InstanceOf :model/Field)]]+ [:metrics [:maybe ::instantiated-metrics]]+ [:name DomainEntityReference]+ [:segments [:maybe ::instantiated-segments]]+ [:source_table ::lib.schema.id/table]+ [:type DomainEntityType]])diff --git a/src/metabase/xrays/related.clj b/src/metabase/xrays/related.cljindex bb09d20b773..675f25ff213 100644--- a/src/metabase/xrays/related.clj+++ b/src/metabase/xrays/related.clj@@ -8,6 +8,7 @@[metabase.models.interface :as mi][metabase.query-processor.util :as qp.util][metabase.util.malli.registry :as mr]+ [metabase.xrays.automagic-dashboards.util :as magic.util][toucan2.core :as t2]))(def ^:private ^Long max-best-matches 3)@@ -27,7 +28,8 @@(defn- collect-context-bearing-forms[form]- (let [form (mbql.normalize/normalize-fragment [:query :filter] form)]+ ;; legacy usage, temporary until we convert X-Rays to Lib+ (let [form #_{:clj-kondo/ignore [:deprecated-var]} (mbql.normalize/normalize-fragment [:query :filter] form)](into #{}(comp (filter (mr/validator ContextBearingForm))(map #(update % 0 qp.util/normalize-token)))@@ -47,8 +49,7 @@[card](-> card:dataset_query- :query- ((juxt :breakout :aggregation :expressions :fields))))+ (magic.util/do-with-legacy-query #(-> % :query ((juxt :breakout :aggregation :expressions :fields))))))(defmethod definition :model/Segment[segment]diff --git a/src/metabase/xrays/transforms/core.clj b/src/metabase/xrays/transforms/core.cljindex 07f3f15548b..7b5fcc4a6f9 100644--- a/src/metabase/xrays/transforms/core.clj+++ b/src/metabase/xrays/transforms/core.clj@@ -2,7 +2,6 @@(:require[medley.core :as m][metabase.driver :as driver]- [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.schema.id :as lib.schema.id][metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi]@@ -12,13 +11,10 @@[metabase.util.malli :as mu][metabase.util.malli.schema :as ms][metabase.warehouse-schema.models.table :as table]- [metabase.xrays.domain-entities.core- :as de- :refer [Bindings DimensionBindings SourceEntity SourceName]]- [metabase.xrays.domain-entities.specs- :refer [domain-entity-specs DomainEntitySpec]]+ [metabase.xrays.domain-entities.core :as de :refer [Bindings DimensionBindings SourceEntity SourceName]]+ [metabase.xrays.domain-entities.specs :as domain-entities.specs :refer [*domain-entity-specs* DomainEntitySpec]][metabase.xrays.transforms.materialize :as tf.materialize]- [metabase.xrays.transforms.specs :refer [Step transform-specs TransformSpec]]+ [metabase.xrays.transforms.specs :as transforms.specs :refer [*transform-specs* Step TransformSpec]][toucan2.core :as t2]))(mu/defn- add-bindings :- Bindings@@ -44,7 +40,7 @@(mu/defn- infer-resulting-dimensions :- DimensionBindings[bindings :- Bindings{:keys [joins name]} :- Step- query :- mbql.s/Query]+ query :- ::transforms.specs/query](let [flattened-bindings (merge (apply merge (map (comp :dimensions bindings :source) joins))(get-in bindings [name :dimensions]))](into {} (for [{:keys [name] :as col} (qp.preprocess/query->expected-cols query)]@@ -142,7 +138,11 @@:dimensions (infer-resulting-dimensions local-bindings step query)})))(def ^:private Tableset- [:sequential (ms/InstanceOf :model/Table)])+ [:sequential+ [:and+ (ms/InstanceOf :model/Table)+ [:map+ [:domain_entity [:maybe ::domain-entities.specs/instantiated-domain-entity]]]]])(mu/defn- find-tables-with-domain-entity :- Tableset[tableset :- Tableset@@ -171,8 +171,8 @@[bindings :- Bindings{:keys [provides]} :- TransformSpec](doseq [domain-entity-name provides]- (assert (de/satisfies-requierments? (get-in bindings [domain-entity-name :entity])- (@domain-entity-specs domain-entity-name))+ (assert (de/satisfies-requirements? (get-in bindings [domain-entity-name :entity])+ (@*domain-entity-specs* domain-entity-name))(str (tru "Resulting transforms do not conform to expectations.\nExpected: {0}"domain-entity-name))))bindings)@@ -181,7 +181,7 @@[tableset :- Tableset{:keys [requires]} :- TransformSpec](let [matches (map (comp (partial find-tables-with-domain-entity tableset)- @domain-entity-specs)+ @*domain-entity-specs*)requires)](when (every? (comp #{1} count) matches)(map first matches))))@@ -218,4 +218,4 @@[table](filter (comp (partial some (comp #{(u/the-id table)} u/the-id))(partial tables-matching-requirements (tableset (:db_id table) (:schema table))))- @transform-specs))+ @*transform-specs*))diff --git a/src/metabase/xrays/transforms/dashboard.clj b/src/metabase/xrays/transforms/dashboard.cljindex 6f929846c61..d9134add2e0 100644--- a/src/metabase/xrays/transforms/dashboard.clj+++ b/src/metabase/xrays/transforms/dashboard.clj@@ -7,7 +7,7 @@[metabase.util.malli :as mu][metabase.xrays.automagic-dashboards.populate :as populate][metabase.xrays.transforms.materialize :as tf.materialize]- [metabase.xrays.transforms.specs :refer [transform-specs]]+ [metabase.xrays.transforms.specs :refer [*transform-specs*]][toucan2.core :as t2][toucan2.realize :as t2.realize]))@@ -15,9 +15,9 @@(def ^:private ^:const ^Long total-width 18)(def ^:private ^:const ^Long height 4)-(defn- cards->section+(mu/defn- cards->section"Build a section of cards and format them according to what the automagic dashboards code expects."- [group cards]+ [group :- :string cards](mapcat (fn [{:keys [name description display] :as card}](cond-> [(assoc card:group group@@ -62,10 +62,12 @@(defn dashboard"Create a (transient) dashboard for transform named `transform-name`."[transform-name]- (let [transform-spec (m/find-first (comp #{transform-name} :name) @transform-specs)+ (let [transform-spec (or (m/find-first (comp #{transform-name} :name) @*transform-specs*)+ (throw (ex-info (format "Failed to find transform with name %s" (pr-str transform-name))+ {:status-code 404}))){steps false provides true} (->> transform-nametf.materialize/get-collection- (t2/select 'Card :collection_id)+ (t2/select :model/Card :collection_id)(group-by (comp some?(-> transform-spec :provides set):name)))diff --git a/src/metabase/xrays/transforms/materialize.clj b/src/metabase/xrays/transforms/materialize.cljindex dedab8fe7d9..171f411e154 100644--- a/src/metabase/xrays/transforms/materialize.clj+++ b/src/metabase/xrays/transforms/materialize.clj@@ -2,8 +2,12 @@(:require[metabase.api.common :as api][metabase.collections.models.collection :as collection]+ [metabase.lib-be.core :as lib-be][metabase.queries.core :as queries]+ [metabase.queries.schema :as queries.schema][metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.util.malli :as mu]+ [metabase.xrays.transforms.specs :as transforms.specs][toucan2.core :as t2]))(declare get-or-create-root-container-collection!)@@ -49,17 +53,19 @@(t2/delete! :model/Card :collection_id collection-id)(create-collection! name description)))-(defn make-card-for-step!+(mu/defn make-card-for-step! :- ::queries.schema/card"Make and save a card for a given transform step and query."- [{:keys [name transform description]} query]- (->> {:creator_id api/*current-user-id*- :dataset_query query- :description description- :name name- :collection_id (get-collection transform)- :result_metadata (qp.preprocess/query->expected-cols query)- :visualization_settings {}- :display :table}- queries/populate-card-query-fields- (t2/insert-returning-instances! :model/Card)- first))+ [{:keys [name transform description]} :- transforms.specs/Step+ query :- ::transforms.specs/query]+ (let [query (lib-be/normalize-query query)]+ (->> {:creator_id api/*current-user-id*+ :dataset_query query+ :description description+ :name name+ :collection_id (get-collection transform)+ :result_metadata (qp.preprocess/query->expected-cols query)+ :visualization_settings {}+ :display :table}+ queries/populate-card-query-fields+ (t2/insert-returning-instances! :model/Card)+ first)))diff --git a/src/metabase/xrays/transforms/specs.clj b/src/metabase/xrays/transforms/specs.cljindex 95a6bace3d0..1b9701d04b7 100644--- a/src/metabase/xrays/transforms/specs.clj+++ b/src/metabase/xrays/transforms/specs.clj@@ -11,6 +11,11 @@[metabase.util.yaml :as yaml][metabase.xrays.domain-entities.specs :refer [MBQL]]))+(mr/def ::query+ "Schema for MBQL queries as handled by the X-Rays transforms code. (Currently, legacy queries; in the very near future+ we can change this schema to MBQL 5 queries and then go fix all the functions that reference this schema.)"+ ::mbql.s/Query)+(def ^:private DecodableString[:string{:decode/transform-spec (fn [x]@@ -133,6 +138,6 @@(def ^:private transforms-dir "transforms/")-(def transform-specs- "List of registered dataset transforms."+(def ^:dynamic *transform-specs*+ "Delay for all transform specs, loaded from YAML and coerced to the [[TransformSpec]] schema."(delay (yaml/load-dir transforms-dir (comp coerce-to-transform-spec add-metadata-to-steps))))diff --git a/test/metabase/actions/api_test.clj b/test/metabase/actions/api_test.cljindex a8ac8e63608..8d814eeb817 100644--- a/test/metabase/actions/api_test.clj+++ b/test/metabase/actions/api_test.clj@@ -2,9 +2,12 @@(:require[clojure.set :as set][clojure.test :refer :all]+ [medley.core :as m][metabase.actions.api :as api.action][metabase.analytics.snowplow-test :as snowplow-test][metabase.collections.models.collection :as collection]+ [metabase.lib.core :as lib]+ [metabase.lib.schema.id :as lib.schema.id][metabase.search.core :as search][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -40,12 +43,14 @@[:id ms/PositiveInt][:type [:= "query"]][:model_id ms/PositiveInt]- [:database_id ms/PositiveInt]+ [:database_id ::lib.schema.id/database][:dataset_query [:map- [:database ms/PositiveInt]- [:type [:= "native"]]- [:native [:map- [:query :string]]]]]+ [:database ::lib.schema.id/database]+ [:stages [:sequential+ {:min 1, :max 1}+ [:map+ [:lib/type [:= "mbql.stage/native"]]+ [:native string?]]]]]][:parameters :any][:parameter_mappings :any][:visualization_settings :map]@@ -54,7 +59,7 @@[:creator_id ms/PositiveInt][:creator DefaultUser]])-(defn all-actions-default+(defn- all-actions-default[card-id][{:name "Get example":description "A dummy HTTP action"@@ -69,15 +74,15 @@:description "A simple update query action":type "query":model_id card-id- :dataset_query (update (mt/native-query {:query "update users set name = 'foo' where id = {{x}}"})- :type name)+ :dataset_query (lib/native-query (mt/metadata-provider) "update users set name = 'foo' where id = {{x}}"):database_id (t2/select-one-fn :database_id :model/Card :id card-id)- :parameters [{:id "x" :type "type/biginteger"}]}+ :parameters [{:id "x" :type "number"}]}{:name "Implicit example":type "implicit":model_id card-id:kind "row/create"- :parameters [{:id "nonexistent" :special "shouldbeignored"} {:id "id" :special "hello"}]}])+ :parameters [{:id "nonexistent" :special "shouldbeignored", :type "number"}+ {:id "id" :special "hello", :type "number"}]}])(deftest list-actions-test(search/reset-tracking!)@@ -252,18 +257,30 @@(assoc :kind "row/update" :description "A new description")(= (:type initial-action) "query")- (assoc :dataset_query (update (mt/native-query {:query "update users set name = 'bar' where id = {{x}}"})- :type name))+ (assoc :dataset_query+ (lib/native-query (mt/metadata-provider) "update users set name = 'bar' where id = {{x}}"))(= (:type initial-action) "http")(-> (assoc :response_handle ".body.result" :description nil))))expected-fn (fn [m]- (cond-> m- (= (:type initial-action) "implicit")- (assoc :database_id (mt/id)- :parameters (if (= "row/create" (:kind initial-action))- []- [{:id "id" :type "type/BigInteger" :special "hello"}]))))+ (-> m+ (cond-> (= (:type initial-action) "implicit")+ (assoc :database_id (mt/id)+ :parameters (if (= "row/create" (:kind initial-action))+ []+ [{:id "id" :type "number" :special "hello"}])))+ (m/update-existing :dataset_query (letfn [(update-template-tags [template-tags]+ (update-vals template-tags #(dissoc % :id)))+ (update-stage [stage]+ (m/update-existing stage :template-tags update-template-tags))+ (update-stages [stages]+ (mapv update-stage stages))+ (update-query [query]+ (-> query+ mt/obj->json->obj+ (dissoc :lib/metadata)+ (update :stages update-stages)))]+ update-query))))updated-action (update-fn initial-action)](testing "Create fails with"(testing "no permission"@@ -335,7 +352,8 @@:type "implicit":model_id model-id:kind "row/create"- :parameters [{:id "nonexistent" :special "shouldbeignored"} {:id "id" :special "hello"}]})))))))+ :parameters [{:type "number", :id "nonexistent" :special "shouldbeignored"}+ {:type "number", :id "id" :special "hello"}]})))))))(deftest snowplow-test(snowplow-test/with-fake-snowplow-collector@@ -352,7 +370,6 @@"num_parameters" (count parameters)"type" type}}(last (snowplow-test/pop-event-data-and-user-id!)))))-(testing (format "update an action of type %s" type)(let [updated-action (mt/user-http-request :crowberto :put 200 (str "action/" (:id new-action)) {:name "new name"})](is (=? {:user-id (str (mt/user->id :crowberto))@@ -360,7 +377,6 @@"event" "action_updated""type" type}}(last (snowplow-test/pop-event-data-and-user-id!))))))-(testing (format "delete an action of type %s" type)(mt/user-http-request :crowberto :delete 204 (str "action/" (:id new-action)))(is (=? {:user-id (str (mt/user->id :crowberto))@@ -394,27 +410,27 @@(is (=? {:errors {:name "string"},:specific-errors {:name ["missing required key, received: nil"]}}(mt/user-http-request :crowberto :post 400 "action" {:type "http"})))- (is (=? {:errors {:model_id "value must be an integer greater than zero."}+ (is (=? {:errors {:model_id "Valid Card ID"}:specific-errors {:model_id ["missing required key, received: nil"]}}(mt/user-http-request :crowberto :post 400 "action" {:type "http" :name "test"}))))(testing "Handles need to be valid jq"- (is (=? {:errors {:response_handle "nullable string, and must be a valid json-query, something like '.item.title'"}+ (is (=? {:errors {:response_handle "nullable must be a valid json-query, something like '.item.title'"}:specific-errors {:response_handle ["must be a valid json-query, something like '.item.title', received: \"body\""]}}(mt/user-http-request :crowberto :post 400 "action" (assoc initial-action :response_handle "body"))))- (is (=? {:errors {:error_handle "nullable string, and must be a valid json-query, something like '.item.title'"}+ (is (=? {:errors {:error_handle "nullable must be a valid json-query, something like '.item.title'"}:specific-errors {:error_handle ["must be a valid json-query, something like '.item.title', received: \"x\""]}}(mt/user-http-request :crowberto :post 400 "action" (assoc initial-action :error_handle "x"))))))(testing "Validate PUT"(testing "Template needs method and url"(is (=? {:specific-errors{:template {:method ["missing required key, received: nil"], :url ["missing required key, received: nil"]}},- :errors {:template "nullable map where {:method -> <enum of GET, POST, PUT, DELETE, PATCH>, :url -> <string with length >= 1>, :body (optional) -> <nullable string>, :headers (optional) -> <nullable string>, :parameters (optional) -> <nullable sequence of map>, :parameter_mappings (optional) -> <nullable map>} with no other keys"}}+ :errors {:template {:method "enum of GET, POST, PUT, DELETE, PATCH", :url "string with length >= 1"}}}(mt/user-http-request :crowberto :put 400 action-path {:type "http" :template {}}))))(testing "Handles need to be valid jq"- (is (=? {:errors {:response_handle "nullable string, and must be a valid json-query, something like '.item.title'"},+ (is (=? {:errors {:response_handle "nullable must be a valid json-query, something like '.item.title'"},:specific-errors {:response_handle ["must be a valid json-query, something like '.item.title', received: \"body\""]}}(mt/user-http-request :crowberto :put 400 action-path (assoc initial-action :response_handle "body"))))- (is (=? {:errors {:error_handle "nullable string, and must be a valid json-query, something like '.item.title'"},+ (is (=? {:errors {:error_handle "nullable must be a valid json-query, something like '.item.title'"},:specific-errors {:error_handle ["must be a valid json-query, something like '.item.title', received: \"x\""]}}(mt/user-http-request :crowberto :put 400 action-path (assoc initial-action :error_handle "x")))))))))))diff --git a/test/metabase/actions/models_test.clj b/test/metabase/actions/models_test.cljindex b535e2bc2bd..28aea6f29f3 100644--- a/test/metabase/actions/models_test.clj+++ b/test/metabase/actions/models_test.clj@@ -33,20 +33,43 @@:result_metadata (assoc-in (vec (qp.preprocess/query->expected-cols (mt/mbql-query categories)))[1 :display_name] "Display Name")}{:keys [action-id] :as _context} {:type :implicit}]- (is (partial= {:id action-id- :name "Update Example"- :database_id (mt/id)- :parameters [{:type (if (= driver/*driver* :h2) :type/BigInteger :type/Integer)- :id "id"- :display-name "ID"}- {:type :type/Text- :id "name"- :display-name "Display Name"}]- :visualization_settings {:fields {"id" {:id "id"- :hidden false}- "name" {:id "name"- :hidden false}}}}- (action/select-action :id action-id)))))+ (is (=? {:id action-id+ :name "Update Example"+ :database_id (mt/id)+ :parameters [{:type :number+ :id "id"+ :display-name "ID"}+ {:type :text+ :id "name"+ :display-name "Display Name"}]+ :visualization_settings {:fields {"id" {:description nil+ :placeholder "ID"+ :title "ID"+ :hidden false+ :id "id"+ :display_name "ID"+ :inputType :number+ :base_type #(isa? % :type/Integer)+ :fieldType :number+ :required true+ :order 0}+ "name" {:description nil+ :placeholder "Name"+ :title "Name"+ :hidden false+ :id "name"+ :display_name "Name"+ :inputType :string+ :base_type :type/Text+ :fieldType :string+ :required true+ :order 1}}}}+ (action/select-action :id action-id)))))))))++(deftest hydrate-implicit-action-test-1a+ (mt/test-drivers (mt/normal-drivers-with-feature :actions/custom)+ (mt/with-actions-test-data-and-actions-enabled+ (let [query (mt/mbql-query categories)](testing "for implicit actions visualization_settings.fields, "(mt/with-actions [_model {:type :model:dataset_query query@@ -58,15 +81,15 @@:hidden true}}}}](let [field-settings (get-in (action/select-action :id action-id) [:visualization_settings :fields])](testing "an existing entry should not update if there's a matching parameter"- (is (= {:id "id"- :hidden true}- (get field-settings "id"))))+ (is (=? {:id "id"+ :hidden true}+ (get field-settings "id"))))(testing "an existing entry should be deleted if there's not a matching parameter"(is (not (contains? field-settings "doesnt_exist"))))(testing "a entry with defaults should be created if there is a new matching parameter"- (is (= {:id "name"- :hidden false}- (get field-settings "name")))))))))))+ (is (=? {:id "name"+ :hidden false}+ (get field-settings "name")))))))))))(deftest hydrate-implicit-action-test-2(testing "Implicit actions do not map parameters to json fields (parents or nested)"@@ -135,7 +158,7 @@:model/DashboardCard {dashcard-id :id} {:action_id action-id:dashboard_id dashboard-id}](is (= 1 (t2/count :model/DashboardCard :id dashcard-id)))- (action/update! {:id action-id, :archived true} {:id action-id})+ (action/update! {:id action-id, :archived true} (action/select-action :id action-id))(is (zero? (t2/count :model/DashboardCard :id dashcard-id)))))))))(deftest dashcard-deletion-test-2@@ -159,14 +182,14 @@(let [action (action/select-action :id action-id)new-id (action/insert! (dissoc action :id :made_public_by_id :public_uuid :entity_id))cloned-action (action/select-action :id new-id)]- (is (partial= {:kind "row/create"} cloned-action))))+ (is (partial= {:kind :row/create} cloned-action))))(testing "Update action"(let [action (action/select-action :id action-id)];; Update columns on both the action and the subtype table(action/update! (assoc action :name "New name" :kind "row/update") action)(let [new-action (action/select-action :id action-id)](is (partial= {:name "New name"- :kind "row/update"} new-action)))))))))+ :kind :row/update} new-action)))))))))(deftest model-to-saved-question-test(mt/test-drivers (mt/normal-drivers-with-feature :actions/custom)@@ -205,14 +228,14 @@(deftest exclude-auto-increment-fields-for-create-implicit-actions-test(mt/test-drivers (mt/normal-drivers-with-feature :actions/custom)(mt/with-actions-enabled- (doseq [kind ["row/create" "row/update" "row/delete"]]+ (doseq [kind [:row/create :row/update :row/delete]](testing (format "for implicit action with kind=%s we should %s include auto incremented fields"- kind (if (= "row/create" kind) "not" ""))+ kind (if (= :row/create kind) "not" ""))(mt/with-actions [{:keys [action-id]} {:type :implicit:kind kind}](let [parameters (:parameters (action/select-action :id action-id))id-parameter (first (filter #(= "id" (:id %)) parameters))]- (if (= "row/create" kind)+ (if (= kind :row/create)(is (nil? id-parameter))(is (some? id-parameter))))))))))@@ -236,14 +259,14 @@:database_type "UUID":name "uuid":semantic_type :type/PK}]- (t2/select ['Field :name :database_type :database_required :database_is_auto_increment :semantic_type]+ (t2/select [:model/Field :name :database_type :database_required :database_is_auto_increment :semantic_type]:table_id (mt/id :default_uuid))))(is (= [{:database_is_auto_increment false:database_required true:database_type "UUID":name "uuid":semantic_type :type/PK}]- (t2/select ['Field :name :database_type :database_required :database_is_auto_increment :semantic_type]+ (t2/select [:model/Field :name :database_type :database_required :database_is_auto_increment :semantic_type]:table_id (mt/id :required_uuid)))))(mt/with-actions-enabled(testing "PK with a default should be excluded from implicit create action parameters"@@ -292,7 +315,7 @@action-id (action/insert! action-data)select-action #(action/select-action :id action-id)action-after-insert (select-action)]- (is (= [:type/Boolean] (map :type (:parameters action-after-insert))))+ (is (= [:boolean] (map :type (:parameters action-after-insert))))(action/update! (assoc (select-action) :name "create foo2") action-after-insert)(is (= (:parameters action-after-insert) (:parameters (select-action)))))))))@@ -315,13 +338,13 @@(sync/sync-database! (mt/db)))](testing "if a column type is varied the parameter types are updated on select-action"(exec! "ALTER TABLE \"FOO\" ALTER COLUMN \"name\" BIGINT;")- (is (= [["name" :type/BigInteger]] (map (juxt :id :type) (:parameters (action/select-action action-id)))))+ (is (= [["name" :number]] (map (juxt :id :type) (:parameters (action/select-action action-id)))))(testing "if the column type is varied after an update to the action, it is reflected in the parameters"(let [action (action/select-action action-id)](action/update! (assoc action :name "create foo2") action)(exec! "ALTER TABLE \"FOO\" ALTER COLUMN \"name\" TEXT;")(sync/sync-database! (mt/db))- (is (= [["name" :type/Text]] (map (juxt :id :type) (:parameters (action/select-action action-id))))))))+ (is (= [["name" :text]] (map (juxt :id :type) (:parameters (action/select-action action-id))))))))(testing "if a column added, the model column set is used, not the table"(exec! "ALTER TABLE \"FOO\" ADD COLUMN \"name2\" BIGINT;")(is (= ["name"] (map :id (:parameters (action/select-action action-id))))))diff --git a/test/metabase/actions/test_util.clj b/test/metabase/actions/test_util.cljindex 1e98d2f9387..5e147d029f4 100644--- a/test/metabase/actions/test_util.clj+++ b/test/metabase/actions/test_util.clj@@ -19,7 +19,8 @@[metabase.test.util :as tu][metabase.util.honey-sql-2 :as h2x][metabase.util.random :as u.random]- [toucan2.core :as t2]))+ [toucan2.core :as t2]+ [toucan2.tools.with-temp :as t2.with-temp]))(set! *warn-on-reflection* true)@@ -80,7 +81,6 @@You can use [[with-actions-test-data-tables]] if you need something other than the `categories` table, e.g. fortesting FK constraints."(ActionsTestDatasetDefinition.))-(map :table-name actions-test-data)(defn do-with-dataset-definition"Impl for [[with-temp-test-data]] and [[with-actions-test-data]] macros."@@ -168,79 +168,94 @@(is (= [[74]](row-count))))))))))-(defn do-with-action+(defmulti ^:private create-action*!+ {:arglists '([options-map model-id])}+ (fn [options-map _model-id]+ (:type options-map)))++(defmethod create-action*! :query+ [options-map model-id]+ (let [action-id (action/insert!+ (merge {:model_id model-id+ :name "Query Example"+ :parameters [{:id "id"+ :slug "id"+ :type "number"+ :target [:variable [:template-tag "id"]]}+ {:id "name"+ :slug "name"+ :type "text"+ :required false+ :target [:variable [:template-tag "name"]]}]+ :visualization_settings {:inline true}+ :public_uuid (str (random-uuid))+ :made_public_by_id (test.users/user->id :crowberto)+ :database_id (data/id)+ :creator_id (test.users/user->id :crowberto)+ :dataset_query {:database (data/id)+ :type :native+ :native {:query (str "UPDATE categories\n"+ "SET name = concat([[{{name}}, ' ',]] 'Sh', 'op')\n"+ "WHERE id = {{id}}")+ :template-tags {"id" {:name "id"+ :display-name "ID"+ :type :number+ :required true}+ "name" {:name "name"+ :display-name "Name"+ :type :text+ :required false}}}}}+ options-map))]+ {:action-id action-id :model-id model-id}))++(defmethod create-action*! :implicit+ [options-map model-id]+ (let [action-id (action/insert! (merge+ {:type :implicit+ :name "Update Example"+ :kind "row/update"+ :public_uuid (str (random-uuid))+ :made_public_by_id (test.users/user->id :crowberto)+ :creator_id (test.users/user->id :crowberto)+ :model_id model-id}+ options-map))]+ {:action-id action-id :model-id model-id}))++(defmethod create-action*! :http+ [options-map model-id]+ (let [action-id (action/insert! (merge+ {:type :http+ :name "Echo Example"+ :template {:url (client/build-url "testing/echo[[?fail={{fail}}]]" {})+ :method "POST"+ :body "{\"the_parameter\": {{id}}}"+ :headers "{\"x-test\": \"{{id}}\"}"}+ :parameters [{:id "id"+ :type "number"+ :target [:dimension [:template-tag "id"]]}+ {:id "fail"+ :type "text"+ :target [:dimension [:template-tag "fail"]]}]+ :response_handle ".body"+ :model_id model-id+ :public_uuid (str (random-uuid))+ :made_public_by_id (test.users/user->id :crowberto)+ :creator_id (test.users/user->id :crowberto)}+ options-map))]+ {:action-id action-id :model-id model-id}))++(defn create-action!"Impl for [[with-action]]."[options-map model-id](let [options-map (merge options-map {:created_at (t/zoned-date-time):updated_at (t/zoned-date-time)})]- (case (:type options-map)- :query- (let [action-id (action/insert!- (merge {:model_id model-id- :name "Query Example"- :parameters [{:id "id"- :slug "id"- :type "number"- :target [:variable [:template-tag "id"]]}- {:id "name"- :slug "name"- :type "text"- :required false- :target [:variable [:template-tag "name"]]}]- :visualization_settings {:inline true}- :public_uuid (str (random-uuid))- :made_public_by_id (test.users/user->id :crowberto)- :database_id (data/id)- :creator_id (test.users/user->id :crowberto)- :dataset_query {:database (data/id)- :type :native- :native {:query (str "UPDATE categories\n"- "SET name = concat([[{{name}}, ' ',]] 'Sh', 'op')\n"- "WHERE id = {{id}}")- :template-tags {"id" {:name "id"- :display-name "ID"- :type :number- :required true}- "name" {:name "name"- :display-name "Name"- :type :text- :required false}}}}}- options-map))]- {:action-id action-id :model-id model-id})-- :implicit- (let [action-id (action/insert! (merge- {:type :implicit- :name "Update Example"- :kind "row/update"- :public_uuid (str (random-uuid))- :made_public_by_id (test.users/user->id :crowberto)- :creator_id (test.users/user->id :crowberto)- :model_id model-id}- options-map))]- {:action-id action-id :model-id model-id})-- :http- (let [action-id (action/insert! (merge- {:type :http- :name "Echo Example"- :template {:url (client/build-url "testing/echo[[?fail={{fail}}]]" {})- :method "POST"- :body "{\"the_parameter\": {{id}}}"- :headers "{\"x-test\": \"{{id}}\"}"}- :parameters [{:id "id"- :type "number"- :target [:template-tag "id"]}- {:id "fail"- :type "text"- :target [:template-tag "fail"]}]- :response_handle ".body"- :model_id model-id- :public_uuid (str (random-uuid))- :made_public_by_id (test.users/user->id :crowberto)- :creator_id (test.users/user->id :crowberto)}- options-map))]- {:action-id action-id :model-id model-id}))))+ (create-action*! options-map model-id)))++(defn do-with-actions! [model-def f]+ (initialize/initialize-if-needed! :web-server)+ (t2.with-temp/with-temp [:model/Card model model-def]+ (tu/with-model-cleanup [:model/Action]+ (f model))));;; TODO FIXME -- rename this to [[with-actions!]] and then remove the Kondo ignore comment below#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}@@ -249,7 +264,7 @@`binding-forms-and-options-maps` is a vector of even number of elements, binding and options-map,similar to a `let` form.The first two elements of `binding-forms-and-options-maps` can describe the model, for this the- first option-map should map :dataset to a truthy value and contain :dataset_query. In this case+ first option-map should have `:type :model` `:dataset_query`. In this casethe first binding is bound to the model card created.For actions, the binding form is bound to a map with :action-id and :model-id set to the ID ofthe created action and model card respectively. The options-map overrides the defaults in@@ -260,30 +275,33 @@{:keys [action-id model-id]} {:type :http :name \"Temp HTTP Action\"}](assert (= model-card-id model-id))(something model-card-id id action-id model-id))"- {:style/indent 1}- [binding-forms-and-option-maps & body]- (assert (vector? binding-forms-and-option-maps)+ {:style/indent 1, :arglists '([action-bindings & body]+ [[model-binding {:type :model, :as model} & action-bindings] & body])}+ [[_maybe-model-bindings maybe-model-def :as bindings] & body]+ (assert (vector? bindings)"binding-forms-and-option-maps should be a vector")- (assert (even? (count binding-forms-and-option-maps))+ (assert (even? (count bindings))"binding-forms-and-option-maps should have an even number of elements")- (let [model (gensym "model-")- [_ maybe-model-def :as model-part] (subvec binding-forms-and-option-maps 0 2)- [[custom-binding model-def] binding-forms-and-option-maps]+ (let [model+ (gensym "model-")++ [model-binding model-def & action-bindings](if (and (map? maybe-model-def)(= (:type maybe-model-def) :model)(contains? maybe-model-def :dataset_query))- [model-part (drop 2 binding-forms-and-option-maps)]- ['[_ {:type :model, :dataset_query (mt/mbql-query categories)}]- binding-forms-and-option-maps])]- `(do- (initialize/initialize-if-needed! :web-server)- (mt/with-temp ~[:model/Card model model-def]- (tu/with-model-cleanup [:model/Action]- (let [~custom-binding ~model- ~@(mapcat (fn [[binding-form option-map]]- [binding-form `(do-with-action (merge {:type :query} ~option-map) (:id ~model))])- (partition 2 binding-forms-and-option-maps))]- ~@body))))))+ bindings+ (list*+ '_+ {:type :model, :dataset_query '(metabase.test/mbql-query categories)}+ bindings))]+ `(do-with-actions!+ ~model-def+ (fn [~model]+ (let [~model-binding ~model+ ~@(mapcat (fn [[action-binding action]]+ [action-binding `(create-action! (merge {:type :query} ~action) (:id ~model))])+ (partition 2 action-bindings))]+ ~@body)))))(comment(with-actions [{id :action-id} {:type :implicit :kind "row/create"}diff --git a/test/metabase/analyze/query_results_test.clj b/test/metabase/analyze/query_results_test.cljindex 60a8a9dc76a..63f1e6332ff 100644--- a/test/metabase/analyze/query_results_test.clj+++ b/test/metabase/analyze/query_results_test.clj@@ -7,7 +7,7 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.sync :as test.sync]diff --git a/test/metabase/cache/api_test.clj b/test/metabase/cache/api_test.cljindex 66869ae753e..71a6be54947 100644--- a/test/metabase/cache/api_test.clj+++ b/test/metabase/cache/api_test.clj@@ -35,7 +35,8 @@(mt/with-model-cleanup [:model/CacheConfig](mt/with-premium-features #{}(testing "But no advanced strategies can be used"- (is (=? {:errors {:strategy string?}}+ (is (=? {:specific-errors {:strategy {:type ["should be either :nocache or :ttl, received: :schedule" string?]}}+ :errors {:strategy {:type "enum of :nocache, :ttl"}}}(mt/user-http-request :crowberto :put 400 "cache/"{:model "root":model_id 0diff --git a/test/metabase/cmd/core_test.clj b/test/metabase/cmd/core_test.cljindex abd738e0e23..3f451aa47de 100644--- a/test/metabase/cmd/core_test.clj+++ b/test/metabase/cmd/core_test.clj@@ -9,22 +9,12 @@(deftest ^:parallel error-message-test(is (=? ["Unrecognized command: 'a-command-that-does-not-exist'"- #"\QValid commands: version, help, drop-entity-ids, import, dump,\E.*"]+ #"\QValid commands: version, help, drop-entity-ids, import,\E.*"](#'cmd/validate "a-command-that-does-not-exist" [])))(is (= ["The 'rotate-encryption-key' command requires the following arguments: [new-key], but received: []."](#'cmd/validate "rotate-encryption-key" [])))(is (nil? (#'cmd/validate "rotate-encryption-key" [:some-arg]))))-(deftest load-command-test- (do-with-captured-call-enterprise-calls!- (fn []- (testing "with no options"- (is (= '(metabase-enterprise.serialization.cmd/v1-load! "/path/" {:mode :skip, :on-error :continue})- (cmd/load "/path/"))))- (testing "with options"- (is (= '(metabase-enterprise.serialization.cmd/v1-load! "/path/" {:mode :skip, :on-error :abort})- (cmd/load "/path/" "--on-error" "abort")))))))-(deftest import-command-test(do-with-captured-call-enterprise-calls!(fn []@@ -38,16 +28,6 @@(is (= '(metabase-enterprise.serialization.cmd/v2-load! "/path/" {:full-stacktrace true})(cmd/import "/path/" "--full-stacktrace")))))))-(deftest dump-command-test- (do-with-captured-call-enterprise-calls!- (fn []- (testing "with no options"- (is (= '(metabase-enterprise.serialization.cmd/v1-dump! "/path/" {:state :all})- (cmd/dump "/path/"))))- (testing "with options"- (is (= '(metabase-enterprise.serialization.cmd/v1-dump! "/path/" {:state :active})- (cmd/dump "/path/" "--state" "active")))))))-(deftest export-command-arg-parsing-test(do-with-captured-call-enterprise-calls!(fn []diff --git a/test/metabase/collections/api_test.clj b/test/metabase/collections/api_test.cljindex 8d3921d99ce..ebbe0549cb7 100644--- a/test/metabase/collections/api_test.clj+++ b/test/metabase/collections/api_test.clj@@ -761,8 +761,11 @@{:name "Electro-Magnetic Pulse", :model "pulse", :entity_id true}])(assoc-in [1 :fully_parameterized] true))(mt/boolean-ids-and-timestamps- (:data (mt/user-http-request :rasta :get 200 (str "collection/" (u/the-id collection) "/items"))))))))+ (:data (mt/user-http-request :rasta :get 200 (str "collection/" (u/the-id collection) "/items")))))))))))+(deftest collection-items-children-test-2+ (testing "GET /api/collection/:id/items"+ (testing "check that you get to see the children as appropriate"(testing "...and that you can also filter so that you only see the children you want to see"(mt/with-temp [:model/Collection collection {:name "Art Collection"}](perms/grant-collection-read-permissions! (perms/all-users-group) collection)diff --git a/test/metabase/core/modules_test.clj b/test/metabase/core/modules_test.cljindex 92675dd7130..4a482d43597 100644--- a/test/metabase/core/modules_test.clj+++ b/test/metabase/core/modules_test.clj@@ -2,8 +2,10 @@"Tests that the modules config file is configured correctly."(:require[clojure.edn :as edn]+ [clojure.set :as set][clojure.string :as str][clojure.test :refer :all]+ [dev.deps-graph][metabase.util.json :as json][rewrite-clj.node :as n][rewrite-clj.parser :as r.parser]@@ -134,3 +136,34 @@(testing (format "\n'%s' module" module)(is (= (sort-module-names uses)uses))))))))++(deftest ^:parallel modules-config-up-to-date-test+ (testing (str "Please update .clj-kondo/config/modules/config.edn 🥰\n"+ "[Pro Tip: use (dev.deps-graph/print-kondo-config-diff) to see the changes you need to make in a nicer format]\n")+ (let [expected (dev.deps-graph/generate-config)+ actual (dev.deps-graph/kondo-config)+ modules (set/union (set (keys expected))+ (set (keys actual)))]+ (doseq [module modules+ :let [_ (testing (format "Remove %s" (pr-str module))+ (is (seq (get expected module))))]+ k [:api :uses]+ :let [ks [module k]+ expected (get-in expected ks)+ actual (get-in actual ks)]+ :when (not= actual :any)+ :let [missing (set/difference expected actual)+ extraneous (set/difference actual expected)]]+ (testing (format "Add %s to %s\nused by %s"+ (pr-str missing)+ (pr-str ks)+ (pr-str (case k+ :uses (reduce+ (partial merge-with set/union)+ {}+ (map #(dev.deps-graph/module-usages-of-other-module module %)+ missing))+ :api (select-keys (dev.deps-graph/external-usages-by-namespace module) missing))))+ (is (empty? missing)))+ (testing (format "Remove %s from %s" (pr-str extraneous) (pr-str ks))+ (is (empty? extraneous)))))))diff --git a/test/metabase/dashboards/api_test.clj b/test/metabase/dashboards/api_test.cljindex 72356a2415b..7987a1649ae 100644--- a/test/metabase/dashboards/api_test.clj+++ b/test/metabase/dashboards/api_test.clj@@ -20,6 +20,8 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu][metabase.models.interface :as mi][metabase.parameters.chain-filter :as chain-filter][metabase.parameters.chain-filter-test :as chain-filter-test]@@ -49,7 +51,7 @@(set! *warn-on-reflection* true)-(use-fixtures :once (fixtures/initialize :test-users))+(use-fixtures :once (fixtures/initialize :test-users :db :web-server))(deftest ^:parallel update-colvalmap-setting-test(testing "update-colvalmap-setting function with regex matching"@@ -729,7 +731,7 @@(mt/with-log-messages-for-level [messages [metabase.parameters.params :error]](is (some? (mt/user-http-request :rasta :get 200 (str "dashboard/" dash-id))))(is (=? [{:level :error- :message "Could not find matching field clause for target: [:dimension [:template-tag not-existed-filter]]"}]+ :message "Could not find matching field clause for target: [:dimension [:template-tag \"not-existed-filter\"]]"}](messages)))))));;; +----------------------------------------------------------------------------------------------------------------+@@ -2152,7 +2154,7 @@:parameter_mappings [{:parameter_id "abc":card_id 123:hash "abc"- :target "foo"}]+ :target [:dimension [:template-tag "foo"]]}]:visualization_settings {}}]:tabs []}))];; extra sure here because the dashcard we given has a negative id@@ -2164,7 +2166,7 @@:row 4:series []:dashboard_tab_id nil- :parameter_mappings [{:parameter_id "abc" :card_id 123, :hash "abc", :target "foo"}]+ :parameter_mappings [{:parameter_id "abc" :card_id 123, :hash "abc", :target ["dimension" ["template-tag" "foo"]]}]:visualization_settings {}:created_at true:updated_at true@@ -2178,7 +2180,7 @@:size_y 4:col 4:row 4- :parameter_mappings [{:parameter_id "abc", :card_id 123, :hash "abc", :target "foo"}]+ :parameter_mappings [{:parameter_id "abc", :card_id 123, :hash "abc", :target [:dimension [:template-tag "foo"]]}]:visualization_settings {}}](map (partial into {})(t2/select [:model/DashboardCard :size_x :size_y :col :row :parameter_mappings :visualization_settings]@@ -2336,7 +2338,7 @@:parameter_mappings [{:parameter_id "abc":card_id 123:hash "abc"- :target "foo"}]+ :target [:dimension [:template-tag "foo"]]}]:visualization_settings {}}]:tabs []}))))))@@ -2657,60 +2659,64 @@form))x))+(defn- fake-query [x]+ (lib/native-query (lib.tu/mock-metadata-provider {:database (assoc meta/database :id 55001)})+ (format "SELECT %s" (pr-str x))))+(deftest ^:parallel dashcard->query-hashes-test(doseq [[dashcard expected]- [[{:card {:dataset_query {:database 1}}}- ["k9Y1XOETkQ31kX+S9DXW/cbDPGF7v4uS5f6dZsXjMRs="- "I6mv3dlN4xat/6R+KQVTLDqNe8/B0oymcDnW/aKppwY="]]-- [{:card {:dataset_query {:database 2}}- :series [{:dataset_query {:database 3}}- {:dataset_query {:database 4}}]}- ["WbWqdd3zu9zvVCVWh8X9ASWLqtaB1rZlU0gKLEuCK0I="- "ysJFZA3Gd0vKIlrZWJDYBLCIQBf10X6QjuFtV/8QzbE="- "pjdBPUgWnbVvMf0VsETyeB6smRC8SYejyTZIVPh2m3I="- "wf9reZSm1Pz+WDHRYtZXmfQ39U+17mq7u28MqPR8fQI="- "rP5XFvxpRDCPXeM0A2Z7uoUkH0zwV0Z0o22obH3c1Uk="- "r+C7dsdRXBN32GK+QHLA/n9pr1hzjteFzDCVezLzImQ="]]]]+ [[{:card {:dataset_query (fake-query 1)}}+ ["RNcJA21084AYlVPIqlxDxMILKsyU6iE9hiUDMYBIrUw="+ "GsfPch7sQZ60fotaEuN6fghF82iX3dUHLlvWtkJXOU0="]]++ [{:card {:dataset_query (fake-query 2)}+ :series [{:dataset_query (fake-query 3)}+ {:dataset_query (fake-query 4)}]}+ ["LyJKZ0rm2YyPWGXIfP/V6SFcHlRvdalSDD55qn/Avng="+ "zYehm4Huu7AD/0GP6XpMhav7+/ystaIeKjax4fqSHUg="+ "nDuISCLB6o8sToOR4GHQ/Po6X3A3nE7oPj+e1g2kfCE="+ "/yzJQIPSdxWHnREe9F2orYWDbPS5ns8WOaIf6fGix1k="+ "oqG69TgkGdzPFaxDFymLAVnb9iuyGnvlX0aiiL1Eojc="+ "GKQrcQTNICx7QOYDQVHW7BIi7gB3/oBUWVel+AvOq0Q="]]]](testing (pr-str dashcard)(is (= expected(base-64-encode-byte-arrays (#'api.dashboard/dashcard->query-hashes dashcard)))))))(deftest ^:parallel dashcards->query-hashes-test- (is (= ["k9Y1XOETkQ31kX+S9DXW/cbDPGF7v4uS5f6dZsXjMRs="- "I6mv3dlN4xat/6R+KQVTLDqNe8/B0oymcDnW/aKppwY="- "WbWqdd3zu9zvVCVWh8X9ASWLqtaB1rZlU0gKLEuCK0I="- "ysJFZA3Gd0vKIlrZWJDYBLCIQBf10X6QjuFtV/8QzbE="- "pjdBPUgWnbVvMf0VsETyeB6smRC8SYejyTZIVPh2m3I="- "wf9reZSm1Pz+WDHRYtZXmfQ39U+17mq7u28MqPR8fQI="- "rP5XFvxpRDCPXeM0A2Z7uoUkH0zwV0Z0o22obH3c1Uk="- "r+C7dsdRXBN32GK+QHLA/n9pr1hzjteFzDCVezLzImQ="]+ (is (= ["RNcJA21084AYlVPIqlxDxMILKsyU6iE9hiUDMYBIrUw="+ "GsfPch7sQZ60fotaEuN6fghF82iX3dUHLlvWtkJXOU0="+ "LyJKZ0rm2YyPWGXIfP/V6SFcHlRvdalSDD55qn/Avng="+ "zYehm4Huu7AD/0GP6XpMhav7+/ystaIeKjax4fqSHUg="+ "nDuISCLB6o8sToOR4GHQ/Po6X3A3nE7oPj+e1g2kfCE="+ "/yzJQIPSdxWHnREe9F2orYWDbPS5ns8WOaIf6fGix1k="+ "oqG69TgkGdzPFaxDFymLAVnb9iuyGnvlX0aiiL1Eojc="+ "GKQrcQTNICx7QOYDQVHW7BIi7gB3/oBUWVel+AvOq0Q="](base-64-encode-byte-arrays(#'api.dashboard/dashcards->query-hashes- [{:card {:dataset_query {:database 1}}}- {:card {:dataset_query {:database 2}}- :series [{:dataset_query {:database 3}}- {:dataset_query {:database 4}}]}])))))+ [{:card {:dataset_query (fake-query 1)}}+ {:card {:dataset_query (fake-query 2)}+ :series [{:dataset_query (fake-query 3)}+ {:dataset_query (fake-query 4)}]}])))))-(deftest add-query-average-duration-to-dashcards-test- (is (= [{:card {:dataset_query {:database 1}, :query_average_duration 111}+(deftest ^:parallel add-query-average-duration-to-dashcards-test+ (is (= [{:card {:dataset_query (fake-query 1), :query_average_duration 111}:series []}- {:card {:dataset_query {:database 2}, :query_average_duration 333}- :series [{:dataset_query {:database 3}, :query_average_duration 555}- {:dataset_query {:database 4}, :query_average_duration 777}]}]+ {:card {:dataset_query (fake-query 2), :query_average_duration 333}+ :series [{:dataset_query (fake-query 3), :query_average_duration 555}+ {:dataset_query (fake-query 4), :query_average_duration 777}]}](#'api.dashboard/add-query-average-duration-to-dashcards- [{:card {:dataset_query {:database 1}}}- {:card {:dataset_query {:database 2}}- :series [{:dataset_query {:database 3}}- {:dataset_query {:database 4}}]}]- (into {} (for [[k v] {"k9Y1XOETkQ31kX+S9DXW/cbDPGF7v4uS5f6dZsXjMRs=" 111- "K6A0F7tRxQ+2xa33kigBwIvUvU+F95UUccWjGTx8kuI=" 222- "WbWqdd3zu9zvVCVWh8X9ASWLqtaB1rZlU0gKLEuCK0I=" 333- "NzgQC4fjR52npCkZV7IiZDb9NfcmKbWHP4krFzkLPyA=" 444- "pjdBPUgWnbVvMf0VsETyeB6smRC8SYejyTZIVPh2m3I=" 555- "dEXUTWQI2L0Z/Bvrb2LTVVPl2Qg/56hKIPb+I2a4mG8=" 666- "rP5XFvxpRDCPXeM0A2Z7uoUkH0zwV0Z0o22obH3c1Uk=" 777- "Wn9nubTcKZX5862pHFaibkqqbsqAfGa3gVhN3D4FrJw=" 888}]+ [{:card {:dataset_query (fake-query 1)}}+ {:card {:dataset_query (fake-query 2)}+ :series [{:dataset_query (fake-query 3)}+ {:dataset_query (fake-query 4)}]}]+ (into {} (for [[k v] {"RNcJA21084AYlVPIqlxDxMILKsyU6iE9hiUDMYBIrUw=" 111+ "GsfPch7sQZ60fotaEuN6fghF82iX3dUHLlvWtkJXOU0=" 222+ "LyJKZ0rm2YyPWGXIfP/V6SFcHlRvdalSDD55qn/Avng=" 333+ "zYehm4Huu7AD/0GP6XpMhav7+/ystaIeKjax4fqSHUg=" 444+ "nDuISCLB6o8sToOR4GHQ/Po6X3A3nE7oPj+e1g2kfCE=" 555+ "/yzJQIPSdxWHnREe9F2orYWDbPS5ns8WOaIf6fGix1k=" 666+ "oqG69TgkGdzPFaxDFymLAVnb9iuyGnvlX0aiiL1Eojc=" 777+ "GKQrcQTNICx7QOYDQVHW7BIi7gB3/oBUWVel+AvOq0Q=" 888}][(mapv int (codec/base64-decode k)) v]))))));;; +----------------------------------------------------------------------------------------------------------------+@@ -3103,62 +3109,64 @@:values [["African"] ["American"] ["Asian"]]}(mt/user-http-request :rasta :get 200(chain-filter-values-url (:id dashboard) (:static-category param-keys)))))-(is (= {:has_more_values false:values [["African" "Af"] ["American" "Am"] ["Asian" "As"]]}(mt/user-http-request :rasta :get 200(chain-filter-values-url (:id dashboard) (:static-category-label param-keys))))))-(testing "we could search the values"(is (= {:has_more_values false:values [["African"]]}(mt/user-http-request :rasta :get 200(chain-filter-search-url (:id dashboard) (:static-category param-keys) "af"))))-(is (= {:has_more_values false:values [["African" "Af"]]}(mt/user-http-request :rasta :get 200(chain-filter-search-url (:id dashboard) (:static-category-label param-keys) "f")))))-(testing "we could edit the values list"- ;; TODO add tests for schema check(let [dashboard (mt/user-http-request :rasta :put 200 (str "dashboard/" (:id dashboard))- {:parameters [{:name "Static Category"- :slug "static_category"- :id "_STATIC_CATEGORY_"- :type "category"- :values_query_type "search"- :values_source_type "static-list"+ {:parameters [{:name "Static Category"+ :slug "static_category"+ :id "_STATIC_CATEGORY_"+ :type "category"+ :values_query_type "search"+ :values_source_type "static-list":values_source_config {"values" ["BBQ" "Bakery" "Bar"]}}]})]- (is (= [{:name "Static Category"- :slug "static_category"- :id "_STATIC_CATEGORY_"- :type "category"- :values_query_type "search"- :values_source_type "static-list"+ (is (= [{:name "Static Category"+ :slug "static_category"+ :id "_STATIC_CATEGORY_"+ :type "category"+ :values_query_type "search"+ :values_source_type "static-list":values_source_config {:values ["BBQ" "Bakery" "Bar"]}}]- (:parameters dashboard))))))+ (:parameters dashboard))))))))- (testing "source-options must be a map and sourcetype must be `card` or `static-list` must be a string"- (is (= "nullable sequence of parameter must be a map with :id and :type keys"+(deftest ^:parallel dashboard-with-static-list-parameters-test-2+ (testing "A dashboard that has parameters that has static values"+ (testing "`values_source_config` must be a map and `values_source_type` must be `card` or `static-list` must be a string"+ (is (= {:values_source_type "nullable enum of :static-list, :card"}(get-in (mt/user-http-request :rasta :post 400 "dashboard"{:name "a dashboard"- :parameters [{:id "_value_"- :name "value"- :type "category"- :values_source_type "random-type"+ :parameters [{:id "_value_"+ :name "value"+ :type "category"+ :values_source_type "random-type":values_source_config {"values" [1 2 3]}}]})- [:errors :parameters])))- (is (= "nullable sequence of parameter must be a map with :id and :type keys"- (get-in (mt/user-http-request :rasta :post 400 "dashboard"- {:name "a dashboard"- :parameters [{:id "_value_"- :name "value"- :type "category"- :values_source_type "static-list"- :values_source_config []}]})[:errors :parameters]))))))+(deftest ^:parallel dashboard-with-static-list-parameters-test-2b+ (testing "A dashboard that has parameters that has static values"+ (testing "`values_source_config` must be a map and `values_source_type` must be `card` or `static-list` must be a string"+ ;; actually, this works now because normalization will remove `values_source_config` when it's not a map.+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (is (=? {:parameters [{:values_source_config nil}]}+ (mt/user-http-request :rasta :post 200 "dashboard"+ {:name "a dashboard"+ :parameters [{:id "_value_"+ :name "value"+ :type "category"+ :values_source_type "static-list"+ :values_source_config []}]})))))))+(deftest native-query-get-params-test(testing "GET /api/dashboard/:id/params/:param-key/values works for native queries"(mt/dataset test-data@@ -3173,7 +3181,7 @@(let [metadata (-> (:dataset_query native-card)qp/process-query :data :results_metadata :columns)](is (seq metadata) "Did not get metadata")- (t2/update! 'Card {:id model-id}+ (t2/update! :model/Card {:id model-id}{:result_metadata (assoc-in metadata [0 :id](mt/id :products :category))}));; ...so instead we create a question on top of this model (note that@@ -3215,13 +3223,11 @@(is (= {:values [["Bar"] ["Gay Bar"] ["Juice Bar"]]:has_more_values false}(chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url)))))))-(mt/let-url [url (chain-filter-search-url dashboard (:category-name param-keys) "house" (:price param-keys) 4)](testing "\nShow me names of categories that include 'house' that have expensive venues (price = 4)"(is (= {:values [["Steakhouse"]]:has_more_values false}(chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url))))))-(testing "Should require a non-empty query"(doseq [query [" " "\n"]](mt/let-url [url (chain-filter-search-url dashboard (:category-name param-keys) query)]@@ -3230,8 +3236,10 @@(doseq [query [nil ""]](mt/let-url [url (chain-filter-search-url dashboard (:category-name param-keys) query)](is (= "API endpoint does not exist."- (mt/user-http-request :rasta :get 404 url)))))))+ (mt/user-http-request :rasta :get 404 url)))))))))+(deftest chain-filter-search-test-2+ (testing "GET /api/dashboard/:id/params/:param-key/search/:query"(testing "Should require perms for the Dashboard"(mt/with-non-admin-groups-no-root-collection-perms(mt/with-temp [:model/Collection collection]@@ -3245,7 +3253,6 @@(mt/with-temp [:model/Dashboard {dashboard-id :id}](testing "GET /api/dashboard/:id/params/:param-key/values returns 400 if param not found"(mt/user-http-request :rasta :get 400 (format "dashboard/%d/params/non-existing-param/values" dashboard-id)))-(testing "GET /api/dashboard/:id/params/:param-key/search/:query returns 400 if param not found"(mt/user-http-request :rasta :get 400 (format "dashboard/%d/params/non-existing-param/search/bar" dashboard-id)))))@@ -3258,7 +3265,8 @@:card_id (:id card):target [:dimension (mt/$ids venues $category_id->categories.name)]}{:parameter_id "_PRICE_"- :card_id (:id card)}]})+ :card_id (:id card)+ :target [:dimension [:template-tag "WOW"]]}]})(testing "Since the _PRICE_ param is not mapped to a valid Field, it should get ignored"(mt/let-url [url (chain-filter-values-url dashboard "_CATEGORY_NAME_" "_PRICE_" 4)](is (= {:values [["African"] ["American"] ["Artisan"]]@@ -3267,7 +3275,7 @@(deftest chain-filter-human-readable-values-remapping-test(testing "Chain filtering for Fields that have Human-Readable values\n"- (chain-filter-test/with-human-readable-values-remapping+ (chain-filter-test/with-human-readable-values-remapping!(with-chain-filter-fixtures [{:keys [dashboard]}](testing "GET /api/dashboard/:id/params/:param-key/values"(mt/let-url [url (chain-filter-values-url dashboard "_CATEGORY_ID_" "_PRICE_" 4)]@@ -3297,7 +3305,6 @@[16 "Pacific Dining Car - Santa Monica"]]:has_more_values false}(chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url))))))-(testing "GET /api/dashboard/:id/params/:param-key/search/:query"(mt/let-url [url (chain-filter-search-url dashboard "_ID_" "fish")](is (= {:values [[90 "Señor Fish"]]@@ -3312,7 +3319,7 @@(deftest chain-filter-fk-field-to-field-remapping-test(testing "Chain filtering for Fields that have a FK Field -> Field remapping\n"- (chain-filter-test/with-fk-field-to-field-remapping+ (chain-filter-test/with-fk-field-to-field-remapping!(with-chain-filter-fixtures [{:keys [dashboard]}](testing "GET /api/dashboard/:id/params/:param-key/values"(mt/let-url [url (chain-filter-values-url dashboard "_CATEGORY_ID_" "_PRICE_" 4)]@@ -3326,7 +3333,7 @@:has_more_values false}(mt/user-http-request :rasta :get 200 url)))))))))-(deftest chain-filter-multiple-test+(deftest ^:parallel chain-filter-multiple-test(testing "Chain filtering works when a few filters are specified"(with-chain-filter-fixtures [{:keys [dashboard param-keys]}](testing "GET /api/dashboard/:id/params/:param-key/values"@@ -3334,12 +3341,22 @@(:not-category-name param-keys) "American")](is (= {:values [["African"] ["Artisan"] ["Asian"]]:has_more_values false}- (chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url)))))+ (chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url)))))))))++(deftest ^:parallel chain-filter-multiple-test-2+ (testing "Chain filtering works when a few filters are specified"+ (with-chain-filter-fixtures [{:keys [dashboard param-keys]}]+ (testing "GET /api/dashboard/:id/params/:param-key/values"(mt/let-url [url (chain-filter-values-url dashboard (:category-name param-keys)(:category-contains param-keys) "m")](is (= {:values [["American"] ["Comedy Club"] ["Dim Sum"]]:has_more_values false}- (chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url)))))+ (chain-filter-test/take-n-values 3 (mt/user-http-request :rasta :get 200 url)))))))))++(deftest ^:parallel chain-filter-multiple-test-3+ (testing "Chain filtering works when a few filters are specified"+ (with-chain-filter-fixtures [{:keys [dashboard param-keys]}]+ (testing "GET /api/dashboard/:id/params/:param-key/values"(testing "contains is case insensitive"(mt/let-url [url (chain-filter-values-url dashboard (:category-name param-keys)(:not-category-name param-keys) "American"@@ -3420,7 +3437,6 @@:value_field (mt/$ids $venues.name)}}]}](mt/let-url [url (chain-filter-values-url dashboard "abc")](is (= mock-chain-filter-result (mt/user-http-request :rasta :get 200 url))))))-(testing "if card is archived"(mt/with-temp [:model/Card {card-id :id} {:archived true}:model/Dashboard dashboard {:parameters [{:id "abc"@@ -3518,7 +3534,7 @@(mt/let-url [url (chain-filter-values-url dash-id (:id param))](is (some? (mt/user-http-request :rasta :get 200 url))))))))))-(deftest valid-filter-fields-test+(deftest ^:parallel valid-filter-fields-test(testing "GET /api/dashboard/params/valid-filter-fields"(letfn [(result= [expected {:keys [filtered filtering]}](testing (format "\nGET dashboard/params/valid-filter-fields")@@ -3534,16 +3550,19 @@%categories.name (sort [%venues.price %categories.name])}{:filtered [%venues.price %categories.name], :filtering [%categories.name %venues.price]}))(testing "filtered-ids cannot be nil"- (is (= {:errors {:filtered "vector of value must be an integer greater than zero."}+ (is (= {:errors {:filtered "vector of Valid Field ID"}:specific-errors{:filtered ["missing required key, received: nil"]}}- (mt/user-http-request :rasta :get 400 "dashboard/params/valid-filter-fields" :filtering [%categories.name]))))))- (testing "should check perms for the Fields in question"- (mt/with-temp-copy-of-db- (perms.test-util/with-no-data-perms-for-all-users!- (is (= "You don't have permissions to do that."- (mt/$ids (mt/user-http-request :rasta :get 403 "dashboard/params/valid-filter-fields"- :filtered [%venues.price] :filtering [%categories.name]))))))))))+ (mt/user-http-request :rasta :get 400 "dashboard/params/valid-filter-fields" :filtering [%categories.name])))))))))++(deftest valid-filter-fields-test-2+ (testing "GET /api/dashboard/params/valid-filter-fields"+ (testing "should check perms for the Fields in question"+ (mt/with-temp-copy-of-db+ (perms.test-util/with-no-data-perms-for-all-users!+ (is (= "You don't have permissions to do that."+ (mt/$ids (mt/user-http-request :rasta :get 403 "dashboard/params/valid-filter-fields"+ :filtered [%venues.price] :filtering [%categories.name])))))))))(deftest uuid-id-column-is-not-implicitly-remapped-test(mt/test-drivers@@ -3617,7 +3636,6 @@(data-perms/set-database-permission! (perms-group/all-users) (mt/id) :perms/create-queries :no)(is (malli= (dashboard-card-query-expected-results-schema)(mt/user-http-request :rasta :post 202 (url)))))))-(testing "Validation"(testing "404s"(testing "Should return 404 if Dashboard doesn't exist"@@ -3626,7 +3644,6 @@(testing "Should return 404 if Card doesn't exist"(is (= "Not found."(mt/user-http-request :rasta :post 404 (url :card-id Integer/MAX_VALUE))))))-(testing "perms"(mt/with-temp [:model/Collection {collection-id :id}](perms/revoke-collection-permissions! (perms-group/all-users) collection-id)@@ -3988,7 +4005,7 @@{:query (format "insert into types (%s) values ({{%s}})"(u/upper-case-en field-name)field-name)- :template-tags {field-name {:id field-name :name field-name :type :text :display_name field-name}}})})+ :template-tags {field-name {:id field-name, :name field-name, :type :text, :display-name field-name}}})})(deftest dashcard-action-execution-type-test(mt/test-drivers (disj (mt/normal-drivers-with-feature :actions) :mysql)@@ -5074,14 +5091,16 @@(is (not (t2/select-one-fn :archived :model/Dashboard dash-id)))(is (not (t2/select-one-fn :archived :model/Card card-id))))))-(deftest dashboard-questions-are-archived-when-unused-and-vice-versa+(deftest ^:parallel dashboard-questions-are-archived-when-unused-and-vice-versa(testing "The dashboard question is archived when it's removed from the dashboard"(mt/with-temp [:model/Dashboard {dash-id :id} {}:model/Card {card-id :id} {:dashboard_id dash-id}:model/DashboardCard _ {:card_id card-id :dashboard_id dash-id}](mt/user-http-request :rasta :put 200 (str "dashboard/" dash-id) {:dashcards []})(is (not (t2/exists? :model/DashboardCard :card_id card-id :dashboard_id dash-id)))- (is (t2/select-one-fn :archived :model/Card card-id))))+ (is (t2/select-one-fn :archived :model/Card card-id)))))++(deftest ^:parallel dashboard-questions-are-archived-when-unused-and-vice-versa-2(testing "The dashboard question is unarchived when it's re-added to the dashboard"(mt/with-temp [:model/Dashboard {dash-id :id} {}:model/Card {card-id :id} {:dashboard_id dash-id}]@@ -5093,13 +5112,15 @@(is (t2/exists? :model/DashboardCard :card_id card-id :dashboard_id dash-id))(is (not (t2/select-one-fn :archived :model/Card card-id))))))-(deftest dashboard-items-works+(deftest ^:parallel dashboard-items-works(testing "Dashboard items is empty when the dashboard is a normal dashboard w/o DQs"(mt/with-temp [:model/Dashboard {dash-id :id} {}:model/Card {card-id :id} {}:model/DashboardCard _ {:card_id card-id :dashboard_id dash-id}](is (= {:total 0 :data [] :models [] :limit nil :offset nil}- (mt/user-http-request :rasta :get 200 (str "dashboard/" dash-id "/items"))))))+ (mt/user-http-request :rasta :get 200 (str "dashboard/" dash-id "/items")))))))++(deftest ^:parallel dashboard-items-works-2(testing "Dashboard items is present when the dashboard has DQs"(mt/with-temp [:model/Dashboard {dash-id :id} {}:model/Card {card-id :id} {:dashboard_id dash-id}@@ -5111,7 +5132,9 @@:models ["card"]}(update (mt/user-http-request :rasta :get 200 (str "dashboard/" dash-id "/items")):data- #(map (fn [card] (select-keys card [:id])) %))))))+ #(map (fn [card] (select-keys card [:id])) %)))))))++(deftest ^:parallel dashboard-items-works-3(testing "DQs don't appear twice even if they appear multiple times in the dashboard"(mt/with-temp [:model/Dashboard {dash-id :id} {}:model/Card {card-id :id} {:dashboard_id dash-id}@@ -5126,7 +5149,7 @@:data#(map (fn [card] (select-keys card [:id])) %)))))))-(deftest dashboard-items-is-the-same-as-collection-items+(deftest ^:parallel dashboard-items-is-the-same-as-collection-items(mt/with-temp [:model/Collection {coll-id :id} {}:model/Dashboard {dash-id :id} {}:model/Card _ {:collection_id coll-id}diff --git a/test/metabase/dashboards/models/dashboard_card_test.clj b/test/metabase/dashboards/models/dashboard_card_test.cljindex 5fee49249b9..b2347caee91 100644--- a/test/metabase/dashboards/models/dashboard_card_test.clj+++ b/test/metabase/dashboards/models/dashboard_card_test.clj@@ -3,7 +3,6 @@[clojure.test :refer :all][metabase.dashboards.models.dashboard :as dashboard][metabase.dashboards.models.dashboard-card :as dashboard-card]- [metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.queries.models.card-test :as card-test][metabase.test :as mt]@@ -31,16 +30,19 @@(testing "retrieve-dashboard-card basic dashcard (no additional series)"(mt/with-temp [:model/Dashboard {dashboard-id :id} {}:model/Card {card-id :id} {}- :model/DashboardCard {dashcard-id :id} {:dashboard_id dashboard-id :card_id card-id :parameter_mappings [{:foo "bar"}]}]- (is (= {:size_x 4- :size_y 4- :col 0- :row 0- :parameter_mappings [{:foo "bar"}]- :inline_parameters []- :visualization_settings {}- :series []}- (remove-ids-and-timestamps (dashboard-card/retrieve-dashboard-card dashcard-id)))))))+ :model/DashboardCard {dashcard-id :id} {:dashboard_id dashboard-id+ :card_id card-id+ :parameter_mappings [{:parameter_id "wow"+ :target [:dimension [:template-tag "id"]]}]}]+ (is (=? {:size_x 4+ :size_y 4+ :col 0+ :row 0+ :parameter_mappings [{:parameter_id "wow"}]+ :inline_parameters []+ :visualization_settings {}+ :series []}+ (dashboard-card/retrieve-dashboard-card dashcard-id))))))(deftest ^:parallel retrieve-dashboard-card-with-additional-series-test(testing "retrieve-dashboard-card dashcard w/ additional series"@@ -117,40 +119,40 @@:size_y 3:row 1:col 1- :parameter_mappings [{:foo "bar"}]+ :parameter_mappings [{:parameter_id "bird", :target [:dimension [:template-tag "nest"]]}]:inline_parameters []:visualization_settings {}:series [card-id]}]))](testing "return value from function"- (is (= {:size_x 4- :size_y 3- :col 1- :row 1- :parameter_mappings [{:foo "bar"}]- :inline_parameters []- :visualization_settings {}- :series [{:name "Test Card"- :description nil- :display :table- :type :question- :dataset_query {}- :visualization_settings {}}]}- (remove-ids-and-timestamps dashboard-card))))+ (is (=? {:size_x 4+ :size_y 3+ :col 1+ :row 1+ :parameter_mappings [{:parameter_id "bird"}]+ :inline_parameters []+ :visualization_settings {}+ :series [{:name "Test Card"+ :description nil+ :display :table+ :type :question+ :dataset_query {}+ :visualization_settings {}}]}+ dashboard-card)))(testing "validate db captured everything"- (is (= {:size_x 4- :size_y 3- :col 1- :row 1- :parameter_mappings [{:foo "bar"}]- :inline_parameters []- :visualization_settings {}- :series [{:name "Test Card"- :description nil- :display :table- :type :question- :dataset_query {}- :visualization_settings {}}]}- (remove-ids-and-timestamps (dashboard-card/retrieve-dashboard-card (:id dashboard-card))))))))))+ (is (=? {:size_x 4+ :size_y 3+ :col 1+ :row 1+ :parameter_mappings [{:parameter_id "bird"}]+ :inline_parameters []+ :visualization_settings {}+ :series [{:name "Test Card"+ :description nil+ :display :table+ :type :question+ :dataset_query {}+ :visualization_settings {}}]}+ (dashboard-card/retrieve-dashboard-card (:id dashboard-card)))))))))(deftest update-dashboard-card!-test(testing (str "update-dashboard-card! basic update. We are testing multiple things here: 1. ability to update all "@@ -161,19 +163,20 @@:model/DashboardCard {dashcard-id :id:as dashboard-card} {:dashboard_id dashboard-id:card_id card-id- :parameter_mappings [{:foo "bar"}]}+ :parameter_mappings [{:parameter_id "x"+ :target [:dimension [:template-tag "x"]]}]}:model/Card {card-id-1 :id} {:name "Test Card 1"}:model/Card {card-id-2 :id} {:name "Test Card 2"}](testing "unmodified dashcard"- (is (= {:size_x 4- :size_y 4- :col 0- :row 0- :parameter_mappings [{:foo "bar"}]- :inline_parameters []- :visualization_settings {}- :series []}- (remove-ids-and-timestamps (dashboard-card/retrieve-dashboard-card dashcard-id)))))+ (is (=? {:size_x 4+ :size_y 4+ :col 0+ :row 0+ :parameter_mappings [{:parameter_id "x"}]+ :inline_parameters []+ :visualization_settings {}+ :series []}+ (dashboard-card/retrieve-dashboard-card dashcard-id))))(testing "return value from the update call should be nil"(is (nil? (dashboard-card/update-dashboard-card!{:id dashcard-id@@ -184,32 +187,32 @@:size_y 3:row 1:col 1- :parameter_mappings [{:foo "barbar"}]+ :parameter_mappings [{:parameter_id "y", :target [:dimension [:template-tag "y"]]}]:inline_parameters []:visualization_settings {}:series [card-id-2 card-id-1]}dashboard-card))))(testing "validate db captured everything"- (is (= {:size_x 5- :size_y 3- :col 1- :row 1- :parameter_mappings [{:foo "barbar"}]- :inline_parameters []- :visualization_settings {}- :series [{:name "Test Card 2"- :description nil- :display :table- :type :question- :dataset_query {}- :visualization_settings {}}- {:name "Test Card 1"- :description nil- :display :table- :type :question- :dataset_query {}- :visualization_settings {}}]}- (remove-ids-and-timestamps (dashboard-card/retrieve-dashboard-card dashcard-id))))))))+ (is (=? {:size_x 5+ :size_y 3+ :col 1+ :row 1+ :parameter_mappings [{:parameter_id "y", :target [:dimension [:template-tag "y"]]}]+ :inline_parameters []+ :visualization_settings {}+ :series [{:name "Test Card 2"+ :description nil+ :display :table+ :type :question+ :dataset_query {}+ :visualization_settings {}}+ {:name "Test Card 1"+ :description nil+ :display :table+ :type :question+ :dataset_query {}+ :visualization_settings {}}]}+ (dashboard-card/retrieve-dashboard-card dashcard-id)))))))(deftest update-dashboard-card!-call-count-test(testing "This tracks the call count of update-dashcards! for the purpose of optimizing the@@ -281,38 +284,6 @@(is (= expected(t2/select-one-fn :visualization_settings :model/DashboardCard :id (u/the-id dashcard))))))))))-(deftest ^:parallel normalize-parameter-mappings-test-2- (testing "make sure parameter mappings correctly normalize things like legacy MBQL clauses"- (is (= [{:target [:dimension [:field 30 {:source-field 23}]]}]- ((:out mi/transform-parameters-list)- (json/encode- [{:target [:dimension [:fk-> 23 30]]}]))))-- (testing "...but parameter mappings we should not normalize things like :target"- (is (= [{:card-id 123, :hash "abc", :target "foo"}]- ((:out mi/transform-parameters-list)- (json/encode- [{:card-id 123, :hash "abc", :target "foo"}])))))))--(deftest ^:parallel keep-empty-parameter-mappings-empty-test- (testing (str "we should keep empty parameter mappings as empty instead of making them nil (if `normalize` removes "- "them because they are empty) (I think this is to prevent NPEs on the FE? Not sure why we do this)")- (is (= []- ((:out mi/transform-parameters-list)- (json/encode []))))))--(deftest ^:parallel normalize-card-parameter-mappings-test- (doseq [parameters [[]- [{:name "Time grouping"- :slug "time_grouping"- :id "8e366c15"- :type :temporal-unit- :sectionId "temporal-unit"- :temporal_units [:minute :quarter-of-year]}]]]- (is (= parameters- ((:out mi/transform-card-parameters-list)- (json/encode parameters))))))-(deftest ^:parallel identity-hash-test(testing "Dashboard card hashes are composed of the card hash, dashboard hash, and visualization settings"(let [now #t "2022-09-01T12:34:56Z"]diff --git a/test/metabase/dashboards/models/dashboard_test.clj b/test/metabase/dashboards/models/dashboard_test.cljindex ce6739492ec..5bd2b9008a0 100644--- a/test/metabase/dashboards/models/dashboard_test.clj+++ b/test/metabase/dashboards/models/dashboard_test.clj@@ -23,26 +23,28 @@(tu/with-temporary-setting-values [enable-public-sharing true](mt/with-temp [:model/Dashboard dashboard {:public_uuid (str (random-uuid))}](is (=? u/uuid-regex- (:public_uuid dashboard)))))+ (:public_uuid dashboard)))))))+(deftest public-sharing-test-2+ (testing "test that a Dashboard's :public_uuid comes back if public sharing is enabled..."(testing "...but if public sharing is *disabled* it should come back as `nil`"(tu/with-temporary-setting-values [enable-public-sharing false](mt/with-temp [:model/Dashboard dashboard {:public_uuid (str (random-uuid))}](is (= nil(:public_uuid dashboard))))))))-(def default-parameter+(def ^:private default-parameter{:id "_CATEGORY_NAME_"- :type "category"+ :type :category:name "Category Name":slug "category_name"})-(deftest migrate-parameters-with-linked-filters-and-values-source-type-test+(deftest ^:parallel migrate-parameters-with-linked-filters-and-values-source-type-test(testing "test that a Dashboard's :parameters filterParameters are cleared if the :values_source_type is not nil"(doseq [[values_source_type- keep-filtering-parameters?] {"card" false- "static-list" false- nil true}]+ keep-filtering-parameters?] {:card false+ :static-list false+ nil true}](testing (format "\nvalues_source_type=%s" values_source_type)(mt/with-temp [:model/Dashboard dashboard {:parameters [(mergedefault-parameter@@ -54,7 +56,7 @@(:filteringParameters parameter)))(is (not (contains? parameter :filteringParameters))))))))))-(deftest migrate-parameters-with-linked-filters-and-values-query-type-test+(deftest ^:parallel migrate-parameters-with-linked-filters-and-values-query-type-test(testing "test that a Dashboard's :parameters filterParameters are cleared if the :values_query_type is 'none'"(doseq [[values_query_typekeep-filtering-parameters?] {"none" false@@ -70,7 +72,7 @@(:filteringParameters parameter)))(is (not (contains? parameter :filteringParameters))))))))))-(deftest migrate-parameters-empty-name-test+(deftest ^:parallel migrate-parameters-empty-name-test(testing "test that a Dashboard's :parameters is selected with a non-nil name and slug"(doseq [[name slug] [["" ""] ["" "slug"] ["name" ""]]](mt/with-temp [:model/Dashboard dashboard {:parameters [(merge@@ -99,7 +101,7 @@(is (nil? (t2/select-one :model/Pulse pulse-id)))(is (= 0 (count (pulse-channel-test/send-pulse-triggers pulse-id))))))))-(deftest parameter-card-test+(deftest ^:parallel parameter-card-test(testing "A new dashboard creates a new ParameterCard"(mt/with-temp [:model/Card {card-id :id} {}:model/Dashboard {dashboard-id :id} {:parameters [(merge default-parameter@@ -109,12 +111,13 @@:parameterized_object_type :dashboard:parameterized_object_id dashboard-id:parameter_id "_CATEGORY_NAME_"}- (t2/select-one 'ParameterCard :card_id card-id)))))+ (t2/select-one :model/ParameterCard :card_id card-id))))))+(deftest parameter-card-test-2(testing "Adding a card_id creates a new ParameterCard"(mt/with-temp [:model/Card {card-id :id} {}:model/Dashboard {dashboard-id :id} {:parameters [default-parameter]}]- (is (nil? (t2/select-one 'ParameterCard :card_id card-id)))+ (is (nil? (t2/select-one :model/ParameterCard :card_id card-id)))(t2/update! :model/Dashboard dashboard-id {:parameters [(merge default-parameter{:values_source_type "card":values_source_config {:card_id card-id}})]})@@ -122,8 +125,9 @@:parameterized_object_type :dashboard:parameterized_object_id dashboard-id:parameter_id "_CATEGORY_NAME_"}- (t2/select-one 'ParameterCard :card_id card-id)))))+ (t2/select-one :model/ParameterCard :card_id card-id))))))+(deftest parameter-card-test-3(testing "Removing a card_id deletes old ParameterCards"(mt/with-temp [:model/Card {card-id :id} {}:model/Dashboard {dashboard-id :id} {:parameters [(merge default-parameter@@ -131,7 +135,7 @@:values_source_config {:card_id card-id}})]}];; same setup as earlier test, we know the ParameterCard exists right now(t2/delete! :model/Dashboard :id dashboard-id)- (is (nil? (t2/select-one 'ParameterCard :card_id card-id))))))+ (is (nil? (t2/select-one :model/ParameterCard :card_id card-id))))))(deftest do-not-update-parameter-card-if-it-doesn't-change-test(testing "Do not update ParameterCard if updating a Dashboard doesn't change the parameters"@@ -219,13 +223,16 @@#"A Dashboard can only go in Collections in the \"default\" or :analytics namespace."(t2/update! :model/Dashboard card-id {:collection_id collection-id})))))))-(deftest validate-parameters-test+(deftest ^:parallel validate-parameters-test(testing "Should validate Dashboard :parameters when"(testing "creating"(is (thrown-with-msg?clojure.lang.ExceptionInfo#":parameters must be a sequence of maps with :id and :type keys"- (mt/with-temp [:model/Dashboard _ {:parameters {:a :b}}]))))+ (mt/with-temp [:model/Dashboard _ {:parameters {:a :b}}]))))))++(deftest validate-parameters-test-2+ (testing "Should validate Dashboard :parameters when"(testing "updating"(mt/with-temp [:model/Dashboard {:keys [id]} {:parameters []}](is (thrown-with-msg?@@ -253,12 +260,12 @@:id "_CATEGORY_NAME_":type :category:target expected- :values_query_type "list",- :values_source_type "card",+ :values_query_type :list+ :values_source_type :card:values_source_config {:card_id card-id, :value_field [:field 2 nil]}}](t2/select-one-fn :parameters :model/Dashboard :id dashboard-id))))))))-(deftest should-add-default-values-source-test+(deftest ^:parallel should-add-default-values-source-test(testing "shoudld add default if not exists"(mt/with-temp [:model/Dashboard {dashboard-id :id} {:parameters [{:name "Category Name":slug "category_name"@@ -268,8 +275,9 @@:slug "category_name":id "_CATEGORY_NAME_":type :category}]- (t2/select-one-fn :parameters :model/Dashboard :id dashboard-id)))))+ (t2/select-one-fn :parameters :model/Dashboard :id dashboard-id))))))+(deftest ^:parallel should-add-default-values-source-test-2(testing "shoudld not override if existsed "(mt/with-temp [:model/Card {card-id :id} {}:model/Dashboard {dashboard-id :id} {:parameters [{:name "Category Name"@@ -284,12 +292,12 @@:slug "category_name":id "_CATEGORY_NAME_":type :category- :values_query_type "list",- :values_source_type "card",+ :values_query_type :list+ :values_source_type :card:values_source_config {:card_id card-id, :value_field [:field 2 nil]}}](t2/select-one-fn :parameters :model/Dashboard :id dashboard-id))))))-(deftest identity-hash-test+(deftest ^:parallel identity-hash-test(testing "Dashboard hashes are composed of the name and parent collection's hash"(let [now #t "2022-09-01T12:34:56Z"](mt/with-temp [:model/Collection c1 {:name "top level" :location "/" :created_at now}@@ -298,7 +306,7 @@(serdes/raw-hash ["my dashboard" (serdes/identity-hash c1) (:created_at dash)])(serdes/identity-hash dash)))))))-(deftest descendants-test+(deftest ^:parallel descendants-test(testing "dashboard which have parameter's source is another card"(mt/with-temp[:model/Field field {:name "A field"}@@ -310,8 +318,9 @@:values_source_config {:card_id (:id card):value_field [:field (:id field) nil]}}]}](is (= {["Card" (:id card)] {"Dashboard" (:id dashboard)}}- (serdes/descendants "Dashboard" (:id dashboard))))))+ (serdes/descendants "Dashboard" (:id dashboard)))))))+(deftest descendants-test-2(testing "dashboard which has a dashcard with an action"(mt/with-actions [{:keys [action-id]} {}](mt/with-temp@@ -321,8 +330,9 @@:parameter_mappings []}](is (= {["Action" action-id] {"Dashboard" (:id dashboard)"DashboardCard" (:id dc)}}- (serdes/descendants "Dashboard" (:id dashboard)))))))+ (serdes/descendants "Dashboard" (:id dashboard))))))))+(deftest ^:parallel descendants-test-3(testing "dashboard in which its dashcards has parameter_mappings to a card"(mt/with-temp[:model/Card card1 {:name "Card attached to dashcard"}@@ -340,8 +350,9 @@"DashboardCard" (:id dc)}["Card" (:id card2)] {"Dashboard" (:id dashboard)"DashboardCard" (:id dc)}}- (serdes/descendants "Dashboard" (:id dashboard))))))+ (serdes/descendants "Dashboard" (:id dashboard)))))))+(deftest ^:parallel descendants-test-4(testing "dashboard in which its dashcards have series"(mt/with-temp[:model/Card card1 {:name "Card attached to dashcard"}diff --git a/test/metabase/driver/common/parameters/dates_test.clj b/test/metabase/driver/common/parameters/dates_test.cljindex 9cf006b9999..4cc8ff16ffc 100644--- a/test/metabase/driver/common/parameters/dates_test.clj+++ b/test/metabase/driver/common/parameters/dates_test.clj@@ -1,11 +1,7 @@(ns metabase.driver.common.parameters.dates-test(:require[clojure.test :refer :all]- [clojure.test.check.clojure-test :refer [defspec]]- [clojure.test.check.generators :as gen]- [clojure.test.check.properties :as prop]- [java-time.api :as t]- [metabase.driver.common.parameters.dates :as params.dates]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.dates :as params.dates][metabase.test :as mt]))(deftest ^:parallel date-string->filter-test@@ -141,344 +137,3 @@(testing "minutes"(is (thrown? clojure.lang.ExceptionInfo #"Don't know how to parse date string \"exclude-minutes-15-30\""(params.dates/date-string->filter "exclude-minutes-15-30" [:field "field" {:base-type :type/DateTime}])))))))--(defn do-date-string-range-test- [s->expected]- (mt/with-clock #t "2016-06-07T12:13:55Z"- (doseq [[s ranges] s->expected- [expected-range option] (map vector ranges [nil- {:inclusive-start? false}- {:inclusive-end? false}- {:inclusive-start? false :inclusive-end? false}])]- (testing (format "%s with options %s should parse to %s" (pr-str s) (pr-str option) (pr-str ranges))- (is (= expected-range- (params.dates/date-string->range s option)))))))--(deftest ^:parallel date-string->range-absolute-datetimes-test- (do-date-string-range-test- {"Q1-2016" [{:start "2016-01-01" :end "2016-03-31"} ;; inclusive start + end = true (default)- {:start "2015-12-31" :end "2016-03-31"} ;; inclusive start = false- {:start "2016-01-01" :end "2016-04-01"} ;; inclusive end = false- {:start "2015-12-31" :end "2016-04-01"}] ;; inclusive start + end = false- "2016-02" [{:start "2016-02-01" :end "2016-02-29"}- {:start "2016-01-31" :end "2016-02-29"}- {:start "2016-02-01" :end "2016-03-01"}- {:start "2016-01-31" :end "2016-03-01"}]- "2016-04-18" [{:start "2016-04-18" :end "2016-04-18"}- {:start "2016-04-17" :end "2016-04-18"}- {:start "2016-04-18" :end "2016-04-19"}- {:start "2016-04-17" :end "2016-04-19"}]- "2016-04-18~2016-04-23" [{:start "2016-04-18" :end "2016-04-23"}- {:start "2016-04-17" :end "2016-04-23"}- {:start "2016-04-18" :end "2016-04-24"}- {:start "2016-04-17" :end "2016-04-24"}]- "2016-04-18T10:30:00~2016-04-23T10:30:00" [{:start "2016-04-18T10:30:00" :end "2016-04-23T10:30:00"}- {:start "2016-04-18T10:29:00" :end "2016-04-23T10:30:00"}- {:start "2016-04-18T10:30:00" :end "2016-04-23T10:31:00"}- {:start "2016-04-18T10:29:00" :end "2016-04-23T10:31:00"}]-- "2016-04-18~" [{:start "2016-04-18"}- {:start "2016-04-17"}- {:start "2016-04-18"}- {:start "2016-04-17"}]- "~2016-04-18" [{:end "2016-04-18"}- {:end "2016-04-18"}- {:end "2016-04-19"}- {:end "2016-04-19"}]}))--(deftest ^:parallel date-string->range-relative-past-test- (do-date-string-range-test- {"past30seconds" [{:start "2016-06-07T12:13:25" :end "2016-06-07T12:13:54"}- {:start "2016-06-07T12:13:24" :end "2016-06-07T12:13:54"}- {:start "2016-06-07T12:13:25" :end "2016-06-07T12:13:55"}- {:start "2016-06-07T12:13:24" :end "2016-06-07T12:13:55"}]- "past5minutes~" [{:start "2016-06-07T12:08:00" :end "2016-06-07T12:13:00"}- {:start "2016-06-07T12:07:00" :end "2016-06-07T12:13:00"}- {:start "2016-06-07T12:08:00" :end "2016-06-07T12:14:00"}- {:start "2016-06-07T12:07:00" :end "2016-06-07T12:14:00"}]- "past3hours" [{:start "2016-06-07T09:00:00" :end "2016-06-07T11:00:00"}- {:start "2016-06-07T08:00:00" :end "2016-06-07T11:00:00"}- {:start "2016-06-07T09:00:00" :end "2016-06-07T12:00:00"}- {:start "2016-06-07T08:00:00" :end "2016-06-07T12:00:00"}]- "past3days" [{:start "2016-06-04" :end "2016-06-06"}- {:start "2016-06-03" :end "2016-06-06"}- {:start "2016-06-04" :end "2016-06-07"}- {:start "2016-06-03" :end "2016-06-07"}]- "past3days~" [{:start "2016-06-04" :end "2016-06-07"}- {:start "2016-06-03" :end "2016-06-07"}- {:start "2016-06-04" :end "2016-06-08"}- {:start "2016-06-03" :end "2016-06-08"}]- "past7days" [{:start "2016-05-31" :end "2016-06-06"}- {:start "2016-05-30" :end "2016-06-06"}- {:start "2016-05-31" :end "2016-06-07"}- {:start "2016-05-30" :end "2016-06-07"}]- "past30days" [{:start "2016-05-08" :end "2016-06-06"}- {:start "2016-05-07" :end "2016-06-06"}- {:start "2016-05-08" :end "2016-06-07"}- {:start "2016-05-07" :end "2016-06-07"}]- "past2months" [{:start "2016-04-01" :end "2016-05-31"}- {:start "2016-03-31" :end "2016-05-31"}- {:start "2016-04-01" :end "2016-06-01"}- {:start "2016-03-31" :end "2016-06-01"}]- "past2months~" [{:start "2016-04-01" :end "2016-06-30"}- {:start "2016-03-31" :end "2016-06-30"}- {:start "2016-04-01" :end "2016-07-01"}- {:start "2016-03-31" :end "2016-07-01"}]- "past13months" [{:start "2015-05-01" :end "2016-05-31"}- {:start "2015-04-30" :end "2016-05-31"}- {:start "2015-05-01" :end "2016-06-01"}- {:start "2015-04-30" :end "2016-06-01"}]- "past2quarters" [{:start "2015-10-01" :end "2016-03-31"}- {:start "2015-09-30" :end "2016-03-31"}- {:start "2015-10-01" :end "2016-04-01"}- {:start "2015-09-30" :end "2016-04-01"}]- "past2quarters~" [{:start "2015-10-01" :end "2016-06-30"}- {:start "2015-09-30" :end "2016-06-30"}- {:start "2015-10-01" :end "2016-07-01"}- {:start "2015-09-30" :end "2016-07-01"}]- "past1years" [{:start "2015-01-01" :end "2015-12-31"}- {:start "2014-12-31" :end "2015-12-31"}- {:start "2015-01-01" :end "2016-01-01"}- {:start "2014-12-31" :end "2016-01-01"}]- "past1years~" [{:start "2015-01-01" :end "2016-12-31"}- {:start "2014-12-31" :end "2016-12-31"}- {:start "2015-01-01" :end "2017-01-01"}- {:start "2014-12-31" :end "2017-01-01"}]- "past16years" [{:start "2000-01-01" :end "2015-12-31"}- {:start "1999-12-31" :end "2015-12-31"}- {:start "2000-01-01" :end "2016-01-01"}- {:start "1999-12-31" :end "2016-01-01"}]}))--(deftest ^:parallel date-string->range-relative-next-test- (do-date-string-range-test- {"next45seconds" [{:start "2016-06-07T12:13:56" :end "2016-06-07T12:14:40"}- {:start "2016-06-07T12:13:55" :end "2016-06-07T12:14:40"}- {:start "2016-06-07T12:13:56" :end "2016-06-07T12:14:41"}- {:start "2016-06-07T12:13:55" :end "2016-06-07T12:14:41"}]- "next20minutes" [{:start "2016-06-07T12:14:00" :end "2016-06-07T12:33:00"}- {:start "2016-06-07T12:13:00" :end "2016-06-07T12:33:00"}- {:start "2016-06-07T12:14:00" :end "2016-06-07T12:34:00"}- {:start "2016-06-07T12:13:00" :end "2016-06-07T12:34:00"}]- "next6hours" [{:start "2016-06-07T13:00:00" :end "2016-06-07T18:00:00"}- {:start "2016-06-07T12:00:00" :end "2016-06-07T18:00:00"}- {:start "2016-06-07T13:00:00" :end "2016-06-07T19:00:00"}- {:start "2016-06-07T12:00:00" :end "2016-06-07T19:00:00"}]- "next3days" [{:start "2016-06-08" :end "2016-06-10"}- {:start "2016-06-07" :end "2016-06-10"}- {:start "2016-06-08" :end "2016-06-11"}- {:start "2016-06-07" :end "2016-06-11"}]- "next3days~" [{:start "2016-06-07" :end "2016-06-10"}- {:start "2016-06-06" :end "2016-06-10"}- {:start "2016-06-07" :end "2016-06-11"}- {:start "2016-06-06" :end "2016-06-11"}]- "next7days" [{:start "2016-06-08" :end "2016-06-14"}- {:start "2016-06-07" :end "2016-06-14"}- {:start "2016-06-08" :end "2016-06-15"}- {:start "2016-06-07" :end "2016-06-15"}]- "next30days" [{:start "2016-06-08" :end "2016-07-07"}- {:start "2016-06-07" :end "2016-07-07"}- {:start "2016-06-08" :end "2016-07-08"}- {:start "2016-06-07" :end "2016-07-08"}]- "next2months" [{:start "2016-07-01" :end "2016-08-31"}- {:start "2016-06-30" :end "2016-08-31"}- {:start "2016-07-01" :end "2016-09-01"}- {:start "2016-06-30" :end "2016-09-01"}]- "next2months~" [{:start "2016-06-01" :end "2016-08-31"}- {:start "2016-05-31" :end "2016-08-31"}- {:start "2016-06-01" :end "2016-09-01"}- {:start "2016-05-31" :end "2016-09-01"}]- "next2quarters" [{:start "2016-07-01" :end "2016-12-31"}- {:start "2016-06-30" :end "2016-12-31"}- {:start "2016-07-01" :end "2017-01-01"}- {:start "2016-06-30" :end "2017-01-01"}]- "next2quarters~" [{:start "2016-04-01" :end "2016-12-31"}- {:start "2016-03-31" :end "2016-12-31"}- {:start "2016-04-01" :end "2017-01-01"}- {:start "2016-03-31" :end "2017-01-01"}]- "next13months" [{:start "2016-07-01" :end "2017-07-31"}- {:start "2016-06-30" :end "2017-07-31"}- {:start "2016-07-01" :end "2017-08-01"}- {:start "2016-06-30" :end "2017-08-01"}]- "next1years" [{:start "2017-01-01" :end "2017-12-31"}- {:start "2016-12-31" :end "2017-12-31"}- {:start "2017-01-01" :end "2018-01-01"}- {:start "2016-12-31" :end "2018-01-01"}]- "next1years~" [{:start "2016-01-01" :end "2017-12-31"}- {:start "2015-12-31" :end "2017-12-31"}- {:start "2016-01-01" :end "2018-01-01"}- {:start "2015-12-31" :end "2018-01-01"}]- "next16years" [{:start "2017-01-01" :end "2032-12-31"}- {:start "2016-12-31" :end "2032-12-31"}- {:start "2017-01-01" :end "2033-01-01"}- {:start "2016-12-31" :end "2033-01-01"}]}))--(deftest ^:parallel date-string->range-relative-this-test- (do-date-string-range-test- {"thissecond" [{:start "2016-06-07T12:13:55" :end "2016-06-07T12:13:55"}- {:start "2016-06-07T12:13:54" :end "2016-06-07T12:13:55"}- {:start "2016-06-07T12:13:55" :end "2016-06-07T12:13:56"}- {:start "2016-06-07T12:13:54" :end "2016-06-07T12:13:56"}]- "thisminute" [{:start "2016-06-07T12:13:00" :end "2016-06-07T12:13:00"}- {:start "2016-06-07T12:12:00" :end "2016-06-07T12:13:00"}- {:start "2016-06-07T12:13:00" :end "2016-06-07T12:14:00"}- {:start "2016-06-07T12:12:00" :end "2016-06-07T12:14:00"}]- "thishour" [{:start "2016-06-07T12:00:00" :end "2016-06-07T12:00:00"}- {:start "2016-06-07T11:00:00" :end "2016-06-07T12:00:00"}- {:start "2016-06-07T12:00:00" :end "2016-06-07T13:00:00"}- {:start "2016-06-07T11:00:00" :end "2016-06-07T13:00:00"}]- "thisday" [{:start "2016-06-07" :end "2016-06-07"}- {:start "2016-06-06" :end "2016-06-07"}- {:start "2016-06-07" :end "2016-06-08"}- {:start "2016-06-06" :end "2016-06-08"}]- "thisweek" [{:start "2016-06-05" :end "2016-06-11"}- {:start "2016-06-04" :end "2016-06-11"}- {:start "2016-06-05" :end "2016-06-12"}- {:start "2016-06-04" :end "2016-06-12"}]- "thismonth" [{:start "2016-06-01" :end "2016-06-30"}- {:start "2016-05-31" :end "2016-06-30"}- {:start "2016-06-01" :end "2016-07-01"}- {:start "2016-05-31" :end "2016-07-01"}]- "thisquarter" [{:start "2016-04-01" :end "2016-06-30"}- {:start "2016-03-31" :end "2016-06-30"}- {:start "2016-04-01" :end "2016-07-01"}- {:start "2016-03-31" :end "2016-07-01"}]- "thisyear" [{:start "2016-01-01" :end "2016-12-31"}- {:start "2015-12-31" :end "2016-12-31"}- {:start "2016-01-01" :end "2017-01-01"}- {:start "2015-12-31" :end "2017-01-01"}]}))--(deftest ^:parallel date-string->range-relative-last-test- (do-date-string-range-test- {"lastsecond" [{:start "2016-06-07T12:13:54" :end "2016-06-07T12:13:54"}- {:start "2016-06-07T12:13:53" :end "2016-06-07T12:13:54"}- {:start "2016-06-07T12:13:54" :end "2016-06-07T12:13:55"}- {:start "2016-06-07T12:13:53" :end "2016-06-07T12:13:55"}]- "lastminute" [{:start "2016-06-07T12:12:00" :end "2016-06-07T12:12:00"}- {:start "2016-06-07T12:11:00" :end "2016-06-07T12:12:00"}- {:start "2016-06-07T12:12:00" :end "2016-06-07T12:13:00"}- {:start "2016-06-07T12:11:00" :end "2016-06-07T12:13:00"}]- "lasthour" [{:start "2016-06-07T11:00:00" :end "2016-06-07T11:00:00"}- {:start "2016-06-07T10:00:00" :end "2016-06-07T11:00:00"}- {:start "2016-06-07T11:00:00" :end "2016-06-07T12:00:00"}- {:start "2016-06-07T10:00:00" :end "2016-06-07T12:00:00"}]- "lastweek" [{:start "2016-05-29" :end "2016-06-04"}- {:start "2016-05-28" :end "2016-06-04"}- {:start "2016-05-29" :end "2016-06-05"}- {:start "2016-05-28" :end "2016-06-05"}]- "lastmonth" [{:start "2016-05-01" :end "2016-05-31"}- {:start "2016-04-30" :end "2016-05-31"}- {:start "2016-05-01" :end "2016-06-01"}- {:start "2016-04-30" :end "2016-06-01"}]- "lastquarter" [{:start "2016-01-01" :end "2016-03-31"}- {:start "2015-12-31" :end "2016-03-31"}- {:start "2016-01-01" :end "2016-04-01"}- {:start "2015-12-31" :end "2016-04-01"}]- "lastyear" [{:start "2015-01-01" :end "2015-12-31"}- {:start "2014-12-31" :end "2015-12-31"}- {:start "2015-01-01" :end "2016-01-01"}- {:start "2014-12-31" :end "2016-01-01"}]}))--(deftest ^:parallel date-string->range-relative-today-yesterday-test- (do-date-string-range-test- {"yesterday" [{:start "2016-06-06" :end "2016-06-06"}- {:start "2016-06-05" :end "2016-06-06"}- {:start "2016-06-06" :end "2016-06-07"}- {:start "2016-06-05" :end "2016-06-07"}]- "today" [{:start "2016-06-07" :end "2016-06-07"}- {:start "2016-06-06" :end "2016-06-07"}- {:start "2016-06-07" :end "2016-06-08"}- {:start "2016-06-06" :end "2016-06-08"}]}))--(deftest ^:parallel date-string->range-relative-past-from-test- (do-date-string-range-test- {"past1days-from-0days" [{:start "2016-06-06" :end "2016-06-06"}- {:start "2016-06-05" :end "2016-06-06"}- {:start "2016-06-06" :end "2016-06-07"}- {:start "2016-06-05" :end "2016-06-07"}]- "past1months-from-0months" [{:start "2016-05-01" :end "2016-05-31"}- {:start "2016-04-30" :end "2016-05-31"}- {:start "2016-05-01" :end "2016-06-01"}- {:start "2016-04-30" :end "2016-06-01"}]- "past1months-from-36months" [{:start "2013-05-01" :end "2013-05-31"}- {:start "2013-04-30" :end "2013-05-31"}- {:start "2013-05-01" :end "2013-06-01"}- {:start "2013-04-30" :end "2013-06-01"}]- "past1years-from-36months" [{:start "2012-01-01" :end "2012-12-31"}- {:start "2011-12-31" :end "2012-12-31"}- {:start "2012-01-01" :end "2013-01-01"}- {:start "2011-12-31" :end "2013-01-01"}]- "past3days-from-3years" [{:start "2013-06-04" :end "2013-06-06"}- {:start "2013-06-03" :end "2013-06-06"}- {:start "2013-06-04" :end "2013-06-07"}- {:start "2013-06-03" :end "2013-06-07"}]}))--(deftest ^:parallel date-string->range-relative-next-from-test- (do-date-string-range-test- {"next2days-from-1months" [{:start "2016-07-08" :end "2016-07-09"}- {:start "2016-07-07" :end "2016-07-09"}- {:start "2016-07-08" :end "2016-07-10"}- {:start "2016-07-07" :end "2016-07-10"}]- "next1months-from-0months" [{:start "2016-07-01" :end "2016-07-31"}- {:start "2016-06-30" :end "2016-07-31"}- {:start "2016-07-01" :end "2016-08-01"}- {:start "2016-06-30" :end "2016-08-01"}]- "next1months-from-36months" [{:start "2019-07-01" :end "2019-07-31"}- {:start "2019-06-30" :end "2019-07-31"}- {:start "2019-07-01" :end "2019-08-01"}- {:start "2019-06-30" :end "2019-08-01"}]- "next1years-from-36months" [{:start "2020-01-01" :end "2020-12-31"}- {:start "2019-12-31" :end "2020-12-31"}- {:start "2020-01-01" :end "2021-01-01"}- {:start "2019-12-31" :end "2021-01-01"}]- "next3days-from-3years" [{:start "2019-06-08" :end "2019-06-10"}- {:start "2019-06-07" :end "2019-06-10"}- {:start "2019-06-08" :end "2019-06-11"}- {:start "2019-06-07" :end "2019-06-11"}]- "next7hours-from-13months" [{:start "2017-07-07T13:00:00" :end "2017-07-07T19:00:00"}- {:start "2017-07-07T12:00:00" :end "2017-07-07T19:00:00"}- {:start "2017-07-07T13:00:00" :end "2017-07-07T20:00:00"}- {:start "2017-07-07T12:00:00" :end "2017-07-07T20:00:00"}]}))--(deftest ^:parallel relative-dates-with-starting-from-zero-must-match- (testing "relative dates need to behave the same way, offset or not."- (mt/with-clock #t "2016-06-07T12:13:55Z"- (testing "'past1months-from-0months' should be the same as: 'past1months'"- (is (= {:start "2016-05-01" :end "2016-05-31"}- (params.dates/date-string->range "past1months")- (params.dates/date-string->range "past1months-from-0months"))))- (testing "'next1months-from-0months' should be the same as: 'next1months'"- (is (= {:start "2016-07-01" :end "2016-07-31"}- (params.dates/date-string->range "next1months")- (params.dates/date-string->range "next1months-from-0months")))))))--(def ^:private time-range-generator- (let [time-units (mapv #(str % "s") (keys @#'params.dates/operations-by-date-unit))]- (gen/fmap- (fn [[frame n unit unit2]]- [(str frame n unit)- (str frame n unit "-from-0" unit2)])- (gen/tuple- (gen/elements #{"next" "past"})- (gen/such-that #(not= % 0) gen/nat)- (gen/elements time-units)- (gen/elements time-units)))))--(defspec ^:parallel date-string->range-spec-test 1000- (prop/for-all [[tr tr+from-zero] time-range-generator]- (mt/with-clock (t/zoned-date-time)- (= (params.dates/date-string->range tr)- (params.dates/date-string->range tr+from-zero)))))--(deftest custom-start-of-week-test- (testing "Relative filters should respect the custom `start-of-week` Setting (#14294)"- (mt/with-clock #t "2021-03-01T14:15:00-08:00[US/Pacific]"- (doseq [[first-day-of-week expected] {"sunday" {:start "2021-02-21" :end "2021-02-27"}- "monday" {:start "2021-02-22" :end "2021-02-28"}- "tuesday" {:start "2021-02-16" :end "2021-02-22"}- "wednesday" {:start "2021-02-17" :end "2021-02-23"}- "thursday" {:start "2021-02-18" :end "2021-02-24"}- "friday" {:start "2021-02-19" :end "2021-02-25"}- "saturday" {:start "2021-02-20" :end "2021-02-26"}}]- (mt/with-temporary-setting-values [start-of-week first-day-of-week]- (is (= expected- (params.dates/date-string->range "past1weeks"))))))))diff --git a/test/metabase/driver/common/parameters/operators_test.clj b/test/metabase/driver/common/parameters/operators_test.cljindex a835d7cc9dd..af42a40b7cf 100644--- a/test/metabase/driver/common/parameters/operators_test.clj+++ b/test/metabase/driver/common/parameters/operators_test.clj@@ -1,7 +1,7 @@(ns metabase.driver.common.parameters.operators-test(:require[clojure.test :refer :all]- [metabase.driver.common.parameters.operators :as params.ops]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.operators :as params.ops][metabase.query-processor.error-type :as qp.error-type]))(deftest ^:parallel to-clause-testdiff --git a/test/metabase/driver/common/parameters/parse_test.clj b/test/metabase/driver/common/parameters/parse_test.cljindex 587e7b0dd79..4ca54490289 100644--- a/test/metabase/driver/common/parameters/parse_test.clj+++ b/test/metabase/driver/common/parameters/parse_test.clj@@ -1,8 +1,8 @@(ns metabase.driver.common.parameters.parse-test(:require[clojure.test :refer :all]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.parse :as params.parse]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse][metabase.lib.parse :as lib.parse]))(defn- param [field-name] (params/->Param field-name))diff --git a/test/metabase/driver/common/parameters/values_test.clj b/test/metabase/driver/common/parameters/values_test.cljindex 043e2236972..02bb10f11e9 100644--- a/test/metabase/driver/common/parameters/values_test.clj+++ b/test/metabase/driver/common/parameters/values_test.clj@@ -4,8 +4,8 @@[clojure.string :as str][clojure.test :refer :all][metabase.driver :as driver]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.values :as params.values]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.values :as params.values][metabase.driver.ddl.interface :as ddl.i][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.lib.metadata :as lib.metadata]@@ -17,7 +17,7 @@[metabase.permissions.models.permissions-group :as perms-group][metabase.query-processor :as qp][metabase.query-processor.middleware.permissions :as qp.perms]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.system.core :as system][metabase.test :as mt][metabase.util :as u]diff --git a/test/metabase/driver/common/table_rows_sample_test.clj b/test/metabase/driver/common/table_rows_sample_test.cljindex ff768e42a9b..bfb3df87272 100644--- a/test/metabase/driver/common/table_rows_sample_test.clj+++ b/test/metabase/driver/common/table_rows_sample_test.clj@@ -2,13 +2,15 @@(:require[clojure.test :refer :all][metabase.driver :as driver]+ [metabase.driver-api.core :as driver-api][metabase.driver.common.table-rows-sample :as table-rows-sample]- [metabase.driver.util :as driver.u]- [metabase.models.interface :as mi]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp][metabase.sync.analyze.fingerprint :as fingerprint][metabase.test :as mt]- [metabase.warehouse-schema.models.table :as table][toucan2.core :as t2]));;; whether to run `field-count` and `field-distinct-count` tests.@@ -21,6 +23,19 @@[_driver _feature _database]false)+(deftest ^:parallel table-rows-sample-query-options-test+ (let [mp meta/metadata-provider+ table (meta/table-metadata :orders)+ fields (driver-api/fields mp (:id table))+ id-column (meta/field-metadata :orders :id)+ opts {:limit 100+ :order-by [(driver-api/order-by-clause id-column :desc)]}]+ (is (=? {:stages [{:source-table (meta/id :orders)+ :fields sequential?+ :order-by [[:desc {} [:field {} (meta/id :orders :id)]]]+ :limit 100}]}+ (#'table-rows-sample/table-rows-sample-query mp table fields opts)))))+(deftest ^:parallel table-rows-sample-test(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc})(let [expected [["20th Century Cafe"]@@ -46,61 +61,95 @@(fetch size))"Did not truncate a text field"))))))-(deftest table-rows-sample-substring-test+(driver/register! ::does-not-support-expressions-driver, :abstract? true)++(defmethod driver/database-supports? [::does-not-support-expressions-driver :expressions]+ [_driver _feature _database]+ false)++(deftest ^:parallel table-rows-sample-substring-test+ (testing "substring checking"+ (let [database (-> meta/database+ (assoc :id 456, :engine :h2))+ table (-> (meta/table-metadata :venues)+ (assoc :id 1234, :db-id 456))+ fields [(-> (meta/field-metadata :venues :name)+ (assoc :table-id 1234, :id 4321, :base-type :type/Text))]+ mp (lib.tu/mock-metadata-provider+ {:database database, :tables [table], :fields fields})]+ (testing "uses substrings if driver supports expressions"+ (let [query (#'table-rows-sample/table-rows-sample-query mp table fields {:truncation-size 4})]+ (is (seq (lib/expressions query)))))+ (testing "doesnt' use substrings if driver doesn't support expressions"+ (let [mp (lib.tu/merged-mock-metadata-provider+ mp+ {:database (assoc database :engine ::does-not-support-expressions-driver)})+ query (#'table-rows-sample/table-rows-sample-query mp table fields {:truncation-size 4})]+ (is (empty? (get-in query [:query :expressions]))))))))++(deftest ^:parallel table-rows-sample-substring-test-2+ (testing "substring checking"+ (testing "serialized json stored in text columns is treated as text (potentially large, OOM risk)"+ (let [database (-> meta/database+ (assoc :id 456, :engine :h2))+ table (-> (meta/table-metadata :venues)+ (assoc :id 1234, :db-id 456))+ fields [(-> (meta/field-metadata :venues :name)+ (assoc :table-id 1234, :id 4321, :base-type :type/Text, :semantic-type :type/SerializedJSON))]+ mp (lib.tu/mock-metadata-provider+ {:database database, :tables [table], :fields fields})+ query (#'table-rows-sample/table-rows-sample-query mp table fields {:truncation-size 4})]+ (is (seq (lib/expressions query)))))))++(deftest ^:parallel table-rows-sample-substring-test-3+ (testing "substring checking"+ (testing "substring is not called upon type/Text derivates"+ (doseq [base-type (filter #(not (or (isa? % :Semantic/*)+ (isa? % :Relation/*)))+ (descendants :type/Text))]+ (let [database (-> meta/database+ (assoc :id 456, :engine :h2))+ table (-> (meta/table-metadata :venues)+ (assoc :id 1234, :db-id 456))+ fields [(-> (meta/field-metadata :venues :name)+ (assoc :table-id 1234, :id 4321, :base-type base-type, :effective-type base-type))]+ mp (lib.tu/mock-metadata-provider+ {:database database, :tables [table], :fields fields})+ query (#'table-rows-sample/table-rows-sample-query mp table fields {:truncation-size 4})]+ (is (empty? (lib/expressions query))))))))++(deftest ^:parallel table-rows-sample-substring-test-4(testing "substring checking"- (with-redefs [driver.u/database->driver (constantly (:engine (mt/db)))- table/database (constantly (mi/instance :model/Database {:id 5678}))]- (let [table (mi/instance :model/Table {:id 1234})- fields [(mi/instance :model/Field {:id 4321 :base_type :type/Text})]]- (testing "uses substrings if driver supports expressions"- (with-redefs [driver.u/supports? (constantly true)]- (let [query (#'table-rows-sample/table-rows-sample-query table fields {:truncation-size 4})]- (is (seq (get-in query [:query :expressions]))))))- (testing "doesnt' use substrings if driver doesn't support expressions"- (with-redefs [driver.u/supports? (constantly false)]- (let [query (#'table-rows-sample/table-rows-sample-query table fields {:truncation-size 4})]- (is (empty? (get-in query [:query :expressions])))))))- (testing "serialized json stored in text columns is treated as text (potentially large, OOM risk)"- (let [table (mi/instance :model/Table {:id 1234})- fields [(mi/instance :model/Field {:id 4321, :base_type :type/Text, :semantic_type :type/SerializedJSON})]]- (with-redefs [driver.u/supports? (constantly true)]- (let [query (#'table-rows-sample/table-rows-sample-query table fields {:truncation-size 4})]- (is (seq (get-in query [:query :expressions])))))))- (testing "substring is not called upon type/Text derivates"- (doseq [base-type (filter #(not (or (isa? % :Semantic/*)- (isa? % :Relation/*)))- (descendants :type/Text))]- (let [table (mi/instance :model/Table {:id 1234})- fields [(mi/instance :model/Field {:id 4321, :base_type base-type})]]- (with-redefs [driver.u/supports? (constantly true)]- (let [query (#'table-rows-sample/table-rows-sample-query table fields {:truncation-size 4})]- (is (empty? (get-in query [:query :expressions]))))))))- (testing "primary check is on effective_type"- (with-redefs [driver.u/supports? (constantly true)]- (let [table (mi/instance :model/Table {:id 1234})- fields [(mi/instance :model/Field {:id 4321, :base_type :type/Number :effective_type :type/Text})]- query (#'table-rows-sample/table-rows-sample-query table fields {:truncation-size 4})]- (is (seq (get-in query [:query :expressions])))))))))+ (testing "primary check is on effective_type"+ (let [database (-> meta/database+ (assoc :id 456, :engine :h2))+ table (-> (meta/table-metadata :venues)+ (assoc :id 1234, :db-id 456))+ fields [(-> (meta/field-metadata :venues :name)+ (assoc :table-id 1234, :id 4321, :base-type :type/Number, :effective-type :type/Text))]+ mp (lib.tu/mock-metadata-provider+ {:database database, :tables [table], :fields fields})+ query (#'table-rows-sample/table-rows-sample-query mp table fields {:truncation-size 4})]+ (is (seq (lib/expressions query)))))))(deftest ^:synchronized coerced-field-substring-integration-test(testing "For coerced fields, effective type is used for fingerprinting (string -> number exmaple)"(mt/datasetcoerced-string-nums-db(doseq [id (t2/select-fn-vec :id :model/Field :table_id (mt/id :string_nums))]- (t2/update! :model/Field :id id {:fingerprint nil+ (t2/update! :model/Field :id id {:fingerprint nil:fingerprint_version 0}))- (let [fingerprints (atom [])- fingerprint-query (atom nil)+ (let [fingerprints (atom [])+ fingerprint-query (atom nil)orig-table-rows-sample-query @#'table-rows-sample/table-rows-sample-query]- (with-redefs [fingerprint/save-fingerprint! (fn [_field fingerprint]- (swap! fingerprints conj fingerprint))+ (with-redefs [fingerprint/save-fingerprint! (fn [_field fingerprint]+ (swap! fingerprints conj fingerprint))table-rows-sample/table-rows-sample-query (fn [& args](reset! fingerprint-query(apply orig-table-rows-sample-query args)))](fingerprint/fingerprint-table! (t2/select-one :model/Table :id (mt/id :string_nums)))(testing "empty expressions = no substring optimization in sample query = use of effective type"- (is (empty? (-> @fingerprint-query :query :expressions)))- (is (seq (-> @fingerprint-query :query :fields))))+ (is (empty? (lib/expressions @fingerprint-query))))(testing "query returns number types due coercion -> numbers are fingerprinted"(is (=? [{:type {:type/Number {}}}{:type {:type/Number {}}}@@ -115,14 +164,18 @@:model/Field field2 {:name "group_id" :table_id (:id table) :database_partitioned true :base_type :type/Integer}](testing "the sample rows query on a table that requires a filter will include a filter";; currently only applied for bigquery tables in which a table can have a required partition filter- (is (=? [:> [:field (:id field2) {:base-type :type/Integer}] (mt/malli=? int?)]- (get-in (#'table-rows-sample/table-rows-sample-query table [field1] {}) [:query :filter]))))+ (let [mp (lib-be/application-database-metadata-provider (:id db))+ table (lib-be/instance->metadata table :metadata/table)+ field1 (lib-be/instance->metadata field1 :metadata/column)]+ (is (=? [[:> {}+ [:field {:base-type :type/Integer} (:id field2)]+ int?]]+ (lib/filters (#'table-rows-sample/table-rows-sample-query mp table [field1] {}))))))(testing "the mbql on a table that requires a filter will include a filter";; currently only applied for bigquery tables in which a table can have a required partition filter- (let [query (atom nil)]- (with-redefs [qp/process-query (fn [& args]- (reset! query (-> args first :query)))]- (is (=? {:filter [:> [:field (:id field2) {:base-type :type/Integer}] (mt/malli=? int?)]}- (table-rows-sample/table-rows-sample table [] (constantly conj))))- (is (=? {:filter [:> [:field (:id field2) {:base-type :type/Integer}] (mt/malli=? int?)]}- @query)))))))+ (with-redefs [qp/process-query (fn [& args]+ (first args))]+ (is (=? [[:> {}+ [:field {:base-type :type/Integer} (:id field2)]+ int?]]+ (lib/filters (table-rows-sample/table-rows-sample table [] (constantly conj)))))))))diff --git a/test/metabase/driver/h2_test.clj b/test/metabase/driver/h2_test.cljindex 070317a6ebb..f17d788e3bd 100644--- a/test/metabase/driver/h2_test.clj+++ b/test/metabase/driver/h2_test.clj@@ -15,8 +15,11 @@[metabase.driver.sql-jdbc.actions :as sql-jdbc.actions][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql.query-processor :as sql.qp]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.test :as mt][metabase.util.honey-sql-2 :as h2x][toucan2.core :as t2]))@@ -37,11 +40,9 @@(testing "Check that we add safe connection options to connection strings"(is (= "file:my-file;LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON=NICE_TRY;IFEXISTS=TRUE"(#'h2/connection-string-set-safe-options "file:my-file;;LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON=NICE_TRY"))))-(testing "Check that we override shady connection string options set by shady admins with safe ones"(is (= "file:my-file;LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON=NICE_TRY;IFEXISTS=TRUE"(#'h2/connection-string-set-safe-options "file:my-file;;LOOK_I_INCLUDED_AN_EXTRA_SEMICOLON=NICE_TRY;IFEXISTS=FALSE;"))))-(testing "Check that we override the INIT connection string option"(is (= "file:my-file;IFEXISTS=TRUE"(#'h2/connection-string-set-safe-options "file:my-file;INIT=ANYTHING_HERE_WILL_BE_IGNORED")))))@@ -50,11 +51,9 @@(testing "make sure we return the USER from db details if it is a keyword key in details..."(is (= "cam"(#'h2/db-details->user {:db "file:my_db.db", :USER "cam"}))))-(testing "or a string key..."(is (= "cam"(#'h2/db-details->user {:db "file:my_db.db", "USER" "cam"}))))-(testing "or part of the `db` connection string itself"(is (= "cam"(#'h2/db-details->user {:db "file:my_db.db;USER=cam"})))))@@ -80,7 +79,9 @@(is (instance? clojure.lang.ExceptionInfo result))(is (partial= {:cause "Malicious keys detected":data {:keys ["TRACE_LEVEL_SYSTEM_OUT"]}}- (Throwable->map result)))))+ (Throwable->map result))))))++(deftest ^:parallel only-connect-when-non-malicious-properties-2(testing "Reject connection details which lie about their driver"(let [conn "mem:fake-h2-db"f (fn f [details]@@ -110,15 +111,21 @@t/zone-id.normalized)))))-(deftest disallow-admin-accounts-test+(deftest ^:parallel disallow-admin-accounts-test(testing "Check that we're not allowed to run SQL against an H2 database with a non-admin account"(mt/with-temp [:model/Database db {:name "Fake-H2-DB", :engine "h2", :details {:db "mem:fake-h2-db"}}]- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"Running SQL queries against H2 databases using the default \(admin\) database user is forbidden\.$"- (qp/process-query {:database (:id db)- :type :native- :native {:query "SELECT 1"}}))))))+ (doseq [[query-type query] {"legacy MBQL query"+ {:database (:id db)+ :type :native+ :native {:query "SELECT 1"}}++ "MBQL 5 query"+ (lib/native-query (lib-be/application-database-metadata-provider (:id db)) "SELECT 1")}]+ (testing query-type+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Running SQL queries against H2 databases using the default \(admin\) database user is forbidden\.$"+ (qp/process-query query))))))))(deftest ^:parallel add-interval-honeysql-form-test(testing "Should convert fractional seconds to milliseconds"@@ -269,13 +276,21 @@:engine :h2:native {:query trigger-creation-attempt}}))))))+(defn- check-read-only-statements [query]+ (mt/with-metadata-provider (mt/id)+ (-> query+ qp.preprocess/preprocess+ qp.compile/attach-compiled-query+ (#'h2/check-read-only-statements))))+(deftest ^:parallel check-read-only-test(testing "read only statements should pass"(are [query] (nil?(mt/with-metadata-provider (mt/id)(#'h2/check-read-only-statements- {:engine :h2- :native {:query query}})))+ {:database 1+ :type :native+ :native {:query query}})))"select * from orders""select 1; select 2;""explain select * from orders"@@ -293,8 +308,9 @@#"Only SELECT statements are allowed in a native query."(mt/with-metadata-provider (mt/id)(#'h2/check-read-only-statements- {:engine :h2- :native {:query query}}))+ {:database 1+ :type :native+ :native {:query query}}))"update venues set name = 'bill'")"insert into venues (name) values ('bill')""delete venues"@@ -344,7 +360,7 @@(mt/test-driver :h2(when config/ee-available?(let [audit-db-expected-id 13371337- original-audit-db (t2/select-one 'Database :is_audit true)]+ original-audit-db (t2/select-one :model/Database :is_audit true)](is (not= ::mbc/noop (mbc/ensure-audit-db-installed!))"Make sure we call the right ensure-audit-db-installed! impl")(trydiff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.cljindex fc5d54a756a..61fbb8fd5b1 100644--- a/test/metabase/driver/mysql_test.clj+++ b/test/metabase/driver/mysql_test.clj@@ -25,7 +25,7 @@[metabase.query-processor :as qp][metabase.query-processor-test.string-extracts-test :as string-extracts-test][metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.analyze.fingerprint :as sync.fingerprint][metabase.sync.core :as sync][metabase.sync.sync-metadata.tables :as sync-tables]diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 95aa1112504..61689668b4f 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -32,7 +32,7 @@[metabase.lib.test-util.metadata-providers.mock :as providers.mock][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.secrets.models.secret :as secret][metabase.sync.core :as sync][metabase.sync.sync-metadata :as sync-metadata]@@ -1354,10 +1354,10 @@(-> {:query (str "SELECT * FROM json_table ""WHERE json_val::jsonb ? 'a' ""AND json_val::jsonb ->> 'a' = {{val}}")- :template-tags {:val+ :template-tags {"val"{:name "val"- :display_name "Val"- :type "text"}}}+ :display-name "Val"+ :type :text}}}mt/native-query(assoc :parameters[{:type "number/="diff --git a/test/metabase/driver/sql/parameters/substitute_test.clj b/test/metabase/driver/sql/parameters/substitute_test.cljindex f4339a1bbca..7e18834c94c 100644--- a/test/metabase/driver/sql/parameters/substitute_test.clj+++ b/test/metabase/driver/sql/parameters/substitute_test.clj@@ -4,8 +4,8 @@[clojure.test :refer :all][java-time.api :as t][metabase.driver :as driver]- [metabase.driver.common.parameters :as params]- [metabase.driver.common.parameters.parse :as params.parse]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters.parse :as params.parse][metabase.driver.sql.parameters.substitute :as sql.params.substitute][metabase.legacy-mbql.normalize :as mbql.normalize][metabase.lib.core :as lib]diff --git a/test/metabase/driver/sql/parameters/substitution_test.clj b/test/metabase/driver/sql/parameters/substitution_test.cljindex d75e3860fe1..183788b621d 100644--- a/test/metabase/driver/sql/parameters/substitution_test.clj+++ b/test/metabase/driver/sql/parameters/substitution_test.clj@@ -5,12 +5,12 @@[clojure.test :refer :all][java-time.api :as t][metabase.driver :as driver]- [metabase.driver.common.parameters :as params]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.driver.common.parameters :as params][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp][metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.util.honey-sql-2 :as h2x]))diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.cljindex 8706e581bd1..b6a9b955866 100644--- a/test/metabase/driver/sql/query_processor_test.clj+++ b/test/metabase/driver/sql/query_processor_test.clj@@ -19,7 +19,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.limit :as limit][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as add][metabase.settings.core :as setting][metabase.test :as mt]diff --git a/test/metabase/driver/sql_jdbc/actions_test.clj b/test/metabase/driver/sql_jdbc/actions_test.cljindex 37c7859f02d..962cb280547 100644--- a/test/metabase/driver/sql_jdbc/actions_test.clj+++ b/test/metabase/driver/sql_jdbc/actions_test.clj@@ -10,7 +10,7 @@[metabase.driver :as driver][metabase.driver.sql-jdbc.actions :as sql-jdbc.actions][metabase.lib.schema.id :as lib.schema.id]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu]diff --git a/test/metabase/driver/sql_jdbc/connection_test.clj b/test/metabase/driver/sql_jdbc/connection_test.cljindex 38d9666c162..3489cec120b 100644--- a/test/metabase/driver/sql_jdbc/connection_test.clj+++ b/test/metabase/driver/sql_jdbc/connection_test.clj@@ -230,7 +230,9 @@(deftest connection-pool-does-not-cache-audit-db(mt/test-drivers app-db-types(when config/ee-available?- (t2/delete! 'Database {:where [:= :is_audit true]})+ ;; TODO (Cam 9/30/25) -- sort of evil to delete databases like this in a test, shouldn't we do this in a+ ;; transaction or something?+ (t2/delete! :model/Database {:where [:= :is_audit true]})(let [status (mbc/ensure-audit-db-installed!)audit-db-id (t2/select-one-fn :id :model/Database {:where [:= :is_audit true]})_ (is (= :metabase-enterprise.audit-app.audit/installed status))diff --git a/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj b/test/metabase/driver/sql_jdbc/sync/describe_database_test.cljindex 2ad77cf6ef6..96bb58149ec 100644--- a/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj+++ b/test/metabase/driver/sql_jdbc/sync/describe_database_test.clj@@ -13,7 +13,7 @@[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface][metabase.driver.util :as driver.u][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.sync.core :as sync][metabase.test :as mt][metabase.test.data.one-off-dbs :as one-off-dbs]diff --git a/test/metabase/driver/util_test.clj b/test/metabase/driver/util_test.cljindex 905bffb19d3..8ceb64acd54 100644--- a/test/metabase/driver/util_test.clj+++ b/test/metabase/driver/util_test.clj@@ -9,7 +9,7 @@[metabase.driver.util :as driver.u][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u])diff --git a/test/metabase/embedding/api/embed_test.clj b/test/metabase/embedding/api/embed_test.cljindex 758940fb3ea..7da76ffd513 100644--- a/test/metabase/embedding/api/embed_test.clj+++ b/test/metabase/embedding/api/embed_test.clj@@ -146,7 +146,8 @@{:description nil:display "table":visualization_settings {}- :dataset_query {:type "query"}+ :dataset_query {:lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/mbql"}]}:parameters []:param_fields {}})@@ -1509,7 +1510,8 @@(with-embedding-enabled-and-new-secret-key!(mt/with-temp [:model/Card card {:dataset_query (mt/native-query{:query "SELECT count(*) FROM orders WHERE quantity = {{qty_locked}}"- :template-tags {"qty_locked" {:name "qty_locked"+ :template-tags {"qty_locked" {:id "0123"+ :name "qty_locked":display-name "Quantity (Locked)":type :number}}}):enable_embedding truediff --git a/test/metabase/embedding/api/preview_embed_test.clj b/test/metabase/embedding/api/preview_embed_test.cljindex b3e053f734b..cdb3aea4ba7 100644--- a/test/metabase/embedding/api/preview_embed_test.clj+++ b/test/metabase/embedding/api/preview_embed_test.clj@@ -367,7 +367,8 @@(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id):type :native:native {:query "SELECT {{num}} AS num"- :template-tags {:num {:name "num"+ :template-tags {:num {:id "X"+ :name "num":display_name "Num":type "number":required truediff --git a/test/metabase/legacy_mbql/normalize_test.cljc b/test/metabase/legacy_mbql/normalize_test.cljcindex 71bd04e07d3..dfd6c3adaf5 100644--- a/test/metabase/legacy_mbql/normalize_test.cljc+++ b/test/metabase/legacy_mbql/normalize_test.cljc@@ -1,4 +1,5 @@(ns metabase.legacy-mbql.normalize-test+ {:clj-kondo/config '{:linters {:deprecated-var {:level :off}}}}(:require#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))[clojure.set :as set]diff --git a/test/metabase/lib/convert_test.cljc b/test/metabase/lib/convert_test.cljcindex d0f9758ef4e..f4b41931274 100644--- a/test/metabase/lib/convert_test.cljc+++ b/test/metabase/lib/convert_test.cljc@@ -3,11 +3,9 @@#?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))[clojure.test :refer [are deftest is testing]][medley.core :as m]- [metabase.legacy-mbql.schema :as mbql.s][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.options :as lib.options]- [metabase.lib.schema :as lib.schema][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -1142,31 +1140,6 @@assoc :name "b"))assoc :name "xixix")))))))-(deftest ^:parallel blank-queries-test- (testing "minimal legacy"- (testing "native queries"- (let [query {:type :native}]- (testing "pass the legacy schema"- (is (mr/validate ::mbql.s/Query query)))- (testing "pass the pMBQL schema after conversion"- (is (nil? (->> query- lib.convert/->pMBQL- (mr/explain ::lib.schema/query)))))- (testing "round trip to pMBQL and back with small changes"- (is (= query- (lib.convert/->legacy-MBQL (lib.convert/->pMBQL query)))))))- (testing "MBQL queries"- (let [query {:type :query}]- (testing "pass the legacy schema"- (is (mr/validate ::mbql.s/Query query)))- (testing "pass the pMBQL schema after conversion"- (is (nil? (->> query- lib.convert/->pMBQL- (mr/explain ::lib.schema/query)))))- (testing "round trip to pMBQL and back with small changes"- (is (= query- (lib.convert/->legacy-MBQL (lib.convert/->pMBQL query)))))))))-(deftest ^:parallel round-trip-expression-literal-test(are [literal] (test-round-trip {:database 1:type :query@@ -1651,3 +1624,13 @@{:stage-number 0, :lib/uuid string?}[:field {:base-type :type/BigInteger, :lib/uuid string?} 49]](lib.convert/->pMBQL [:dimension [:field 49 {:base-type :type/BigInteger}] {:stage-number 0}]))))++(deftest ^:parallel ->legacy-MBQL-idempotence-test+ (let [query {:database 1493+ :type :query+ :query {:aggregation [[:count]]+ :breakout [[:field 76313 {:source-field 76299}]]+ :source-table 11759+ :expressions {"TestColumn" [:+ 1 1]}}}]+ (is (= query+ (lib.convert/->legacy-MBQL query)))))diff --git a/test/metabase/lib/drill_thru/underlying_records_test.cljc b/test/metabase/lib/drill_thru/underlying_records_test.cljcindex b34e16f4d33..60eb00eb120 100644--- a/test/metabase/lib/drill_thru/underlying_records_test.cljc+++ b/test/metabase/lib/drill_thru/underlying_records_test.cljc@@ -16,6 +16,7 @@[metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros][metabase.lib.test-util.metadata-providers.mock :as providers.mock]+ [metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.lib.underlying :as lib.underlying]))#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))@@ -696,3 +697,38 @@[:field {:temporal-unit :year} "CREATED_AT"]"2023-01-01T00:00:00Z"]]}](:stages (lib/drill-thru query drill))))))++(deftest ^:parallel remove-original-binning-test+ (testing "We should remove :lib/original-binning from display info as well"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/count))+ (lib/breakout (-> (meta/field-metadata :orders :total)+ (lib/with-binning {:strategy :num-bins, :num-bins 100})))+ (as-> $query (lib/filter $query (lib/>= (meta/field-metadata :orders :total) 90))))+ cols (lib/returned-columns query)+ count-col (lib.tu.notebook/find-col-with-spec query cols {} {:display-name "Count"})+ total-col (lib.tu.notebook/find-col-with-spec query cols {} {:display-name "Total: 100 bins"})+ context {:column count-col+ :column-ref (lib/ref count-col)+ :value 37+ :row [{:column total-col, :column-ref (lib/ref total-col), :value 90}+ {:column count-col, :column-ref (lib/ref count-col), :value 37}]+ :dimensions [{:column total-col, :column-ref (lib/ref total-col), :value 90}]}+ drill (m/find-first #(= (:type %) :drill-thru/underlying-records)+ (lib/available-drill-thrus query context))]+ (is (some? drill))+ (let [query' (lib/drill-thru query drill)]+ (is (=? [{:source-table (meta/id :orders)+ :filters [[:>= {}+ [:field {} (meta/id :orders :total)]+ 90]+ [:<+ {}+ [:field {} (meta/id :orders :total)]+ 92.0]]}]+ (:stages query')))+ (is (= ["Total is greater than or equal to 90"+ #?(:clj "Total is less than 92.0"+ :cljs "Total is less than 92")]+ (map #(:long-display-name (lib/display-info query' %))+ (lib/filters query'))))))))diff --git a/test/metabase/lib/equality_test.cljc b/test/metabase/lib/equality_test.cljcindex 44e5b594abc..6b982be614d 100644--- a/test/metabase/lib/equality_test.cljc+++ b/test/metabase/lib/equality_test.cljc@@ -320,6 +320,8 @@[:field {} "Cat__NAME"]] ; 7refs))(testing (str "\n" (with-out-str+ ;; usage is ok since we're wrapping it in `with-out-str`+ #_{:clj-kondo/ignore [:discouraged-var]}(pprint/print-table[:name :lib/original-join-alias :lib/original-name :lib/deduplicated-name :lib/source-column-alias :lib/desired-column-alias]cols)))diff --git a/test/metabase/lib/fe_util_test.cljc b/test/metabase/lib/fe_util_test.cljcindex a78b8733407..24710e61972 100644--- a/test/metabase/lib/fe_util_test.cljc+++ b/test/metabase/lib/fe_util_test.cljc@@ -971,7 +971,9 @@:widget-type :text:display-name "foo":dimension [:field {:lib/uuid (str (random-uuid))} 1]}})- (lib/dependent-metadata nil :question)))))+ (lib/dependent-metadata nil :question))))))++(deftest ^:parallel dependent-metadata-test-2(testing "simple query"(are [query] (=? [{:type :database, :id (meta/id)}{:type :schema, :id (meta/id)}@@ -979,7 +981,9 @@{:type :table, :id (meta/id :categories)}](lib/dependent-metadata query nil :question))(lib.tu/venues-query)- (lib/append-stage (lib.tu/venues-query))))+ (lib/append-stage (lib.tu/venues-query)))))++(deftest ^:parallel dependent-metadata-test-3(testing "join query"(are [query] (=? [{:type :database, :id (meta/id)}{:type :schema, :id (meta/id)}@@ -987,7 +991,9 @@{:type :table, :id (meta/id :categories)}](lib/dependent-metadata query nil :question))(lib.tu/query-with-join)- (lib/append-stage (lib.tu/query-with-join))))+ (lib/append-stage (lib.tu/query-with-join)))))++(deftest ^:parallel dependent-metadata-test-4(testing "source card based query"(are [query] (=? [{:type :database, :id (meta/id)}{:type :schema, :id (meta/id)}@@ -995,7 +1001,9 @@{:type :table, :id (meta/id :users)}](lib/dependent-metadata query nil :question))(lib.tu/query-with-source-card)- (lib/append-stage (lib.tu/query-with-source-card))))+ (lib/append-stage (lib.tu/query-with-source-card)))))++(deftest ^:parallel dependent-metadata-test-5(testing "source card based query with result metadata"(are [query] (=? [{:type :database, :id (meta/id)}{:type :schema, :id (meta/id)}@@ -1003,7 +1011,9 @@{:type :table, :id (meta/id :users)}](lib/dependent-metadata query nil :question))(lib.tu/query-with-source-card-with-result-metadata)- (lib/append-stage (lib.tu/query-with-source-card-with-result-metadata))))+ (lib/append-stage (lib.tu/query-with-source-card-with-result-metadata)))))++(deftest ^:parallel dependent-metadata-test-6(testing "model based query"(let [query (assoc (lib.tu/query-with-source-card) :lib/metadata lib.tu/metadata-provider-with-model)](are [query] (=? [{:type :database, :id (meta/id)}@@ -1012,7 +1022,9 @@{:type :table, :id (meta/id :users)}](lib/dependent-metadata query nil :question))query- (lib/append-stage query))))+ (lib/append-stage query)))))++(deftest ^:parallel dependent-metadata-test-7(testing "metric based query"(let [query (assoc (lib.tu/query-with-source-card) :lib/metadata lib.tu/metadata-provider-with-metric)](are [query] (=? [{:type :database, :id (meta/id)}@@ -1023,7 +1035,9 @@{:type :table, :id (meta/id :venues)}](lib/dependent-metadata query nil :question))query- (lib/append-stage query))))+ (lib/append-stage query)))))++(deftest ^:parallel dependent-metadata-test-8(testing "editing a model"(let [metadata-provider lib.tu/metadata-provider-with-modelquery (->> (lib.metadata/card metadata-provider 1)@@ -1037,7 +1051,9 @@{:type :table, :id "card__1"}](lib/dependent-metadata query 1 :model))query- (lib/append-stage query))))+ (lib/append-stage query)))))++(deftest ^:parallel dependent-metadata-test-9(testing "editing a metric"(let [metadata-provider lib.tu/metadata-provider-with-metricquery (->> (lib.metadata/card metadata-provider 1)@@ -1051,7 +1067,9 @@{:type :table, :id "card__1"}](lib/dependent-metadata query 1 :metric))query- (lib/append-stage query))))+ (lib/append-stage query)))))++(deftest ^:parallel dependent-metadata-test-10(testing "Native query snippets should be included in dependent metadata"(let [;; lib/native-query would try to look up the snippets:query {:lib/type :mbql/querydiff --git a/test/metabase/lib/join_test.cljc b/test/metabase/lib/join_test.cljcindex 1a403bc7ab8..2034f9e6ea9 100644--- a/test/metabase/lib/join_test.cljc+++ b/test/metabase/lib/join_test.cljc@@ -9,6 +9,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.options :as lib.options]+ [metabase.lib.schema.util :as lib.schema.util][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -298,8 +299,9 @@:lib/source-column-alias "count"::lib.join/join-alias "checkins_by_user"}](lib.join/join-returned-columns-relative-to-parent-stage query -1 join)))- (is (= (lib.metadata/card metadata-provider 1)- (lib.join/joined-thing query join)))))+ (is (=? (-> (lib.metadata/card metadata-provider 1)+ (update :dataset-query lib.schema.util/remove-lib-uuids))+ (lib.join/joined-thing query join)))))(deftest ^:parallel joins-source-and-desired-aliases-test(let [query (-> (lib.tu/venues-query)diff --git a/test/metabase/lib/js/metadata_test.cljs b/test/metabase/lib/js/metadata_test.cljsindex 25264a098d9..1c6a6015607 100644--- a/test/metabase/lib/js/metadata_test.cljs+++ b/test/metabase/lib/js/metadata_test.cljs@@ -1,6 +1,7 @@(ns metabase.lib.js.metadata-test(:require[clojure.test :refer [deftest is]]+ [metabase.lib.core :as lib][metabase.lib.js.metadata :as lib.js.metadata][metabase.lib.metadata :as lib.metadata]))@@ -60,3 +61,10 @@:id 66:name "ID [internal remap]"}}(lib.metadata/field metadata-provider 33)))))++(deftest ^:parallel do-not-remove-during-normalization-test+ (let [metadata #js {:fields #js {"36" mock-field-metadata-with-external-remap}}+ mp (lib.js.metadata/metadata-provider 1 metadata)+ query {:lib/type :mbql/query, :database 1, :lib/metadata mp, :stages [{:lib/type :mbql.stage/mbql, :source-table 2}]}]+ (is (= query+ (lib/normalize query)))))diff --git a/test/metabase/lib/js_test.cljs b/test/metabase/lib/js_test.cljsindex 3df11b2f0f7..5e19ef5b55a 100644--- a/test/metabase/lib/js_test.cljs+++ b/test/metabase/lib/js_test.cljs@@ -9,12 +9,15 @@[metabase.lib.js :as lib.js][metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options]+ [metabase.lib.schema.util :as lib.schema.util][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.test-runner.assert-exprs.approximately-equal][metabase.test.util.js :as test.js][metabase.util.malli.registry :as mr]))+(comment metabase.test-runner.assert-exprs.approximately-equal/keep-me)+(deftest ^:parallel query=-test(doseq [q1 [nil js/undefined]q2 [nil js/undefined]]@@ -624,3 +627,60 @@:keyword "too":value nil}](is (js= expected (lib.js/display-info->js input))))))++(deftest ^:parallel query-to-js-test+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/count))+ lib/append-stage+ (as-> $query (lib/filter $query (lib/> (first (lib/returned-columns $query)) 100))))+ js-query (lib.js/to-js-query query)+ ag-uuid (lib.options/uuid (first (lib/aggregations query 0)))+ filter-uuid (lib.options/uuid (first (lib/filters query)))+ field-uuid (-> (first (lib/filters query))+ (nth 2)+ lib.options/uuid)]+ (testing "CLJS to JS"+ (is (js= #js {"database" (meta/id)+ "lib/type" "mbql/query"+ "stages" #js [#js {"source-table" (meta/id :orders)+ "lib/type" "mbql.stage/mbql"+ "aggregation" #js [#js ["count" #js {"lib/uuid" ag-uuid}]]}+ #js {"lib/type" "mbql.stage/mbql"+ "filters" #js [#js [">"+ #js {"lib/uuid" filter-uuid}+ #js ["field"+ #js {"effective-type" "type/Integer"+ "lib/uuid" field-uuid+ "base-type" "type/Integer"}+ "count"]+ 100]]}]}+ js-query)))+ (testing "JS to CLJS"+ (testing "MBQL 5"+ (is (= query+ (lib.js/from-js-query (:lib/metadata query) js-query))))+ (testing "MBQL 4 (legacy)"+ (is (=? (lib.schema.util/remove-lib-uuids query)+ (lib.js/from-js-query (:lib/metadata query) (lib.js/legacy-query js-query)))))+ (testing "Cache converted queries"+ (let [q1 (lib.js/from-js-query (:lib/metadata query) js-query)+ q2 (lib.js/from-js-query (:lib/metadata query) js-query)]+ (is (identical? q1 q2)))))))++(deftest ^:parallel from-js-query-preserve-database-id-test+ (is (=? {:lib/type :mbql/query,+ :stages [{:lib/type :mbql.stage/mbql, :source-table 2}]+ :database 1}+ (lib.js/from-js-query+ (lib.js/metadataProvider 1 #js {})+ #js {:database 1, :type "query", :query #js {:source-table 2}}))))++(deftest ^:parallel query-equals-should-work-on-js-and-cljs-mbql-4-and-5+ (let [queries {"Cljs MBQL 4" {:database 1, :type :query, :query {:source-table 2}}+ "JS MBQL 4" #js {:database 1, :type :query, :query #js {:source-table 2}}+ "Cljs MBQL 5" {:database 1, :lib/type :mbql/query, :stages [{:lib/type :mbql.stage/mbql, :source-table 2}]}+ "JS MBQL 5" #js {:database 1, "lib/type" :mbql/query, :stages #js [#js {"lib/type" :mbql.stage/mbql, :source-table 2}]}}]+ (doseq [[q1-type q1] queries+ [q2-type q2] queries]+ (testing (str "q1 = " q1-type ", q2 = " q2-type)+ (lib.js/query= q1 q2)))))diff --git a/test/metabase/lib/metric_test.cljc b/test/metabase/lib/metric_test.cljcindex c6f4d40340f..190bdd8c770 100644--- a/test/metabase/lib/metric_test.cljc+++ b/test/metabase/lib/metric_test.cljc@@ -8,7 +8,8 @@[metabase.lib.metric :as lib.metric][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]- [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))+ [metabase.lib.test-util.metadata-providers.mock :as providers.mock]+ [metabase.util.malli :as mu]))#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))@@ -128,6 +129,27 @@metric-clausemetric-metadata))+(deftest ^:parallel type-of-normalization-test+ (testing "Metric type-of should normalize the metric :dataset-query as needed"+ (let [metric {:description "A metric"+ :archived false+ :lib/type :metadata/metric+ :database-id (meta/id)+ :table-id (meta/id :orders)+ :name "Count"+ :type :metric+ :source-card-id nil+ :id 80+ :dataset-query {:lib/type "mbql/query"+ :database (meta/id)+ :stages [{:source-table (meta/id :orders)+ :lib/type "mbql.stage/mbql"+ :aggregation [["count" {:lib/uuid "bd23f4c1-c973-4a04-8dc2-342236fe2b0f"}]]}]}}]+ ;; disable Malli enforcement to make sure this works in the FE+ (mu/disable-enforcement+ (is (= :type/Integer+ (lib/type-of (lib/query meta/metadata-provider (meta/table-metadata :orders)) metric)))))))+(deftest ^:parallel unknown-type-of-test(is (= :type/*(lib/type-of query-with-metric [:metric {} 1]))))diff --git a/test/metabase/lib/native_test.cljc b/test/metabase/lib/native_test.cljcindex 461779379cf..84f45cc177f 100644--- a/test/metabase/lib/native_test.cljc+++ b/test/metabase/lib/native_test.cljc@@ -239,6 +239,22 @@(-> (lib.tu/venues-query)(lib/with-template-tags {"myid" (assoc (get original-tags "myid") :display-name "My ID")}))))))+(deftest ^:parallel with-template-tags-update-map-order-test+ ;; yes, I know template tags are sorted as a map, but for small maps we should preserve the order passed in by the+ ;; FE. See+ ;; https://metaboat.slack.com/archives/C0645JP1W81/p1759974826834279?thread_ts=1759289751.539169&cid=C0645JP1W81+ (testing "it should be possible to reorder template tags with with-template-tags"+ (let [query (lib/native-query meta/metadata-provider "{{x}} {{y}} {{z}}")+ original-tags (lib/template-tags query)]+ (is (=? {"x" {}, "y" {}, "z" {}}+ original-tags))+ (is (= ["x" "y" "z"]+ (keys original-tags)))+ (let [updated-tags {"y" (get original-tags "y"), "x" (get original-tags "x")}+ query' (lib/with-template-tags query updated-tags)]+ (is (= ["y" "x" "z"]+ (keys (lib/template-tags query'))))))))+(defn ^:private metadata-provider-requiring-collection [](meta/updated-metadata-provider update :features conj :native-requires-specified-collection))diff --git a/test/metabase/lib/normalize_test.cljc b/test/metabase/lib/normalize_test.cljcindex 462559553fd..a3a29ddf9c5 100644--- a/test/metabase/lib/normalize_test.cljc+++ b/test/metabase/lib/normalize_test.cljc@@ -103,3 +103,112 @@"source-table" 1"aggregation" [["count" {}]]"filters" [["=" {} ["field" {} 1] 4]]}]})))))++(deftest ^:parallel normalize-from-json-test+ (let [query '{:lib/type "mbql/query"+ :lib/metadata nil+ :database 1493+ :stages ({:joins ({:lib/type "mbql/join"+ :alias "CategoriesStats"+ :fields "all"+ :conditions+ (("="+ #:lib{:uuid "e27ffec5-b743-47ff-a715-91f6d68f0ec1"}+ ("field"+ {:base-type "type/Integer", :lib/uuid "3edfdf40-092e-419e-8637-8df75d11c10e", :effective-type "type/Integer"}+ 76329)+ ("field"+ {:base-type "type/Integer"+ :join-alias "CategoriesStats"+ :lib/uuid "ff3dc321-27a7-42d8-a7fc-518160250984"+ :effective-type "type/Integer"}+ 76329)))+ :stages ({:lib/type "mbql.stage/mbql", :source-card 13281})+ :lib/options #:lib{:uuid "517699ac-dede-4f6b-bc53-272def3eea8b"}})+ :lib/type "mbql.stage/mbql"+ :source-table 11763+ :fields (("field"+ {:base-type "type/BigInteger"+ :lib/uuid "9aeaa3ff-0c78-48b5-803c-feb828e92c42"+ :effective-type "type/BigInteger"}+ 76331)+ ("field"+ {:base-type "type/Text", :lib/uuid "69d0b0d9-31ea-486a-a2b1-d5cfc052e1e3", :effective-type "type/Text"}+ 76332)+ ("field"+ {:base-type "type/Integer", :lib/uuid "8680f1f3-7d32-466d-a19d-6dd0a13570dc", :effective-type "type/Integer"}+ 76329)+ ("field"+ {:base-type "type/Float", :lib/uuid "1ce9c56b-1a41-4a55-a760-1ead55f9f54f", :effective-type "type/Float"}+ 76333)+ ("field"+ {:base-type "type/Float", :lib/uuid "41cd94d5-ee1a-4410-910e-979ba593ad32", :effective-type "type/Float"}+ 76334)+ ("field"+ {:base-type "type/Integer", :lib/uuid "9e3d4356-82a4-4145-8004-23a1548c9cba", :effective-type "type/Integer"}+ 76330)+ ("expression" #:lib{:uuid "3ed4feb6-d9b0-4acd-b331-b34607b68422"} "RelativePrice"))+ :expressions (("/"+ #:lib{:uuid "0123167c-80e2-4175-b270-71a3fba58c51", :expression-name "RelativePrice"}+ ("field"+ {:base-type "type/Integer", :lib/uuid "349d9db5-340e-44e2-a7f3-d513d3b9b0cd", :effective-type "type/Integer"}+ 76330)+ ("field"+ {:join-alias "CategoriesStats", :base-type "type/Float", :lib/uuid "a7d60dc3-529a-434c-9bb7-dde67c2af90e"}+ "AvgPrice")))+ :limit 3})}]+ (is (= {:database 1493+ :stages [{:expressions+ [[:/ {:lib/expression-name "RelativePrice", :lib/uuid "0123167c-80e2-4175-b270-71a3fba58c51"}+ [:field+ {:base-type :type/Integer, :effective-type :type/Integer, :lib/uuid "349d9db5-340e-44e2-a7f3-d513d3b9b0cd"}+ 76330]+ [:field+ {:base-type :type/Float, :join-alias "CategoriesStats", :lib/uuid "a7d60dc3-529a-434c-9bb7-dde67c2af90e"}+ "AvgPrice"]]]+ :fields [[:field+ {:base-type :type/BigInteger+ :lib/uuid "9aeaa3ff-0c78-48b5-803c-feb828e92c42"+ :effective-type :type/BigInteger}+ 76331]+ [:field+ {:base-type :type/Text, :lib/uuid "69d0b0d9-31ea-486a-a2b1-d5cfc052e1e3", :effective-type :type/Text}+ 76332]+ [:field+ {:base-type :type/Integer+ :lib/uuid "8680f1f3-7d32-466d-a19d-6dd0a13570dc"+ :effective-type :type/Integer}+ 76329]+ [:field+ {:base-type :type/Float, :lib/uuid "1ce9c56b-1a41-4a55-a760-1ead55f9f54f", :effective-type :type/Float}+ 76333]+ [:field+ {:base-type :type/Float, :lib/uuid "41cd94d5-ee1a-4410-910e-979ba593ad32", :effective-type :type/Float}+ 76334]+ [:field+ {:base-type :type/Integer+ :lib/uuid "9e3d4356-82a4-4145-8004-23a1548c9cba"+ :effective-type :type/Integer}+ 76330]+ [:expression {:lib/uuid "3ed4feb6-d9b0-4acd-b331-b34607b68422"} "RelativePrice"]]+ :joins [{:alias "CategoriesStats"+ :conditions [[:=+ {:lib/uuid "e27ffec5-b743-47ff-a715-91f6d68f0ec1"}+ [:field+ {:base-type :type/Integer, :effective-type :type/Integer, :lib/uuid "3edfdf40-092e-419e-8637-8df75d11c10e"}+ 76329]+ [:field+ {:base-type :type/Integer+ :effective-type :type/Integer+ :join-alias "CategoriesStats"+ :lib/uuid "ff3dc321-27a7-42d8-a7fc-518160250984"}+ 76329]]]+ :fields :all+ :stages [{:source-card 13281, :lib/type :mbql.stage/mbql}]+ :lib/options {:lib/uuid "517699ac-dede-4f6b-bc53-272def3eea8b"}+ :lib/type :mbql/join}]+ :limit 3+ :source-table 11763+ :lib/type :mbql.stage/mbql}]+ :lib/type :mbql/query}+ (lib/normalize query)))))diff --git a/test/metabase/lib/parameters/parse_test.cljc b/test/metabase/lib/parameters/parse_test.cljcnew file mode 100644index 00000000000..80112688861--- /dev/null+++ b/test/metabase/lib/parameters/parse_test.cljc@@ -0,0 +1,155 @@+(ns metabase.lib.parameters.parse-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.lib.parameters.parse :as params.parse]+ [metabase.lib.parameters.parse.types :as lib.parms.parse.types]+ [metabase.lib.parse :as lib.parse]+ [metabase.lib.util :as lib.util]))++(defn- param [field-name] (lib.parms.parse.types/param {:k field-name}))+(defn- optional [& args] (lib.parms.parse.types/optional {:args (vec args)}))++(defn- normalize-tokens+ [tokens]+ (for [token tokens]+ (or (:token token) token)))++(deftest ^:parallel tokenize-test+ (doseq [[query expected]+ {"{{num_toucans}}"+ [:param-begin "num_toucans" :param-end]++ "[[AND num_toucans > {{num_toucans}}]]"+ [:optional-begin "AND num_toucans > " :param-begin "num_toucans" :param-end :optional-end]++ "}}{{]][["+ [:param-end :param-begin :optional-end :optional-begin]++ "SELECT * FROM toucanneries WHERE TRUE [[AND num_toucans > {{num_toucans}}]]"+ ["SELECT * FROM toucanneries WHERE TRUE " :optional-begin "AND num_toucans > " :param-begin "num_toucans" :param-end :optional-end]++ "SELECT * FROM toucanneries WHERE TRUE [[AND num_toucans > {{num_toucans}}]] [[AND total_birds > {{total_birds}}]]"+ ["SELECT * FROM toucanneries WHERE TRUE " :optional-begin "AND num_toucans > " :param-begin "num_toucans" :param-end :optional-end+ " " :optional-begin "AND total_birds > " :param-begin "total_birds" :param-end :optional-end]++ "SELECT * FROM -- {{foo}}\n" ["SELECT * FROM " :line-comment-begin " " :param-begin "foo" :param-end :newline]++ "/*SELECT {{foo}}*/" [:block-comment-begin "SELECT " :param-begin "foo" :param-end :block-comment-end]}]+ (is (= expected+ (normalize-tokens (#'lib.parse/tokenize query true)))+ (lib.util/format "%s should get tokenized to %s" (pr-str query) (pr-str expected)))))++(deftest ^:parallel tokenize-no-sql-comments-test+ (doseq [[query expected]+ {"-- {{num_toucans}}"+ ["-- " :param-begin "num_toucans" :param-end]++ "/* [[AND num_toucans > {{num_toucans}} --]] */"+ ["/* " :optional-begin "AND num_toucans > " :param-begin "num_toucans" :param-end " --" :optional-end " */"]}]+ (is (= expected+ (normalize-tokens (#'lib.parse/tokenize query false)))+ (lib.util/format "%s should get tokenized to %s" (pr-str query) (pr-str expected)))))++(deftest ^:parallel parse-test+ (doseq [[group s->expected]+ {"queries with one param"+ {"select * from foo where bar=1" ["select * from foo where bar=1"]+ "select * from foo where bar={{baz}}" ["select * from foo where bar=" (param "baz")]+ "select * from foo [[where bar = {{baz}} ]]" ["select * from foo " (optional "where bar = " (param "baz") " ")]}++ "multiple params"+ {"SELECT * FROM bird_facts WHERE toucans_are_cool = {{toucans_are_cool}} AND bird_type = {{bird_type}}"+ ["SELECT * FROM bird_facts WHERE toucans_are_cool = " (param "toucans_are_cool")+ " AND bird_type = " (param "bird_type")]}++ "Multiple optional clauses"+ {(str "select * from foo where bar1 = {{baz}} "+ "[[and bar2 = {{baz}}]] "+ "[[and bar3 = {{baz}}]] "+ "[[and bar4 = {{baz}}]]")+ ["select * from foo where bar1 = " (param "baz") " "+ (optional "and bar2 = " (param "baz")) " "+ (optional "and bar3 = " (param "baz")) " "+ (optional "and bar4 = " (param "baz"))]++ "SELECT * FROM toucanneries WHERE TRUE [[AND num_toucans > {{num_toucans}}]] [[AND total_birds > {{total_birds}}]]"+ ["SELECT * FROM toucanneries WHERE TRUE "+ (optional "AND num_toucans > " (param "num_toucans"))+ " "+ (optional "AND total_birds > " (param "total_birds"))]++ "select * from foobars [[ where foobars.id in (string_to_array({{foobar_id}}, ',')::integer[]) ]]"+ ["select * from foobars "+ (optional " where foobars.id in (string_to_array(" (param "foobar_id") ", ',')::integer[]) ")]}++ "single square brackets shouldn't get parsed"+ (let [query (str "SELECT [test_data.checkins.venue_id] AS [venue_id], "+ " [test_data.checkins.user_id] AS [user_id], "+ " [test_data.checkins.id] AS [checkins_id] "+ "FROM [test_data.checkins] "+ "LIMIT 2")]+ {query [query]})++ "Valid syntax in PG -- shouldn't get parsed"+ (let [query "SELECT array_dims(1 || '[0:1]={2,3}'::int[])"]+ {query [query]})++ "Queries with newlines (#11526)"+ {"SELECT count(*)\nFROM products\nWHERE category = {{category}}"+ ["SELECT count(*)\nFROM products\nWHERE category = " (param "category")]}++ "Queries with params in SQL comments (#7742)"+ {"SELECT -- {{foo}}" ["SELECT -- {{foo}}"]+ "[[{{this}} -- and]] that" [(optional (param "this") " -- and") " that"]+ "SELECT /* \n --{{foo}} */ {{bar}}" ["SELECT /* \n --{{foo}} */ " (param "bar")]}++ "JSON queries that contain non-param fragments like '}}'"+ {"{x: {y: \"{{param}}\"}}" ["{x: {y: \"" (param "param") "\"}}"]+ "{$match: {{{date}}, field: 1}}}" ["{$match: {" (param "date") ", field: 1}}}"]}++ "Queries that contain non-param fragments like '}}'"+ {"select ']]' from t [[where x = {{foo}}]]" ["select ']]' from t " (optional "where x = " (param "foo"))]+ "select '}}' from t [[where x = {{foo}}]]" ["select '}}' from t " (optional "where x = " (param "foo"))]}}]+ (testing group+ (doseq [[s expected] s->expected]+ (is (= expected+ (normalize-tokens (params.parse/parse s)))+ (lib.util/format "%s should get parsed to %s" (pr-str s) (pr-str expected))))))++ (testing "Testing that invalid/unterminated template lib.parms.parse.types/clauses throw an exception"+ (doseq [invalid ["select * from foo [[where bar = {{baz}} "+ "select * from foo [[where bar = {{baz]]"+ "select * from foo {{bar}} {{baz"+ "select * from foo [[clause 1 {{bar}}]] [[clause 2"+ "select * from foo where bar = {{baz]]"+ "select * from foo [[where bar = {{baz}}}}"]]+ (is (thrown? #?(:clj clojure.lang.ExceptionInfo :cljs :default)+ (params.parse/parse invalid))+ (lib.util/format "Parsing %s should throw an exception" (pr-str invalid))))))++(deftest ^:parallel disable-comment-handling-test+ (testing "SQL comments are ignored when handle-sql-comments = false, e.g. in Mongo driver queries"+ (doseq [[query result] [["{{{foo}}: -- {{bar}}}"+ ["{" (param "foo") ": -- " (param "bar") "}"]]++ ["{{{foo}}: \"/* {{bar}} */\"}"+ ["{" (param "foo") ": \"/* " (param "bar") " */\"}"]]]]+ (is (= result (normalize-tokens (params.parse/parse query false)))))))++(deftest ^:parallel tokens-in-strings-test+ (testing "skip malformed parameter tokens in strings"+ (is (= ["'{{'"] (params.parse/parse "'{{'")))+ (is (= ["'{{"] (params.parse/parse "'{{")))+ (is (= ["'{{}}'"] (params.parse/parse "'{{}}'")))+ (is (= ["'[['"] (params.parse/parse "'[['")))+ (is (= ["'[[]]'"] (params.parse/parse "'[[]]'")))+ (is (= ["'[[{{]]'"] (params.parse/parse "'[[{{]]'"))))+ (testing "Parse well-formed tokens in strings"+ (is (= ["'" (param "x") "'"] (params.parse/parse "'{{x}}'")))+ (is (= ["'" (param "snippet: 'test'") "'"] (params.parse/parse "'{{snippet: 'test'}}'")))+ (is (= ["'{{" (param "x") "'"] (params.parse/parse "'{{{{x}}'")))+ (is (= ["'[[" (param "x") "'"] (params.parse/parse "'[[{{x}}'")))+ (is (= ["'" (optional (param "x")) "'"] (params.parse/parse "'[[{{x}}]]'"))))+ (testing "skip comment start in strings"+ (is (= ["concat('--'," (param "x") ")"] (params.parse/parse "concat('--',{{x}})")))+ (is (= ["concat('/*'," (param "x") ")"] (params.parse/parse "concat('/*',{{x}})")))))diff --git a/test/metabase/lib/query_test.cljc b/test/metabase/lib/query_test.cljcindex f41f5af0244..df7ddb08b73 100644--- a/test/metabase/lib/query_test.cljc+++ b/test/metabase/lib/query_test.cljc@@ -615,3 +615,11 @@(is (lib.metadata.protocols/cached-metadata-provider? mp))(is (lib.metadata.protocols/cached-metadata-provider-with-cache? mp))(is (identical? mp (:lib/metadata (#'lib.query/ensure-cached-metadata-provider {:lib/metadata mp})))))))++(deftest ^:parallel preserve-database-id-with-invalid-metadata-provider-test+ (mu/disable-enforcement+ (is (=? {:database 1+ :stages [{:source-table 2, :lib/type :mbql.stage/mbql}]+ :lib/type :mbql/query}+ (lib.query/query (lib.tu/mock-metadata-provider {})+ {"database" 1, "type" "query", "query" {"source-table" 2}})))))diff --git a/test/metabase/lib/schema/common_test.cljc b/test/metabase/lib/schema/common_test.cljcindex bad7151ac66..3ee3fb416d0 100644--- a/test/metabase/lib/schema/common_test.cljc+++ b/test/metabase/lib/schema/common_test.cljc@@ -26,3 +26,22 @@(mu/disable-enforcement(is (= :type/CreationTime(lib/normalize ::lib.schema.common/base-type "type/creationtime")))))))++(deftest ^:parallel url-encoded-string-regex-test+ (are [s] (re-matches lib.schema.common/url-encoded-string-regex s)+ "a"+ "Hello%20World"+ "user%40example.com"+ "test_string-123.txt"+ "%3Cscript%3E")+ (are [s] (not (re-matches lib.schema.common/url-encoded-string-regex s))+ "a/b"+ "Hello World"+ "<script>"))++(deftest ^:parallel unfussy-sorted-map-test+ (is (nil? (:lib/type (lib.schema.common/unfussy-sorted-map "a" 1))))+ (is (nil? (get (lib.schema.common/unfussy-sorted-map :a 1) "lib/type")))+ (is (= {"a" 1 :b 2}+ (lib.schema.common/unfussy-sorted-map "a" 1 :b 2))))diff --git a/test/metabase/lib/schema/join_test.cljc b/test/metabase/lib/schema/join_test.cljcindex 3bfd512d6b9..8532b7c7b78 100644--- a/test/metabase/lib/schema/join_test.cljc+++ b/test/metabase/lib/schema/join_test.cljc@@ -18,9 +18,10 @@(mu.humanize/humanize (mr/explain ::lib.schema.join/join {:stages []}))));; not sure why these errors are repeated.(is (=? {:lib/type "missing required key"- :stages [{:joins [{:stages [[{:lib/type "missing required key"}- "Invalid stage :lib/type: expected :mbql.stage/native or :mbql.stage/mbql"]]- :conditions "should have at least 1 elements"}]}]}+ :stages [[{:joins [{:stages [[{:lib/type "missing required key"}+ "Invalid stage :lib/type: expected :mbql.stage/native or :mbql.stage/mbql"]]+ :conditions "should have at least 1 elements"}]}+ "Initial MBQL stage must have either :source-table or :source-card (but not both)"]]}(mu.humanize/humanize (mr/explain ::lib.schema/query {:stages [{:lib/type :mbql.stage/mbql:joins [{:lib/type :mbql/join:stages [{}]diff --git a/test/metabase/lib/schema/parameter_test.cljc b/test/metabase/lib/schema/parameter_test.cljcindex d47746994f0..5464e723620 100644--- a/test/metabase/lib/schema/parameter_test.cljc+++ b/test/metabase/lib/schema/parameter_test.cljc@@ -61,3 +61,26 @@(lib/normalize ::lib.schema.parameter/dimension clause))[:dimension ["field-id" 76331]] [:dimension [:field 76331 nil]]["dimension" ["fk->" 23 30]] [:dimension [:field 30 {:source-field 23}]]))++(deftest ^:parallel normalize-dimension-with-raw-field-id-test+ (is (= [:dimension [:field 100 nil]]+ (lib/normalize ::lib.schema.parameter/dimension [:dimension 100]))))++(deftest ^:parallel normalize-target-test+ (is (= [:variable [:template-tag "state"]]+ (lib/normalize ::lib.schema.parameter/target [:variable ["template-tag" "state"]])+ (lib/normalize ::lib.schema.parameter/target [:variable ["template-tag" :state]]))))++(deftest ^:parallel fix-unwrapped-template-tag-target-test+ (is (= [:variable [:template-tag "x"]]+ (lib/normalize ::lib.schema.parameter/target ["template-tag" :x])+ (lib/normalize ::lib.schema.parameter/target [:template-tag "x"]))))++(deftest ^:parallel decode-busted-param-type-test+ (is (= :text+ (lib/normalize ::lib.schema.parameter/type "type/Text")+ (lib/normalize ::lib.schema.parameter/type :type/Text))))++(deftest ^:parallel default-to-type-text-test+ (is (= {:id "x", :type :text}+ (lib/normalize ::lib.schema.parameter/parameter {:id "x"}))))diff --git a/test/metabase/lib/schema/ref_test.cljc b/test/metabase/lib/schema/ref_test.cljcindex 7bf11caaf8d..050ab0e35b1 100644--- a/test/metabase/lib/schema/ref_test.cljc+++ b/test/metabase/lib/schema/ref_test.cljc@@ -2,6 +2,7 @@(:require[clojure.test :refer [are deftest is testing]][malli.error :as me]+ [metabase.lib.core :as lib][metabase.lib.schema.expression :as expression][metabase.lib.schema.ref :as lib.schema.ref][metabase.util.malli.registry :as mr]))@@ -50,3 +51,17 @@(are [schema] (not (me/humanize (mr/explain schema field-ref))):mbql.clause/field::lib.schema.ref/ref)))))++(deftest ^:parallel normalize-original-binning-test+ (is (= [:field+ {:base-type :type/Float+ :effective-type :type/Float+ :lib/original-binning {:num-bins 50, :strategy :num-bins}+ :lib/uuid "d01f4c83-0fe5-4329-80f3-2bbea1f27c3b"}+ "TOTAL_2"]+ (lib/normalize ["field"+ {"base-type" "type/Float"+ "lib/uuid" "d01f4c83-0fe5-4329-80f3-2bbea1f27c3b"+ "effective-type" "type/Float"+ "lib/original-binning" {"strategy" "num-bins", "num-bins" 50}}+ "TOTAL_2"]))))diff --git a/test/metabase/lib/schema/template_tag_test.cljc b/test/metabase/lib/schema/template_tag_test.cljcindex caa13d42f21..1a1d3af2817 100644--- a/test/metabase/lib/schema/template_tag_test.cljc+++ b/test/metabase/lib/schema/template_tag_test.cljc@@ -1,9 +1,12 @@(ns metabase.lib.schema.template-tag-test(:require- [clojure.test :refer [deftest is testing]]+ #?@(:cljs ([metabase.test-runner.assert-exprs.approximately-equal]))+ [clojure.test :refer [are deftest is testing]][metabase.lib.core :as lib][metabase.lib.schema.template-tag :as lib.schema.template-tag]))+#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))+(deftest ^:parallel normalize-template-tag-remove-dimension-test(testing "Normalization should be able to remove :dimension from template tag types where it is disallowed"(is (= {:name "number_comma"@@ -15,3 +18,45 @@:display-name "Number Comma":type "number":dimension ["field" {"lib/uuid" "eef18794-7d5f-409c-8017-00d306f9aec3"} 1]})))))++(deftest ^:parallel normalize-legacy-ref-test+ (are [x] (=? {:dimension [:field {:lib/uuid string?} 100]}+ (lib/normalize ::lib.schema.template-tag/field-filter {:type :dimension, :dimension x}))+ [:field 100]+ [:field-id 100]+ [:field 100 nil]))++(deftest ^:parallel fix-names-test+ (testing "names in the map values need to match keys in the map"+ (is (=? {"time-unit" {:name "time-unit"+ :display-name "id"+ :type :temporal-unit+ :dimension [:field {:lib/uuid string?} 1]}}+ (lib/normalize ::lib.schema.template-tag/template-tag-map+ {"time-unit" {:name "id"+ :display-name "id"+ :type :temporal-unit+ :dimension [:field 1]}})))))++(deftest ^:parallel normalize-template-tag-options-map-test+ (is (= {:default nil+ :id "f7672b4d-1e84-1fa8-bf02-b5e584cd4535"+ :name "state"+ :display-name "State"+ :type :dimension+ :options {:case-sensitive false}+ :dimension [:field+ {:base-type :type/Text, :lib/uuid "15f3559e-5c7a-4684-9a7a-d906da2eaf61", :effective-type :type/Text}+ 1]+ :widget-type :string/contains}+ (lib/normalize ::lib.schema.template-tag/template-tag+ {:default nil+ :id "f7672b4d-1e84-1fa8-bf02-b5e584cd4535"+ :name "state"+ :display-name "State"+ :type :dimension+ :options {"case-sensitive" false}+ :dimension [:field+ {:base-type :type/Text, :lib/uuid "15f3559e-5c7a-4684-9a7a-d906da2eaf61", :effective-type :type/Text}+ 1]+ :widget-type :string/contains}))))diff --git a/test/metabase/lib/schema_test.cljc b/test/metabase/lib/schema_test.cljcindex 00a6df8ce14..e4ecbfb5a4a 100644--- a/test/metabase/lib/schema_test.cljc+++ b/test/metabase/lib/schema_test.cljc@@ -343,3 +343,22 @@(deftest ^:parallel normalize-remove-disallowed-keys-test(is (= {:source-table 1, :lib/type :mbql.stage/mbql}(lib/normalize ::lib.schema/stage {:source-table 1, :type "query"}))))++(deftest ^:parallel remove-empty-stage-metadata-test+ (is (= {:lib/type :mbql/query+ :database 1493+ :stages [{:template-tags {"x" {:id "6c3d5730-6f9b-4bd6-ae25-3496e8b95011", :type :text, :name "x", :display-name "X"}}+ :lib/type :mbql.stage/native+ :native "update users set name = 'foo' where id = {{x}}"}]}+ (lib/normalize+ '{:lib/type "mbql/query"+ :database 1493+ :stages ({:template-tags {:x {:id "6c3d5730-6f9b-4bd6-ae25-3496e8b95011", :type "text", :name "x", :display-name "X"}}+ :lib/type "mbql.stage/native"+ :lib/stage-metadata nil+ :native "update users set name = 'foo' where id = {{x}}"})}))))++(deftest ^:parallel first-mbql-stage-cannot-be-empty-test+ (is (= {:stages [["Initial MBQL stage must have either :source-table or :source-card (but not both)"]]}+ (me/humanize (mr/explain ::lib.schema/query+ {:lib/type :mbql/query, :database 2378, :stages [{:lib/type :mbql.stage/mbql}]})))))diff --git a/test/metabase/lib_be/hash_test.clj b/test/metabase/lib_be/hash_test.cljnew file mode 100644index 00000000000..fc9a80aff28--- /dev/null+++ b/test/metabase/lib_be/hash_test.clj@@ -0,0 +1,222 @@+(ns metabase.lib-be.hash-test+ (:require+ [buddy.core.codecs :as codecs]+ [clojure.test :refer :all]+ [metabase.lib-be.hash :as lib-be.hash]+ [metabase.lib.test-metadata :as meta]+ [metabase.test :as mt]+ [metabase.util.malli :as mu]))++(defn- query-hash-hex [query]+ (codecs/bytes->hex+ (mu/disable-enforcement+ (lib-be.hash/query-hash (merge {:database Integer/MAX_VALUE}+ (cond+ (:query query) {:type :query}+ (:native query) {:type :native}+ :else {:lib/type :mbql/query})+ query)))))++(deftest ^:parallel query-hash-test+ (testing "lib-be.hash/query-hash"+ (testing "should always hash something the same way, every time"+ (is (= "171fc4cb48d83f4a32e0851ec301426a14c7554ea1b0c81976c7f7b55399c572"+ (query-hash-hex {:query {:source-table 1}})))+ (is (= (query-hash-hex {:query {:source-table 2}})+ (query-hash-hex {:query {:source-table 2}})))+ (let [q (mt/mbql-query products+ {:aggregation [[:count]]+ :breakout [$category]+ :order-by [[:asc [:aggregation 0]]]})]+ (is (= (query-hash-hex q)+ (query-hash-hex q)))))))++(deftest ^:parallel query-hash-test-1b+ (testing "lib-be.hash/query-hash"+ (testing "should handle parameter values that mix regular numbers with bigintegers stored as strings"+ (let [q1 (-> (mt/mbql-query orders)+ (assoc :parameters [{:type :number, :name "p1", :value [1 "9223372036854775808"]}]))+ q2 (-> (mt/mbql-query orders)+ (assoc :parameters [{:type :number, :name "p1", :value ["9223372036854775808" 1]}]))]+ (is (some? (query-hash-hex q1)))+ (is (= (query-hash-hex q1) (query-hash-hex q2)))))))++(deftest ^:parallel ignore-lib-uuids-test+ (letfn [(query []+ {:lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 1+ :filters [[:=+ {:lib/uuid (str (random-uuid))}+ 1+ 2]]}]})]+ (is (= "1494a478589d855b78e4eaf6bfc9d19f080682f89d0eb110c14af6a740436f2e"+ (query-hash-hex (query))+ (query-hash-hex (query))))))++(deftest ^:parallel query-hash-test-2+ (testing "lib-be.hash/query-hash"+ (testing "different queries should produce different hashes"+ (are [x y] (not= (query-hash-hex x)+ (query-hash-hex y))+ {:lib/type :mbql/query, :stages [{:stage-type :abc}]}+ {:lib/type :mbql/query, :stages [{:stage-type :def}]}++ {:lib/type :mbql/query, :database 1}+ {:lib/type :mbql/query, :database 2}++ {:lib/type :mbql/query, :stages [{:lib/type :mbql.stage/mbql}]}+ {:lib/type :mbql/query, :stages [{:lib/type :mbql.stage/native}]}++ {:lib/type :mbql/query, :parameters [{:value 1}]}+ {:lib/type :mbql/query, :parameters [{:value 2}]}++ {:lib/type :mbql/query, :constraints {:max-rows 1000}}+ {:lib/type :mbql/query, :constraints nil}++ (mt/mbql-query products+ {:aggregation [[:count] [:cum-count]]+ :breakout [$category]+ :order-by [[:asc [:aggregation 0]]]})+ (mt/mbql-query products+ {:aggregation [[:count] [:cum-count]]+ :breakout [$category]+ :order-by [[:asc [:aggregation 1]]]})++ (mt/mbql-query products+ {:aggregation [[:count] [:cum-count]]+ :breakout [$category]+ :order-by [[:asc $created_at]]})+ (mt/mbql-query products+ {:aggregation [[:count] [:cum-count]]+ :breakout [$category]+ :order-by [[:asc $rating]]})))))++(deftest ^:parallel query-hash-test-3+ (testing "lib-be.hash/query-hash"+ (testing "keys that are irrelevant to the query should be ignored"+ (is (= (query-hash-hex {:query {:source-table 1}, :random :def})+ (query-hash-hex {:query {:source-table 1}, :random :xyz}))))))++(deftest ^:parallel query-hash-test-4+ (testing "lib-be.hash/query-hash"+ (testing "empty `:parameters` lists should not affect the hash"+ (is (= (query-hash-hex {:query {:source-table 1}})+ (query-hash-hex {:query {:source-table 1}, :parameters []})+ (query-hash-hex {:query {:source-table 1}, :parameters nil}))))))++(deftest ^:parallel query-hash-test-5+ (testing "lib-be.hash/query-hash"+ (testing "empty `:parameters` lists should not affect the hash"+ (testing "...but non-empty ones should"+ (is (not (= (query-hash-hex {:query {:source-table 1}})+ (query-hash-hex {:query {:source-table 1}, :parameters [{:id "ABC"}]}))))))))++(deftest ^:parallel query-hash-test-6+ (testing "lib-be.hash/query-hash"+ (testing "empty `:parameters` lists should not affect the hash"+ (testing (str "the presence of a `nil` value for `:constraints` should produce the same hash as not including "+ "the key at all")+ (is (= (query-hash-hex {:query {:source-table 1}})+ (query-hash-hex {:query {:source-table 1}, :constraints nil})+ (query-hash-hex {:query {:source-table 1}, :constraints {}})))))))++(deftest ^:parallel query-hash-test-7+ (testing "lib-be.hash/query-hash"+ (testing "empty `:parameters` lists should not affect the hash"+ (testing (str "the presence of a `nil` value for `:constraints` should produce the same hash as not including "+ "the key at all")+ (is (= (query-hash-hex {:query {:source-table 1}})+ (query-hash-hex {:query {:source-table 1}, :constraints nil})+ (query-hash-hex {:query {:source-table 1}, :constraints {}})))))))++(deftest ^:parallel query-hash-test-8+ (testing "lib-be.hash/query-hash"+ (testing "make sure two different native queries have different hashes!"+ (is (not= (query-hash-hex {:database (mt/id)+ :type :native+ :native {:query "SELECT pg_sleep(15), 1 AS one"}})+ (query-hash-hex {:database (mt/id)+ :type :native+ :native {:query "SELECT pg_sleep(15), 2 AS two"}}))))))++(deftest ^:parallel key-order-should-not-affect-query-hash-test+ (is (= "915d52e11063543602921c37e792e60ec5667846a20f517a1080ad64600a92e6"+ (query-hash-hex {:parameters [{:value 1, :name "parameter"}]})+ (query-hash-hex {:parameters [{:name "parameter", :value 1}]}))))++(deftest ^:parallel parameter-order-should-not-affect-query-hash-test+ (is (= "6e569604fa10aed0cb3b4625991e9f07ba264462ea72fe6251bb2060709af9e0"+ (query-hash-hex {:parameters [{:name "parameter", :value ["a" "b"]}]})+ (query-hash-hex {:parameters [{:name "parameter", :value ["b" "a"]}]})))+ (is (= "5cf17924f3e572edbad58dfd8a6f78953d25d234ed5bc23d98d922b48e9bfa61"+ (query-hash-hex {:parameters [{:name "parameter", :value [1 2]}]})+ (query-hash-hex {:parameters [{:name "parameter", :value [2 1]}]}))))++(deftest ^:parallel ignore-nonsense-when-hashing-queries-test+ ;; these two queries differ in a few ways:+ ;;+ ;; - `q1` has `:metabase.lib.query/transformation-added-base-type` while `q2` does not+ ;; - `q1` has empty `:constraints`+ ;; - the order or parameters is flipped between the two queries+ ;; - the order of parameter :value is filled for parameter "A"+ (let [q1 {:lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql+ :fields [[:field+ {:lib/uuid "40dfd708-6c4e-4ff1-ae9e-c3863ae2fe46"+ :metabase.lib.query/transformation-added-base-type true+ :base-type :type/BigInteger+ :effective-type :type/BigInteger}+ 89952]+ [:field+ {:lib/uuid "719f9e44-eb66-4488-be9b-62078f2a677c"+ :metabase.lib.query/transformation-added-base-type true+ :base-type :type/Text+ :effective-type :type/Text}+ 89953]]+ :source-table 14303}]+ :database 2378+ :parameters [{:value [1 2], :target [:dimension [:field 89952 nil]], :id "a", :type :id}+ {:value [1], :target [:dimension [:field 89952 nil]], :id "b", :type :id}]+ :lib.convert/converted? true+ :lib/metadata meta/metadata-provider+ :constraints {}}+ q2 {:constraints nil+ :lib/type :mbql/query+ :lib/metadata meta/metadata-provider+ :stages [{:fields [[:field+ {:base-type :type/BigInteger+ :lib/uuid "373849ed-7038-4071-b038-db3dd7039181"+ :effective-type :type/BigInteger}+ 89952]+ [:field+ {:base-type :type/Text, :lib/uuid "b007dc3f-b12e-459e-89dc-4706580fed0a", :effective-type :type/Text}+ 89953]]+ :lib/type :mbql.stage/mbql+ :source-table 14303}]+ :middleware {:userland-query? true}+ :cache-strategy nil+ :info {:executed-by nil, :context :action, :action-id 1735}+ :database 2378+ :parameters [{:value [1], :target [:dimension [:field 89952 nil]], :id "b", :type :id}+ {:value [2 1], :target [:dimension [:field 89952 nil]], :id "a", :type :id}]}]+ (is (= (query-hash-hex q1)+ (query-hash-hex q2)))))++(deftest ^:parallel normalize-before-hashing-test+ (testing "Empty :constraints should get normalized out before calculating query hash"+ (is (= (query-hash-hex {:constraints nil+ :lib/type :mbql/query+ :database Integer/MAX_VALUE+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 14303}]})+ (query-hash-hex {:constraints {}+ :lib/type :mbql/query+ :database Integer/MAX_VALUE+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 14303}]})+ (query-hash-hex {:lib/type :mbql/query+ :database Integer/MAX_VALUE+ :stages [{:lib/type :mbql.stage/mbql+ :source-table 14303}]})))))diff --git a/test/metabase/lib_be/metadata/jvm_test.clj b/test/metabase/lib_be/metadata/jvm_test.cljindex d587bf48eeb..92ba49b17f7 100644--- a/test/metabase/lib_be/metadata/jvm_test.clj+++ b/test/metabase/lib_be/metadata/jvm_test.clj@@ -259,3 +259,13 @@{:lib/type :metadata/segment, :table-id 1}{:lib/type :metadata/native-query-snippet, :id #{1}}{:lib/type :metadata/native-query-snippet, :name #{"Snippet"}}))))++(deftest ^:parallel return-database-require-filter-test+ (mt/with-temp [:model/Database db {:engine :h2}+ :model/Table buyer {:name "BUYER" :database_require_filter true :db_id (:id db)}+ :model/Field buyer-id {:name "ID" :table_id (:id buyer) :base_type :type/Integer :database_partitioned true}]+ (let [mp (lib.metadata.jvm/application-database-metadata-provider (:id db))]+ (is (=? {:database-require-filter true}+ (lib.metadata/table mp (:id buyer))))+ (is (=? {:database-partitioned true}+ (lib.metadata/field mp (:id buyer-id)))))))diff --git a/test/metabase/lib_be/models/transforms_test.clj b/test/metabase/lib_be/models/transforms_test.cljindex 3dccad305cf..ce17f8ffedb 100644--- a/test/metabase/lib_be/models/transforms_test.clj+++ b/test/metabase/lib_be/models/transforms_test.clj@@ -2,7 +2,8 @@(:require[clojure.test :refer :all][metabase.lib-be.core :as lib-be]- [metabase.util.json :as json]))+ [metabase.util.json :as json]+ [metabase.util.malli :as mu]))(deftest ^:parallel handle-bad-template-tags-test(testing (str "an malformed template tags map like the one below is invalid. Rather than potentially destroy an entire API "@@ -27,3 +28,35 @@(deftest ^:parallel normalize-empty-query-test(is (= {}((:out lib-be/transform-query) "{}"))))++(deftest ^:parallel normalize-busted-query-test+ ;; this validation should be done even in prod+ (mu/disable-enforcement+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"\QQuery must include :database\E"+ (lib-be/normalize-query {:query {:source-table "card__117"}})))))++(deftest ^:parallel normalize-busted-query-test-2+ ;; this validation should be done even in prod+ (mu/disable-enforcement+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"\QQuery must include :lib/type or :type\E"+ (lib-be/normalize-query {:database 1, :query {:source-table "card__117"}})))))++(deftest ^:parallel normalize-busted-query-test-3+ ;; this validation should be done even in prod+ (mu/disable-enforcement+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"\QMBQL 4 keys like :type, :query, or :native are not allowed in MBQL 5 queries with :lib/type\E"+ (lib-be/normalize-query {:database 1, :lib/type :mbql/query, :query {:source-table "card__117"}})))))++(deftest ^:parallel normalize-busted-query-test-4+ ;; this validation should be done even in prod+ (mu/disable-enforcement+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"\QMBQL 5 :stages is not allowed in an MBQL 4 query with :type\E"+ (lib-be/normalize-query {:database 1, :type :query, :stages []})))))diff --git a/test/metabase/models/interface_test.clj b/test/metabase/models/interface_test.cljindex 2657e06cda4..6fe1bdcdc53 100644--- a/test/metabase/models/interface_test.clj+++ b/test/metabase/models/interface_test.clj@@ -1,8 +1,8 @@(ns metabase.models.interface-test+ {:clj-kondo/config '{:linters {:deprecated-var {:level :off}}}}(:require[clojure.test :refer :all][java-time.api :as t]- [metabase.legacy-mbql.normalize :as mbql.normalize][metabase.models.interface :as mi][metabase.test :as mt][metabase.util :as u]@@ -10,51 +10,11 @@[metabase.util.encryption-test :as encryption-test][metabase.util.json :as json][toucan2.core :as t2])- (:import (com.fasterxml.jackson.core JsonParseException)))+ (:import+ (com.fasterxml.jackson.core JsonParseException)))-;; let's make sure the `transform-metabase-query`/`transform-metric-segment-definition`/`transform-parameters-list`-;; normalization functions respond gracefully to invalid stuff when pulling them out of the Database. See #8914--(deftest ^:parallel handle-bad-template-tags-test- (testing (str "an malformed template tags map like the one below is invalid. Rather than potentially destroy an entire API "- "response because of one malformed Card, dump the error to the logs and return nil.")- (is (= nil- ((:out mi/transform-metabase-query)- (json/encode- {:database 1- :type :native- :native {:template-tags 1000}}))))))--(deftest ^:parallel template-tag-validate-saves-test- (testing "on the other hand we should be a little more strict on the way and disallow you from saving the invalid stuff"- ;; TODO -- we should make sure this returns a good error message so we don't have to dig thru the exception chain.- (is (thrown?- Exception- ((:in mi/transform-metabase-query)- {:database 1- :type :native- :native {:template-tags {100 [:field-id "WOW"]}}})))))--(deftest ^:parallel normalize-empty-query-test- (is (= {}- ((:out mi/transform-metabase-query) "{}"))))--(deftest handle-errors-gracefully-test- (testing (str "Cheat and override the `normalization-tokens` function to always throw an Exception so we can make "- "sure the Toucan type fn handles the error gracefully")- (with-redefs [mbql.normalize/normalize-tokens (fn [& _] (throw (Exception. "BARF")))]- (is (= nil- ((:out mi/transform-parameters-list)- (json/encode- [{:target [:dimension [:field "ABC" nil]]}])))))))--(deftest do-not-eat-exceptions-test- (testing "should not eat Exceptions if normalization barfs when saving"- (is (thrown?- Exception- (with-redefs [mbql.normalize/normalize-tokens (fn [& _] (throw (Exception. "BARF")))]- ((:in mi/transform-parameters-list)- [{:target [:dimension [:field "ABC" nil]]}]))))))+;; Let's make sure `transform-metric-segment-definition`/`transform-parameters-list` normalization functions respond+;; gracefully to invalid stuff when pulling them out of the Database. See #8914(deftest timestamped-property-test(testing "Make sure updated_at gets updated for timestamped models"@@ -68,13 +28,16 @@:updated_at (partial not= updated-at)}(t2/select-one [:model/Table :id :name :updated_at] (u/the-id table))))))))-(deftest timestamped-property-do-not-stomp-on-explicit-values-test+(deftest ^:parallel timestamped-property-do-not-stomp-on-explicit-values-test(testing "The :timestamped property should not stomp on :created_at/:updated_at if they are explicitly specified"(mt/with-temp [:model/Field field](testing "Nothing specified: use now() for both"(is (=? {:created_at java.time.temporal.Temporal:updated_at java.time.temporal.Temporal}- field))))+ field))))))++(deftest ^:parallel timestamped-property-do-not-stomp-on-explicit-values-test-2+ (testing "The :timestamped property should not stomp on :created_at/:updated_at if they are explicitly specified"(let [t #t "2022-10-13T19:21:00Z"expected-timestamp (t/offset-date-time "2022-10-13T19:21:00Z")](testing "Explicitly specify :created_at"@@ -91,12 +54,14 @@(defmethod mi/non-timestamped-fields :test-model/updated-at-tester [_]#{:non_timestamped :other})-(deftest timestamped-property-skips-non-timestamped-fields-test+(deftest ^:parallel timestamped-property-skips-non-timestamped-fields-test(testing "Does not add a timestamp if it only includes non-timestamped fields"(let [instance (-> (t2/instance :test-model/updated-at-tester {:non_timestamped nil})(assoc :non_timestamped 1))](is (= {:non_timestamped 1}- (#'mi/add-updated-at-timestamp instance)))))+ (#'mi/add-updated-at-timestamp instance))))))++(deftest ^:parallel timestamped-property-skips-non-timestamped-fields-test-2(testing "Adds a timestamp if it includes other fields"(let [instance (-> (t2/instance :test-model/updated-at-tester {:non_timestamped nil :included nil})(assoc :non_timestamped 1)@@ -219,9 +184,7 @@{"toucan2.jdbc.query/sql-args" (str (apply str (repeat 247 "b")) "...")}]]}})(#'mi/json-in-with-eliding {"ex-data" {"toucan2/context-trace" [["execute SQL with class com.mchange.v2.c3p0.impl.NewProxyConnection"{"toucan2.jdbc.query/sql-args" (apply str (repeat 500 "b"))}]]}})))-(is (= (json/encode {:a (repeat 50 "x")}) (#'mi/json-in-with-eliding {:a (repeat 500 "x")})))-(testing "A passed string is not elided"(is (= (apply str (repeat 1000 "a")) (#'mi/json-in-with-eliding (apply str (repeat 1000 "a")))))))diff --git a/test/metabase/parameters/chain_filter_test.clj b/test/metabase/parameters/chain_filter_test.cljindex b755375d6ad..d33a540ba49 100644--- a/test/metabase/parameters/chain_filter_test.clj+++ b/test/metabase/parameters/chain_filter_test.clj@@ -1,22 +1,20 @@(ns metabase.parameters.chain-filter-test(:require[clojure.test :refer :all]- [malli.error :as me]- [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.parameters.chain-filter :as chain-filter][metabase.parameters.field-values :as params.field-values][metabase.test :as mt][metabase.util :as u][metabase.util.json :as json]- [metabase.util.malli.registry :as mr][metabase.warehouse-schema.models.field-values :as field-values][toucan2.core :as t2]))(defn shorthand->constraint [field-id v](if-not (vector? v){:field-id field-id- :op :=- :value v}+ :op := :value v}(let [op (when (keyword? (first v)) (first v))options (when (map? (last v)) (last v))v (cond-> v@@ -54,80 +52,105 @@(deftest ^:parallel special-form-filter-clause-test(testing "Can handle multi-arg string filters when chaining (#57287)"- (doseq [value ["Omer" ["Omer"] ["Omer" "Clovis"]]]- (testing (str "with value " (pr-str value))- (are [op] (nil? (->> (#'chain-filter/filter-clause (mt/id :venues)- {:field-id (mt/id :venues :name)- :op op- :value value- :options nil})- (mr/explain mbql.s/Filter)- me/humanize))- :starts-with :ends-with :contains :does-not-contain)))))--(deftest chain-filter-test+ (let [mp (mt/metadata-provider)]+ (doseq [value ["Omer" ["Omer"] ["Omer" "Clovis"]]]+ (testing (str "with value " (pr-str value))+ (are [op] (some? (#'chain-filter/add-filter+ (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (mt/id :venues)+ {:field-id (mt/id :venues :name)+ :op op+ :value value+ :options nil}))+ :starts-with :ends-with :contains :does-not-contain))))))++(deftest ^:parallel chain-filter-test(testing "Show me expensive restaurants"- (is (= {:values [["Dal Rae Restaurant"]- ["Lawry's The Prime Rib"]- ["Pacific Dining Car - Santa Monica"]- ["Sushi Nakazawa"]- ["Sushi Yasuda"]- ["Tanoshi Sushi & Sake Bar"]]+ (is (= {:values+ [["Dal Rae Restaurant"]+ ["Lawry's The Prime Rib"]+ ["Pacific Dining Car - Santa Monica"]+ ["Sushi Nakazawa"]+ ["Sushi Yasuda"]+ ["Tanoshi Sushi & Sake Bar"]],:has_more_values false}- (chain-filter venues.name {venues.price 4}))))+ (chain-filter venues.name {venues.price 4})))))++(deftest ^:parallel chain-filter-test-2+ (testing "Show me categories that have expensive restaurants"+ (is (= {:values [["Japanese"] ["Steakhouse"]], :has_more_values false} (chain-filter categories.name {venues.price 4})))))++(deftest ^:parallel chain-filter-test-2b(testing "Show me categories that have expensive restaurants"- (is (= {:values [["Japanese"] ["Steakhouse"]]- :has_more_values false}- (chain-filter categories.name {venues.price 4})))(testing "Should work with string versions of param values"- (is (= {:values [["Japanese"] ["Steakhouse"]]- :has_more_values false}- (chain-filter categories.name {venues.price "4"})))))+ (is (= {:values [["Japanese"] ["Steakhouse"]], :has_more_values false}+ (chain-filter categories.name {venues.price "4"}))))))++(deftest ^:parallel chain-filter-test-3(testing "Show me categories starting with s (case-insensitive) that have expensive restaurants"- (is (= {:values [["Steakhouse"]]- :has_more_values false}- (chain-filter categories.name {venues.price 4, categories.name [:starts-with "s" {:case-sensitive false}]}))))+ (is (= {:values [["Steakhouse"]], :has_more_values false}+ (chain-filter categories.name {venues.price 4, categories.name [:starts-with "s" {:case-sensitive false}]})))))++(deftest ^:parallel chain-filter-test-4(testing "Show me cheap Thai restaurants"- (is (= {:values [["Kinaree Thai Bistro"] ["Krua Siri"]]- :has_more_values false}- (chain-filter venues.name {venues.price 1, categories.name "Thai"}))))+ (is (= {:values [["Kinaree Thai Bistro"] ["Krua Siri"]], :has_more_values false}+ (chain-filter venues.name {venues.price 1, categories.name "Thai"})))))++(deftest ^:parallel chain-filter-test-5(testing "Show me the categories that have cheap restaurants"- (is (= {:values [["Asian"] ["BBQ"] ["Bakery"] ["Bar"] ["Burger"] ["Caribbean"] ["Deli"]- ["Karaoke"] ["Mexican"] ["Pizza"] ["Southern"] ["Thai"]]+ (is (= {:values+ [["Asian"]+ ["BBQ"]+ ["Bakery"]+ ["Bar"]+ ["Burger"]+ ["Caribbean"]+ ["Deli"]+ ["Karaoke"]+ ["Mexican"]+ ["Pizza"]+ ["Southern"]+ ["Thai"]],:has_more_values false}- (chain-filter categories.name {venues.price 1}))))+ (chain-filter categories.name {venues.price 1})))))++(deftest ^:parallel chain-filter-test-6(testing "Show me cheap restaurants with the word 'taco' in their name (case-insensitive)"- (is (= {:values [["Tacos Villa Corona"] ["Tito's Tacos"]]- :has_more_values false}- (chain-filter venues.name {venues.price 1, venues.name [:contains "tAcO" {:case-sensitive false}]}))))+ (is (= {:values [["Tacos Villa Corona"] ["Tito's Tacos"]], :has_more_values false}+ (chain-filter venues.name {venues.price 1, venues.name [:contains "tAcO" {:case-sensitive false}]})))))++(deftest ^:parallel chain-filter-test-7(testing "Show me the first 3 expensive restaurants"- (is (= {:values [["Dal Rae Restaurant"] ["Lawry's The Prime Rib"] ["Pacific Dining Car - Santa Monica"]]+ (is (= {:values [["Dal Rae Restaurant"] ["Lawry's The Prime Rib"] ["Pacific Dining Car - Santa Monica"]],:has_more_values true}- (chain-filter venues.name {venues.price 4} :limit 3))))+ (chain-filter venues.name {venues.price 4} :limit 3)))))++(deftest ^:parallel chain-filter-test-8(testing "Oh yeah, we actually support arbitrary MBQL filter clauses. Neat!"- (is (= {:values [["Festa"] ["Fred 62"]]- :has_more_values false}- (chain-filter venues.name {venues.price [:between 2 3]- venues.name [:starts-with "f" {:case-sensitive false}]})))))+ (is (= {:values [["Festa"] ["Fred 62"]], :has_more_values false}+ (chain-filter venues.name {venues.price [:between 2 3], venues.name [:starts-with "f" {:case-sensitive false}]})))))-(deftest multiple-values-test+(deftest ^:parallel multiple-values-test(testing "Chain filtering should support multiple values for a single parameter (as a vector or set of values)"(testing "Show me restaurants with price = 1 or 2 with the word 'BBQ' in their name (case-sensitive)"(is (= {:values [["Baby Blues BBQ"] ["Beachwood BBQ & Brewing"] ["Bludso's BBQ"]]:has_more_values false}- (chain-filter venues.name {venues.price #{1 2}, venues.name [:contains "BBQ"]}))))+ (chain-filter venues.name {venues.price #{1 2}, venues.name [:contains "BBQ"]}))))))++(deftest ^:parallel multiple-values-test-2+ (testing "Chain filtering should support multiple values for a single parameter (as a vector or set of values)"(testing "Show me the possible values of price for Bakery *or* BBQ restaurants"(is (= {:values [[1] [2] [3]]:has_more_values false}(chain-filter venues.price {categories.name ["Bakery" "BBQ"]}))))))-(deftest auto-parse-string-params-test+(deftest ^:parallel auto-parse-string-params-test(testing "Parameters that come in as strings (i.e., all of them that come in via the API) should work as intended"(is (= {:values [["Baby Blues BBQ"] ["Beachwood BBQ & Brewing"] ["Bludso's BBQ"]]:has_more_values false}(chain-filter venues.name {venues.price ["1" "2"], venues.name [:contains "BBQ"]})))))-(deftest unrelated-params-test+(deftest ^:parallel unrelated-params-test(testing "Parameters that are completely unrelated (don't apply to this Table) should just get ignored entirely";; there is no way to join from venues -> users so users.id should get ignored(binding [chain-filter/*enable-reverse-joins* false]@@ -178,54 +201,58 @@{90 {200 [[90 200]]}200 {:end [[200 :end]]}}))-(deftest traverse-graph-test+(deftest ^:parallel traverse-graph-test(testing "If no need to join, returns immediately"- (is (nil? (#'chain-filter/traverse-graph {} :start :start 5))))+ (is (nil? (#'chain-filter/traverse-graph {} :start :start 5)))))++(deftest ^:parallel traverse-graph-test-2(testing "Finds a simple hop"(let [graph {:start {:end [:start->end]}}](is (= [:start->end]- (#'chain-filter/traverse-graph graph :start :end 5))))- (testing "Finds over a few hops"- (let [graph {:start {:a [:start->a]}- :a {:b [:a->b]}- :b {:c [:b->c]}- :c {:end [:c->end]}}]- (is (= [:start->a :a->b :b->c :c->end]- (#'chain-filter/traverse-graph graph :start :end 5)))- (testing "But will not exceed the max depth"- (is (nil? (#'chain-filter/traverse-graph graph :start :end 2))))))- (testing "Can find a path in a dense and large graph"- (is (= [[:start 50] [50 :end]]- (#'chain-filter/traverse-graph megagraph :start :end 5)))- (is (= [[:start 90] [90 200] [200 :end]]- (#'chain-filter/traverse-graph megagraph-single-path :start :end 5))))- (testing "Returns nil if there is no path"- (let [graph {:start {1 [[:start 1]]}- 1 {2 [[1 2]]}- ;; no way to get to 3- 3 {4 [[3 4]]}- 4 {:end [[4 :end]]}}]- (is (nil? (#'chain-filter/traverse-graph graph :start :end 5)))))- (testing "Not fooled by loops"- (let [graph {:start {:a [:start->a]}- :a {:b [:a->b]- :a [:b->a]}- :b {:c [:b->c]- :a [:c->a]- :b [:c->b]}- :c {:end [:c->end]}}]- (is (= [:start->a :a->b :b->c :c->end]- (#'chain-filter/traverse-graph graph :start :end 5)))- (testing "But will not exceed the max depth"- (is (nil? (#'chain-filter/traverse-graph graph :start :end 2))))))))--(deftest find-joins-test+ (#'chain-filter/traverse-graph graph :start :end 5))))))++(deftest ^:parallel traverse-graph-test-3+ (testing "Finds over a few hops"+ (let [graph {:start {:a [:start->a]}, :a {:b [:a->b]}, :b {:c [:b->c]}, :c {:end [:c->end]}}]+ (is (= [:start->a :a->b :b->c :c->end]+ (#'chain-filter/traverse-graph graph :start :end 5)))+ (testing "But will not exceed the max depth"+ (is (nil? (#'chain-filter/traverse-graph graph :start :end 2)))))))++(deftest ^:parallel traverse-graph-test-4+ (testing "Can find a path in a dense and large graph"+ (is (= [[:start 50] [50 :end]]+ (#'chain-filter/traverse-graph megagraph :start :end 5)))+ (is (= [[:start 90] [90 200] [200 :end]]+ (#'chain-filter/traverse-graph megagraph-single-path :start :end 5)))))++(deftest ^:parallel traverse-graph-test-5+ (testing "Returns nil if there is no path"+ (let [graph {:start {1 [[:start 1]]}, 1 {2 [[1 2]]}, 3 {4 [[3 4]]}, 4 {:end [[4 :end]]}}]+ (is (nil? (#'chain-filter/traverse-graph graph :start :end 5))))))++(deftest ^:parallel traverse-graph-test-6+ (testing "Not fooled by loops"+ (let [graph+ {:start {:a [:start->a]},+ :a {:b [:a->b], :a [:b->a]},+ :b {:c [:b->c], :a [:c->a], :b [:c->b]},+ :c {:end [:c->end]}}]+ (is (= [:start->a :a->b :b->c :c->end] (#'chain-filter/traverse-graph graph :start :end 5)))+ (testing "But will not exceed the max depth"+ (is (nil? (#'chain-filter/traverse-graph graph :start :end 2)))))))++(deftest ^:parallel find-joins-test(mt/dataset airports(mt/$ids nil(testing "airport -> municipality"(is (= [{:lhs {:table $$airport, :field %airport.municipality_id}:rhs {:table $$municipality, :field %municipality.id}}]- (#'chain-filter/find-joins (mt/id) $$airport $$municipality))))+ (#'chain-filter/find-joins (mt/id) $$airport $$municipality)))))))++(deftest ^:parallel find-joins-test-2+ (mt/dataset airports+ (mt/$ids nil(testing "airport [-> municipality -> region] -> country"(is (= [{:lhs {:table $$airport, :field %airport.municipality_id}:rhs {:table $$municipality, :field %municipality.id}}@@ -233,12 +260,21 @@:rhs {:table $$region, :field %region.id}}{:lhs {:table $$region, :field %region.country_id}:rhs {:table $$country, :field %country.id}}]- (#'chain-filter/find-joins (mt/id) $$airport $$country))))+ (#'chain-filter/find-joins (mt/id) $$airport $$country)))))))++(deftest ^:parallel find-joins-test-3+ (mt/dataset airports+ (mt/$ids nil(testing "[backwards]"(testing "municipality -> airport"(is (= [{:lhs {:table $$municipality, :field %municipality.id}:rhs {:table $$airport, :field %airport.municipality_id}}]- (#'chain-filter/find-joins (mt/id) $$municipality $$airport))))+ (#'chain-filter/find-joins (mt/id) $$municipality $$airport))))))))++(deftest ^:parallel find-joins-test-4+ (mt/dataset airports+ (mt/$ids nil+ (testing "[backwards]"(testing "country [-> region -> municipality] -> airport"(is (= [{:lhs {:table $$country, :field %country.id}:rhs {:table $$region, :field %region.country_id}}@@ -248,12 +284,14 @@:rhs {:table $$airport, :field %airport.municipality_id}}](#'chain-filter/find-joins (mt/id) $$country $$airport))))))))-(deftest find-all-joins-test+(deftest ^:parallel find-all-joins-test(testing "With reverse joins disabled"(binding [chain-filter/*enable-reverse-joins* false](mt/$ids nil(is (= [{:lhs {:table $$venues, :field %venues.category_id}, :rhs {:table $$categories, :field %categories.id}}]- (#'chain-filter/find-all-joins $$venues #{%categories.name %users.id}))))))+ (#'chain-filter/find-all-joins $$venues #{%categories.name %users.id})))))))++(deftest ^:parallel find-all-joins-test-2(mt/dataset airports(mt/$ids nil(testing "airport [-> municipality] -> region"@@ -264,80 +302,127 @@:rhs {:table $$region, :field %region.id}}](#'chain-filter/find-all-joins $$airport #{%region.name %municipality.name %region.id}))))))))-(deftest multi-hop-test+(deftest ^:parallel multi-hop-test(mt/dataset airports(testing "Should be able to filter against other tables with that require multiple joins\n"(testing "single direct join: Airport -> Municipality"(is (= {:values [["San Francisco International Airport"]]:has_more_values false}- (chain-filter airport.name {municipality.name ["San Francisco"]}))))+ (chain-filter airport.name {municipality.name ["San Francisco"]})))))))++(deftest ^:parallel multi-hop-test-2+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"(testing "2 joins required: Airport -> Municipality -> Region"(is (= {:values [["Beale Air Force Base"]["Edwards Air Force Base"]["John Wayne Airport-Orange County Airport"]]:has_more_values false}- (take-n-values 3 (chain-filter airport.name {region.name ["California"]})))))+ (take-n-values 3 (chain-filter airport.name {region.name ["California"]}))))))))++(deftest ^:parallel multi-hop-test-3+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"(testing "3 joins required: Airport -> Municipality -> Region -> Country"(is (= {:values [["Abraham Lincoln Capital Airport"]["Albuquerque International Sunport"]["Altus Air Force Base"]]:has_more_values false}- (take-n-values 3 (chain-filter airport.name {country.name ["United States"]})))))+ (take-n-values 3 (chain-filter airport.name {country.name ["United States"]}))))))))++(deftest ^:parallel multi-hop-test-4+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"(testing "4 joins required: Airport -> Municipality -> Region -> Country -> Continent"(is (= {:values [["Afonso Pena Airport"]["Alejandro Velasco Astete International Airport"]["Carrasco International /General C L Berisso Airport"]]:has_more_values false}- (take-n-values 3 (chain-filter airport.name {continent.name ["South America"]})))))+ (take-n-values 3 (chain-filter airport.name {continent.name ["South America"]}))))))))++(deftest ^:parallel multi-hop-test-5a+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"(testing "[backwards]"(testing "single direct join: Municipality -> Airport"(is (= {:values [["San Francisco"]]:has_more_values false}- (chain-filter municipality.name {airport.name ["San Francisco International Airport"]}))))+ (chain-filter municipality.name {airport.name ["San Francisco International Airport"]}))))))))++(deftest ^:parallel multi-hop-test-5b+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"+ (testing "[backwards]"(testing "2 joins required: Region -> Municipality -> Airport"(is (= {:values [["California"]]:has_more_values false}- (chain-filter region.name {airport.name ["San Francisco International Airport"]}))))+ (chain-filter region.name {airport.name ["San Francisco International Airport"]}))))))))++(deftest ^:parallel multi-hop-test-5c+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"+ (testing "[backwards]"(testing "3 joins required: Country -> Region -> Municipality -> Airport"(is (= {:values [["United States"]]:has_more_values false}- (chain-filter country.name {airport.name ["San Francisco International Airport"]}))))+ (chain-filter country.name {airport.name ["San Francisco International Airport"]}))))))))++(deftest ^:parallel multi-hop-test-5d+ (mt/dataset airports+ (testing "Should be able to filter against other tables with that require multiple joins\n"+ (testing "[backwards]"(testing "4 joins required: Continent -> Region -> Municipality -> Airport"(is (= {:values [["North America"]]:has_more_values false}(chain-filter continent.name {airport.name ["San Francisco International Airport"]}))))))))-(deftest filterable-field-ids-test+(deftest ^:parallel filterable-field-ids-test(mt/$ids(testing (format "venues.price = %d categories.name = %d users.id = %d\n" %venues.price %categories.name %users.id)(is (= #{%categories.name %users.id}- (chain-filter/filterable-field-ids %venues.price #{%categories.name %users.id})))+ (chain-filter/filterable-field-ids %venues.price #{%categories.name %users.id}))))))++(deftest ^:parallel filterable-field-ids-test-2+ (mt/$ids+ (testing (format "venues.price = %d categories.name = %d users.id = %d\n" %venues.price %categories.name %users.id)(testing "reverse joins disabled: should exclude users.id"(binding [chain-filter/*enable-reverse-joins* false](is (= #{%categories.name}- (chain-filter/filterable-field-ids %venues.price #{%categories.name %users.id})))))+ (chain-filter/filterable-field-ids %venues.price #{%categories.name %users.id}))))))))++(deftest ^:parallel filterable-field-ids-test-3+ (mt/$ids+ (testing (format "venues.price = %d categories.name = %d users.id = %d\n" %venues.price %categories.name %users.id)(testing "return nil if filtering-field-ids is empty"(is (= nil(chain-filter/filterable-field-ids %venues.price #{})))))))-(deftest chain-filter-search-test+(deftest ^:parallel chain-filter-search-test(testing "Show me categories containing 'eak' (case-insensitive) that have expensive restaurants"(is (= {:values [["Steakhouse"]]:has_more_values false}- (chain-filter-search categories.name {venues.price 4} "eak"))))+ (chain-filter-search categories.name {venues.price 4} "eak")))))++(deftest ^:parallel chain-filter-search-test-2(testing "Show me cheap restaurants including with 'taco' (case-insensitive)"(is (= {:values [["Tacos Villa Corona"] ["Tito's Tacos"]]:has_more_values false}- (chain-filter-search venues.name {venues.price 1} "tAcO"))))+ (chain-filter-search venues.name {venues.price 1} "tAcO")))))++(deftest ^:parallel chain-filter-search-test-3(testing "search for something crazy = should return empty results"(is (= {:values []:has_more_values false}- (chain-filter-search categories.name {venues.price 4} "zzzzz"))))+ (chain-filter-search categories.name {venues.price 4} "zzzzz")))))++(deftest ^:parallel chain-filter-search-test-4(testing "Field that doesn't exist should throw a 404"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Field [\d,]+ does not exist"- (chain-filter/chain-filter-search Integer/MAX_VALUE nil "s"))))+ (chain-filter/chain-filter-search Integer/MAX_VALUE nil "s")))))++(deftest ^:parallel chain-filter-search-test-5(testing "Field that isn't type/Text should throw a 400"(is (thrown-with-msg?clojure.lang.ExceptionInfo@@ -346,15 +431,15 @@;;; --------------------------------------------------- Remapping -----------------------------------------------------(defn do-with-human-readable-values-remapping [thunk]+(defn do-with-human-readable-values-remapping! [thunk](mt/with-column-remappings [venues.category_id (values-of categories.name)](thunk)))-(defmacro with-human-readable-values-remapping {:style/indent 0} [& body]- `(do-with-human-readable-values-remapping (fn [] ~@body)))+(defmacro with-human-readable-values-remapping! {:style/indent 0} [& body]+ `(do-with-human-readable-values-remapping! (fn [] ~@body)))(deftest human-readable-values-remapped-chain-filter-test- (with-human-readable-values-remapping+ (with-human-readable-values-remapping!(testing "Show me category IDs for categories";; there are no restaurants with category 1(is (= {:values [[2 "American"]@@ -373,7 +458,7 @@(take-n-values 3 (chain-filter venues.category_id {venues.category_id 40})))))))(deftest human-readable-values-remapped-chain-filter-search-test- (with-human-readable-values-remapping+ (with-human-readable-values-remapping!(testing "Show me category IDs [whose name] contains 'bar'"(testing "\nconstraints = {}"(is (= {:values [[7 "Bar"]@@ -385,7 +470,6 @@[74 "Wine Bar"]]:has_more_values false}(chain-filter-search venues.category_id nil "bar"))))-(testing "Show me category IDs [whose name] contains 'house' that have expensive restaurants"(is (= {:values [[67 "Steakhouse"]]:has_more_values false}@@ -395,22 +479,25 @@:has_more_values false}(chain-filter-search venues.category_id {venues.price 4} "zzzzz"))))))-(deftest field-to-field-remapped-field-id-test+(deftest ^:parallel field-to-field-remapped-field-id-test(is (= (mt/id :venues :name)(#'chain-filter/remapped-field-id (mt/id :venues :id)))))-(deftest fk-field-to-pk-field-to-name-field-remapped-field-id-test+(deftest ^:parallel fk-field-to-pk-field-to-name-field-remapped-field-id-test(is (= (mt/id :people :name)(#'chain-filter/remapped-field-id (mt/id :orders :user_id)))))-(deftest field-to-field-remapped-chain-filter-test+(deftest ^:parallel field-to-field-remapped-chain-filter-test(testing "Field-to-field remapping: venues.category_id -> categories.name\n"(testing "Show me venue IDs (names)"(is (= {:values [[29 "20th Century Cafe"][8 "25°"][93 "33 Taps"]]:has_more_values false}- (take-n-values 3 (chain-filter venues.id nil)))))+ (take-n-values 3 (chain-filter venues.id nil)))))))++(deftest ^:parallel field-to-field-remapped-chain-filter-test-2+ (testing "Field-to-field remapping: venues.category_id -> categories.name\n"(testing "Show me expensive venue IDs (names)"(is (= {:values [[55 "Dal Rae Restaurant"][61 "Lawry's The Prime Rib"]@@ -418,36 +505,42 @@:has_more_values false}(take-n-values 3 (chain-filter venues.id {venues.price 4})))))))-(deftest field-to-field-remapped-chain-filter-search-test+(deftest ^:parallel field-to-field-remapped-chain-filter-search-test(testing "Field-to-field remapping: venues.category_id -> categories.name\n"(testing "Show me venue IDs that [have a remapped name that] contains 'sushi'"(is (= {:values [[76 "Beyond Sushi"][80 "Blue Ribbon Sushi"][77 "Sushi Nakazawa"]]:has_more_values false}- (take-n-values 3 (chain-filter-search venues.id nil "sushi")))))+ (take-n-values 3 (chain-filter-search venues.id nil "sushi")))))))++(deftest ^:parallel field-to-field-remapped-chain-filter-search-test-2+ (testing "Field-to-field remapping: venues.category_id -> categories.name\n"(testing "Show me venue IDs that [have a remapped name that] contain 'sushi' that are expensive"(is (= {:values [[77 "Sushi Nakazawa"][79 "Sushi Yasuda"][81 "Tanoshi Sushi & Sake Bar"]]:has_more_values false}- (chain-filter-search venues.id {venues.price 4} "sushi"))))+ (chain-filter-search venues.id {venues.price 4} "sushi"))))))++(deftest ^:parallel field-to-field-remapped-chain-filter-search-test-3+ (testing "Field-to-field remapping: venues.category_id -> categories.name\n"(testing "search for something crazy = should return empty results"(is (= {:values []:has_more_values false}(chain-filter-search venues.id {venues.price 4} "zzzzz"))))))-(defmacro with-fk-field-to-field-remapping {:style/indent 0} [& body]+(defmacro with-fk-field-to-field-remapping! {:style/indent 0} [& body]`(mt/with-column-remappings [~'venues.category_id ~'categories.name]~@body))(deftest fk-field-to-field-remapped-field-id-test- (with-fk-field-to-field-remapping+ (with-fk-field-to-field-remapping!(is (= (mt/id :categories :name)(#'chain-filter/remapped-field-id (mt/id :venues :category_id))))))(deftest fk-field-to-field-remapped-chain-filter-test- (with-fk-field-to-field-remapping+ (with-fk-field-to-field-remapping!(testing "Show me category IDs for categories";; there are no restaurants with category 1(is (= {:values [[2 "American"]@@ -466,7 +559,7 @@(chain-filter venues.category_id {venues.category_id 40}))))))(deftest fk-field-to-field-remapped-chain-filter-search-test- (with-fk-field-to-field-remapping+ (with-fk-field-to-field-remapping!(testing "Show me categories containing 'ar'"(testing "\nconstraints = {}"(is (= {:values [[3 "Artisan"]@@ -504,7 +597,6 @@:has_more_values false}(take-n-values 3 (chain-filter categories.name nil))))(is (= 1 (t2/count :model/FieldValues :field_id field-id :type :full)))))-(testing "should create a linked-filter FieldValues when have constraints";; make sure we have a clean start(field-values/clear-advanced-field-values-for-field! field-id)@@ -516,7 +608,6 @@:has_more_values false}(chain-filter categories.name {venues.price 4})))(is (= 1 (t2/count :model/FieldValues :field_id field-id :type :advanced)))))-(testing "should search with the cached FieldValues when search without constraints"(mt/with-temp[:model/Field field (-> (t2/select-one :model/Field (mt/id :categories :name))@@ -528,7 +619,6 @@(is (= {:values [["Goooood"]]:has_more_values false}(chain-filter-search categories.name2 nil "oooood")))))-(testing "search with constraints";; make sure we have a clean start(field-values/clear-advanced-field-values-for-field! field-id)@@ -536,7 +626,6 @@;; warm up the cache(chain-filter categories.name {venues.price 4})(is (= 1 (t2/count :model/FieldValues :field_id field-id :type :advanced))))-(testing "should search for the values of linked-filter FieldValues"(t2/update! :model/FieldValues {:field_id field-id:type :advanced}@@ -564,18 +653,24 @@(is (= {:values [[2 "American"] [3 "Artisan"] [4 "Asian"]]:has_more_values false}(take-n-values 3 (chain-filter/chain-filter (mt/id :venues :category_id) nil))))-(is (= {:values [[4 "Asian"]]:has_more_values false}(chain-filter/chain-filter-search (mt/id :venues :category_id) nil "sian")))))))-(deftest time-interval-test+(deftest ^:parallel time-interval-test(testing "chain-filter should accept time interval strings like `past32weeks` for temporal Fields"(mt/$ids- (is (= [:time-interval $checkins.date -32 :week {:include-current false}]- (#'chain-filter/filter-clause $$checkins {:field-id %checkins.date- :op :=- :value "past32weeks"}))))))+ (is (=? {:stages [{:filters [[:time-interval {:include-current false}+ [:field {} (mt/id :checkins :date)]+ -32+ :week]]}]}+ (#'chain-filter/add-filter+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id :checkins))))+ $$checkins+ {:field-id %checkins.date+ :op :=+ :value "past32weeks"}))))))(mt/defdataset nil-values-dataset[["tbl"@@ -596,7 +691,7 @@:has_more_values false}}](testing "chain-filter";; sorting can differ a bit based on whether we use FieldValues or not... not sure why this is- ;; the case, but that's not important for this test anyway. Just sort everything+ ;; ;; the case, but that's not important for this test anyway. Just sort everything(is (= expected-values(update (chain-filter/chain-filter (mt/id :tbl field) []) :values sort))))(testing "chain-filter-search"@@ -726,7 +821,6 @@(#'chain-filter/find-joins (mt/id) $$messages $$users))))(finally(t2/update! :model/Field {:id %messages.receiver_id} {:active true})))-;; mark field(t2/update! :model/Field {:id %users.id} {:active false})(testing "there are no connections when PK is inactive"diff --git a/test/metabase/parameters/custom_values_test.clj b/test/metabase/parameters/custom_values_test.cljindex 24ff34b75c7..86aced8c1ba 100644--- a/test/metabase/parameters/custom_values_test.clj+++ b/test/metabase/parameters/custom_values_test.clj@@ -5,6 +5,8 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures]))+(set! *warn-on-reflection* true)+(use-fixtures :once (fixtures/initialize :db :row-lock));;; --------------------------------------------- source=card ----------------------------------------------@@ -174,7 +176,7 @@:has_more_values true}(custom-values/values-from-cardcard- [:field "Categories__NAME" {:base_type :type/Text}]+ [:field "Categories__NAME" {:base-type :type/Text}]{:stage-number 1}))))))))(deftest ^:parallel with-mbql-card-test-6-expressions@@ -270,8 +272,8 @@{:name "Card as source":slug "card":id "_CARD_"- :type "category"- :values_source_type "card"+ :type :category+ :values_source_type :card:values_source_config {:card_id (:id card):value_field (mt/$ids $venues.name)}}nil@@ -290,8 +292,8 @@{:name "Card as source":slug "card":id "_CARD_"- :type "category"- :values_source_type "card"+ :type :category+ :values_source_type :card:values_source_config {:card_id (:id card):value_field (mt/$ids $venues.name)}}nil@@ -310,10 +312,10 @@{:name "Card as source":slug "card":id "_CARD_"- :type "category"- :values_source_type "card"+ :type :category+ :values_source_type :card:values_source_config {:card_id (:id card)- :value_field [:field 0 nil]}}+ :value_field [:field Integer/MAX_VALUE nil]}}nil(constantly mock-default-result))))))))))diff --git a/test/metabase/parameters/params_test.clj b/test/metabase/parameters/params_test.cljindex 35c41a819f7..068d23a65a4 100644--- a/test/metabase/parameters/params_test.clj+++ b/test/metabase/parameters/params_test.clj@@ -119,7 +119,9 @@:dimensions []}]}(-> (t2/hydrate dashboard :param_fields):param_fields- mt/derecordize)))))+ mt/derecordize))))))++(deftest hydrate-param-fields-for-dashboard-test-2(testing "should ignore invalid parameter mappings"(mt/with-temporary-setting-values [enable-public-sharing true](mt/with-temp [:model/Dashboard dashboard {:parameters [{:id "p1" :type :number/=}@@ -180,26 +182,23 @@(is (not (contains? param-fields "p7"))))))))(deftest ^:parallel card->template-tag-test- (let [card {:dataset_query (mt/native-query {:query "SELECT *"+ (let [card {:dataset_query (mt/native-query {:query "SELECT *":template-tags {"id" {:name "id"- :display_name "ID"+ :display-name "ID":id "11111111":type :dimension+ :widget-type :number:dimension [:field (mt/id :venues :id) nil]}"name" {:name "name"- :display_name "Name"+ :display-name "Name":id "aaaaaaaa":type :dimension+ :widget-type :number:dimension [:field "name" {:base-type :type/Text}]}}})}]- (testing "card->template-tag-param-id->field-clauses"- (is (= {"11111111" #{[:field (mt/id :venues :id) nil]}- "aaaaaaaa" #{[:field "name" {:base-type :type/Text}]}}- (#'params/card->template-tag-param-id->field-clauses card))))-(testing "card->template-tag-param-id->field-ids"(is (= {"11111111" #{(mt/id :venues :id)}"aaaaaaaa" #{}}- (#'params/card->template-tag-param-id->field-ids card))))+ (#'params/card->template-tag-id->field-ids card))))(testing "card->template-tag-field-ids"(is (= #{(mt/id :venues :id)}@@ -212,7 +211,9 @@(params/get-linked-field-ids[{:parameter_mappings[{:parameter_id "foo" :target [:dimension [:field 256 nil]]}- {:parameter_id "bar" :target [:dimension [:field 267 nil]]}]}]))))+ {:parameter_id "bar" :target [:dimension [:field 267 nil]]}]}])))))++(deftest ^:parallel get-linked-field-ids-test-2(testing "get-linked-field-ids multiple fields to one param test"(is (= {"foo" #{256 10}"bar" #{267}}@@ -221,7 +222,9 @@[{:parameter_id "foo" :target [:dimension [:field 256 nil]]}{:parameter_id "bar" :target [:dimension [:field 267 nil]]}]}{:parameter_mappings- [{:parameter_id "foo" :target [:dimension [:field 10 nil]]}]}]))))+ [{:parameter_id "foo" :target [:dimension [:field 10 nil]]}]}])))))++(deftest ^:parallel get-linked-field-ids-test-3(testing "get-linked-field-ids-test misc fields"(is (= {"1" #{1} "2" #{2} "3" #{3} "4" #{4} "5" #{5}}(params/get-linked-field-ids@@ -231,7 +234,9 @@{:parameter_id "wow" :target [:dimension [:field "wow" {:base-type :type/Integer}]]}{:parameter_id "3" :target [:dimension [:field 3 {:source-field 1}]]}{:parameter_id "4" :target [:dimension [:field 4 {:binning {:strategy :num-bins, :num-bins 1}}]]}- {:parameter_id "5" :target [:dimension [:field 5]]}]}]))))+ {:parameter_id "5" :target [:dimension [:field 5 nil]]}]}])))))++(deftest ^:parallel get-linked-field-ids-test-4(testing "get-linked-field-ids-test no fields"(is (= {}(params/get-linked-field-ids@@ -243,25 +248,21 @@(let [card {:dataset_query (mt/native-query {:query "SELECT *":template-tags{"tag1" {:name "tag1"- :display_name "Tag 1"+ :display-name "Tag 1":id "11111111":type :dimension+ :widget-type :number:dimension [:field (mt/id :orders :id) nil]}"tag2" {:name "tag2"- :display_name "Tag 2"+ :display-name "Tag 2":id "aaaaaaaa":type :dimension+ :widget-type :number:dimension [:field (mt/id :products :id) nil]}}})}]- (testing "card->template-tag-param-id->field-clauses"- (is (= {"11111111" #{[:field (mt/id :orders :id) nil]}- "aaaaaaaa" #{[:field (mt/id :products :id) nil]}}- (#'params/card->template-tag-param-id->field-clauses card))))-(testing "card->template-tag-param-id->field-ids"(is (= {"11111111" #{(mt/id :orders :id)}"aaaaaaaa" #{(mt/id :products :id)}}- (#'params/card->template-tag-param-id->field-ids card))))-+ (#'params/card->template-tag-id->field-ids card))))(testing "card->template-tag-field-ids"(is (= #{(mt/id :orders :id)(mt/id :products :id)}diff --git a/test/metabase/parameters/schema_test.clj b/test/metabase/parameters/schema_test.cljdeleted file mode 100644index 8ebf05421a8..00000000000--- a/test/metabase/parameters/schema_test.clj+++ /dev/null@@ -1,55 +0,0 @@-(ns metabase.parameters.schema-test- (:require- [clojure.test :refer :all]- [metabase.parameters.schema :as parameters.schema]- [metabase.util.malli.registry :as mr]))--(deftest ^:parallel legacy-field-or-expression-reference-test- (testing "Failures"- (are [v] (not (mr/validate ::parameters.schema/legacy-field-or-expression-reference v))- [:aggregation 0]- [:field "name" {}]))- (testing "Successes"- (are [v] (mr/validate ::parameters.schema/legacy-field-or-expression-reference v)- [:field 3 nil]- ["field" "name" {:base-type :type/Float}])))--(deftest ^:parallel parameter-test- (testing "Failures"- (are [v] (not (mr/validate ::parameters.schema/parameter v))- {:id "param-id"- :name "param-name"}- {:id "param-id"- :type "number"- :values_source_type "invalid-type"- :values_source_config {:values [[1 2 3]]}}- {:id "param-id"- :type "number"- :values_source_type "card"- :values_source_config {:card_id 3- :value_field [:aggregation 0]}}))- (testing "Successes"- (are [v] (mr/validate ::parameters.schema/parameter v)- {:id "param-id"- :type "number"- :values_source_type "card"- :values_source_config {:card_id 3- :value_field [:field 3 nil]- :label_field [:field "name" {:base-type :type/Float}]}}- {:id "param-id"- :type "number"- :values_source_type "static-list"- :values_source_config {:values [[1 2 3]]}})))--(deftest ^:parallel parameter-mapping-test- (testing "Failures"- (are [v] (not (mr/validate ::parameters.schema/parameter-mapping v))- {:parameter_id "param-id"}- {:parameter_id "param-id"- :target [:field 3 nil]- :card_id "a"}))- (testing "Successes"- (are [v] (mr/validate ::parameters.schema/parameter-mapping v)- {:parameter_id "param-id"- :target [:field 3 nil]- :card_id 3})))diff --git a/test/metabase/parameters/schema_test.cljc b/test/metabase/parameters/schema_test.cljcnew file mode 100644index 00000000000..7aea7c47543--- /dev/null+++ b/test/metabase/parameters/schema_test.cljc@@ -0,0 +1,19 @@+(ns metabase.parameters.schema-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [metabase.parameters.schema :as parameters.schema]))++(deftest ^:parallel default-to-type-text-test+ (is (= {:id "x", :type :text}+ (parameters.schema/normalize-parameter {:id "x"}))))++(deftest ^:parallel normalize-parameter-from-json-test+ (is (= {:type :text, :id "e7f8ca", :name "variable", :slug "foo_bar", :value 15, :values_source_type :card}+ (parameters.schema/normalize-parameter+ {"id" "e7f8ca", "name" "variable", "slug" "foo_bar", "type" "text", "value" 15, "values_source_type" "card"}))))++(deftest ^:parallel normalize-parameter-mapping-remove-negative-card-ids+ (testing "Apparently some FE code accidentally sent `card_id -1` and now we have to work around it"+ (is (= {:parameter_id "53bb6214", :target [:variable [:template-tag "id"]]}+ (parameters.schema/normalize-parameter-mapping+ {"parameter_id" "53bb6214", "card_id" -1, "target" ["variable" ["template-tag" "id"]]})))))diff --git a/test/metabase/parameters/shared_test.cljc b/test/metabase/parameters/shared_test.cljcindex 74fca3535dc..5655abd2aa8 100644--- a/test/metabase/parameters/shared_test.cljc+++ b/test/metabase/parameters/shared_test.cljc@@ -1,6 +1,6 @@(ns metabase.parameters.shared-test(:require- [clojure.test :as t]+ [clojure.test :refer [deftest is are testing]][metabase.parameters.shared :as params]))(defn- tag-names@@ -9,9 +9,9 @@#?(:clj result:cljs (set (js->clj result)))))-(t/deftest ^:parallel parse-tag-names-test- (t/testing "Tag names are correctly parsed from text card contents"- (t/are [text tags] (= tags (tag-names text))+(deftest ^:parallel parse-tag-names-test+ (testing "Tag names are correctly parsed from text card contents"+ (are [text tags] (= tags (tag-names text));; Valid tags"{{foo}}" #{"foo"}"{{ foo }}" #{"foo"}@@ -38,9 +38,12 @@"{{*foo*}}" #{}"{{&()'}}" #{})))-(t/deftest ^:parallel substitute-tags-test- (t/testing "Tags are correctly substituted into card text, and formatted appropriately based on their type"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(defn- substitute-tags [text tag->param & args]+ (apply params/substitute-tags text (update-vals tag->param #(assoc % :id "_DATE_")) args))++(deftest ^:parallel substitute-tags-test+ (testing "Tags are correctly substituted into card text, and formatted appropriately based on their type"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param))"{{foo}}"{"foo" {:type :string/= :value "bar"}}"bar"@@ -114,10 +117,11 @@"{{foo}}"{"foo" {:type :date/relative :value "thismonth"}}- "This month"))+ "This month")))- (t/testing "Special characters (with semantic meaning in Markdown) are escaped in formatted values"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(deftest ^:parallel substitute-tags-test-2+ (testing "Special characters (with semantic meaning in Markdown) are escaped in formatted values"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param))"{{foo}}"{"foo" {:type :string/= :value "*bar*"}}"\\*bar\\*"@@ -129,10 +133,11 @@;; Characters in the original text are not escaped"_*{{foo}}*_"{"foo" {:type :string/= :value "*bar*"}}- "_*\\*bar\\**_"))+ "_*\\*bar\\**_")))- (t/testing "Special characters (with semantic meaning in Markdown) are not escaped in formatted values when escape-markdown is set to true"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param "en" false))+(deftest ^:parallel substitute-tags-test-3+ (testing "Special characters (with semantic meaning in Markdown) are not escaped in formatted values when escape-markdown is set to true"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param "en" false))"{{foo}}"{"foo" {:type :string/= :value "*bar*"}}"*bar*"@@ -144,10 +149,11 @@;; Characters in the original text are not escaped"_*{{foo}}*_"{"foo" {:type :string/= :value "*bar*"}}- "_**bar**_"))+ "_**bar**_")))- (t/testing "No substitution is done when no parameter is provided, or the parameter is invalid"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(deftest ^:parallel substitute-tags-test-4+ (testing "No substitution is done when no parameter is provided, or the parameter is invalid"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param));; Nil inputnil{}@@ -173,14 +179,15 @@{"foo" {:type :string/=}}"{{foo}}"- ;; Parameter with no type: stringify the value with no additional formatting+ ;; Parameter with no type: stringify the value with no additional formatting (i.e., treat this as a 'text' type+ ;; parameter)"{{foo}}"{"foo" {:value "today"}}"today")))-(t/deftest ^:parallel substitute-tags-date-filters- (t/testing "Basic date values are formatted correctly"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(deftest ^:parallel substitute-tags-date-filters+ (testing "Basic date values are formatted correctly"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param))"{{foo}}"{"foo" {:type :date/single :value "2022-07-09"}}"July 9\\, 2022"@@ -211,10 +218,11 @@"{{foo}}"{"foo" {:type :date/all-options :value "2022-07-06~2022-07-09"}}- "July 6\\, 2022 \\- July 9\\, 2022"))+ "July 6\\, 2022 \\- July 9\\, 2022")))- (t/testing "Exclude options are formatted correctly"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(deftest ^:parallel substitute-tags-date-filters-2+ (testing "Exclude options are formatted correctly"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param))"{{foo}}"{"foo" {:type :date/all-options :value "exclude-months-Jul"}}"Exclude July"@@ -265,10 +273,11 @@"{{foo}}"{"foo" {:type :date/all-options :value "exclude-quarters-2-1-4"}}- "Exclude 3 selections"))+ "Exclude 3 selections")))- (t/testing "Relative date values are formatted correctly"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(deftest ^:parallel substitute-tags-date-filters-3+ (testing "Relative date values are formatted correctly"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param))"{{foo}}"{"foo" {:type :date/all-options :value "thisday"}}"Today"@@ -299,10 +308,11 @@"{{foo}}"{"foo" {:type :date/all-options :value "next5years"}}- "Next 5 years"))+ "Next 5 years")))- (t/testing "Date values are formatted using the locale passed in as an argument"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param "es" true))+(deftest ^:parallel substitute-tags-date-filters-4+ (testing "Date values are formatted using the locale passed in as an argument"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param "es" true))"{{foo}}"{"foo" {:type :date/single :value "2022-07-09"}}"julio 9\\, 2022"@@ -315,9 +325,9 @@{"foo" {:type :date/month-year :value "2019-08"}}"agosto\\, 2019")))-(t/deftest ^:parallel substitute-tags-optional-blocks-test- (t/testing "Optional blocks are removed when necessary"- (t/are [text tag->param expected] (= expected (params/substitute-tags text tag->param))+(deftest ^:parallel substitute-tags-optional-blocks-test+ (testing "Optional blocks are removed when necessary"+ (are [text tag->param expected] (= expected (substitute-tags text tag->param))"[[{{foo}}]]"{}""@@ -361,7 +371,17 @@{"foo" {:type :string/= :value "[[bar]]"}}"\\[\\[bar\\]\\]")))-(t/deftest ^:parallel value-string-test+#?(:cljs+ (deftest ^:parallel substitute-tags-js-test+ (is (= "Variable: 15"+ (params/substitute-tags "Variable: {{variable}}"+ ;; the `:name` here is technically wrong, since it should match with the one that+ ;; serves as the map key; but this code doesn't currently care.+ #js {:variable #js {:id "e7f8ca", :name "foo bar", :slug "foo_bar", :type "text", :value 15}}+ nil+ false)))))++(deftest ^:parallel value-string-test(let [parameters [{:name "State",:slug "state",:id "63e719d0",@@ -380,24 +400,24 @@:id "acd0dfab",:type "string/contains",:sectionId "string"}]]- (t/testing "If a filter has multiple values, they are concatenated into a comma-separated string"- (t/is (= "CA, NY, and NJ"- (params/value-string (first parameters) "en"))))+ (testing "If a filter has multiple values, they are concatenated into a comma-separated string"+ (is (= "CA, NY, and NJ"+ (params/value-string (first parameters) "en"))))- (t/testing "If a filter has a single default value, it is formatted appropriately"- (t/is (= "Q1, 2021"- (params/value-string (second parameters) "en"))))))+ (testing "If a filter has a single default value, it is formatted appropriately"+ (is (= "Q1, 2021"+ (params/value-string (second parameters) "en"))))))-(t/deftest param-val-or-default-test+(deftest param-val-or-default-test(let [param-val-or-default #'params/param-val-or-default]- (t/testing "When the parameter’s :value key is missing, fallback to the :default key"- (t/is (= "my default value"- (param-val-or-default {:default "my default value"}))))- (t/testing "When the parameter’s :value is explicitly nil (i.e. for no-op filters), do not fallback to the :default key"- (t/is (nil? (param-val-or-default {:value nil :default "my default value"}))))))--(t/deftest ^:parallel value-string-contains-test- (t/testing "string/contains parameters are correctly formatted"+ (testing "When the parameter’s :value key is missing, fallback to the :default key"+ (is (= "my default value"+ (param-val-or-default {:default "my default value"}))))+ (testing "When the parameter’s :value is explicitly nil (i.e. for no-op filters), do not fallback to the :default key"+ (is (nil? (param-val-or-default {:value nil :default "my default value"}))))))++(deftest ^:parallel value-string-contains-test+ (testing "string/contains parameters are correctly formatted"(let [format-param (fn [default] (params/value-string {:name "State",:slug "state",:id "63e719d0",@@ -405,9 +425,9 @@:type "string/contains",:sectionId "location"}"en"))]- (t/is (= "contains CA"- (format-param ["CA"])))- (t/is (= "contains CA or NY"- (format-param ["CA" "NY"])))- (t/is (= "contains CA, NY, or NJ"- (format-param ["CA" "NY" "NJ"]))))))+ (is (= "contains CA"+ (format-param ["CA"])))+ (is (= "contains CA or NY"+ (format-param ["CA" "NY"])))+ (is (= "contains CA, NY, or NJ"+ (format-param ["CA" "NY" "NJ"]))))))diff --git a/test/metabase/public_sharing/api_test.clj b/test/metabase/public_sharing/api_test.cljindex 39d851c275c..7eb0f7e8ac0 100644--- a/test/metabase/public_sharing/api_test.clj+++ b/test/metabase/public_sharing/api_test.clj@@ -70,7 +70,7 @@:name "Venue ID":slug "venue_id":type "id"- :target [:dimension (mt/id :venues :id)]+ :target [:dimension (mt/id :venues :id)] ; this is wrong, we have never allowed [:dimension <id>] ...:default nil}]})(shared-obj)m)]@@ -353,7 +353,8 @@{:database (mt/id):type :native:native {:query "SELECT count(*) AS Count FROM venues [[WHERE id = {{venue_id}}]]"- :template-tags {"venue_id" {:name "venue_id"+ :template-tags {"venue_id" {:id "_VENUE_ID_"+ :name "venue_id":display-name "Venue ID":type :number:required false}}}}@@ -398,7 +399,6 @@(client/client :get 202 (str "public/card/" uuid "/query"):parameters (json/encode [{:id "_VENUE_ID_":value 2}]))))))-;; see longer explanation in [[metabase.legacy-mbql.schema/parameter-types]](testing "If the FE client is incorrectly passing in the parameter as a `:category` type, allow it for now"(with-temp-public-card [{uuid :public_uuid} {:dataset_query {:database (mt/id)@@ -475,7 +475,8 @@{:database (mt/id):type :native:native {:query "SELECT count(*) FROM venues v WHERE price = {{price}}"- :template-tags {"price" {:name "price"+ :template-tags {"price" {:id "_PRICE_"+ :name "price":display-name "Price":type :number:required true}}}}}]@@ -693,8 +694,10 @@(mt/with-temporary-setting-values [enable-public-sharing false](with-temp-public-dashboard-and-card [dash card dashcard](is (= "An error occurred."- (client/client :get 400 (dashcard-url dash card dashcard)))))))+ (client/client :get 400 (dashcard-url dash card dashcard)))))))))+(deftest execute-public-dashcard-errors-test-2+ (testing "GET /api/public/dashboard/:uuid/card/:card-id"(testing "Should get a 404"(mt/with-temporary-setting-values [enable-public-sharing true](with-temp-public-dashboard-and-card [dash card dashcard]@@ -844,7 +847,8 @@(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id):type :native:native {:query "SELECT {{num}} AS num"- :template-tags {:num {:name "num"+ :template-tags {:num {:id "01234"+ :name "num":display-name "Num":type "number":required true@@ -927,8 +931,7 @@(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id):type :native:native {:query "SELECT {{msg}} AS message"- :template-tags {:msg {:id "_MSG_-"+ :template-tags {:msg {:id "_MSG_":name "msg":display-name "Message":type "text"@@ -1173,7 +1176,7 @@:model/DashboardCard _ {:dashboard_id (:id dash):card_id (:id card):parameter_mappings [{:parameter_id "_CATEGORY_NAME_"- :target [:dimension (mt/$ids *categories.name)]}]}]+ :target [:dimension [:field "NAME" {:base-type :type/Text}]]}]}](is (=? {:param_fields {(keyword "_CATEGORY_NAME_")[{:semantic_type "type/Name",:table_id (mt/id :categories)@@ -1421,10 +1424,10 @@(is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))(is (= ["CO" "Affiliate" "Gadget" 0 62 211] (nth rows 100)))(is (= [nil nil nil 7 18760 69540] (last rows))))))-(testing "with parameters"(let [result (results :parameters (json/encode [{:name "State":id "_STATE_"+ :type :text:slug :state:target [:dimension (mt/$ids $orders.user_id->people.state)]:value ["CA" "WA"]}]))]diff --git a/test/metabase/pulse/api/pulse_test.clj b/test/metabase/pulse/api/pulse_test.cljindex 3994a512b75..de951312687 100644--- a/test/metabase/pulse/api/pulse_test.clj+++ b/test/metabase/pulse/api/pulse_test.clj@@ -126,7 +126,8 @@{:name "abc":cards ["abc"]}- default-post-card-ref-validation-error+ {:errors {:cards "value must be a map with the keys `include_csv`, `include_xls`, and `dashboard_card_id`.",+ :channels "one or more map"}}{:name "abc":cards [{:id 100, :include_csv false, :include_xls false, :dashboard_card_id nil}@@ -464,7 +465,7 @@default-put-card-ref-validation-error{:cards ["abc"]}- default-put-card-ref-validation-error+ {:errors {:cards "value must be a map with the keys `include_csv`, `include_xls`, and `dashboard_card_id`."}}{:channels 123}{:errors {:channels "nullable one or more map"}}diff --git a/test/metabase/queries/api/card_test.clj b/test/metabase/queries/api/card_test.cljindex 4d54d369fd7..a6e2e59f87d 100644--- a/test/metabase/queries/api/card_test.clj+++ b/test/metabase/queries/api/card_test.clj@@ -17,7 +17,7 @@[metabase.content-verification.models.moderation-review :as moderation-review][metabase.driver :as driver][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -44,6 +44,7 @@[metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json]+ [ring.util.codec :as codec][toucan2.core :as t2])(:import(java.io ByteArrayInputStream)@@ -107,7 +108,7 @@(pmbql-count-query (mt/id) (mt/id :venues)))([db-or-id table-or-id]- (let [metadata-provider (lib.metadata.jvm/application-database-metadata-provider (u/the-id db-or-id))+ (let [metadata-provider (lib-be/application-database-metadata-provider (u/the-id db-or-id))venues (lib.metadata/table metadata-provider (u/the-id table-or-id))query (lib/query metadata-provider venues)](lib/aggregate query (lib/count)))))@@ -125,7 +126,7 @@:dataset_query query:visualization_settings {:global {:title nil}}}))-(defn- do-with-temp-native-card!+(defn- do-with-temp-native-card[f](mt/with-temp [:model/Database db {:details (:details (mt/db)), :engine :h2}:model/Table _ {:db_id (u/the-id db), :name "CATEGORIES"}@@ -134,11 +135,11 @@:native {:query "SELECT COUNT(*) FROM CATEGORIES;"}}}](f db card)))-(defmacro ^:private with-temp-native-card!+(defmacro ^:private with-temp-native-card{:style/indent 1}[[db-binding card-binding] & body]- `(do-with-temp-native-card! (fn [~(or db-binding '_) ~(or card-binding '_)]- ~@body)))+ `(do-with-temp-native-card (fn [~(or db-binding '_) ~(or card-binding '_)]+ ~@body)))(defn do-with-cards-in-a-collection! [card-or-cards-or-ids grant-perms-fn! f](mt/with-non-admin-groups-no-root-collection-perms@@ -166,7 +167,7 @@[card-or-cards-or-ids & body]`(do-with-cards-in-a-collection! ~card-or-cards-or-ids perms/grant-collection-readwrite-permissions! (fn [] ~@body)))-(defn- do-with-temp-native-card-with-params! [f]+(defn- do-with-temp-native-card-with-params [f](mt/with-temp[:model/Database db {:details (:details (mt/db)), :engine :h2}:model/Table _ {:db_id (u/the-id db), :name "VENUES"}@@ -181,13 +182,13 @@:required true}}}}}](f db card)))-(defmacro ^:private with-temp-native-card-with-params! {:style/indent 1} [[db-binding card-binding] & body]- `(do-with-temp-native-card-with-params! (fn [~(or db-binding '_) ~(or card-binding '_)] ~@body)))+(defmacro ^:private with-temp-native-card-with-params {:style/indent 1} [[db-binding card-binding] & body]+ `(do-with-temp-native-card-with-params (fn [~(or db-binding '_) ~(or card-binding '_)] ~@body)))-(deftest run-query-with-parameters-test+(deftest ^:parallel run-query-with-parameters-test(testing "POST /api/card/:id/query"(testing "should respect `:parameters`"- (with-temp-native-card-with-params! [{db-id :id} {card-id :id}]+ (with-temp-native-card-with-params [{db-id :id} {card-id :id}](is (=? {:database_id db-id:row_count 1:data {:rows [[8]]}}@@ -323,7 +324,7 @@"x-metabase-client-version" "2"}}})(is (=? {:embedding_client "client-B", :embedding_version "2"};; The query metadata is handled asynchronously, so we need to poll until it's available:- (tu/poll-until 100+ (tu/poll-until 500(t2/select-one [:model/QueryExecution :embedding_client :embedding_version]:card_id (u/the-id card-1))))))))@@ -448,7 +449,6 @@:user-id user-id:is-creation? true:object {:id card-1-id}}))-(doseq [user-id [(mt/user->id :crowberto) (mt/user->id :rasta)]](revision/push-revision!{:entity :model/Card@@ -607,7 +607,6 @@{:name "Card with dimension is unixtimestmap":visualization_settings {:graph.dimensions ["timestamp"]:graph.metrics ["severity"]}-:display :line:result_metadata [{:base_type :type/BigInteger:coercion_strategy :Coercion/UNIXMilliSeconds->DateTime@@ -621,87 +620,79 @@:semantic_type :type/Number}]})(deftest series-are-compatible-test- (mt/dataset test-data- (let [database-id->metadata-provider {(mt/id) (mt/metadata-provider)}]- (testing "area-line-bar charts"- (mt/with-temp- [:model/Card datetime-card (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query orders {:aggregation [[:sum $orders.total]]- :breakout [!month.orders.created_at]}))- {:visualization_settings {:graph.metrics ["sum"]- :graph.dimensions ["CREATED_AT"]}}- {:name "datetime card"- :display :line})- :model/Card number-card (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query orders {:aggregation [:count]- :breakout [$orders.quantity]}))- {:visualization_settings {:graph.metrics ["count"]- :graph.dimensions ["QUANTITY"]}}- {:name "number card"- :display :line})- :model/Card without-metric-card (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query orders {:breakout [!month.orders.created_at]}))- {:visualization_settings {:graph.dimensions ["CREATED_AT"]}}- {:name "card has no metric"- :display :line})- :model/Card combo-card (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query orders {:aggregation [[:sum $orders.total]]- :breakout [!month.orders.created_at]}))- {:visualization_settings {:graph.metrics ["sum"]- :graph.dimensions ["CREATED_AT"]}}- {:name "table card"- :display :combo})]- (testing "2 datetime cards can be combined"- (is (true? (api.card/series-are-compatible? datetime-card datetime-card database-id->metadata-provider))))-- (testing "2 number cards can be combined"- (is (true? (api.card/series-are-compatible? number-card number-card database-id->metadata-provider))))-- (testing "number card can't be combined with datetime cards"- (is (false? (api.card/series-are-compatible? number-card datetime-card database-id->metadata-provider)))- (is (false? (api.card/series-are-compatible? datetime-card number-card database-id->metadata-provider))))-- (testing "can combine series with UNIX millisecond timestamp and datetime"- (is (true? (api.card/series-are-compatible? millisecond-card datetime-card database-id->metadata-provider)))- (is (true? (api.card/series-are-compatible? datetime-card millisecond-card database-id->metadata-provider))))-- (testing "can't combines series with UNIX milliseceond timestamp and number"- (is (false? (api.card/series-are-compatible? millisecond-card number-card database-id->metadata-provider)))- (is (false? (api.card/series-are-compatible? number-card millisecond-card database-id->metadata-provider))))-- (testing "second card must has a metric"- (is (false? (api.card/series-are-compatible? datetime-card without-metric-card database-id->metadata-provider))))-- (testing "can't combine card of any other types rather than line/bar/area"- (is (nil? (api.card/series-are-compatible? datetime-card combo-card database-id->metadata-provider))))))-- (testing "scalar test"- (mt/with-temp- [:model/Card scalar-1 (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query venues {:aggregation [[:count]]}))- {:name "A Scalar 1" :display :scalar})- :model/Card scalar-2 (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query venues {:aggregation [[:count]]}))- {:name "A Scalar 2" :display :scalar})-- :model/Card scalar-2-cols (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query venues {:aggregation [[:count]- [:sum $venues.price]]}))- {:name "A Scalar with 2 columns" :display :scalar})- :model/Card line-card (merge (mt/card-with-source-metadata-for-query- (mt/mbql-query venues {:aggregation [[:sum $venues.price]]- :breakout [$venues.category_id]}))+ (testing "area-line-bar charts"+ (mt/with-temp+ [:model/Card datetime-card (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query orders {:aggregation [[:sum $orders.total]]+ :breakout [!month.orders.created_at]})){:visualization_settings {:graph.metrics ["sum"]- :graph.dimensions ["CATEGORY_ID"]}}- {:name "Line card" :display :line})]- (testing "2 scalars with 1 column can be combined"- (is (true? (api.card/series-are-compatible? scalar-1 scalar-2 database-id->metadata-provider))))- (testing "can't be combined if either one of 2 cards has more than one column"- (is (false? (api.card/series-are-compatible? scalar-1 scalar-2-cols database-id->metadata-provider)))- (is (false? (api.card/series-are-compatible? scalar-2-cols scalar-2 database-id->metadata-provider))))-- (testing "can only be cominbed with scalar cards"- (is (false? (api.card/series-are-compatible? scalar-1 line-card database-id->metadata-provider)))))))))+ :graph.dimensions ["CREATED_AT"]}}+ {:name "datetime card"+ :display :line})+ :model/Card number-card (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query orders {:aggregation [:count]+ :breakout [$orders.quantity]}))+ {:visualization_settings {:graph.metrics ["count"]+ :graph.dimensions ["QUANTITY"]}}+ {:name "number card"+ :display :line})+ :model/Card without-metric-card (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query orders {:breakout [!month.orders.created_at]}))+ {:visualization_settings {:graph.dimensions ["CREATED_AT"]}}+ {:name "card has no metric"+ :display :line})+ :model/Card combo-card (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query orders {:aggregation [[:sum $orders.total]]+ :breakout [!month.orders.created_at]}))+ {:visualization_settings {:graph.metrics ["sum"]+ :graph.dimensions ["CREATED_AT"]}}+ {:name "table card"+ :display :combo})]+ (testing "2 datetime cards can be combined"+ (is (true? (api.card/series-are-compatible? datetime-card datetime-card))))+ (testing "2 number cards can be combined"+ (is (true? (api.card/series-are-compatible? number-card number-card))))+ (testing "number card can't be combined with datetime cards"+ (is (false? (api.card/series-are-compatible? number-card datetime-card)))+ (is (false? (api.card/series-are-compatible? datetime-card number-card))))+ (testing "can combine series with UNIX millisecond timestamp and datetime"+ (is (true? (api.card/series-are-compatible? millisecond-card datetime-card)))+ (is (true? (api.card/series-are-compatible? datetime-card millisecond-card))))+ (testing "can't combines series with UNIX milliseceond timestamp and number"+ (is (false? (api.card/series-are-compatible? millisecond-card number-card)))+ (is (false? (api.card/series-are-compatible? number-card millisecond-card))))+ (testing "second card must has a metric"+ (is (false? (api.card/series-are-compatible? datetime-card without-metric-card))))+ (testing "can't combine card of any other types rather than line/bar/area"+ (is (nil? (api.card/series-are-compatible? datetime-card combo-card)))))))++(deftest series-are-compatible-test-2+ (testing "scalar test"+ (mt/with-temp+ [:model/Card scalar-1 (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query venues {:aggregation [[:count]]}))+ {:name "A Scalar 1" :display :scalar})+ :model/Card scalar-2 (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query venues {:aggregation [[:count]]}))+ {:name "A Scalar 2" :display :scalar})++ :model/Card scalar-2-cols (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query venues {:aggregation [[:count]+ [:sum $venues.price]]}))+ {:name "A Scalar with 2 columns" :display :scalar})+ :model/Card line-card (merge (mt/card-with-source-metadata-for-query+ (mt/mbql-query venues {:aggregation [[:sum $venues.price]]+ :breakout [$venues.category_id]}))+ {:visualization_settings {:graph.metrics ["sum"]+ :graph.dimensions ["CATEGORY_ID"]}}+ {:name "Line card" :display :line})]+ (testing "2 scalars with 1 column can be combined"+ (is (true? (api.card/series-are-compatible? scalar-1 scalar-2))))+ (testing "can't be combined if either one of 2 cards has more than one column"+ (is (false? (api.card/series-are-compatible? scalar-1 scalar-2-cols)))+ (is (false? (api.card/series-are-compatible? scalar-2-cols scalar-2))))+ (testing "can only be cominbed with scalar cards"+ (is (false? (api.card/series-are-compatible? scalar-1 line-card)))))));;; +----------------------------------------------------------------------------------------------------------------+;;; | CREATING A CARD (POST /api/card) |@@ -709,30 +700,27 @@(deftest ^:parallel doc-test(testing "Make sure generated docstring resolves Malli schemas in the registry correctly (#46799)"- (let [openapi-object (open-api/open-api-spec (api.macros/ns-handler 'metabase.queries.api.card) "/api/card")- schemas (get-in openapi-object [:components :schemas])- body-properties (get-in openapi-object [:paths "/api/card/" :post :requestBody :content "application/json" :schema :properties])- _ (is (some? body-properties))- type-schema-ref (some-> (get-in body-properties ["type" :$ref])- (str/replace #"^#/components/schemas/" "")- (str/replace #"\Q~1\E" "/"))- _ (is (some? type-schema-ref))- type-schema (get schemas type-schema-ref)- result-metadata-schema-ref (some-> (get-in body-properties ["result_metadata" :$ref])- (str/replace #"^#/components/schemas/" "")- (str/replace #"\Q~1\E" "/"))- _ (is (some? result-metadata-schema-ref))- result-metadata-schema (get schemas result-metadata-schema-ref)]+ (let [openapi-object (open-api/open-api-spec (api.macros/ns-handler 'metabase.queries.api.card) "/api/card")+ schemas (get-in openapi-object [:components :schemas])+ body-properties (get-in openapi-object [:paths "/api/card/" :post :requestBody :content "application/json" :schema :properties])+ _ (is (some? body-properties))+ resolve-schema (fn resolve-schema [schema]+ (or (some-> schema+ :$ref+ (str/replace #"^#/components/schemas/" "")+ (->> (get schemas))+ resolve-schema)+ schema))+ type-schema (resolve-schema (get body-properties "type"))+ result-metadata-schema (resolve-schema (get body-properties "result_metadata"))](testing 'type- (testing (pr-str type-schema-ref)- (is (=? {:type :string, :enum [:question :metric :model]}- type-schema))))+ (is (=? {:type :string, :enum [:question :metric :model]}+ type-schema)))(testing 'result_metadata- (testing (pr-str result-metadata-schema-ref)- (is (=? {:type :array- :description "value must be an array of valid results column metadata maps."- :optional true}- result-metadata-schema)))))))+ (is (=? {:type :array+ :description "value must be an array of valid results column metadata maps."+ :optional true}+ result-metadata-schema))))))(deftest create-a-card(testing "POST /api/card"@@ -749,7 +737,7 @@:collection_id (u/the-id collection):parameters [{:id "abc123", :name "test", :type "date"}]:parameter_mappings [{:parameter_id "abc123", :card_id 10,- :target [:dimension [:template-tags "category"]]}])]+ :target [:dimension [:template-tag "category"]]}])](is (=? (mergecard-defaults{:name (:name card)@@ -758,7 +746,7 @@:creator_id (mt/user->id :rasta):parameters [{:id "abc123", :name "test", :type "date"}]:parameter_mappings [{:parameter_id "abc123", :card_id 10,- :target ["dimension" ["template-tags" "category"]]}]+ :target ["dimension" ["template-tag" "category"]]}]:dataset_query true:query_type "query":visualization_settings {:global {:title nil}}@@ -819,7 +807,7 @@(deftest create-card-validation-test-2(testing "POST /api/card"- (with-temp-native-card-with-params! [db card]+ (with-temp-native-card-with-params [db card](testing "You cannot create a card with variables as a model"(is (= "A model made from a native SQL question cannot have a variable or field filter."(mt/user-http-request :rasta :post 400 "card"@@ -937,7 +925,7 @@(deftest save-card-with-empty-result-metadata-test(testing "we should be able to save a Card if the `result_metadata` is *empty* (but not nil) (#9286)"(mt/with-model-cleanup [:model/Card]- (let [card (card-with-name-and-query)]+ (let [card (card-with-name-and-query)](is (=? {:id pos-int?}(mt/user-http-request :crowberto:post@@ -945,7 +933,7 @@"card"(assoc card :result_metadata []))))))))-(deftest cache-ttl-save+(deftest cache-ttl-save-1(testing "POST /api/card/:id"(testing "saving cache ttl by post actually saves it"(mt/with-model-cleanup [:model/Card]@@ -955,7 +943,9 @@:post200"card"- (assoc card :cache_ttl 1234)))))))))+ (assoc card :cache_ttl 1234))))))))))++(deftest cache-ttl-save-2(testing "PUT /api/card/:id"(testing "saving cache ttl by put actually saves it"(mt/with-temp [:model/Card card]@@ -964,7 +954,10 @@:put200(str "card/" (u/the-id card))- {:cache_ttl 1234}))))))+ {:cache_ttl 1234}))))))))++(deftest cache-ttl-save-3+ (testing "PUT /api/card/:id"(testing "nilling out cache ttl works"(mt/with-temp [:model/Card card](is (= nil@@ -1061,11 +1054,11 @@(is (= 1@called)))))))))-(deftest updating-card-updates-metadata-3+(deftest ^:parallel updating-card-updates-metadata-3(let [query (updating-card-updates-metadata-query)](testing "Patching the card _without_ the query does not clear the metadata";; in practice the application does not do this. But cypress does and it poisons the state of the frontend- (mt/with-model-cleanup [:model/Card]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [card (mt/user-http-request :crowberto :post 200 "card"(card-with-name-and-query "card-name"query))]@@ -1075,10 +1068,10 @@:type :model})](is (= ["ID" "NAME"] (map norm (:result_metadata updated))))))))))-(deftest updating-card-updates-metadata-4+(deftest ^:parallel updating-card-updates-metadata-4(let [query (updating-card-updates-metadata-query)](testing "You can update just the metadata"- (mt/with-model-cleanup [:model/Card]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [card (mt/user-http-request :crowberto :post 200 "card"(card-with-name-and-query "card-name"query))]@@ -1089,14 +1082,14 @@(is (= ["UPDATED" "UPDATED"](map :display_name (:result_metadata updated))))))))))-(deftest updating-native-card-preserves-metadata+(deftest ^:parallel updating-native-card-preserves-metadata(testing "A trivial change in a native question should not remove result_metadata (#37009)"(let [query (to-native (updating-card-updates-metadata-query))updated-query (update-in query [:native :query] str/replace #"\d+$" "1000")];; sanity check(is (not= query updated-query));; the actual test- (mt/with-model-cleanup [:model/Card]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [card (mt/user-http-request :rasta :post 200 "card"(card-with-name-and-query "card-name"query))@@ -1183,7 +1176,12 @@(testing "Permissions errors should be meaningful and include info for debugging (#14931)"(is (malli= [:map[:message [:= "You cannot save this Question because you do not have permissions to run its query."]]- [:query [:= {} (mt/obj->json->obj query)]]+ [:query [:map+ [:lib/type [:= "mbql/query"]]+ [:stages [:sequential+ {:min 1, :max 1}+ [:map+ [:source-table [:= (mt/id :venues)]]]]]]][:required-perms :map][:actual-perms [:sequential perms.u/PathSchema]][:trace [:sequential :any]]]@@ -1220,7 +1218,6 @@:result_metadata base-metadata}(mt/user-http-request :crowberto :get 200 (str "card/" (:id card))))"initial result_metadata is inferred correctly")-(is (=? {:type "model":result_metadata base-metadata}(mt/user-http-request :crowberto :put 200 (str "card/" (:id card)) {:type "model"})))@@ -1279,7 +1276,7 @@(testing "can update a metric"(mt/with-temp [:model/Card card {:dataset_query (mbql-count-query):type "metric"}]- (is (=? {:dataset_query {:query {:source-table (mt/id :checkins)}}+ (is (=? {:dataset_query {:stages [{:source-table (mt/id :checkins)}]}:type "metric"}(mt/user-http-request :crowberto :put 200 (str "card/" (:id card)){:dataset_query (mbql-count-query (mt/id) (mt/id :checkins))}))))))@@ -1355,7 +1352,9 @@:last_name "Toucan":first_name "Rasta":email "[email protected]"})- :dataset_query (mt/obj->json->obj (:dataset_query card))+ :dataset_query (-> (:dataset_query card)+ (dissoc :lib/metadata)+ mt/obj->json->obj):display "table":query_type "query":visualization_settings {}@@ -1527,16 +1526,18 @@(mt/with-temp [:model/Card card](testing "successfully update with valid parameter_mappings"(is (partial= {:parameter_mappings [{:parameter_id "abc123", :card_id 10,- :target ["dimension" ["template-tags" "category"]]}]}+ :target ["dimension" ["template-tag" "category"]]}]}(mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)){:parameter_mappings [{:parameter_id "abc123", :card_id 10,- :target ["dimension" ["template-tags" "category"]]}]})))))+ :target ["dimension" ["template-tag" "category"]]}]})))))))+(deftest update-card-parameter-mappings-test-2+ (testing "PUT /api/card/:id"(mt/with-temp [:model/Card card {:parameter_mappings [{:parameter_id "abc123", :card_id 10,- :target ["dimension" ["template-tags" "category"]]}]}]+ :target ["dimension" ["template-tag" "category"]]}]}](testing "nil parameters will no-op"(is (partial= {:parameter_mappings [{:parameter_id "abc123", :card_id 10,- :target ["dimension" ["template-tags" "category"]]}]}+ :target ["dimension" ["template-tag" "category"]]}]}(mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)){:parameters nil}))))(testing "an empty list will remove parameter_mappings"@@ -1609,7 +1610,7 @@(deftest update-card-validation-test(testing "PUT /api/card"- (with-temp-native-card-with-params! [_db card]+ (with-temp-native-card-with-params [_db card](testing "You cannot update a model to have variables"(is (= "A model made from a native SQL question cannot have a variable or field filter."(mt/user-http-request :rasta :put 400 (format "card/%d" (:id card)) {:type :model})))))))@@ -1863,119 +1864,140 @@(testing "\nadmin"(testing "*should* be allowed to update query"(is (=? {:id card-id- :dataset_query (mt/obj->json->obj (mt/mbql-query checkins))}+ :dataset_query {:lib/type "mbql/query"+ :stages [{:source-table (mt/id :checkins)}]}}(update-card! :crowberto 200 {:dataset_query (mt/mbql-query checkins)})))))-(testing "\nnon-admin"(testing "should be allowed to update fields besides query"(is (=? {:id card-id:name "Updated name"}(update-card! :rasta 200 {:name "Updated name"}))))-(testing "should *not* be allowed to update query"(testing "Permissions errors should be meaningful and include info for debugging (#14931)"(is (malli= [:map[:message [:= "You cannot save this Question because you do not have permissions to run its query."]]- [:query [:= {} (mt/obj->json->obj (mt/mbql-query users))]]+ [:query [:map+ [:stages [:sequential+ {:min 1, :max 1}+ [:map+ [:source-table [:= (mt/id :users)]]]]]]][:required-perms :map][:actual-perms [:sequential perms.u/PathSchema]][:trace [:sequential :any]]](update-card! :rasta 403 {:dataset_query (mt/mbql-query users)}))))(testing "make sure query hasn't changed in the DB"- (is (= (mt/mbql-query checkins)- (t2/select-one-fn :dataset_query :model/Card :id card-id)))))-+ (is (=? {:lib/type :mbql/query+ :stages [{:source-table (mt/id :checkins)}]}+ (t2/select-one-fn :dataset_query :model/Card :id card-id)))))(testing "should be allowed to update other fields if query is passed in but hasn't changed (##11719)"(is (=? {:id card-id:name "Another new name"- :dataset_query (mt/obj->json->obj (mt/mbql-query checkins))}- (update-card! :rasta 200 {:name "Another new name", :dataset_query (mt/mbql-query checkins)})))))))))))+ :dataset_query {:lib/type "mbql/query"+ :stages [{:source-table (mt/id :checkins)}]}}+ (update-card! :rasta 200 {:name "Another new name", :dataset_query (t2/select-one-fn :dataset_query :model/Card card-id)})))))))))));;; +----------------------------------------------------------------------------------------------------------------+;;; | Card updates that impact alerts |;;; +----------------------------------------------------------------------------------------------------------------++(defn- test-alert-deletion! [{:keys [message card deleted? expected-email-re f]}]+ (testing message+ (notification.tu/with-channel-fixtures [:channel/email]+ (api.notification-test/with-send-messages-sync!+ (notification.tu/with-card-notification+ [notification {:card (merge {:name "YOLO"} card)+ :handlers [{:channel_type :channel/email+ :recipients [{:type :notification-recipient/user+ :user_id (mt/user->id :crowberto)}+ {:type :notification-recipient/user+ :user_id (mt/user->id :rasta)}+ {:type :notification-recipient/raw-value+ :details {:value "[email protected]"}}]}]}]+ (when deleted?+ (let [[email] (notification.tu/with-mock-inbox-email!+ (f (->> notification :payload :card_id (t2/select-one :model/Card))))]+ :subject "One of your alerts has stopped working"+ :body [{(str expected-email-re) true}]}+ (mt/summarize-multipart-single-email email expected-email-re)))))+ (if deleted?+ (is (not (t2/exists? :model/Notification :id (:id notification)))+ "Alert should have been deleted")+ (is (t2/exists? :model/Notification :id (:id notification))+ "Alert should not have been deleted")))))))+(deftest alert-deletion-test- (doseq [{:keys [message card deleted? expected-email-re f]}- [{:message "Archiving a Card should trigger Alert deletion"- :deleted? true- :expected-email-re #"Alerts about [A-Za-z]+ \(#\d+\) have stopped because the question was archived by Rasta Toucan"- :f (fn [card]- (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:archived true}))}- {:message "Validate changing a display type triggers alert deletion"- :card {:display :table}- :deleted? true- :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Rasta Toucan"- :f (fn [card]- (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:display :line}))}- {:message "Changing the display type from line to table should force a delete"- :card {:display :line}- :deleted? true- :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Rasta Toucan"- :f (fn [card]- (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:display :table}))}- {:message "Removing the goal value will trigger the alert to be deleted"- :card {:display :line- :visualization_settings {:graph.goal_value 10}}- :deleted? true- :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Rasta Toucan"- :f (fn [card]- (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:visualization_settings {:something "else"}}))}- {:message "Adding an additional breakout does not cause the alert to be removed if no goal is set"- :card {:display :line- :visualization_settings {}- :dataset_query (assoc-in- (mbql-count-query (mt/id) (mt/id :checkins))- [:query :breakout]- [[:field- (mt/id :checkins :date)- {:temporal-unit :hour}]])}- :deleted? false- :f (fn [card]- (mt/user-http-request :crowberto :put 200 (str "card/" (u/the-id card))- {:dataset_query (assoc-in (mbql-count-query (mt/id) (mt/id :checkins))- [:query :breakout] [[:field (mt/id :checkins :date) {:temporal-unit :hour}]- [:field (mt/id :checkins :date) {:temporal-unit :minute}]])}))}- {:message "Adding an additional breakout will cause the alert to be removed if a goal is set"- :card {:display :line- :visualization_settings {:graph.goal_value 10}- :dataset_query (assoc-in- (mbql-count-query (mt/id) (mt/id :checkins))- [:query :breakout]- [[:field- (mt/id :checkins :date)- {:temporal-unit :hour}]])}- :deleted? true- :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Crowberto Corv"- :f (fn [card]- (mt/user-http-request :crowberto :put 200 (str "card/" (u/the-id card))- {:dataset_query (assoc-in (mbql-count-query (mt/id) (mt/id :checkins))- [:query :breakout] [[:field (mt/id :checkins :date) {:temporal-unit :hour}]- [:field (mt/id :checkins :date) {:temporal-unit :minute}]])}))}]]- (testing message- (notification.tu/with-channel-fixtures [:channel/email]- (api.notification-test/with-send-messages-sync!- (notification.tu/with-card-notification- [notification {:card (merge {:name "YOLO"} card)- :handlers [{:channel_type :channel/email- :recipients [{:type :notification-recipient/user- :user_id (mt/user->id :crowberto)}- {:type :notification-recipient/user- :user_id (mt/user->id :rasta)}- {:type :notification-recipient/raw-value- :details {:value "[email protected]"}}]}]}]- (when deleted?- (let [[email] (notification.tu/with-mock-inbox-email!- (f (->> notification :payload :card_id (t2/select-one :model/Card))))]- :subject "One of your alerts has stopped working"- :body [{(str expected-email-re) true}]}- (mt/summarize-multipart-single-email email expected-email-re)))))- (if deleted?- (is (= nil (t2/select-one :model/Notification :id (:id notification)))- "Alert should have been deleted")- (is (not= nil (t2/select-one :model/Notification :id (:id notification)))- "Alert should not have been deleted"))))))))+ (test-alert-deletion!+ {:message "Archiving a Card should trigger Alert deletion"+ :deleted? true+ :expected-email-re #"Alerts about [A-Za-z]+ \(#\d+\) have stopped because the question was archived by Rasta Toucan"+ :f (fn [card]+ (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:archived true}))}))++(deftest alert-deletion-test-2+ (test-alert-deletion!+ {:message "Validate changing a display type triggers alert deletion"+ :card {:display :table}+ :deleted? true+ :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Rasta Toucan"+ :f (fn [card]+ (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:display :line}))}))++(deftest alert-deletion-test-3+ (test-alert-deletion!+ {:message "Changing the display type from line to table should force a delete"+ :card {:display :line}+ :deleted? true+ :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Rasta Toucan"+ :f (fn [card]+ (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:display :table}))}))++(deftest alert-deletion-test-4+ (test-alert-deletion!+ {:message "Removing the goal value will trigger the alert to be deleted"+ :card {:display :line+ :visualization_settings {:graph.goal_value 10}}+ :deleted? true+ :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Rasta Toucan"+ :f (fn [card]+ (mt/user-http-request :rasta :put 200 (str "card/" (u/the-id card)) {:visualization_settings {:something "else"}}))}))++(deftest alert-deletion-test-5+ (test-alert-deletion!+ {:message "Adding an additional breakout does not cause the alert to be removed if no goal is set"+ :card {:display :line+ :visualization_settings {}+ :dataset_query (assoc-in+ (mbql-count-query (mt/id) (mt/id :checkins))+ [:query :breakout]+ [[:field+ (mt/id :checkins :date)+ {:temporal-unit :month}]])}+ :deleted? false+ :f (fn [card]+ (mt/user-http-request :crowberto :put 200 (str "card/" (u/the-id card))+ {:dataset_query (assoc-in (mbql-count-query (mt/id) (mt/id :checkins))+ [:query :breakout] [[:field (mt/id :checkins :date) {:temporal-unit :month}]+ [:field (mt/id :checkins :date) {:temporal-unit :year}]])}))}))++(deftest alert-deletion-test-6+ (test-alert-deletion!+ {:message "Adding an additional breakout will cause the alert to be removed if a goal is set"+ :card {:display :line+ :visualization_settings {:graph.goal_value 10}+ :dataset_query (assoc-in+ (mbql-count-query (mt/id) (mt/id :checkins))+ [:query :breakout]+ [[:field+ (mt/id :checkins :date)+ {:temporal-unit :month}]])}+ :deleted? true+ :expected-email-re #"Alerts about <a href=\"https?://[^\/]+\/question/\d+\">([^<]+)<\/a> have stopped because the question was edited by Crowberto Corv"+ :f (fn [card]+ (mt/user-http-request :crowberto :put 200 (str "card/" (u/the-id card))+ {:dataset_query (assoc-in (mbql-count-query (mt/id) (mt/id :checkins))+ [:query :breakout] [[:field (mt/id :checkins :date) {:temporal-unit :month}]+ [:field (mt/id :checkins :date) {:temporal-unit :year}]])}))}))(deftest changing-the-display-type-from-line-to-area-bar-is-fine-and-doesnt-delete-the-alert(doseq [{:keys [message display]}@@ -2030,7 +2052,7 @@(deftest csv-download-test(testing "no parameters"- (with-temp-native-card! [_ card]+ (with-temp-native-card [_ card](with-cards-in-readable-collection! card(let [response (mt/user-http-request :rasta :post 200 (format "card/%d/query/csv" (u/the-id card)))](is (= ["COUNT(*)"@@ -2040,7 +2062,7 @@(deftest csv-download-test-2(testing "with parameters"- (with-temp-native-card-with-params! [_ card]+ (with-temp-native-card-with-params [_ card](with-cards-in-readable-collection! card(let [response (mt/user-http-request :rasta :post 200 (format "card/%d/query/csv" (u/the-id card)){:parameters test-params})]@@ -2051,14 +2073,14 @@(deftest json-download-test(testing "no parameters"- (with-temp-native-card! [_ card]+ (with-temp-native-card [_ card](with-cards-in-readable-collection! card(is (= [{(keyword "COUNT(*)") "75"}](mt/user-http-request :rasta :post 200 (format "card/%d/query/json" (u/the-id card)) {:format_rows true})))))))(deftest json-download-test-2(testing "with parameters"- (with-temp-native-card-with-params! [_ card]+ (with-temp-native-card-with-params [_ card](with-cards-in-readable-collection! card(is (= [{(keyword "COUNT(*)") "8"}](mt/user-http-request :rasta :post 200 (format "card/%d/query/json" (u/the-id card))@@ -2161,7 +2183,7 @@(deftest xlsx-download-test(testing "no parameters"- (with-temp-native-card! [_ card]+ (with-temp-native-card [_ card](with-cards-in-readable-collection! card(is (= [{:col "COUNT(*)"} {:col 75.0}](parse-xlsx-results@@ -2169,7 +2191,7 @@(deftest xlsx-download-test-2(testing "with parameters"- (with-temp-native-card-with-params! [_ card]+ (with-temp-native-card-with-params [_ card](with-cards-in-readable-collection! card(is (= [{:col "COUNT(*)"} {:col 8.0}](parse-xlsx-results@@ -2178,7 +2200,7 @@(defn- parse-xlsx-results-to-strings"Parse an excel response into a 2-D array of formatted values"- [results]+ [^bytes results](let [df (DataFormatter.)](->> resultsByteArrayInputStream.@@ -2188,7 +2210,7 @@(mapv (fn [row](mapv (fn [cell] (.formatCellValue df cell)) (spreadsheet/cell-seq row)))))))-(deftest xlsx-timestamp-formatting-test+(deftest ^:parallel xlsx-timestamp-formatting-test(testing "A timestamp should format correctly in an excel export (#14393)"(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id):type :native@@ -2204,7 +2226,7 @@(mt/user-http-request :rasta :post 200 (format "card/%d/query/xlsx" (u/the-id card)){:format_rows true}))))))))-(deftest xlsx-default-currency-formatting-test+(deftest ^:parallel xlsx-default-currency-formatting-test(testing "The default currency is USD"(mt/with-temp [:model/Card card {:dataset_query {:database (mt/id):type :native@@ -2463,7 +2485,7 @@(update "Content-Disposition" #(some-> % (str/replace #"my_awesome_card_.+(\.\w+)""my_awesome_card_<timestamp>$1")))))-(deftest download-response-headers-test+(deftest ^:parallel download-response-headers-test(testing "Make sure CSV/etc. download requests come back with the correct headers"(mt/with-temp [:model/Card card {:name "My Awesome Card"}](is (= {"Cache-Control" "max-age=0, no-cache, must-revalidate, proxy-revalidate"@@ -2611,10 +2633,9 @@(testing "Changing core attributes un-verifies the card"(with-card :verified(is (verified? card))- (is (=? {:dataset_query {:query {:source-table pos-int?}}}- card)- "(This test is written to expect a legacy query)")- (update-card card (update-in card [:dataset_query :query :source-table] inc))+ (is (=? {:dataset_query {:stages [{:source-table pos-int?}]}}+ card))+ (update-card card (update-in card [:dataset_query :stages 0 :source-table] inc))(is (not (verified? card)))(testing "The unverification edit has explanatory text"(is (= "Unverified due to edit"@@ -3060,8 +3081,8 @@(param-values-url card-or-id param-key nil))([card-or-id param-key query](if query- (format "card/%d/params/%s/search/%s" (u/the-id card-or-id) (name param-key) query)- (format "card/%d/params/%s/values" (u/the-id card-or-id) (name param-key)))))+ (format "card/%d/params/%s/search/%s" (u/the-id card-or-id) (codec/form-encode (name param-key)) query)+ (format "card/%d/params/%s/values" (u/the-id card-or-id) (codec/form-encode (name param-key))))))(defn do-with-card-param-values-fixtures"Impl of `with-card-param-values-fixtures` macro."@@ -3144,7 +3165,7 @@[[binding card-values] & body]`(do-with-card-param-values-fixtures ~card-values (fn [~binding] ~@body)))-(deftest parameter-remapping-test+(deftest ^:parallel parameter-remapping-test(with-card-param-values-fixtures [{:keys [card field-filter-card name-mapped-card param-keys]}](letfn [(request [{:keys [id] :as _card} value-source value](mt/user-http-request :crowberto :get 200@@ -3160,7 +3181,7 @@(is (= ["African" "Af"] (request card :static-list-label "African")))(is (= [42 "Reyes Strosin"] (request name-mapped-card :labeled-field-values "42"))))))-(deftest parameters-with-source-is-card-test+(deftest ^:parallel parameters-with-source-is-card-test(testing "getting values"(binding [custom-values/*max-rows* 5](with-card-param-values-fixtures [{:keys [card param-keys]}]@@ -3280,7 +3301,7 @@(is (not ((into #{} (mapcat identity) (:values response)) "The Virgil")))))))))(deftest parameters-with-field-to-field-remapping-test- (let [param-key "id_param_id"]+ (let [param-key "id/param"](mt/with-temp[:model/Card card {:dataset_query{:database (mt/id)@@ -3668,15 +3689,18 @@(deftest ^:parallel query-metadata-test(mt/with-temp[:model/Card {card-id-1 :id} {:dataset_query (mt/mbql-query products)- :database_id (mt/id)}- :model/Card {card-id-2 :id} (native-card-with-template-tags)]+ :database_id (mt/id)}](testing "Simple card"(is (=?{:fields empty?:tables (sort-by :id [{:id (mt/id :products)}]):databases [{:id (mt/id) :engine string?}]}(-> (mt/user-http-request :crowberto :get 200 (str "card/" card-id-1 "/query_metadata"))- (api.test-util/select-query-metadata-keys-for-debugging)))))+ (api.test-util/select-query-metadata-keys-for-debugging)))))))++(deftest ^:parallel query-metadata-test-2+ (mt/with-temp+ [:model/Card {card-id-2 :id} (native-card-with-template-tags)](testing "Parameterized native query"(is (=?{:fields (sort-by :id@@ -4526,3 +4550,15 @@(finally(when existing-config(t2/insert! :model/CacheConfig existing-config)))))))++(deftest ^:parallel return-legacy-query-test+ (testing "GET /api/card/:id?legacy-mbql=true"+ (mt/with-temp [:model/Card card {:dataset_query (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/aggregate (lib/count))))}]+ (is (=? {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :orders)+ :aggregation [["count"]]}}+ (-> (mt/user-http-request :crowberto :get 200 (str "card/" (:id card)) :legacy-mbql true)+ :dataset_query))))))diff --git a/test/metabase/queries/metadata_test.clj b/test/metabase/queries/metadata_test.cljnew file mode 100644index 00000000000..bfeae8a3729--- /dev/null+++ b/test/metabase/queries/metadata_test.clj@@ -0,0 +1,11 @@+(ns metabase.queries.metadata-test+ (:require+ [clojure.test :refer :all]+ [metabase.queries.metadata :as queries.metadata]+ [metabase.util.malli :as mu]))++(deftest ^:parallel batch-fetch-card-metadata-empty-queries-test+ ;; disable Malli because we want to make sure this works in prod+ (mu/disable-enforcement+ (is (= {:databases [], :fields [], :snippets [], :tables []}+ (queries.metadata/batch-fetch-card-metadata [{}])))))diff --git a/test/metabase/queries/models/card_test.clj b/test/metabase/queries/models/card_test.cljindex 9df5130e91e..f43e5252a24 100644--- a/test/metabase/queries/models/card_test.clj+++ b/test/metabase/queries/models/card_test.clj@@ -112,19 +112,22 @@(mt/with-actions-enabled(testing "when updating a model to include any clauses will disable implicit actions if they exist\n"(testing "happy paths\n"- (let [base (mt/mbql-query users)]- (doseq [query-change [{:limit 1}- {:expressions {"id + 1" [:+ (mt/$ids $users.id) 1]}}- {:filter [:> (mt/$ids $users.id) 2]}- {:breakout [(mt/$ids !month.users.last_login)]}- {:aggregation [[:count]]}- {:joins [{:fields :all- :source-table (mt/id :checkins)- :condition [:= (mt/$ids $users.id) (mt/$ids $checkins.user_id)]- :alias "People"}]}- {:order-by [[(mt/$ids $users.id) :asc]]}- {:fields [(mt/$ids $users.id)]}]]- (testing (format "when adding %s to the query" (first (keys query-change)))+ (let [mp (mt/metadata-provider)+ base (lib/query+ mp+ (mt/mbql-query users))]+ (doseq [[f & args] [[#'lib/limit 1]+ [#'lib/expression "id + 1" (lib/+ (lib.metadata/field mp (mt/id :users :id)) 1)]+ [#'lib/filter (lib/> (lib.metadata/field mp (mt/id :users :id)) 2)]+ [#'lib/breakout (-> (lib.metadata/field mp (mt/id :users :last_login))+ (lib/with-temporal-bucket :month))]+ [#'lib/aggregate (lib/count)]+ [#'lib/join (-> (lib/join-clause (lib.metadata/table mp (mt/id :checkins)))+ (lib/with-join-fields :all)+ (lib/with-join-alias "People"))]+ [#'lib/order-by (lib.metadata/field mp (mt/id :users :id))]+ [#'lib/with-fields [(lib.metadata/field mp (mt/id :users :id))]]]]+ (testing (format "when applying %s to the query" (pr-str f))(mt/with-actions [{model-id :idquery :dataset_query} {:type :model, :dataset_query base}{action-id-1 :action-id} {:type :implicit@@ -133,13 +136,13 @@:kind "row/update"}];; make sure we have thing exists to start with(is (= 2 (t2/count :model/Action :id [:in [action-id-1 action-id-2]])))- (is (= 1 (t2/update! :model/Card :id model-id {:dataset_query (update query :query merge query-change)})))+ (is (= 1 (t2/update! :model/Card :id model-id {:dataset_query (apply f query args)})));; should be gone by now(is (= 0 (t2/count :model/Action :id [:in [action-id-1 action-id-2]])))(is (= 0 (t2/count :model/ImplicitAction :action_id [:in [action-id-1 action-id-2]])))- ;; call it twice to make we don't get delete error if no actions are found- ;; Returns zero because there are no actual changes happening here- (is (= 0 (t2/update! :model/Card :id model-id {:dataset_query (update query :query merge query-change)})))))))))))+ ;; call it twice to make we don't get delete error if no actions are found Returns either zero or one+ ;; depending on the change because the query will possibly have different UUIDs+ (is (#{0 1} (t2/update! :model/Card :id model-id {:dataset_query (apply f query args)})))))))))))(deftest disable-implicit-actions-if-needed-test-2(mt/with-actions-enabled@@ -404,7 +407,7 @@(testing "creating"(is (thrown-with-msg?clojure.lang.ExceptionInfo- #":parameters must be a sequence of maps with :id and :type keys"+ #"Invalid output: \{:parameters \[\"invalid type, got: \{:a :b\}\"\]\}"(mt/with-temp [:model/Card _ {:parameters {:a :b}}])))(mt/with-temp [:model/Card card {:parameters [{:id "valid-id":type "id"}]}]@@ -416,7 +419,7 @@(mt/with-temp [:model/Card {:keys [id]} {:parameters []}](is (thrown-with-msg?clojure.lang.ExceptionInfo- #":parameters must be a sequence of maps with :id and :type keys"+ #"Invalid output:.*:parameters"(t2/update! :model/Card id {:parameters [{:id 100}]})))(is (pos? (t2/update! :model/Card id {:parameters [{:id "new-valid-id":type "id"}]})))))))@@ -438,7 +441,7 @@(testing "creating"(is (thrown-with-msg?clojure.lang.ExceptionInfo- #":parameter_mappings must be a sequence of maps with :parameter_id and :type keys"+ #"Invalid output:.*:parameter_mappings"(mt/with-temp [:model/Card _ {:parameter_mappings {:a :b}}])))(mt/with-temp [:model/Card card {:parameter_mappings [{:parameter_id "valid-id":target [:field 1000 nil]}]}]@@ -450,12 +453,12 @@(mt/with-temp [:model/Card {:keys [id]} {:parameter_mappings []}](is (thrown-with-msg?clojure.lang.ExceptionInfo- #":parameter_mappings must be a sequence of maps with :parameter_id and :type keys"+ #"Invalid output:.*:parameter_mappings"(t2/update! :model/Card id {:parameter_mappings [{:parameter_id 100}]})))(is (pos? (t2/update! :model/Card id {:parameter_mappings [{:parameter_id "new-valid-id":target [:field 1000 nil]}]})))))))-(deftest normalize-parameter-mappings-test+(deftest ^:parallel normalize-parameter-mappings-test(testing ":parameter_mappings should get normalized when coming out of the DB"(mt/with-temp [:model/Card {card-id :id} {:parameter_mappings [{:parameter_id "22486e00":card_id 1@@ -465,7 +468,7 @@:target [:dimension [:field 1 nil]]}](t2/select-one-fn :parameter_mappings :model/Card :id card-id))))))-(deftest identity-hash-test+(deftest ^:parallel identity-hash-test(testing "Card hashes are composed of the name and the collection's hash"(let [now #t "2022-09-01T12:34:56Z"](mt/with-temp [:model/Collection coll {:name "field-db" :location "/" :created_at now}@@ -600,7 +603,7 @@(is (=? [{:name "Param 1":id "param_1":type :category- :values_source_type "card"+ :values_source_type :card:values_source_config {:card_id source-card-id:value_field (mt/$ids $products.title)}}](t2/select-one-fn :parameters :model/Card :id (:id card)))))))@@ -830,7 +833,7 @@:database_id (mt/id)}(t2/select-one :model/Card :id (u/the-id card))))))))))-(deftest can-run-adhoc-query-test+(deftest ^:parallel can-run-adhoc-query-test(let [metadata-provider (mt/metadata-provider)venues (lib.metadata/table metadata-provider (mt/id :venues))query (lib/query metadata-provider venues)]@@ -859,43 +862,48 @@(is (false? (mi/can-read? card)))(is (false? (mi/can-write? card)))))))))-(deftest breakouts-->identifier->action-fn-test- (are [b1 b2 expected--identifier->action] (= expected--identifier->action- (#'card/breakouts-->identifier->action b1 b2))- [[:field 10 {:temporal-unit :day}]]+(deftest ^:parallel breakouts->identifier->action-fn-test+ (are [b1 b2 expected-identifier->action] (=? expected-identifier->action+ (#'card/breakouts->identifier->action+ (map lib/normalize b1)+ (map lib/normalize b2)))+ [[:field {:temporal-unit :day} 10]]nilnil- [[:expression "x" {:temporal-unit :day}]]+ [[:expression {:temporal-unit :day} "x"]]nilnil- [[:expression "x" {:temporal-unit :day}]]- [[:expression "x" {:temporal-unit :month}]]- {[:expression "x"] [:update [:expression "x" {:temporal-unit :month}]]}+ [[:expression {:temporal-unit :day} "x"]]+ [[:expression {:temporal-unit :month} "x"]]+ {[:expression "x"] [:update [:expression {:temporal-unit :month} "x"]]}- [[:expression "x" {:temporal-unit :day}]]- [[:expression "x" {:temporal-unit :day}]]+ [[:expression {:temporal-unit :day} "x"]]+ [[:expression {:temporal-unit :day} "x"]]nil- [[:field 10 {:temporal-unit :day}] [:expression "x" {:temporal-unit :day}]]- [[:expression "x" {:temporal-unit :day}] [:field 10 {:temporal-unit :month}]]- {[:field 10] [:update [:field 10 {:temporal-unit :month}]]}+ [[:field {:temporal-unit :day} 10]+ [:expression {:temporal-unit :day} "x"]]+ [[:expression {:temporal-unit :day} "x"]+ [:field {:temporal-unit :month} 10]]+ {[:field 10] [:update [:field {:temporal-unit :month} 10]]}- [[:field 10 {:temporal-unit :year}] [:field 10 {:temporal-unit :day-of-week}]]- [[:field 10 {:temporal-unit :year}]]+ [[:field {:temporal-unit :year} 10]+ [:field {:temporal-unit :day-of-week} 10]]+ [[:field {:temporal-unit :year} 10]]nil))-(deftest update-for-dashcard-fn-test+(deftest ^:parallel update-for-dashcard-fn-test(are [indetifier->action quasi-dashcards expected-quasi-dashcards](= expected-quasi-dashcards(#'card/updates-for-dashcards indetifier->action quasi-dashcards))- {[:field 10] [:update [:field 10 {:temporal-unit :month}]]}+ {[:field 10] [:update [:field {:temporal-unit :month} 10]]}[{:parameter_mappings []}]nil- {[:field 10] [:update [:field 10 {:temporal-unit :month}]]}+ {[:field 10] [:update [:field {:temporal-unit :month} 10]]}[{:id 1 :parameter_mappings [{:target [:dimension [:field 10 nil]]}]}][[1 {:parameter_mappings [{:target [:dimension [:field 10 {:temporal-unit :month}]]}]}]]@@ -903,7 +911,7 @@[{:id 1 :parameter_mappings [{:target [:dimension [:field 10 nil]]}]}]nil- {[:field 10] [:update [:field 10 {:temporal-unit :month}]]}+ {[:field 10] [:update [:field {:temporal-unit :month} 10]]}[{:id 1 :parameter_mappings [{:target [:dimension [:field 10 {:temporal-unit :year}]]}{:target [:dimension [:field 33 {:temporal-unit :month}]]}{:target [:dimension [:field 10 {:temporal-unit :day}]]}]}]@@ -1034,8 +1042,8 @@(t2/update! :model/Card card-id {:dataset_query unnormalized-query});; Verify the query was normalized (field-id -> field)(let [updated-query (t2/select-one-fn :dataset_query :model/Card :id card-id)]- (is (= [:= [:field (mt/id :venues :name) nil] "Test"]- (get-in updated-query [:query :filter]))))))))+ (is (=? [:= {} [:field {} (mt/id :venues :name)] "Test"]+ (get-in updated-query [:stages 0 :filters 0]))))))))(deftest before-update-query-fields-population-test(testing "populate-query-fields is called"@@ -1086,3 +1094,10 @@(let [doc (first search-docs)](testing "native-query field is nil for non-native queries"(is (nil? (:native_query doc)))))))))++(deftest normalize-card-on-update-test+ (mt/with-temp [:model/Card card {:name "some card", :type "model"}]+ (let [card' (assoc card :type "question")]+ (t2/save! card')+ (is (= :question+ (t2/select-one-fn :type :model/Card :id (:id card)))))))diff --git a/test/metabase/queries/models/parameter_card_test.clj b/test/metabase/queries/models/parameter_card_test.cljindex c0661820de8..d09482a1cc2 100644--- a/test/metabase/queries/models/parameter_card_test.clj+++ b/test/metabase/queries/models/parameter_card_test.clj@@ -5,13 +5,14 @@[metabase.test :as mt][toucan2.core :as t2]))-(deftest validate-parameterized-object-type-test+(deftest ^:parallel validate-parameterized-object-type-test(testing "valid parameterized_object_type values are accepted"(is (nil? (#'parameter-card/validate-parameterized-object-type{:parameterized_object_type "dashboard"})))(is (nil? (#'parameter-card/validate-parameterized-object-type- {:parameterized_object_type "card"}))))+ {:parameterized_object_type "card"})))))+(deftest ^:parallel validate-parameterized-object-type-test-2(testing "invalid parameterized_object_type throws exception"(is (thrown-with-msg?clojure.lang.ExceptionInfo@@ -33,13 +34,11 @@:parameterized_object_id 2:parameter_id "param3":card_id card-id}]-(testing "deletes all ParameterCards for given object when no exclusions"(parameter-card/delete-all-for-parameterized-object! "dashboard" 1)(is (nil? (t2/select-one :model/ParameterCard :id pc1-id)))(is (nil? (t2/select-one :model/ParameterCard :id pc2-id)))(is (some? (t2/select-one :model/ParameterCard :id pc3-id))))-(testing "preserves ParameterCards with parameter IDs still in use"(parameter-card/delete-all-for-parameterized-object! "dashboard" 2 ["param3"])(is (some? (t2/select-one :model/ParameterCard :id pc3-id))))))@@ -48,7 +47,6 @@(mt/with-temp [:model/Database {db-id :id} {}:model/Card {card-id-1 :id} {:database_id db-id}:model/Card {card-id-2 :id} {:database_id db-id}]-(testing "creates new ParameterCards for parameters with card sources"(let [parameters [{:id "param1":values_source_type "card"@@ -57,7 +55,6 @@:values_source_type "card":values_source_config {:card_id card-id-2}}]](#'parameter-card/upsert-from-parameters! "dashboard" 1 parameters)-(let [pc1 (t2/select-one :model/ParameterCard:parameterized_object_type "dashboard":parameterized_object_id 1@@ -83,13 +80,11 @@:values_source_config {:card_id card-id-1}}]];; test that we ran without throwing(is (nil? (#'parameter-card/upsert-from-parameters! "dashboard" 1 parameters)))))-(testing "updates existing ParameterCard"(let [parameters [{:id "param1":values_source_type "card":values_source_config {:card_id card-id-2}}]](#'parameter-card/upsert-from-parameters! "dashboard" 1 parameters)-(let [pc (t2/select-one :model/ParameterCard:parameterized_object_type "dashboard":parameterized_object_id 1@@ -100,22 +95,20 @@(mt/with-temp [:model/Card {card-id-1 :id} {}:model/Card {card-id-2 :id} {}:model/Card {card-id-3 :id} {}]-(testing "creates ParameterCards for valid card-sourced parameters"(let [parameters [{:id "param1"- :type :test- :values_source_type "card"+ :type :number+ :values_source_type :card:values_source_config {:card_id card-id-1}}{:id "param2"- :type :test- :values_source_type "static-list"+ :type :number+ :values_source_type :static-list:values_source_config {:values ["a" "b" "c"]}}{:id "param3"- :type :test- :values_source_type "card"+ :type :number+ :values_source_type :card:values_source_config {:card_id card-id-2}}]](parameter-card/upsert-or-delete-from-parameters! "dashboard" 1 parameters)-(let [pcs (t2/select :model/ParameterCard:parameterized_object_type "dashboard":parameterized_object_id 1)]@@ -125,39 +118,32 @@(is (some #(and (= "param3" (:parameter_id %))(= card-id-2 (:card_id %))) pcs))(is (not (some #(= "param2" (:parameter_id %)) pcs))))))-(testing "deletes ParameterCards no longer in use"(mt/with-temp [:model/ParameterCard _ {:parameterized_object_type "dashboard":parameterized_object_id 1:parameter_id "old-param":card_id card-id-3}](let [parameters [{:id "param1"- :type :test- :values_source_type "card"+ :type :number+ :values_source_type :card:values_source_config {:card_id card-id-1}}]](parameter-card/upsert-or-delete-from-parameters! "dashboard" 1 parameters)-(let [pcs (t2/select :model/ParameterCard:parameterized_object_type "dashboard":parameterized_object_id 1)](is (= 1 (count pcs)))(is (= "param1" (:parameter_id (first pcs))))(is (= card-id-1 (:card_id (first pcs))))))))-(testing "handles nil parameters gracefully"(parameter-card/upsert-or-delete-from-parameters! "dashboard" 1 nil)- (let [pcs (t2/select :model/ParameterCard+ (is (not (t2/exists? :model/ParameterCard:parameterized_object_type "dashboard"- :parameterized_object_id 1)]- (is (empty? pcs))))-+ :parameterized_object_id 1))))(testing "handles empty parameters list"(parameter-card/upsert-or-delete-from-parameters! "dashboard" 1 [])- (let [pcs (t2/select :model/ParameterCard+ (is (not (t2/exists? :model/ParameterCard:parameterized_object_type "dashboard"- :parameterized_object_id 1)]- (is (empty? pcs))))-+ :parameterized_object_id 1))))(testing "parameters without required fields"(let [parameters [{:id "param1":type :test@@ -175,14 +161,12 @@clojure.lang.ExceptionInfo#"Invalid input:"(parameter-card/upsert-or-delete-from-parameters! "dashboard" 1 parameters)))))-(testing "works with card parameterized objects"(let [parameters [{:id "param1"- :type :test- :values_source_type "card"+ :type :number+ :values_source_type :card:values_source_config {:card_id card-id-1}}]](parameter-card/upsert-or-delete-from-parameters! "card" 1 parameters)-(let [pcs (t2/select :model/ParameterCard:parameterized_object_type "card":parameterized_object_id 1)]@@ -192,20 +176,17 @@(deftest model-validation-test(mt/with-temp [:model/Card {card-id :id} {}]-(testing "ParameterCard creation with valid parameterized_object_type"(is (some? (t2/insert! :model/ParameterCard{:parameterized_object_type "dashboard":parameterized_object_id 1:parameter_id "param1":card_id card-id})))-(is (some? (t2/insert! :model/ParameterCard{:parameterized_object_type "card":parameterized_object_id 1:parameter_id "param2":card_id card-id}))))-(testing "ParameterCard creation with invalid parameterized_object_type throws exception"(is (thrown-with-msg?clojure.lang.ExceptionInfo@@ -215,7 +196,6 @@:parameterized_object_id 1:parameter_id "param1":card_id card-id}))))-(testing "ParameterCard update with invalid parameterized_object_type throws exception"(mt/with-temp [:model/ParameterCard {pc-id :id} {:parameterized_object_type "dashboard":parameterized_object_id 1diff --git a/test/metabase/queries/models/query_test.clj b/test/metabase/queries/models/query_test.cljindex 9a4fb429e67..c1d5ef48a80 100644--- a/test/metabase/queries/models/query_test.clj+++ b/test/metabase/queries/models/query_test.clj@@ -39,17 +39,20 @@:query {:source-query {:source-table 6}}}}}](testing message(is (=? expected- (into {} (query/query->database-and-table-ids query))))))))+ (into {} (query/query->database-and-table-ids (lib/query+ (lib.tu/mock-metadata-provider+ {:database (assoc meta/database :id (:database query))})+ query)))))))))-(deftest ^:parallel query->database-and-table-ids-pMBQL-test- (testing "Should work for pMBQL queries"+(deftest ^:parallel query->database-and-table-ids-mbql5-test+ (testing "Should work for MBQL 5 queries"(let [venues (lib.metadata/table meta/metadata-provider (meta/id :venues))query (lib/query meta/metadata-provider venues)](is (= {:database-id (meta/id), :table-id (meta/id :venues)}(query/query->database-and-table-ids query))))))-(deftest ^:parallel query->database-and-table-ids-pMBQL-source-card-test- (testing "Should handle source-card lookup for pMBQL queries"+(deftest ^:parallel query->database-and-table-ids-mbql5-source-card-test+ (testing "Should handle source-card lookup for MBQL 5 queries"(let [metadata-provider (lib.tu/metadata-provider-with-cards-for-queriesmeta/metadata-provider[(lib/query meta/metadata-provider (lib.metadata/table meta/metadata-provider (meta/id :venues)))])@@ -58,47 +61,3 @@query-with-source-card))(is (= {:database-id (meta/id), :table-id (meta/id :venues)}(query/query->database-and-table-ids query-with-source-card))))))--(deftest ^:parallel collect-card-ids-legacy-native-query-template-tags-test- (let [query {:database 1- :type :native- :native {:query "SELECT *;"- :template-tags {"tag_1" {:type :card- :card-id 100}- "tag_2" {:type :card- :card-id 200}}}}]- #_{:clj-kondo/ignore [:deprecated-var]}- (is (= #{100 200}- (set (query/collect-card-ids query))))))--(deftest ^:parallel collect-card-ids-legacy-query-source-card-test- (let [query {:database 1- :type :query- :query {:source-query {:source-table "card__1000"}}}]- #_{:clj-kondo/ignore [:deprecated-var]}- (is (= #{1000}- (set (query/collect-card-ids query))))))--(deftest ^:parallel collect-card-ids-pmbql-native-query-template-tags-test- (let [query (lib/query meta/metadata-provider {:lib/type :mbql.stage/native- :native "SELECT *;"- :template-tags {"tag_1" {:name "tag_1"- :display-name "Tag 1"- :type :card- :card-id 100}- "tag_2" {:name "tag_2"- :display-name "Tag 2"- :type :card- :card-id 200}}})]- #_{:clj-kondo/ignore [:deprecated-var]}- (is (= #{100 200}- (set (query/collect-card-ids query))))))--(deftest ^:parallel collect-card-ids-pmbql-query-source-card-test- (let [metadata-provider (lib.tu/metadata-provider-with-cards-for-queries- meta/metadata-provider- [(lib/query meta/metadata-provider (lib.metadata/table meta/metadata-provider (meta/id :venues)))])- query-with-source-card (lib/query metadata-provider (lib.metadata/card metadata-provider 1))]- #_{:clj-kondo/ignore [:deprecated-var]}- (is (= #{1}- (set (query/collect-card-ids query-with-source-card))))))diff --git a/test/metabase/queries/schema_test.clj b/test/metabase/queries/schema_test.cljnew file mode 100644index 00000000000..489c71f786d--- /dev/null+++ b/test/metabase/queries/schema_test.clj@@ -0,0 +1,12 @@+(ns metabase.queries.schema-test+ (:require+ [clojure.test :refer :all]+ [malli.error :as me]+ [metabase.queries.schema :as queries.schema]+ [metabase.util.malli.registry :as mr]))++(deftest ^:parallel validate-query-test+ (is (= {:dataset_query {:lib/metadata ["missing required key"]+ :stages [["Initial MBQL stage must have either :source-table or :source-card (but not both)"]]}}+ (me/humanize (mr/explain ::queries.schema/card+ {:dataset_query {:lib/type :mbql/query, :database 2378, :stages [{:lib/type :mbql.stage/mbql}]}})))))diff --git a/test/metabase/query_permissions/impl_test.clj b/test/metabase/query_permissions/impl_test.cljindex d30eeceecf3..5723a1d18b4 100644--- a/test/metabase/query_permissions/impl_test.clj+++ b/test/metabase/query_permissions/impl_test.clj@@ -16,7 +16,7 @@;;; ---------------------------------------------- Permissions Checking ----------------------------------------------(defn- card []- {:dataset_query {:database (mt/id), :type "native"}})+ {:dataset_query {:database (mt/id), :type :native, :native {:query "SELECT 1;"}}})(defn- card-in-collection [collection-or-id](assoc (card) :collection_id (u/the-id collection-or-id)))@@ -27,52 +27,41 @@(mt/with-temp [:model/Card card (card)](binding [*current-user-permissions-set* (delay #{})](is (not (mi/can-read? card)))))-(testing "...or one in a Collection either!"(mt/with-temp [:model/Card card (card-in-collection collection)](binding [*current-user-permissions-set* (delay #{})](is (not (mi/can-read? card)))))))-(testing "*should* be allowed to read a Card not in a Collection if you have Root collection perms"(mt/with-temp [:model/Card card (card)](binding [*current-user-permissions-set* (delay #{"/collection/root/read/"})](is (mi/can-read? card)))-(testing "...but not if you have perms for some other Collection"(binding [*current-user-permissions-set* (delay #{"/collection/1337/read/"})](is (not (mi/can-read? card)))))))-(testing "should be allowed to *read* a Card in a Collection if you have read perms for that Collection"(mt/with-temp [:model/Card card (card-in-collection collection)](binding [*current-user-permissions-set* (delay #{(permissions.path/collection-read-path collection)})](is (mi/can-read? card)))-(testing "...but not if you only have Root Collection perms"(binding [*current-user-permissions-set* (delay #{"/collection/root/read/"})](is (not (mi/can-read? card)))))))-(testing "to *write* a Card not in a Collection you need Root Collection Write Perms"(mt/with-temp [:model/Card card (card)](binding [*current-user-permissions-set* (delay #{"/collection/root/"})](is (mi/can-write? card)))-(testing "...root Collection Read Perms shouldn't work"(binding [*current-user-permissions-set* (delay #{"/collection/root/read/"})](is (not (mi/can-write? card))))-(testing "...nor should write perms for another collection"(binding [*current-user-permissions-set* (delay #{"/collection/1337/"})](is (not (mi/can-write? card))))))))-(testing "to *write* a Card *in* a Collection you need Collection Write Perms"(mt/with-temp [:model/Card card (card-in-collection collection)](binding [*current-user-permissions-set* (delay #{(permissions.path/collection-readwrite-path collection)})](is (mi/can-write? card)))-(testing "...Collection read perms shouldn't work"(binding [*current-user-permissions-set* (delay #{(permissions.path/collection-read-path collection)})](is (not (mi/can-write? card))))-(testing "...nor should write perms for the Root Collection"(binding [*current-user-permissions-set* (delay #{"/collection/root/"})](is (not (mi/can-write? card))))))))))diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 8416ba96628..79b92456294 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -25,7 +25,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.constraints :as qp.constraints][metabase.query-processor.pivot.test-util :as qp.pivot.test-util]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.query-processor.util :as qp.util][metabase.test :as mt]diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.cljindex 8915208084f..62a60d55d2a 100644--- a/test/metabase/query_processor/card_test.clj+++ b/test/metabase/query_processor/card_test.clj@@ -7,6 +7,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.models.interface :as mi][metabase.permissions.models.data-permissions :as data-perms]@@ -15,7 +16,7 @@[metabase.query-processor :as qp][metabase.query-processor.card :as qp.card][metabase.query-processor.middleware.results-metadata :as qp.results-metadata]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.users :as test.users]@@ -281,9 +282,9 @@{:dataset_query (:dataset-query card-1):type :model:result_metadata (add-user-edits (:result-metadata card-1))})- :model/Card card-2 (let [card-2 (lib.metadata/card mp 2)]- {:dataset_query (-> (:dataset-query card-2)- (assoc-in [:query :source-table] (format "card__%d" (:id card-1))))+ :model/Card card-2 (let [card-2 (lib.metadata/card mp 2)+ mp (mt/metadata-provider)]+ {:dataset_query (lib/query mp (lib.metadata/card mp (:id card-1))):result_metadata (:result-metadata card-2)})](doseq [[card-type card-id] {"model" (:id card-1)"card with model as source" (:id card-2)}]diff --git a/test/metabase/query_processor/dashboard_test.clj b/test/metabase/query_processor/dashboard_test.cljindex 3df931a9c47..4ae0bc221d3 100644--- a/test/metabase/query_processor/dashboard_test.clj+++ b/test/metabase/query_processor/dashboard_test.clj@@ -1,9 +1,6 @@(ns metabase.query-processor.dashboard-test"There are more e2e tests in [[metabase.dashboards.api-test]]."- {:clj-kondo/config '{:linters- ;; allowing `with-temp` here for now since this tests the REST API which doesn't fully use- ;; metadata providers.- {:discouraged-var {metabase.test/with-temp {:level :off}}}}}+ {:clj-kondo/config '{:linters {:discouraged-var {metabase.test/with-temp {:level :off}}}}}(:require[clojure.test :refer :all][metabase.dashboards.api-test :as api.dashboard-test]@@ -174,9 +171,9 @@[:model/Card {card-id :id} {:database_id (mt/id):table_id (mt/id :venues):dataset_query (mt/mbql-query venues- {:aggregation [:count]- :breakout [$category_id- $price]})}+ {:aggregation [:count]+ :breakout [$category_id+ $price]})}:model/Dashboard {dashboard-id :id} {:parameters[{:slug "venue_id":id "_VENUE_ID_"diff --git a/test/metabase/query_processor/middleware/add_implicit_clauses_test.clj b/test/metabase/query_processor/middleware/add_implicit_clauses_test.cljindex 408233edcce..f45a76e3f5b 100644--- a/test/metabase/query_processor/middleware/add_implicit_clauses_test.clj+++ b/test/metabase/query_processor/middleware/add_implicit_clauses_test.clj@@ -152,8 +152,9 @@[:field "count" {:base-type :type/Integer}]]}(add-implicit-clauses-to-legacy-inner-query(:query (lib.tu.macros/mbql-query checkins- {:source-query {:aggregation [[:count]]- :breakout [!month.$date]}})))))))+ {:source-query {:source-table $$checkins+ :aggregation [[:count]]+ :breakout [!month.$date]}})))))))(deftest ^:parallel expression-with-only-field-in-source-query-test(testing "Field coming from expression in source query should have string id"diff --git a/test/metabase/query_processor/middleware/add_remaps_test.clj b/test/metabase/query_processor/middleware/add_remaps_test.cljindex df997ecf87a..3035c9f600e 100644--- a/test/metabase/query_processor/middleware/add_remaps_test.clj+++ b/test/metabase/query_processor/middleware/add_remaps_test.clj@@ -11,7 +11,7 @@[metabase.query-processor.middleware.add-remaps :as qp.add-remaps][metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.reducible :as qp.reducible]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.test.fixtures :as fixtures]))diff --git a/test/metabase/query_processor/middleware/add_timezone_info_test.clj b/test/metabase/query_processor/middleware/add_timezone_info_test.cljindex 97a93d8efc9..cc2e072562c 100644--- a/test/metabase/query_processor/middleware/add_timezone_info_test.clj+++ b/test/metabase/query_processor/middleware/add_timezone_info_test.clj@@ -2,8 +2,9 @@(:require[clojure.test :refer :all][metabase.driver :as driver]- [metabase.query-processor.middleware.add-timezone-info- :as add-timezone-info]+ [metabase.lib.test-metadata :as meta]+ [metabase.query-processor.middleware.add-timezone-info :as add-timezone-info]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(driver/register! ::timezone-driver, :abstract? true)@@ -28,6 +29,7 @@(testing driver(mt/with-temporary-setting-values [report-timezone timezone](driver/with-driver driver- (mt/with-database-timezone-id nil- (is (= expected- (add-timezone-info {})))))))))+ (qp.store/with-metadata-provider meta/metadata-provider+ (mt/with-database-timezone-id nil+ (is (= expected+ (add-timezone-info {}))))))))))diff --git a/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj b/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.cljindex 15b144f8a67..abaf2e2122b 100644--- a/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj+++ b/test/metabase/query_processor/middleware/annotate/legacy_helper_fns_test.clj@@ -4,7 +4,7 @@[clojure.walk :as walk][metabase.lib.test-metadata :as meta][metabase.query-processor.middleware.annotate.legacy-helper-fns :as annotate.legacy-helper-fns]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u]))(deftest ^:parallel aggregation-name-testdiff --git a/test/metabase/query_processor/middleware/annotate_test.clj b/test/metabase/query_processor/middleware/annotate_test.cljindex e8bfd2563e8..a6a5ceb8195 100644--- a/test/metabase/query_processor/middleware/annotate_test.clj+++ b/test/metabase/query_processor/middleware/annotate_test.clj@@ -9,7 +9,7 @@[metabase.lib.test-util.macros :as lib.tu.macros][metabase.query-processor.middleware.annotate :as annotate][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.util :as u][metabase.util.malli :as mu]))@@ -125,7 +125,7 @@:source :native:field_ref [:field "a_2" {:base-type :type/*}]}](column-info- (lib/query meta/metadata-provider {:type :native})+ (lib/query meta/metadata-provider {:type :native, :native {:query "SELECT 1;"}}){:cols [{:name "a"} {:name "a"}]:rows rows})))))))diff --git a/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj b/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.cljindex d59733fb522..e270e3d79a5 100644--- a/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj+++ b/test/metabase/query_processor/middleware/auto_bucket_datetimes_test.clj@@ -8,7 +8,7 @@[metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros][metabase.query-processor.middleware.auto-bucket-datetimes :as qp.auto-bucket-datetimes]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util.malli :as mu]))(deftest ^:parallel should-not-be-autobucketed?-testdiff --git a/test/metabase/query_processor/middleware/binning_test.clj b/test/metabase/query_processor/middleware/binning_test.cljindex 67e881563c0..4864b4a233c 100644--- a/test/metabase/query_processor/middleware/binning_test.clj+++ b/test/metabase/query_processor/middleware/binning_test.clj@@ -16,7 +16,7 @@[metabase.query-processor :as qp][metabase.query-processor.middleware.binning :as binning][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(deftest ^:parallel filter->field-map-testdiff --git a/test/metabase/query_processor/middleware/cache_test.clj b/test/metabase/query_processor/middleware/cache_test.cljindex a32ce2dbd8c..3edb8143b79 100644--- a/test/metabase/query_processor/middleware/cache_test.clj+++ b/test/metabase/query_processor/middleware/cache_test.clj@@ -21,7 +21,7 @@[metabase.query-processor.middleware.process-userland-query :as process-userland-query][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.reducible :as qp.reducible]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming :as qp.streaming][metabase.query-processor.test-util :as qp.test-util][metabase.query-processor.util :as qp.util]@@ -134,7 +134,11 @@:min-duration-ms *query-caching-min-ttl*})(defn- test-query [query-kvs]- (merge {:cache-strategy (ttl-strategy), :lib/type :mbql/query, :database 1, :stages [{:abc :def}]} query-kvs))+ (merge {:cache-strategy (ttl-strategy)+ :lib/type :mbql/query+ :database 1+ :stages [{:lib/type :mbql.stage/mbql, :source-table 2, :abc :def}]}+ query-kvs))(defn- run-query* [& {:as query-kvs}];; clear out stale values in save/purge channels@@ -234,7 +238,7 @@(deftest max-ttl-test(testing (str "Check that `query-caching-max-ttl` is respected. Whenever a new query is cached the cache should ""evict any entries older that `query-caching-max-ttl`. Set max-ttl to 100 ms, run query `:abc`, "- "then wait 200 ms, and run `:def`. This should trigger the cache flush for entries past "+ "then wait 200 ms, and run the query. This should trigger the cache flush for entries past ""`:max-ttl`; and the cached entry for `:abc` should be deleted. Running `:abc` a subsequent time ""should not return cached results")(with-mock-cache! [purge-chan]@@ -242,7 +246,7 @@(run-query)(mt/wait-for-result purge-chan)(Thread/sleep 200)- (run-query :query :def)+ (run-query :stages [{:lib/type :mbql.stage/native, :native "SELECT abc;"}])(mt/wait-for-result purge-chan)(is (= :not-cached(run-query)))))))@@ -465,7 +469,8 @@(is (=? {:cache/details {:cached true:updated_at #t "2020-02-19T04:44:26.056Z[UTC]":hash some?- ;; TODO: this check is not working if the key is not present in the data+ ;; TODO: this check is not working if the key is not present in the+ ;; data:cache-hash some?}:row_count 5:status :completed}diff --git a/test/metabase/query_processor/middleware/expand_macros_test.clj b/test/metabase/query_processor/middleware/expand_macros_test.cljindex 1bcefd7cdf3..2d30825d7af 100644--- a/test/metabase/query_processor/middleware/expand_macros_test.clj+++ b/test/metabase/query_processor/middleware/expand_macros_test.clj@@ -8,7 +8,7 @@[metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros][metabase.query-processor.middleware.expand-macros :as expand-macros]- [metabase.query-processor.store :as qp.store]))+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]))(defn- expand-macros"If input is a legacy query, convert to pMBQL, call [[expand-macros/expand-macros]], then convert back to legacy. Thisdiff --git a/test/metabase/query_processor/middleware/fetch_source_query_test.clj b/test/metabase/query_processor/middleware/fetch_source_query_test.cljindex f468ea13ba2..551e9eda10c 100644--- a/test/metabase/query_processor/middleware/fetch_source_query_test.clj+++ b/test/metabase/query_processor/middleware/fetch_source_query_test.clj@@ -14,7 +14,7 @@[metabase.query-processor :as qp][metabase.query-processor.middleware.fetch-source-query :as fetch-source-query][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt]))diff --git a/test/metabase/query_processor/middleware/large_int_test.clj b/test/metabase/query_processor/middleware/large_int_test.cljindex 4c2e36f050d..c82857d8910 100644--- a/test/metabase/query_processor/middleware/large_int_test.clj+++ b/test/metabase/query_processor/middleware/large_int_test.clj@@ -2,7 +2,7 @@(:require[clojure.test :refer :all][metabase.query-processor.middleware.large-int :as large-int]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(set! *warn-on-reflection* true)diff --git a/test/metabase/query_processor/middleware/limit_test.clj b/test/metabase/query_processor/middleware/limit_test.cljindex 984e46f88ca..990a0154e57 100644--- a/test/metabase/query_processor/middleware/limit_test.clj+++ b/test/metabase/query_processor/middleware/limit_test.clj@@ -29,7 +29,7 @@(deftest limit-results-rows-test(testing "Apply to an infinite sequence and make sure it gets capped at `qp.settings/absolute-max-results`"(is (= test-max-results- (-> (limit! {:type :native}) mt/rows count)))))+ (-> (limit! {:type :native, :native {:query "SELECT 1;"}}) mt/rows count)))))(deftest ^:parallel disable-max-results-test(testing "Apply `absolute-max-results` limit in the default case"@@ -63,7 +63,7 @@(testing "Apply a max-results-bare-rows limit specifically on no-aggregation query"(let [query {:constraints {:max-results 46}:type :query- :query {}}+ :query {:source-table 1}}result (limit! query)](is (= 46(-> result mt/rows count))@@ -80,11 +80,11 @@(deftest ^:parallel download-row-max-results-test(testing "Apply `absolute-max-results` when `download-row-limit` is not set."(let [query {:type :query- :query {}+ :query {:source-table (meta/id :venues)}:info {:context :csv-download}}]- (is (=? {:type :query- :query {:limit qp.settings/absolute-max-results}- :info {:context :csv-download}}+ (is (=? {:type :query+ :query {:limit qp.settings/absolute-max-results}+ :info {:context :csv-download}}(add-default-limit query))))))(deftest download-row-limit-test@@ -104,7 +104,7 @@(is (= expected(get-in (add-default-limit{:type :query- :query {}+ :query {:source-table (meta/id :venues)}:info {:context context}})[:query :limit]))))))))@@ -124,7 +124,7 @@(is (= expected(get-in (add-default-limit{:type :query- :query {}+ :query {:source-table (meta/id :venues)};; setting a constraint here will result in `(mbql.u/query->max-rows-limit query)` returning that limit;; so we can use this to check the behaviour of `add-default-limit` when download-row-limit is unset:constraints (when limit {:max-results-bare-rows limit})diff --git a/test/metabase/query_processor/middleware/normalize_query_test.clj b/test/metabase/query_processor/middleware/normalize_query_test.cljindex 12724c5a9c0..69619ec41d9 100644--- a/test/metabase/query_processor/middleware/normalize_query_test.clj+++ b/test/metabase/query_processor/middleware/normalize_query_test.clj@@ -4,7 +4,7 @@[metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.test-metadata :as meta][metabase.query-processor.middleware.normalize-query :as normalize-query]- [metabase.query-processor.store :as qp.store]))+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]))(deftest ^:parallel normalize-test(testing "handle legacy queries as they look when coming in from REST API"diff --git a/test/metabase/query_processor/middleware/parameters_test.clj b/test/metabase/query_processor/middleware/parameters_test.cljindex f34ac1c4cab..0d3e42b39d8 100644--- a/test/metabase/query_processor/middleware/parameters_test.clj+++ b/test/metabase/query_processor/middleware/parameters_test.clj@@ -174,7 +174,7 @@(defn- card-template-tag[card-id](let [tag (str "#" card-id)]- {:id tag, :name tag, :display-name tag, :type "card", :card-id card-id}))+ {:id tag, :name tag, :display-name tag, :type :card, :card-id card-id}))(defn card-template-tags"Generate the map representing card template tags (sub-queries) for the given `card-ids`."@@ -324,7 +324,7 @@{"Venue fields" 1"Filter: expensive venues" 2})})}]})](testing "multiple snippets are correctly expanded in parent query"- (is (=? {:native {:query "SELECT name, price FROM venues WHERE price > 2", :params nil}}+ (is (=? {:stages [{:native "SELECT name, price FROM venues WHERE price > 2", :params nil}]}(substitute-params mp (:dataset-query (lib.metadata/card mp 1))))))(testing "multiple snippets are expanded from saved sub-query"(is (=? {:native {:query "SELECT * FROM (SELECT name, price FROM venues WHERE price > 2) AS x", :params []}}diff --git a/test/metabase/query_processor/middleware/permissions_test.clj b/test/metabase/query_processor/middleware/permissions_test.cljindex e10213af1f3..fa941ca84c0 100644--- a/test/metabase/query_processor/middleware/permissions_test.clj+++ b/test/metabase/query_processor/middleware/permissions_test.clj@@ -1,18 +1,17 @@(ns metabase.query-processor.middleware.permissions-test"Tests for the middleware that checks whether the current user has permissions to run a given query."- {:clj-kondo/config '{:linters- ;; allowing `with-temp` here for now since perms is mostly still app-DB based.- {:discouraged-var {metabase.test/with-temp {:level :off}}}}}+ {:clj-kondo/config '{:linters {:discouraged-var {metabase.test/with-temp {:level :off}}}}}(:require[clojure.test :refer :all][metabase.api.common :as api]+ [metabase.lib-be.core :as lib-be][metabase.lib.test-metadata :as meta][metabase.permissions.core :as perms][metabase.query-processor :as qp][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.setup :as qp.setup]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.util :as u][metabase.util.malli.fn :as mu.fn])@@ -246,7 +245,7 @@(mt/with-no-data-perms-for-all-users!(perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/view-data :unrestricted)(perms/set-database-permission! (perms/all-users-group) (mt/id) :perms/create-queries :no)- (let [query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 2})+ (let [query (lib-be/normalize-query (mt/mbql-query venues {:order-by [[:asc $id]], :limit 2}))check! (fn [query](binding [api/*current-user-id* (mt/user->id :rasta)](qp.store/with-metadata-provider (mt/id)diff --git a/test/metabase/query_processor/middleware/process_userland_query_test.clj b/test/metabase/query_processor/middleware/process_userland_query_test.cljindex 1204e6758de..b0ee5301781 100644--- a/test/metabase/query_processor/middleware/process_userland_query_test.clj+++ b/test/metabase/query_processor/middleware/process_userland_query_test.clj@@ -12,7 +12,7 @@[metabase.query-processor.middleware.process-userland-query :as process-userland-query][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.reducible :as qp.reducible]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util :as qp.util][metabase.test :as mt][methodical.core :as methodical]))diff --git a/test/metabase/query_processor/middleware/remove_inactive_field_refs_test.clj b/test/metabase/query_processor/middleware/remove_inactive_field_refs_test.cljindex 7ad3e96fb1e..a9689a79ac3 100644--- a/test/metabase/query_processor/middleware/remove_inactive_field_refs_test.clj+++ b/test/metabase/query_processor/middleware/remove_inactive_field_refs_test.clj@@ -13,7 +13,7 @@[metabase.query-processor :as qp][metabase.query-processor.middleware.remove-inactive-field-refs :as remove-inactive-field-refs][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt]))diff --git a/test/metabase/query_processor/middleware/results_metadata_test.clj b/test/metabase/query_processor/middleware/results_metadata_test.cljindex 266f3736dae..779ae788a59 100644--- a/test/metabase/query_processor/middleware/results_metadata_test.clj+++ b/test/metabase/query_processor/middleware/results_metadata_test.clj@@ -16,7 +16,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.results-metadata :as middleware.results-metadata][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util :as qp.util][metabase.test :as mt][metabase.util :as u]diff --git a/test/metabase/query_processor/middleware/update_used_cards_test.clj b/test/metabase/query_processor/middleware/update_used_cards_test.cljindex 024fe8b5356..f9947c680e1 100644--- a/test/metabase/query_processor/middleware/update_used_cards_test.clj+++ b/test/metabase/query_processor/middleware/update_used_cards_test.clj@@ -69,8 +69,8 @@:template-tags {"#card" {:card-id card-id:display-name "card":id "card"- :name "card"- :type "card"}}}))))))+ :name "#card"+ :type :card}}}))))))(deftest alert-test(with-used-cards-setup!diff --git a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj b/test/metabase/query_processor/middleware/wrap_value_literals_test.cljindex 78aea2688ae..8ce933fb482 100644--- a/test/metabase/query_processor/middleware/wrap_value_literals_test.clj+++ b/test/metabase/query_processor/middleware/wrap_value_literals_test.clj@@ -10,7 +10,7 @@[metabase.lib.test-util.macros :as lib.tu.macros][metabase.query-processor.middleware.wrap-value-literals :as qp.wrap-value-literals][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.test :as mt]))@@ -59,20 +59,21 @@[:> $id 50][:< $price 5]]})))))-(defn- parse-with-timezone! [datetime-str ^String timezone-id]+(defn- parse-with-timezone [datetime-str ^String timezone-id](driver/with-driver ::tz-driver- (mt/with-report-timezone-id! timezone-id- (is (= (qp.timezone/results-timezone-id)- timezone-id)- "Make sure `results-timezone-id` is returning the bound value")- (-> (#'qp.wrap-value-literals/add-type-info datetime-str {:unit :day})- (nth 2)))))+ (binding [qp.timezone/*report-timezone-id-override* timezone-id]+ (qp.store/with-metadata-provider meta/metadata-provider+ (is (= (qp.timezone/results-timezone-id)+ timezone-id)+ "Make sure `results-timezone-id` is returning the bound value")+ (-> (#'qp.wrap-value-literals/add-type-info datetime-str {:unit :day})+ (nth 2))))))-(deftest parse-datetime-literal-strings-test+(deftest ^:parallel parse-datetime-literal-strings-test(doseq [[timezone expected] {"UTC" (t/zoned-date-time "2018-10-01T00:00:00Z[UTC]")"US/Pacific" (t/zoned-date-time "2018-10-01T00:00:00-07:00[US/Pacific]")}](is (= expected- (parse-with-timezone! "2018-10-01" timezone))+ (parse-with-timezone "2018-10-01" timezone))(format "datetime literal string '2018-10-01' parsed with the %s timezone should be %s" timezone expected))))(deftest ^:parallel wrap-datetime-literal-strings-testdiff --git a/test/metabase/query_processor/parameters/temporal_units_test.clj b/test/metabase/query_processor/parameters/temporal_units_test.cljindex 2ddc5e1212c..819d57e4703 100644--- a/test/metabase/query_processor/parameters/temporal_units_test.clj+++ b/test/metabase/query_processor/parameters/temporal_units_test.clj@@ -36,10 +36,10 @@(let [base-query (mt/arbitrary-select-query driver/*driver* :orders "{{time-unit}}")native-query (mt/native-query(assoc base-query- :template-tags {"time-unit" {:name "id"+ :template-tags {"time-unit" {:name "time-unit":display-name "id":type :temporal-unit- :dimension [:field (mt/id :orders :created_at)]}}))+ :dimension [:field (mt/id :orders :created_at) nil]}}))parameterized-query (assoc native-query:parameters [{:type :temporal-unit:name "time-unit"@@ -84,10 +84,10 @@(let [base-query (mt/arbitrary-select-query driver/*driver* :orders "{{time-unit}}")native-query (mt/native-query(assoc base-query- :template-tags {"time-unit" {:name "id"+ :template-tags {"time-unit" {:name "time-unit":display-name "id":type :temporal-unit- :dimension [:field (mt/id :orders :created_at)]+ :dimension [:field (mt/id :orders :created_at) nil]:alias (mt/make-alias driver/*driver* "created_at")}}))parameterized-query (assoc native-query:parameters [{:type :temporal-unit@@ -133,10 +133,10 @@(let [base-query (mt/arbitrary-select-query driver/*driver* :orders "{{time-unit}}")query (assoc (mt/native-query(assoc base-query- :template-tags {"time-unit" {:name "id"+ :template-tags {"time-unit" {:name "time-unit":display-name "id":type :temporal-unit- :dimension [:field 1000000000]}}))+ :dimension [:field 1000000000 nil]}})):parameters [{:type :temporal-unit:name "time-unit":target [:variable [:template-tag "time-unit"]]@@ -152,10 +152,10 @@(let [base-query (mt/arbitrary-select-query driver/*driver* :orders "{{time-unit}}")query (assoc (mt/native-query(assoc base-query- :template-tags {"time-unit" {:name "id"+ :template-tags {"time-unit" {:name "time-unit":display-name "id":type :temporal-unit- :dimension [:field (mt/id :orders :created_at)]}}))+ :dimension [:field (mt/id :orders :created_at) nil]}})):parameters [{:type :temporal-unit:name "time-unit":target [:dimension [:template-tag "time-unit"]]diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex 2c8052d3ed5..60d516c5808 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -16,7 +16,7 @@[metabase.query-processor :as qp][metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.pivot.test-util :as qp.pivot.test-util]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.test.data :as data][metabase.util :as u]diff --git a/test/metabase/query_processor/preprocess_test.clj b/test/metabase/query_processor/preprocess_test.cljindex 797cc05e96d..aea228a797f 100644--- a/test/metabase/query_processor/preprocess_test.clj+++ b/test/metabase/query_processor/preprocess_test.clj@@ -17,7 +17,7 @@[metabase.query-processor :as qp][metabase.query-processor.middleware.annotate :as annotate][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.interface :as tx]diff --git a/test/metabase/query_processor/store_test.clj b/test/metabase/query_processor/store_test.cljindex 6fbe189d364..2a825b9c4ff 100644--- a/test/metabase/query_processor/store_test.clj+++ b/test/metabase/query_processor/store_test.clj@@ -6,7 +6,7 @@[metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u]))(deftest ^:parallel cached-testdiff --git a/test/metabase/query_processor/streaming_test.clj b/test/metabase/query_processor/streaming_test.cljindex a5973e51034..bafa3e537ef 100644--- a/test/metabase/query_processor/streaming_test.clj+++ b/test/metabase/query_processor/streaming_test.clj@@ -11,7 +11,7 @@[metabase.query-processor :as qp][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.schema :as qp.schema]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming :as qp.streaming][metabase.query-processor.streaming.test-util :as streaming.test-util][metabase.query-processor.streaming.xlsx-test :as xlsx-test]diff --git a/test/metabase/query_processor/test_util.clj b/test/metabase/query_processor/test_util.cljindex 3e37315d35f..e9cb9af2891 100644--- a/test/metabase/query_processor/test_util.clj+++ b/test/metabase/query_processor/test_util.clj@@ -15,14 +15,14 @@[metabase.driver :as driver][metabase.driver.test-util :as driver.tu][metabase.driver.util :as driver.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.test.data :as data][metabase.test.data.env :as tx.env]@@ -446,7 +446,7 @@"A mock metadata provider composed with the application database metadata provider that adds FK relationshipsfor Tables that would normally have them in drivers that have formal FK constraints."([]- (mock-fks-application-database-metadata-provider (lib.metadata.jvm/application-database-metadata-provider (data/id))))+ (mock-fks-application-database-metadata-provider (lib-be/application-database-metadata-provider (data/id))))([parent-metadata-provider](lib.tu/merged-mock-metadata-provider@@ -502,7 +502,7 @@in `queries`. Cards do not include result metadata. Cards have IDs starting at `1` and increasing sequentially."([queries](metadata-provider-with-cards-for-queries- (lib.metadata.jvm/application-database-metadata-provider (data/id))+ (lib-be/application-database-metadata-provider (data/id))queries))([parent-metadata-provider :- ::lib.schema.metadata/metadata-provider@@ -518,7 +518,7 @@([queries :- [:sequential {:min 1} :map]transforms](metadata-provider-with-cards-with-transformed-metadata-for-queries- (lib.metadata.jvm/application-database-metadata-provider (data/id))+ (lib-be/application-database-metadata-provider (data/id))queriestransforms))@@ -554,7 +554,7 @@subsequent Cards."([queries](metadata-provider-with-cards-with-metadata-for-queries- (lib.metadata.jvm/application-database-metadata-provider (data/id))+ (lib-be/application-database-metadata-provider (data/id))queries))([parent-metadata-provider :- ::lib.schema.metadata/metadata-provider@@ -566,7 +566,8 @@[(data/mbql-query venues)])](is (partial= {:id 1:name "Card 1"- :dataset-query {:type :query}+ :dataset-query {:lib/type :mbql/query+ :stages [{:lib/type :mbql.stage/mbql}]}:result-metadata [{:name "ID"}{:name "NAME"}{:name "CATEGORY_ID"}@@ -580,7 +581,7 @@[(data/native-query (qp.compile/compile (data/mbql-query venues)))])](is (partial= {:id 1:name "Card 1"- :dataset-query {:type :native}+ :dataset-query {:stages [{:lib/type :mbql.stage/native}]}:result-metadata [{:name "ID"}{:name "NAME"}{:name "CATEGORY_ID"}diff --git a/test/metabase/query_processor/util/add_alias_info_test.clj b/test/metabase/query_processor/util/add_alias_info_test.cljindex deba39f61fc..0a3c75f3897 100644--- a/test/metabase/query_processor/util/add_alias_info_test.clj+++ b/test/metabase/query_processor/util/add_alias_info_test.clj@@ -18,7 +18,7 @@[metabase.lib.util :as lib.util][metabase.lib.util.match :as lib.util.match][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as add][metabase.test :as mt]))diff --git a/test/metabase/query_processor/util/nest_query_test.clj b/test/metabase/query_processor/util/nest_query_test.cljindex 71eac9387c1..b6790b57c06 100644--- a/test/metabase/query_processor/util/nest_query_test.clj+++ b/test/metabase/query_processor/util/nest_query_test.clj@@ -11,7 +11,7 @@[metabase.lib.test-util.macros :as lib.tu.macros][metabase.query-processor :as qp][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.util.add-alias-info :as add][metabase.query-processor.util.nest-query :as nest-query][metabase.test :as mt]diff --git a/test/metabase/query_processor/util_test.clj b/test/metabase/query_processor/util_test.cljdeleted file mode 100644index 8f3e99cca54..00000000000--- a/test/metabase/query_processor/util_test.clj+++ /dev/null@@ -1,149 +0,0 @@-(ns metabase.query-processor.util-test- "Tests for various functions that provide information about the query."- (:require- [buddy.core.codecs :as codecs]- [clojure.test :refer :all]- [metabase.query-processor.util :as qp.util]- [metabase.test :as mt]- [metabase.util.malli :as mu]))--(set! *warn-on-reflection* true)--(defn- query-hash-hex [query]- (codecs/bytes->hex- (mu/disable-enforcement- (qp.util/query-hash query))))--(deftest ^:parallel query-hash-test- (testing "qp.util/query-hash"- (testing "should always hash something the same way, every time"- (is (= "840eb7aa2a9935de63366bacbe9d97e978a859e93dc792a0334de60ed52f8e99"- (query-hash-hex {:query :abc})))- (is (= (query-hash-hex {:query :def})- (query-hash-hex {:query :def})))- (let [q (mt/mbql-query products- {:aggregation [[:count]]- :breakout [$category]- :order-by [[:asc [:aggregation 0]]]})]- (is (= (query-hash-hex q)- (query-hash-hex q)))))- (testing "should handle parameter values that mix regular numbers with bigintegers stored as strings"- (let [q1 (-> (mt/mbql-query orders)- (assoc :parameters [{:type :number, :name "p1", :value [1 "9223372036854775808"]}]))- q2 (-> (mt/mbql-query orders)- (assoc :parameters [{:type :number, :name "p1", :value ["9223372036854775808" 1]}]))]- (is (some? (query-hash-hex q1)))- (is (= (query-hash-hex q1) (query-hash-hex q2)))))))--(deftest ^:parallel ignore-lib-uuids-test- (letfn [(query []- {:lib/type :mbql/query- :database 1- :stages [{:lib/type :mbql.stage/mbql- :source-table 1- :filters [[:=- {:lib/uuid (str (random-uuid))}- 1- 2]]}]})]- (is (= "1494a478589d855b78e4eaf6bfc9d19f080682f89d0eb110c14af6a740436f2e"- (query-hash-hex (query))- (query-hash-hex (query))))))--(deftest ^:parallel query-hash-test-2- (testing "qp.util/query-hash"- (testing "different queries should produce different hashes"- (are [x y] (not= (query-hash-hex x)- (query-hash-hex y))- {:lib/type :type/query, :stages [{:stage-type :abc}]}- {:lib/type :type/query, :stages [{:stage-type :def}]}-- {:lib/type :type/query, :database 1}- {:lib/type :type/query, :database 2}-- {:lib/type :type/query, :stages [{:lib/type :mbql.stage/mbql}]}- {:lib/type :type/query, :stages [{:lib/type :mbql.stage/native}]}-- {:lib/type :type/query, :parameters [1]}- {:lib/type :type/query, :parameters [2]}-- {:lib/type :type/query, :constraints {:max-rows 1000}}- {:lib/type :type/query, :constraints nil}-- (mt/mbql-query products- {:aggregation [[:count] [:cum-count]]- :breakout [$category]- :order-by [[:asc [:aggregation 0]]]})- (mt/mbql-query products- {:aggregation [[:count] [:cum-count]]- :breakout [$category]- :order-by [[:asc [:aggregation 1]]]})-- (mt/mbql-query products- {:aggregation [[:count] [:cum-count]],- :breakout [$category],- :order-by [[:asc $created_at]]})- (mt/mbql-query products- {:aggregation [[:count] [:cum-count]],- :breakout [$category],- :order-by [[:asc $rating]]})))))--(deftest ^:parallel query-hash-test-3- (testing "qp.util/query-hash"- (testing "keys that are irrelevant to the query should be ignored"- (is (= (query-hash-hex {:query :abc, :random :def})- (query-hash-hex {:query :abc, :random :xyz}))))))--(deftest ^:parallel query-hash-test-4- (testing "qp.util/query-hash"- (testing "empty `:parameters` lists should not affect the hash"- (is (= (query-hash-hex {:query :abc})- (query-hash-hex {:query :abc, :parameters []})- (query-hash-hex {:query :abc, :parameters nil}))))))--(deftest ^:parallel query-hash-test-5- (testing "qp.util/query-hash"- (testing "empty `:parameters` lists should not affect the hash"- (testing "...but non-empty ones should"- (is (not (= (query-hash-hex {:query :abc})- (query-hash-hex {:query :abc, :parameters ["ABC"]}))))))))--(deftest ^:parallel query-hash-test-6- (testing "qp.util/query-hash"- (testing "empty `:parameters` lists should not affect the hash"- (testing (str "the presence of a `nil` value for `:constraints` should produce the same hash as not including "- "the key at all")- (is (= (query-hash-hex {:query :abc})- (query-hash-hex {:query :abc, :constraints nil})- (query-hash-hex {:query :abc, :constraints {}})))))))--(deftest ^:parallel query-hash-test-7- (testing "qp.util/query-hash"- (testing "empty `:parameters` lists should not affect the hash"- (testing (str "the presence of a `nil` value for `:constraints` should produce the same hash as not including "- "the key at all")- (is (= (query-hash-hex {:query :abc})- (query-hash-hex {:query :abc, :constraints nil})- (query-hash-hex {:query :abc, :constraints {}})))))))--(deftest ^:parallel query-hash-test-8- (testing "qp.util/query-hash"- (testing "make sure two different native queries have different hashes!"- (is (not= (query-hash-hex {:database (mt/id)- :type :native- :native {:query "SELECT pg_sleep(15), 1 AS one"}})- (query-hash-hex {:database (mt/id)- :type :native- :native {:query "SELECT pg_sleep(15), 2 AS two"}}))))))--(deftest ^:parallel key-order-should-not-affect-query-hash-test- (is (= "7e144bc5b43ee850648f353cda978b2911e2f66260ac03c5e1744bce6ca668ff"- (query-hash-hex {:parameters [{:value 1, :name "parameter"}]})- (query-hash-hex {:parameters [{:name "parameter", :value 1}]}))))--(deftest ^:parallel parameter-order-should-not-affect-query-hash-test- (is (= "db4583f6192234b4bfc076446a0836a4438e5d0182a1ef9a9d737ad6ea180617"- (query-hash-hex {:parameters [{:name "parameter", :value ["a" "b"]}]})- (query-hash-hex {:parameters [{:name "parameter", :value ["b" "a"]}]})))- (is (= "e51a6617a34f3955a8d9b275535c2d0a611ce0bace4f52630229ba9975ed44c3"- (query-hash-hex {:parameters [{:name "parameter", :value [1 2]}]})- (query-hash-hex {:parameters [{:name "parameter", :value [2 1]}]}))))diff --git a/test/metabase/query_processor_test/aggregation_test.clj b/test/metabase/query_processor_test/aggregation_test.cljindex 183a84335dc..c1e52bb05c8 100644--- a/test/metabase/query_processor_test/aggregation_test.clj+++ b/test/metabase/query_processor_test/aggregation_test.clj@@ -7,7 +7,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.util :as u]))diff --git a/test/metabase/query_processor_test/breakout_test.clj b/test/metabase/query_processor_test/breakout_test.cljindex 69716163f07..a39ed74aece 100644--- a/test/metabase/query_processor_test/breakout_test.clj+++ b/test/metabase/query_processor_test/breakout_test.clj@@ -9,7 +9,7 @@[metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp][metabase.query-processor.middleware.add-remaps :as qp.add-remaps]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.dataset-definitions :as defs]diff --git a/test/metabase/query_processor_test/coercion_test.clj b/test/metabase/query_processor_test/coercion_test.cljindex eb12a845036..f11e7f96009 100644--- a/test/metabase/query_processor_test/coercion_test.clj+++ b/test/metabase/query_processor_test/coercion_test.clj@@ -6,7 +6,7 @@[metabase.lib.test-util :as lib.tu][metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.types.core :as types]diff --git a/test/metabase/query_processor_test/count_where_test.clj b/test/metabase/query_processor_test/count_where_test.cljindex 31201ce585b..d63b3ce3256 100644--- a/test/metabase/query_processor_test/count_where_test.clj+++ b/test/metabase/query_processor_test/count_where_test.clj@@ -3,7 +3,7 @@[clojure.test :refer :all][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(deftest ^:parallel basic-testdiff --git a/test/metabase/query_processor_test/date_bucketing_test.clj b/test/metabase/query_processor_test/date_bucketing_test.cljindex faffab44c40..79afb1d5093 100644--- a/test/metabase/query_processor_test/date_bucketing_test.clj+++ b/test/metabase/query_processor_test/date_bucketing_test.clj@@ -31,7 +31,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.format-rows :as format-rows][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.interface :as tx]diff --git a/test/metabase/query_processor_test/date_time_zone_functions_test.clj b/test/metabase/query_processor_test/date_time_zone_functions_test.cljindex 5c3a5c8fe4a..637fb48215f 100644--- a/test/metabase/query_processor_test/date_time_zone_functions_test.clj+++ b/test/metabase/query_processor_test/date_time_zone_functions_test.clj@@ -12,7 +12,7 @@[metabase.query-processor :as qp][metabase.query-processor.middleware.annotate :as annotate][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.timezone :as qp.timezone][metabase.test :as mt][metabase.test.data.interface :as tx]diff --git a/test/metabase/query_processor_test/distinct_where_test.clj b/test/metabase/query_processor_test/distinct_where_test.cljindex db07c49f58b..539ffeeee51 100644--- a/test/metabase/query_processor_test/distinct_where_test.clj+++ b/test/metabase/query_processor_test/distinct_where_test.clj@@ -2,7 +2,7 @@(:require[clojure.test :refer :all][metabase.lib.test-util :as lib.tu]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(deftest ^:parallel basic-testdiff --git a/test/metabase/query_processor_test/drill_thru_e2e_test.clj b/test/metabase/query_processor_test/drill_thru_e2e_test.cljindex d2fff4a4567..e76cae3b96d 100644--- a/test/metabase/query_processor_test/drill_thru_e2e_test.clj+++ b/test/metabase/query_processor_test/drill_thru_e2e_test.clj@@ -7,7 +7,7 @@[metabase.lib.ref :as lib.ref][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(deftest ^:parallel quick-filter-on-bucketed-date-testdiff --git a/test/metabase/query_processor_test/explicit_joins_test.clj b/test/metabase/query_processor_test/explicit_joins_test.cljindex 1d7a28cee22..a69f926cc7f 100644--- a/test/metabase/query_processor_test/explicit_joins_test.clj+++ b/test/metabase/query_processor_test/explicit_joins_test.clj@@ -16,7 +16,7 @@[metabase.query-processor-test.timezones-test :as timezones-test][metabase.query-processor.compile :as qp.compile][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.interface :as tx]diff --git a/test/metabase/query_processor_test/field_ref_repro_test.clj b/test/metabase/query_processor_test/field_ref_repro_test.cljindex 7839ef3994d..485738773be 100644--- a/test/metabase/query_processor_test/field_ref_repro_test.clj+++ b/test/metabase/query_processor_test/field_ref_repro_test.clj@@ -10,7 +10,7 @@[metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt]))diff --git a/test/metabase/query_processor_test/filter_test.clj b/test/metabase/query_processor_test/filter_test.cljindex 99674be6f93..f05c1f2c963 100644--- a/test/metabase/query_processor_test/filter_test.clj+++ b/test/metabase/query_processor_test/filter_test.clj@@ -12,7 +12,7 @@[metabase.query-processor-test.timezones-test :as timezones-test][metabase.query-processor.compile :as qp.compile][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.util :as u]))diff --git a/test/metabase/query_processor_test/implicit_joins_test.clj b/test/metabase/query_processor_test/implicit_joins_test.cljindex 2c1b7a93440..71b59cc93e4 100644--- a/test/metabase/query_processor_test/implicit_joins_test.clj+++ b/test/metabase/query_processor_test/implicit_joins_test.clj@@ -7,7 +7,7 @@[metabase.lib.core :as lib][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.util.date-2 :as u.date]))diff --git a/test/metabase/query_processor_test/native_test.clj b/test/metabase/query_processor_test/native_test.cljindex f773b71eb25..1de12377e48 100644--- a/test/metabase/query_processor_test/native_test.clj+++ b/test/metabase/query_processor_test/native_test.clj@@ -4,7 +4,7 @@[metabase.lib.core :as lib][metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.util :as u]))diff --git a/test/metabase/query_processor_test/nested_queries_test.clj b/test/metabase/query_processor_test/nested_queries_test.cljindex 03f0d9f8d4f..c45000f7cbe 100644--- a/test/metabase/query_processor_test/nested_queries_test.clj+++ b/test/metabase/query_processor_test/nested_queries_test.clj@@ -24,7 +24,7 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.interface :as tx]diff --git a/test/metabase/query_processor_test/parameters_test.clj b/test/metabase/query_processor_test/parameters_test.cljindex 8eab1d6f488..f3a46506319 100644--- a/test/metabase/query_processor_test/parameters_test.clj+++ b/test/metabase/query_processor_test/parameters_test.clj@@ -16,7 +16,6 @@[metabase.permissions.models.permissions-group :as perms-group][metabase.query-processor :as qp][metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.util :as u][metabase.util.date-2 :as u.date]))@@ -51,8 +50,8 @@(assoc (mt/count-with-template-tag-query driver/*driver* table field param-type):template-tags {(name field) {:name (name field):display-name (name field)- :type (or (namespace param-type)- (name param-type))}}))]+ :type (keyword (or (namespace param-type)+ (name param-type)))}}))](if defaults?(query-with-default-parameter-value query field param-value)(assoc query :parameters [{:type param-type@@ -253,30 +252,32 @@(deftest ^:parallel filter-nested-queries-test(mt/test-drivers (mt/normal-drivers-with-feature :native-parameters :nested-queries)- (qp.store/with-metadata-provider (lib.tu/mock-metadata-provider- (mt/metadata-provider)- {:cards [{:id 1- :dataset-query (mt/native-query (qp.compile/compile (mt/mbql-query checkins)))}]})- (let [date (lib.metadata/field (qp.store/metadata-provider) (mt/id :checkins :date))- query (assoc (mt/mbql-query nil- {:source-table "card__1", :limit 5})- :parameters [{:type :date/all-options- :target [:dimension [:field (:name date) {:base-type :type/Date}]]- :value "2014-01-06"}])]- (testing "We should be able to apply filters to queries that use native queries with parameters as their source (#9802)"+ (let [mp (lib.tu/mock-metadata-provider+ (mt/metadata-provider)+ {:cards [{:id 1+ :dataset-query (mt/native-query (qp.compile/compile (mt/mbql-query checkins)))}]})+ date (lib.metadata/field mp (mt/id :checkins :date))+ query (lib/query+ mp+ (assoc (mt/mbql-query nil+ {:source-table "card__1", :limit 5})+ :parameters [{:type :date/all-options+ :target [:dimension [:field (:name date) {:base-type :type/Date}]]+ :value "2014-01-06"}]))]+ (testing "We should be able to apply filters to queries that use native queries with parameters as their source (#9802)"+ (is (= [[182 "2014-01-06T00:00:00Z" 5 31]]+ (mt/formatted-rows+ :checkins+ (qp/process-query query)))))+ (testing "We should be able to apply filters explicitly targeting nested native stages (#48258)"+ (let [query' (assoc-in query [:parameters 0 :target 2 :stage-number] 0)]+ (is (=? {:parameters [{:target [:dimension [:field (:name date) {:base-type :type/Date}] {:stage-number 0}]+ :value "2014-01-06"}]}+ query'))(is (= [[182 "2014-01-06T00:00:00Z" 5 31]](mt/formatted-rows:checkins- (qp/process-query query)))))- (testing "We should be able to apply filters explicitly targeting nested native stages (#48258)"- (let [query' (assoc-in query [:parameters 0 :target 2 :stage-number] 0)]- (is (=? {:parameters [{:target [:dimension [:field (:name date) {:base-type :type/Date}] {:stage-number 0}]- :value "2014-01-06"}]}- query'))- (is (= [[182 "2014-01-06T00:00:00Z" 5 31]]- (mt/formatted-rows- :checkins- (qp/process-query query'))))))))))+ (qp/process-query query')))))))))(deftest ^:parallel string-escape-test;; test `:sql` drivers that support native parameters@@ -526,6 +527,7 @@:name "cat":display-name "Cat":type :dimension+ :widget-type :number:dimension [:field (mt/id :products :category) nil]}}})](is (= [200](mt/first-row (qp/process-query query))))))))@@ -693,6 +695,7 @@(qp/process-query {:database (mt/id):type :native:native (dissoc (qp.compile/compile (:dataset_query card-2))+ :lib/type:query-permissions/referenced-card-ids)}))))))(let [query (mt/native-query{:query (mt/native-query-with-card-template-tag driver/*driver* "card")diff --git a/test/metabase/query_processor_test/remapping_test.clj b/test/metabase/query_processor_test/remapping_test.cljindex 9d41b9bd24d..fad77daf11a 100644--- a/test/metabase/query_processor_test/remapping_test.clj+++ b/test/metabase/query_processor_test/remapping_test.clj@@ -10,7 +10,7 @@[metabase.query-processor.middleware.add-remaps :as qp.add-remaps][metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.test :as mt][metabase.test.data.dataset-definitions :as defs]diff --git a/test/metabase/query_processor_test/sum_where_test.clj b/test/metabase/query_processor_test/sum_where_test.cljindex d7075d24959..527269b95b6 100644--- a/test/metabase/query_processor_test/sum_where_test.clj+++ b/test/metabase/query_processor_test/sum_where_test.clj@@ -2,7 +2,7 @@(:require[clojure.test :refer :all][metabase.lib.test-util :as lib.tu]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]))(deftest ^:parallel basic-testdiff --git a/test/metabase/query_processor_test/timezones_test.clj b/test/metabase/query_processor_test/timezones_test.cljindex 470c10e49a3..54ada2bd9dd 100644--- a/test/metabase/query_processor_test/timezones_test.clj+++ b/test/metabase/query_processor_test/timezones_test.clj@@ -9,7 +9,7 @@[metabase.driver.util :as driver.u][metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt][metabase.test.data.sql :as sql.tx][metabase.util.date-2 :as u.date]))diff --git a/test/metabase/revisions/impl/card_test.clj b/test/metabase/revisions/impl/card_test.cljindex 29ea1bfb0f2..a30142b41f8 100644--- a/test/metabase/revisions/impl/card_test.clj+++ b/test/metabase/revisions/impl/card_test.clj@@ -160,10 +160,8 @@changes (update before :result_metadata drop-last)](t2/update! :model/Card (:id card) changes)(create-card-revision! (:id card) false)-(testing "we should track when :result_metadata changes on model"(is (= 1 (t2/count :model/Revision :model "Card" :model_id (:id card)))))-(testing "we should have a revision description for :result_metadata on model"(is (some? (u/build-sentence(revision/diff-strings@@ -181,7 +179,6 @@serialized-card (revision/serialize-instance :model/Card card-id full-card);; Remove card_schema to simulate pre-v0.55 revisionold-card-data (dissoc serialized-card :card_schema)]-;; Manually create a revision without :card_schema to simulate pre-v0.55 data(t2/insert! :model/Revision{:model "Card"@@ -195,7 +192,6 @@(is (seq revisions))(is (= "Test revision without card_schema"(-> revisions first :message)))))-(testing "Revision object has card_schema added with legacy default after after-select"(let [revision (t2/select-one :model/Revision:model "Card"@@ -204,7 +200,6 @@;; The after-select should have added `:card_schema`revision-obj (:object revision)](is (= queries/starting-card-schema-version (:card_schema revision-obj)))))-(testing "Card object from revision can go through upgrade-card-schema-to-latest";; Actual regression test; this used to throw:;; "Cannot SELECT a Card without including :card_schema"diff --git a/test/metabase/revisions/models/revision_test.clj b/test/metabase/revisions/models/revision_test.cljindex f894698d052..5873545d019 100644--- a/test/metabase/revisions/models/revision_test.clj+++ b/test/metabase/revisions/models/revision_test.clj@@ -58,14 +58,15 @@(testing (str "make sure we call the appropriate post-select methods on `:object` when a revision comes out of the ""DB. This is especially important for things like Cards where we need to make sure query is ""normalized")- (is (= {:model "Card", :object {:card_schema queries/starting-card-schema-version- :dataset_query {:database (mt/id)- :type :query- :query {:source-table (mt/id :venues)}}}}- (mt/derecordize- (mi/do-after-select :model/Revision {:model "Card", :object {:dataset_query {:database (mt/id)- :type "query"- :query {:source-table (mt/id :venues)}}}}))))))+ (is (=? {:model "Card"+ :object {:card_schema queries/starting-card-schema-version+ :dataset_query {:database (mt/id)+ :lib/type :mbql/query+ :stages [{:source-table (mt/id :venues)}]}}}+ (mt/derecordize+ (mi/do-after-select :model/Revision {:model "Card", :object {:dataset_query {:database (mt/id)+ :type "query"+ :query {:source-table (mt/id :venues)}}}}))))));;; # Default diff-* implementationsdiff --git a/test/metabase/search/api_test.clj b/test/metabase/search/api_test.cljindex 5d561b17022..3cbacbbd53e 100644--- a/test/metabase/search/api_test.clj+++ b/test/metabase/search/api_test.clj@@ -1385,14 +1385,14 @@:model/Card {mbql-model :id} {:name search-term :type :model}:model/Card {native-model-in-name :id} {:name search-term :type :model}:model/Card {native-model-in-query :id} {:dataset_query (mt/native-query {:query (format "select %s" search-term)}) :type :model}]+ (is (= :native+ (t2/select-one-fn :query_type :model/Card :id native-card-in-query)))(mt/with-actions[_ {:type :model :dataset_query (mt/mbql-query venues)}{http-action :action-id} {:type :http :name search-term}{query-action :action-id} {:type :query :dataset_query (mt/native-query {:query (format "delete from %s" search-term)})}]-;; TODO investigate why the actions don't get indexed automatically(search/reindex! {:async? false :in-place? true})-(testing "by default do not search for native content"(is (= #{["card" mbql-card]["card" native-card-in-name]@@ -1403,14 +1403,12 @@:data(map (juxt :model :id))set))))-(testing "if search-native-query is true, search both dataset_query and the name"(is (= #{["card" mbql-card]["card" native-card-in-name]["dataset" mbql-model]["dataset" native-model-in-name]["action" http-action]-["card" native-card-in-query]["dataset" native-model-in-query]["action" query-action]}@@ -1431,21 +1429,18 @@:name search-term}:model/Card {card-id-2 :id} {:creator_id user-id-2:name search-term}]-(revision/push-revision!{:entity :model/Card:id card-id-1:user-id user-id-1:is-creation? true:object {:id card-id-1 :type "question"}})-(revision/push-revision!{:entity :model/Card:id card-id-2:user-id user-id-2:is-creation? true:object {:id card-id-2 :type "question"}})-(testing "search result should returns creator_common_name and last_editor_common_name"(is (= #{["card" card-id-1 "Ngoc Khuat" "Ngoc Khuat"];; for user that doesn't have first_name or last_name, should fall backs to emaildiff --git a/test/metabase/search/appdb/index_test.clj b/test/metabase/search/appdb/index_test.cljindex 2e7b84091e7..d945f62f431 100644--- a/test/metabase/search/appdb/index_test.clj+++ b/test/metabase/search/appdb/index_test.clj@@ -5,6 +5,8 @@[metabase.app-db.core :as mdb][metabase.config.core :as config][metabase.indexed-entities.models.model-index :as model-index]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.search.appdb.index :as search.index][metabase.search.core :as search][metabase.search.engine :as search.engine]@@ -172,7 +174,7 @@(ingest! model [:= :this.name entity-name])(fetch-one model :name entity-name))-(def default-index-entity+(def ^:parallel default-index-entity{:model nil:model_id nil:name nil@@ -464,9 +466,10 @@:collection_id coll-id})](fetch "indexed-entity" :name (:name miv)))))(testing "Changing values syncs the index"- (t2/update! :model/Card (:id model) (assoc-in model- [:dataset_query :query :filter]- [:!= (mt/$ids :products $id) (:model_pk miv)]))+ (t2/update! :model/Card (:id model) (update model :dataset_query (fn [query]+ (let [products-id (lib.metadata/field query (mt/id :products :id))]+ (-> query+ (lib/filter (lib/!= products-id (:model_pk miv))))))))(model-index/add-values! model-index)(let [miv2 (t2/select-one :model/ModelIndexValue :model_index_id (:id model-index))](is (=? [(index-entitydiff --git a/test/metabase/setup/api_test.clj b/test/metabase/setup/api_test.cljindex 779d8ba5aec..5682a69c60f 100644--- a/test/metabase/setup/api_test.clj+++ b/test/metabase/setup/api_test.clj@@ -175,11 +175,11 @@(testing "site locale"(testing "invalid format"(is (=? {:specific-errors {:prefs {:site_locale ["valid locale, received: \"eng-USA\""]}}- :errors {:prefs #(str/includes? % ":site_locale (optional) -> <nullable String must be a valid two-letter ISO language or language-country code e.g. 'en' or 'en_US'.>")}}+ :errors {:prefs {:site_locale "nullable String must be a valid two-letter ISO language or language-country code e.g. 'en' or 'en_US'."}}}(setup! assoc-in [:prefs :site_locale] "eng-USA"))))(testing "non-existent locale"(is (=? {:specific-errors {:prefs {:site_locale ["valid locale, received: \"en-EN\""]}}- :errors {:prefs #(str/includes? % ":site_locale (optional) -> <nullable String must be a valid two-letter ISO language or language-country code e.g. 'en' or 'en_US'.>")}}+ :errors {:prefs {:site_locale "nullable String must be a valid two-letter ISO language or language-country code e.g. 'en' or 'en_US'."}}}(setup! assoc-in [:prefs :site_locale] "en-EN")))))))(deftest setup-validation-test-4@@ -194,17 +194,17 @@(is (:id (setup! 200 assoc-in [:user :last_name] nil)))))(testing "email"(testing "missing"- (is (=? {:errors {:user #(str/includes? % ":email -> <value must be a valid email address.>")}+ (is (=? {:errors {:user {:email "value must be a valid email address."}}:specific-errors {:user {:email ["missing required key, received: nil"]}}}(setup! m/dissoc-in [:user :email]))))(testing "invalid"- (is (=? {:errors {:user #(str/includes? % ":email -> <value must be a valid email address.>")}+ (is (=? {:errors {:user {:email "value must be a valid email address."}}:specific-errors {:user {:email ["valid email address, received: \"anything\""]}}}(setup! assoc-in [:user :email] "anything")))))(testing "password"(testing "missing"(is (=? {:specific-errors {:user {:password ["missing required key, received: nil"]}}- :errors {:user #(str/includes? % ":password -> <password is too common.>")}}+ :errors {:user {:password "password is too common."}}}(setup! m/dissoc-in [:user :password]))))(testing "invalid"(is (=? {:specific-errors {:user {:password ["valid password that is not too common, received: \"anything\""]}}}diff --git a/test/metabase/task_history/api_test.clj b/test/metabase/task_history/api_test.cljindex e684d3795dd..830dfb03705 100644--- a/test/metabase/task_history/api_test.clj+++ b/test/metabase/task_history/api_test.clj@@ -162,11 +162,11 @@(let [response (mt/user-http-request :crowberto :get 200 "task/")](is (= 3 (-> response :data count)))))(testing "Error is returned on explicit nil status"- (is (= "enum of :started, :unknown, :success, :failed"- (-> (mt/user-http-request :crowberto :get 400 "task/" :status nil) :errors :status first))))+ (is (=? {:errors {:status "enum of :started, :unknown, :success, :failed"}}+ (mt/user-http-request :crowberto :get 400 "task/" :status nil))))(testing "Error is returned for unexpected status values"- (is (= "enum of :started, :unknown, :success, :failed"- (-> (mt/user-http-request :crowberto :get 400 "task/" :status 1) :errors :status first))))+ (is (=? {:errors {:status "enum of :started, :unknown, :success, :failed"}}+ (mt/user-http-request :crowberto :get 400 "task/" :status 1))))(letfn [(task-test-filtering-response[task](testing (format "Filtering for `%s` named task works correctly" task)diff --git a/test/metabase/test.clj b/test/metabase/test.cljindex f9aea4e9ad6..cd6a2a4f034 100644--- a/test/metabase/test.clj+++ b/test/metabase/test.clj@@ -16,12 +16,12 @@[metabase.core.init][metabase.driver :as driver][metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.model-persistence.test-util][metabase.permissions.test-util :as perms.test-util][metabase.premium-features.test-util :as premium-features.test-util][metabase.query-processor :as qp]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test-util :as qp.test-util][metabase.request.core][metabase.test-runner.assert-exprs :as test-runner.assert-exprs]@@ -71,7 +71,7 @@driver/keep-mei18n.tu/keep-meinitialize/keep-me- lib.metadata.jvm/keep-me+ lib-be/keep-memb.hawk.init/keep-memdb.test-util/keep-memetabase.channel.email-test/keep-me@@ -168,7 +168,7 @@[initializeinitialize-if-needed!]- [lib.metadata.jvm+ [lib-beapplication-database-metadata-provider][metabase.util.log.capturediff --git a/test/metabase/test/data.clj b/test/metabase/test/data.cljindex f3fe19907a3..571da5f4b4e 100644--- a/test/metabase/test/data.clj+++ b/test/metabase/test/data.clj@@ -43,7 +43,9 @@[metabase.driver :as driver][metabase.driver.ddl.interface :as ddl.i][metabase.driver.util :as driver.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.legacy-mbql.normalize :as mbql.normalize]+ [metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib-be.core :as lib-be][metabase.lib.schema.id :as lib.schema.id][metabase.permissions.models.permissions-group :as perms-group][metabase.query-processor :as qp]@@ -177,13 +179,13 @@(declare id)-(mu/defn native-query+(mu/defn native-query :- ::mbql.s/Query"Like `mbql-query`, but for native queries."- [inner-native-query :- [:map- [:query some?]]]+ [inner-native-query :- :map]+ #_{:clj-kondo/ignore [:deprecated-var]}{:database (id):type :native- :native inner-native-query})+ :native (mbql.normalize/normalize-fragment [:native] inner-native-query)})(defn run-mbql-query* [query];; catch the Exception and rethrow with the query itself so we can have a little extra info for debugging if it fails.@@ -234,7 +236,7 @@(defn metadata-provider"Get a metadata-provider for the current database."[]- (lib.metadata.jvm/application-database-metadata-provider (id)))+ (lib-be/application-database-metadata-provider (id)))(defmacro dataset"Create a database and load it with the data defined by `dataset`, then do a quick metadata-only sync; make it thediff --git a/test/metabase/test/data/mbql_query_impl.cljc b/test/metabase/test/data/mbql_query_impl.cljcindex ffbf318623a..7e94378e6a8 100644--- a/test/metabase/test/data/mbql_query_impl.cljc+++ b/test/metabase/test/data/mbql_query_impl.cljc@@ -1,8 +1,8 @@(ns metabase.test.data.mbql-query-impl"Internal implementation of [[metabase.test.data/$ids]] and [[metabase.test.data/$ids]] and related macros."(:require- #?@(:clj ([metabase.query-processor.store :as qp.store]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ #?@(:clj (^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]+ [metabase.lib-be.core :as lib-be][metabase.lib.metadata :as lib.metadata]))[clojure.string :as str][clojure.walk :as walk]@@ -113,7 +113,7 @@(defn- metadata-provider [](if (qp.store/initialized?)(qp.store/metadata-provider)- (lib.metadata.jvm/application-database-metadata-provider ((requiring-resolve 'metabase.test.data/id))))))+ (lib-be/application-database-metadata-provider ((requiring-resolve 'metabase.test.data/id))))))#?(:clj(defn field-name [field-id]diff --git a/test/metabase/test/generate.clj b/test/metabase/test/generate.cljindex 8aeae265d41..4fa48e56d82 100644--- a/test/metabase/test/generate.clj+++ b/test/metabase/test/generate.clj@@ -97,8 +97,8 @@;; * card(s/def ::display #{:table})-(s/def ::visualization_settings #{"{}"})-(s/def ::dataset_query #{"{}"})+(s/def ::visualization_settings #{{}})+(s/def ::dataset_query #{{}});; * dashboardcard_series(s/def ::position pos-int?)@@ -138,7 +138,7 @@(s/def ::col pos-int?)(s/def ::size_x pos-int?)(s/def ::size_y pos-int?)-(s/def ::parameter_mappings #{[{}]})+(s/def ::parameter_mappings #{[]})(s/def ::action (s/keys :req-un [::id :action/type ::name]))(s/def ::query-action (s/keys :req-un [::dataset_query]))@@ -147,7 +147,6 @@(s/def ::core-user (s/keys :req-un [::id ::first_name ::last_name ::email ::password]))(s/def ::collection (s/keys :req-un [::id ::name]))-(s/def ::activity (s/keys :req-un [::id ::topic ::details ::timestamp]))(s/def ::pulse (s/keys :req-un [::id ::name]))(s/def ::permissions-group (s/keys :req-un [::id ::name]))(s/def ::permissions-group-membership (s/keys :req-un [::user_id ::group_id]))diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex 4797457f1e0..c8c5819ceb4 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -878,7 +878,16 @@(created-query-cache!)(is cached?))- Only works for models that have a numeric primary key e.g. `:id`."+ Only works for models that have a numeric primary key e.g. `:id`.++ # TODO (Cam 9/29/25)++ I'm planning on deprecating and removing this in near future. Instead of using this you can do++ (t2/with-transaction [_conn nil {:rollback-only true}]+ ...)++ which is thread-safe."[models & body]`(do-with-model-cleanup ~models (fn [] ~@body)))diff --git a/test/metabase/tiles/api_test.clj b/test/metabase/tiles/api_test.cljindex ec1f0fac559..38421095822 100644--- a/test/metabase/tiles/api_test.clj+++ b/test/metabase/tiles/api_test.clj@@ -1,8 +1,9 @@(ns metabase.tiles.api-test"Tests for `/api/tiles` endpoints."(:require- [clojure.set :as set][clojure.test :refer :all]+ [metabase.lib.core :as lib]+ [metabase.lib.test-metadata :as meta][metabase.test :as mt][metabase.tiles.api :as api.tiles][metabase.util :as u]@@ -42,57 +43,74 @@(defn encoded-lat-field-ref"JSON-encoded latitude field ref for the People table"- []- (json/encode (mt/$ids $people.latitude)))+ ([]+ (encoded-lat-field-ref :mbql))+ ([mbql-or-native]+ (json/encode+ (case mbql-or-native+ :mbql (mt/$ids $people.latitude)+ :native [:field "LATITUDE" {:base-type :type/Float}]))))(defn encoded-lon-field-ref"JSON-encoded longitude field ref for the People table"- []- (json/encode (mt/$ids $people.longitude)))+ ([]+ (encoded-lon-field-ref :mbql))+ ([mbql-or-native]+ (json/encode+ (case mbql-or-native+ :mbql (mt/$ids $people.longitude)+ :native [:field "LONGITUDE" {:base-type :type/Float}]))))-(deftest ad-hoc-query-test+(deftest ^:parallel ad-hoc-query-test(testing "GET /api/tiles/:zoom/:x/:y with latField and lonField query params"(is (png? (mt/user-http-request:crowberto :get 200 "tiles/4/2/4":query (json/encode (venues-query))- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)))))+ :latField (encoded-lat-field-ref :mbql)+ :lonField (encoded-lon-field-ref :mbql))))))++(deftest ^:parallel ad-hoc-query-test-2(testing "Works on native queries"(is (png? (mt/user-http-request:crowberto :get 200 "tiles/1/1/1":query (json/encode (native-query))- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref))))))+ :latField (encoded-lat-field-ref :native)+ :lonField (encoded-lon-field-ref :native))))))-(deftest saved-card-test+(deftest ^:parallel saved-card-test(testing "GET /api/tiles/:card-id/:zoom/:x/:y with latField and lonField query params"(testing "MBQL saved card"(mt/with-temp [:model/Card card {:dataset_query (venues-query)}](is (png? (mt/user-http-request:crowberto :get 200 (format "tiles/%d/1/1/1" (u/id card))- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref))))))+ :latField (encoded-lat-field-ref :mbql)+ :lonField (encoded-lon-field-ref :mbql))))))))+(deftest ^:parallel saved-card-test-2+ (testing "GET /api/tiles/:card-id/:zoom/:x/:y with latField and lonField query params"(testing "Native saved card"(mt/with-temp [:model/Card card {:dataset_query (native-query)}](testing "GET /api/tiles/:card-id/:zoom/:x/:y with latField and lonField query params"(is (png? (mt/user-http-request:crowberto :get 200 (format "tiles/%d/1/1/1" (u/id card))- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)))))))+ :latField (encoded-lat-field-ref :native)+ :lonField (encoded-lon-field-ref :native)))))))))+(deftest ^:parallel saved-card-test-3+ (testing "GET /api/tiles/:card-id/:zoom/:x/:y with latField and lonField query params"(testing "Parameterized native saved card"(mt/with-temp [:model/Card card {:dataset_query (parameterized-native-query)}](testing "GET /api/tiles/:card-id/:zoom/:x/:y with latField and lonField query params"(is (png? (mt/user-http-request:crowberto :get 200 (format "tiles/%d/1/1/1" (u/id card))- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)- :parameters (json/encode [{:type :text+ :latField (encoded-lat-field-ref :native)+ :lonField (encoded-lon-field-ref :native)+ :parameters (json/encode [{:id "_STATE_"+ :type :text:target [:variable [:template-tag :state]]:value "CA"}])))))))))-(deftest dashcard-test+(deftest ^:parallel dashcard-test(testing "GET /api/tiles/:dashboard-id/dashcard/:dashcard-id/card/:card-id/:zoom/:x/:y with latField and lonField query params"(testing "MBQL dashcard"(mt/with-temp [:model/Dashboard {dashboard-id :id} {}@@ -104,9 +122,11 @@dashboard-iddashcard-idcard-id)- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref))))))+ :latField (encoded-lat-field-ref :mbql)+ :lonField (encoded-lon-field-ref :mbql))))))))+(deftest ^:parallel dashcard-test-2+ (testing "GET /api/tiles/:dashboard-id/dashcard/:dashcard-id/card/:card-id/:zoom/:x/:y with latField and lonField query params"(testing "Native dashcard"(mt/with-temp [:model/Dashboard {dashboard-id :id} {}:model/Card {card-id :id} {:dataset_query (native-query)}@@ -118,10 +138,10 @@dashboard-iddashcard-idcard-id)- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)))))))))+ :latField (encoded-lat-field-ref :native)+ :lonField (encoded-lon-field-ref :native)))))))))-(deftest parameterized-dashcard-test+(deftest ^:parallel parameterized-dashcard-test(testing "Parameterized mbql dashcard"(mt/with-temp [:model/Dashboard {dashboard-id :id} {:parameters [{:name "State":id "_STATE_"@@ -139,12 +159,13 @@dashboard-iddashcard-idcard-id)- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)+ :latField (encoded-lat-field-ref :mbql)+ :lonField (encoded-lon-field-ref :mbql):parameters (json/encode [{:id "_STATE_"+ :type "text":value ["CA"]}]))))))))-(deftest parameterized-dashcard-test-2+(deftest ^:parallel parameterized-dashcard-test-2(testing "Parameterized native dashcard"(mt/with-temp [:model/Dashboard {dashboard-id :id} {:parameters [{:name "State":id "_STATE_"@@ -162,52 +183,55 @@dashboard-iddashcard-idcard-id)- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)+ :latField (encoded-lat-field-ref :native)+ :lonField (encoded-lon-field-ref :native):parameters (json/encode [{:id "_STATE_"+ :type "text":value ["CA"]}]))))))))-(deftest tiles-query-test- (letfn [(clean [q]- (-> q- (update-in [:query :filter] #(take 3 %))))]- (testing "mbql"- (testing "adds the inside filter and only selects the lat/lon fields"- (let [query {:database 19- :query {:source-table 88- :fields [[:field 562 nil]- [:field 574 nil] ; lat- [:field 576 nil]] ; lon-- :limit 50000}- :type :query}]- (is (= {:database 19- :query {:source-table 88- :fields [[:field 574 nil]- [:field 576 nil]]- :limit 2000- :filter [:inside [:field 574 nil] [:field 576 nil]]}- :type :query}- (clean (#'api.tiles/tiles-query query 2 3 1 [:field 574 nil] [:field 576 nil])))))))-- (testing "native"- (testing "nests the query, selects fields"- (let [query {:type :native- :native {:query "select name, latitude, longitude from zomato limit 5000;"- :template-tags {}}- :database 19}]- (is (= {:database 19- :query {:source-query (-> query :native (set/rename-keys {:query :native}))- :fields [[:field "latitude" {:base-type :type/Float}]- [:field "longitude" {:base-type :type/Float}]]- :filter [:inside- [:field "latitude" {:base-type :type/Float}]- [:field "longitude" {:base-type :type/Float}]]- :limit 2000}- :type :query}- (clean (@#'api.tiles/tiles-query query 2 2 1- [:field "latitude" {:base-type :type/Float}]- [:field "longitude" {:base-type :type/Float}])))))))))+(deftest ^:parallel tiles-query-test+ (testing "mbql"+ (testing "adds the inside filter and only selects the lat/lon fields"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :people))+ (lib/limit 50000)+ (lib/with-fields [(meta/field-metadata :people :id)+ (meta/field-metadata :people :latitude)+ (meta/field-metadata :people :longitude)]))]+ (is (=? {:stages [{}+ ;; should append a new stage+ {:fields [[:field {} "LATITUDE"]+ [:field {} "LONGITUDE"]]+ :limit 2000+ :filters [[:inside {}+ [:field {} "LATITUDE"]+ [:field {} "LONGITUDE"]+ double?+ double?+ double?+ double?]]}]}+ (#'api.tiles/tiles-query query 2 3 1+ [:field (meta/id :people :latitude) nil]+ [:field (meta/id :people :longitude) nil])))))))++(deftest ^:parallel tiles-query-test-2+ (testing "native"+ (testing "nests the query, selects fields"+ (let [query (lib/native-query meta/metadata-provider "select name, latitude, longitude from zomato limit 5000;")]+ (is (=? {:stages [{:native "select name, latitude, longitude from zomato limit 5000;"}+ {:fields [[:field {:base-type :type/Float} "latitude"]+ [:field {:base-type :type/Float} "longitude"]]+ :filters [[:inside+ {}+ [:field {:base-type :type/Float} "latitude"]+ [:field {:base-type :type/Float} "longitude"]+ double?+ double?+ double?+ double?]]+ :limit 2000}]}+ (@#'api.tiles/tiles-query query 2 2 1+ [:field "latitude" {:base-type :type/Float}]+ [:field "longitude" {:base-type :type/Float}])))))))(deftest breakout-query-test(testing "the appropriate lat/lon fields are selected from the results, if the query contains a :breakout clause (#20182)"@@ -216,8 +240,8 @@api.tiles/tile->byte-array identity](let [result (mt/user-http-request:crowberto :get 200 "tiles/7/30/49"- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)+ :latField (encoded-lat-field-ref :mbql)+ :lonField (encoded-lon-field-ref :mbql):query (json/encode(mt/mbql-query people{:breakout [[:field (mt/id :people :latitude)]@@ -233,6 +257,6 @@(is (=? {:status "failed"}(mt/user-http-request:rasta :get 400 "tiles/1/1/1"- :latField (encoded-lat-field-ref)- :lonField (encoded-lon-field-ref)- :query (json/encode (mt/mbql-query venues {:filter [:= $users.id 1]})))))))+ :latField (encoded-lat-field-ref :mbql)+ :lonField (encoded-lon-field-ref :mbql)+ :query (json/encode (mt/mbql-query people {:filter [:= $people.id "X"]})))))))diff --git a/test/metabase/upload/impl_test.clj b/test/metabase/upload/impl_test.cljindex 7fa55ab729f..51f703777d3 100644--- a/test/metabase/upload/impl_test.clj+++ b/test/metabase/upload/impl_test.clj@@ -15,7 +15,7 @@[metabase.driver.mysql :as mysql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.util :as driver.u]- [metabase.lib-be.metadata.jvm :as lib.metadata.jvm]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id]@@ -1148,14 +1148,14 @@{:schema-name schema-name, :auxiliary-sync-steps :synchronous}(fn [model](with-upload-table! [new-table (card->table model)]- (is (=? {:display :table- :database_id db-id- :dataset_query {:database db-id- :query {:source-table (:id new-table)}- :type :query}- :creator_id (mt/user->id :rasta)- :name #"(?i)example csv file(.*)"- :collection_id nil}+ (is (=? {:display :table+ :database_id db-id+ :dataset_query {:database db-id+ :stages [{:source-table (:id new-table)}]+ :lib/type :mbql/query}+ :creator_id (mt/user->id :rasta)+ :name #"(?i)example csv file(.*)"+ :collection_id nil}(t2/select-one :model/Card :table_id (:id new-table)))"A new model is created")(is (=? {:name #"(?i)example(.*)"@@ -1915,7 +1915,7 @@file (csv-file-with csv-rows)other-id (mt/id :venues)other-table (t2/select-one :model/Table other-id)- mp (lib.metadata.jvm/application-database-metadata-provider (:db_id table))]+ mp (lib-be/application-database-metadata-provider (:db_id table))](mt/with-temp [:model/Card {question-id :id} {:table_id table-id, :dataset_query (mbql mp table)}:model/Card {model-id :id} {:table_id table-id, :type :model, :dataset_query (mbql mp table)}diff --git a/test/metabase/warehouse_schema/api/table_test.clj b/test/metabase/warehouse_schema/api/table_test.cljindex 83e6e01a259..f07ac68df8d 100644--- a/test/metabase/warehouse_schema/api/table_test.clj+++ b/test/metabase/warehouse_schema/api/table_test.clj@@ -889,7 +889,10 @@:database_id (mt/id):name "Venues metric":type "metric"- :dataset_query metric-query+ :dataset_query {:database (mt/id)+ :lib/type "mbql/query"+ :stages [{:source-card (:id model)+ :aggregation [["count" {}]]}]}:id (:id metric)}]}(mt/user-http-request :crowberto :get 200(format "table/card__%d/query_metadata" (u/the-id model))))))diff --git a/test/metabase/warehouse_schema/metadata_queries_test.clj b/test/metabase/warehouse_schema/metadata_queries_test.cljindex b19ad63911d..9e7a9a496ac 100644--- a/test/metabase/warehouse_schema/metadata_queries_test.clj+++ b/test/metabase/warehouse_schema/metadata_queries_test.clj@@ -1,68 +1,132 @@(ns metabase.warehouse-schema.metadata-queries-test(:require[clojure.test :refer :all]- [metabase.test :as mt]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.test-util :as lib.tu]+ [metabase.lib.util.match :as lib.util.match][metabase.warehouse-schema.metadata-queries :as metadata-queries]))-(defn- ordered-filter [query]- ;; sort by id [:field id option]- (update query :filter (fn [filter-clause]- (if (#{:and :or} (first filter-clause))- (into [(first filter-clause)] (sort-by second (rest filter-clause)))- filter-clause))))+(defn- sort-by-field-id [clauses]+ (sort-by #(lib.util.match/match-one %+ [:field _opts (id :guard pos-int?)]+ id)+ clauses))-(deftest add-required-filter-if-needed-test- (mt/with-temp- [:model/Database db {:engine :h2}- :model/Table product {:name "PRODUCT" :db_id (:id db)}- :model/Field _product-id {:name "ID" :table_id (:id product) :base_type :type/Integer}- :model/Field _product-name {:name "NAME" :table_id (:id product) :base_type :type/Integer}- :model/Table buyer {:name "BUYER" :database_require_filter true :db_id (:id db)}- :model/Field buyer-id {:name "ID" :table_id (:id buyer) :base_type :type/Integer :database_partitioned true}- :model/Table order {:name "ORDER" :database_require_filter true :db_id (:id db)}- :model/Field order-id {:name "ID" :table_id (:id order) :base_type :type/Integer}- :model/Field _order-buyer-id {:name "BUYER_ID" :table_id (:id order) :base_type :type/Integer}- :model/Field order-product-id {:name "PRODUCT_ID" :table_id (:id order) :base_type :type/Integer :database_partitioned true}]- (mt/with-db db- (testing "no op for tables that do not require filter"- (let [query (:query (mt/mbql-query product))]- (is (= query- (metadata-queries/add-required-filters-if-needed query)))))+(defn- id+ ([table]+ (case table+ :product 1+ :buyer 2+ :order 3))+ ([table field]+ (case table+ :product (case field+ :id 1+ :name 2)+ :buyer (case field+ :id 3)+ :order (case field+ :id 4+ :buyer-id 5+ :product-id 6))))- (testing "if the source table requires a filter, add the partitioned filter"- (let [query (:query (mt/mbql-query order))]- (is (= (assoc query- :filter [:> [:field (:id order-product-id) {:base-type :type/Integer}] -9223372036854775808])- (metadata-queries/add-required-filters-if-needed query)))))+(defn- mp []+ (lib.tu/mock-metadata-provider+ {:database (assoc meta/database :id 1)+ :tables [(assoc (meta/table-metadata :products) :name "PRODUCT", :db-id 1, :id (id :product))+ (assoc (meta/table-metadata :products) :name "BUYER", :db-id 1, :id (id :buyer), :database-require-filter true)+ (assoc (meta/table-metadata :products) :name "ORDER", :db-id 1, :id (id :order), :database-require-filter true)]+ :fields [;; PRODUCT+ (merge (meta/field-metadata :products :id)+ {:table-id (id :product), :name "ID", :id (id :product :id)})+ (merge (meta/field-metadata :products :id)+ {:table-id (id :product), :name "NAME", :id (id :product :name), :base-type :type/Text})+ ;; BUYER+ (merge (meta/field-metadata :products :id)+ {:table-id (id :buyer), :name "ID", :id (id :buyer :id), :database-partitioned true})+ ;; ORDER+ (merge (meta/field-metadata :products :id)+ {:table-id (id :order), :name "ID", :id (id :order :id)})+ (merge (meta/field-metadata :products :id)+ {:table-id (id :order)+ :name "BUYER_ID"+ :id (id :order :buyer-id)+ :semantic-type :type/FK+ :fk-target-field-id (id :buyer :id)})+ (merge (meta/field-metadata :products :id)+ {:table-id (id :order)+ :name "PRODUCT_ID"+ :id (id :order :product-id)+ :semantic-type :type/FK+ :fk-target-field-id (id :product :id)+ :database-partitioned true})]}))- (testing "if a joined table require a filter, add the partitioned filter"- (let [query (:query (mt/mbql-query product {:joins [{:source-table (:id order)- :condition [:= $order.product_id $product.id]- :alias "Product"}]}))]- (is (= (assoc query- :filter [:> [:field (:id order-product-id) {:base-type :type/Integer}] -9223372036854775808])- (metadata-queries/add-required-filters-if-needed query)))))+(deftest ^:parallel add-required-filter-if-needed-test+ (let [mp (mp)]+ (testing "no op for tables that do not require filter"+ (let [query (lib/query mp (lib.metadata/table mp (id :product)))]+ (is (= query+ (metadata-queries/add-required-filters-if-needed query)))))))- (testing "if both source tables and joined table require a filter, add both"- (let [query (:query (mt/mbql-query order {:joins [{:source-table (:id buyer)- :condition [:= $order.buyer_id $buyer.id]- :alias "BUYER"}]}))]- (is (= (-> query- (assoc :filter [:and- [:> [:field (:id buyer-id) {:base-type :type/Integer}] -9223372036854775808]- [:> [:field (:id order-product-id) {:base-type :type/Integer}] -9223372036854775808]])- ordered-filter)- (-> query- metadata-queries/add-required-filters-if-needed- ordered-filter)))))+(deftest ^:parallel add-required-filter-if-needed-test-2+ (let [mp (mp)]+ (testing "if the source table requires a filter, add the partitioned filter"+ (let [query (lib/query mp (lib.metadata/table mp (id :order)))]+ (is (=? [[:> {}+ [:field {} (id :order :product-id)]+ -9223372036854775808]]+ (-> (metadata-queries/add-required-filters-if-needed query)+ :stages+ first+ :filters)))))))- (testing "Should add an and clause for existing filter"- (let [query (:query (mt/mbql-query order {:filter [:> $order.id 1]}))]- (is (= (-> query- (assoc :filter [:and- [:> [:field (:id order-id) nil] 1]- [:> [:field (:id order-product-id) {:base-type :type/Integer}] -9223372036854775808]])- ordered-filter)- (-> query- metadata-queries/add-required-filters-if-needed- ordered-filter))))))))+(deftest ^:parallel add-required-filter-if-needed-test-3+ (let [mp (mp)]+ (testing "if a joined table require a filter, add the partitioned filter"+ (let [query (-> (lib/query mp (lib.metadata/table mp (id :product)))+ (lib/join (lib.metadata/table mp (id :order))))]+ (is (=? [[:> {}+ [:field {:join-alias string?} (id :order :product-id)]+ -9223372036854775808]]+ (-> (metadata-queries/add-required-filters-if-needed query)+ :stages+ first+ :filters)))))))++(deftest ^:parallel add-required-filter-if-needed-test-4+ (let [mp (mp)]+ (testing "if both source tables and joined table require a filter, add both"+ (let [query (-> (lib/query mp (lib.metadata/table mp (id :order)))+ (lib/join (lib.metadata/table mp (id :buyer))))]+ (is (=? [[:> {}+ [:field {:join-alias string?} (id :buyer :id)]+ -9223372036854775808]+ [:> {}+ [:field {} (id :order :product-id)]+ -9223372036854775808]]+ (-> query+ metadata-queries/add-required-filters-if-needed+ :stages+ first+ :filters+ sort-by-field-id)))))))++(deftest ^:parallel add-required-filter-if-needed-test-5+ (let [mp (mp)]+ (testing "Should add an and clause for existing filter"+ (let [query (-> (lib/query mp (lib.metadata/table mp (id :order)))+ (lib/filter (lib/> (lib.metadata/field mp (id :order :id)) 1)))]+ (is (=? [[:> {}+ [:field {} (id :order :id)]+ 1]+ [:> {}+ [:field {} (id :order :product-id)]+ -9223372036854775808]]+ (-> query+ metadata-queries/add-required-filters-if-needed+ :stages+ first+ :filters+ sort-by-field-id)))))))diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.cljindex a78965ea50a..f3bb14ed642 100644--- a/test/metabase/warehouses/models/database_test.clj+++ b/test/metabase/warehouses/models/database_test.clj@@ -13,7 +13,7 @@[metabase.models.serialization :as serdes][metabase.permissions.core :as perms][metabase.permissions.models.data-permissions :as data-perms]- [metabase.query-processor.store :as qp.store]+ ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.request.core :as request][metabase.secrets.core :as secret][metabase.sync.task.sync-databases :as task.sync-databases]@@ -606,17 +606,7 @@(is (partial= {:details {}}db))))))-(deftest ^:parallel after-select-driver-features-realize-db-row-test- ;; This test is necessary because driver multimethods should be able to assume that the db argument is a Database- ;; instance, not a transient row. Otherwise a call like `(mi/instance-of :model/Database db)` will return false- ;; when it should return true.- (testing "Make sure selecting a database calls `driver/database-supports?` with a database instance"- (mt/with-temp [:model/Database {db-id :id} {:engine (u/qualified-name ::test)}]- (mt/with-dynamic-fn-redefs [driver.u/supports? (fn [_ _ db]- (is (true? (mi/instance-of? :model/Database db))))]- (is (some? (t2/select-one-fn :features :model/Database :id db-id)))))))--(deftest hydrate-tables-test+(deftest ^:parallel hydrate-tables-test(is (= ["CATEGORIES""CHECKINS""ORDERS"diff --git a/test/metabase/xrays/api/automagic_dashboards_test.clj b/test/metabase/xrays/api/automagic_dashboards_test.cljindex 0b118ab3ac4..0904983667e 100644--- a/test/metabase/xrays/api/automagic_dashboards_test.clj+++ b/test/metabase/xrays/api/automagic_dashboards_test.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [malli.core :as mc][metabase.indexed-entities.models.model-index :as model-index][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group]@@ -10,9 +11,11 @@[metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.xrays.api.automagic-dashboards :as api.magic][metabase.xrays.automagic-dashboards.util :as magic.util]- [metabase.xrays.test-util.automagic-dashboards :refer [with-dashboard-cleanup!]]+ [metabase.xrays.test-util.automagic-dashboards :refer [with-rollback-only-transaction]][metabase.xrays.test-util.domain-entities :as test.de][metabase.xrays.test-util.transforms :as transforms.test][metabase.xrays.transforms.core :as tf]@@ -20,7 +23,12 @@[metabase.xrays.transforms.specs :as tf.specs][toucan2.core :as t2]))-(use-fixtures :once (fixtures/initialize :db :web-server :test-users :test-users-personal-collections))+(use-fixtures :once+ (fixtures/initialize :db :web-server :test-users :test-users-personal-collections)+ ;; initialize the test data+ (fn [thunk]+ (mt/id)+ (thunk)))(defn- dashcards-schema-check[dashcards]@@ -49,7 +57,7 @@([template args revoke-fn validation-fn](mt/with-test-user :rasta- (with-dashboard-cleanup!+ (with-rollback-only-transaction(mt/with-full-data-perms-for-all-users!(let [api-endpoint (apply format (str "automagic-dashboards/" template) args)resp (mt/user-http-request :rasta :get 200 api-endpoint)@@ -97,10 +105,10 @@:sectionId "location"}]}(mt/user-http-request :crowberto :get 200 (format "automagic-dashboards/table/%d" (mt/id :people))))))))-(deftest save-table-xray-dashboard-test+(deftest ^:parallel save-table-xray-dashboard-test(testing "Should be able to save a Dashboard generated by X-Rays"(mt/dataset test-data- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [generated-dashboard (mt/user-http-request :crowberto :get 200 (format "automagic-dashboards/table/%d" (mt/id :orders)))](is (=? {:description "Some metrics we found about transactions."}generated-dashboard))@@ -117,7 +125,6 @@:definition {:filter [:> [:field (mt/id :venues :price) nil] 10]}}](testing "GET /api/automagic-dashboards/segment/:id"(is (some? (api-call! "segment/%s" [segment-id]))))-(testing "GET /api/automagic-dashboards/segment/:id/rule/example/indepth"(is (some? (api-call! "segment/%s/rule/example/indepth" [segment-id]))))))@@ -275,12 +282,12 @@;;; ------------------- Transforms -------------------(deftest transforms-test- (testing "GET /api/automagic-dashboards/transform/:id"+ (testing "GET /api/automagic-dashboards/transform/:transform-name"(mt/with-test-user :crowberto- (transforms.test/with-test-transform-specs!- (test.de/with-test-domain-entity-specs!- (mt/with-model-cleanup [:model/Card :model/Collection]- (tf/apply-transform! (mt/id) "PUBLIC" (first @tf.specs/transform-specs))+ (transforms.test/with-test-transform-specs+ (test.de/with-test-domain-entity-specs+ (t2/with-transaction [_conn nil {:rollback-only true}]+ (tf/apply-transform! (mt/id) "PUBLIC" (first @tf.specs/*transform-specs*))(is (= [[1 "Red Medicine" 4 10.065 -165.374 3 1.5 4 3 2 1][2 "Stout Burgers & Beers" 11 34.1 -118.329 2 1.1 11 2 1 1][3 "The Apple Pan" 11 34.041 -118.428 2 1.1 11 2 1 1]]@@ -289,7 +296,15 @@(api-call! "transform/%s" ["Test transform"]#(revoke-collection-permissions!(tf.materialize/get-collection "Test transform"))- (fn [dashboard]+ (mu/fn [dashboard :- [:map+ [:dashcards+ [:sequential+ {:min 1}+ [:map+ [:card+ {:optional true}+ [:map+ [:dataset_query :map]]]]]]]](->> dashboard:dashcards(sort-by (juxt :row :col))@@ -323,7 +338,7 @@[{:keys [query pk-ref value-ref]} f](mt/with-temp [:model/Card model {:type :model:dataset_query query}]- (mt/with-model-cleanup [:model/ModelIndex]+ (t2/with-transaction [_conn nil {:rollback-only true}](let [model-index (model-index/create {:model-id (:id model):pk-ref pk-ref:value-ref value-ref@@ -520,11 +535,11 @@;; ------------------------------------------------ `show` limit test -------------------------------------------------;; Historically, the used params are `nil` and "all", so this tests the integer case.-(defn- card-count-check!+(defn- card-count-check"Create a dashboard via API twice, once with a limit and once without, and return the results."[limit template args](mt/with-test-user :crowberto- (with-dashboard-cleanup!+ (with-rollback-only-transaction(let [api-endpoint (apply format (str "automagic-dashboards/" template) args)resp (mt/user-http-request :crowberto :get 200 api-endpoint)slimmed (mt/user-http-request :crowberto :get 200 api-endpoint :show limit)@@ -532,60 +547,60 @@{:base-count (card-count-fn resp):show-count (card-count-fn slimmed)}))))-(deftest table-show-param-test+(deftest ^:parallel table-show-param-test(testing "x-ray of a table with show set reduces the number of returned cards"(let [show-limit 1- {:keys [base-count show-count]} (card-count-check! show-limit "table/%s" [(mt/id :venues)])]+ {:keys [base-count show-count]} (card-count-check show-limit "table/%s" [(mt/id :venues)])](testing "The non-slimmed dashboard isn't already at \"limit\" cards"(is (< show-count base-count)))(testing "Only \"limit\" cards are produced"(is (= show-limit show-count))))))-(deftest segment-xray-show-param-test+(deftest ^:parallel segment-xray-show-param-test(testing "x-ray of a segment with show set reduces the number of returned cards"(mt/with-temp [:model/Segment {segment-id :id} {:table_id (mt/id :venues):definition {:filter [:> [:field (mt/id :venues :price) nil] 10]}}](let [show-limit 1- {:keys [base-count show-count]} (card-count-check! show-limit "segment/%s" [segment-id])]+ {:keys [base-count show-count]} (card-count-check show-limit "segment/%s" [segment-id])](testing "The non-slimmed dashboard isn't already at \"limit\" cards"(is (< show-count base-count)))(testing "Only \"limit\" cards are produced"(is (= show-limit show-count)))))))-(deftest field-xray-show-param-test+(deftest ^:parallel field-xray-show-param-test(testing "x-ray of a field with show set reduces the number of returned cards"(let [show-limit 1- {:keys [base-count show-count]} (card-count-check! show-limit "field/%s" [(mt/id :venues :price)])]+ {:keys [base-count show-count]} (card-count-check show-limit "field/%s" [(mt/id :venues :price)])](testing "The non-slimmed dashboard isn't already at \"limit\" cards"(is (< show-count base-count)))(testing "Only \"limit\" cards are produced"(is (= show-limit show-count))))))-(deftest cell-query-xray-show-param-test+(deftest ^:parallel cell-query-xray-show-param-test(testing "x-ray of a cell-query with show set reduces the number of returned cards"(mt/with-temp [:model/Card {card-id :id} {:table_id (mt/id :venues):dataset_query (mt/mbql-query venues{:filter [:> $price 10]})}](let [cell-query (magic.util/encode-base64-json [:> [:field (mt/id :venues :price) nil] 5])show-limit 2- {:keys [base-count show-count]} (card-count-check! show-limit "question/%s/cell/%s" [card-id cell-query])]+ {:keys [base-count show-count]} (card-count-check show-limit "question/%s/cell/%s" [card-id cell-query])](testing "The non-slimmed dashboard isn't already at \"limit\" cards"(is (< show-count base-count)))(testing "Only \"limit\" cards are produced"(is (= show-limit show-count)))))))-(deftest comparison-xray-show-param-test+(deftest ^:parallel comparison-xray-show-param-test(testing "x-ray of a comparison with show set reduces the number of returned cards"(mt/with-temp [:model/Segment {segment-id :id} @segment](let [show-limit 1- {:keys [base-count show-count]} (card-count-check! show-limit- "adhoc/%s/cell/%s/compare/segment/%s"- [(->> (mt/mbql-query venues- {:filter [:> $price 10]})- (magic.util/encode-base64-json))- (->> [:= [:field (mt/id :venues :price) nil] 15]- (magic.util/encode-base64-json))- segment-id])]+ {:keys [base-count show-count]} (card-count-check show-limit+ "adhoc/%s/cell/%s/compare/segment/%s"+ [(->> (mt/mbql-query venues+ {:filter [:> $price 10]})+ (magic.util/encode-base64-json))+ (->> [:= [:field (mt/id :venues :price) nil] 15]+ (magic.util/encode-base64-json))+ segment-id])](testing "The slimmed dashboard produces less than the base dashboard";;NOTE - Comparisons produce multiple dashboards and merge the results, so you don't get exactly `show-limit` cards(is (< show-count base-count)))))))@@ -595,3 +610,12 @@{:tables (sort-by :id [{:id (mt/id :venues)}{:id (mt/id :categories)}])}(mt/user-http-request :crowberto :get 200 (str "automagic-dashboards/table/" (mt/id :venues) "/query_metadata")))))++(deftest ^:paralell allow-equals-in-base-64-encoding-for-now-test+ (testing "Allow `=` in the route part of X-Rays API requests... for now"+ (doseq [schema [::api.magic/base-64-encoded-json+ ::api.magic/entity-id-or-query]]+ (testing schema+ (let [pattern (:api/regex (mc/properties (mr/resolve-schema schema)))]+ (assert (instance? java.util.regex.Pattern pattern))+ (is (re= pattern "0IjoieWVhciJ9XV19LCJkYXRhYmFzZSI6MX0=")))))))diff --git a/test/metabase/xrays/automagic_dashboards/comparison_test.clj b/test/metabase/xrays/automagic_dashboards/comparison_test.cljindex 5e6057a8c9e..8bd843a4eff 100644--- a/test/metabase/xrays/automagic_dashboards/comparison_test.clj+++ b/test/metabase/xrays/automagic_dashboards/comparison_test.clj@@ -5,7 +5,7 @@[metabase.xrays.api.automagic-dashboards :as api.automagic-dashboards][metabase.xrays.automagic-dashboards.comparison :as c][metabase.xrays.automagic-dashboards.core :as magic]- [metabase.xrays.test-util.automagic-dashboards :refer [with-dashboard-cleanup!]]+ [metabase.xrays.test-util.automagic-dashboards :refer [with-rollback-only-transaction]][toucan2.core :as t2]))(def ^:private segment@@ -24,36 +24,36 @@;; TODO -- I don't know what these are supposed to test. So I have no idea what to name them.-(deftest test-1+(deftest ^:parallel test-1(mt/with-temp [:model/Segment {segment-id :id} @segment](mt/with-test-user :rasta- (with-dashboard-cleanup!+ (with-rollback-only-transaction(is (some? (test-comparison (t2/select-one :model/Table :id (mt/id :venues)) (t2/select-one :model/Segment :id segment-id))))(is (some? (test-comparison (t2/select-one :model/Segment :id segment-id) (t2/select-one :model/Table :id (mt/id :venues)))))))))-(deftest test-2+(deftest ^:parallel test-2(mt/with-temp [:model/Segment {segment1-id :id} @segment:model/Segment {segment2-id :id} {:table_id (mt/id :venues):definition {:filter [:< [:field (mt/id :venues :price) nil] 4]}}](mt/with-test-user :rasta- (with-dashboard-cleanup!+ (with-rollback-only-transaction(is (some? (test-comparison (t2/select-one :model/Segment :id segment1-id) (t2/select-one :model/Segment :id segment2-id))))))))-(deftest test-3+(deftest ^:parallel test-3(mt/with-test-user :rasta- (with-dashboard-cleanup!+ (with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter (-> @segment :definition :filter):source-table (mt/id :venues)}:type :query:database (mt/id)})](is (some? (test-comparison (t2/select-one :model/Table :id (mt/id :venues)) q)))))))-(deftest test-4+(deftest ^:parallel test-4(mt/with-temp [:model/Card {card-id :id} {:table_id (mt/id :venues):dataset_query {:query {:filter (-> @segment :definition :filter):source-table (mt/id :venues)}:type :query:database (mt/id)}}](mt/with-test-user :rasta- (with-dashboard-cleanup!+ (with-rollback-only-transaction(is (some? (test-comparison (t2/select-one :model/Table :id (mt/id :venues)) (t2/select-one :model/Card :id card-id))))))))diff --git a/test/metabase/xrays/automagic_dashboards/core_test.clj b/test/metabase/xrays/automagic_dashboards/core_test.cljindex 356c8dfc29d..811fdd32bda 100644--- a/test/metabase/xrays/automagic_dashboards/core_test.clj+++ b/test/metabase/xrays/automagic_dashboards/core_test.clj@@ -5,9 +5,11 @@[clojure.test :refer :all][medley.core :as m][metabase.legacy-mbql.schema :as mbql.s]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema.id :as lib.schema.id]+ [metabase.lib.util.match :as lib.util.match][metabase.models.interface :as mi][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group]@@ -24,6 +26,7 @@[metabase.xrays.automagic-dashboards.core :as magic][metabase.xrays.automagic-dashboards.dashboard-templates :as dashboard-templates][metabase.xrays.automagic-dashboards.interesting :as interesting]+ [metabase.xrays.automagic-dashboards.util :as magic.util][metabase.xrays.test-util.automagic-dashboards :as automagic-dashboards.test][toucan2.core :as t2]))@@ -31,14 +34,15 @@;;; ------------------- Dashboard template matching --------------------(deftest ^:parallel dashboard-template-matching-test+(deftest dashboard-template-matching-test(is (= [:entity/UserTable :entity/GenericTable :entity/*](->> (mt/id :users)(t2/select-one :model/Table :id)(#'magic/->root)(#'magic/matching-dashboard-templates (dashboard-templates/get-dashboard-templates ["table"]))- (map (comp first :applies_to)))))+ (map (comp first :applies_to))))))+(deftest dashboard-template-matching-test-2(testing "Test fallback to GenericTable"(is (= [:entity/GenericTable :entity/*](->> (-> (t2/select-one :model/Table :id (mt/id :users))@@ -49,7 +53,7 @@;;; ------------------- `->root source` --------------------(deftest ^:parallel source-root-table-test+(deftest source-root-table-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Table"(mt/dataset test-data(testing "The source of a table is the table itself"@@ -59,7 +63,7 @@(is (= entity table))(is (= source entity)))))))-(deftest ^:parallel source-root-field-test+(deftest source-root-field-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Field"(mt/dataset test-data(testing "The source of a field is the originating table of the field"@@ -69,7 +73,7 @@(is (= source table))(is (= entity field)))))))-(deftest ^:parallel source-root-card-test+(deftest source-root-card-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -83,10 +87,12 @@(let [{:keys [entity source]} (#'magic/->root card)](is (=? {:type :model}card))- (is (= entity card))- (is (= source (assoc card :entity_type :entity/GenericTable))))))))))+ (is (=? (dissoc entity :dataset_query)+ card))+ (is (=? (dissoc source :dataset_query)+ (assoc card :entity_type :entity/GenericTable))))))))))-(deftest ^:parallel source-root-card-test-2+(deftest source-root-card-test-2(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -107,10 +113,12 @@(is (=? {:type :question}card))(is (true? (#'magic/nested-query? card)))- (is (= entity card))- (is (= source (assoc nested-query :entity_type :entity/GenericTable))))))))))+ (is (=? (dissoc entity :dataset_query)+ card))+ (is (=? (dissoc source :entity_type)+ nested-query)))))))))-(deftest ^:parallel source-root-card-test-3+(deftest source-root-card-test-3(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -121,10 +129,12 @@(is (=? {:type :question}card))(is (true? (#'magic/native-query? card)))- (is (= entity card))- (is (= source (assoc card :entity_type :entity/GenericTable)))))))))))+ (is (=? (dissoc entity :dataset_query)+ card))+ (is (=? (dissoc source :dataset_query :entity_type)+ card))))))))))-(deftest ^:parallel source-root-card-test-4+(deftest source-root-card-test-4(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Card"(mt/dataset test-data(testing "Card sourcing has four branches..."@@ -141,10 +151,11 @@card))(is (false? (#'magic/nested-query? card)))(is (false? (#'magic/native-query? card)))- (is (= entity card))+ (is (=? (dissoc entity :dataset_query)+ card))(is (= source (t2/select-one :model/Table :id table-id))))))))))-(deftest ^:parallel source-root-query-test+(deftest source-root-query-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Query"(mt/dataset test-data(testing "The source of a query is the underlying datasource of the query"@@ -160,7 +171,7 @@(is (= entity query))(is (= source (t2/select-one :model/Table (mt/id :orders)))))))))-(deftest ^:parallel source-root-segment-test+(deftest source-root-segment-test(testing "Demonstrate the stated methods in which ->root computes the source of a :model/Segment"(testing "The source of a segment is its underlying table."(mt/with-temp [:model/Segment segment {:table_id (mt/id :venues)@@ -189,7 +200,7 @@(deftest automagic-analysis-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(doseq [[table cardinality] (map vector(t2/select :model/Table :db_id (mt/id) {:order-by [[:name :asc]]})[2 8 11 11 15 17 5 7])]@@ -197,7 +208,7 @@(deftest automagic-analysis-test-2(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(is (= 1(->> (magic/automagic-analysis (t2/select-one :model/Table :id (mt/id :venues)) {:show 1}):dashcards@@ -206,7 +217,7 @@(deftest weird-characters-in-names-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(-> (t2/select-one :model/Table :id (mt/id :venues))(assoc :display_name "%Venues")(test-automagic-analysis 7)))))@@ -215,7 +226,7 @@;; Not just with respect to drivers, but all sorts of other stuff that makes it chaotic(deftest mass-field-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(doseq [field (t2/select :model/Field:table_id [:in (t2/select-fn-set :id :model/Table :db_id (mt/id))]:visibility_type "normal"@@ -263,7 +274,7 @@:type :query:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(test-automagic-analysis (t2/select-one :model/Card :id card-id) 7))))))@@ -279,7 +290,7 @@:type :query:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(test-automagic-analysis (t2/select-one :model/Card :id card-id) 17))))))(deftest native-query-test@@ -292,7 +303,7 @@:type :native:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(test-automagic-analysis (t2/select-one :model/Card :id card-id) 2))))))@@ -319,7 +330,7 @@:type :query:database lib.schema.id/saved-questions-virtual-database-id}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(test-automagic-analysis (t2/select-one :model/Card :id card-id) 7)))))))@@ -346,7 +357,7 @@:type :query:database lib.schema.id/saved-questions-virtual-database-id}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(test-automagic-analysis (t2/select-one :model/Card :id card-id) 6))))))))@@ -529,16 +540,17 @@magic-card-database-id :database_idmagic-card-query :dataset_query:as _magic-card}]- (let [valid-source-ids (set (map (fn [{:keys [name id]}] (or id name)) source-meta))- {query-db-id :database- query-actual :query} magic-card-query- {source-table :source-table- breakout :breakout} query-actual]+ (let [valid-source-ids (set (map (fn [{:keys [name id]}] (or id name)) source-meta))+ magic-card-query (lib-be/normalize-query magic-card-query)+ {query-db-id :database} magic-card-query](is (= source-database-id magic-card-database-id))(is (= source-database-id query-db-id))(is (= source-table-id magic-card-table-id))- (is (= (format "card__%s" source-card-id) source-table))- (is (true? (every? (fn [[_ id]] (valid-source-ids id)) breakout)))))+ (is (= source-card-id+ (lib/source-card-id magic-card-query)))+ (doseq [breakout (lib/breakouts magic-card-query)+ field-id (lib/all-field-ids breakout)]+ (is (contains? valid-source-ids field-id)))))(defn- ensure-dashboard-sourcing [source-card dashboard](doseq [magic-card (->> dashboard@@ -580,7 +592,7 @@(result-metadata-for-querysource-query)):type :model}]- (let [dashboard (mt/with-test-user :rasta (magic/automagic-analysis card nil))+ (let [dashboard (mt/with-test-user :rasta (magic/automagic-analysis card nil))binned-field-id (mt/id :products :price)](ensure-single-table-sourced (mt/id :products) dashboard);; Count of records@@ -592,11 +604,14 @@;; This ensures we get a card that does binning on price(is (= binned-field-id(first- (for [card (:dashcards dashboard)- :let [fields (get-in card [:card :dataset_query :query :breakout])]- [_ field-id m] fields- :when (:binning m)]- field-id)))))))))))+ (for [dashcard (:dashcards dashboard)+ :let [query (get-in dashcard [:card :dataset_query])]+ :when query+ :let [breakouts (magic.util/do-with-mbql5-query query lib/breakouts)]+ id (lib.util.match/match breakouts+ [:field (_opts :guard :binning) (id :guard pos-int?)]+ id)]+ id)))))))))))(deftest basic-root-model-test-2(testing "Simple model with a temporal dimension detected"@@ -622,12 +637,15 @@(result-metadata-for-querysource-query)):type :model}]- (let [dashboard (mt/with-test-user :rasta (magic/automagic-analysis card nil))- temporal-field-ids (for [card (:dashcards dashboard)- :let [fields (get-in card [:card :dataset_query :query :breakout])]- [_ field-id m] fields- :when (:temporal-unit m)]- field-id)]+ (let [dashboard (mt/with-test-user :rasta (magic/automagic-analysis card nil))+ temporal-field-ids (for [dashcard (:dashcards dashboard)+ :let [query (get-in dashcard [:card :dataset_query])]+ :when query+ :let [breakouts (magic.util/do-with-mbql5-query query lib/breakouts)]+ id (lib.util.match/match breakouts+ [:field (_opts :guard :temporal-unit) (id :guard pos-int?)]+ id)]+ id)](ensure-single-table-sourced (mt/id :products) dashboard)(ensure-dashboard-sourcing card dashboard);; We want to produce at least one temporal axis card@@ -767,7 +785,8 @@(:name (mt/with-test-user :rasta (magic/automagic-analysis field nil)))))))))(deftest test-segment-title-test- (testing "Given the current automagic_dashboards/metric/GenericTable.yaml template (This is the default template for segments), produce the expected dashboard title"+ (testing (str "Given the current automagic_dashboards/metric/GenericTable.yaml template (This is the default"+ " template for segments), produce the expected dashboard title")(mt/with-non-admin-groups-no-root-collection-perms(mt/with-temp [:model/Segment {table-id :table_idsegment-name :name@@ -841,7 +860,7 @@:type :query:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(test-automagic-analysis (t2/select-one :model/Card :id card-id) 17))))))@@ -854,7 +873,7 @@:type :native:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(test-automagic-analysis (t2/select-one :model/Card :id card-id) 2))))))@@ -868,7 +887,7 @@:type :query:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(-> (t2/select-one :model/Card :id card-id)(test-automagic-analysis [:= [:field (mt/id :venues :category_id) nil] 2] 7)))))))@@ -886,12 +905,14 @@(let [entity (t2/select-one :model/Card :id card-id)cell-query [:= [:field (mt/id :venues :category_id) nil] 2]](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id) =- (let [all-dashcard-filters (->> (magic/automagic-analysis entity {:cell-query cell-query :show :all})- :dashcards- (keep (comp :filter :query :dataset_query :card)))- filter-contains-cell-query? #(= cell-query (some #{cell-query} %))]+ (let [all-dashcard-filters (->> (magic/automagic-analysis entity {:cell-query cell-query :show :all})+ :dashcards+ (keep #(-> %+ (get-in [:card :dataset_query])+ (magic.util/do-with-legacy-query get-in [:query :filter]))))+ filter-contains-cell-query? #(= cell-query (some #{cell-query} %))](is (pos? (count all-dashcard-filters)))(is (every? filter-contains-cell-query? all-dashcard-filters))))))))))@@ -905,14 +926,14 @@:type :query:database (mt/id)}}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(perms/grant-collection-readwrite-permissions! (perms-group/all-users) collection-id)(-> (t2/select-one :model/Card :id card-id)(test-automagic-analysis [:= [:field (mt/id :venues :category_id) nil] 2] 7)))))))(deftest adhoc-filter-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter [:> [:field (mt/id :venues :price) nil] 10]:source-table (mt/id :venues)}:type :query@@ -921,7 +942,7 @@(deftest adhoc-count-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:aggregation [[:count]]:breakout [[:field (mt/id :venues :category_id) nil]]:source-table (mt/id :venues)}@@ -931,7 +952,7 @@(deftest adhoc-fk-breakout-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:aggregation [[:count]]:breakout [[:field (mt/id :venues :category_id) {:source-field (mt/id :checkins)}]]:source-table (mt/id :checkins)}@@ -941,7 +962,7 @@(deftest adhoc-filter-cell-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [q (api.automagic-dashboards/adhoc-query-instance {:query {:filter [:> [:field (mt/id :venues :price) nil] 10]:source-table (mt/id :venues)}:type :query@@ -950,7 +971,7 @@(deftest join-splicing-test(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [join-vec [{:source-table (mt/id :categories):condition [:= [:field (mt/id :categories :id) nil] 1]:strategy :left-join@@ -971,21 +992,23 @@(testing "/candidates"(testing "should work with the normal test-data DB"(mt/with-test-user :rasta- (is (malli= [:cat- [:map- [:tables [:sequential {:min 8, :max 8} :any]]]- [:* :any]]- (magic/candidate-tables (mt/db))))))-+ (mt/with-full-data-perms-for-all-users!+ (is (malli= [:cat+ [:map+ [:tables [:sequential {:min 8, :max 8} :any]]]+ [:* :any]]+ (magic/candidate-tables (mt/db)))))))))++(deftest candidates-test-2+ (testing "/candidates"(testing "should work with unanalyzed tables"(mt/with-test-user :rasta(mt/with-temp [:model/Database {db-id :id} {}:model/Table {table-id :id} {:db_id db-id}:model/Field _ {:table_id table-id}:model/Field _ {:table_id table-id}]- (automagic-dashboards.test/with-dashboard-cleanup!- (is (=? [{:tables [{:table {:id table-id}}]}]- (magic/candidate-tables (t2/select-one :model/Database :id db-id))))))))))+ (is (=? [{:tables [{:table {:id table-id}}]}]+ (magic/candidate-tables (t2/select-one :model/Database :id db-id)))))))))(deftest call-count-test(mt/with-temp [:model/Database {db-id :id} {}@@ -993,7 +1016,7 @@:model/Field _ {:table_id table-id}:model/Field _ {:table_id table-id}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [database (t2/select-one :model/Database :id db-id)](t2/with-call-count [call-count](magic/candidate-tables database)@@ -1013,7 +1036,7 @@:model/Field _ {:table_id table-id :semantic_type :type/PK}:model/Field _ {:table_id table-id}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(is (partial= {:list-like? true:num-fields 2}(-> (#'magic/load-tables-with-enhanced-table-stats [[:= :id table-id]])@@ -1026,14 +1049,14 @@:model/Field _ {:table_id table-id :semantic_type :type/FK}:model/Field _ {:table_id table-id :semantic_type :type/FK}](mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(testing "filters out link-tables"(is (empty?(#'magic/load-tables-with-enhanced-table-stats [[:= :id table-id]]))))))));;; ------------------- Definition overloading --------------------(deftest ^:parallel most-specific-definition-test+(deftest most-specific-definition-test(testing "Identity"(is (= :d1(-> [{:d1 {:field_type [:type/Category] :score 100}}]@@ -1041,7 +1064,7 @@firstkey)))))-(deftest ^:parallel ancestors-definition-test+(deftest ancestors-definition-test(testing "Base case: more ancestors"(is (= :d2(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1050,7 +1073,7 @@firstkey)))))-(deftest ^:parallel definition-tiebreak-test+(deftest definition-tiebreak-test(testing "Break ties based on the number of additional filters"(is (= :d3(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1062,7 +1085,7 @@firstkey)))))-(deftest ^:parallel definition-tiebreak-score-test+(deftest definition-tiebreak-score-test(testing "Break ties on score"(is (= :d2(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1072,7 +1095,7 @@firstkey)))))-(deftest ^:parallel definition-tiebreak-precedence-test+(deftest definition-tiebreak-precedence-test(testing "Number of additional filters has precedence over score"(is (= :d3(-> [{:d1 {:field_type [:type/Category] :score 100}}@@ -1086,11 +1109,6 @@;;; -------------------- Filters ---------------------(defn field! [table column]- (or (t2/select-one :model/Field :id (mt/id table column))- (throw (ex-info (format "Did not find %s.%s" (name table) (name column))- {:table table :column column}))))-(deftest most-specific-definition-inner-shape-test(testing "Ensure we have examples to understand the shape returned from most-specific-definition"(mt/dataset test-data@@ -1261,7 +1279,7 @@;;; -------------------- Ensure generation of subcards via related (includes indepth, drilldown) ---------------------(deftest ^:parallel related-card-generation-test+(deftest related-card-generation-test(testing "Ensure that the `related` function is called and the right cards are created."(mt/with-test-user :rasta(mt/dataset test-data@@ -1295,7 +1313,7 @@(update :zoom-in (comp vec (partial sort-by :title)))(update :related (comp vec (partial sort-by :title)))))))))))-(deftest ^:parallel singular-cell-dimensions-test+(deftest singular-cell-dimensions-test(testing "Find the cell dimensions for a cell query"(is (= #{1 2 "TOTAL"}(#'magic/singular-cell-dimension-field-ids@@ -1310,7 +1328,7 @@X-rays fails on explicit joins, when metric is for the joined table"(mt/dataset test-data(mt/with-test-user :rasta- (automagic-dashboards.test/with-dashboard-cleanup!+ (automagic-dashboards.test/with-rollback-only-transaction(let [query-with-joins {:database (mt/id):type :query:query {:source-table (mt/id :reviews)@@ -1461,14 +1479,16 @@nil](->>(#'magic/preserve-entity-element- {:dashcards [{:card {:dataset_query {:query {:joins [{:strategy :left-join :alias "Orders"}]}}}}- {:card {:dataset_query {:query {:joins []}}}}- {:card {:dataset_query {:query {}}}}+ {:dashcards [{:card {:dataset_query {:database 1, :type :query, :query {:joins [{:strategy :left-join :alias "Orders"}]}}}}+ {:card {:dataset_query {:database 1, :type :query, :query {:joins []}}}}+ {:card {:dataset_query {:database 1, :type :query, :query {}}}}{:viz_settings nil}]}- {:dataset_query {:query {:joins [{:strategy :left-join :alias "Products"}]}}}+ {:dataset_query {:database 1, :type :query, :query {:joins [{:strategy :left-join :alias "Products"}]}}}:joins):dashcards- (mapv (comp :joins :query :dataset_query :card))))))+ (mapv (comp :joins :query :dataset_query :card)))))))++(deftest preserve-entity-element-test-2(testing "Expression preservation scenarios: merge, empty expressions, no expressions, no card"(is (= [{"Existing" [:- 1 1] "TestColumn" [:+ 1 1]}{"TestColumn" [:+ 1 1]}@@ -1476,11 +1496,11 @@nil](->>(#'magic/preserve-entity-element- {:dashcards [{:card {:dataset_query {:query {:expressions {"Existing" [:- 1 1]}}}}}- {:card {:dataset_query {:query {:expressions {}}}}}- {:card {:dataset_query {:query {}}}}+ {:dashcards [{:card {:dataset_query {:database 1, :type :query, :query {:expressions {"Existing" [:- 1 1]}}}}}+ {:card {:dataset_query {:database 1, :type :query, :query {:expressions {}}}}}+ {:card {:dataset_query {:database 1, :type :query, :query {}}}}{:viz_settings nil}]}- {:dataset_query {:query {:expressions {"TestColumn" [:+ 1 1]}}}}+ {:dataset_query {:database 1, :type :query, :query {:expressions {"TestColumn" [:+ 1 1]}}}}:expressions):dashcards(mapv (comp :expressions :query :dataset_query :card)))))))@@ -1550,10 +1570,10 @@;;; this is a fixed version of the test above that correctly generates the second card so that it matches the equivalent;;; Cypress test spec.-(deftest ^:parallel compare-to-the-rest-15655-fixed-test+(deftest compare-to-the-rest-15655-fixed-test(testing "Questions based on native questions should produce a valid dashboard. (#15655)"(mt/with-temp [:model/Card {native-card-id :id} (merge (mt/card-with-source-metadata-for-query- (lib/->legacy-MBQL (lib/native-query (mt/metadata-provider) "select * from people LIMIT 100;")))+ (lib/native-query (mt/metadata-provider) "select * from people LIMIT 100;")){:name "15655"}):model/Card card (let [metadata-provider (mt/metadata-provider)query (lib/query metadata-provider@@ -1563,7 +1583,7 @@(lib/breakout(m/find-first #(= (:name %) "SOURCE")(lib/breakoutable-columns query))))]- (qp.test-util/card-with-source-metadata-for-query (lib/->legacy-MBQL query)))]+ (qp.test-util/card-with-source-metadata-for-query query))](let [cell-query ["=" ["field" "SOURCE" {"base-type" "type/Text"}] "Affiliate"]](is (= ["Over time""Number of 15655 per day of the week"@@ -1596,7 +1616,9 @@(= "Distinct Product ID"(get-in dashcard [:card :name]))))first)- aggregate (get-in dashcard [:card :dataset_query :query :aggregation])]+ aggregate (-> dashcard+ (get-in [:card :dataset_query])+ (magic.util/do-with-legacy-query get-in [:query :aggregation]))](testing "Fields requiring a join should have :source-field populated in the aggregate."(is (= [[:distinct [:field (mt/id :products :id);; This should be present vs. nil (value before issue)diff --git a/test/metabase/xrays/automagic_dashboards/interesting_test.clj b/test/metabase/xrays/automagic_dashboards/interesting_test.cljindex 5f5b743a432..afcb0675e3a 100644--- a/test/metabase/xrays/automagic_dashboards/interesting_test.clj+++ b/test/metabase/xrays/automagic_dashboards/interesting_test.clj@@ -39,7 +39,9 @@;; ...but it isn't a candidate dimension because it is an id column...(is (false? ((#'interesting/fieldspec-matcher :type/*) id-field)));; ...unless you're looking explicitly for a primary key- (is (true? ((#'interesting/fieldspec-matcher :type/PK) id-field))))))+ (is (true? ((#'interesting/fieldspec-matcher :type/PK) id-field)))))))++(deftest ^:parallel field-matching-predicates-test-2(testing "The fieldspec-matcher should match fields by their fieldspec"(mt/dataset test-data(let [price-field (field :products :price)@@ -49,20 +51,26 @@(is (false? (pred price-field)))(is (true? (pred latitude-field)))(is (true? ((#'interesting/fieldspec-matcher :type/CreationTimestamp) created-at-field)))- (is (true? ((#'interesting/fieldspec-matcher :type/*) created-at-field))))))+ (is (true? ((#'interesting/fieldspec-matcher :type/*) created-at-field)))))))++(deftest ^:parallel field-matching-predicates-test-3(testing "The name-regex-matcher should return fields with string/regex matches"(mt/dataset test-data(let [price-field (field :products :price)category-field (field :products :category)ice-pred (#'interesting/name-regex-matcher "ice")](is (some? (ice-pred price-field)))- (is (nil? (ice-pred category-field))))))+ (is (nil? (ice-pred category-field)))))))++(deftest ^:parallel field-matching-predicates-test-4(testing "The max-cardinality-matcher should return fields with cardinality <= the specified cardinality"(mt/dataset test-data(let [category-field (field :products :category)](is (false? ((#'interesting/max-cardinality-matcher 3) category-field)))(is (true? ((#'interesting/max-cardinality-matcher 4) category-field)))- (is (true? ((#'interesting/max-cardinality-matcher 100) category-field))))))+ (is (true? ((#'interesting/max-cardinality-matcher 100) category-field)))))))++(deftest ^:parallel field-matching-predicates-test-5(testing "Roll the above together and test filter-fields"(mt/dataset test-data(let [category-field (field :products :category)diff --git a/test/metabase/xrays/domain_entities/converters_test.cljs b/test/metabase/xrays/domain_entities/converters_test.cljsindex 8a15e2bf32c..c10e7ccbb36 100644--- a/test/metabase/xrays/domain_entities/converters_test.cljs+++ b/test/metabase/xrays/domain_entities/converters_test.cljs@@ -4,14 +4,14 @@[metabase.test.util.js :as test.js][metabase.xrays.domain-entities.converters :as converters]))-(deftest incoming-basics-test+(deftest ^:parallel incoming-basics-test(testing "simple values are not transformed"(is (= identity (converters/incoming number?)))(is (= identity (converters/incoming string?)))(is (= identity (converters/incoming nil?)))(is (= identity (converters/incoming boolean?)))))-(deftest keywords-test+(deftest ^:parallel keywords-test(testing "plain keywords are transformed without changing their spelling"(let [->kw (converters/incoming keyword?)kw-> (converters/outgoing keyword?)@@ -23,8 +23,9 @@(is (= kw (->kw s)))(is (= s (kw-> kw)))(is (= kw (-> kw kw-> ->kw)))- (is (= s (-> s ->kw kw->))))))+ (is (= s (-> s ->kw kw->)))))))+(deftest ^:parallel keywords-test-2(testing "qualified keywords are transformed without changing their spelling"(let [->kw (converters/incoming :qualified-keyword)kw-> (converters/outgoing :qualified-keyword)@@ -38,16 +39,16 @@(is (= kw (-> kw kw-> ->kw)))(is (= s (-> s ->kw kw->)))))))-(def HalfDeclared+(def ^:private HalfDeclared[:map[:declared-camel {:js/prop "declaredCamel"} string?][:declared-snake string?][:declared-kebab {:js/prop "declared-kebab"} string?]])-(def ->half-declared+(def ^:private ->half-declared(converters/incoming HalfDeclared))-(deftest map-basics-test+(deftest ^:parallel map-basics-test(testing "incoming maps"(testing "become CLJS maps"(is (map? ((converters/incoming [:map]) #js {}))))@@ -119,7 +120,7 @@(def Grandparent[:map [:parent Parent]])-(deftest nesting-test+(deftest ^:parallel nesting-test(testing "deeply nested maps"(let [input #js {"parent" #js {"child" #js {"innerValue" "asdf"}}}exp-clj {:parent {:child {:inner-value "asdf"}}}@@ -187,13 +188,13 @@(testing "round-trips as expected"(is (test.js/= input (-> input ->sink sink->)))))))-(deftest idempotency-test+(deftest ^:parallel idempotency-test(testing "CLJS maps are not further converted"(let [->parent (converters/incoming Parent)input {:child {:inner-value "foo"}}](is (identical? input (->parent input))))))-(deftest opaque-any-test+(deftest ^:parallel opaque-any-test(testing ":any values are not touched, and round-trip as identical?"(let [schema [:map[:wrapper [:map@@ -214,7 +215,7 @@(is (identical? js-obj (let [^Object wrapper (.-wrapper returned)](.-inner wrapper)))))))-(deftest uuid-test+(deftest ^:parallel uuid-test(testing "UUIDs are converted to strings in JS and back to #uuid objects in CLJS"(let [uuid (random-uuid)](is (= (str uuid)diff --git a/test/metabase/xrays/domain_entities/core_test.clj b/test/metabase/xrays/domain_entities/core_test.cljindex c9dba0b8f9c..bca69a99a4d 100644--- a/test/metabase/xrays/domain_entities/core_test.clj+++ b/test/metabase/xrays/domain_entities/core_test.clj@@ -9,7 +9,6 @@(deftest ^:parallel mbql-reference-test(is (= [:field (data/id :venues :price) nil](#'de/mbql-reference (t2/select-one :model/Field :id (data/id :venues :price)))))-(is (= [:field "PRICE" {:base-type :type/Integer}](#'de/mbql-reference (dissoc (t2/select-one :model/Field :id (data/id :venues :price)) :id)))))@@ -19,16 +18,16 @@(t2/hydrate :fields)))(deftest ^:parallel satisfies-requierments?-test- (is (de/satisfies-requierments? (hydrated-table :venues) (test.de/test-domain-entity-specs "Venues"))))+ (is (de/satisfies-requirements? (hydrated-table :venues) (test.de/test-domain-entity-specs "Venues"))))(deftest ^:parallel best-match-test(testing "Do we correctly pick the best (most specific and most defined) candidate"(is (= "Venues"(-> test.de/test-domain-entity-specs vals (#'de/best-match) :name)))))-(deftest instantiate-snippets-test+(deftest ^:parallel instantiate-snippets-test(testing "Do all the MBQL snippets get instantiated correctly"- (test.de/with-test-domain-entity-specs!+ (test.de/with-test-domain-entity-specs(is (= {:metrics {"Avg Price" {:name "Avg Price":aggregation [:avg (#'de/mbql-reference (t2/select-one :model/Field :id (data/id :venues :price)))]}}:segments nildiff --git a/test/metabase/xrays/domain_entities/specs_test.clj b/test/metabase/xrays/domain_entities/specs_test.cljindex 7c53a9cf92c..32745204140 100644--- a/test/metabase/xrays/domain_entities/specs_test.clj+++ b/test/metabase/xrays/domain_entities/specs_test.clj@@ -6,6 +6,6 @@(deftest ^:parallel validate-specs-test(testing "All specs should be valid YAML (the parser will raise an exception if not) and conforming to the schema."- (doseq [[spec-name spec] @de.specs/domain-entity-specs]+ (doseq [[spec-name spec] @de.specs/*domain-entity-specs*](testing spec-name(is (mr/validate de.specs/DomainEntitySpec spec))))))diff --git a/test/metabase/xrays/related_test.clj b/test/metabase/xrays/related_test.cljindex d44bbea73f9..cce624f27ec 100644--- a/test/metabase/xrays/related_test.clj+++ b/test/metabase/xrays/related_test.clj@@ -17,7 +17,7 @@["segment" 1]][:metric 1]]))))-(deftest similiarity-test+(deftest ^:parallel similiarity-test(mt/with-temp [:model/Card {card-id-1 :id} {:dataset_query (mt/mbql-query venues{:aggregation [[:sum $price]]:breakout [$category_id]})}@@ -104,7 +104,7 @@(m/update-existing :collections (partial filter (partial = (:collection-id *world*))))(m/update-existing :tables set))))-(deftest related-segments-test+(deftest ^:parallel related-segments-test(with-world(is (= {:table (mt/id :venues):metrics (sort [metric-id-a metric-id-b])@@ -113,7 +113,7 @@(->> (mt/user-http-request :crowberto :get 200 (format "segment/%s/related" segment-id-a))result-mask)))))-(deftest related-tables-test+(deftest ^:parallel related-tables-test(with-world(is (= {:metrics (sort [metric-id-a metric-id-b]):segments (sort [segment-id-a segment-id-b])@@ -147,7 +147,7 @@(is (= 0(count-related-fields)))))))-(deftest recommended-dashboards-test+(deftest ^:parallel recommended-dashboards-test(mt/with-temp [:model/Card card-1 {}:model/Card card-2 {}:model/Card card-3 {}@@ -161,4 +161,6 @@(binding [api/*current-user-id* (mt/user->id :rasta)api/*current-user-permissions-set* (atom #{"/"})](is (=? [{:id dash-id}]- (#'related/recommended-dashboards [card-1 card-2 card-3]))))))+ ;; filter out dashboards unrelated to this test+ (filter #(= (:id %) dash-id)+ (#'related/recommended-dashboards [card-1 card-2 card-3])))))))diff --git a/test/metabase/xrays/test_util/automagic_dashboards.clj b/test/metabase/xrays/test_util/automagic_dashboards.cljindex 9bd4a521f71..9e69601cdf1 100644--- a/test/metabase/xrays/test_util/automagic_dashboards.clj+++ b/test/metabase/xrays/test_util/automagic_dashboards.clj@@ -2,17 +2,20 @@"Helper functions and macros for writing tests for automagic dashboards."(:require[clojure.test :refer :all]- [metabase.legacy-mbql.normalize :as mbql.normalize]- [metabase.legacy-mbql.schema :as mbql.s][metabase.test :as mt][metabase.util :as u]- [metabase.util.malli.schema :as ms]))+ [metabase.util.malli.schema :as ms]+ [metabase.xrays.automagic-dashboards.schema :as ads]+ [toucan2.core :as t2]))-(defmacro with-dashboard-cleanup!+(defmacro with-rollback-only-transaction"Execute body and cleanup all dashboard elements created."[& body]- `(mt/with-model-cleanup [:model/Card :model/Dashboard :model/Collection :model/DashboardCard]- ~@body))+ `(do+ ;; make sure test data is initialized outside of the rollback-only transaction+ (mt/id)+ (t2/with-transaction [_conn# nil {:rollback-only true}]+ ~@body)))(defn- collect-urls[dashboard]@@ -24,19 +27,21 @@(defn- test-urls-are-valid[dashboard]- (doseq [url (collect-urls dashboard)]- (testing (format "\nURL = %s" (pr-str url))+ (doseq [url (collect-urls dashboard)+ :let [url (format "automagic-dashboards/%s" (subs url 16))]]+ (testing (format "\nendpoint = GET /api/%s" url)(is (malli= [:map[:name ms/NonBlankString][:description ms/NonBlankString]]- (mt/user-http-request :crowberto :get 200 (format "automagic-dashboards/%s" (subs url 16))))))))+ (mt/user-http-request :crowberto :get 200 url))))))(defn- test-card-is-valid [{query :dataset_query, :as card}](testing "Card should be valid"(testing (format "\nCard =\n%s\n" (u/pprint-to-str card))(testing "Card query should be valid"- (is (malli= mbql.s/Query- (mbql.normalize/normalize query)))))))+ (is (map? query))+ (is (malli= ::ads/query+ query))))))(defn test-dashboard-is-valid"Is generated dashboard valid?diff --git a/test/metabase/xrays/test_util/domain_entities.clj b/test/metabase/xrays/test_util/domain_entities.cljindex 757fdf20dc3..0f0271559ca 100644--- a/test/metabase/xrays/test_util/domain_entities.clj+++ b/test/metabase/xrays/test_util/domain_entities.clj@@ -27,10 +27,10 @@(#'de.specs/coerce-to-domain-entity-spec))]))(into {})))-(defmacro with-test-domain-entity-specs!+(defmacro with-test-domain-entity-specs"Evaluate `body` in a context where `domain-entities.specs/domain-entity-specs` have been swapped for`test-domain-entity-specs`"[& body]`(testing "with-test-domain-entity-specs\n"- (with-redefs [de.specs/domain-entity-specs (delay test-domain-entity-specs)]+ (binding [de.specs/*domain-entity-specs* (delay test-domain-entity-specs)]~@body)))diff --git a/test/metabase/xrays/test_util/transforms.clj b/test/metabase/xrays/test_util/transforms.cljindex 6c56ff18253..3983131e3ae 100644--- a/test/metabase/xrays/test_util/transforms.clj+++ b/test/metabase/xrays/test_util/transforms.clj@@ -24,9 +24,9 @@(#'tf.specs/add-metadata-to-steps)(#'tf.specs/coerce-to-transform-spec)))-(defmacro with-test-transform-specs!+(defmacro with-test-transform-specs"Evaluate `body` in a context where `transforms.specs/transform-specs` have been swapped for `test-transform-specs`"[& body]`(testing "with-test-transform-specs\n"- (with-redefs [tf.specs/transform-specs (delay [test-transform-spec])]+ (binding [tf.specs/*transform-specs* (delay [test-transform-spec])]~@body)))diff --git a/test/metabase/xrays/transforms/core_test.clj b/test/metabase/xrays/transforms/core_test.cljindex 3e441f2175e..67964cbae5c 100644--- a/test/metabase/xrays/transforms/core_test.clj+++ b/test/metabase/xrays/transforms/core_test.clj@@ -5,27 +5,35 @@[metabase.models.interface :as mi][metabase.query-processor :as qp][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util :as u][metabase.xrays.domain-entities.core :as de][metabase.xrays.domain-entities.specs :as de.specs]- [metabase.xrays.test-util.domain-entities :refer [with-test-domain-entity-specs!]]- [metabase.xrays.test-util.transforms :refer [test-transform-spec with-test-transform-specs!]]+ [metabase.xrays.test-util.domain-entities :refer [with-test-domain-entity-specs]]+ [metabase.xrays.test-util.transforms :refer [test-transform-spec with-test-transform-specs]][metabase.xrays.transforms.core :as tf][metabase.xrays.transforms.specs :as tf.specs][toucan2.core :as t2]))+(use-fixtures :once+ (fixtures/initialize :db)+ ;; make sure test data is initialized, we don't want to do it inside the rollback-only transactions+ (fn [thunk]+ (mt/db)+ (thunk)))+(use-fixtures :each (fn [thunk]- (mt/with-model-cleanup [:model/Card :model/Collection]+ (t2/with-transaction [_conn nil {:rollback-only true}](thunk))))(def ^:private test-bindings(delay- (with-test-domain-entity-specs!+ (with-test-domain-entity-specs(let [table (m/find-first (comp #{(mt/id :venues)} u/the-id) (#'tf/tableset (mt/id) "PUBLIC"))]{"Venues" {:dimensions (m/map-vals de/mbql-reference (get-in table [:domain_entity :dimensions])):entity table}}))))-(deftest add-bindings-test+(deftest ^:parallel add-bindings-test(testing "Can we accure bindings?"(let [new-bindings {"D2" [:sum [:field 4 nil]]"D3" [:field 5 nil]}]@@ -36,7 +44,7 @@(is (= @test-bindings(#'tf/add-bindings @test-bindings "Venues" nil)))))-(deftest mbql-reference->col-name-test+(deftest ^:parallel mbql-reference->col-name-test(is (= "PRICE"(#'tf/mbql-reference->col-name [:field (mt/id :venues :price) nil])))(is (= "PRICE"@@ -44,7 +52,7 @@(is (= "PRICE"(#'tf/mbql-reference->col-name [{:foo [:field (mt/id :venues :price) nil]}]))))-(deftest ->source-table-reference-test+(deftest ^:parallel ->source-table-reference-test(testing "Can we turn a given entity into a format suitable for a query's `:source_table`?"(testing "for a Table"(is (= (mt/id :venues)@@ -55,83 +63,87 @@(is (= (str "card__" card-id)(#'tf/->source-table-reference (t2/select-one :model/Card :id card-id))))))))-(deftest tableset-test+(deftest ^:parallel tableset-test(testing "Can we get a tableset for a given schema?"(is (= (t2/select-pks-set :model/Table :db_id (mt/id))(set (map u/the-id (#'tf/tableset (mt/id) "PUBLIC")))))))-(deftest find-tables-with-domain-entity-test- (with-test-domain-entity-specs!+(deftest ^:parallel find-tables-with-domain-entity-test+ (with-test-domain-entity-specs(testing "Can we filter a tableset by domain entity?"(is (= [(mt/id :venues)](map u/the-id (#'tf/find-tables-with-domain-entity (#'tf/tableset (mt/id) "PUBLIC")- (@de.specs/domain-entity-specs "Venues"))))))+ (@de.specs/*domain-entity-specs* "Venues"))))))))++(deftest ^:parallel find-tables-with-domain-entity-test-2+ (with-test-domain-entity-specs(testing "Gracefully handle no-match"- (with-test-domain-entity-specs!+ (with-test-domain-entity-specs(is (= nil(not-empty- (#'tf/find-tables-with-domain-entity [] (@de.specs/domain-entity-specs "Venues")))))))))+ (#'tf/find-tables-with-domain-entity [] (@de.specs/*domain-entity-specs* "Venues")))))))))-(deftest resulting-entities-test+(deftest ^:parallel resulting-entities-test(testing "Can we extract results from the final bindings?"- (with-test-transform-specs!+ (with-test-transform-specs(is (= [(mt/id :venues)](map u/the-id (#'tf/resulting-entities {"VenuesEnhanced" {:entity (t2/select-one :model/Table :id (mt/id :venues)):dimensions {"D1" [:field 1 nil]}}}- (first @tf.specs/transform-specs))))))))+ (first @tf.specs/*transform-specs*))))))))-(deftest tables-matching-requirements-test+(deftest ^:parallel tables-matching-requirements-test(testing "Can we find a table set matching requirements of a given spec?"- (with-test-transform-specs!- (with-test-domain-entity-specs!+ (with-test-transform-specs+ (with-test-domain-entity-specs(is (= [(mt/id :venues)](map u/the-id (#'tf/tables-matching-requirements (#'tf/tableset (mt/id) "PUBLIC")- (first @tf.specs/transform-specs)))))))))+ (first @tf.specs/*transform-specs*)))))))))-(deftest tableset->bindings-test+(deftest ^:parallel tableset->bindings-test(testing "Can we turn a tableset into corresponding bindings?"- (with-test-domain-entity-specs!+ (with-test-domain-entity-specs(is (= @test-bindings(#'tf/tableset->bindings (filter (comp #{(mt/id :venues)} u/the-id) (#'tf/tableset (mt/id) "PUBLIC"))))))))-(deftest validation-test- (with-test-domain-entity-specs!- (with-test-transform-specs!+(deftest ^:parallel validation-test+ (with-test-domain-entity-specs+ (with-test-transform-specs(testing "Is the validation of results working?"(is (#'tf/validate-results {"VenuesEnhanced" {:entity (mi/instance:model/Card- {:result_metadata [{:name "AvgPrice"}- {:name "MaxPrice"}- {:name "MinPrice"}]})+ {:result_metadata [{:base_type :type/Number, :name "AvgPrice"}+ {:base_type :type/Number, :name "MaxPrice"}+ {:base_type :type/Number, :name "MinPrice"}]}):dimensions {"D1" [:field 1 nil]}}}- (first @tf.specs/transform-specs))))+ (first @tf.specs/*transform-specs*))))(testing "... and do we throw if we didn't get what we expected?"(is (thrown?java.lang.AssertionError(#'tf/validate-results {"VenuesEnhanced" {:entity (t2/select-one :model/Table :id (mt/id :venues)):dimensions {"D1" [:field 1 nil]}}}- (first @tf.specs/transform-specs))))))))+ (first @tf.specs/*transform-specs*))))))))(deftest transform-test(testing "Run the transform and make sure it produces the correct result"- (mt/with-full-data-perms-for-all-users!- (mt/with-test-user :rasta- (with-test-domain-entity-specs!+ (mt/with-test-user :crowberto+ (with-test-domain-entity-specs+ (let [cards (tf/apply-transform! (mt/id) "PUBLIC" test-transform-spec)]+ (is (seq cards))(is (= [[1 "Red Medicine" 4 10.065 -165.374 3 1.5 4 3 2 1][2 "Stout Burgers & Beers" 11 34.1 -118.329 2 1.1 11 2 1 1][3 "The Apple Pan" 11 34.041 -118.428 2 1.1 11 2 1 1]](mt/formatted-rows[int str int 3.0 3.0 int 1.0 int int int int]- (-> (tf/apply-transform! (mt/id) "PUBLIC" test-transform-spec)+ (-> cardsfirst:dataset_queryqp/process-query)))))))))-(deftest correct-transforms-for-table-test+(deftest ^:parallel correct-transforms-for-table-test(testing "Can we find the right transform(s) for a given table"- (with-test-transform-specs!- (with-test-domain-entity-specs!+ (with-test-transform-specs+ (with-test-domain-entity-specs(is (= "Test transform"(-> (tf/candidates (t2/select-one :model/Table :id (mt/id :venues)))firstdiff --git a/test/metabase/xrays/transforms/specs_test.clj b/test/metabase/xrays/transforms/specs_test.cljindex 36145eabac7..26da1d98a1f 100644--- a/test/metabase/xrays/transforms/specs_test.clj+++ b/test/metabase/xrays/transforms/specs_test.clj@@ -14,4 +14,4 @@(deftest ^:parallel validate-yaml-test(testing "All specs should be valid YAML (the parser will raise an exception if not) and conforming to the schema."(is (malli= [:sequential {:min 1} tf.specs/TransformSpec]- @tf.specs/transform-specs))))+ @tf.specs/*transform-specs*))))diff --git a/test_resources/serialization_baseline/transforms/EFFQTUPv83TcPakYN_Rlu_test_transform.yaml b/test_resources/serialization_baseline/transforms/EFFQTUPv83TcPakYN_Rlu_test_transform.yamlindex 7d70abd83aa..5991095bc03 100644--- a/test_resources/serialization_baseline/transforms/EFFQTUPv83TcPakYN_Rlu_test_transform.yaml+++ b/test_resources/serialization_baseline/transforms/EFFQTUPv83TcPakYN_Rlu_test_transform.yaml@@ -3,13 +3,14 @@ description: A test transform for serializationentity_id: EFFQTUPv83TcPakYN_Rluname: Test Transformsource:- database: My Databasequery:- source-table:- - My Database- - null- - Schemaless Table- type: query+ database: My Database+ query:+ source-table:+ - My Database+ - null+ - Schemaless Table+ type: querytags:- entity_id: 2vO2leHROJ38q2j19gNdbposition: 0diff --git a/test_resources/serialization_baseline/transforms/txz-QxFUj623m2n-gC1Qz_test_transform.yaml b/test_resources/serialization_baseline/transforms/txz-QxFUj623m2n-gC1Qz_test_transform.yamlindex 21022788f14..11c02caf9da 100644--- a/test_resources/serialization_baseline/transforms/txz-QxFUj623m2n-gC1Qz_test_transform.yaml+++ b/test_resources/serialization_baseline/transforms/txz-QxFUj623m2n-gC1Qz_test_transform.yaml@@ -3,13 +3,14 @@ description: A test transform for serializationentity_id: txz-QxFUj623m2n-gC1Qzname: Test Transformsource:- database: My Databasequery:- source-table:- - My Database- - null- - Schemaless Table- type: query+ database: My Database+ query:+ source-table:+ - My Database+ - null+ - Schemaless Table+ type: querytags:- entity_id: qvqHB8VgfxMzZx4LIVYGtposition: 0