The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Remove table_ids from data apps (#84358)
Alexander Polyankin8997e86d4839
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 0ded1266a8b..4a26230229e 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -3818,11 +3818,9 @@api-scopeaudit-appcollections- drivereventsliblib.be- lib.metadatalib.schemalib.typesmetabot@@ -3837,7 +3835,9 @@systemusersutil+ util.graphwarehouse-schema-overlay+ enterprise/dependenciesenterprise/serialization}:model-imports #{:model/Action:model/Card@@ -3847,7 +3847,6 @@:model/Field:model/Measure:model/Permissions- :model/QueryAction:model/Table}:model-exports #{:model/DataApp}}diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 826a93dcc43..ed4034778c4 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -10,6 +10,13 @@ title: Driver interface changelogwith CTEs instead of nested subselects. Drivers should only do this if they satisfy all of the criteriain the docstring of this method.+- `driver/native-query-table-refs` has been removed. Nothing calls it anymore, so drivers that implement it should+ delete their `defmethod`, which no longer compiles.++- `driver/native-query-deps` can now return `{:unresolved-table {:schema ... :table ...}}` for a table the query names+ that matches no synced table or transform target. Dependency tracking recomputes such a query once sync finds the+ table. Drivers built on `:sql` get this from `metabase.sql-tools.core/referenced-tables`.+## Metabase 0.64.0- Date and time functions that depended on the `start-of-week` setting now take a `time-config` map as their firstdiff --git a/e2e/support/helpers/e2e-data-app-helpers.ts b/e2e/support/helpers/e2e-data-app-helpers.tsindex e8dfb286470..f8f3d52e515 100644--- a/e2e/support/helpers/e2e-data-app-helpers.ts+++ b/e2e/support/helpers/e2e-data-app-helpers.ts@@ -48,7 +48,6 @@ export const fakeDataApp = (overrides: Partial<DataApp> = {}): DataApp => ({bundle_path: "dist/index.js",enabled: true,resource_collection_id: 1,- table_ids: [],allowed_hosts: [],bundle_hash: "e2e-bundle-hash",created_at: "2024-01-01T00:00:00Z",diff --git a/e2e/test-host-app/data-apps/resources.cy.spec.ts b/e2e/test-host-app/data-apps/resources.cy.spec.tsindex 8e3f9501e2f..386457be851 100644--- a/e2e/test-host-app/data-apps/resources.cy.spec.ts+++ b/e2e/test-host-app/data-apps/resources.cy.spec.ts@@ -146,8 +146,6 @@ describe("Embedding SDK: data-app resources (queries)", () => {cards: [ordersQuestion(question)],});H.publishDataApp(APP_ROOT(), APP_SLUG).then((app) => {- expect(app.table_ids).to.deep.eq([ORDERS_ID]);-cy.request<Card>(`/api/card/${question}`).then(({ body: card }) => {expect(card.collection_id).to.eq(app.resource_collection_id);expect(card.name).to.eq("Orders");diff --git a/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts b/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.tsindex 418ce86e8c4..8f8ab93a183 100644--- a/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts@@ -30,6 +30,7 @@ describe("scenarios > data apps > group access warnings (EMB-2416)", () => {}),],});+ H.waitForBackfillComplete();cy.request<GroupInfo>("POST", "/api/permissions/group", {name: GROUP_NAME,diff --git a/e2e/test/scenarios/data-apps/sync.cy.spec.ts b/e2e/test/scenarios/data-apps/sync.cy.spec.tsindex e45c87614d2..c391d5607ca 100644--- a/e2e/test/scenarios/data-apps/sync.cy.spec.ts+++ b/e2e/test/scenarios/data-apps/sync.cy.spec.ts@@ -1,10 +1,7 @@-import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";import type { DataApp } from "metabase-types/api";const { H } = cy;-const { ORDERS_ID } = SAMPLE_DATABASE;-/*** Drives a real remote-sync pull of a repo whose `data_apps/` holds two apps, each a `data_app.yaml` with its bundle* next to it, and asserts that pulls materialize and remove them.@@ -54,7 +51,6 @@ describe("scenarios > data apps > repo sync", () => {cy.request("/api/card/goodAppOrdersQuestion").its("body.collection_id").should("eq", app.resource_collection_id);- expect(app.table_ids).to.deep.eq([ORDERS_ID]);});// An author deletes an app by deleting its directory and its collection'sdiff --git a/enterprise/backend/src/metabase_enterprise/data_apps/README.md b/enterprise/backend/src/metabase_enterprise/data_apps/README.mdindex b0de8e2a916..cb8ad126499 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/README.md+++ b/enterprise/backend/src/metabase_enterprise/data_apps/README.md@@ -50,7 +50,7 @@ the YAML, and ingestion reads the paths `serdes/resource-paths` returns back in.must stay inside the entity's directory.`enabled` is admin-owned and never leaves the instance; group assignments also stay local to the-instance. `table_ids` are server-managed, and `bundle_hash` is recomputed from the bundle on import.+instance, and `bundle_hash` is recomputed from the bundle on import.A targeted export of an app brings its collection and what it holds along (`serdes/descendants`).An import matches an app by `entity_id` and reasserts the app's resources; a manifest whose slug an app@@ -146,7 +146,7 @@ The Data Apps feature is required for all app API endpoints, including group lis| Namespace | Responsibility || -------------------------------------- | ----------------------------------------------------------------------------------------------- || `apps.clj` | Creating apps; the connected repository's URL. |-| `core.clj` | Public access checks, resource file problems, and table dependencies. |+| `core.clj` | Public access checks and resource file problems. || `config.clj` | The serialized layout and data app contract version constants. || `schema.clj` | Column schemas, with the normalization and validation every write goes through. || `api.clj` | The `/api/apps` endpoints, bundle serving, ETag handling. |@@ -159,9 +159,8 @@ The Data Apps feature is required for all app API endpoints, including group lis| `generate/schemas/` | The schema's stages: data access (`source`), entities (`table`, `metric`, `action`, `common`), rendering (`render`, `javascript`). || `query_definition.clj` | The closed schema of a `defineQuery` definition the serialization accepts. || `resource_validation.clj` | What the files of an app's collection may hold, checked on the whole snapshot before an import. |-| `resource_tables.clj` | The tables an app's resources read, recorded on the app after an import. || `access.clj` | App access through assigned groups. |-| `group_access.clj` | Assignment management and collection grant reconciliation. |+| `group_access.clj` | Assignment management, collection grant reconciliation, and the permission warnings, which walk the dependency graph (`:dependencies` feature) from the app's cards and actions to the tables they read. || `models/data_app_group_assignment.clj` | App-to-group assignments. || `db.clj` | The module's application-database queries. || `csp.clj` | `allowed_hosts` lookup for the core CSP middleware. |diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj b/enterprise/backend/src/metabase_enterprise/data_apps/api.cljindex 11a786581d1..1735cce9de4 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj@@ -72,7 +72,6 @@[:enabled :boolean][:allowed_hosts [:sequential :string]][:resource_collection_id ms/PositiveInt]- [:table_ids [:sequential ms/PositiveInt]][:bundle_hash [:maybe :string]][:created_at :any][:updated_at :any]])diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/core.clj b/enterprise/backend/src/metabase_enterprise/data_apps/core.cljindex 7403401fd0f..87fab58d44a 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/core.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/core.clj@@ -1,9 +1,7 @@(ns metabase-enterprise.data-apps.core- "What other modules may ask the data-apps module: what the files of an app's collection may hold, and the tables- those read."+ "What other modules may ask the data-apps module: what the files of an app's collection may hold."(:require[metabase-enterprise.data-apps.db :as data-apps.db]- [metabase-enterprise.data-apps.resource-tables :as resource-tables][metabase-enterprise.data-apps.resource-validation :as resource-validation][metabase.api.common :as api][metabase.models.interface :as mi]@@ -11,8 +9,6 @@[potemkin :as p]))(p/import-vars- [resource-tables- record-table-dependencies!][resource-validationproblemswarnings])diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.cljindex 7ec8c32c162..5c96e66a020 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj@@ -5,7 +5,6 @@[metabase-enterprise.data-apps.schema :as data-apps.schema][metabase.lib.schema.id :as lib.schema.id][metabase.queries.core :as queries]- [metabase.util :as u][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]@@ -14,7 +13,7 @@(def non-blob-columns"Every DataApp column except the raw bundle blob."[:id :entity_id :name :display_name :description :version :bundle_path :enabled :allowed_hosts- :resource_collection_id :table_ids :bundle_hash :created_at :updated_at])+ :resource_collection_id :bundle_hash :created_at :updated_at])(def ^:private non-blob-model(into [:model/DataApp] non-blob-columns))@@ -113,11 +112,6 @@[entity-id](t2/select-one [:model/DataApp :id :entity_id :resource_collection_id] :entity_id entity-id))-(defn data-apps-with-resource-collections- "The `:id` and `:resource_collection_id` of every DataApp that has a resource collection."- []- (t2/select [:model/DataApp :id :resource_collection_id] :resource_collection_id [:not= nil]))-(defn active-table?"Whether the table with `table-id` is active."[table-id]@@ -151,32 +145,15 @@(t2/select [:model/Action :entity_id :collection_id] :entity_id [:in entity-ids])[]))-(defn table-ids-named- "The IDs of the active tables in the database with `database-id` that `refs` (maps of `:schema` and `:table`)- name: by name, and by schema when the reference has one, ignoring case."- [database-id refs]- (if (seq refs)- (let [lower #(some-> % u/lower-case-en)- tables (t2/select [:model/Table :id :schema :name] :db_id database-id :active true- {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]- :where [:in [:lower :name] (into [] (keep (comp lower :table)) refs)]})]- (into (sorted-set)- (for [{:keys [schema table]} refs- {:keys [id] :as named} tables- :when (and (= (lower table) (lower (:name named)))- (or (nil? schema) (= (lower schema) (lower (:schema named)))))]- id)))- #{}))+(defn collection-card-ids+ "The IDs of the cards in the collection with `collection-id`."+ [collection-id]+ (t2/select-pks-vec :model/Card :collection_id collection-id))-(defn collection-dataset-queries- "The queries of the cards and of the query actions in the collection with `collection-id`."+(defn collection-action-ids+ "The IDs of the actions in the collection with `collection-id`."[collection-id]- (into (t2/select-fn-vec :dataset_query :model/Card :collection_id collection-id)- (t2/select-fn-vec :dataset_query :model/QueryAction- {:select [:qa.*]- :from [[:query_action :qa]]- :join [[:action :a] [:= :a.id :qa.action_id]]- :where [:= :a.collection_id collection-id]})))+ (t2/select-pks-vec :model/Action :collection_id collection-id))(defn resource-collection"The resource collection with `collection-id`, or nil."diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj b/enterprise/backend/src/metabase_enterprise/data_apps/group_access.cljindex ed1ac22d51e..458f58d87fb 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj@@ -2,7 +2,9 @@(:require[metabase-enterprise.data-apps.db :as data-apps.db][metabase-enterprise.data-apps.resources :as resources]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency][metabase.api.common :as api]+ [metabase.graph.core :as graph][metabase.permissions.core :as perms][metabase.settings.core :as setting][metabase.util.i18n :refer [tru]]@@ -68,10 +70,28 @@(sort-by (juxt :database_name :schema :name))vec))+(defn- resource-table-ids+ "The IDs of the tables the cards and actions in the resource collection with `collection-id` read, directly or+ through the cards, measures, segments and snippets they use, with a transform they name standing for its output."+ [collection-id]+ (let [nodes (models.dependency/entities->nodes {:card (data-apps.db/collection-card-ids collection-id)+ :action (data-apps.db/collection-action-ids collection-id)})+ graph (models.dependency/filtered-graph-dependencies nil {:types #{:action :card :measure :segment :snippet}})+ {:keys [table transform]} (models.dependency/group-nodes (graph/transitive graph (vec nodes)))+ outputs (->> (models.dependency/entities->nodes {:transform transform})+ models.dependency/direct-dependents+ vals+ (apply concat)+ models.dependency/group-nodes+ :table)]+ (vec (into (set table) outputs))))+(defn permission-warnings"Missing table access for assigned groups, including access inherited from All Users."- [{app-id :id table-ids :table_ids}]- (let [assigned-ids (mapv :permission_group_id (data-apps.db/app-assignments [app-id]))]+ [{app-id :id collection-id :resource_collection_id}]+ (let [assigned-ids (mapv :permission_group_id (data-apps.db/app-assignments [app-id]))+ table-ids (when (and (seq assigned-ids) collection-id)+ (resource-table-ids collection-id))](if (and (seq assigned-ids) (seq table-ids))(let [tables (table-details table-ids)all-users-id (:id (perms/all-users-group))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.cljindex e9d8dbb20b1..518dc8f318e 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj@@ -40,8 +40,7 @@(t2/deftransforms :model/DataApp{:bundle transform-bundle;; JSON array of origins the sandboxed bundle may fetch/XHR (see config.clj).- :allowed_hosts mi/transform-json- :table_ids mi/transform-json})+ :allowed_hosts mi/transform-json})(doto :model/DataApp(derive :metabase/model)@@ -100,8 +99,7 @@(t2/define-after-select :model/DataApp[app](cond-> app- (contains? app :allowed_hosts) (update :allowed_hosts #(or % []))- (contains? app :table_ids) (update :table_ids #(or % []))))+ (contains? app :allowed_hosts) (update :allowed_hosts #(or % []))))(defmethod mi/can-read? :model/DataApp([app] (mi/can-read? :model/DataApp (:id app)))@@ -157,9 +155,7 @@:skip [;; admin-owned state of this instance:enabled;; set by the import itself- :bundle_hash- ;; server-managed resources, recreated on import- :table_ids]+ :bundle_hash]:transform {:created_at (serdes/date);; the app's resource collection, a collection in the `data-apps` namespace that loads before the app:resource_collection_id (assoc (serdes/fk :model/Collection) :as :collection)diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/resource_tables.clj b/enterprise/backend/src/metabase_enterprise/data_apps/resource_tables.cljdeleted file mode 100644index aceaf93a2b7..00000000000--- a/enterprise/backend/src/metabase_enterprise/data_apps/resource_tables.clj+++ /dev/null@@ -1,77 +0,0 @@-(ns metabase-enterprise.data-apps.resource-tables- "The tables a data app's resources read, recorded on the app for the permission warnings an admin sees when- granting access: a viewer who can't read one of them gets an app that fails for them."- (:require- [clojure.string :as str]- [metabase-enterprise.data-apps.db :as data-apps.db]- [metabase.driver :as driver]- [metabase.driver.util :as driver.u]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.util.log :as log]))--(set! *warn-on-reflection* true)--(defn- parseable-sql- "`sql` as a parser reads it: optional clauses kept as the SQL they hold, a card tag as a plain name, a snippet tag- as its snippet's SQL with the snippets inside it, looked up through `query`'s metadata provider, and any other- template tag as it is."- ([query sql]- (parseable-sql query sql #{}))- ([query sql expanded]- (letfn [(render [token]- (cond- (string? token) token- ;; a parser reads neither `[[` nor `]]`, and a table named inside a clause is read like one outside- (= :metabase.lib.parse/optional (:type token)) (apply str (map render (:contents token)))- ;; a card tag stands where a table does- (= \# (first (:name token))) "mb_card"- (str/starts-with? (:name token) "snippet:")- (let [snippet-name (str/trim (subs (:name token) (count "snippet:")))]- ;; a snippet that includes itself is read once- (if (contains? expanded snippet-name)- ""- (parseable-sql query- (or (:content (lib.metadata/native-query-snippet-by-name query snippet-name)) "")- (conj expanded snippet-name))))- :else (str "{{" (:name token) "}}")))]- (apply str (map render (lib/parse {} sql))))))--(defn- native-table-ids- "The tables a native query names in its SQL, as its driver's parser reads them, and in its table template tags."- [{database-id :database, :as query}]- (let [parseable (lib/with-native-query query (parseable-sql query (lib/raw-native-query query)))]- (into (data-apps.db/table-ids-named database-id- (driver/native-query-table-refs (driver.u/database->driver database-id) parseable))- (map :table)- (lib/native-query-table-references query))))--(defn- query-table-ids [dataset-query]- (let [query (lib/query (lib-be/application-database-metadata-provider (:database dataset-query)) dataset-query)]- (if (lib/native? query)- (native-table-ids query)- (into (set (lib/all-source-table-ids query))- (lib/all-implicitly-joined-table-ids query)))))--(defn collection-table-ids- "The tables the cards and the query actions in the collection with `collection-id` read, including- the ones a query reaches only through an implicit join."- [collection-id]- (->> (data-apps.db/collection-dataset-queries collection-id)- (into #{} (mapcat (fn [dataset-query]- ;; a query whose tables can't be read, such as a native query on a driver that can't- ;; parse one, must not fail the pull that loaded it- (try- (query-table-ids dataset-query)- (catch Exception e- (log/warn e "Could not read the tables of a data app query")- nil)))))- sort- vec))--(defn record-table-dependencies!- "Record on every data app the tables its resources read. Runs after an import, since that is what changes them."- []- (doseq [{:keys [id resource_collection_id]} (data-apps.db/data-apps-with-resource-collections)]- (data-apps.db/update-data-app! id {:table_ids (collection-table-ids resource_collection_id)})))diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj b/enterprise/backend/src/metabase_enterprise/data_apps/schema.cljindex fd1cc6ef934..e0a0d5b2643 100644--- a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj@@ -113,8 +113,7 @@[:updated_at {:optional true} [:maybe ms/TemporalInstant]][:description {:optional true} ::description][:version {:optional true} ::version]- [:resource_collection_id {:optional true} [:maybe ms/PositiveInt]]- [:table_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]])+ [:resource_collection_id {:optional true} [:maybe ms/PositiveInt]]])(mr/def ::data-app.insert"What an insert of a DataApp accepts: [[::data-app.update]] with its slug, display name, and bundle path required."diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.cljindex 4ef9dd2b7bb..ee424bdee36 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj@@ -737,6 +737,30 @@;;; ------------------------------------------------ Dependency status ------------------------------------------------+(def ^:private entity-database-queries+ "For each entity type that can hold a native query, a query of the IDs of its entities reading from `db-id`."+ {:card (fn [db-id] ^:allow-subquery {:select [:id]+ :from [(t2/table-name :model/Card)]+ :where [:= :database_id db-id]})+ :transform (fn [db-id] ^:allow-subquery {:select [:id]+ :from [(t2/table-name :model/Transform)]+ :where [:= :source_database_id db-id]})+ :action (fn [db-id] ^:allow-subquery {:select [:action_id]+ :from [(t2/table-name :model/QueryAction)]+ :where [:= :database_id db-id]})})++(mu/defn unresolved-table-entity-ids :- [:map-of ::deps.dependency-types/dependency-types [:set ms/PositiveInt]]+ "The IDs, by entity type, of the entities reading from the Database with `db-id` that name tables matching nothing."+ [db-id :- ::lib.schema.id/database]+ (into {}+ (keep (fn [[entity-type entity-ids-query]]+ (when-let [ids (not-empty (t2/select-fn-set :entity_id :model/DependencyStatus+ :entity_type entity-type+ :unresolved_tables true+ :entity_id [:in (entity-ids-query db-id)]))]+ [entity-type ids])))+ entity-database-queries))+(mu/defn dependency-status"The DependencyStatus of the entity `entity-type` `entity-id`, or nil."[entity-type :- EntityType@@ -778,10 +802,11 @@(mu/defn upsert-dependency-status!"Upsert the DependencyStatus of `entity-type` `entity-id`, setting stale=false, `dependency_analysis_version` to- `current-version`, and clearing any failure state."- [entity-type :- EntityType- entity-id :- ms/PositiveInt- current-version :- ms/PositiveInt]+ `current-version`, `unresolved_tables` to `unresolved-tables?`, and clearing any failure state."+ [entity-type :- EntityType+ entity-id :- ms/PositiveInt+ current-version :- ms/PositiveInt+ unresolved-tables? :- :boolean](mdb/update-or-insert!:model/DependencyStatus{:entity_type entity-type :entity_id entity-id}@@ -789,7 +814,8 @@{:dependency_analysis_version current-version:stale false:fail_count 0- :next_retry_at nil})))+ :next_retry_at nil+ :unresolved_tables unresolved-tables?})))(mu/defn record-dependency-status-failure!"Upsert the DependencyStatus of `entity-type` `entity-id`, creating it if needed. `update-fn` receives thediff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.cljindex 1edde6ea56c..9b042052562 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj@@ -219,9 +219,13 @@(before-delete* model instance))(defenterprise after-database-sync- "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed."+ "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed, and the+ entities reading from it that name tables that matched nothing, which sync may have found now.":feature :dependencies[database-id]+ (doseq [[entity-type entity-ids] (dependencies.db/unresolved-table-entity-ids database-id)+ entity-id entity-ids]+ (mark-changed! entity-type entity-id))(when (deps.findings/mark-transitive-dependents-stale!{:table (dependencies.db/table-ids-with-outdated-findings database-id)})(trigger-jobs!)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex 772c7036612..5812611cb8f 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -23,8 +23,18 @@(def current-dependency-analysis-version"Current version of the dependency analysis logic.- This should be incremented when the dependency analysis logic changes."- 6)+ This should be incremented when the dependency analysis logic changes.++ Version history:+ - 1: Initial version+ - 2: Added implicitly joined tables+ - 3: Added segments+ - 4: Tracked unreferenced and broken entities+ - 5: Added the source tables of Python transforms+ - 6: Enabled native entity validation+ - 7: Native queries depend on the cards they reference but no longer on those cards' tables, and record whether+ they name tables that match nothing"+ 7)(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.cljindex 28e3dffc71a..362646d79eb 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj@@ -24,11 +24,11 @@(dependencies.db/mark-dependency-status-stale! entity-type id)))(defn upsert-status!- "Upsert a dependency_status entry, setting stale=false, version to current,- and clearing any failure state."- [entity-type entity-id]+ "Upsert a dependency_status entry, setting stale=false, version to current, whether the entity names tables that+ match nothing, and clearing any failure state."+ [entity-type entity-id unresolved-tables?](dependencies.db/upsert-dependency-status!- entity-type entity-id models.dependency/current-dependency-analysis-version))+ entity-type entity-id models.dependency/current-dependency-analysis-version unresolved-tables?))(defmulti hydrate-for-deps"Hydrate a batch of instances with data needed for dependency calculation.diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.cljindex f65b4c025df..5496a60a5e0 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj@@ -56,15 +56,16 @@"Compile a query replacing card and table references with placeholder table names.Card refs like {{#1}} are replaced with mb__validat_card__1.Table refs like {{table_one}} are replaced with mb__validat_table__N.+ Only the refs in the query's own SQL are replaced; those inside its snippets expand with them.Returns the compiled native-only-query, or nil if the original SQL containsthe placeholder prefix (collision guard)."[query](let [stage (first (:stages query))sql (:native stage)- ttags (:template-tags stage)- card-tags (filter #(= (:type %) :card) ttags)- table-tags (filter #(= (:type %) :table) ttags)]+ own-tags (filter #(str/includes? sql (str "{{" (:name %) "}}")) (:template-tags stage))+ card-tags (filter #(= (:type %) :card) own-tags)+ table-tags (filter #(= (:type %) :table) own-tags)](when-not (or (str/includes? sql card-placeholder-prefix)(str/includes? sql table-placeholder-prefix))(let [;; Replace card tags@@ -86,7 +87,7 @@table-tags)remaining-tags- (into [] (remove #(#{:card :table} (:type %))) ttags)+ (into [] (remove (set (concat card-tags table-tags))) (:template-tags stage))modified-query(-> query@@ -307,17 +308,26 @@(compile-query query))](driver/native-result-metadata driver compiled))))+(defn- placeholder-dep?+ "Whether `dep` is a card or table placeholder that [[compile-toplevel-query]] put in place of a template tag."+ [dep]+ (when-let [table-name (-> dep :unresolved-table :table)]+ (or (parse-card-placeholder table-name)+ (table-placeholder? table-name))))+(mu/defn native-query-deps :- [:set[:or::driver/native-query-deps.table-dep::driver/native-query-deps.transform-dep+ ::driver/native-query-deps.unresolved-table-dep[:map {:closed true} [:snippet ::lib.schema.id/snippet]][:map {:closed true} [:card ::lib.schema.id/card]]]]- "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs."+ "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs, with a card it references+ standing in as a placeholder table so that the card's own tables stay out of them."[driver :- :keywordquery :- ::lib.schema/native-only-query]- (let [compiled (compile-query query)]- (into (driver/native-query-deps driver compiled)+ (let [compiled (or (compile-toplevel-query query) (compile-query query))]+ (into (into #{} (remove placeholder-dep?) (driver/native-query-deps driver compiled));; TODO (Cam 10/1/25) -- Even this much MBQL manipulation outside of Lib is illegal. Move this sort of stuff;; into Lib.(keep #(case (:type %)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj b/enterprise/backend/src/metabase_enterprise/dependencies/schema.cljindex 219f8caaad5..fe54439a1d9 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj@@ -14,7 +14,10 @@[:dashboard {:optional true} [:set ::lib.schema.id/dashboard]][:document {:optional true} [:set ::documents.schema/document.id]][:sandbox {:optional true} [:set ::lib.schema.id/sandbox]]- [:segment {:optional true} [:set ::lib.schema.id/segment]]])+ [:segment {:optional true} [:set ::lib.schema.id/segment]]+ [:unresolved-table {:optional true} [:set [:map+ [:schema {:optional true} :string]+ [:table :string]]]]])(mr/def ::analysis-finding"A AnalysisFinding as selected from the app DB: every column of `:analysis_finding`."@@ -81,4 +84,5 @@[:stale {:optional true} [:maybe :boolean]][:fail_count {:optional true} [:maybe :int]][:next_retry_at {:optional true} [:maybe ms/TemporalInstant]]- [:terminal {:optional true} [:maybe :boolean]]])+ [:terminal {:optional true} [:maybe :boolean]]+ [:unresolved_tables {:optional true} [:maybe :boolean]]])diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljindex 3e50587b6eb..cda348a3988 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj@@ -21,6 +21,7 @@[metabase-enterprise.dependencies.settings :as deps.settings][metabase-enterprise.dependencies.task-util :as deps.task-util][metabase-enterprise.dependencies.task.entity-check :as task.entity-check]+ [metabase.graph.core :as graph][metabase.lib-be.core :as lib-be][metabase.premium-features.core :as premium-features][metabase.task.core :as task]@@ -31,12 +32,18 @@(set! *warn-on-reflection* true)+(def ^:private expanded-into-dependents+ "The entity types whose content a native query using them expands into its own SQL, so that a change to one changes+ the table dependencies of the entities using it, which no graph edge carries."+ #{:snippet})+(def ^:private entity-types- "The list of entity types to backfill.+ "The list of entity types to backfill, those expanded into their dependents first, so that the dependents they mark+ are recomputed in the same run.This is not the same as deps.dependency-types/dependency-types, because tables shouldn't be backfilled. Instead, linksinvolving tables are found via analysis of the other side of the relation."- (vec deps.dependency-types/backfillable-dependency-types))+ (vec (sort-by (complement expanded-into-dependents) deps.dependency-types/backfillable-dependency-types)))(def ^:private max-retries 5)@@ -75,19 +82,29 @@entity-type(deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))+(defn- mark-expanding-entities-stale!+ "Mark for recalculation the entities whose native SQL expands the `entity-type` entity `entity-id`, directly or+ through the entities of the same kind that include it."+ [entity-type entity-id]+ (let [graph (models.dependency/filtered-graph-dependents nil {:types expanded-into-dependents})+ type->ids (models.dependency/group-nodes (graph/transitive graph [[entity-type entity-id]]))]+ (deps.dependency-status/mark-stale! (apply dissoc type->ids expanded-into-dependents))))+(defn- compute-deps-for-entity!- "Mark the dependents of an entity that a write marked stale for re-analysis, then compute and store its- dependencies and update its dependency_status. Entities are expected to be pre-hydrated by- [[deps.dependency-status/hydrate-for-deps]]."+ "Mark the dependents of an entity that a write marked stale for re-analysis, and those that expand it for+ recalculation, then compute and store its dependencies and update its dependency_status, flagging the tables it+ names that match nothing. Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."[entity-type entity](log/debug "Computing dependencies for" (name entity-type) (:id entity))(when (:stale (dependencies.db/dependency-status entity-type (:id entity)))- (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}}))+ (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}})+ (when (expanded-into-dependents entity-type)+ (mark-expanding-entities-stale! entity-type (:id entity))))(t2/with-transaction [_](let [deps (deps.calculation/calculate-deps entity-type entity)]- (models.dependency/replace-dependencies! entity-type (:id entity) deps)+ (models.dependency/replace-dependencies! entity-type (:id entity) (dissoc deps :unresolved-table))(post-deps-cleanup! entity-type entity)- (deps.dependency-status/upsert-status! entity-type (:id entity)))))+ (deps.dependency-status/upsert-status! entity-type (:id entity) (contains? deps :unresolved-table)))))(defn- remove-deleted-entity-batch!"Mark the dependents of up to `batch-size` deleted entities of `entity-type` for re-analysis and drop theirdiff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.cljindex cfeedb6e517..3087fedb117 100644--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj@@ -3,7 +3,6 @@[clojure.string :as str][diehard.core :as dh][java-time.api :as t]- [metabase-enterprise.data-apps.core :as data-apps][metabase-enterprise.remote-sync.db :as remote-sync.db][metabase-enterprise.remote-sync.guards :as guards][metabase-enterprise.remote-sync.merge :as remote-sync.merge]@@ -579,15 +578,6 @@(remote-sync.db/update-rso! (:id existing) {:status status :status_changed_at timestamp})(remote-sync.db/insert-rso! (-> row (dissoc :id) (assoc :status_changed_at timestamp))))))-(defn- record-data-app-tables!- "Record the tables each data app's resources read, after a load, logging a failure: the load's transaction has- committed by then, so a failure must not undo its record."- []- (try- (data-apps/record-table-dependencies!)- (catch Exception e- (log/warn e "Could not record the tables of the data apps"))))-(defn- import-merged!"Import in merge mode. Should only be called when you have a base-snapshot and its version differs from snaphot's version.@@ -618,7 +608,6 @@:finalize! (fn [](restore-dirty-objects! dirty-objects sync-timestamp)(finalize!)))- (record-data-app-tables!)(log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"(apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)(count dirty-objects))@@ -718,8 +707,7 @@:else(let [_ (log/info "Remote sync full import: forced")- imported-data (u/prog1 (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)- (record-data-app-tables!))]+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)](log/info "Successfully reloaded entities from git repository"){:status :success:version snapshot-version@@ -757,8 +745,7 @@first-import? "first import":else "changes not incrementally loadable")_ (log/infof "Remote sync full import: %s" reason)- imported-data (u/prog1 (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)- (record-data-app-tables!))]+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)](log/info "Successfully reloaded entities from git repository"){:status :success:version snapshot-version@@ -856,7 +843,6 @@:finalize! (fn [](remote-sync.db/mark-all-rsos-synced! sync-timestamp)(remote-sync.task/set-version! task-id version)))- (record-data-app-tables!)(log/infof "Exported with merge: folded in %d remote change(s) (added %d, updated %d, removed %d); pushed %d"pulled (:added summary) (:updated summary) (:removed summary) (if empty? 0 pushed-count)){:status :success :version version :merge-summary summarydiff --git a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.cljindex c788bd6d305..ef97f08d808 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj@@ -2,6 +2,10 @@(:require[clojure.test :refer :all][metabase-enterprise.data-apps.group-access :as group-access]+ [metabase-enterprise.dependencies.test-util :as deps.test]+ [metabase.dependencies.hooks :as dependencies.hooks]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.permissions-rest.data-permissions.graph :as data-perms.graph][metabase.permissions.core :as perms][metabase.test :as mt]@@ -9,118 +13,212 @@(set! *warn-on-reflection* true)+(defn- do-with-app-reading!+ "Call `f` with a data app whose resource collection holds a card that reads `table-ids`."+ [table-ids f]+ (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"}+ :model/Card card {:collection_id (:resource_collection_id app)}]+ (mt/with-model-cleanup [:model/Dependency]+ (when (seq table-ids)+ (t2/insert! :model/Dependency+ (for [table-id table-ids]+ {:from_entity_type :card :from_entity_id (:id card)+ :to_entity_type :table :to_entity_id table-id})))+ (f app))))++(defmacro ^{:private true :clj-kondo/lint-as 'clojure.core/let} with-app-reading!+ "Bind `app-sym` to a data app whose resource collection reads `table-ids`, and run `body`."+ [[app-sym table-ids] & body]+ `(do-with-app-reading! ~table-ids (fn [~app-sym] ~@body)))++(deftest group-permission-warnings-walk-the-dependency-graph-test+ (testing "the warnings list the tables a native card, a card on another card, and a query action read"+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}+ (mt/with-no-data-perms-for-all-users!+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}+ :model/Card {base-id :id} {:collection_id collection-id+ :dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}+ :model/Card _ {:collection_id collection-id+ :dataset_query (lib/native-query mp "SELECT * FROM VENUES")}+ :model/PermissionsGroup group {}]+ (mt/with-temp [:model/Card _ {:collection_id collection-id+ :dataset_query (lib/query mp (lib.metadata/card mp base-id))}+ :model/Action {action-id :id} {:type :query :name "Delete" :collection_id collection-id}+ :model/QueryAction _ {:action_id action-id+ :database_id (mt/id)+ :dataset_query (lib/native-query mp "DELETE FROM CHECKINS WHERE ID = 1")}]+ (deps.test/synchronously-run-backfill!)+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (is (=? [{:group_id (:id group)+ :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :orders)} {:id (mt/id :venues)}]}]+ (group-access/permission-warnings app)))))))))))++(deftest group-permission-warnings-include-tables-in-optional-clauses-test+ (testing "the tables an optional clause names count, whether or not its parameter is set"+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}+ (mt/with-no-data-perms-for-all-users!+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]+ (doseq [sql ["SELECT * FROM VENUES [[WHERE ID IN (SELECT VENUE_ID FROM CHECKINS WHERE ID = {{id}})]]"+ "SELECT * FROM VENUES v [[JOIN CHECKINS c ON c.VENUE_ID = v.ID AND c.ID = {{id}}]]"]]+ (testing sql+ (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}+ :model/Card _ {:collection_id collection-id+ :dataset_query (lib/native-query (mt/metadata-provider) sql)}+ :model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (deps.test/synchronously-run-backfill!)+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}]+ (group-access/permission-warnings app)))))))))))++(deftest group-permission-warnings-follow-snippet-edit-test+ (testing "after a snippet edit and the jobs, the warnings list the tables of the snippet's new SQL"+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}+ (mt/with-no-data-perms-for-all-users!+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "birds source" :content "VENUES"}+ :model/DataApp {collection-id :resource_collection_id :as app}+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}+ :model/Card _ {:collection_id collection-id+ :dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT * FROM {{snippet: birds source}}")}+ :model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (deps.test/synchronously-run-backfill!)+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "CHECKINS"})+ (deps.test/synchronously-run-backfill!)+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)}]}]+ (group-access/permission-warnings app)))))))))++(deftest group-permission-warnings-include-table-synced-after-analysis-test+ (testing "a table that sync finds after the app's card was analyzed enters the warnings once the jobs run"+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}+ (mt/with-no-data-perms-for-all-users!+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]+ (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}+ :model/Card _ {:collection_id collection-id+ :dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT * FROM PUBLIC.BIRDS_LATE")}+ :model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (deps.test/synchronously-run-backfill!)+ (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]+ (dependencies.hooks/after-database-sync (mt/id))+ (deps.test/synchronously-run-backfill!)+ (is (=? [{:group_id (:id group) :missing_tables [{:id late-id}]}]+ (group-access/permission-warnings app))))))))))+(deftest group-permission-warnings-exclude-inactive-tables-test(mt/with-premium-features #{:data-apps :advanced-permissions}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders) (mt/id :categories)]}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (group-access/add-groups! app [(:id group)])- (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}- (mt/with-temp-vals-in-db :model/Table (mt/id :orders) {:visibility_type :hidden}- ;; exclude inactive tables, but keep active visible and hidden tables.- ;; permissions editor also follow this filtering.- (is (=? [{:group_id (:id group)- :missing_tables [{:id (mt/id :categories)} {:id (mt/id :orders)}]}]- (group-access/permission-warnings app)))))))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders) (mt/id :categories)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}+ (mt/with-temp-vals-in-db :model/Table (mt/id :orders) {:visibility_type :hidden}+ ;; exclude inactive tables, but keep active visible and hidden tables.+ ;; permissions editor also follow this filtering.+ (is (=? [{:group_id (:id group)+ :missing_tables [{:id (mt/id :categories)} {:id (mt/id :orders)}]}]+ (group-access/permission-warnings app))))))))))(deftest group-permission-warnings-ignore-apps-with-only-inactive-tables-test(mt/with-premium-features #{:data-apps :advanced-permissions}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues)]}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (group-access/add-groups! app [(:id group)])+ (with-app-reading! [app [(mt/id :venues)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)]); if data apps only references inactive tables, there will be no access warnings- (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}- (is (= [] (group-access/permission-warnings app))))))))+ (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}+ (is (= [] (group-access/permission-warnings app)))))))))(deftest group-permission-warnings-ignore-unassigned-groups-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp _app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [_app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))(deftest group-permission-warnings-include-inherited-access-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (group-access/add-groups! app [(:id group)])- (perms/set-table-permission! (perms/all-users-group) (mt/id :venues) :perms/view-data :unrestricted)- (is (=? [{:group_id (:id group)- :missing_tables [{:id (mt/id :orders) :name "Orders" :database_id (mt/id)}]}]- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (perms/set-table-permission! (perms/all-users-group) (mt/id :venues) :perms/view-data :unrestricted)+ (is (=? [{:group_id (:id group)+ :missing_tables [{:id (mt/id :orders) :name "Orders" :database_id (mt/id)}]}]+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))(deftest group-permission-warnings-preserve-permissions-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (group-access/add-groups! app [(:id group)])- (let [before (t2/select :model/DataPermissions)]- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")- (is (= before (t2/select :model/DataPermissions))))))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (let [before (t2/select :model/DataPermissions)]+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")+ (is (= before (t2/select :model/DataPermissions)))))))))(deftest group-permission-warnings-include-direct-access-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup group {}]- (group-access/add-groups! app [(:id group)])- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (perms/set-table-permission! (:id group) (mt/id :venues) :perms/view-data :unrestricted)- (perms/set-table-permission! (:id group) (mt/id :orders) :perms/view-data :unrestricted)- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (group-access/add-groups! app [(:id group)])+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (perms/set-table-permission! (:id group) (mt/id :venues) :perms/view-data :unrestricted)+ (perms/set-table-permission! (:id group) (mt/id :orders) :perms/view-data :unrestricted)+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))(deftest group-permission-warnings-include-sandbox-access-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup group {}]- (group-access/add-groups! app [(:id group)])- (mt/user-http-request :crowberto :put 200 "permissions/graph"- (-> (data-perms.graph/api-graph)- (assoc-in [:groups (:id group) (mt/id) :view-data]- {"PUBLIC" {(mt/id :venues) :unrestricted- (mt/id :orders) :sandboxed}})- (assoc :sandboxes [{:group_id (:id group) :table_id (mt/id :orders)}])))- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))- (mt/user-http-request :crowberto :put 200 "permissions/graph"- (assoc-in (data-perms.graph/api-graph)- [:groups (:id group) (mt/id) :view-data "PUBLIC" (mt/id :orders)]- :blocked))- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :orders)}]}]- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (group-access/add-groups! app [(:id group)])+ (mt/user-http-request :crowberto :put 200 "permissions/graph"+ (-> (data-perms.graph/api-graph)+ (assoc-in [:groups (:id group) (mt/id) :view-data]+ {"PUBLIC" {(mt/id :venues) :unrestricted+ (mt/id :orders) :sandboxed}})+ (assoc :sandboxes [{:group_id (:id group) :table_id (mt/id :orders)}])))+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))+ (mt/user-http-request :crowberto :put 200 "permissions/graph"+ (assoc-in (data-perms.graph/api-graph)+ [:groups (:id group) (mt/id) :view-data "PUBLIC" (mt/id :orders)]+ :blocked))+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :orders)}]}]+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))(deftest group-permission-warnings-require-admin-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup group {}]- (group-access/add-groups! app [(:id group)])- (mt/user-http-request :rasta :get 403 "apps/birds/group-permission-warnings")))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (group-access/add-groups! app [(:id group)])+ (mt/user-http-request :rasta :get 403 "apps/birds/group-permission-warnings"))))))(deftest group-permission-warnings-without-dependencies-test(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids []}- :model/PermissionsGroup group {}]- (group-access/add-groups! app [(:id group)])- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [app []]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (group-access/add-groups! app [(:id group)])+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))(deftest group-permission-warnings-missing-app-test(mt/with-premium-features #{:data-apps}@@ -129,61 +227,57 @@(deftest group-permission-warnings-use-group-access-not-memberships-test(mt/with-premium-features #{:data-apps :advanced-permissions}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues)]}- :model/PermissionsGroup group {}- :model/PermissionsGroup other-group {}]- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)- (perms/set-database-permission! (:id other-group) (mt/id) :perms/view-data :unrestricted)- (group-access/add-groups! app [(:id group)])- (perms/add-user-to-group! (mt/user->id :rasta) (:id group))- (perms/add-user-to-group! (mt/user->id :rasta) (:id other-group))- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]- (group-access/permission-warnings app)))))))+ (with-app-reading! [app [(mt/id :venues)]]+ (mt/with-temp [:model/PermissionsGroup group {}+ :model/PermissionsGroup other-group {}]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (perms/set-database-permission! (:id other-group) (mt/id) :perms/view-data :unrestricted)+ (group-access/add-groups! app [(:id group)])+ (perms/add-user-to-group! (mt/user->id :rasta) (:id group))+ (perms/add-user-to-group! (mt/user->id :rasta) (:id other-group))+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]+ (group-access/permission-warnings app))))))))(deftest group-permission-warnings-include-impersonated-access-test(mt/with-premium-features #{:data-apps :advanced-permissions}(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues)]}- :model/PermissionsGroup group {}]- (group-access/add-groups! app [(:id group)])- (mt/user-http-request :crowberto :put 200 "permissions/graph"- (-> (data-perms.graph/api-graph)- (assoc-in [:groups (:id group) (mt/id) :view-data] :impersonated)- (assoc :impersonations [{:group_id (:id group) :db_id (mt/id)- :attribute "role"}])))- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))- (mt/user-http-request :crowberto :put 200 "permissions/graph"- (assoc-in (data-perms.graph/api-graph)- [:groups (:id group) (mt/id) :view-data]- :blocked))- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [app [(mt/id :venues)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (group-access/add-groups! app [(:id group)])+ (mt/user-http-request :crowberto :put 200 "permissions/graph"+ (-> (data-perms.graph/api-graph)+ (assoc-in [:groups (:id group) (mt/id) :view-data] :impersonated)+ (assoc :impersonations [{:group_id (:id group) :db_id (mt/id)+ :attribute "role"}])))+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))+ (mt/user-http-request :crowberto :put 200 "permissions/graph"+ (assoc-in (data-perms.graph/api-graph)+ [:groups (:id group) (mt/id) :view-data]+ :blocked))+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))(deftest group-permission-warning-lookups-are-batched-test(mt/with-no-data-perms-for-all-users!- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues) (mt/id :orders)]}- :model/PermissionsGroup finches {}- :model/PermissionsGroup owls {}]- (group-access/add-groups! app [(:id finches)])- (group-access/permission-warnings app)- (let [query-count #(t2/with-call-count [calls]- (group-access/permission-warnings app)- (calls))- one-group-count (query-count)]- (group-access/add-groups! app [(:id owls)])- (is (= one-group-count (query-count)))))))+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]+ (mt/with-temp [:model/PermissionsGroup finches {}+ :model/PermissionsGroup owls {}]+ (group-access/add-groups! app [(:id finches)])+ (group-access/permission-warnings app)+ (let [query-count #(t2/with-call-count [calls]+ (group-access/permission-warnings app)+ (calls))+ one-group-count (query-count)]+ (group-access/add-groups! app [(:id owls)])+ (is (= one-group-count (query-count))))))))(deftest group-permission-warnings-require-feature-test- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(mt/id :venues)]}- :model/PermissionsGroup group {}]- (group-access/add-groups! app [(:id group)])- (mt/with-premium-features #{}- (mt/user-http-request :crowberto :get 402 "apps/birds/group-permission-warnings")- (mt/user-http-request :rasta :get 402 "apps/birds/group-permission-warnings"))))+ (with-app-reading! [app [(mt/id :venues)]]+ (mt/with-temp [:model/PermissionsGroup group {}]+ (group-access/add-groups! app [(:id group)])+ (mt/with-premium-features #{}+ (mt/user-http-request :crowberto :get 402 "apps/birds/group-permission-warnings")+ (mt/user-http-request :rasta :get 402 "apps/birds/group-permission-warnings")))))(deftest group-permission-warning-lookups-exclude-unrelated-permissions-test(mt/with-temp [:model/Database other-db {}@@ -220,18 +314,17 @@:model/Table albatrosses {:db_id (:id zoo) :name "albatrosses":display_name "Albatrosses" :schema nil}:model/TableUserSettings _ {:table_id (:id finches) :display_name "Finches"}- :model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"- :table_ids [(:id albatrosses) (:id owls) (:id finches)]}:model/PermissionsGroup group {}]- (doseq [group-id [(:id (perms/all-users-group)) (:id group)]- database-id [(:id aviary) (:id zoo)]]- (perms/set-database-permission! group-id database-id :perms/view-data :blocked))- (group-access/add-groups! app [(:id group)])- (is (= [{:group_id (:id group)- :missing_tables [{:id (:id finches) :name "Finches" :schema "PUBLIC"- :database_id (:id aviary) :database_name "Aviary"}- {:id (:id owls) :name "Owls" :schema "PUBLIC"- :database_id (:id aviary) :database_name "Aviary"}- {:id (:id albatrosses) :name "Albatrosses" :schema nil- :database_id (:id zoo) :database_name "Zoo"}]}]- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))+ (with-app-reading! [app [(:id albatrosses) (:id owls) (:id finches)]]+ (doseq [group-id [(:id (perms/all-users-group)) (:id group)]+ database-id [(:id aviary) (:id zoo)]]+ (perms/set-database-permission! group-id database-id :perms/view-data :blocked))+ (group-access/add-groups! app [(:id group)])+ (is (= [{:group_id (:id group)+ :missing_tables [{:id (:id finches) :name "Finches" :schema "PUBLIC"+ :database_id (:id aviary) :database_name "Aviary"}+ {:id (:id owls) :name "Owls" :schema "PUBLIC"+ :database_id (:id aviary) :database_name "Aviary"}+ {:id (:id albatrosses) :name "Albatrosses" :schema nil+ :database_id (:id zoo) :database_name "Zoo"}]}]+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/resource_tables_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/resource_tables_test.cljdeleted file mode 100644index b5c4a77d596..00000000000--- a/enterprise/backend/test/metabase_enterprise/data_apps/resource_tables_test.clj+++ /dev/null@@ -1,46 +0,0 @@-(ns metabase-enterprise.data-apps.resource-tables-test- "The tables a data app's resources read, recorded for the permission warnings an admin sees."- (:require- [clojure.test :refer :all]- [metabase-enterprise.data-apps.resource-tables :as resource-tables]- [metabase.lib.core :as lib]- [metabase.test :as mt]))--(set! *warn-on-reflection* true)--(deftest a-table-in-an-optional-clause-is-recorded-test- (testing "an optional clause is ordinary query syntax that a parser doesn't read, and the tables inside it are read- like the ones outside"- (let [native (fn [sql] (lib/native-query (mt/metadata-provider) sql))]- (mt/with-temp [:model/Collection {collection-id :id} {:namespace :data-apps}- :model/Card _ {:collection_id collection-id- :dataset_query (native "SELECT * FROM venues WHERE 1=1 [[AND name = {{name}}]]")}- :model/Card _ {:collection_id collection-id- :dataset_query (native "SELECT * FROM venues v [[JOIN checkins c ON c.venue_id = v.id AND c.id = {{id}}]]")}]- (is (= (sort [(mt/id :venues) (mt/id :checkins)])- (resource-tables/collection-table-ids collection-id))- "checkins is named only inside the optional clause")))))--(deftest a-table-joined-to-a-card-tag-is-recorded-test- (testing "a card tag stands where a table does, and the parser reads the query around it"- (let [native (fn [sql] (lib/native-query (mt/metadata-provider) sql))]- (mt/with-temp [:model/Collection {collection-id :id} {:namespace :data-apps}- :model/Card {venues-id :id} {:collection_id collection-id- :dataset_query (native "SELECT * FROM venues")}- :model/Card _ {:collection_id collection-id- :dataset_query (native (format "SELECT * FROM {{#%d}} x JOIN checkins c ON c.venue_id = x.id" venues-id))}]- (is (= (sort [(mt/id :venues) (mt/id :checkins)])- (resource-tables/collection-table-ids collection-id)))))))--(deftest a-table-in-a-snippet-is-recorded-test- (testing "a snippet tag stands for the SQL its snippet holds, snippets inside it included, so the tables it names- are read like the ones around it"- (let [native (fn [sql] (lib/native-query (mt/metadata-provider) sql))]- (mt/with-temp [:model/NativeQuerySnippet _ {:name "venues table" :content "venues"}- :model/NativeQuerySnippet _ {:name "checked in" :content "id IN (SELECT venue_id FROM checkins)"}- :model/NativeQuerySnippet _ {:name "open" :content "price > 1 AND {{snippet: checked in}}"}- :model/Collection {collection-id :id} {:namespace :data-apps}- :model/Card _ {:collection_id collection-id- :dataset_query (native "SELECT * FROM {{snippet: venues table}} v WHERE {{snippet: open}}")}]- (is (= (sort [(mt/id :venues) (mt/id :checkins)])- (resource-tables/collection-table-ids collection-id)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.cljindex 9402ccaf1ce..d04510c780d 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj@@ -704,7 +704,7 @@:display-name "Venues":card-id card-id}}))}](is (= {:card #{card-id}- :table #{(mt/id :checkins) (mt/id :venues)}}+ :table #{(mt/id :checkins)}}(calculation/calculate-deps :action (actions/select-action :id action-id)))))))(deftest upstream-deps-implicit-action-testdiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex 353dd9f2436..b07825ca508 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -781,8 +781,8 @@(deps.findings/upsert-analysis! parent-card)(deps.findings/upsert-analysis! child-card)(deps.findings/upsert-analysis! other-card)- (deps.dependency-status/upsert-status! :card child-card-id)- (deps.dependency-status/upsert-status! :card other-card-id)+ (deps.dependency-status/upsert-status! :card child-card-id false)+ (deps.dependency-status/upsert-status! :card other-card-id false)(testing "Checking that the initial analyses exist"(assert-has-analyses{:card {parent-card-id old-version@@ -817,7 +817,7 @@:to_entity_type :card :to_entity_id parent-id}](deps.findings/upsert-analysis! parent)(deps.findings/upsert-analysis! child)- (deps.dependency-status/upsert-status! :card child-id)+ (deps.dependency-status/upsert-status! :card child-id false)(t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")})(deps.test/synchronously-run-backfill!)(testing "Parent card should be marked stale"@@ -993,7 +993,7 @@:to_entity_type :segment :to_entity_id segment-id}](deps.findings/upsert-analysis! segment)(deps.findings/upsert-analysis! card)- (deps.dependency-status/upsert-status! :card card-id)+ (deps.dependency-status/upsert-status! :card card-id false)(assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})(t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})(deps.test/synchronously-run-backfill!)diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.cljindex d3e8dec01e6..15586d21787 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj@@ -34,7 +34,7 @@(mt/with-premium-features #{:dependencies}(mt/with-model-cleanup [:model/DependencyStatus](mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]- (deps.dependency-status/upsert-status! :card card-id)+ (deps.dependency-status/upsert-status! :card card-id false)(t2/update! :model/Card card-id {:name "Renamed"})(is (not (stale? :card card-id)))(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})@@ -58,7 +58,7 @@(mt/with-premium-features #{}(mt/with-model-cleanup [:model/DependencyStatus](mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]- (deps.dependency-status/upsert-status! :card card-id)+ (deps.dependency-status/upsert-status! :card card-id false)(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})(is (not (stale? :card card-id))))))))@@ -69,7 +69,7 @@(mt/with-temp [:model/Dashboard {dashboard-id :id} {}:model/Card {card-id :id} {:dataset_query (table-query :products)}:model/Card {other-id :id} {:dataset_query (table-query :orders)}]- (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id)]+ (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id false)](processed!)(let [dashcard-id (t2/insert-returning-pk! :model/DashboardCard{:dashboard_id dashboard-id :card_id card-id@@ -98,7 +98,7 @@(mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}](t2/insert! :model/Dependency {:from_entity_type "card" :from_entity_id card-id:to_entity_type "table" :to_entity_id (mt/id :products)})- (deps.dependency-status/upsert-status! :card card-id)+ (deps.dependency-status/upsert-status! :card card-id false)(t2/delete! :model/Card :id card-id)(is (stale? :card card-id))(deps.test/synchronously-run-backfill!)@@ -131,7 +131,7 @@(t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-b:to_entity_type :card :to_entity_id card-a})(doseq [card-id [card-a card-b]]- (deps.dependency-status/upsert-status! :card card-id)+ (deps.dependency-status/upsert-status! :card card-id false)(deps.analysis-finding/upsert-analysis! :card card-id true []))(mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!(fn [type->ids]@@ -160,7 +160,7 @@triggered (atom 0)](t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id:row 4 :col 0 :size_x 4 :size_y 4})- (deps.dependency-status/upsert-status! :dashboard dashboard-id)+ (deps.dependency-status/upsert-status! :dashboard dashboard-id false)(mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))task.backfill/trigger-backfill-job! (fn [] (swap! triggered inc))](deps.test/do-with-immediate-hook-writes@@ -182,7 +182,7 @@:model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id:visualization_settings {}}](let [exported (ts/extract-one "Dashboard" dashboard-id)]- (deps.dependency-status/upsert-status! :dashboard dashboard-id)+ (deps.dependency-status/upsert-status! :dashboard dashboard-id false)(mt/with-dynamic-fn-redefs [task.backfill/trigger-backfill-job! (fn [])](deps.test/do-with-immediate-hook-writes#(serdes/with-cachediff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.cljindex 6c2e4ab094f..a9a5ae489e8 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj@@ -136,10 +136,8 @@(deps.test/synchronously-run-backfill!)(testing "raw deps are recorded correctly"(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))- (is (=? #{(depends-on-> :card id2 :table (mt/id :orders))- (depends-on-> :card id2 :card id1)} (upstream-of :card id2)))- (is (=? #{(depends-on-> :card id3 :table (mt/id :orders))- (depends-on-> :card id3 :card id2)} (upstream-of :card id3))))+ (is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))+ (is (=? #{(depends-on-> :card id3 :card id2)} (upstream-of :card id3))))(testing "transitive deps are computed correctly"(testing "for each card"(is (=? {:card #{id2 id3}}diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.cljindex 7b2745f85a3..6cce6c3ae00 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj@@ -67,8 +67,7 @@:dataset-query(lib-be/normalize-query mp)(deps.native-validation/native-query-deps driver))))- (is (= #{{:table (mt/id :products)}- {:card 1}}+ (is (= #{{:card 1}}(->> (lib.metadata/card mp 5):dataset-query(lib-be/normalize-query mp)@@ -88,9 +87,7 @@:dataset-query(lib-be/normalize-query mp)(deps.native-validation/native-query-deps driver))))- (is (= #{{:table (mt/id :products)}- {:table (mt/id :orders)}- {:card 1}+ (is (= #{{:card 1}{:card 2}}(->> (lib.metadata/card mp 9):dataset-querydiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljindex e84d4e5159f..8f2ec778799 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -11,6 +11,8 @@[metabase-enterprise.dependencies.task.backfill :as dependencies.backfill][metabase-enterprise.dependencies.test-util :as deps.test][metabase.dependencies.hooks :as dependencies.hooks]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.premium-features.core :as premium-features][metabase.task.core :as task][metabase.test :as mt]@@ -585,3 +587,59 @@(#'dependencies.backfill/run-and-reschedule! nil))))(testing "and the next run was scheduled anyway"(is (= 1 (count @scheduled))))))))++(defn- table-dependency-ids+ "The IDs of the tables the `entity-type` entity `entity-id` depends on directly."+ [entity-type entity-id]+ (t2/select-fn-set :to_entity_id :model/Dependency+ :from_entity_type entity-type :from_entity_id entity-id :to_entity_type :table))++(deftest ^:synchronized backfill-snippet-edit-recomputes-queries-expanding-it-test+ (testing "a snippet's tables are the card's own dependencies, so editing a snippet, even one nested in another,+ recomputes the cards whose SQL expands it"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "birds inner" :content "VENUES"}+ :model/NativeQuerySnippet _ {:name "birds outer" :content "SELECT * FROM {{snippet: birds inner}}"}+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "{{snippet: birds outer}}")}]+ (backfill-all-existing-entities!)+ (is (= #{(mt/id :venues)} (table-dependency-ids :card card-id)))+ (t2/update! :model/NativeQuerySnippet inner-id {:content "CHECKINS"})+ (backfill-all-existing-entities!)+ (is (= #{(mt/id :checkins)} (table-dependency-ids :card card-id)))))))++(deftest ^:synchronized backfill-card-reference-leaves-out-card-tables-test+ (testing "a native query depends on the card it references, not on the card's tables, which it reads through the card"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies}+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card {orders-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}+ :model/Card {card-id :id} {:dataset_query+ (lib/native-query+ mp+ (format "SELECT * FROM {{#%d}} o JOIN PRODUCTS p ON o.PRODUCT_ID = p.ID"+ orders-id))}]+ (backfill-all-existing-entities!)+ (is (= #{(mt/id :products)} (table-dependency-ids :card card-id)))+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :card :to_entity_id orders-id)))))))++(deftest ^:synchronized backfill-records-table-sync-finds-later-test+ (testing "a table a native query names before sync finds it gets its dependency once a sync of the database finds it"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT * FROM PUBLIC.BIRDS_LATE")}+ :model/Card {known-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)+ "SELECT * FROM VENUES")}]+ (backfill-all-existing-entities!)+ (is (empty? (table-dependency-ids :card card-id)))+ (is (true? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id card-id)))+ (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id known-id)))+ (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]+ (dependencies.hooks/after-database-sync (mt/id))+ (backfill-all-existing-entities!)+ (is (= #{late-id} (table-dependency-ids :card card-id)))+ (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus+ :entity_type :card :entity_id card-id))))))))diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.cljindex d7067c66e16..6e4c065e2cf 100644--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj@@ -5,8 +5,9 @@(:require[clojure.string :as str][clojure.test :refer :all]- [metabase-enterprise.data-apps.core :as data-apps]+ [metabase-enterprise.data-apps.group-access :as group-access][metabase-enterprise.data-apps.test-util :as data-apps.tu]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.remote-sync.impl :as impl][metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task][metabase-enterprise.remote-sync.source :as source]@@ -15,10 +16,10 @@[metabase-enterprise.remote-sync.test-helpers :as test-helpers][metabase.actions.core :as actions][metabase.collections.models.collection :as collection]- [metabase.driver :as driver][metabase.events.core :as events][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.permissions.core :as perms][metabase.search.test-util :as search.tu][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -226,9 +227,7 @@(is (=? {:type :query :collection_id collection-id :model_id nil}(t2/select-one :model/Action :entity_id (copy-eid "action" :model/Action action-id)))))(testing "the source is untouched"- (is (nil? (t2/select-one-fn :collection_id :model/Action :id action-id))))- (testing "the app records the tables its resources read"- (is (= [(mt/id :venues)] (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))))))+ (is (nil? (t2/select-one-fn :collection_id :model/Action :id action-id)))))))))))(deftest pull-updates-and-removes-resources-with-their-files-test(with-data-apps-sync@@ -329,6 +328,41 @@[mock stem](some #(str/starts-with? (last (str/split % #"/")) stem) (resource-files mock)))+(deftest pull-reaches-permission-warnings-test+ (testing "once the jobs run after a pull, the warnings list the tables the app's native question and action name"+ (with-data-apps-sync+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}+ (mt/with-no-data-perms-for-all-users!+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]+ (let [action-id (actions/insert! {:name "Clear checkins"+ :type :query+ :database_id (mt/id)+ :dataset_query (lib/native-query (mt/metadata-provider)+ "DELETE FROM checkins WHERE id = {{id}}")+ :parameters [{:id "id" :slug "id" :type :number/=}]})+ resources (data-apps.tu/build-resources+ shop-collection-name shop-collection-eid+ [{:entity_id question-eid :name "VenuesList" :query (venues-query)}]+ [action-id])+ path (some #(when (str/includes? % question-eid) %) (keys resources))+ parsed (yaml/parse-string (get resources path))+ native (assoc parsed :dataset_query {:database (:database (:dataset_query parsed))+ :lib/type "mbql/query"+ :stages [{:lib/type "mbql.stage/native"+ :native "select * from Venues"}]})+ src (test-helpers/versioned-source+ :trees {"v0" (shop-tree (assoc resources path (yaml/generate-string native)))}+ :current "v0")+ result (import-at! src "v0" :force? true)]+ (is (= :success (:status result)) (:message result))+ (mt/with-temp [:model/PermissionsGroup group {}]+ (let [app (t2/select-one :model/DataApp :name "shop")]+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)+ (group-access/add-groups! app [(:id group)])+ (deps.test/synchronously-run-backfill!)+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}]+ (group-access/permission-warnings app))))))))))))+(deftest an-apps-collection-is-read-only-test(testing "an admin can't write an app's collection, its cards or its actions, whatever the sync mode"(with-data-apps-sync@@ -597,32 +631,6 @@(is (not (t2/exists? :model/DataApp :name [:in ["first" "second"]])))(is (not (t2/exists? :model/Collection :entity_id [:in (map data-apps.tu/collection-entity-id ["first" "second"])])))))))-;; a driver that can't name the tables of a native query: `driver/native-query-table-refs` has no method for it-(driver/register! ::no-table-refs :abstract? true)--(deftest pull-survives-a-query-whose-tables-cant-be-read-test- (testing "a native query on a driver that can't name its tables doesn't fail the pull; the app records the rest"- (with-data-apps-sync- (mt/with-temp [:model/Database _ {:engine ::no-table-refs :name "no-table-refs"}]- (let [resources (data-apps.tu/build-resources- shop-collection-name shop-collection-eid- [{:entity_id question-eid :name "VenuesList" :query (venues-query)}- {:entity_id "shopQuestionNative000" :name "Native" :query (venues-query)}]- [])- path (some #(when (str/includes? % "shopQuestionNative000") %) (keys resources))- native (assoc (yaml/parse-string (get resources path))- :database_id "no-table-refs"- :dataset_query {:database "no-table-refs"- :lib/type "mbql/query"- :stages [{:lib/type "mbql.stage/native" :native "{\"find\": \"orders\"}"}]})- src (test-helpers/versioned-source- :trees {"v0" (shop-tree (assoc resources path (yaml/generate-string native)))}- :current "v0")- result (import-at! src "v0" :force? true)]- (is (= :success (:status result)) (:message result))- (is (= "v0" (remote-sync.task/last-version)))- (is (= [(mt/id :venues)] (t2/select-one-fn :table_ids :model/DataApp :name "shop"))))))))-(deftest pull-refuses-a-manifest-that-names-a-collection-it-doesnt-define-test(testing "a manifest with no resource files can't claim a collection that is already here"(with-data-apps-sync@@ -667,7 +675,7 @@(impl/handle-task-result! result task)result))-(deftest an-export-that-merges-checks-and-records-like-a-pull-test+(deftest an-export-that-merges-checks-like-a-pull-test(with-data-apps-sync(testing "a remote file that a pull refuses is refused by an export that merges it in"(mt/with-temp [:model/Collection {elsewhere :id} {:name "Elsewhere"}@@ -686,22 +694,6 @@(is (=? {:name "Someone else's" :collection_id elsewhere}(t2/select-one :model/Card :id foreign-id))))))))-(deftest an-export-that-merges-records-the-tables-test- (with-data-apps-sync- (let [remote (data-apps.tu/build-resources- shop-collection-name shop-collection-eid- [{:entity_id question-eid :name "VenuesList" :query (venues-query)}- {:entity_id "shopQuestionCheckins0" :name "Checkins"- :query {:stages [{:source {:type "table" :id (mt/id :checkins)} :limit 5}]}}]- [])- src (test-helpers/versioned-source- :trees {"v0" (shop-tree (question-resources)) "v1" (shop-tree remote)}- :current "v1")]- (is (= :success (:status (import-at! src "v0" :force? true))))- (is (= :success (:status (export-merged! src "v0"))))- (is (= (sort [(mt/id :venues) (mt/id :checkins)])- (t2/select-one-fn :table_ids :model/DataApp :name "shop"))))))-(deftest pull-loads-a-snippet-with-the-card-that-uses-it-test(testing "a resource may use a snippet that the same pull brings"(with-data-apps-sync@@ -765,23 +757,6 @@(is (=? {:database_id db-id :collection_id (shop-collection-id)}(t2/select-one :model/Card :entity_id question-eid))))))))-(deftest pull-records-the-table-a-native-query-names-test- (testing "a native query's table is matched by name whatever its case"- (with-data-apps-sync- (let [resources (question-resources)- path (some #(when (str/includes? % question-eid) %) (keys resources))- parsed (yaml/parse-string (get resources path))- native (assoc parsed :dataset_query {:database (:database (:dataset_query parsed))- :lib/type "mbql/query"- :stages [{:lib/type "mbql.stage/native"- :native "select * from Venues"}]})- src (test-helpers/versioned-source- :trees {"v0" (shop-tree (assoc resources path (yaml/generate-string native)))}- :current "v0")- result (import-at! src "v0" :force? true)]- (is (= :success (:status result)) (:message result))- (is (= [(mt/id :venues)] (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))-(deftest an-apps-collection-cant-be-moved-to-the-trash-test(testing "trashing it would drop the app's resource files on the next export"(with-data-apps-sync@@ -797,25 +772,6 @@(mt/user-http-request :crowberto :delete 204 "apps/shop")(is (not (t2/exists? :model/Collection :id collection-id))))))))))-(deftest pull-records-the-table-a-query-action-names-test- (testing "the tables an action's SQL names are recorded apart from the tables the questions read"- (with-data-apps-sync- (let [mp (mt/metadata-provider)- action-id (actions/insert! {:name "Clear checkins"- :type :query- :database_id (mt/id)- :dataset_query (lib/native-query mp "DELETE FROM checkins WHERE id = {{id}}")- :parameters [{:id "id" :slug "id" :type :number/=}]})- resources (data-apps.tu/build-resources- shop-collection-name shop-collection-eid- [{:entity_id question-eid :name "VenuesList" :query (venues-query)}]- [action-id])- src (test-helpers/versioned-source :trees {"v0" (shop-tree resources)} :current "v0")- result (import-at! src "v0" :force? true)]- (is (= :success (:status result)) (:message result))- (is (= (sort [(mt/id :venues) (mt/id :checkins)])- (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))-(deftest an-apps-own-yaml-is-left-alone-test(testing "a YAML file in the app's directory that isn't its manifest is the app's own: not loaded, not removed"(with-data-apps-sync@@ -1017,16 +973,6 @@(is (= :success (:status (export! mock))))(is (= :success (:status (import-at! mock "main" :force? true)))))))))))-(deftest a-pull-whose-table-record-fails-still-loads-test- (testing "the tables are recorded after the load's transaction, so a failure there is logged and the pull stands"- (with-data-apps-sync- (let [src (test-helpers/versioned-source :trees {"v0" (shop-tree (question-resources))} :current "v0")]- (mt/with-dynamic-fn-redefs [data-apps/record-table-dependencies! (fn [] (throw (ex-info "the tables can't be recorded" {})))]- (let [result (import-at! src "v0" :force? true)]- (is (= :success (:status result)) (:message result))))- (is (t2/exists? :model/DataApp :name "shop"))- (is (= "v0" (remote-sync.task/last-version)))))))-(deftest a-refused-pull-names-the-file-whatever-its-name-test(testing "a validator failure is reported as what it is when a path has \"branch\" in it, not as a branch error"(with-data-apps-sync@@ -1045,17 +991,6 @@(is (str/includes? (:message result) "must be a question or metric"))(is (str/includes? (:message result) card-file))))))-(deftest an-export-leaves-the-tables-an-app-reads-alone-test- (testing "only a pull changes an app's cards, so a pull records the tables they read and an export leaves the record- as it is"- (with-data-apps-sync- (let [mock (test-helpers/create-mock-source :initial-files {"main" (shop-tree (question-resources))})]- (is (= :success (:status (import-at! mock "main" :force? true))))- (mt/user-http-request :crowberto :put 200 "apps/shop" {:bundle "B2"})- (t2/update! :model/DataApp :name "shop" {:table_ids [(mt/id :checkins)]})- (is (= :success (:status (export! mock))))- (is (= [(mt/id :checkins)] (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))-(defn- generated-app-tree"The repo files of the `shop` app as `/api/apps/generate` writes them: its manifest and collection, a questionaggregating `metric-id`, the copy of that metric, the copy of `action-id`, and its bundle."diff --git a/frontend/src/metabase-types/api/data-app.ts b/frontend/src/metabase-types/api/data-app.tsindex 833921ff15f..02c5a615e94 100644--- a/frontend/src/metabase-types/api/data-app.ts+++ b/frontend/src/metabase-types/api/data-app.ts@@ -26,8 +26,6 @@ export interface DataApp {enabled: boolean;/** The collection that contains this app's saved questions and models. */resource_collection_id: number;- /** Tables used by the last successful resource synchronization. */- table_ids: number[];/*** External origins the app's sandboxed bundle may `fetch`/XHR. Empty means* none (Metabase data still flows through the SDK). Each entry is an origin,diff --git a/frontend/src/metabase-types/api/mocks/data-app.ts b/frontend/src/metabase-types/api/mocks/data-app.tsindex 61ef6177ea9..c3b80b45729 100644--- a/frontend/src/metabase-types/api/mocks/data-app.ts+++ b/frontend/src/metabase-types/api/mocks/data-app.ts@@ -11,7 +11,6 @@ export const createMockDataApp = (opts?: Partial<DataApp>): DataApp => ({bundle_path: "dist/index.js",enabled: true,resource_collection_id: 1,- table_ids: [],allowed_hosts: [],bundle_hash: "abc123",created_at: "2024-01-01T00:00:00Z",diff --git a/resources/migrations/065/20261014_drop_data_app_table_ids.yaml b/resources/migrations/065/20261014_drop_data_app_table_ids.yamlnew file mode 100644index 00000000000..cd95d702474--- /dev/null+++ b/resources/migrations/065/20261014_drop_data_app_table_ids.yaml@@ -0,0 +1,20 @@+## The tables a data app's resources read are now walked from the dependency graph when an admin opens the+## permission warnings, instead of being recorded on the app after every import.++databaseChangeLog:+ - changeSet:+ id: v65.2026-10-14T00:00:00+ author: ranquild+ comment: Drop data_app.table_ids, the dependency graph records the tables data apps read+ changes:+ - dropColumn:+ tableName: data_app+ columnName: table_ids+ rollback:+ - addColumn:+ tableName: data_app+ columns:+ - column:+ name: table_ids+ type: ${text.type}+ remarks: JSON array of table ids used by the last successful resource syncdiff --git a/resources/migrations/065/20261016_dependency_status_unresolved_tables.yaml b/resources/migrations/065/20261016_dependency_status_unresolved_tables.yamlnew file mode 100644index 00000000000..fd1db922578--- /dev/null+++ b/resources/migrations/065/20261016_dependency_status_unresolved_tables.yaml@@ -0,0 +1,23 @@+## A native query can name a table that sync hasn't found yet. The backfill flags such entities, so that the next sync+## of their database recomputes them and records the table once it exists.++databaseChangeLog:+ - changeSet:+ id: v65.2026-10-16T00:00:00+ author: ranquild+ comment: Add dependency_status.unresolved_tables, flagging entities that name tables matching nothing+ changes:+ - addColumn:+ tableName: dependency_status+ columns:+ - column:+ name: unresolved_tables+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity names tables that match no synced table, recomputed after each sync+ constraints:+ nullable: false+ rollback:+ - dropColumn:+ tableName: dependency_status+ columnName: unresolved_tablesdiff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 81e637522f6..0d56c7e9aa5 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -1567,10 +1567,19 @@{:closed true}[:transform :metabase.lib.schema.id/transform]])+(mr/def ::native-query-deps.unresolved-table-dep+ [:map+ {:closed true}+ [:unresolved-table [:map+ {:closed true}+ [:schema {:optional true} :string]+ [:table :string]]]])+(mr/def ::native-query-deps[:set [:or::native-query-deps.table-dep- ::native-query-deps.transform-dep]])+ ::native-query-deps.transform-dep+ ::native-query-deps.unresolved-table-dep]])(defmulti native-query-deps"Gets the table dependencies of a given sql string (or equivalent).@@ -1580,6 +1589,9 @@`query` is a Lib `:metabase.lib.schema/native-only-query`; you can use [[metabase.driver-api.core/raw-native-query]]to get the raw native query as needed.+ A table the query names that matches no synced table or transform target can be returned as+ `{:unresolved-table {:schema ... :table ...}}`, so dependency tracking can recompute the query once sync finds it.+The return value should match the `:metabase.driver/native-query-deps` schema."{:added "0.57.0" :arglists '([driver query])}dispatch-on-initialized-driver@@ -1593,31 +1605,6 @@:args args}))#{}))-(mr/def ::native-query-table-refs.table-ref- [:map- {:closed true}- [:schema {:optional true} [:maybe :string]]- [:table :string]])--(mr/def ::native-query-table-refs- [:set ::native-query-table-refs.table-ref])--(defmulti native-query-table-refs- "Gets the raw table references from a native query without resolving them to IDs.-- Unlike [[native-query-deps]] which looks up tables in the database to return IDs,- this method returns just the schema and table names as they appear in the query.- This is useful for dependency tracking where referenced tables may not- exist yet.-- `query` is a Lib `:metabase.lib.schema/native-only-query`; you can use- [[metabase.driver-api.core/raw-native-query]] to get the raw native query as needed.-- The return value should match the `:metabase.driver/native-query-table-refs` schema."- {:changelog-test/ignore true :added "0.58.0" :arglists '([driver query])}- dispatch-on-initialized-driver- :hierarchy #'hierarchy)-(defmulti native-result-metadata"Gets the result-metadata for a native query using static analysis (i.e., without actuallygoing to the database).diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex 047d9547a3c..b6acb3d4aed 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -320,16 +320,12 @@;;; | Dependencies |;;; +----------------------------------------------------------------------------------------------------------------+-(mu/defmethod driver/native-query-table-refs :sql :- ::driver/native-query-table-refs- [driver :- :keyword- query :- :metabase.lib.schema/native-only-query]- (into #{} (sql-tools/referenced-tables-raw driver (driver-api/raw-native-query query))))-(mu/defmethod driver/native-query-deps :sql :- ::driver/native-query-deps[driver :- :keywordquery :- :metabase.lib.schema/native-only-query](into (driver-api/native-query-table-references query)- (sql-tools/referenced-tables driver query)))+ (cond-> (sql-tools/referenced-tables driver query)+ (seq (lib/all-template-tags query)) (->> (remove :unresolved-table)))))(mu/defmethod driver/native-result-metadata :sql[driver :- :keyworddiff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.cljindex 00cde57922a..e5a1b6d5c4b 100644--- a/src/metabase/sql_tools/common.clj+++ b/src/metabase/sql_tools/common.clj@@ -61,6 +61,15 @@{:transform id}))transforms))))+(defn resolve-table-specs+ "Resolve the table `specs` parsed out of a native query with [[find-table-or-transform]], keeping each spec that+ matches nothing as `{:unresolved-table spec}`."+ [driver database tables transforms specs]+ (into #{}+ (map #(or (find-table-or-transform driver database tables transforms %)+ {:unresolved-table (cond-> % (nil? (:schema %)) (dissoc :schema))}))+ specs))+(mu/defn table-name :- [:maybe :string]"Computes a table name from a table reference"[raw-col :- [:map {:closed true}diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.cljindex 73cd8e31c5b..dc39d210414 100644--- a/src/metabase/sql_tools/core.clj+++ b/src/metabase/sql_tools/core.clj@@ -89,7 +89,8 @@(interface/returned-columns-impl parser driver native-query))))(mu/defn referenced-tables :- [:set :map]- "Return tables referenced by the `native-query`"+ "Return the tables and transforms the `native-query` references, keeping each name that matches neither as+ `{:unresolved-table spec}`."[driver :- :keywordnative-query :- :metabase.lib.schema/native-only-query](let [parser (sql-tools.settings/current-parser-backend)]diff --git a/src/metabase/sql_tools/macaw/core.clj b/src/metabase/sql_tools/macaw/core.cljindex 808c391b13d..f214777a2b6 100644--- a/src/metabase/sql_tools/macaw/core.clj+++ b/src/metabase/sql_tools/macaw/core.clj@@ -107,9 +107,7 @@table-ids (sql-tools.common/table-ids-by-name (lib/database-id query) (keep :table specs))db-tables (lib.metadata/bulk-metadata query :metadata/table table-ids)db-transforms (lib.metadata/transforms query)]- (into #{}- (keep #(sql-tools.common/find-table-or-transform driver (lib.metadata/database query) db-tables db-transforms %))- specs)))+ (sql-tools.common/resolve-table-specs driver (lib.metadata/database query) db-tables db-transforms specs)))(defmethod sql-tools/referenced-tables-impl :macaw[_parser driver query]diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.cljindex 50d519050fc..b206278567e 100644--- a/src/metabase/sql_tools/sqlglot/core.clj+++ b/src/metabase/sql_tools/sqlglot/core.clj@@ -63,9 +63,7 @@table-ids (sql-tools.common/table-ids-by-name (lib/database-id query) (keep :table specs))db-tables (lib.metadata/bulk-metadata query :metadata/table table-ids)db-transforms (lib.metadata/transforms query)]- (into #{}- (keep #(sql-tools.common/find-table-or-transform driver (lib.metadata/database query) db-tables db-transforms %))- specs))+ (sql-tools.common/resolve-table-specs driver (lib.metadata/database query) db-tables db-transforms specs))(catch Exception e;; Return empty sequence on parse error to follow the Macaw implementation behavior.(if (sql-parsing/parse-error? e)diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.cljindex b8f8768eafb..7c14282c353 100644--- a/src/metabase/transforms_base/ordering.clj+++ b/src/metabase/transforms_base/ordering.clj@@ -35,7 +35,7 @@lib.metadata/database:engine)](if (lib/native-only-query? query)- (driver/native-query-deps driver query)+ (into #{} (remove :unresolved-table) (driver/native-query-deps driver query))(into #{}(map (fn [table-id]{:table table-id}))