Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove table_ids from data apps (#84358)

Alexander Polyankin8997e86d4839
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 0ded1266a8b..4a26230229e 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3818,11 +3818,9 @@
api-scope
audit-app
collections
- driver
events
lib
lib.be
- lib.metadata
lib.schema
lib.types
metabot
@@ -3837,7 +3835,9 @@
system
users
util
+ util.graph
warehouse-schema-overlay
+ enterprise/dependencies
enterprise/serialization}
:model-imports #{:model/Action
:model/Card
@@ -3847,7 +3847,6 @@
:model/Field
:model/Measure
:model/Permissions
- :model/QueryAction
:model/Table}
:model-exports #{:model/DataApp}}
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 826a93dcc43..ed4034778c4 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -10,6 +10,13 @@ title: Driver interface changelog
with CTEs instead of nested subselects. Drivers should only do this if they satisfy all of the criteria
in the docstring of this method.
+- `driver/native-query-table-refs` has been removed. Nothing calls it anymore, so drivers that implement it should
+ delete their `defmethod`, which no longer compiles.
+
+- `driver/native-query-deps` can now return `{:unresolved-table {:schema ... :table ...}}` for a table the query names
+ that matches no synced table or transform target. Dependency tracking recomputes such a query once sync finds the
+ table. Drivers built on `:sql` get this from `metabase.sql-tools.core/referenced-tables`.
+
## Metabase 0.64.0
- Date and time functions that depended on the `start-of-week` setting now take a `time-config` map as their first
diff --git a/e2e/support/helpers/e2e-data-app-helpers.ts b/e2e/support/helpers/e2e-data-app-helpers.ts
index e8dfb286470..f8f3d52e515 100644
--- a/e2e/support/helpers/e2e-data-app-helpers.ts
+++ b/e2e/support/helpers/e2e-data-app-helpers.ts
@@ -48,7 +48,6 @@ export const fakeDataApp = (overrides: Partial<DataApp> = {}): DataApp => ({
bundle_path: "dist/index.js",
enabled: true,
resource_collection_id: 1,
- table_ids: [],
allowed_hosts: [],
bundle_hash: "e2e-bundle-hash",
created_at: "2024-01-01T00:00:00Z",
diff --git a/e2e/test-host-app/data-apps/resources.cy.spec.ts b/e2e/test-host-app/data-apps/resources.cy.spec.ts
index 8e3f9501e2f..386457be851 100644
--- a/e2e/test-host-app/data-apps/resources.cy.spec.ts
+++ b/e2e/test-host-app/data-apps/resources.cy.spec.ts
@@ -146,8 +146,6 @@ describe("Embedding SDK: data-app resources (queries)", () => {
cards: [ordersQuestion(question)],
});
H.publishDataApp(APP_ROOT(), APP_SLUG).then((app) => {
- expect(app.table_ids).to.deep.eq([ORDERS_ID]);
-
cy.request<Card>(`/api/card/${question}`).then(({ body: card }) => {
expect(card.collection_id).to.eq(app.resource_collection_id);
expect(card.name).to.eq("Orders");
diff --git a/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts b/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts
index 418ce86e8c4..8f8ab93a183 100644
--- a/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts
+++ b/e2e/test/scenarios/data-apps/group-access-warnings.cy.spec.ts
@@ -30,6 +30,7 @@ describe("scenarios > data apps > group access warnings (EMB-2416)", () => {
}),
],
});
+ H.waitForBackfillComplete();
cy.request<GroupInfo>("POST", "/api/permissions/group", {
name: GROUP_NAME,
diff --git a/e2e/test/scenarios/data-apps/sync.cy.spec.ts b/e2e/test/scenarios/data-apps/sync.cy.spec.ts
index e45c87614d2..c391d5607ca 100644
--- a/e2e/test/scenarios/data-apps/sync.cy.spec.ts
+++ b/e2e/test/scenarios/data-apps/sync.cy.spec.ts
@@ -1,10 +1,7 @@
-import { SAMPLE_DATABASE } from "e2e/support/cypress_sample_database";
import type { DataApp } from "metabase-types/api";
const { H } = cy;
-const { ORDERS_ID } = SAMPLE_DATABASE;
-
/**
* Drives a real remote-sync pull of a repo whose `data_apps/` holds two apps, each a `data_app.yaml` with its bundle
* next to it, and asserts that pulls materialize and remove them.
@@ -54,7 +51,6 @@ describe("scenarios > data apps > repo sync", () => {
cy.request("/api/card/goodAppOrdersQuestion")
.its("body.collection_id")
.should("eq", app.resource_collection_id);
- expect(app.table_ids).to.deep.eq([ORDERS_ID]);
});
// An author deletes an app by deleting its directory and its collection's
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/README.md b/enterprise/backend/src/metabase_enterprise/data_apps/README.md
index b0de8e2a916..cb8ad126499 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/README.md
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/README.md
@@ -50,7 +50,7 @@ the YAML, and ingestion reads the paths `serdes/resource-paths` returns back in.
must stay inside the entity's directory.
`enabled` is admin-owned and never leaves the instance; group assignments also stay local to the
-instance. `table_ids` are server-managed, and `bundle_hash` is recomputed from the bundle on import.
+instance, and `bundle_hash` is recomputed from the bundle on import.
A targeted export of an app brings its collection and what it holds along (`serdes/descendants`).
An import matches an app by `entity_id` and reasserts the app's resources; a manifest whose slug an app
@@ -146,7 +146,7 @@ The Data Apps feature is required for all app API endpoints, including group lis
| Namespace | Responsibility |
| -------------------------------------- | ----------------------------------------------------------------------------------------------- |
| `apps.clj` | Creating apps; the connected repository's URL. |
-| `core.clj` | Public access checks, resource file problems, and table dependencies. |
+| `core.clj` | Public access checks and resource file problems. |
| `config.clj` | The serialized layout and data app contract version constants. |
| `schema.clj` | Column schemas, with the normalization and validation every write goes through. |
| `api.clj` | The `/api/apps` endpoints, bundle serving, ETag handling. |
@@ -159,9 +159,8 @@ The Data Apps feature is required for all app API endpoints, including group lis
| `generate/schemas/` | The schema's stages: data access (`source`), entities (`table`, `metric`, `action`, `common`), rendering (`render`, `javascript`). |
| `query_definition.clj` | The closed schema of a `defineQuery` definition the serialization accepts. |
| `resource_validation.clj` | What the files of an app's collection may hold, checked on the whole snapshot before an import. |
-| `resource_tables.clj` | The tables an app's resources read, recorded on the app after an import. |
| `access.clj` | App access through assigned groups. |
-| `group_access.clj` | Assignment management and collection grant reconciliation. |
+| `group_access.clj` | Assignment management, collection grant reconciliation, and the permission warnings, which walk the dependency graph (`:dependencies` feature) from the app's cards and actions to the tables they read. |
| `models/data_app_group_assignment.clj` | App-to-group assignments. |
| `db.clj` | The module's application-database queries. |
| `csp.clj` | `allowed_hosts` lookup for the core CSP middleware. |
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj
index 11a786581d1..1735cce9de4 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/api.clj
@@ -72,7 +72,6 @@
[:enabled :boolean]
[:allowed_hosts [:sequential :string]]
[:resource_collection_id ms/PositiveInt]
- [:table_ids [:sequential ms/PositiveInt]]
[:bundle_hash [:maybe :string]]
[:created_at :any]
[:updated_at :any]])
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/core.clj b/enterprise/backend/src/metabase_enterprise/data_apps/core.clj
index 7403401fd0f..87fab58d44a 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/core.clj
@@ -1,9 +1,7 @@
(ns metabase-enterprise.data-apps.core
- "What other modules may ask the data-apps module: what the files of an app's collection may hold, and the tables
- those read."
+ "What other modules may ask the data-apps module: what the files of an app's collection may hold."
(:require
[metabase-enterprise.data-apps.db :as data-apps.db]
- [metabase-enterprise.data-apps.resource-tables :as resource-tables]
[metabase-enterprise.data-apps.resource-validation :as resource-validation]
[metabase.api.common :as api]
[metabase.models.interface :as mi]
@@ -11,8 +9,6 @@
[potemkin :as p]))
(p/import-vars
- [resource-tables
- record-table-dependencies!]
[resource-validation
problems
warnings])
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
index 7ec8c32c162..5c96e66a020 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/db.clj
@@ -5,7 +5,6 @@
[metabase-enterprise.data-apps.schema :as data-apps.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.queries.core :as queries]
- [metabase.util :as u]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
@@ -14,7 +13,7 @@
(def non-blob-columns
"Every DataApp column except the raw bundle blob."
[:id :entity_id :name :display_name :description :version :bundle_path :enabled :allowed_hosts
- :resource_collection_id :table_ids :bundle_hash :created_at :updated_at])
+ :resource_collection_id :bundle_hash :created_at :updated_at])
(def ^:private non-blob-model
(into [:model/DataApp] non-blob-columns))
@@ -113,11 +112,6 @@
[entity-id]
(t2/select-one [:model/DataApp :id :entity_id :resource_collection_id] :entity_id entity-id))
-(defn data-apps-with-resource-collections
- "The `:id` and `:resource_collection_id` of every DataApp that has a resource collection."
- []
- (t2/select [:model/DataApp :id :resource_collection_id] :resource_collection_id [:not= nil]))
-
(defn active-table?
"Whether the table with `table-id` is active."
[table-id]
@@ -151,32 +145,15 @@
(t2/select [:model/Action :entity_id :collection_id] :entity_id [:in entity-ids])
[]))
-(defn table-ids-named
- "The IDs of the active tables in the database with `database-id` that `refs` (maps of `:schema` and `:table`)
- name: by name, and by schema when the reference has one, ignoring case."
- [database-id refs]
- (if (seq refs)
- (let [lower #(some-> % u/lower-case-en)
- tables (t2/select [:model/Table :id :schema :name] :db_id database-id :active true
- {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]
- :where [:in [:lower :name] (into [] (keep (comp lower :table)) refs)]})]
- (into (sorted-set)
- (for [{:keys [schema table]} refs
- {:keys [id] :as named} tables
- :when (and (= (lower table) (lower (:name named)))
- (or (nil? schema) (= (lower schema) (lower (:schema named)))))]
- id)))
- #{}))
+(defn collection-card-ids
+ "The IDs of the cards in the collection with `collection-id`."
+ [collection-id]
+ (t2/select-pks-vec :model/Card :collection_id collection-id))
-(defn collection-dataset-queries
- "The queries of the cards and of the query actions in the collection with `collection-id`."
+(defn collection-action-ids
+ "The IDs of the actions in the collection with `collection-id`."
[collection-id]
- (into (t2/select-fn-vec :dataset_query :model/Card :collection_id collection-id)
- (t2/select-fn-vec :dataset_query :model/QueryAction
- {:select [:qa.*]
- :from [[:query_action :qa]]
- :join [[:action :a] [:= :a.id :qa.action_id]]
- :where [:= :a.collection_id collection-id]})))
+ (t2/select-pks-vec :model/Action :collection_id collection-id))
(defn resource-collection
"The resource collection with `collection-id`, or nil."
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj b/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj
index ed1ac22d51e..458f58d87fb 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/group_access.clj
@@ -2,7 +2,9 @@
(:require
[metabase-enterprise.data-apps.db :as data-apps.db]
[metabase-enterprise.data-apps.resources :as resources]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
[metabase.api.common :as api]
+ [metabase.graph.core :as graph]
[metabase.permissions.core :as perms]
[metabase.settings.core :as setting]
[metabase.util.i18n :refer [tru]]
@@ -68,10 +70,28 @@
(sort-by (juxt :database_name :schema :name))
vec))
+(defn- resource-table-ids
+ "The IDs of the tables the cards and actions in the resource collection with `collection-id` read, directly or
+ through the cards, measures, segments and snippets they use, with a transform they name standing for its output."
+ [collection-id]
+ (let [nodes (models.dependency/entities->nodes {:card (data-apps.db/collection-card-ids collection-id)
+ :action (data-apps.db/collection-action-ids collection-id)})
+ graph (models.dependency/filtered-graph-dependencies nil {:types #{:action :card :measure :segment :snippet}})
+ {:keys [table transform]} (models.dependency/group-nodes (graph/transitive graph (vec nodes)))
+ outputs (->> (models.dependency/entities->nodes {:transform transform})
+ models.dependency/direct-dependents
+ vals
+ (apply concat)
+ models.dependency/group-nodes
+ :table)]
+ (vec (into (set table) outputs))))
+
(defn permission-warnings
"Missing table access for assigned groups, including access inherited from All Users."
- [{app-id :id table-ids :table_ids}]
- (let [assigned-ids (mapv :permission_group_id (data-apps.db/app-assignments [app-id]))]
+ [{app-id :id collection-id :resource_collection_id}]
+ (let [assigned-ids (mapv :permission_group_id (data-apps.db/app-assignments [app-id]))
+ table-ids (when (and (seq assigned-ids) collection-id)
+ (resource-table-ids collection-id))]
(if (and (seq assigned-ids) (seq table-ids))
(let [tables (table-details table-ids)
all-users-id (:id (perms/all-users-group))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
index e9d8dbb20b1..518dc8f318e 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/models/data_app.clj
@@ -40,8 +40,7 @@
(t2/deftransforms :model/DataApp
{:bundle transform-bundle
;; JSON array of origins the sandboxed bundle may fetch/XHR (see config.clj).
- :allowed_hosts mi/transform-json
- :table_ids mi/transform-json})
+ :allowed_hosts mi/transform-json})
(doto :model/DataApp
(derive :metabase/model)
@@ -100,8 +99,7 @@
(t2/define-after-select :model/DataApp
[app]
(cond-> app
- (contains? app :allowed_hosts) (update :allowed_hosts #(or % []))
- (contains? app :table_ids) (update :table_ids #(or % []))))
+ (contains? app :allowed_hosts) (update :allowed_hosts #(or % []))))
(defmethod mi/can-read? :model/DataApp
([app] (mi/can-read? :model/DataApp (:id app)))
@@ -157,9 +155,7 @@
:skip [;; admin-owned state of this instance
:enabled
;; set by the import itself
- :bundle_hash
- ;; server-managed resources, recreated on import
- :table_ids]
+ :bundle_hash]
:transform {:created_at (serdes/date)
;; the app's resource collection, a collection in the `data-apps` namespace that loads before the app
:resource_collection_id (assoc (serdes/fk :model/Collection) :as :collection)
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/resource_tables.clj b/enterprise/backend/src/metabase_enterprise/data_apps/resource_tables.clj
deleted file mode 100644
index aceaf93a2b7..00000000000
--- a/enterprise/backend/src/metabase_enterprise/data_apps/resource_tables.clj
+++ /dev/null
@@ -1,77 +0,0 @@
-(ns metabase-enterprise.data-apps.resource-tables
- "The tables a data app's resources read, recorded on the app for the permission warnings an admin sees when
- granting access: a viewer who can't read one of them gets an app that fails for them."
- (:require
- [clojure.string :as str]
- [metabase-enterprise.data-apps.db :as data-apps.db]
- [metabase.driver :as driver]
- [metabase.driver.util :as driver.u]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.util.log :as log]))
-
-(set! *warn-on-reflection* true)
-
-(defn- parseable-sql
- "`sql` as a parser reads it: optional clauses kept as the SQL they hold, a card tag as a plain name, a snippet tag
- as its snippet's SQL with the snippets inside it, looked up through `query`'s metadata provider, and any other
- template tag as it is."
- ([query sql]
- (parseable-sql query sql #{}))
- ([query sql expanded]
- (letfn [(render [token]
- (cond
- (string? token) token
- ;; a parser reads neither `[[` nor `]]`, and a table named inside a clause is read like one outside
- (= :metabase.lib.parse/optional (:type token)) (apply str (map render (:contents token)))
- ;; a card tag stands where a table does
- (= \# (first (:name token))) "mb_card"
- (str/starts-with? (:name token) "snippet:")
- (let [snippet-name (str/trim (subs (:name token) (count "snippet:")))]
- ;; a snippet that includes itself is read once
- (if (contains? expanded snippet-name)
- ""
- (parseable-sql query
- (or (:content (lib.metadata/native-query-snippet-by-name query snippet-name)) "")
- (conj expanded snippet-name))))
- :else (str "{{" (:name token) "}}")))]
- (apply str (map render (lib/parse {} sql))))))
-
-(defn- native-table-ids
- "The tables a native query names in its SQL, as its driver's parser reads them, and in its table template tags."
- [{database-id :database, :as query}]
- (let [parseable (lib/with-native-query query (parseable-sql query (lib/raw-native-query query)))]
- (into (data-apps.db/table-ids-named database-id
- (driver/native-query-table-refs (driver.u/database->driver database-id) parseable))
- (map :table)
- (lib/native-query-table-references query))))
-
-(defn- query-table-ids [dataset-query]
- (let [query (lib/query (lib-be/application-database-metadata-provider (:database dataset-query)) dataset-query)]
- (if (lib/native? query)
- (native-table-ids query)
- (into (set (lib/all-source-table-ids query))
- (lib/all-implicitly-joined-table-ids query)))))
-
-(defn collection-table-ids
- "The tables the cards and the query actions in the collection with `collection-id` read, including
- the ones a query reaches only through an implicit join."
- [collection-id]
- (->> (data-apps.db/collection-dataset-queries collection-id)
- (into #{} (mapcat (fn [dataset-query]
- ;; a query whose tables can't be read, such as a native query on a driver that can't
- ;; parse one, must not fail the pull that loaded it
- (try
- (query-table-ids dataset-query)
- (catch Exception e
- (log/warn e "Could not read the tables of a data app query")
- nil)))))
- sort
- vec))
-
-(defn record-table-dependencies!
- "Record on every data app the tables its resources read. Runs after an import, since that is what changes them."
- []
- (doseq [{:keys [id resource_collection_id]} (data-apps.db/data-apps-with-resource-collections)]
- (data-apps.db/update-data-app! id {:table_ids (collection-table-ids resource_collection_id)})))
diff --git a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj
index fd1cc6ef934..e0a0d5b2643 100644
--- a/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/data_apps/schema.clj
@@ -113,8 +113,7 @@
[:updated_at {:optional true} [:maybe ms/TemporalInstant]]
[:description {:optional true} ::description]
[:version {:optional true} ::version]
- [:resource_collection_id {:optional true} [:maybe ms/PositiveInt]]
- [:table_ids {:optional true} [:maybe [:sequential ms/PositiveInt]]]])
+ [:resource_collection_id {:optional true} [:maybe ms/PositiveInt]]])
(mr/def ::data-app.insert
"What an insert of a DataApp accepts: [[::data-app.update]] with its slug, display name, and bundle path required."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index 4ef9dd2b7bb..ee424bdee36 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -737,6 +737,30 @@
;;; ------------------------------------------------ Dependency status ------------------------------------------------
+(def ^:private entity-database-queries
+ "For each entity type that can hold a native query, a query of the IDs of its entities reading from `db-id`."
+ {:card (fn [db-id] ^:allow-subquery {:select [:id]
+ :from [(t2/table-name :model/Card)]
+ :where [:= :database_id db-id]})
+ :transform (fn [db-id] ^:allow-subquery {:select [:id]
+ :from [(t2/table-name :model/Transform)]
+ :where [:= :source_database_id db-id]})
+ :action (fn [db-id] ^:allow-subquery {:select [:action_id]
+ :from [(t2/table-name :model/QueryAction)]
+ :where [:= :database_id db-id]})})
+
+(mu/defn unresolved-table-entity-ids :- [:map-of ::deps.dependency-types/dependency-types [:set ms/PositiveInt]]
+ "The IDs, by entity type, of the entities reading from the Database with `db-id` that name tables matching nothing."
+ [db-id :- ::lib.schema.id/database]
+ (into {}
+ (keep (fn [[entity-type entity-ids-query]]
+ (when-let [ids (not-empty (t2/select-fn-set :entity_id :model/DependencyStatus
+ :entity_type entity-type
+ :unresolved_tables true
+ :entity_id [:in (entity-ids-query db-id)]))]
+ [entity-type ids])))
+ entity-database-queries))
+
(mu/defn dependency-status
"The DependencyStatus of the entity `entity-type` `entity-id`, or nil."
[entity-type :- EntityType
@@ -778,10 +802,11 @@
(mu/defn upsert-dependency-status!
"Upsert the DependencyStatus of `entity-type` `entity-id`, setting stale=false, `dependency_analysis_version` to
- `current-version`, and clearing any failure state."
- [entity-type :- EntityType
- entity-id :- ms/PositiveInt
- current-version :- ms/PositiveInt]
+ `current-version`, `unresolved_tables` to `unresolved-tables?`, and clearing any failure state."
+ [entity-type :- EntityType
+ entity-id :- ms/PositiveInt
+ current-version :- ms/PositiveInt
+ unresolved-tables? :- :boolean]
(mdb/update-or-insert!
:model/DependencyStatus
{:entity_type entity-type :entity_id entity-id}
@@ -789,7 +814,8 @@
{:dependency_analysis_version current-version
:stale false
:fail_count 0
- :next_retry_at nil})))
+ :next_retry_at nil
+ :unresolved_tables unresolved-tables?})))
(mu/defn record-dependency-status-failure!
"Upsert the DependencyStatus of `entity-type` `entity-id`, creating it if needed. `update-fn` receives the
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
index 1edde6ea56c..9b042052562 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
@@ -219,9 +219,13 @@
(before-delete* model instance))
(defenterprise after-database-sync
- "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed."
+ "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed, and the
+ entities reading from it that name tables that matched nothing, which sync may have found now."
:feature :dependencies
[database-id]
+ (doseq [[entity-type entity-ids] (dependencies.db/unresolved-table-entity-ids database-id)
+ entity-id entity-ids]
+ (mark-changed! entity-type entity-id))
(when (deps.findings/mark-transitive-dependents-stale!
{:table (dependencies.db/table-ids-with-outdated-findings database-id)})
(trigger-jobs!)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 772c7036612..5812611cb8f 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -23,8 +23,18 @@
(def current-dependency-analysis-version
"Current version of the dependency analysis logic.
- This should be incremented when the dependency analysis logic changes."
- 6)
+ This should be incremented when the dependency analysis logic changes.
+
+ Version history:
+ - 1: Initial version
+ - 2: Added implicitly joined tables
+ - 3: Added segments
+ - 4: Tracked unreferenced and broken entities
+ - 5: Added the source tables of Python transforms
+ - 6: Enabled native entity validation
+ - 7: Native queries depend on the cards they reference but no longer on those cards' tables, and record whether
+ they name tables that match nothing"
+ 7)
(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
index 28e3dffc71a..362646d79eb 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
@@ -24,11 +24,11 @@
(dependencies.db/mark-dependency-status-stale! entity-type id)))
(defn upsert-status!
- "Upsert a dependency_status entry, setting stale=false, version to current,
- and clearing any failure state."
- [entity-type entity-id]
+ "Upsert a dependency_status entry, setting stale=false, version to current, whether the entity names tables that
+ match nothing, and clearing any failure state."
+ [entity-type entity-id unresolved-tables?]
(dependencies.db/upsert-dependency-status!
- entity-type entity-id models.dependency/current-dependency-analysis-version))
+ entity-type entity-id models.dependency/current-dependency-analysis-version unresolved-tables?))
(defmulti hydrate-for-deps
"Hydrate a batch of instances with data needed for dependency calculation.
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
index f65b4c025df..5496a60a5e0 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/native_validation.clj
@@ -56,15 +56,16 @@
"Compile a query replacing card and table references with placeholder table names.
Card refs like {{#1}} are replaced with mb__validat_card__1.
Table refs like {{table_one}} are replaced with mb__validat_table__N.
+ Only the refs in the query's own SQL are replaced; those inside its snippets expand with them.
Returns the compiled native-only-query, or nil if the original SQL contains
the placeholder prefix (collision guard)."
[query]
(let [stage (first (:stages query))
sql (:native stage)
- ttags (:template-tags stage)
- card-tags (filter #(= (:type %) :card) ttags)
- table-tags (filter #(= (:type %) :table) ttags)]
+ own-tags (filter #(str/includes? sql (str "{{" (:name %) "}}")) (:template-tags stage))
+ card-tags (filter #(= (:type %) :card) own-tags)
+ table-tags (filter #(= (:type %) :table) own-tags)]
(when-not (or (str/includes? sql card-placeholder-prefix)
(str/includes? sql table-placeholder-prefix))
(let [;; Replace card tags
@@ -86,7 +87,7 @@
table-tags)
remaining-tags
- (into [] (remove #(#{:card :table} (:type %))) ttags)
+ (into [] (remove (set (concat card-tags table-tags))) (:template-tags stage))
modified-query
(-> query
@@ -307,17 +308,26 @@
(compile-query query))]
(driver/native-result-metadata driver compiled))))
+(defn- placeholder-dep?
+ "Whether `dep` is a card or table placeholder that [[compile-toplevel-query]] put in place of a template tag."
+ [dep]
+ (when-let [table-name (-> dep :unresolved-table :table)]
+ (or (parse-card-placeholder table-name)
+ (table-placeholder? table-name))))
+
(mu/defn native-query-deps :- [:set
[:or
::driver/native-query-deps.table-dep
::driver/native-query-deps.transform-dep
+ ::driver/native-query-deps.unresolved-table-dep
[:map {:closed true} [:snippet ::lib.schema.id/snippet]]
[:map {:closed true} [:card ::lib.schema.id/card]]]]
- "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs."
+ "Returns the upstream dependencies of a native query, as a set of `{:kind id}` pairs, with a card it references
+ standing in as a placeholder table so that the card's own tables stay out of them."
[driver :- :keyword
query :- ::lib.schema/native-only-query]
- (let [compiled (compile-query query)]
- (into (driver/native-query-deps driver compiled)
+ (let [compiled (or (compile-toplevel-query query) (compile-query query))]
+ (into (into #{} (remove placeholder-dep?) (driver/native-query-deps driver compiled))
;; TODO (Cam 10/1/25) -- Even this much MBQL manipulation outside of Lib is illegal. Move this sort of stuff
;; into Lib.
(keep #(case (:type %)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj
index 219f8caaad5..fe54439a1d9 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/schema.clj
@@ -14,7 +14,10 @@
[:dashboard {:optional true} [:set ::lib.schema.id/dashboard]]
[:document {:optional true} [:set ::documents.schema/document.id]]
[:sandbox {:optional true} [:set ::lib.schema.id/sandbox]]
- [:segment {:optional true} [:set ::lib.schema.id/segment]]])
+ [:segment {:optional true} [:set ::lib.schema.id/segment]]
+ [:unresolved-table {:optional true} [:set [:map
+ [:schema {:optional true} :string]
+ [:table :string]]]]])
(mr/def ::analysis-finding
"A AnalysisFinding as selected from the app DB: every column of `:analysis_finding`."
@@ -81,4 +84,5 @@
[:stale {:optional true} [:maybe :boolean]]
[:fail_count {:optional true} [:maybe :int]]
[:next_retry_at {:optional true} [:maybe ms/TemporalInstant]]
- [:terminal {:optional true} [:maybe :boolean]]])
+ [:terminal {:optional true} [:maybe :boolean]]
+ [:unresolved_tables {:optional true} [:maybe :boolean]]])
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
index 3e50587b6eb..cda348a3988 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
@@ -21,6 +21,7 @@
[metabase-enterprise.dependencies.settings :as deps.settings]
[metabase-enterprise.dependencies.task-util :as deps.task-util]
[metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
+ [metabase.graph.core :as graph]
[metabase.lib-be.core :as lib-be]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
@@ -31,12 +32,18 @@
(set! *warn-on-reflection* true)
+(def ^:private expanded-into-dependents
+ "The entity types whose content a native query using them expands into its own SQL, so that a change to one changes
+ the table dependencies of the entities using it, which no graph edge carries."
+ #{:snippet})
+
(def ^:private entity-types
- "The list of entity types to backfill.
+ "The list of entity types to backfill, those expanded into their dependents first, so that the dependents they mark
+ are recomputed in the same run.
This is not the same as deps.dependency-types/dependency-types, because tables shouldn't be backfilled. Instead, links
involving tables are found via analysis of the other side of the relation."
- (vec deps.dependency-types/backfillable-dependency-types))
+ (vec (sort-by (complement expanded-into-dependents) deps.dependency-types/backfillable-dependency-types)))
(def ^:private max-retries 5)
@@ -75,19 +82,29 @@
entity-type
(deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))
+(defn- mark-expanding-entities-stale!
+ "Mark for recalculation the entities whose native SQL expands the `entity-type` entity `entity-id`, directly or
+ through the entities of the same kind that include it."
+ [entity-type entity-id]
+ (let [graph (models.dependency/filtered-graph-dependents nil {:types expanded-into-dependents})
+ type->ids (models.dependency/group-nodes (graph/transitive graph [[entity-type entity-id]]))]
+ (deps.dependency-status/mark-stale! (apply dissoc type->ids expanded-into-dependents))))
+
(defn- compute-deps-for-entity!
- "Mark the dependents of an entity that a write marked stale for re-analysis, then compute and store its
- dependencies and update its dependency_status. Entities are expected to be pre-hydrated by
- [[deps.dependency-status/hydrate-for-deps]]."
+ "Mark the dependents of an entity that a write marked stale for re-analysis, and those that expand it for
+ recalculation, then compute and store its dependencies and update its dependency_status, flagging the tables it
+ names that match nothing. Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."
[entity-type entity]
(log/debug "Computing dependencies for" (name entity-type) (:id entity))
(when (:stale (dependencies.db/dependency-status entity-type (:id entity)))
- (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}}))
+ (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}})
+ (when (expanded-into-dependents entity-type)
+ (mark-expanding-entities-stale! entity-type (:id entity))))
(t2/with-transaction [_]
(let [deps (deps.calculation/calculate-deps entity-type entity)]
- (models.dependency/replace-dependencies! entity-type (:id entity) deps)
+ (models.dependency/replace-dependencies! entity-type (:id entity) (dissoc deps :unresolved-table))
(post-deps-cleanup! entity-type entity)
- (deps.dependency-status/upsert-status! entity-type (:id entity)))))
+ (deps.dependency-status/upsert-status! entity-type (:id entity) (contains? deps :unresolved-table)))))
(defn- remove-deleted-entity-batch!
"Mark the dependents of up to `batch-size` deleted entities of `entity-type` for re-analysis and drop their
diff --git a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
index cfeedb6e517..3087fedb117 100644
--- a/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/remote_sync/impl.clj
@@ -3,7 +3,6 @@
[clojure.string :as str]
[diehard.core :as dh]
[java-time.api :as t]
- [metabase-enterprise.data-apps.core :as data-apps]
[metabase-enterprise.remote-sync.db :as remote-sync.db]
[metabase-enterprise.remote-sync.guards :as guards]
[metabase-enterprise.remote-sync.merge :as remote-sync.merge]
@@ -579,15 +578,6 @@
(remote-sync.db/update-rso! (:id existing) {:status status :status_changed_at timestamp})
(remote-sync.db/insert-rso! (-> row (dissoc :id) (assoc :status_changed_at timestamp))))))
-(defn- record-data-app-tables!
- "Record the tables each data app's resources read, after a load, logging a failure: the load's transaction has
- committed by then, so a failure must not undo its record."
- []
- (try
- (data-apps/record-table-dependencies!)
- (catch Exception e
- (log/warn e "Could not record the tables of the data apps"))))
-
(defn- import-merged!
"Import in merge mode. Should only be called when you have a base-snapshot and its version differs from snaphot's version.
@@ -618,7 +608,6 @@
:finalize! (fn []
(restore-dirty-objects! dirty-objects sync-timestamp)
(finalize!)))
- (record-data-app-tables!)
(log/infof "Pull merge: folded in %d remote change(s) (added %d, updated %d, removed %d); kept %d local change(s)"
(apply + (vals summary)) (:added summary) (:updated summary) (:removed summary)
(count dirty-objects))
@@ -718,8 +707,7 @@
:else
(let [_ (log/info "Remote sync full import: forced")
- imported-data (u/prog1 (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)
- (record-data-app-tables!))]
+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)]
(log/info "Successfully reloaded entities from git repository")
{:status :success
:version snapshot-version
@@ -757,8 +745,7 @@
first-import? "first import"
:else "changes not incrementally loadable")
_ (log/infof "Remote sync full import: %s" reason)
- imported-data (u/prog1 (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)
- (record-data-app-tables!))]
+ imported-data (load-snapshot! snapshot report sync-timestamp :finalize! finalize!)]
(log/info "Successfully reloaded entities from git repository")
{:status :success
:version snapshot-version
@@ -856,7 +843,6 @@
:finalize! (fn []
(remote-sync.db/mark-all-rsos-synced! sync-timestamp)
(remote-sync.task/set-version! task-id version)))
- (record-data-app-tables!)
(log/infof "Exported with merge: folded in %d remote change(s) (added %d, updated %d, removed %d); pushed %d"
pulled (:added summary) (:updated summary) (:removed summary) (if empty? 0 pushed-count))
{:status :success :version version :merge-summary summary
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj
index c788bd6d305..ef97f08d808 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/group_access_warnings_test.clj
@@ -2,6 +2,10 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.data-apps.group-access :as group-access]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
+ [metabase.dependencies.hooks :as dependencies.hooks]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.permissions-rest.data-permissions.graph :as data-perms.graph]
[metabase.permissions.core :as perms]
[metabase.test :as mt]
@@ -9,118 +13,212 @@
(set! *warn-on-reflection* true)
+(defn- do-with-app-reading!
+ "Call `f` with a data app whose resource collection holds a card that reads `table-ids`."
+ [table-ids f]
+ (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
+ :model/Card card {:collection_id (:resource_collection_id app)}]
+ (mt/with-model-cleanup [:model/Dependency]
+ (when (seq table-ids)
+ (t2/insert! :model/Dependency
+ (for [table-id table-ids]
+ {:from_entity_type :card :from_entity_id (:id card)
+ :to_entity_type :table :to_entity_id table-id})))
+ (f app))))
+
+(defmacro ^{:private true :clj-kondo/lint-as 'clojure.core/let} with-app-reading!
+ "Bind `app-sym` to a data app whose resource collection reads `table-ids`, and run `body`."
+ [[app-sym table-ids] & body]
+ `(do-with-app-reading! ~table-ids (fn [~app-sym] ~@body)))
+
+(deftest group-permission-warnings-walk-the-dependency-graph-test
+ (testing "the warnings list the tables a native card, a card on another card, and a query action read"
+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}
+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
+ :model/Card {base-id :id} {:collection_id collection-id
+ :dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
+ :model/Card _ {:collection_id collection-id
+ :dataset_query (lib/native-query mp "SELECT * FROM VENUES")}
+ :model/PermissionsGroup group {}]
+ (mt/with-temp [:model/Card _ {:collection_id collection-id
+ :dataset_query (lib/query mp (lib.metadata/card mp base-id))}
+ :model/Action {action-id :id} {:type :query :name "Delete" :collection_id collection-id}
+ :model/QueryAction _ {:action_id action-id
+ :database_id (mt/id)
+ :dataset_query (lib/native-query mp "DELETE FROM CHECKINS WHERE ID = 1")}]
+ (deps.test/synchronously-run-backfill!)
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (is (=? [{:group_id (:id group)
+ :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :orders)} {:id (mt/id :venues)}]}]
+ (group-access/permission-warnings app)))))))))))
+
+(deftest group-permission-warnings-include-tables-in-optional-clauses-test
+ (testing "the tables an optional clause names count, whether or not its parameter is set"
+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
+ (doseq [sql ["SELECT * FROM VENUES [[WHERE ID IN (SELECT VENUE_ID FROM CHECKINS WHERE ID = {{id}})]]"
+ "SELECT * FROM VENUES v [[JOIN CHECKINS c ON c.VENUE_ID = v.ID AND c.ID = {{id}}]]"]]
+ (testing sql
+ (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}
+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
+ :model/Card _ {:collection_id collection-id
+ :dataset_query (lib/native-query (mt/metadata-provider) sql)}
+ :model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (deps.test/synchronously-run-backfill!)
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}]
+ (group-access/permission-warnings app)))))))))))
+
+(deftest group-permission-warnings-follow-snippet-edit-test
+ (testing "after a snippet edit and the jobs, the warnings list the tables of the snippet's new SQL"
+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]
+ (mt/with-temp [:model/NativeQuerySnippet snippet {:name "birds source" :content "VENUES"}
+ :model/DataApp {collection-id :resource_collection_id :as app}
+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
+ :model/Card _ {:collection_id collection-id
+ :dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM {{snippet: birds source}}")}
+ :model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (deps.test/synchronously-run-backfill!)
+ (t2/update! :model/NativeQuerySnippet (:id snippet) {:content "CHECKINS"})
+ (deps.test/synchronously-run-backfill!)
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)}]}]
+ (group-access/permission-warnings app)))))))))
+
+(deftest group-permission-warnings-include-table-synced-after-analysis-test
+ (testing "a table that sync finds after the app's card was analyzed enters the warnings once the jobs run"
+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus :model/AnalysisFinding]
+ (mt/with-temp [:model/DataApp {collection-id :resource_collection_id :as app}
+ {:name "birds" :display_name "Birds" :bundle_path "birds.js"}
+ :model/Card _ {:collection_id collection-id
+ :dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM PUBLIC.BIRDS_LATE")}
+ :model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (deps.test/synchronously-run-backfill!)
+ (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]
+ (dependencies.hooks/after-database-sync (mt/id))
+ (deps.test/synchronously-run-backfill!)
+ (is (=? [{:group_id (:id group) :missing_tables [{:id late-id}]}]
+ (group-access/permission-warnings app))))))))))
+
(deftest group-permission-warnings-exclude-inactive-tables-test
(mt/with-premium-features #{:data-apps :advanced-permissions}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders) (mt/id :categories)]}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (group-access/add-groups! app [(:id group)])
- (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}
- (mt/with-temp-vals-in-db :model/Table (mt/id :orders) {:visibility_type :hidden}
- ;; exclude inactive tables, but keep active visible and hidden tables.
- ;; permissions editor also follow this filtering.
- (is (=? [{:group_id (:id group)
- :missing_tables [{:id (mt/id :categories)} {:id (mt/id :orders)}]}]
- (group-access/permission-warnings app)))))))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders) (mt/id :categories)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}
+ (mt/with-temp-vals-in-db :model/Table (mt/id :orders) {:visibility_type :hidden}
+ ;; exclude inactive tables, but keep active visible and hidden tables.
+ ;; permissions editor also follow this filtering.
+ (is (=? [{:group_id (:id group)
+ :missing_tables [{:id (mt/id :categories)} {:id (mt/id :orders)}]}]
+ (group-access/permission-warnings app))))))))))
(deftest group-permission-warnings-ignore-apps-with-only-inactive-tables-test
(mt/with-premium-features #{:data-apps :advanced-permissions}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues)]}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (group-access/add-groups! app [(:id group)])
+ (with-app-reading! [app [(mt/id :venues)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
; if data apps only references inactive tables, there will be no access warnings
- (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}
- (is (= [] (group-access/permission-warnings app))))))))
+ (mt/with-temp-vals-in-db :model/Table (mt/id :venues) {:active false}
+ (is (= [] (group-access/permission-warnings app)))))))))
(deftest group-permission-warnings-ignore-unassigned-groups-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp _app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [_app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
(deftest group-permission-warnings-include-inherited-access-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (group-access/add-groups! app [(:id group)])
- (perms/set-table-permission! (perms/all-users-group) (mt/id :venues) :perms/view-data :unrestricted)
- (is (=? [{:group_id (:id group)
- :missing_tables [{:id (mt/id :orders) :name "Orders" :database_id (mt/id)}]}]
- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (perms/set-table-permission! (perms/all-users-group) (mt/id :venues) :perms/view-data :unrestricted)
+ (is (=? [{:group_id (:id group)
+ :missing_tables [{:id (mt/id :orders) :name "Orders" :database_id (mt/id)}]}]
+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
(deftest group-permission-warnings-preserve-permissions-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (group-access/add-groups! app [(:id group)])
- (let [before (t2/select :model/DataPermissions)]
- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")
- (is (= before (t2/select :model/DataPermissions))))))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (let [before (t2/select :model/DataPermissions)]
+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")
+ (is (= before (t2/select :model/DataPermissions)))))))))
(deftest group-permission-warnings-include-direct-access-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup group {}]
- (group-access/add-groups! app [(:id group)])
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (perms/set-table-permission! (:id group) (mt/id :venues) :perms/view-data :unrestricted)
- (perms/set-table-permission! (:id group) (mt/id :orders) :perms/view-data :unrestricted)
- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (group-access/add-groups! app [(:id group)])
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (perms/set-table-permission! (:id group) (mt/id :venues) :perms/view-data :unrestricted)
+ (perms/set-table-permission! (:id group) (mt/id :orders) :perms/view-data :unrestricted)
+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
(deftest group-permission-warnings-include-sandbox-access-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup group {}]
- (group-access/add-groups! app [(:id group)])
- (mt/user-http-request :crowberto :put 200 "permissions/graph"
- (-> (data-perms.graph/api-graph)
- (assoc-in [:groups (:id group) (mt/id) :view-data]
- {"PUBLIC" {(mt/id :venues) :unrestricted
- (mt/id :orders) :sandboxed}})
- (assoc :sandboxes [{:group_id (:id group) :table_id (mt/id :orders)}])))
- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))
- (mt/user-http-request :crowberto :put 200 "permissions/graph"
- (assoc-in (data-perms.graph/api-graph)
- [:groups (:id group) (mt/id) :view-data "PUBLIC" (mt/id :orders)]
- :blocked))
- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :orders)}]}]
- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (group-access/add-groups! app [(:id group)])
+ (mt/user-http-request :crowberto :put 200 "permissions/graph"
+ (-> (data-perms.graph/api-graph)
+ (assoc-in [:groups (:id group) (mt/id) :view-data]
+ {"PUBLIC" {(mt/id :venues) :unrestricted
+ (mt/id :orders) :sandboxed}})
+ (assoc :sandboxes [{:group_id (:id group) :table_id (mt/id :orders)}])))
+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))
+ (mt/user-http-request :crowberto :put 200 "permissions/graph"
+ (assoc-in (data-perms.graph/api-graph)
+ [:groups (:id group) (mt/id) :view-data "PUBLIC" (mt/id :orders)]
+ :blocked))
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :orders)}]}]
+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
(deftest group-permission-warnings-require-admin-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup group {}]
- (group-access/add-groups! app [(:id group)])
- (mt/user-http-request :rasta :get 403 "apps/birds/group-permission-warnings")))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (group-access/add-groups! app [(:id group)])
+ (mt/user-http-request :rasta :get 403 "apps/birds/group-permission-warnings"))))))
(deftest group-permission-warnings-without-dependencies-test
(mt/with-premium-features #{:data-apps :advanced-permissions :sandboxes}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids []}
- :model/PermissionsGroup group {}]
- (group-access/add-groups! app [(:id group)])
- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [app []]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (group-access/add-groups! app [(:id group)])
+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
(deftest group-permission-warnings-missing-app-test
(mt/with-premium-features #{:data-apps}
@@ -129,61 +227,57 @@
(deftest group-permission-warnings-use-group-access-not-memberships-test
(mt/with-premium-features #{:data-apps :advanced-permissions}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues)]}
- :model/PermissionsGroup group {}
- :model/PermissionsGroup other-group {}]
- (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
- (perms/set-database-permission! (:id other-group) (mt/id) :perms/view-data :unrestricted)
- (group-access/add-groups! app [(:id group)])
- (perms/add-user-to-group! (mt/user->id :rasta) (:id group))
- (perms/add-user-to-group! (mt/user->id :rasta) (:id other-group))
- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]
- (group-access/permission-warnings app)))))))
+ (with-app-reading! [app [(mt/id :venues)]]
+ (mt/with-temp [:model/PermissionsGroup group {}
+ :model/PermissionsGroup other-group {}]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (perms/set-database-permission! (:id other-group) (mt/id) :perms/view-data :unrestricted)
+ (group-access/add-groups! app [(:id group)])
+ (perms/add-user-to-group! (mt/user->id :rasta) (:id group))
+ (perms/add-user-to-group! (mt/user->id :rasta) (:id other-group))
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]
+ (group-access/permission-warnings app))))))))
(deftest group-permission-warnings-include-impersonated-access-test
(mt/with-premium-features #{:data-apps :advanced-permissions}
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues)]}
- :model/PermissionsGroup group {}]
- (group-access/add-groups! app [(:id group)])
- (mt/user-http-request :crowberto :put 200 "permissions/graph"
- (-> (data-perms.graph/api-graph)
- (assoc-in [:groups (:id group) (mt/id) :view-data] :impersonated)
- (assoc :impersonations [{:group_id (:id group) :db_id (mt/id)
- :attribute "role"}])))
- (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))
- (mt/user-http-request :crowberto :put 200 "permissions/graph"
- (assoc-in (data-perms.graph/api-graph)
- [:groups (:id group) (mt/id) :view-data]
- :blocked))
- (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]
- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [app [(mt/id :venues)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (group-access/add-groups! app [(:id group)])
+ (mt/user-http-request :crowberto :put 200 "permissions/graph"
+ (-> (data-perms.graph/api-graph)
+ (assoc-in [:groups (:id group) (mt/id) :view-data] :impersonated)
+ (assoc :impersonations [{:group_id (:id group) :db_id (mt/id)
+ :attribute "role"}])))
+ (is (= [] (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))
+ (mt/user-http-request :crowberto :put 200 "permissions/graph"
+ (assoc-in (data-perms.graph/api-graph)
+ [:groups (:id group) (mt/id) :view-data]
+ :blocked))
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :venues)}]}]
+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
(deftest group-permission-warning-lookups-are-batched-test
(mt/with-no-data-perms-for-all-users!
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues) (mt/id :orders)]}
- :model/PermissionsGroup finches {}
- :model/PermissionsGroup owls {}]
- (group-access/add-groups! app [(:id finches)])
- (group-access/permission-warnings app)
- (let [query-count #(t2/with-call-count [calls]
- (group-access/permission-warnings app)
- (calls))
- one-group-count (query-count)]
- (group-access/add-groups! app [(:id owls)])
- (is (= one-group-count (query-count)))))))
+ (with-app-reading! [app [(mt/id :venues) (mt/id :orders)]]
+ (mt/with-temp [:model/PermissionsGroup finches {}
+ :model/PermissionsGroup owls {}]
+ (group-access/add-groups! app [(:id finches)])
+ (group-access/permission-warnings app)
+ (let [query-count #(t2/with-call-count [calls]
+ (group-access/permission-warnings app)
+ (calls))
+ one-group-count (query-count)]
+ (group-access/add-groups! app [(:id owls)])
+ (is (= one-group-count (query-count))))))))
(deftest group-permission-warnings-require-feature-test
- (mt/with-temp [:model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(mt/id :venues)]}
- :model/PermissionsGroup group {}]
- (group-access/add-groups! app [(:id group)])
- (mt/with-premium-features #{}
- (mt/user-http-request :crowberto :get 402 "apps/birds/group-permission-warnings")
- (mt/user-http-request :rasta :get 402 "apps/birds/group-permission-warnings"))))
+ (with-app-reading! [app [(mt/id :venues)]]
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (group-access/add-groups! app [(:id group)])
+ (mt/with-premium-features #{}
+ (mt/user-http-request :crowberto :get 402 "apps/birds/group-permission-warnings")
+ (mt/user-http-request :rasta :get 402 "apps/birds/group-permission-warnings")))))
(deftest group-permission-warning-lookups-exclude-unrelated-permissions-test
(mt/with-temp [:model/Database other-db {}
@@ -220,18 +314,17 @@
:model/Table albatrosses {:db_id (:id zoo) :name "albatrosses"
:display_name "Albatrosses" :schema nil}
:model/TableUserSettings _ {:table_id (:id finches) :display_name "Finches"}
- :model/DataApp app {:name "birds" :display_name "Birds" :bundle_path "birds.js"
- :table_ids [(:id albatrosses) (:id owls) (:id finches)]}
:model/PermissionsGroup group {}]
- (doseq [group-id [(:id (perms/all-users-group)) (:id group)]
- database-id [(:id aviary) (:id zoo)]]
- (perms/set-database-permission! group-id database-id :perms/view-data :blocked))
- (group-access/add-groups! app [(:id group)])
- (is (= [{:group_id (:id group)
- :missing_tables [{:id (:id finches) :name "Finches" :schema "PUBLIC"
- :database_id (:id aviary) :database_name "Aviary"}
- {:id (:id owls) :name "Owls" :schema "PUBLIC"
- :database_id (:id aviary) :database_name "Aviary"}
- {:id (:id albatrosses) :name "Albatrosses" :schema nil
- :database_id (:id zoo) :database_name "Zoo"}]}]
- (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings")))))))
+ (with-app-reading! [app [(:id albatrosses) (:id owls) (:id finches)]]
+ (doseq [group-id [(:id (perms/all-users-group)) (:id group)]
+ database-id [(:id aviary) (:id zoo)]]
+ (perms/set-database-permission! group-id database-id :perms/view-data :blocked))
+ (group-access/add-groups! app [(:id group)])
+ (is (= [{:group_id (:id group)
+ :missing_tables [{:id (:id finches) :name "Finches" :schema "PUBLIC"
+ :database_id (:id aviary) :database_name "Aviary"}
+ {:id (:id owls) :name "Owls" :schema "PUBLIC"
+ :database_id (:id aviary) :database_name "Aviary"}
+ {:id (:id albatrosses) :name "Albatrosses" :schema nil
+ :database_id (:id zoo) :database_name "Zoo"}]}]
+ (mt/user-http-request :crowberto :get 200 "apps/birds/group-permission-warnings"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/resource_tables_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/resource_tables_test.clj
deleted file mode 100644
index b5c4a77d596..00000000000
--- a/enterprise/backend/test/metabase_enterprise/data_apps/resource_tables_test.clj
+++ /dev/null
@@ -1,46 +0,0 @@
-(ns metabase-enterprise.data-apps.resource-tables-test
- "The tables a data app's resources read, recorded for the permission warnings an admin sees."
- (:require
- [clojure.test :refer :all]
- [metabase-enterprise.data-apps.resource-tables :as resource-tables]
- [metabase.lib.core :as lib]
- [metabase.test :as mt]))
-
-(set! *warn-on-reflection* true)
-
-(deftest a-table-in-an-optional-clause-is-recorded-test
- (testing "an optional clause is ordinary query syntax that a parser doesn't read, and the tables inside it are read
- like the ones outside"
- (let [native (fn [sql] (lib/native-query (mt/metadata-provider) sql))]
- (mt/with-temp [:model/Collection {collection-id :id} {:namespace :data-apps}
- :model/Card _ {:collection_id collection-id
- :dataset_query (native "SELECT * FROM venues WHERE 1=1 [[AND name = {{name}}]]")}
- :model/Card _ {:collection_id collection-id
- :dataset_query (native "SELECT * FROM venues v [[JOIN checkins c ON c.venue_id = v.id AND c.id = {{id}}]]")}]
- (is (= (sort [(mt/id :venues) (mt/id :checkins)])
- (resource-tables/collection-table-ids collection-id))
- "checkins is named only inside the optional clause")))))
-
-(deftest a-table-joined-to-a-card-tag-is-recorded-test
- (testing "a card tag stands where a table does, and the parser reads the query around it"
- (let [native (fn [sql] (lib/native-query (mt/metadata-provider) sql))]
- (mt/with-temp [:model/Collection {collection-id :id} {:namespace :data-apps}
- :model/Card {venues-id :id} {:collection_id collection-id
- :dataset_query (native "SELECT * FROM venues")}
- :model/Card _ {:collection_id collection-id
- :dataset_query (native (format "SELECT * FROM {{#%d}} x JOIN checkins c ON c.venue_id = x.id" venues-id))}]
- (is (= (sort [(mt/id :venues) (mt/id :checkins)])
- (resource-tables/collection-table-ids collection-id)))))))
-
-(deftest a-table-in-a-snippet-is-recorded-test
- (testing "a snippet tag stands for the SQL its snippet holds, snippets inside it included, so the tables it names
- are read like the ones around it"
- (let [native (fn [sql] (lib/native-query (mt/metadata-provider) sql))]
- (mt/with-temp [:model/NativeQuerySnippet _ {:name "venues table" :content "venues"}
- :model/NativeQuerySnippet _ {:name "checked in" :content "id IN (SELECT venue_id FROM checkins)"}
- :model/NativeQuerySnippet _ {:name "open" :content "price > 1 AND {{snippet: checked in}}"}
- :model/Collection {collection-id :id} {:namespace :data-apps}
- :model/Card _ {:collection_id collection-id
- :dataset_query (native "SELECT * FROM {{snippet: venues table}} v WHERE {{snippet: open}}")}]
- (is (= (sort [(mt/id :venues) (mt/id :checkins)])
- (resource-tables/collection-table-ids collection-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
index 9402ccaf1ce..d04510c780d 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
@@ -704,7 +704,7 @@
:display-name "Venues"
:card-id card-id}}))}]
(is (= {:card #{card-id}
- :table #{(mt/id :checkins) (mt/id :venues)}}
+ :table #{(mt/id :checkins)}}
(calculation/calculate-deps :action (actions/select-action :id action-id)))))))
(deftest upstream-deps-implicit-action-test
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 353dd9f2436..b07825ca508 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -781,8 +781,8 @@
(deps.findings/upsert-analysis! parent-card)
(deps.findings/upsert-analysis! child-card)
(deps.findings/upsert-analysis! other-card)
- (deps.dependency-status/upsert-status! :card child-card-id)
- (deps.dependency-status/upsert-status! :card other-card-id)
+ (deps.dependency-status/upsert-status! :card child-card-id false)
+ (deps.dependency-status/upsert-status! :card other-card-id false)
(testing "Checking that the initial analyses exist"
(assert-has-analyses
{:card {parent-card-id old-version
@@ -817,7 +817,7 @@
:to_entity_type :card :to_entity_id parent-id}]
(deps.findings/upsert-analysis! parent)
(deps.findings/upsert-analysis! child)
- (deps.dependency-status/upsert-status! :card child-id)
+ (deps.dependency-status/upsert-status! :card child-id false)
(t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")})
(deps.test/synchronously-run-backfill!)
(testing "Parent card should be marked stale"
@@ -993,7 +993,7 @@
:to_entity_type :segment :to_entity_id segment-id}]
(deps.findings/upsert-analysis! segment)
(deps.findings/upsert-analysis! card)
- (deps.dependency-status/upsert-status! :card card-id)
+ (deps.dependency-status/upsert-status! :card card-id false)
(assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})
(t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})
(deps.test/synchronously-run-backfill!)
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
index d3e8dec01e6..15586d21787 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
@@ -34,7 +34,7 @@
(mt/with-premium-features #{:dependencies}
(mt/with-model-cleanup [:model/DependencyStatus]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
- (deps.dependency-status/upsert-status! :card card-id)
+ (deps.dependency-status/upsert-status! :card card-id false)
(t2/update! :model/Card card-id {:name "Renamed"})
(is (not (stale? :card card-id)))
(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
@@ -58,7 +58,7 @@
(mt/with-premium-features #{}
(mt/with-model-cleanup [:model/DependencyStatus]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
- (deps.dependency-status/upsert-status! :card card-id)
+ (deps.dependency-status/upsert-status! :card card-id false)
(t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
(is (not (stale? :card card-id))))))))
@@ -69,7 +69,7 @@
(mt/with-temp [:model/Dashboard {dashboard-id :id} {}
:model/Card {card-id :id} {:dataset_query (table-query :products)}
:model/Card {other-id :id} {:dataset_query (table-query :orders)}]
- (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id)]
+ (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id false)]
(processed!)
(let [dashcard-id (t2/insert-returning-pk! :model/DashboardCard
{:dashboard_id dashboard-id :card_id card-id
@@ -98,7 +98,7 @@
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
(t2/insert! :model/Dependency {:from_entity_type "card" :from_entity_id card-id
:to_entity_type "table" :to_entity_id (mt/id :products)})
- (deps.dependency-status/upsert-status! :card card-id)
+ (deps.dependency-status/upsert-status! :card card-id false)
(t2/delete! :model/Card :id card-id)
(is (stale? :card card-id))
(deps.test/synchronously-run-backfill!)
@@ -131,7 +131,7 @@
(t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-b
:to_entity_type :card :to_entity_id card-a})
(doseq [card-id [card-a card-b]]
- (deps.dependency-status/upsert-status! :card card-id)
+ (deps.dependency-status/upsert-status! :card card-id false)
(deps.analysis-finding/upsert-analysis! :card card-id true []))
(mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!
(fn [type->ids]
@@ -160,7 +160,7 @@
triggered (atom 0)]
(t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id
:row 4 :col 0 :size_x 4 :size_y 4})
- (deps.dependency-status/upsert-status! :dashboard dashboard-id)
+ (deps.dependency-status/upsert-status! :dashboard dashboard-id false)
(mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))
task.backfill/trigger-backfill-job! (fn [] (swap! triggered inc))]
(deps.test/do-with-immediate-hook-writes
@@ -182,7 +182,7 @@
:model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id
:visualization_settings {}}]
(let [exported (ts/extract-one "Dashboard" dashboard-id)]
- (deps.dependency-status/upsert-status! :dashboard dashboard-id)
+ (deps.dependency-status/upsert-status! :dashboard dashboard-id false)
(mt/with-dynamic-fn-redefs [task.backfill/trigger-backfill-job! (fn [])]
(deps.test/do-with-immediate-hook-writes
#(serdes/with-cache
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
index 6c2e4ab094f..a9a5ae489e8 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -136,10 +136,8 @@
(deps.test/synchronously-run-backfill!)
(testing "raw deps are recorded correctly"
(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
- (is (=? #{(depends-on-> :card id2 :table (mt/id :orders))
- (depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
- (is (=? #{(depends-on-> :card id3 :table (mt/id :orders))
- (depends-on-> :card id3 :card id2)} (upstream-of :card id3))))
+ (is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
+ (is (=? #{(depends-on-> :card id3 :card id2)} (upstream-of :card id3))))
(testing "transitive deps are computed correctly"
(testing "for each card"
(is (=? {:card #{id2 id3}}
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
index 7b2745f85a3..6cce6c3ae00 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
@@ -67,8 +67,7 @@
:dataset-query
(lib-be/normalize-query mp)
(deps.native-validation/native-query-deps driver))))
- (is (= #{{:table (mt/id :products)}
- {:card 1}}
+ (is (= #{{:card 1}}
(->> (lib.metadata/card mp 5)
:dataset-query
(lib-be/normalize-query mp)
@@ -88,9 +87,7 @@
:dataset-query
(lib-be/normalize-query mp)
(deps.native-validation/native-query-deps driver))))
- (is (= #{{:table (mt/id :products)}
- {:table (mt/id :orders)}
- {:card 1}
+ (is (= #{{:card 1}
{:card 2}}
(->> (lib.metadata/card mp 9)
:dataset-query
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
index e84d4e5159f..8f2ec778799 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
@@ -11,6 +11,8 @@
[metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.dependencies.hooks :as dependencies.hooks]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
[metabase.test :as mt]
@@ -585,3 +587,59 @@
(#'dependencies.backfill/run-and-reschedule! nil))))
(testing "and the next run was scheduled anyway"
(is (= 1 (count @scheduled))))))))
+
+(defn- table-dependency-ids
+ "The IDs of the tables the `entity-type` entity `entity-id` depends on directly."
+ [entity-type entity-id]
+ (t2/select-fn-set :to_entity_id :model/Dependency
+ :from_entity_type entity-type :from_entity_id entity-id :to_entity_type :table))
+
+(deftest ^:synchronized backfill-snippet-edit-recomputes-queries-expanding-it-test
+ (testing "a snippet's tables are the card's own dependencies, so editing a snippet, even one nested in another,
+ recomputes the cards whose SQL expands it"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/NativeQuerySnippet {inner-id :id} {:name "birds inner" :content "VENUES"}
+ :model/NativeQuerySnippet _ {:name "birds outer" :content "SELECT * FROM {{snippet: birds inner}}"}
+ :model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "{{snippet: birds outer}}")}]
+ (backfill-all-existing-entities!)
+ (is (= #{(mt/id :venues)} (table-dependency-ids :card card-id)))
+ (t2/update! :model/NativeQuerySnippet inner-id {:content "CHECKINS"})
+ (backfill-all-existing-entities!)
+ (is (= #{(mt/id :checkins)} (table-dependency-ids :card card-id)))))))
+
+(deftest ^:synchronized backfill-card-reference-leaves-out-card-tables-test
+ (testing "a native query depends on the card it references, not on the card's tables, which it reads through the card"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies}
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card {orders-id :id} {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
+ :model/Card {card-id :id} {:dataset_query
+ (lib/native-query
+ mp
+ (format "SELECT * FROM {{#%d}} o JOIN PRODUCTS p ON o.PRODUCT_ID = p.ID"
+ orders-id))}]
+ (backfill-all-existing-entities!)
+ (is (= #{(mt/id :products)} (table-dependency-ids :card card-id)))
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :card :to_entity_id orders-id)))))))
+
+(deftest ^:synchronized backfill-records-table-sync-finds-later-test
+ (testing "a table a native query names before sync finds it gets its dependency once a sync of the database finds it"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM PUBLIC.BIRDS_LATE")}
+ :model/Card {known-id :id} {:dataset_query (lib/native-query (mt/metadata-provider)
+ "SELECT * FROM VENUES")}]
+ (backfill-all-existing-entities!)
+ (is (empty? (table-dependency-ids :card card-id)))
+ (is (true? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id card-id)))
+ (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus :entity_type :card :entity_id known-id)))
+ (mt/with-temp [:model/Table {late-id :id} {:db_id (mt/id) :schema "PUBLIC" :name "BIRDS_LATE" :active true}]
+ (dependencies.hooks/after-database-sync (mt/id))
+ (backfill-all-existing-entities!)
+ (is (= #{late-id} (table-dependency-ids :card card-id)))
+ (is (false? (t2/select-one-fn :unresolved_tables :model/DependencyStatus
+ :entity_type :card :entity_id card-id))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
index d7067c66e16..6e4c065e2cf 100644
--- a/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/remote_sync/data_apps_test.clj
@@ -5,8 +5,9 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
- [metabase-enterprise.data-apps.core :as data-apps]
+ [metabase-enterprise.data-apps.group-access :as group-access]
[metabase-enterprise.data-apps.test-util :as data-apps.tu]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.remote-sync.impl :as impl]
[metabase-enterprise.remote-sync.models.remote-sync-task :as remote-sync.task]
[metabase-enterprise.remote-sync.source :as source]
@@ -15,10 +16,10 @@
[metabase-enterprise.remote-sync.test-helpers :as test-helpers]
[metabase.actions.core :as actions]
[metabase.collections.models.collection :as collection]
- [metabase.driver :as driver]
[metabase.events.core :as events]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.permissions.core :as perms]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -226,9 +227,7 @@
(is (=? {:type :query :collection_id collection-id :model_id nil}
(t2/select-one :model/Action :entity_id (copy-eid "action" :model/Action action-id)))))
(testing "the source is untouched"
- (is (nil? (t2/select-one-fn :collection_id :model/Action :id action-id))))
- (testing "the app records the tables its resources read"
- (is (= [(mt/id :venues)] (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))))))
+ (is (nil? (t2/select-one-fn :collection_id :model/Action :id action-id)))))))))))
(deftest pull-updates-and-removes-resources-with-their-files-test
(with-data-apps-sync
@@ -329,6 +328,41 @@
[mock stem]
(some #(str/starts-with? (last (str/split % #"/")) stem) (resource-files mock)))
+(deftest pull-reaches-permission-warnings-test
+ (testing "once the jobs run after a pull, the warnings list the tables the app's native question and action name"
+ (with-data-apps-sync
+ (mt/with-premium-features #{:data-apps :advanced-permissions :dependencies}
+ (mt/with-no-data-perms-for-all-users!
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
+ (let [action-id (actions/insert! {:name "Clear checkins"
+ :type :query
+ :database_id (mt/id)
+ :dataset_query (lib/native-query (mt/metadata-provider)
+ "DELETE FROM checkins WHERE id = {{id}}")
+ :parameters [{:id "id" :slug "id" :type :number/=}]})
+ resources (data-apps.tu/build-resources
+ shop-collection-name shop-collection-eid
+ [{:entity_id question-eid :name "VenuesList" :query (venues-query)}]
+ [action-id])
+ path (some #(when (str/includes? % question-eid) %) (keys resources))
+ parsed (yaml/parse-string (get resources path))
+ native (assoc parsed :dataset_query {:database (:database (:dataset_query parsed))
+ :lib/type "mbql/query"
+ :stages [{:lib/type "mbql.stage/native"
+ :native "select * from Venues"}]})
+ src (test-helpers/versioned-source
+ :trees {"v0" (shop-tree (assoc resources path (yaml/generate-string native)))}
+ :current "v0")
+ result (import-at! src "v0" :force? true)]
+ (is (= :success (:status result)) (:message result))
+ (mt/with-temp [:model/PermissionsGroup group {}]
+ (let [app (t2/select-one :model/DataApp :name "shop")]
+ (perms/set-database-permission! (:id group) (mt/id) :perms/view-data :blocked)
+ (group-access/add-groups! app [(:id group)])
+ (deps.test/synchronously-run-backfill!)
+ (is (=? [{:group_id (:id group) :missing_tables [{:id (mt/id :checkins)} {:id (mt/id :venues)}]}]
+ (group-access/permission-warnings app))))))))))))
+
(deftest an-apps-collection-is-read-only-test
(testing "an admin can't write an app's collection, its cards or its actions, whatever the sync mode"
(with-data-apps-sync
@@ -597,32 +631,6 @@
(is (not (t2/exists? :model/DataApp :name [:in ["first" "second"]])))
(is (not (t2/exists? :model/Collection :entity_id [:in (map data-apps.tu/collection-entity-id ["first" "second"])])))))))
-;; a driver that can't name the tables of a native query: `driver/native-query-table-refs` has no method for it
-(driver/register! ::no-table-refs :abstract? true)
-
-(deftest pull-survives-a-query-whose-tables-cant-be-read-test
- (testing "a native query on a driver that can't name its tables doesn't fail the pull; the app records the rest"
- (with-data-apps-sync
- (mt/with-temp [:model/Database _ {:engine ::no-table-refs :name "no-table-refs"}]
- (let [resources (data-apps.tu/build-resources
- shop-collection-name shop-collection-eid
- [{:entity_id question-eid :name "VenuesList" :query (venues-query)}
- {:entity_id "shopQuestionNative000" :name "Native" :query (venues-query)}]
- [])
- path (some #(when (str/includes? % "shopQuestionNative000") %) (keys resources))
- native (assoc (yaml/parse-string (get resources path))
- :database_id "no-table-refs"
- :dataset_query {:database "no-table-refs"
- :lib/type "mbql/query"
- :stages [{:lib/type "mbql.stage/native" :native "{\"find\": \"orders\"}"}]})
- src (test-helpers/versioned-source
- :trees {"v0" (shop-tree (assoc resources path (yaml/generate-string native)))}
- :current "v0")
- result (import-at! src "v0" :force? true)]
- (is (= :success (:status result)) (:message result))
- (is (= "v0" (remote-sync.task/last-version)))
- (is (= [(mt/id :venues)] (t2/select-one-fn :table_ids :model/DataApp :name "shop"))))))))
-
(deftest pull-refuses-a-manifest-that-names-a-collection-it-doesnt-define-test
(testing "a manifest with no resource files can't claim a collection that is already here"
(with-data-apps-sync
@@ -667,7 +675,7 @@
(impl/handle-task-result! result task)
result))
-(deftest an-export-that-merges-checks-and-records-like-a-pull-test
+(deftest an-export-that-merges-checks-like-a-pull-test
(with-data-apps-sync
(testing "a remote file that a pull refuses is refused by an export that merges it in"
(mt/with-temp [:model/Collection {elsewhere :id} {:name "Elsewhere"}
@@ -686,22 +694,6 @@
(is (=? {:name "Someone else's" :collection_id elsewhere}
(t2/select-one :model/Card :id foreign-id))))))))
-(deftest an-export-that-merges-records-the-tables-test
- (with-data-apps-sync
- (let [remote (data-apps.tu/build-resources
- shop-collection-name shop-collection-eid
- [{:entity_id question-eid :name "VenuesList" :query (venues-query)}
- {:entity_id "shopQuestionCheckins0" :name "Checkins"
- :query {:stages [{:source {:type "table" :id (mt/id :checkins)} :limit 5}]}}]
- [])
- src (test-helpers/versioned-source
- :trees {"v0" (shop-tree (question-resources)) "v1" (shop-tree remote)}
- :current "v1")]
- (is (= :success (:status (import-at! src "v0" :force? true))))
- (is (= :success (:status (export-merged! src "v0"))))
- (is (= (sort [(mt/id :venues) (mt/id :checkins)])
- (t2/select-one-fn :table_ids :model/DataApp :name "shop"))))))
-
(deftest pull-loads-a-snippet-with-the-card-that-uses-it-test
(testing "a resource may use a snippet that the same pull brings"
(with-data-apps-sync
@@ -765,23 +757,6 @@
(is (=? {:database_id db-id :collection_id (shop-collection-id)}
(t2/select-one :model/Card :entity_id question-eid))))))))
-(deftest pull-records-the-table-a-native-query-names-test
- (testing "a native query's table is matched by name whatever its case"
- (with-data-apps-sync
- (let [resources (question-resources)
- path (some #(when (str/includes? % question-eid) %) (keys resources))
- parsed (yaml/parse-string (get resources path))
- native (assoc parsed :dataset_query {:database (:database (:dataset_query parsed))
- :lib/type "mbql/query"
- :stages [{:lib/type "mbql.stage/native"
- :native "select * from Venues"}]})
- src (test-helpers/versioned-source
- :trees {"v0" (shop-tree (assoc resources path (yaml/generate-string native)))}
- :current "v0")
- result (import-at! src "v0" :force? true)]
- (is (= :success (:status result)) (:message result))
- (is (= [(mt/id :venues)] (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))
-
(deftest an-apps-collection-cant-be-moved-to-the-trash-test
(testing "trashing it would drop the app's resource files on the next export"
(with-data-apps-sync
@@ -797,25 +772,6 @@
(mt/user-http-request :crowberto :delete 204 "apps/shop")
(is (not (t2/exists? :model/Collection :id collection-id))))))))))
-(deftest pull-records-the-table-a-query-action-names-test
- (testing "the tables an action's SQL names are recorded apart from the tables the questions read"
- (with-data-apps-sync
- (let [mp (mt/metadata-provider)
- action-id (actions/insert! {:name "Clear checkins"
- :type :query
- :database_id (mt/id)
- :dataset_query (lib/native-query mp "DELETE FROM checkins WHERE id = {{id}}")
- :parameters [{:id "id" :slug "id" :type :number/=}]})
- resources (data-apps.tu/build-resources
- shop-collection-name shop-collection-eid
- [{:entity_id question-eid :name "VenuesList" :query (venues-query)}]
- [action-id])
- src (test-helpers/versioned-source :trees {"v0" (shop-tree resources)} :current "v0")
- result (import-at! src "v0" :force? true)]
- (is (= :success (:status result)) (:message result))
- (is (= (sort [(mt/id :venues) (mt/id :checkins)])
- (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))
-
(deftest an-apps-own-yaml-is-left-alone-test
(testing "a YAML file in the app's directory that isn't its manifest is the app's own: not loaded, not removed"
(with-data-apps-sync
@@ -1017,16 +973,6 @@
(is (= :success (:status (export! mock))))
(is (= :success (:status (import-at! mock "main" :force? true)))))))))))
-(deftest a-pull-whose-table-record-fails-still-loads-test
- (testing "the tables are recorded after the load's transaction, so a failure there is logged and the pull stands"
- (with-data-apps-sync
- (let [src (test-helpers/versioned-source :trees {"v0" (shop-tree (question-resources))} :current "v0")]
- (mt/with-dynamic-fn-redefs [data-apps/record-table-dependencies! (fn [] (throw (ex-info "the tables can't be recorded" {})))]
- (let [result (import-at! src "v0" :force? true)]
- (is (= :success (:status result)) (:message result))))
- (is (t2/exists? :model/DataApp :name "shop"))
- (is (= "v0" (remote-sync.task/last-version)))))))
-
(deftest a-refused-pull-names-the-file-whatever-its-name-test
(testing "a validator failure is reported as what it is when a path has \"branch\" in it, not as a branch error"
(with-data-apps-sync
@@ -1045,17 +991,6 @@
(is (str/includes? (:message result) "must be a question or metric"))
(is (str/includes? (:message result) card-file))))))
-(deftest an-export-leaves-the-tables-an-app-reads-alone-test
- (testing "only a pull changes an app's cards, so a pull records the tables they read and an export leaves the record
- as it is"
- (with-data-apps-sync
- (let [mock (test-helpers/create-mock-source :initial-files {"main" (shop-tree (question-resources))})]
- (is (= :success (:status (import-at! mock "main" :force? true))))
- (mt/user-http-request :crowberto :put 200 "apps/shop" {:bundle "B2"})
- (t2/update! :model/DataApp :name "shop" {:table_ids [(mt/id :checkins)]})
- (is (= :success (:status (export! mock))))
- (is (= [(mt/id :checkins)] (t2/select-one-fn :table_ids :model/DataApp :name "shop")))))))
-
(defn- generated-app-tree
"The repo files of the `shop` app as `/api/apps/generate` writes them: its manifest and collection, a question
aggregating `metric-id`, the copy of that metric, the copy of `action-id`, and its bundle."
diff --git a/frontend/src/metabase-types/api/data-app.ts b/frontend/src/metabase-types/api/data-app.ts
index 833921ff15f..02c5a615e94 100644
--- a/frontend/src/metabase-types/api/data-app.ts
+++ b/frontend/src/metabase-types/api/data-app.ts
@@ -26,8 +26,6 @@ export interface DataApp {
enabled: boolean;
/** The collection that contains this app's saved questions and models. */
resource_collection_id: number;
- /** Tables used by the last successful resource synchronization. */
- table_ids: number[];
/**
* External origins the app's sandboxed bundle may `fetch`/XHR. Empty means
* none (Metabase data still flows through the SDK). Each entry is an origin,
diff --git a/frontend/src/metabase-types/api/mocks/data-app.ts b/frontend/src/metabase-types/api/mocks/data-app.ts
index 61ef6177ea9..c3b80b45729 100644
--- a/frontend/src/metabase-types/api/mocks/data-app.ts
+++ b/frontend/src/metabase-types/api/mocks/data-app.ts
@@ -11,7 +11,6 @@ export const createMockDataApp = (opts?: Partial<DataApp>): DataApp => ({
bundle_path: "dist/index.js",
enabled: true,
resource_collection_id: 1,
- table_ids: [],
allowed_hosts: [],
bundle_hash: "abc123",
created_at: "2024-01-01T00:00:00Z",
diff --git a/resources/migrations/065/20261014_drop_data_app_table_ids.yaml b/resources/migrations/065/20261014_drop_data_app_table_ids.yaml
new file mode 100644
index 00000000000..cd95d702474
--- /dev/null
+++ b/resources/migrations/065/20261014_drop_data_app_table_ids.yaml
@@ -0,0 +1,20 @@
+## The tables a data app's resources read are now walked from the dependency graph when an admin opens the
+## permission warnings, instead of being recorded on the app after every import.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-10-14T00:00:00
+ author: ranquild
+ comment: Drop data_app.table_ids, the dependency graph records the tables data apps read
+ changes:
+ - dropColumn:
+ tableName: data_app
+ columnName: table_ids
+ rollback:
+ - addColumn:
+ tableName: data_app
+ columns:
+ - column:
+ name: table_ids
+ type: ${text.type}
+ remarks: JSON array of table ids used by the last successful resource sync
diff --git a/resources/migrations/065/20261016_dependency_status_unresolved_tables.yaml b/resources/migrations/065/20261016_dependency_status_unresolved_tables.yaml
new file mode 100644
index 00000000000..fd1db922578
--- /dev/null
+++ b/resources/migrations/065/20261016_dependency_status_unresolved_tables.yaml
@@ -0,0 +1,23 @@
+## A native query can name a table that sync hasn't found yet. The backfill flags such entities, so that the next sync
+## of their database recomputes them and records the table once it exists.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-10-16T00:00:00
+ author: ranquild
+ comment: Add dependency_status.unresolved_tables, flagging entities that name tables matching nothing
+ changes:
+ - addColumn:
+ tableName: dependency_status
+ columns:
+ - column:
+ name: unresolved_tables
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity names tables that match no synced table, recomputed after each sync
+ constraints:
+ nullable: false
+ rollback:
+ - dropColumn:
+ tableName: dependency_status
+ columnName: unresolved_tables
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 81e637522f6..0d56c7e9aa5 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -1567,10 +1567,19 @@
{:closed true}
[:transform :metabase.lib.schema.id/transform]])
+(mr/def ::native-query-deps.unresolved-table-dep
+ [:map
+ {:closed true}
+ [:unresolved-table [:map
+ {:closed true}
+ [:schema {:optional true} :string]
+ [:table :string]]]])
+
(mr/def ::native-query-deps
[:set [:or
::native-query-deps.table-dep
- ::native-query-deps.transform-dep]])
+ ::native-query-deps.transform-dep
+ ::native-query-deps.unresolved-table-dep]])
(defmulti native-query-deps
"Gets the table dependencies of a given sql string (or equivalent).
@@ -1580,6 +1589,9 @@
`query` is a Lib `:metabase.lib.schema/native-only-query`; you can use [[metabase.driver-api.core/raw-native-query]]
to get the raw native query as needed.
+ A table the query names that matches no synced table or transform target can be returned as
+ `{:unresolved-table {:schema ... :table ...}}`, so dependency tracking can recompute the query once sync finds it.
+
The return value should match the `:metabase.driver/native-query-deps` schema."
{:added "0.57.0" :arglists '([driver query])}
dispatch-on-initialized-driver
@@ -1593,31 +1605,6 @@
:args args}))
#{}))
-(mr/def ::native-query-table-refs.table-ref
- [:map
- {:closed true}
- [:schema {:optional true} [:maybe :string]]
- [:table :string]])
-
-(mr/def ::native-query-table-refs
- [:set ::native-query-table-refs.table-ref])
-
-(defmulti native-query-table-refs
- "Gets the raw table references from a native query without resolving them to IDs.
-
- Unlike [[native-query-deps]] which looks up tables in the database to return IDs,
- this method returns just the schema and table names as they appear in the query.
- This is useful for dependency tracking where referenced tables may not
- exist yet.
-
- `query` is a Lib `:metabase.lib.schema/native-only-query`; you can use
- [[metabase.driver-api.core/raw-native-query]] to get the raw native query as needed.
-
- The return value should match the `:metabase.driver/native-query-table-refs` schema."
- {:changelog-test/ignore true :added "0.58.0" :arglists '([driver query])}
- dispatch-on-initialized-driver
- :hierarchy #'hierarchy)
-
(defmulti native-result-metadata
"Gets the result-metadata for a native query using static analysis (i.e., without actually
going to the database).
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index 047d9547a3c..b6acb3d4aed 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -320,16 +320,12 @@
;;; | Dependencies |
;;; +----------------------------------------------------------------------------------------------------------------+
-(mu/defmethod driver/native-query-table-refs :sql :- ::driver/native-query-table-refs
- [driver :- :keyword
- query :- :metabase.lib.schema/native-only-query]
- (into #{} (sql-tools/referenced-tables-raw driver (driver-api/raw-native-query query))))
-
(mu/defmethod driver/native-query-deps :sql :- ::driver/native-query-deps
[driver :- :keyword
query :- :metabase.lib.schema/native-only-query]
(into (driver-api/native-query-table-references query)
- (sql-tools/referenced-tables driver query)))
+ (cond-> (sql-tools/referenced-tables driver query)
+ (seq (lib/all-template-tags query)) (->> (remove :unresolved-table)))))
(mu/defmethod driver/native-result-metadata :sql
[driver :- :keyword
diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.clj
index 00cde57922a..e5a1b6d5c4b 100644
--- a/src/metabase/sql_tools/common.clj
+++ b/src/metabase/sql_tools/common.clj
@@ -61,6 +61,15 @@
{:transform id}))
transforms))))
+(defn resolve-table-specs
+ "Resolve the table `specs` parsed out of a native query with [[find-table-or-transform]], keeping each spec that
+ matches nothing as `{:unresolved-table spec}`."
+ [driver database tables transforms specs]
+ (into #{}
+ (map #(or (find-table-or-transform driver database tables transforms %)
+ {:unresolved-table (cond-> % (nil? (:schema %)) (dissoc :schema))}))
+ specs))
+
(mu/defn table-name :- [:maybe :string]
"Computes a table name from a table reference"
[raw-col :- [:map {:closed true}
diff --git a/src/metabase/sql_tools/core.clj b/src/metabase/sql_tools/core.clj
index 73cd8e31c5b..dc39d210414 100644
--- a/src/metabase/sql_tools/core.clj
+++ b/src/metabase/sql_tools/core.clj
@@ -89,7 +89,8 @@
(interface/returned-columns-impl parser driver native-query))))
(mu/defn referenced-tables :- [:set :map]
- "Return tables referenced by the `native-query`"
+ "Return the tables and transforms the `native-query` references, keeping each name that matches neither as
+ `{:unresolved-table spec}`."
[driver :- :keyword
native-query :- :metabase.lib.schema/native-only-query]
(let [parser (sql-tools.settings/current-parser-backend)]
diff --git a/src/metabase/sql_tools/macaw/core.clj b/src/metabase/sql_tools/macaw/core.clj
index 808c391b13d..f214777a2b6 100644
--- a/src/metabase/sql_tools/macaw/core.clj
+++ b/src/metabase/sql_tools/macaw/core.clj
@@ -107,9 +107,7 @@
table-ids (sql-tools.common/table-ids-by-name (lib/database-id query) (keep :table specs))
db-tables (lib.metadata/bulk-metadata query :metadata/table table-ids)
db-transforms (lib.metadata/transforms query)]
- (into #{}
- (keep #(sql-tools.common/find-table-or-transform driver (lib.metadata/database query) db-tables db-transforms %))
- specs)))
+ (sql-tools.common/resolve-table-specs driver (lib.metadata/database query) db-tables db-transforms specs)))
(defmethod sql-tools/referenced-tables-impl :macaw
[_parser driver query]
diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.clj
index 50d519050fc..b206278567e 100644
--- a/src/metabase/sql_tools/sqlglot/core.clj
+++ b/src/metabase/sql_tools/sqlglot/core.clj
@@ -63,9 +63,7 @@
table-ids (sql-tools.common/table-ids-by-name (lib/database-id query) (keep :table specs))
db-tables (lib.metadata/bulk-metadata query :metadata/table table-ids)
db-transforms (lib.metadata/transforms query)]
- (into #{}
- (keep #(sql-tools.common/find-table-or-transform driver (lib.metadata/database query) db-tables db-transforms %))
- specs))
+ (sql-tools.common/resolve-table-specs driver (lib.metadata/database query) db-tables db-transforms specs))
(catch Exception e
;; Return empty sequence on parse error to follow the Macaw implementation behavior.
(if (sql-parsing/parse-error? e)
diff --git a/src/metabase/transforms_base/ordering.clj b/src/metabase/transforms_base/ordering.clj
index b8f8768eafb..7c14282c353 100644
--- a/src/metabase/transforms_base/ordering.clj
+++ b/src/metabase/transforms_base/ordering.clj
@@ -35,7 +35,7 @@
lib.metadata/database
:engine)]
(if (lib/native-only-query? query)
- (driver/native-query-deps driver query)
+ (into #{} (remove :unresolved-table) (driver/native-query-deps driver query))
(into #{}
(map (fn [table-id]
{:table table-id}))