Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Filter server-side when all target schemas are exactly named (#81250)

Ben Grabowf73eb6d00e8e
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.clj
index 898333b561a..14c1f1c9c7c 100644
--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj
+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj
@@ -120,47 +120,87 @@
[driver database database-types]
((get-method driver/dynamic-database-types-lookup :sql-jdbc) driver database database-types))
-(def ^:private get-tables-sql
+(def ^:private regex-metacharacters
+ "Characters [[metabase.driver.sync/schema-pattern->re-pattern]] can turn into something other than themselves.
+
+ It compiles a filter segment into a regex by expanding an unescaped `*` into `.*` and handing every other
+ character to [[re-pattern]] as regex source. A segment containing none of these therefore compiles to a regex that
+ matches itself and nothing else, which is what lets an `in (...)` predicate select the schemas the client-side
+ filter would have kept.
+
+ Stray `]` and `}` are literals to Java and could be admitted; they are refused anyway rather than resting on that.
+
+ [[metabase.driver.redshift-test/exactly-named-schemas-agrees-with-filter-test]] pins the agreement, and
+ [[metabase.driver.redshift-test/regex-metacharacters-is-complete-test]] pins this set against every ASCII
+ character."
+ (set "\\.[]{}()*+?^$|"))
+
+(defn- exactly-named-schemas
+ "The schema names an inclusion filter names outright, or `nil` when evaluating the filter needs every schema.
+
+ Only a segment free of [[regex-metacharacters]] qualifies: one that carries regex syntax asks a different question
+ than `in (...)` does, and answering it needs every schema. Blank patterns mean \"include everything\", so they fall
+ through to the unfiltered query too."
+ [inclusion-patterns]
+ (when-not (str/blank? inclusion-patterns)
+ (let [segments (map str/trim (str/split inclusion-patterns #","))]
+ (when (perf/every? #(and (seq %) (not-any? regex-metacharacters %)) segments)
+ (distinct segments)))))
+
+(defn- get-tables-sql
+ "Query listing every syncable relation, restricted to `schema-names` when the filter named them outright.
+
+ Without that restriction this scans the whole catalog and the caller drops what the filter rejects. On a shared
+ cluster that is most of the rows -- CI runs measured ~1100 relations fetched to keep ~10."
+ [schema-names]
;; Cal 2024-04-09 This query uses tables that the JDBC redshift driver currently uses.
;; It does not return tables from datashares, which is a relatively new feature of redshift.
;; See https://github.com/dbt-labs/dbt-redshift/issues/742 for an implementation for DBT's integration with redshift
;; for inspiration, and the JDBC driver itself:
;; https://github.com/aws/amazon-redshift-jdbc-driver/blob/master/src/main/java/com/amazon/redshift/jdbc/RedshiftDatabaseMetaData.java#L1794
- ;; This is a vector so adding parameters doesn't require a change to describe-database-tables in the future.
- [(str/join
- "\n"
- ["select"
- " c.relname as name,"
- " n.nspname as schema,"
- " case c.relkind"
- " when 'r' then 'table'"
- " when 'p' then 'partitioned table'"
- " when 'v' then 'view'"
- " when 'f' then 'foreign table'"
- " when 'm' then 'materialized view'"
- " end as type,"
- " d.description"
- " from pg_catalog.pg_namespace n, pg_catalog.pg_class c"
- " left join pg_catalog.pg_description d on c.oid = d.objoid and d.objsubid = 0"
- " left join pg_catalog.pg_class dc on d.classoid=dc.oid and dc.relname='pg_class'"
- " left join pg_catalog.pg_namespace dn on dn.oid=dc.relnamespace and dn.nspname='pg_catalog'"
- " where c.relnamespace = n.oid"
- " and n.nspname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"
- " and c.relkind in ('r', 'p', 'v', 'f', 'm')"
- " and pg_catalog.has_schema_privilege(n.oid, 'USAGE')"
- " and (pg_catalog.has_table_privilege(c.oid,'SELECT')"
- " or pg_catalog.has_any_column_privilege(c.oid,'SELECT'))"
- "union all"
- "select"
- " tablename as name,"
- " schemaname as schema,"
- " 'EXTERNAL TABLE' as type,"
- ;; external tables don't have descriptions
- " null as description"
- "from svv_external_tables t"
- "where schemaname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"
- ;; for external tables, USAGE privileges on a schema is sufficient to select
- " and pg_catalog.has_schema_privilege(t.schemaname, 'USAGE')"])])
+ (let [placeholders (when (seq schema-names)
+ (str "(" (str/join ", " (repeat (count schema-names) "?")) ")"))]
+ (into
+ [(str/join
+ "\n"
+ (remove
+ nil?
+ ["select"
+ " c.relname as name,"
+ " n.nspname as schema,"
+ " case c.relkind"
+ " when 'r' then 'table'"
+ " when 'p' then 'partitioned table'"
+ " when 'v' then 'view'"
+ " when 'f' then 'foreign table'"
+ " when 'm' then 'materialized view'"
+ " end as type,"
+ " d.description"
+ " from pg_catalog.pg_namespace n, pg_catalog.pg_class c"
+ " left join pg_catalog.pg_description d on c.oid = d.objoid and d.objsubid = 0"
+ " left join pg_catalog.pg_class dc on d.classoid=dc.oid and dc.relname='pg_class'"
+ " left join pg_catalog.pg_namespace dn on dn.oid=dc.relnamespace and dn.nspname='pg_catalog'"
+ " where c.relnamespace = n.oid"
+ " and n.nspname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"
+ " and c.relkind in ('r', 'p', 'v', 'f', 'm')"
+ (when placeholders (str " and n.nspname in " placeholders))
+ " and pg_catalog.has_schema_privilege(n.oid, 'USAGE')"
+ " and (pg_catalog.has_table_privilege(c.oid,'SELECT')"
+ " or pg_catalog.has_any_column_privilege(c.oid,'SELECT'))"
+ "union all"
+ "select"
+ " tablename as name,"
+ " schemaname as schema,"
+ " 'EXTERNAL TABLE' as type,"
+ ;; external tables don't have descriptions
+ " null as description"
+ "from svv_external_tables t"
+ "where schemaname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"
+ (when placeholders (str " and t.schemaname in " placeholders))
+ ;; for external tables, USAGE privileges on a schema is sufficient to select
+ " and pg_catalog.has_schema_privilege(t.schemaname, 'USAGE')"]))]
+ ;; once per union branch
+ (concat schema-names schema-names))))
(defn- describe-database-tables
[database]
@@ -169,9 +209,11 @@
syncable? (fn [schema]
(sql-jdbc.describe-database/include-schema-logging-exclusion inclusion-patterns exclusion-patterns schema))]
(eduction
+ ;; kept over the narrowed query too: `syncable?` stays the definition of what syncs, and an exclusion filter
+ ;; still arrives here with every schema.
(comp (filter (comp syncable? :schema))
(map #(dissoc % :type)))
- (sql-jdbc.execute/reducible-query database get-tables-sql))))
+ (sql-jdbc.execute/reducible-query database (get-tables-sql (exactly-named-schemas inclusion-patterns))))))
(defmethod driver/describe-database* :redshift
[driver database]
diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
index 5d18d77c9d2..17df5c0ded6 100644
--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
@@ -5,12 +5,14 @@
[clojure.string :as str]
[clojure.test :refer :all]
[metabase.driver :as driver]
+ [metabase.driver.redshift :as redshift]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.sync :as driver.s]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-util :as lib.tu]
@@ -723,3 +725,86 @@
;; None (special role in Postgres to revert back to login role; should not be quoted)
"none" "SET SESSION AUTHORIZATION none;"
"NONE" "SET SESSION AUTHORIZATION NONE;"))))))
+
+(deftest ^:parallel exactly-named-schemas-agrees-with-filter-test
+ (testing "when the inclusion filter names its schemas outright, membership decides exactly what the filter keeps"
+ (let [candidates ["spectrum"
+ "2026_08_27_18_abc_schema"
+ "2026_08_27_18_abc_schema_extra"
+ "prefix_spectrum"
+ "spectrumx"
+ "SPECTRUM"
+ "other"
+ "with space"
+ "withxspace"
+ "hyphen-schema"
+ "hyphenxschema"
+ "unicodé"
+ "unicode"
+ "at@sign"
+ "atxsign"]]
+ (doseq [patterns ["spectrum"
+ "spectrum,2026_08_27_18_abc_schema"
+ " spectrum , 2026_08_27_18_abc_schema "
+ "spectrum,spectrum"
+ ;; legal Redshift schema names that carry no regex syntax
+ "with space"
+ "hyphen-schema"
+ "unicodé"
+ "at@sign"]]
+ (testing (pr-str patterns)
+ (let [named (#'redshift/exactly-named-schemas patterns)]
+ (is (some? named))
+ (doseq [candidate candidates]
+ (is (= (driver.s/include-schema? patterns nil candidate)
+ (contains? (set named) candidate))
+ (pr-str candidate))))))))
+ (testing "a filter that needs every schema to evaluate falls through to the unrestricted query"
+ (are [patterns] (nil? (#'redshift/exactly-named-schemas patterns))
+ nil
+ ""
+ " "
+ "test*"
+ "*_schema"
+ "spectrum,test*"
+ "crazy\\*schema"
+ "a.c"
+ "a|b"
+ "a$b"
+ "a+b"
+ "a(b)"
+ "a[b]"
+ "a^b"
+ "a?b"
+ ;; an interior empty segment names nothing, so it cannot stand in for the filter
+ "spectrum,,other"))
+ (testing "a trailing comma leaves no empty segment behind, so it still qualifies"
+ (is (= ["spectrum"] (#'redshift/exactly-named-schemas "spectrum,")))))
+
+(deftest ^:parallel regex-metacharacters-is-complete-test
+ (testing "every character the guard admits really does compile to a regex matching only itself"
+ ;; Pins the metacharacter enumeration itself: a character missing from it would be admitted here and show up as
+ ;; a disagreement, rather than as a silently wrong `in (...)` against a real cluster.
+ (let [chars (concat (map char (range 32 127)) [\é \ü \空])
+ ;; the filter splits on commas, so a comma never reaches a segment
+ chars (remove #{\,} chars)]
+ (doseq [c chars
+ :let [segment (str "a" c "c")
+ named (#'redshift/exactly-named-schemas segment)]
+ :when named]
+ (testing (pr-str segment)
+ (doseq [candidate [segment "abc" "axc" "ac" "aXc" "a" "other"]]
+ (is (= (boolean (driver.s/include-schema? segment nil candidate))
+ (contains? (set named) candidate))
+ (pr-str candidate))))))))
+
+(deftest ^:parallel get-tables-sql-test
+ (testing "named schemas are bound once per union branch"
+ (let [[sql & params] (#'redshift/get-tables-sql ["spectrum" "sess"])]
+ (is (= ["spectrum" "sess" "spectrum" "sess"] params))
+ (is (= 2 (count (re-seq #"in \(\?, \?\)" sql))))))
+ (testing "no named schemas leaves the query and its params untouched"
+ (let [[sql & params] (#'redshift/get-tables-sql nil)]
+ (is (empty? params))
+ (is (not (str/includes? sql "nspname in")))
+ (is (not (str/includes? sql "schemaname in"))))))