The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Filter server-side when all target schemas are exactly named (#81250)
Ben Grabowf73eb6d00e8e
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex 898333b561a..14c1f1c9c7c 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -120,47 +120,87 @@[driver database database-types]((get-method driver/dynamic-database-types-lookup :sql-jdbc) driver database database-types))-(def ^:private get-tables-sql+(def ^:private regex-metacharacters+ "Characters [[metabase.driver.sync/schema-pattern->re-pattern]] can turn into something other than themselves.++ It compiles a filter segment into a regex by expanding an unescaped `*` into `.*` and handing every other+ character to [[re-pattern]] as regex source. A segment containing none of these therefore compiles to a regex that+ matches itself and nothing else, which is what lets an `in (...)` predicate select the schemas the client-side+ filter would have kept.++ Stray `]` and `}` are literals to Java and could be admitted; they are refused anyway rather than resting on that.++ [[metabase.driver.redshift-test/exactly-named-schemas-agrees-with-filter-test]] pins the agreement, and+ [[metabase.driver.redshift-test/regex-metacharacters-is-complete-test]] pins this set against every ASCII+ character."+ (set "\\.[]{}()*+?^$|"))++(defn- exactly-named-schemas+ "The schema names an inclusion filter names outright, or `nil` when evaluating the filter needs every schema.++ Only a segment free of [[regex-metacharacters]] qualifies: one that carries regex syntax asks a different question+ than `in (...)` does, and answering it needs every schema. Blank patterns mean \"include everything\", so they fall+ through to the unfiltered query too."+ [inclusion-patterns]+ (when-not (str/blank? inclusion-patterns)+ (let [segments (map str/trim (str/split inclusion-patterns #","))]+ (when (perf/every? #(and (seq %) (not-any? regex-metacharacters %)) segments)+ (distinct segments)))))++(defn- get-tables-sql+ "Query listing every syncable relation, restricted to `schema-names` when the filter named them outright.++ Without that restriction this scans the whole catalog and the caller drops what the filter rejects. On a shared+ cluster that is most of the rows -- CI runs measured ~1100 relations fetched to keep ~10."+ [schema-names];; Cal 2024-04-09 This query uses tables that the JDBC redshift driver currently uses.;; It does not return tables from datashares, which is a relatively new feature of redshift.;; See https://github.com/dbt-labs/dbt-redshift/issues/742 for an implementation for DBT's integration with redshift;; for inspiration, and the JDBC driver itself:;; https://github.com/aws/amazon-redshift-jdbc-driver/blob/master/src/main/java/com/amazon/redshift/jdbc/RedshiftDatabaseMetaData.java#L1794- ;; This is a vector so adding parameters doesn't require a change to describe-database-tables in the future.- [(str/join- "\n"- ["select"- " c.relname as name,"- " n.nspname as schema,"- " case c.relkind"- " when 'r' then 'table'"- " when 'p' then 'partitioned table'"- " when 'v' then 'view'"- " when 'f' then 'foreign table'"- " when 'm' then 'materialized view'"- " end as type,"- " d.description"- " from pg_catalog.pg_namespace n, pg_catalog.pg_class c"- " left join pg_catalog.pg_description d on c.oid = d.objoid and d.objsubid = 0"- " left join pg_catalog.pg_class dc on d.classoid=dc.oid and dc.relname='pg_class'"- " left join pg_catalog.pg_namespace dn on dn.oid=dc.relnamespace and dn.nspname='pg_catalog'"- " where c.relnamespace = n.oid"- " and n.nspname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"- " and c.relkind in ('r', 'p', 'v', 'f', 'm')"- " and pg_catalog.has_schema_privilege(n.oid, 'USAGE')"- " and (pg_catalog.has_table_privilege(c.oid,'SELECT')"- " or pg_catalog.has_any_column_privilege(c.oid,'SELECT'))"- "union all"- "select"- " tablename as name,"- " schemaname as schema,"- " 'EXTERNAL TABLE' as type,"- ;; external tables don't have descriptions- " null as description"- "from svv_external_tables t"- "where schemaname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"- ;; for external tables, USAGE privileges on a schema is sufficient to select- " and pg_catalog.has_schema_privilege(t.schemaname, 'USAGE')"])])+ (let [placeholders (when (seq schema-names)+ (str "(" (str/join ", " (repeat (count schema-names) "?")) ")"))]+ (into+ [(str/join+ "\n"+ (remove+ nil?+ ["select"+ " c.relname as name,"+ " n.nspname as schema,"+ " case c.relkind"+ " when 'r' then 'table'"+ " when 'p' then 'partitioned table'"+ " when 'v' then 'view'"+ " when 'f' then 'foreign table'"+ " when 'm' then 'materialized view'"+ " end as type,"+ " d.description"+ " from pg_catalog.pg_namespace n, pg_catalog.pg_class c"+ " left join pg_catalog.pg_description d on c.oid = d.objoid and d.objsubid = 0"+ " left join pg_catalog.pg_class dc on d.classoid=dc.oid and dc.relname='pg_class'"+ " left join pg_catalog.pg_namespace dn on dn.oid=dc.relnamespace and dn.nspname='pg_catalog'"+ " where c.relnamespace = n.oid"+ " and n.nspname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"+ " and c.relkind in ('r', 'p', 'v', 'f', 'm')"+ (when placeholders (str " and n.nspname in " placeholders))+ " and pg_catalog.has_schema_privilege(n.oid, 'USAGE')"+ " and (pg_catalog.has_table_privilege(c.oid,'SELECT')"+ " or pg_catalog.has_any_column_privilege(c.oid,'SELECT'))"+ "union all"+ "select"+ " tablename as name,"+ " schemaname as schema,"+ " 'EXTERNAL TABLE' as type,"+ ;; external tables don't have descriptions+ " null as description"+ "from svv_external_tables t"+ "where schemaname !~ '^information_schema|catalog_history|pg_|metabase_cache_'"+ (when placeholders (str " and t.schemaname in " placeholders))+ ;; for external tables, USAGE privileges on a schema is sufficient to select+ " and pg_catalog.has_schema_privilege(t.schemaname, 'USAGE')"]))]+ ;; once per union branch+ (concat schema-names schema-names))))(defn- describe-database-tables[database]@@ -169,9 +209,11 @@syncable? (fn [schema](sql-jdbc.describe-database/include-schema-logging-exclusion inclusion-patterns exclusion-patterns schema))](eduction+ ;; kept over the narrowed query too: `syncable?` stays the definition of what syncs, and an exclusion filter+ ;; still arrives here with every schema.(comp (filter (comp syncable? :schema))(map #(dissoc % :type)))- (sql-jdbc.execute/reducible-query database get-tables-sql))))+ (sql-jdbc.execute/reducible-query database (get-tables-sql (exactly-named-schemas inclusion-patterns))))))(defmethod driver/describe-database* :redshift[driver database]diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.cljindex 5d18d77c9d2..17df5c0ded6 100644--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj@@ -5,12 +5,14 @@[clojure.string :as str][clojure.test :refer :all][metabase.driver :as driver]+ [metabase.driver.redshift :as redshift][metabase.driver.sql-jdbc :as driver.sql-jdbc][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table][metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync][metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.sync :as driver.s][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.test-util :as lib.tu]@@ -723,3 +725,86 @@;; None (special role in Postgres to revert back to login role; should not be quoted)"none" "SET SESSION AUTHORIZATION none;""NONE" "SET SESSION AUTHORIZATION NONE;"))))))++(deftest ^:parallel exactly-named-schemas-agrees-with-filter-test+ (testing "when the inclusion filter names its schemas outright, membership decides exactly what the filter keeps"+ (let [candidates ["spectrum"+ "2026_08_27_18_abc_schema"+ "2026_08_27_18_abc_schema_extra"+ "prefix_spectrum"+ "spectrumx"+ "SPECTRUM"+ "other"+ "with space"+ "withxspace"+ "hyphen-schema"+ "hyphenxschema"+ "unicodé"+ "unicode"+ "at@sign"+ "atxsign"]]+ (doseq [patterns ["spectrum"+ "spectrum,2026_08_27_18_abc_schema"+ " spectrum , 2026_08_27_18_abc_schema "+ "spectrum,spectrum"+ ;; legal Redshift schema names that carry no regex syntax+ "with space"+ "hyphen-schema"+ "unicodé"+ "at@sign"]]+ (testing (pr-str patterns)+ (let [named (#'redshift/exactly-named-schemas patterns)]+ (is (some? named))+ (doseq [candidate candidates]+ (is (= (driver.s/include-schema? patterns nil candidate)+ (contains? (set named) candidate))+ (pr-str candidate))))))))+ (testing "a filter that needs every schema to evaluate falls through to the unrestricted query"+ (are [patterns] (nil? (#'redshift/exactly-named-schemas patterns))+ nil+ ""+ " "+ "test*"+ "*_schema"+ "spectrum,test*"+ "crazy\\*schema"+ "a.c"+ "a|b"+ "a$b"+ "a+b"+ "a(b)"+ "a[b]"+ "a^b"+ "a?b"+ ;; an interior empty segment names nothing, so it cannot stand in for the filter+ "spectrum,,other"))+ (testing "a trailing comma leaves no empty segment behind, so it still qualifies"+ (is (= ["spectrum"] (#'redshift/exactly-named-schemas "spectrum,")))))++(deftest ^:parallel regex-metacharacters-is-complete-test+ (testing "every character the guard admits really does compile to a regex matching only itself"+ ;; Pins the metacharacter enumeration itself: a character missing from it would be admitted here and show up as+ ;; a disagreement, rather than as a silently wrong `in (...)` against a real cluster.+ (let [chars (concat (map char (range 32 127)) [\é \ü \空])+ ;; the filter splits on commas, so a comma never reaches a segment+ chars (remove #{\,} chars)]+ (doseq [c chars+ :let [segment (str "a" c "c")+ named (#'redshift/exactly-named-schemas segment)]+ :when named]+ (testing (pr-str segment)+ (doseq [candidate [segment "abc" "axc" "ac" "aXc" "a" "other"]]+ (is (= (boolean (driver.s/include-schema? segment nil candidate))+ (contains? (set named) candidate))+ (pr-str candidate))))))))++(deftest ^:parallel get-tables-sql-test+ (testing "named schemas are bound once per union branch"+ (let [[sql & params] (#'redshift/get-tables-sql ["spectrum" "sess"])]+ (is (= ["spectrum" "sess" "spectrum" "sess"] params))+ (is (= 2 (count (re-seq #"in \(\?, \?\)" sql))))))+ (testing "no named schemas leaves the query and its params untouched"+ (let [[sql & params] (#'redshift/get-tables-sql nil)]+ (is (empty? params))+ (is (not (str/includes? sql "nspname in")))+ (is (not (str/includes? sql "schemaname in"))))))