Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: skip temp and toast schema (#17996)
rebelice6c52b2bdaee3
diff --git a/backend/plugin/db/pg/sync.go b/backend/plugin/db/pg/sync.goindex ae4eeedd44..9879a1a672 100644--- a/backend/plugin/db/pg/sync.go+++ b/backend/plugin/db/pg/sync.go@@ -257,7 +257,9 @@ SELECT nsp.nspname, rel.relname, con.conname, pg_get_constraintdef(con.oid, trueFROM pg_catalog.pg_constraint conINNER JOIN pg_catalog.pg_class rel ON rel.oid = con.conrelidINNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = connamespace- WHERE contype = 'c' and ` + fmt.Sprintf(`nsp.nspname NOT IN (%s)`, pgparser.SystemSchemaWhereClause)+ WHERE contype = 'c' and ` + fmt.Sprintf(`nsp.nspname NOT IN (%s)+ AND nsp.nspname NOT LIKE 'pg_temp%%'+ AND nsp.nspname NOT LIKE 'pg_toast%%'`, pgparser.SystemSchemaWhereClause)func getChecks(txn *sql.Tx) (map[db.TableKey][]*storepb.CheckConstraintMetadata, error) {checksMap := make(map[db.TableKey][]*storepb.CheckConstraintMetadata)@@ -300,6 +302,8 @@ FROMJOIN pg_namespace n ON n.oid = c.connamespace` + fmt.Sprintf(`WHEREn.nspname NOT IN(%s)+ AND n.nspname NOT LIKE 'pg_temp%%'+ AND n.nspname NOT LIKE 'pg_toast%%'AND c.contype = 'f'AND c.conparentid = 0ORDER BY fk_schema, fk_table, fk_name;`, pgparser.SystemSchemaWhereClause)@@ -423,10 +427,12 @@ func formatTableNameFromRegclass(name string) string {}var listSchemaQuery = fmt.Sprintf(`-SELECT oid, nspname, pg_catalog.pg_get_userbyid(nspowner) as schema_owner,+SELECT oid, nspname, pg_catalog.pg_get_userbyid(nspowner) as schema_owner,obj_description(oid, 'pg_namespace') as schema_commentFROM pg_catalog.pg_namespaceWHERE nspname NOT IN (%s)+ AND nspname NOT LIKE 'pg_temp%%'+ AND nspname NOT LIKE 'pg_toast%%'ORDER BY nspname;`, pgparser.SystemSchemaWhereClause)@@ -486,7 +492,9 @@ func getListTableQuery(isAtLeastPG10 bool) string {tbl.tableownerFROM pg_catalog.pg_tables tblLEFT JOIN pg_class as pc ON pc.oid = format('%s.%s', quote_ident(tbl.schemaname), quote_ident(tbl.tablename))::regclass` + fmt.Sprintf(`- WHERE tbl.schemaname NOT IN (%s)%s+ WHERE tbl.schemaname NOT IN (%s)+ AND tbl.schemaname NOT LIKE 'pg_temp%%'+ AND tbl.schemaname NOT LIKE 'pg_toast%%'%sORDER BY tbl.schemaname, tbl.tablename;`, pgparser.SystemSchemaWhereClause, relisPartition)}@@ -611,6 +619,8 @@ WHERE((c.relkind = 'r'::"char") OR (c.relkind = 'f'::"char") OR (c.relkind = 'p'::"char"))AND c.relispartition IS TRUE ` + fmt.Sprintf(`AND n.nspname NOT IN (%s)+ AND n.nspname NOT LIKE 'pg_temp%%'+ AND n.nspname NOT LIKE 'pg_toast%%'ORDER BY c.oid;`, pgparser.SystemSchemaWhereClause)func getTablePartitions(txn *sql.Tx, indexMap map[db.TableKey][]*storepb.IndexMetadata, checksMap map[db.TableKey][]*storepb.CheckConstraintMetadata) (map[db.TableKey][]*storepb.TablePartitionMetadata, error) {@@ -725,6 +735,8 @@ SELECTpg_catalog.col_description(format('%s.%s', quote_ident(table_schema), quote_ident(table_name))::regclass, cols.ordinal_position::int) as column_commentFROM INFORMATION_SCHEMA.COLUMNS AS cols` + fmt.Sprintf(`WHERE cols.table_schema NOT IN (%s)+ AND cols.table_schema NOT LIKE 'pg_temp%%'+ AND cols.table_schema NOT LIKE 'pg_toast%%'ORDER BY cols.table_schema, cols.table_name, cols.ordinal_position;`, pgparser.SystemSchemaWhereClause)// getTableColumns gets the columns of a table.@@ -825,6 +837,8 @@ SELECT pc.oid, schemaname, matviewname, definition, obj_description(format('%s.%FROM pg_catalog.pg_matviewsLEFT JOIN pg_class as pc ON pc.oid = format('%s.%s', quote_ident(schemaname), quote_ident(matviewname))::regclass` + fmt.Sprintf(`WHERE schemaname NOT IN (%s)+ AND schemaname NOT LIKE 'pg_temp%%'+ AND schemaname NOT LIKE 'pg_toast%%'ORDER BY schemaname, matviewname;`, pgparser.SystemSchemaWhereClause)func getMaterializedViews(txn *sql.Tx, indexMap map[db.TableKey][]*storepb.IndexMetadata, triggerMap map[db.TableKey][]*storepb.TriggerMetadata, extensionDepend map[int]bool) (map[string][]*storepb.MaterializedViewMetadata, map[int]*db.TableKey, error) {@@ -889,6 +903,8 @@ SELECT pc.oid, schemaname, viewname, definition, obj_description(format('%s.%s',FROM pg_catalog.pg_viewsLEFT JOIN pg_class as pc ON pc.oid = format('%s.%s', quote_ident(schemaname), quote_ident(viewname))::regclass` + fmt.Sprintf(`WHERE schemaname NOT IN (%s)+ AND schemaname NOT LIKE 'pg_temp%%'+ AND schemaname NOT LIKE 'pg_toast%%'ORDER BY schemaname, viewname;`, pgparser.SystemSchemaWhereClause)// getViews gets all views of a database.@@ -1110,6 +1126,8 @@ func getEnumTypes(txn *sql.Tx, extensionDepend map[int]bool) (map[string][]*storLEFT JOIN pg_type as pt ON pe.enumtypid = pt.oidLEFT JOIN pg_namespace as pn ON pt.typnamespace = pn.oidWHERE pn.nspname NOT IN (%s)+ AND pn.nspname NOT LIKE 'pg_temp%%'+ AND pn.nspname NOT LIKE 'pg_toast%%'ORDER BY pn.nspname, pt.typname, pe.enumsortorder;`rows, err := txn.Query(fmt.Sprintf(query, pgparser.SystemSchemaWhereClause))if err != nil {@@ -1175,7 +1193,7 @@ func getSequences(txn *sql.Tx, tableOidMap map[int]*db.TableKeyWithColumns, extereturn nil, errors.Wrapf(err, "failed to get sequence owners")}- query := `+ query := fmt.Sprintf(`SELECTpc.oid,schemaname,@@ -1190,8 +1208,11 @@ func getSequences(txn *sql.Tx, tableOidMap map[int]*db.TableKeyWithColumns, extelast_value,pg_catalog.obj_description(pc.oid) as sequence_commentFROM pg_sequences- LEFT JOIN pg_class as pc ON pc.oid = format('%s.%s', quote_ident(schemaname), quote_ident(sequencename))::regclass- ORDER BY schemaname, sequencename;`+ LEFT JOIN pg_class as pc ON pc.oid = format('%%s.%%s', quote_ident(schemaname), quote_ident(sequencename))::regclass+ WHERE schemaname NOT IN (%s)+ AND schemaname NOT LIKE 'pg_temp%%%%'+ AND schemaname NOT LIKE 'pg_toast%%%%'+ ORDER BY schemaname, sequencename;`, pgparser.SystemSchemaWhereClause)rows, err := txn.Query(query)if err != nil {return nil, err@@ -1378,6 +1399,8 @@ JOIN pg_class t ON ix.indrelid = t.oidJOIN pg_namespace n ON t.relnamespace = n.oidJOIN pg_am am ON i.relam = am.oid` + fmt.Sprintf(`WHERE n.nspname NOT IN (%s)+ AND n.nspname NOT LIKE 'pg_temp%%'+ AND n.nspname NOT LIKE 'pg_toast%%'ORDER BY n.nspname, t.relname, i.relname;`, pgparser.SystemSchemaWhereClause)// parseIndexOptions parses PostgreSQL indoption int2vector to extract sort order information@@ -1499,7 +1522,10 @@ from pg_proc pleft join pg_depend d on p.oid = d.objidleft join pg_type pt on d.refobjid = pt.oidleft join pg_namespace n on p.pronamespace = n.oid` + fmt.Sprintf(`-where n.nspname not in (%s) AND pt.typrelid IS NOT NULL+where n.nspname not in (%s)+ AND n.nspname NOT LIKE 'pg_temp%%'+ AND n.nspname NOT LIKE 'pg_toast%%'+ AND pt.typrelid IS NOT NULL`, pgparser.SystemSchemaWhereClause)func getFunctionDependencyTables(txn *sql.Tx) (map[int][]int, error) {@@ -1539,6 +1565,8 @@ left join pg_namespace n on p.pronamespace = n.oidleft join pg_language l on p.prolang = l.oidleft join pg_type t on t.oid = p.prorettype ` + fmt.Sprintf(`where n.nspname not in (%s)+ AND n.nspname NOT LIKE 'pg_temp%%'+ AND n.nspname NOT LIKE 'pg_toast%%'order by function_schema, function_name;`, pgparser.SystemSchemaWhereClause)// getFunctions gets all functions of a database.diff --git a/backend/tests/sync_schema_test.go b/backend/tests/sync_schema_test.goindex 670b87d65d..186916f63b 100644--- a/backend/tests/sync_schema_test.go+++ b/backend/tests/sync_schema_test.go@@ -2,6 +2,7 @@ package testsimport ("context"+ "database/sql""fmt""testing"@@ -123,3 +124,108 @@ DROP SCHEMA IF EXISTS "schema_a";a.NoError(err)a.Equal(expectedDiff, diff)}++// TestSyncSchemaWithTempSchema tests that schema sync works correctly even when pg_temp schemas exist+// This reproduces the issue from SUP-3 where users get permission errors on pg_temp schemas+func TestSyncSchemaWithTempSchema(t *testing.T) {+ databaseName := "sync_schema_temp"++ t.Parallel()+ a := require.New(t)+ ctx := context.Background()+ ctl := &controller{}+ ctx, err := ctl.StartServerWithExternalPg(ctx)+ a.NoError(err)+ defer ctl.Close(ctx)++ pgContainer, err := getPgContainer(ctx)+ defer func() {+ pgContainer.Close(ctx)+ }()+ a.NoError(err)++ pgDB := pgContainer.db+ err = pgDB.Ping()+ a.NoError(err)++ // Create database and user+ _, err = pgDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %v", databaseName))+ a.NoError(err)+ _, err = pgDB.Exec(fmt.Sprintf("CREATE DATABASE %v", databaseName))+ a.NoError(err)++ // Create a limited user (not superuser) to simulate real-world scenario+ _, err = pgDB.Exec("DROP USER IF EXISTS testuser")+ a.NoError(err)+ _, err = pgDB.Exec("CREATE USER testuser WITH ENCRYPTED PASSWORD 'testpass'")+ a.NoError(err)+ _, err = pgDB.Exec(fmt.Sprintf("GRANT ALL PRIVILEGES ON DATABASE %v TO testuser", databaseName))+ a.NoError(err)++ // Connect to the test database to create temp tables and schemas+ testDB, err := sql.Open("pgx", fmt.Sprintf("host=%s port=%s user=postgres password=root-password dbname=%s sslmode=disable", pgContainer.host, pgContainer.port, databaseName))+ a.NoError(err)+ defer testDB.Close()++ // Create some regular schemas and tables+ _, err = testDB.Exec(`+ CREATE SCHEMA app;+ CREATE TABLE app.users (id INT PRIMARY KEY, name TEXT);+ GRANT USAGE ON SCHEMA app TO testuser;+ GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA app TO testuser;+ `)+ a.NoError(err)++ // Create temporary tables which will create pg_temp schemas+ // These temp tables will be in pg_temp_N schemas that testuser doesn't have permission to access+ _, err = testDB.Exec(`+ CREATE TEMP TABLE temp_table1 (id INT);+ CREATE TEMP TABLE temp_table2 (data TEXT);+ `)+ a.NoError(err)++ // Verify that pg_temp schemas exist+ var tempSchemaCount int+ err = testDB.QueryRow(`+ SELECT COUNT(*)+ FROM pg_namespace+ WHERE nspname LIKE 'pg_temp%'+ `).Scan(&tempSchemaCount)+ a.NoError(err)+ a.Greater(tempSchemaCount, 0, "pg_temp schemas should exist")++ // Now try to sync schema using the limited testuser account+ // This should NOT fail even though testuser doesn't have permissions on pg_temp schemas+ instanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("instance"),+ Instance: &v1pb.Instance{+ Title: "pgTestSyncSchemaWithTemp",+ Engine: v1pb.Engine_POSTGRES,+ Environment: stringPtr("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: pgContainer.host, Port: pgContainer.port, Username: "testuser", Password: "testpass", Id: "admin"}},+ },+ }))+ a.NoError(err)+ instance := instanceResp.Msg++ // Sync the database schema - this should succeed and ignore pg_temp schemas+ _, err = ctl.databaseServiceClient.SyncDatabase(ctx, connect.NewRequest(&v1pb.SyncDatabaseRequest{+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, databaseName),+ }))+ a.NoError(err, "Schema sync should succeed even with pg_temp schemas present")++ // Verify that we can get the database schema metadata without errors+ schemaResp, err := ctl.databaseServiceClient.GetDatabaseSchema(ctx, connect.NewRequest(&v1pb.GetDatabaseSchemaRequest{+ Name: fmt.Sprintf("%s/databases/%s/schema", instance.Name, databaseName),+ }))+ a.NoError(err)++ // Verify that the schema string doesn't contain pg_temp or pg_toast references+ a.NotContains(schemaResp.Msg.Schema, "pg_temp", "pg_temp schemas should be filtered out")+ a.NotContains(schemaResp.Msg.Schema, "pg_toast", "pg_toast schemas should be filtered out")++ // Verify that our regular schema is present in the result+ a.Contains(schemaResp.Msg.Schema, "app", "app schema should be present in synced metadata")+ a.Contains(schemaResp.Msg.Schema, "users", "users table should be present in synced metadata")+}