Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: support multi file dump (#17781)
rebelice5ac6c3ac5891
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.goindex 6032987b0d..46a020ebcf 100644--- a/backend/api/v1/database_service.go+++ b/backend/api/v1/database_service.go@@ -1,6 +1,8 @@package v1import (+ "archive/zip"+ "bytes""context""fmt""slices"@@ -1534,6 +1536,37 @@ func (*DatabaseService) getSingleFileSDL(_ context.Context, engine storepb.Engin}), nil}-func (*DatabaseService) getMultiFileSDL(_ context.Context, _ storepb.Engine, _ *storepb.DatabaseSchemaMetadata, _ string) (*connect.Response[v1pb.DatabaseSDLSchema], error) {- return nil, connect.NewError(connect.CodeUnimplemented, errors.Errorf("multi-file SDL format is not yet implemented"))+func (*DatabaseService) getMultiFileSDL(_ context.Context, engine storepb.Engine, metadata *storepb.DatabaseSchemaMetadata, _ string) (*connect.Response[v1pb.DatabaseSDLSchema], error) {+ // Get multi-file schema from schema package+ result, err := schema.GetMultiFileDatabaseDefinition(engine, schema.GetDefinitionContext{+ SkipBackupSchema: true,+ SDLFormat: true,+ MultiFileFormat: true,+ }, metadata)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to generate multi-file SDL schema: %v", err))+ }++ // Create ZIP archive+ var buf bytes.Buffer+ zipWriter := zip.NewWriter(&buf)++ for _, file := range result.Files {+ writer, err := zipWriter.Create(file.Name)+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create zip entry %s: %v", file.Name, err))+ }+ if _, err := writer.Write([]byte(file.Content)); err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to write content to %s: %v", file.Name, err))+ }+ }++ if err := zipWriter.Close(); err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to close zip writer: %v", err))+ }++ return connect.NewResponse(&v1pb.DatabaseSDLSchema{+ Schema: buf.Bytes(),+ ContentType: "application/zip",+ }), nil}diff --git a/backend/plugin/schema/pg/get_database_definition.go b/backend/plugin/schema/pg/get_database_definition.goindex dacb8c25ae..b18a937758 100644--- a/backend/plugin/schema/pg/get_database_definition.go+++ b/backend/plugin/schema/pg/get_database_definition.go@@ -40,6 +40,7 @@ func init() {schema.RegisterGetMaterializedViewDefinition(storepb.Engine_POSTGRES, GetMaterializedViewDefinition)schema.RegisterGetFunctionDefinition(storepb.Engine_POSTGRES, GetFunctionDefinition)schema.RegisterGetSequenceDefinition(storepb.Engine_POSTGRES, GetSequenceDefinition)+ schema.RegisterGetMultiFileDatabaseDefinition(storepb.Engine_POSTGRES, GetMultiFileDatabaseDefinition)}func GetDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (string, error) {@@ -3003,3 +3004,174 @@ func writeForeignKeyConstraintSDL(out io.Writer, fk *storepb.ForeignKeyMetadata)}return nil}++// GetMultiFileDatabaseDefinition generates multi-file SDL schema for PostgreSQL.+func GetMultiFileDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (*schema.MultiFileSchemaResult, error) {+ metadata = filterBackupSchemaIfNecessary(ctx, metadata)++ if len(metadata.Schemas) == 0 {+ return &schema.MultiFileSchemaResult{Files: []schema.File{}}, nil+ }++ var files []schema.File++ // Build skip sequences map (for serial and identity columns)+ skipSequences := buildSkipSequencesMap(metadata)++ // Build table sequences map for each table+ tableSequencesMap := buildTableSequencesMap(metadata)++ // Generate files for each schema+ for _, schemaMetadata := range metadata.Schemas {+ if schemaMetadata.SkipDump {+ continue+ }++ schemaName := schemaMetadata.Name+ if schemaName == "" {+ schemaName = "public"+ }++ // Generate sequence files (standalone sequences only)+ for _, sequence := range schemaMetadata.Sequences {+ if sequence.SkipDump {+ continue+ }+ // Skip sequences that belong to serial or identity columns+ sequenceKey := schemaName + "." + sequence.Name+ if skipSequences[sequenceKey] {+ continue+ }++ var buf strings.Builder+ if err := writeSequenceSDL(&buf, schemaName, sequence); err != nil {+ return nil, errors.Wrapf(err, "failed to generate sequence SDL for %s.%s", schemaName, sequence.Name)+ }+ files = append(files, schema.File{+ Name: fmt.Sprintf("schemas/%s/sequences/%s.sql", schemaName, sequence.Name),+ Content: buf.String() + ";\n",+ })+ }++ // Generate table files+ for _, table := range schemaMetadata.Tables {+ if table.SkipDump {+ continue+ }++ // Get sequences for this table+ tableKey := schemaName + "." + table.Name+ tableSequences := tableSequencesMap[tableKey]++ var buf strings.Builder+ // Write CREATE TABLE statement in SDL format+ if err := writeCreateTableSDL(&buf, schemaName, table, tableSequences); err != nil {+ return nil, errors.Wrapf(err, "failed to generate table SDL for %s.%s", schemaName, table.Name)+ }+ buf.WriteString(";\n\n")++ // Write CREATE INDEX statements for non-constraint indexes+ if err := writeIndexesSDL(&buf, schemaName, table); err != nil {+ return nil, errors.Wrapf(err, "failed to generate indexes SDL for %s.%s", schemaName, table.Name)+ }++ files = append(files, schema.File{+ Name: fmt.Sprintf("schemas/%s/tables/%s.sql", schemaName, table.Name),+ Content: buf.String(),+ })+ }++ // Generate view files+ for _, view := range schemaMetadata.Views {+ if view.SkipDump {+ continue+ }++ var buf strings.Builder+ if err := writeViewSDL(&buf, schemaName, view); err != nil {+ return nil, errors.Wrapf(err, "failed to generate view SDL for %s.%s", schemaName, view.Name)+ }+ files = append(files, schema.File{+ Name: fmt.Sprintf("schemas/%s/views/%s.sql", schemaName, view.Name),+ Content: buf.String() + ";\n",+ })+ }++ // Generate function files+ for _, function := range schemaMetadata.Functions {+ if function.SkipDump {+ continue+ }++ var buf strings.Builder+ if err := writeFunctionSDL(&buf, schemaName, function); err != nil {+ return nil, errors.Wrapf(err, "failed to generate function SDL for %s.%s", schemaName, function.Name)+ }+ files = append(files, schema.File{+ Name: fmt.Sprintf("schemas/%s/functions/%s.sql", schemaName, function.Name),+ Content: buf.String() + ";\n",+ })+ }+ }++ return &schema.MultiFileSchemaResult{Files: files}, nil+}++// buildSkipSequencesMap builds a map of sequences that should be skipped (serial and identity sequences).+func buildSkipSequencesMap(metadata *storepb.DatabaseSchemaMetadata) map[string]bool {+ skipSequences := make(map[string]bool)+ for _, schema := range metadata.Schemas {+ if schema.SkipDump {+ continue+ }+ for _, table := range schema.Tables {+ if table.SkipDump {+ continue+ }+ for _, column := range table.Columns {+ // Check for serial columns+ isSerial, _ := isSerialColumn(column, table.Name, schema.Sequences)+ if isSerial {+ for _, sequence := range schema.Sequences {+ if sequence.OwnerTable == table.Name && sequence.OwnerColumn == column.Name {+ sequenceKey := schema.Name + "." + sequence.Name+ skipSequences[sequenceKey] = true+ break+ }+ }+ }+ // Check for identity columns+ if isIdentityColumn(column) {+ for _, sequence := range schema.Sequences {+ if sequence.OwnerTable == table.Name && sequence.OwnerColumn == column.Name {+ sequenceKey := schema.Name + "." + sequence.Name+ skipSequences[sequenceKey] = true+ break+ }+ }+ }+ }+ }+ }+ return skipSequences+}++// buildTableSequencesMap builds a map of sequences by table for easy lookup during table creation.+func buildTableSequencesMap(metadata *storepb.DatabaseSchemaMetadata) map[string][]*storepb.SequenceMetadata {+ tableSequencesMap := make(map[string][]*storepb.SequenceMetadata)+ for _, schema := range metadata.Schemas {+ if schema.SkipDump {+ continue+ }+ for _, sequence := range schema.Sequences {+ if sequence.SkipDump {+ continue+ }+ if sequence.OwnerTable != "" {+ tableKey := schema.Name + "." + sequence.OwnerTable+ tableSequencesMap[tableKey] = append(tableSequencesMap[tableKey], sequence)+ }+ }+ }+ return tableSequencesMap+}diff --git a/backend/plugin/schema/schema.go b/backend/plugin/schema/schema.goindex 65060795cf..bcb472d11d 100644--- a/backend/plugin/schema/schema.go+++ b/backend/plugin/schema/schema.go@@ -11,21 +11,23 @@ import ()var (- mux sync.Mutex- getDatabaseDefinitions = make(map[storepb.Engine]getDatabaseDefinition)- getSchemaDefinitions = make(map[storepb.Engine]getSchemaDefinition)- getTableDefinitions = make(map[storepb.Engine]getTableDefinition)- getViewDefinitions = make(map[storepb.Engine]getViewDefinition)- getMaterializedViewDefinitions = make(map[storepb.Engine]getMaterializedViewDefinition)- getFunctionDefinitions = make(map[storepb.Engine]getFunctionDefinition)- getProcedureDefinitions = make(map[storepb.Engine]getProcedureDefinition)- getSequenceDefinitions = make(map[storepb.Engine]getSequenceDefinition)- getDatabaseMetadataMap = make(map[storepb.Engine]getDatabaseMetadata)- generateMigrations = make(map[storepb.Engine]generateMigration)- getSDLDiffs = make(map[storepb.Engine]getSDLDiff)+ mux sync.Mutex+ getDatabaseDefinitions = make(map[storepb.Engine]getDatabaseDefinition)+ getSchemaDefinitions = make(map[storepb.Engine]getSchemaDefinition)+ getTableDefinitions = make(map[storepb.Engine]getTableDefinition)+ getViewDefinitions = make(map[storepb.Engine]getViewDefinition)+ getMaterializedViewDefinitions = make(map[storepb.Engine]getMaterializedViewDefinition)+ getFunctionDefinitions = make(map[storepb.Engine]getFunctionDefinition)+ getProcedureDefinitions = make(map[storepb.Engine]getProcedureDefinition)+ getSequenceDefinitions = make(map[storepb.Engine]getSequenceDefinition)+ getDatabaseMetadataMap = make(map[storepb.Engine]getDatabaseMetadata)+ generateMigrations = make(map[storepb.Engine]generateMigration)+ getSDLDiffs = make(map[storepb.Engine]getSDLDiff)+ getMultiFileDatabaseDefinitions = make(map[storepb.Engine]getMultiFileDatabaseDefinition))type getDatabaseDefinition func(GetDefinitionContext, *storepb.DatabaseSchemaMetadata) (string, error)+type getMultiFileDatabaseDefinition func(GetDefinitionContext, *storepb.DatabaseSchemaMetadata) (*MultiFileSchemaResult, error)type getSchemaDefinition func(*storepb.SchemaMetadata) (string, error)type getTableDefinition func(string, *storepb.TableMetadata, []*storepb.SequenceMetadata) (string, error)type getViewDefinition func(string, *storepb.ViewMetadata) (string, error)@@ -41,6 +43,23 @@ type GetDefinitionContext struct {SkipBackupSchema boolPrintHeader boolSDLFormat bool+ // MultiFileFormat indicates whether to generate multi-file SDL output.+ // When true, the result should be organized as multiple files.+ MultiFileFormat bool+}++// File represents a single file in a multi-file schema output.+type File struct {+ // Name is the file path or name (e.g., "schemas/public/tables/users.sql")+ Name string+ // Content is the file content+ Content string+}++// MultiFileSchemaResult represents the result of multi-file schema generation.+type MultiFileSchemaResult struct {+ // Files is the list of schema files organized by type+ Files []File}func RegisterGetSequenceDefinition(engine storepb.Engine, f getSequenceDefinition) {@@ -229,3 +248,20 @@ func GetSDLDiff(engine storepb.Engine, currentSDLText, previousUserSDLText strin}return f(currentSDLText, previousUserSDLText, currentSchema, previousSchema)}++func RegisterGetMultiFileDatabaseDefinition(engine storepb.Engine, f getMultiFileDatabaseDefinition) {+ mux.Lock()+ defer mux.Unlock()+ if _, dup := getMultiFileDatabaseDefinitions[engine]; dup {+ panic(fmt.Sprintf("Register called twice %s", engine))+ }+ getMultiFileDatabaseDefinitions[engine] = f+}++func GetMultiFileDatabaseDefinition(engine storepb.Engine, ctx GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (*MultiFileSchemaResult, error) {+ f, ok := getMultiFileDatabaseDefinitions[engine]+ if !ok {+ return nil, errors.Errorf("engine %s is not supported for multi-file database definition", engine)+ }+ return f(ctx, metadata)+}