Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: support multi file dump (#17781)

rebelice5ac6c3ac5891
diff --git a/backend/api/v1/database_service.go b/backend/api/v1/database_service.go
index 6032987b0d..46a020ebcf 100644
--- a/backend/api/v1/database_service.go
+++ b/backend/api/v1/database_service.go
@@ -1,6 +1,8 @@
package v1
import (
+ "archive/zip"
+ "bytes"
"context"
"fmt"
"slices"
@@ -1534,6 +1536,37 @@ func (*DatabaseService) getSingleFileSDL(_ context.Context, engine storepb.Engin
}), nil
}
-func (*DatabaseService) getMultiFileSDL(_ context.Context, _ storepb.Engine, _ *storepb.DatabaseSchemaMetadata, _ string) (*connect.Response[v1pb.DatabaseSDLSchema], error) {
- return nil, connect.NewError(connect.CodeUnimplemented, errors.Errorf("multi-file SDL format is not yet implemented"))
+func (*DatabaseService) getMultiFileSDL(_ context.Context, engine storepb.Engine, metadata *storepb.DatabaseSchemaMetadata, _ string) (*connect.Response[v1pb.DatabaseSDLSchema], error) {
+ // Get multi-file schema from schema package
+ result, err := schema.GetMultiFileDatabaseDefinition(engine, schema.GetDefinitionContext{
+ SkipBackupSchema: true,
+ SDLFormat: true,
+ MultiFileFormat: true,
+ }, metadata)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to generate multi-file SDL schema: %v", err))
+ }
+
+ // Create ZIP archive
+ var buf bytes.Buffer
+ zipWriter := zip.NewWriter(&buf)
+
+ for _, file := range result.Files {
+ writer, err := zipWriter.Create(file.Name)
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to create zip entry %s: %v", file.Name, err))
+ }
+ if _, err := writer.Write([]byte(file.Content)); err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to write content to %s: %v", file.Name, err))
+ }
+ }
+
+ if err := zipWriter.Close(); err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to close zip writer: %v", err))
+ }
+
+ return connect.NewResponse(&v1pb.DatabaseSDLSchema{
+ Schema: buf.Bytes(),
+ ContentType: "application/zip",
+ }), nil
}
diff --git a/backend/plugin/schema/pg/get_database_definition.go b/backend/plugin/schema/pg/get_database_definition.go
index dacb8c25ae..b18a937758 100644
--- a/backend/plugin/schema/pg/get_database_definition.go
+++ b/backend/plugin/schema/pg/get_database_definition.go
@@ -40,6 +40,7 @@ func init() {
schema.RegisterGetMaterializedViewDefinition(storepb.Engine_POSTGRES, GetMaterializedViewDefinition)
schema.RegisterGetFunctionDefinition(storepb.Engine_POSTGRES, GetFunctionDefinition)
schema.RegisterGetSequenceDefinition(storepb.Engine_POSTGRES, GetSequenceDefinition)
+ schema.RegisterGetMultiFileDatabaseDefinition(storepb.Engine_POSTGRES, GetMultiFileDatabaseDefinition)
}
func GetDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (string, error) {
@@ -3003,3 +3004,174 @@ func writeForeignKeyConstraintSDL(out io.Writer, fk *storepb.ForeignKeyMetadata)
}
return nil
}
+
+// GetMultiFileDatabaseDefinition generates multi-file SDL schema for PostgreSQL.
+func GetMultiFileDatabaseDefinition(ctx schema.GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (*schema.MultiFileSchemaResult, error) {
+ metadata = filterBackupSchemaIfNecessary(ctx, metadata)
+
+ if len(metadata.Schemas) == 0 {
+ return &schema.MultiFileSchemaResult{Files: []schema.File{}}, nil
+ }
+
+ var files []schema.File
+
+ // Build skip sequences map (for serial and identity columns)
+ skipSequences := buildSkipSequencesMap(metadata)
+
+ // Build table sequences map for each table
+ tableSequencesMap := buildTableSequencesMap(metadata)
+
+ // Generate files for each schema
+ for _, schemaMetadata := range metadata.Schemas {
+ if schemaMetadata.SkipDump {
+ continue
+ }
+
+ schemaName := schemaMetadata.Name
+ if schemaName == "" {
+ schemaName = "public"
+ }
+
+ // Generate sequence files (standalone sequences only)
+ for _, sequence := range schemaMetadata.Sequences {
+ if sequence.SkipDump {
+ continue
+ }
+ // Skip sequences that belong to serial or identity columns
+ sequenceKey := schemaName + "." + sequence.Name
+ if skipSequences[sequenceKey] {
+ continue
+ }
+
+ var buf strings.Builder
+ if err := writeSequenceSDL(&buf, schemaName, sequence); err != nil {
+ return nil, errors.Wrapf(err, "failed to generate sequence SDL for %s.%s", schemaName, sequence.Name)
+ }
+ files = append(files, schema.File{
+ Name: fmt.Sprintf("schemas/%s/sequences/%s.sql", schemaName, sequence.Name),
+ Content: buf.String() + ";\n",
+ })
+ }
+
+ // Generate table files
+ for _, table := range schemaMetadata.Tables {
+ if table.SkipDump {
+ continue
+ }
+
+ // Get sequences for this table
+ tableKey := schemaName + "." + table.Name
+ tableSequences := tableSequencesMap[tableKey]
+
+ var buf strings.Builder
+ // Write CREATE TABLE statement in SDL format
+ if err := writeCreateTableSDL(&buf, schemaName, table, tableSequences); err != nil {
+ return nil, errors.Wrapf(err, "failed to generate table SDL for %s.%s", schemaName, table.Name)
+ }
+ buf.WriteString(";\n\n")
+
+ // Write CREATE INDEX statements for non-constraint indexes
+ if err := writeIndexesSDL(&buf, schemaName, table); err != nil {
+ return nil, errors.Wrapf(err, "failed to generate indexes SDL for %s.%s", schemaName, table.Name)
+ }
+
+ files = append(files, schema.File{
+ Name: fmt.Sprintf("schemas/%s/tables/%s.sql", schemaName, table.Name),
+ Content: buf.String(),
+ })
+ }
+
+ // Generate view files
+ for _, view := range schemaMetadata.Views {
+ if view.SkipDump {
+ continue
+ }
+
+ var buf strings.Builder
+ if err := writeViewSDL(&buf, schemaName, view); err != nil {
+ return nil, errors.Wrapf(err, "failed to generate view SDL for %s.%s", schemaName, view.Name)
+ }
+ files = append(files, schema.File{
+ Name: fmt.Sprintf("schemas/%s/views/%s.sql", schemaName, view.Name),
+ Content: buf.String() + ";\n",
+ })
+ }
+
+ // Generate function files
+ for _, function := range schemaMetadata.Functions {
+ if function.SkipDump {
+ continue
+ }
+
+ var buf strings.Builder
+ if err := writeFunctionSDL(&buf, schemaName, function); err != nil {
+ return nil, errors.Wrapf(err, "failed to generate function SDL for %s.%s", schemaName, function.Name)
+ }
+ files = append(files, schema.File{
+ Name: fmt.Sprintf("schemas/%s/functions/%s.sql", schemaName, function.Name),
+ Content: buf.String() + ";\n",
+ })
+ }
+ }
+
+ return &schema.MultiFileSchemaResult{Files: files}, nil
+}
+
+// buildSkipSequencesMap builds a map of sequences that should be skipped (serial and identity sequences).
+func buildSkipSequencesMap(metadata *storepb.DatabaseSchemaMetadata) map[string]bool {
+ skipSequences := make(map[string]bool)
+ for _, schema := range metadata.Schemas {
+ if schema.SkipDump {
+ continue
+ }
+ for _, table := range schema.Tables {
+ if table.SkipDump {
+ continue
+ }
+ for _, column := range table.Columns {
+ // Check for serial columns
+ isSerial, _ := isSerialColumn(column, table.Name, schema.Sequences)
+ if isSerial {
+ for _, sequence := range schema.Sequences {
+ if sequence.OwnerTable == table.Name && sequence.OwnerColumn == column.Name {
+ sequenceKey := schema.Name + "." + sequence.Name
+ skipSequences[sequenceKey] = true
+ break
+ }
+ }
+ }
+ // Check for identity columns
+ if isIdentityColumn(column) {
+ for _, sequence := range schema.Sequences {
+ if sequence.OwnerTable == table.Name && sequence.OwnerColumn == column.Name {
+ sequenceKey := schema.Name + "." + sequence.Name
+ skipSequences[sequenceKey] = true
+ break
+ }
+ }
+ }
+ }
+ }
+ }
+ return skipSequences
+}
+
+// buildTableSequencesMap builds a map of sequences by table for easy lookup during table creation.
+func buildTableSequencesMap(metadata *storepb.DatabaseSchemaMetadata) map[string][]*storepb.SequenceMetadata {
+ tableSequencesMap := make(map[string][]*storepb.SequenceMetadata)
+ for _, schema := range metadata.Schemas {
+ if schema.SkipDump {
+ continue
+ }
+ for _, sequence := range schema.Sequences {
+ if sequence.SkipDump {
+ continue
+ }
+ if sequence.OwnerTable != "" {
+ tableKey := schema.Name + "." + sequence.OwnerTable
+ tableSequencesMap[tableKey] = append(tableSequencesMap[tableKey], sequence)
+ }
+ }
+ }
+ return tableSequencesMap
+}
diff --git a/backend/plugin/schema/schema.go b/backend/plugin/schema/schema.go
index 65060795cf..bcb472d11d 100644
--- a/backend/plugin/schema/schema.go
+++ b/backend/plugin/schema/schema.go
@@ -11,21 +11,23 @@ import (
)
var (
- mux sync.Mutex
- getDatabaseDefinitions = make(map[storepb.Engine]getDatabaseDefinition)
- getSchemaDefinitions = make(map[storepb.Engine]getSchemaDefinition)
- getTableDefinitions = make(map[storepb.Engine]getTableDefinition)
- getViewDefinitions = make(map[storepb.Engine]getViewDefinition)
- getMaterializedViewDefinitions = make(map[storepb.Engine]getMaterializedViewDefinition)
- getFunctionDefinitions = make(map[storepb.Engine]getFunctionDefinition)
- getProcedureDefinitions = make(map[storepb.Engine]getProcedureDefinition)
- getSequenceDefinitions = make(map[storepb.Engine]getSequenceDefinition)
- getDatabaseMetadataMap = make(map[storepb.Engine]getDatabaseMetadata)
- generateMigrations = make(map[storepb.Engine]generateMigration)
- getSDLDiffs = make(map[storepb.Engine]getSDLDiff)
+ mux sync.Mutex
+ getDatabaseDefinitions = make(map[storepb.Engine]getDatabaseDefinition)
+ getSchemaDefinitions = make(map[storepb.Engine]getSchemaDefinition)
+ getTableDefinitions = make(map[storepb.Engine]getTableDefinition)
+ getViewDefinitions = make(map[storepb.Engine]getViewDefinition)
+ getMaterializedViewDefinitions = make(map[storepb.Engine]getMaterializedViewDefinition)
+ getFunctionDefinitions = make(map[storepb.Engine]getFunctionDefinition)
+ getProcedureDefinitions = make(map[storepb.Engine]getProcedureDefinition)
+ getSequenceDefinitions = make(map[storepb.Engine]getSequenceDefinition)
+ getDatabaseMetadataMap = make(map[storepb.Engine]getDatabaseMetadata)
+ generateMigrations = make(map[storepb.Engine]generateMigration)
+ getSDLDiffs = make(map[storepb.Engine]getSDLDiff)
+ getMultiFileDatabaseDefinitions = make(map[storepb.Engine]getMultiFileDatabaseDefinition)
)
type getDatabaseDefinition func(GetDefinitionContext, *storepb.DatabaseSchemaMetadata) (string, error)
+type getMultiFileDatabaseDefinition func(GetDefinitionContext, *storepb.DatabaseSchemaMetadata) (*MultiFileSchemaResult, error)
type getSchemaDefinition func(*storepb.SchemaMetadata) (string, error)
type getTableDefinition func(string, *storepb.TableMetadata, []*storepb.SequenceMetadata) (string, error)
type getViewDefinition func(string, *storepb.ViewMetadata) (string, error)
@@ -41,6 +43,23 @@ type GetDefinitionContext struct {
SkipBackupSchema bool
PrintHeader bool
SDLFormat bool
+ // MultiFileFormat indicates whether to generate multi-file SDL output.
+ // When true, the result should be organized as multiple files.
+ MultiFileFormat bool
+}
+
+// File represents a single file in a multi-file schema output.
+type File struct {
+ // Name is the file path or name (e.g., "schemas/public/tables/users.sql")
+ Name string
+ // Content is the file content
+ Content string
+}
+
+// MultiFileSchemaResult represents the result of multi-file schema generation.
+type MultiFileSchemaResult struct {
+ // Files is the list of schema files organized by type
+ Files []File
}
func RegisterGetSequenceDefinition(engine storepb.Engine, f getSequenceDefinition) {
@@ -229,3 +248,20 @@ func GetSDLDiff(engine storepb.Engine, currentSDLText, previousUserSDLText strin
}
return f(currentSDLText, previousUserSDLText, currentSchema, previousSchema)
}
+
+func RegisterGetMultiFileDatabaseDefinition(engine storepb.Engine, f getMultiFileDatabaseDefinition) {
+ mux.Lock()
+ defer mux.Unlock()
+ if _, dup := getMultiFileDatabaseDefinitions[engine]; dup {
+ panic(fmt.Sprintf("Register called twice %s", engine))
+ }
+ getMultiFileDatabaseDefinitions[engine] = f
+}
+
+func GetMultiFileDatabaseDefinition(engine storepb.Engine, ctx GetDefinitionContext, metadata *storepb.DatabaseSchemaMetadata) (*MultiFileSchemaResult, error) {
+ f, ok := getMultiFileDatabaseDefinitions[engine]
+ if !ok {
+ return nil, errors.Errorf("engine %s is not supported for multi-file database definition", engine)
+ }
+ return f(ctx, metadata)
+}