Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(schema/redshift): serve the table and view definitions GetSchemaString asks for (#21352)
Danny Xuf66052224d6b
diff --git a/backend/plugin/schema/redshift/get_database_definition.go b/backend/plugin/schema/redshift/get_database_definition.goindex fd027a3ef1..078d233636 100644--- a/backend/plugin/schema/redshift/get_database_definition.go+++ b/backend/plugin/schema/redshift/get_database_definition.go@@ -11,6 +11,26 @@ import (func init() {schema.RegisterGetDatabaseDefinition(storepb.Engine_REDSHIFT, GetDatabaseDefinition)+ schema.RegisterGetTableDefinition(storepb.Engine_REDSHIFT, GetTableDefinition)+ schema.RegisterGetViewDefinition(storepb.Engine_REDSHIFT, GetViewDefinition)+}++// GetTableDefinition renders one table exactly as the whole-database output+// renders it, minus the section banner.+func GetTableDefinition(schemaName string, table *storepb.TableMetadata, _ []*storepb.SequenceMetadata) (string, error) {+ var sb strings.Builder+ if err := writeTable(&sb, convertToTableState(0, schemaName, table)); err != nil {+ return "", err+ }+ return sb.String(), nil+}++func GetViewDefinition(schemaName string, view *storepb.ViewMetadata) (string, error) {+ var sb strings.Builder+ if err := writeView(&sb, convertToViewState(0, schemaName, view)); err != nil {+ return "", err+ }+ return sb.String(), nil}func GetDatabaseDefinition(_ schema.GetDefinitionContext, to *storepb.DatabaseSchemaMetadata) (string, error) {@@ -48,24 +68,7 @@ func writeTables(w io.StringWriter, to *storepb.DatabaseSchemaMetadata, state *dreturn err}- buf := &strings.Builder{}- if err := table.toString(buf); err != nil {- return err- }- // Generate comment for table and columns.- if table.comment != "" {- if _, err := fmt.Fprintf(buf, "COMMENT ON TABLE %s IS '%s';\n", table.name, table.comment); err != nil {- return err- }- }- for _, column := range table.columns {- if column.comment != "" {- if _, err := fmt.Fprintf(buf, "COMMENT ON COLUMN %s.%s IS '%s';\n", table.name, column.name, column.comment); err != nil {- return err- }- }- }- if _, err := w.WriteString(buf.String()); err != nil {+ if err := writeTable(w, table); err != nil {return err}delete(schemaState.tables, table.name)@@ -91,17 +94,7 @@ func writeViews(w io.StringWriter, to *storepb.DatabaseSchemaMetadata, state *dareturn err}- buf := &strings.Builder{}- if err := view.toString(buf); err != nil {- return err- }- // Generate comment for view.- if view.comment != "" {- if _, err := fmt.Fprintf(buf, "COMMENT ON VIEW %s IS '%s';\n", view.name, view.comment); err != nil {- return err- }- }- if _, err := w.WriteString(buf.String()); err != nil {+ if err := writeView(w, view); err != nil {return err}delete(schemaState.views, view.name)@@ -110,6 +103,46 @@ func writeViews(w io.StringWriter, to *storepb.DatabaseSchemaMetadata, state *dareturn nil}+// writeTable writes one table and the comments that belong to it. Column+// comments follow the column order the CREATE TABLE body uses; ranging over the+// map directly, as this did before, ordered them differently run to run.+func writeTable(w io.StringWriter, table *tableState) error {+ buf := &strings.Builder{}+ if err := table.toString(buf); err != nil {+ return err+ }+ if table.comment != "" {+ if _, err := fmt.Fprintf(buf, "COMMENT ON TABLE %s IS '%s';\n", table.qualifiedName(), escapeSingleQuote(table.comment)); err != nil {+ return err+ }+ }+ for _, column := range sortedColumns(table.columns) {+ if column.comment == "" {+ continue+ }+ if _, err := fmt.Fprintf(buf, "COMMENT ON COLUMN %s.%s IS '%s';\n", table.qualifiedName(), quoteIdentifier(column.name), escapeSingleQuote(column.comment)); err != nil {+ return err+ }+ }+ _, err := w.WriteString(buf.String())+ return err+}++// writeView writes one view and its comment.+func writeView(w io.StringWriter, view *viewState) error {+ buf := &strings.Builder{}+ if err := view.toString(buf); err != nil {+ return err+ }+ if view.comment != "" {+ if _, err := fmt.Fprintf(buf, "COMMENT ON VIEW %s IS '%s';\n", view.qualifiedName(), escapeSingleQuote(view.comment)); err != nil {+ return err+ }+ }+ _, err := w.WriteString(buf.String())+ return err+}+func getTableAnnouncement(name string) string {return fmt.Sprintf("\n--\n-- Table structure for `%s`\n--\n", name)}diff --git a/backend/plugin/schema/redshift/get_database_definition_test.go b/backend/plugin/schema/redshift/get_database_definition_test.gonew file mode 100644index 0000000000..2d6c1c3d32--- /dev/null+++ b/backend/plugin/schema/redshift/get_database_definition_test.go@@ -0,0 +1,283 @@+package redshift++import (+ "strings"+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/schema"+)++func TestGetObjectDefinition(t *testing.T) {+ tests := []struct {+ name string+ get func() (string, error)+ want string+ }{+ {+ name: "table with comments",+ get: func() (string, error) {+ return GetTableDefinition("public", &storepb.TableMetadata{+ Name: "orders",+ Comment: "one row per order",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "integer", Comment: "surrogate key"},+ {Name: "total", Type: "numeric(10,2)", Nullable: true},+ {Name: "label", Type: "character varying(50)", Nullable: true, Comment: "display name"},+ },+ }, nil)+ },+ want: `CREATE TABLE "public"."orders" (+ "id" integer NOT NULL,+ "total" numeric(10,2),+ "label" character varying(50)+);+COMMENT ON TABLE "public"."orders" IS 'one row per order';+COMMENT ON COLUMN "public"."orders"."id" IS 'surrogate key';+COMMENT ON COLUMN "public"."orders"."label" IS 'display name';+`,+ },+ {+ name: "view with comment",+ get: func() (string, error) {+ return GetViewDefinition("public", &storepb.ViewMetadata{+ Name: "recent_orders",+ Definition: "SELECT id, total FROM orders",+ Comment: "last 30 days",+ })+ },+ want: `CREATE OR REPLACE VIEW "public"."recent_orders" AS SELECT id, total FROM orders;+COMMENT ON VIEW "public"."recent_orders" IS 'last 30 days';+`,+ },+ {+ // GetSchemaString passes the requested schema, and a table outside+ // the search path has to carry it or the DDL lands somewhere else.+ name: "table outside the default schema keeps its qualifier",+ get: func() (string, error) {+ return GetTableDefinition("analytics", &storepb.TableMetadata{+ Name: "orders",+ Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}},+ }, nil)+ },+ want: `CREATE TABLE "analytics"."orders" (+ "id" integer NOT NULL+);+`,+ },+ {+ // An apostrophe would close the literal and produce DDL that will+ // not replay.+ name: "comments with apostrophes stay valid",+ get: func() (string, error) {+ return GetTableDefinition("public", &storepb.TableMetadata{+ Name: "orders",+ Comment: "owner's orders",+ Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer", Comment: "don't drop"}},+ }, nil)+ },+ want: `CREATE TABLE "public"."orders" (+ "id" integer NOT NULL+);+COMMENT ON TABLE "public"."orders" IS 'owner''s orders';+COMMENT ON COLUMN "public"."orders"."id" IS 'don''t drop';+`,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.get()+ require.NoError(t, err)+ require.Equal(t, tt.want, got)+ })+ }+}++// TestColumnCommentOrderIsStable pins the ordering of the trailing COMMENT ON+// COLUMN statements. They used to come from ranging over a map, so the dump+// differed between runs for any table with more than one commented column.+func TestColumnCommentOrderIsStable(t *testing.T) {+ table := &storepb.TableMetadata{Name: "wide"}+ for _, name := range []string{"a", "b", "c", "d", "e", "f", "g", "h"} {+ table.Columns = append(table.Columns, &storepb.ColumnMetadata{+ Name: name,+ Type: "integer",+ Nullable: true,+ Comment: "note " + name,+ })+ }++ first, err := GetTableDefinition("public", table, nil)+ require.NoError(t, err)+ require.Equal(t, 8, strings.Count(first, "COMMENT ON COLUMN"))++ for i := 0; i < 200; i++ {+ got, err := GetTableDefinition("public", table, nil)+ require.NoError(t, err)+ require.Equalf(t, first, got, "column comment order changed on iteration %d", i)+ }+}++// TestObjectDefinitionsRegisteredForRedshift goes through the schema registry+// rather than calling the functions directly, because the registry lookup is+// what was missing: GetSchemaString served only DATABASE for Redshift and+// returned "engine REDSHIFT is not supported" for TABLE and VIEW, both of which+// the SQL editor offers because supportGetStringSchema lists REDSHIFT.+func TestObjectDefinitionsRegisteredForRedshift(t *testing.T) {+ tests := []struct {+ name string+ get func() (string, error)+ }{+ {+ name: "table",+ get: func() (string, error) {+ return schema.GetTableDefinition(storepb.Engine_REDSHIFT, "public", &storepb.TableMetadata{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer"}},+ }, nil)+ },+ },+ {+ name: "view",+ get: func() (string, error) {+ return schema.GetViewDefinition(storepb.Engine_REDSHIFT, "public", &storepb.ViewMetadata{+ Name: "v",+ Definition: "SELECT 1",+ })+ },+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := tt.get()+ require.NoError(t, err)+ require.NotEmpty(t, got)+ })+ }+}++// TestGetDatabaseDefinitionKeepsSections guards the extraction above: the+// whole-database output still frames each object with its banner and renders+// the same body the single-object entry points return.+func TestGetDatabaseDefinitionKeepsSections(t *testing.T) {+ metadata := &storepb.DatabaseSchemaMetadata{+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "public",+ Tables: []*storepb.TableMetadata{+ {+ Name: "orders",+ Comment: "one row per order",+ Columns: []*storepb.ColumnMetadata{{Name: "id", Type: "integer", Comment: "surrogate key"}},+ },+ },+ Views: []*storepb.ViewMetadata{+ {Name: "recent_orders", Definition: "SELECT id FROM orders"},+ },+ },+ },+ }++ got, err := GetDatabaseDefinition(schema.GetDefinitionContext{}, metadata)+ require.NoError(t, err)++ require.Equal(t, `--+-- Table structure for `+"`orders`"+`+--+CREATE TABLE "public"."orders" (+ "id" integer NOT NULL+);+COMMENT ON TABLE "public"."orders" IS 'one row per order';+COMMENT ON COLUMN "public"."orders"."id" IS 'surrogate key';++--+-- View structure for `+"`recent_orders`"+`+--+CREATE OR REPLACE VIEW "public"."recent_orders" AS SELECT id FROM orders;+`, got)++ table, err := GetTableDefinition("public", metadata.Schemas[0].Tables[0], nil)+ require.NoError(t, err)+ require.Contains(t, got, table)+}++// TestForeignKeyKeepsReferencedSchema pins the qualifier on the REFERENCES+// target. The Redshift sync strips it off ReferencedTable and keeps it in+// ReferencedSchema, so dropping it binds the constraint to whatever the search+// path resolves instead of the table the metadata names.+func TestForeignKeyKeepsReferencedSchema(t *testing.T) {+ got, err := GetTableDefinition("analytics", &storepb.TableMetadata{+ Name: "child",+ Columns: []*storepb.ColumnMetadata{+ {Name: "id", Type: "integer"},+ {Name: "parent_id", Type: "integer", Nullable: true},+ },+ ForeignKeys: []*storepb.ForeignKeyMetadata{+ {+ Name: "child_parent_fk",+ Columns: []string{"parent_id"},+ ReferencedSchema: "warehouse",+ ReferencedTable: "parent",+ ReferencedColumns: []string{"id"},+ },+ },+ }, nil)+ require.NoError(t, err)+ require.Contains(t, got, `REFERENCES "warehouse"."parent"("id")`)+ require.NotContains(t, got, `REFERENCES "parent"("id")`)+}++// TestIdentifiersNeedingQuotesSurvive covers the names that unquoted output+// would corrupt: Redshift folds an unquoted identifier to lower case and rejects+// one containing a space, and an embedded double quote has to be doubled.+func TestIdentifiersNeedingQuotesSurvive(t *testing.T) {+ got, err := GetTableDefinition("Sales Data", &storepb.TableMetadata{+ Name: "MixedCase",+ Columns: []*storepb.ColumnMetadata{+ {Name: `od"d`, Type: "integer", Comment: "quoted name"},+ },+ }, nil)+ require.NoError(t, err)+ require.Equal(t, `CREATE TABLE "Sales Data"."MixedCase" (+ "od""d" integer NOT NULL+);+COMMENT ON COLUMN "Sales Data"."MixedCase"."od""d" IS 'quoted name';+`, got)+}++// TestIndexKeysAreVerbatimAndForeignKeyColumnsAreQuoted pins the split between+// the two: index expressions arrive from pg_get_indexdef(.., true) already+// quoted, while getColumnList strips the quotes off foreign key columns, so one+// must be written as-is and the other must be quoted.+func TestIndexKeysAreVerbatimAndForeignKeyColumnsAreQuoted(t *testing.T) {+ got, err := GetTableDefinition("public", &storepb.TableMetadata{+ Name: "t",+ Columns: []*storepb.ColumnMetadata{+ {Name: "MixedCase", Type: "integer"},+ {Name: "parent_id", Type: "integer", Nullable: true},+ },+ Indexes: []*storepb.IndexMetadata{+ {Name: "t_pkey", Primary: true, Expressions: []string{`"MixedCase"`}},+ },+ ForeignKeys: []*storepb.ForeignKeyMetadata{+ {+ Name: "t_parent_fk",+ Columns: []string{"parent_id"},+ ReferencedSchema: "public",+ ReferencedTable: "parent",+ ReferencedColumns: []string{"id"},+ },+ },+ }, nil)+ require.NoError(t, err)++ // The canonical expression passes through untouched, not re-quoted.+ require.Contains(t, got, `PRIMARY KEY ("MixedCase")`)+ require.NotContains(t, got, `"""MixedCase"""`)+ // Foreign key columns arrive bare and are quoted here.+ require.Contains(t, got, `FOREIGN KEY ("parent_id") REFERENCES "public"."parent"("id")`)+}diff --git a/backend/plugin/schema/redshift/state.go b/backend/plugin/schema/redshift/state.goindex d62dbd3dbb..e4525c71e0 100644--- a/backend/plugin/schema/redshift/state.go+++ b/backend/plugin/schema/redshift/state.go@@ -35,6 +35,38 @@ type schemaState struct {views map[string]*viewState}+// qualifiedName prefixes the object with its schema, so DDL for analytics.orders+// cannot land in public. The prefix is dropped when no schema is known, which is+// how the single-object entry points behave when the caller passes none.+func qualifyName(schemaName, objectName string) string {+ if schemaName == "" {+ return quoteIdentifier(objectName)+ }+ return quoteIdentifier(schemaName) + "." + quoteIdentifier(objectName)+}++// quoteIdentifier renders a name as a quoted SQL identifier. Redshift folds an+// unquoted name to lower case and rejects one containing a space or a reserved+// word outright, so a name the catalog reports verbatim has to be quoted to come+// back as itself. An embedded double quote is doubled, as in PostgreSQL.+func quoteIdentifier(name string) string {+ return `"` + strings.ReplaceAll(name, `"`, `""`) + `"`+}++func (t *tableState) qualifiedName() string {+ return qualifyName(t.schema, t.name)+}++func (v *viewState) qualifiedName() string {+ return qualifyName(v.schema, v.name)+}++// escapeSingleQuote encodes a comment for a SQL string literal; an apostrophe in+// "owner's orders" would otherwise close the literal and produce invalid DDL.+func escapeSingleQuote(s string) string {+ return strings.ReplaceAll(s, "'", "''")+}+func newSchemaState() *schemaState {return &schemaState{tables: make(map[string]*tableState),@@ -46,16 +78,17 @@ func convertToSchemaState(schema *storepb.SchemaMetadata) *schemaState {state := newSchemaState()state.name = schema.Namefor i, table := range schema.Tables {- state.tables[table.Name] = convertToTableState(i, table)+ state.tables[table.Name] = convertToTableState(i, schema.Name, table)}for i, view := range schema.Views {- state.views[view.Name] = convertToViewState(i, view)+ state.views[view.Name] = convertToViewState(i, schema.Name, view)}return state}type tableState struct {id int+ schema stringname stringcolumns map[string]*columnStateindexes map[string]*indexState@@ -64,22 +97,10 @@ type tableState struct {}func (t *tableState) toString(buf *strings.Builder) error {- if _, err := fmt.Fprintf(buf, "CREATE TABLE %s (\n ", t.name); err != nil {+ if _, err := fmt.Fprintf(buf, "CREATE TABLE %s (\n ", t.qualifiedName()); err != nil {return err}- columns := []*columnState{}- for _, column := range t.columns {- columns = append(columns, column)- }- slices.SortFunc(columns, func(a, b *columnState) int {- if a.id < b.id {- return -1- }- if a.id > b.id {- return 1- }- return 0- })+ columns := sortedColumns(t.columns)for i, column := range columns {if i > 0 {if _, err := buf.WriteString(",\n "); err != nil {@@ -163,8 +184,29 @@ func newTableState(id int, name string) *tableState {}}-func convertToTableState(id int, table *storepb.TableMetadata) *tableState {+// sortedColumns returns a table's columns in declaration order. Both the+// CREATE TABLE body and the trailing COMMENT ON COLUMN statements go through+// it, so the two agree and neither depends on map iteration order.+func sortedColumns(columns map[string]*columnState) []*columnState {+ sorted := make([]*columnState, 0, len(columns))+ for _, column := range columns {+ sorted = append(sorted, column)+ }+ slices.SortFunc(sorted, func(a, b *columnState) int {+ if a.id < b.id {+ return -1+ }+ if a.id > b.id {+ return 1+ }+ return 0+ })+ return sorted+}++func convertToTableState(id int, schemaName string, table *storepb.TableMetadata) *tableState {state := newTableState(id, table.Name)+ state.schema = schemaNamestate.comment = table.Commentfor i, column := range table.Columns {state.columns[column.Name] = convertToColumnState(i, column)@@ -182,6 +224,7 @@ type foreignKeyState struct {id intname stringcolumns []string+ referencedSchema stringreferencedTable stringreferencedColumns []string}@@ -191,6 +234,7 @@ func convertToForeignKeyState(id int, foreignKey *storepb.ForeignKeyMetadata) *fid: id,name: foreignKey.Name,columns: foreignKey.Columns,+ referencedSchema: foreignKey.ReferencedSchema,referencedTable: foreignKey.ReferencedTable,referencedColumns: foreignKey.ReferencedColumns,}@@ -201,14 +245,18 @@ func (f *foreignKeyState) toString(buf *strings.Builder) error {if _, err := buf.WriteString("FOREIGN KEY ("); err != nil {return err}- if _, err := buf.WriteString(column); err != nil {+ if _, err := buf.WriteString(quoteIdentifier(column)); err != nil {return err}if _, err := buf.WriteString(") REFERENCES "); err != nil {return err}referencedColumn := f.referencedColumns[i]- if _, err := fmt.Fprintf(buf, "%s(%s)", f.referencedTable, referencedColumn); err != nil {+ // The sync strips the qualifier off ReferencedTable and keeps it in+ // ReferencedSchema, so an unqualified target here would bind to whatever+ // the search path resolves rather than the table the constraint names.+ referenced := qualifyName(f.referencedSchema, f.referencedTable)+ if _, err := fmt.Fprintf(buf, "%s(%s)", referenced, quoteIdentifier(referencedColumn)); err != nil {return err}}@@ -248,6 +296,9 @@ func (i *indexState) toString(buf *strings.Builder) error {return err}}+ // Written verbatim: these come from pg_get_indexdef(.., true), which+ // already quotes what needs it and can be an expression rather than+ // a column, so quoting again would name something else.if _, err := buf.WriteString(key); err != nil {return err}@@ -265,6 +316,9 @@ func (i *indexState) toString(buf *strings.Builder) error {return err}}+ // Written verbatim: these come from pg_get_indexdef(.., true), which+ // already quotes what needs it and can be an expression rather than+ // a column, so quoting again would name something else.if _, err := buf.WriteString(key); err != nil {return err}@@ -313,7 +367,7 @@ type columnState struct {}func (c *columnState) toString(buf *strings.Builder) error {- if _, err := fmt.Fprintf(buf, "%s ", c.name); err != nil {+ if _, err := fmt.Fprintf(buf, "%s ", quoteIdentifier(c.name)); err != nil {return err}if _, err := buf.WriteString(c.tp); err != nil {@@ -369,14 +423,16 @@ func isExpression(value string) bool {type viewState struct {id int+ schema stringname stringdefinition stringcomment string}-func convertToViewState(id int, view *storepb.ViewMetadata) *viewState {+func convertToViewState(id int, schemaName string, view *storepb.ViewMetadata) *viewState {return &viewState{id: id,+ schema: schemaName,name: view.Name,definition: view.Definition,comment: view.Comment,@@ -384,7 +440,7 @@ func convertToViewState(id int, view *storepb.ViewMetadata) *viewState {}func (v *viewState) toString(buf io.StringWriter) error {- stmt := fmt.Sprintf("CREATE OR REPLACE VIEW %s AS %s", v.name, v.definition)+ stmt := fmt.Sprintf("CREATE OR REPLACE VIEW %s AS %s", v.qualifiedName(), v.definition)if !strings.HasSuffix(stmt, ";") {stmt += ";"}