Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(oracle): sync object tables, invisible and virtual columns (#21674)
rebelicefa4306ecce50
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.goindex d9910e420d..e1d15fa14b 100644--- a/backend/common/testcontainer/shared.go+++ b/backend/common/testcontainer/shared.go@@ -4,6 +4,7 @@ import ("context""database/sql""fmt"+ "strings""sync""sync/atomic""testing"@@ -23,6 +24,7 @@ var (sharedMSSQL = startOnce(getMSSQLContainer)sharedTiDB = startOnce(getTiDBContainer)sharedMongo = startOnce(getMongoDBContainer)+ sharedOracle = startOnce(getOracleContainer)sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))// 10.7 is the oldest MariaDB release Bytebase supports.sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))@@ -114,6 +116,28 @@ func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMa// oldest supported release; tests take a database each with NewMariaDB107Database.func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, sharedMariaDB107) }+// SharedOracleContainer is SharedPgContainer for Oracle; tests take a schema+// each with NewOracleSchema. Its admin user is a DBA.+func SharedOracleContainer(t testing.TB) *Container { return shared(t, sharedOracle) }++// NewOracleSchema creates an empty schema for the test on the shared Oracle+// container and returns the container with the schema's name. An Oracle schema+// is a user; this one may own the tables and materialized views the admin+// creates in it.+func NewOracleSchema(t testing.TB) (*Container, string) {+ t.Helper()+ container := SharedOracleContainer(t)+ name := strings.ToUpper(newDatabaseName())+ for _, statement := range []string{+ fmt.Sprintf(`CREATE USER %s IDENTIFIED BY "%s" QUOTA UNLIMITED ON USERS`, name, container.GetPassword()),+ fmt.Sprintf(`GRANT CREATE SESSION, CREATE TABLE, CREATE MATERIALIZED VIEW TO %s`, name),+ } {+ _, err := container.GetDB().Exec(statement)+ require.NoError(t, err)+ }+ return container, name+}+// NewPgDatabase creates an empty database for the test on the shared// PostgreSQL container and returns its name with a superuser handle to it.// Anything cluster-wide the test creates, such as a role, should be nameddiff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.goindex 8bace1dbf8..965e06e048 100644--- a/backend/common/testcontainer/testcontainer.go+++ b/backend/common/testcontainer/testcontainer.go@@ -15,10 +15,12 @@ import ("net""os""path/filepath"+ "strconv""strings""time""github.com/pkg/errors"+ goora "github.com/sijms/go-ora/v2""github.com/testcontainers/testcontainers-go""github.com/testcontainers/testcontainers-go/wait")@@ -353,6 +355,32 @@ func getMongoDBContainer(ctx context.Context) (*Container, error) {})}+// OracleServiceName is the pluggable database the Oracle container serves.+const OracleServiceName = "FREEPDB1"++// getOracleContainer starts Oracle AI Database 26ai Free. The connect timeout+// makes a listener that never answers fail each ping instead of hanging it.+func getOracleContainer(ctx context.Context) (*Container, error) {+ const password = "test-Passw0rd"+ return start(ctx, testcontainers.ContainerRequest{+ Image: "gvenzl/oracle-free:23.26.3-slim-faststart",+ Env: map[string]string{"ORACLE_PASSWORD": password},+ WaitingFor: wait.ForLog("DATABASE IS READY TO USE!").WithStartupTimeout(5 * time.Minute),+ }, spec{+ port: "1521/tcp",+ username: "system",+ password: password,+ driver: "oracle",+ dsn: func(c *Container) string {+ port, err := strconv.Atoi(c.port)+ if err != nil {+ return ""+ }+ return goora.BuildUrl(c.host, port, OracleServiceName, c.username, c.password, map[string]string{"CONNECTION TIMEOUT": "10"})+ },+ })+}+// createTLSMaterial issues a server certificate for localhost and ips from a// fresh CA, and writes the CA to a temporary directory the caller removes.func createTLSMaterial(ips ...net.IP) (string, string, []byte, []byte, error) {diff --git a/backend/plugin/db/oracle/main_test.go b/backend/plugin/db/oracle/main_test.gonew file mode 100644index 0000000000..e8a5706f97--- /dev/null+++ b/backend/plugin/db/oracle/main_test.go@@ -0,0 +1,9 @@+package oracle++import (+ "testing"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+)++func TestMain(m *testing.M) { testcontainer.Main(m) }diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.goindex c9f5d40c6e..eb6a9b447e 100644--- a/backend/plugin/db/oracle/sync.go+++ b/backend/plugin/db/oracle/sync.go@@ -13,6 +13,7 @@ import ("github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util""github.com/bytebase/bytebase/backend/plugin/parser/plsql"@@ -101,7 +102,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMeif err != nil {return nil, errors.Wrapf(err, "failed to get triggers from database %q", d.databaseName)}- tableMap, err := getTables(txn, d.databaseName, columnMap, tableTriggerMap)+ indexMap, checkConstraintMap, foreignKeyMap, err := getIndexesAndConstraints(txn, d.databaseName)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get indices")+ }+ tableMap, err := getTables(txn, d.databaseName, columnMap, indexMap, checkConstraintMap, foreignKeyMap, tableTriggerMap)if err != nil {return nil, errors.Wrapf(err, "failed to get tables from database %q", d.databaseName)}@@ -121,7 +126,7 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMeif err != nil {return nil, errors.Wrapf(err, "failed to get routines from database %q", d.databaseName)}- materializedViews, err := getMaterializedViews(txn, d.databaseName, columnMap)+ materializedViews, err := getMaterializedViews(txn, d.databaseName, indexMap, tableTriggerMap)if err != nil {return nil, errors.Wrapf(err, "failed to get materialized views from database %q", d.databaseName)}@@ -235,11 +240,13 @@ func getTriggers(txn *sql.Tx, schemaName string) (map[db.TableKey][]*metadatapb.return nil, nil, errors.Wrapf(err, "failed to get trigger comments")}+ // A trigger the schema owns on another schema's table is left out: keyed by+ // TABLE_NAME alone, it would land on this schema's table of the same name.query := fmt.Sprintf(`SELECT TRIGGER_NAME, TABLE_NAME, DESCRIPTION, TRIGGER_BODY, BASE_OBJECT_TYPE, TRIGGER_TYPE, TRIGGERING_EVENTFROM sys.all_triggers- WHERE OWNER = '%s' AND TABLE_NAME is NOT NULL AND (BASE_OBJECT_TYPE = 'TABLE' OR BASE_OBJECT_TYPE = 'VIEW')- ORDER BY TRIGGER_NAME`, schemaName)+ WHERE OWNER = '%s' AND TABLE_OWNER = '%s' AND TABLE_NAME is NOT NULL AND (BASE_OBJECT_TYPE = 'TABLE' OR BASE_OBJECT_TYPE = 'VIEW')+ ORDER BY TRIGGER_NAME`, schemaName, schemaName)slog.Debug("running get triggers query")rows, err := txn.Query(query)@@ -301,11 +308,15 @@ func constructTriggerBody(description, triggerBody string) string {}// getTables gets all tables of a database.-func getTables(txn *sql.Tx, schemaName string, columnMap map[db.TableKey][]*metadatapb.ColumnMetadata, triggerMap map[db.TableKey][]*metadatapb.TriggerMetadata) (map[string][]*metadatapb.TableMetadata, error) {- indexMap, checkConstraintMap, foreignKeyMap, err := getIndexesAndConstraints(txn, schemaName)- if err != nil {- return nil, errors.Wrapf(err, "failed to get indices")- }+func getTables(+ txn *sql.Tx,+ schemaName string,+ columnMap map[db.TableKey][]*metadatapb.ColumnMetadata,+ indexMap map[db.TableKey][]*metadatapb.IndexMetadata,+ checkConstraintMap map[db.TableKey][]*metadatapb.CheckConstraintMetadata,+ foreignKeyMap map[db.TableKey][]*metadatapb.ForeignKeyMetadata,+ triggerMap map[db.TableKey][]*metadatapb.TriggerMetadata,+) (map[string][]*metadatapb.TableMetadata, error) {columnCommentMap, err := getTableColumnComments(txn, schemaName)if err != nil {return nil, errors.Wrapf(err, "failed to get table column comments")@@ -323,6 +334,41 @@ func getTables(txn *sql.Tx, schemaName string, columnMap map[db.TableKey][]*metareturn nil, errors.Wrapf(err, "failed to get table comments")}tableMap := make(map[string][]*metadatapb.TableMetadata)+ appendTables := func(query string, skipDump bool) error {+ rows, err := txn.Query(query)+ if err != nil {+ return util.FormatErrorWithQuery(err, query)+ }+ defer rows.Close()++ for rows.Next() {+ table := &metadatapb.TableMetadata{SkipDump: skipDump}+ var schemaName string+ // https://github.com/rana/ora/issues/57#issuecomment-179909837+ // NUMBER in Oracle can hold 38 decimal digits, so int64 is not enough with its 19 decimal digits.+ // float64 is a little bit better - not precise enough, but won't overflow.+ var count sql.NullFloat64+ if err := rows.Scan(&schemaName, &table.Name, &count); err != nil {+ return err+ }+ table.RowCount = int64(count.Float64)+ key := db.TableKey{Schema: schemaName, Table: table.Name}+ table.Columns = columnMap[key]+ table.Indexes = indexMap[key]+ table.CheckConstraints = checkConstraintMap[key]+ table.ForeignKeys = foreignKeyMap[key]+ table.Triggers = triggerMap[key]+ if comment, ok := tableCommentMap[db.TableKey{Schema: schemaName, Table: table.Name}]; ok {+ table.Comment = comment+ }++ tableMap[schemaName] = append(tableMap[schemaName], table)+ }+ if err := rows.Err(); err != nil {+ return util.FormatErrorWithQuery(err, query)+ }+ return rows.Close()+ }query := fmt.Sprintf(`SELECT OWNER, TABLE_NAME, NUM_ROWS@@ -332,42 +378,27 @@ func getTables(txn *sql.Tx, schemaName string, columnMap map[db.TableKey][]*metaSELECT MVIEW_NAME FROM all_mviews WHERE OWNER = '%s')ORDER BY TABLE_NAME`, schemaName, schemaName)-slog.Debug("running get tables query")- rows, err := txn.Query(query)- if err != nil {- return nil, util.FormatErrorWithQuery(err, query)+ if err := appendTables(query, false); err != nil {+ return nil, err}- defer rows.Close()-- for rows.Next() {- table := &metadatapb.TableMetadata{}- var schemaName string- // https://github.com/rana/ora/issues/57#issuecomment-179909837- // NUMBER in Oracle can hold 38 decimal digits, so int64 is not enough with its 19 decimal digits.- // float64 is a little bit better - not precise enough, but won't overflow.- var count sql.NullFloat64- if err := rows.Scan(&schemaName, &table.Name, &count); err != nil {- return nil, err- }- table.RowCount = int64(count.Float64)- key := db.TableKey{Schema: schemaName, Table: table.Name}- table.Columns = columnMap[key]- table.Indexes = indexMap[key]- table.CheckConstraints = checkConstraintMap[key]- table.ForeignKeys = foreignKeyMap[key]- table.Triggers = triggerMap[key]- if comment, ok := tableCommentMap[db.TableKey{Schema: schemaName, Table: table.Name}]; ok {- table.Comment = comment- }- tableMap[schemaName] = append(tableMap[schemaName], table)- }- if err := rows.Err(); err != nil {- return nil, util.FormatErrorWithQuery(err, query)- }- if err := rows.Close(); err != nil {- return nil, errors.Wrapf(err, "failed to close rows")+ // ALL_TABLES omits object and XMLType tables (CREATE TABLE ... OF), which+ // DROP TABLE drops all the same, so they are listed too, with SkipDump. The+ // ALL_MVIEWS subquery drops an object materialized view's container and keeps+ // the plan on the owner index: without it Oracle full-scans OBJ$, which holds+ // every schema's objects.+ objectTableQuery := fmt.Sprintf(`+ SELECT OWNER, TABLE_NAME, NUM_ROWS+ FROM all_object_tables+ WHERE OWNER = '%s'+ AND TABLE_NAME NOT IN (+ SELECT MVIEW_NAME FROM all_mviews WHERE OWNER = '%s'+ )+ ORDER BY TABLE_NAME`, schemaName, schemaName)+ slog.Debug("running get object tables query")+ if err := appendTables(objectTableQuery, true); err != nil {+ return nil, err}return tableMap, nil@@ -460,9 +491,12 @@ func getTableColumnComments(txn *sql.Tx, schemaName string) (map[db.ColumnKey]stfunc getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (map[db.TableKey][]*metadatapb.ColumnMetadata, error) {columnsMap := make(map[db.TableKey][]*metadatapb.ColumnMetadata)- // https://github.com/bytebase/bytebase/issues/6663- // Invisible columns don't have column ID so that we need to filter out them.+ // Invisible columns have no COLUMN_ID. They sort after the visible columns+ // and take the positions that follow. ALL_TAB_COLUMNS is ALL_TAB_COLS with+ // USER_GENERATED = 'YES', which leaves out hidden system columns; the newer+ // query reads ALL_TAB_COLS under that filter to get VIRTUAL_COLUMN too.query := ""+ var virtualColumns map[db.ColumnKey]bool// https://docs.oracle.com/en/database/oracle/oracle-database/12.2/refrn/ALL_TAB_COLS.html#GUID-85036F42-140A-406B-BE11-0AC49A00DBA3equalOrHigherThan12c2release := version.First > 12 || (version.First == 12 && version.Second >= 2)if equalOrHigherThan12c2release {@@ -479,11 +513,19 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (maDATA_DEFAULT,NULLABLE,COLLATION,- DEFAULT_ON_NULL- FROM sys.all_tab_columns- WHERE OWNER = '%s' AND COLUMN_ID IS NOT NULL- ORDER BY TABLE_NAME, COLUMN_ID`, schemaName)+ DEFAULT_ON_NULL,+ VIRTUAL_COLUMN+ FROM sys.all_tab_cols+ WHERE OWNER = '%s' AND USER_GENERATED = 'YES'+ ORDER BY TABLE_NAME, COLUMN_ID NULLS LAST, COLUMN_NAME`, schemaName)} else {+ // Failing to tell virtual columns apart only keeps their expressions+ // in Default, as before, so it must not fail the sync.+ var err error+ virtualColumns, err = getVirtualColumns(txn, schemaName)+ if err != nil {+ slog.Warn("failed to list Oracle virtual columns", slog.String("schema", schemaName), log.BBError(err))+ }query = fmt.Sprintf(`SELECTOWNER,@@ -497,10 +539,11 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (maDATA_DEFAULT,NULLABLE,NULL,+ NULL,NULLFROM sys.all_tab_columns- WHERE OWNER = '%s' AND COLUMN_ID IS NOT NULL- ORDER BY TABLE_NAME, COLUMN_ID`, schemaName)+ WHERE OWNER = '%s'+ ORDER BY TABLE_NAME, COLUMN_ID NULLS LAST, COLUMN_NAME`, schemaName)}slog.Debug("running get columns query")@@ -512,8 +555,9 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (mafor rows.Next() {column := &metadatapb.ColumnMetadata{}var schemaName, tableName, nullable string- var defaultStr, collation, defaultOnNull sql.NullString+ var defaultStr, collation, defaultOnNull, virtualColumn sql.NullStringvar dataLength, dataPrecision, dataScale sql.NullInt64+ var columnID sql.NullInt32if err := rows.Scan(&schemaName,&tableName,@@ -522,11 +566,12 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (ma&dataLength,&dataPrecision,&dataScale,- &column.Position,+ &columnID,&defaultStr,&nullable,&collation,&defaultOnNull,+ &virtualColumn,); err != nil {return nil, err}@@ -534,8 +579,15 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (maif defaultStr.Valid {// Clean up the default expression to remove current schema name and fix NEXTVAL syntaxcleanedDefault := cleanDefaultExpression(defaultStr.String, schemaName)- // Store in Default field (migration from DefaultExpression to Default)- column.Default = cleanedDefault+ // DATA_DEFAULT holds a virtual column's expression.+ if virtualColumn.String == "YES" || virtualColumns[db.ColumnKey{Schema: schemaName, Table: tableName, Column: column.Name}] {+ column.Generation = &metadatapb.GenerationMetadata{+ Type: metadatapb.GenerationMetadata_TYPE_VIRTUAL,+ Expression: cleanedDefault,+ }+ } else {+ column.Default = cleanedDefault+ }}isNullBool, err := util.ConvertYesNo(nullable)if err != nil {@@ -550,6 +602,15 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (ma}key := db.TableKey{Schema: schemaName, Table: tableName}+ if columnID.Valid {+ column.Position = columnID.Int32+ } else {+ column.IsInvisible = true+ column.Position = 1+ if columns := columnsMap[key]; len(columns) > 0 {+ column.Position = columns[len(columns)-1].Position + 1+ }+ }columnsMap[key] = append(columnsMap[key], column)}if err := rows.Err(); err != nil {@@ -562,6 +623,33 @@ func getTableColumns(txn *sql.Tx, schemaName string, version *plsql.Version) (mareturn columnsMap, nil}+// getVirtualColumns lists the virtual columns for versions whose column query+// reads ALL_TAB_COLUMNS, which has no VIRTUAL_COLUMN.+func getVirtualColumns(txn *sql.Tx, schemaName string) (map[db.ColumnKey]bool, error) {+ query := fmt.Sprintf(`+ SELECT TABLE_NAME, COLUMN_NAME+ FROM sys.all_tab_cols+ WHERE OWNER = '%s' AND VIRTUAL_COLUMN = 'YES'`, schemaName)+ slog.Debug("running get virtual columns query")+ rows, err := txn.Query(query)+ if err != nil {+ return nil, util.FormatErrorWithQuery(err, query)+ }+ defer rows.Close()+ virtualColumns := make(map[db.ColumnKey]bool)+ for rows.Next() {+ var tableName, columnName string+ if err := rows.Scan(&tableName, &columnName); err != nil {+ return nil, err+ }+ virtualColumns[db.ColumnKey{Schema: schemaName, Table: tableName, Column: columnName}] = true+ }+ if err := rows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, query)+ }+ return virtualColumns, nil+}+func getTypeString(dataType string, dataLength, dataPrecision, dataScale sql.NullInt64) string {switch dataType {case "VARCHAR2", "CHAR":@@ -881,8 +969,8 @@ func getIndexesAndConstraints(txn *sql.Tx, schemaName string) (map[db.TableKey][continue}- // Skip system-generated indexes for materialized views and virtual columns- if strings.HasPrefix(index.Name, "I_SNAP$_") || strings.HasPrefix(index.Name, "SYS_") {+ // Skip system-generated indexes, such as those of LOBs, IOTs, and nested tables.+ if strings.HasPrefix(index.Name, "SYS_") {continue}@@ -1111,7 +1199,7 @@ func getAllMaterializedViewDependencies(txn *sql.Tx, schemaName string) (map[db.}// getMaterializedViews gets all materialized views of a database.-func getMaterializedViews(txn *sql.Tx, schemaName string, _ map[db.TableKey][]*metadatapb.ColumnMetadata) ([]*metadatapb.MaterializedViewMetadata, error) {+func getMaterializedViews(txn *sql.Tx, schemaName string, indexMap map[db.TableKey][]*metadatapb.IndexMetadata, triggerMap map[db.TableKey][]*metadatapb.TriggerMetadata) ([]*metadatapb.MaterializedViewMetadata, error) {var materializedViews []*metadatapb.MaterializedViewMetadata// Get materialized view comments@@ -1157,6 +1245,9 @@ func getMaterializedViews(txn *sql.Tx, schemaName string, _ map[db.TableKey][]*mif comment, ok := materializedViewCommentMap[key]; ok {materializedView.Comment = comment}+ // Indexes and triggers are on the container table, which has the view's name.+ materializedView.Indexes = indexMap[key]+ materializedView.Triggers = triggerMap[key]materializedViews = append(materializedViews, materializedView)}diff --git a/backend/plugin/db/oracle/sync_oracle_test.go b/backend/plugin/db/oracle/sync_oracle_test.gonew file mode 100644index 0000000000..d8ad2f4006--- /dev/null+++ b/backend/plugin/db/oracle/sync_oracle_test.go@@ -0,0 +1,216 @@+package oracle++import (+ "context"+ "database/sql"+ "fmt"+ "slices"+ "testing"+ "time"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+ "github.com/bytebase/bytebase/backend/plugin/parser/plsql"+)++// newSyncTarget returns a driver opened on a fresh schema the way Bytebase+// opens one for a database: as the admin, with CURRENT_SCHEMA set to it.+func newSyncTarget(ctx context.Context, t *testing.T) (*Driver, *sql.DB, string) {+ t.Helper()+ container, schemaName := testcontainer.NewOracleSchema(t)+ opened, err := newDriver().Open(ctx, storepb.Engine_ORACLE, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Host: container.GetHost(),+ Port: container.GetPort(),+ Username: container.GetUsername(),+ ServiceName: testcontainer.OracleServiceName,+ },+ Password: container.GetPassword(),+ ConnectionContext: db.ConnectionContext{DatabaseName: schemaName},+ })+ require.NoError(t, err)+ t.Cleanup(func() { _ = opened.Close(context.Background()) })+ driver, ok := opened.(*Driver)+ require.True(t, ok)+ instance, err := driver.SyncInstance(ctx)+ require.NoError(t, err)+ driver.connectionCtx.EngineVersion = instance.Version+ return driver, container.GetDB(), schemaName+}++func syncTestContext(t *testing.T) context.Context {+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Minute)+ t.Cleanup(cancel)+ return ctx+}++func execAll(ctx context.Context, t *testing.T, conn *sql.DB, statements ...string) {+ t.Helper()+ for _, statement := range statements {+ _, err := conn.ExecContext(ctx, statement)+ require.NoError(t, err, statement)+ }+}++func syncedSchema(ctx context.Context, t *testing.T, driver *Driver) *metadatapb.SchemaMetadata {+ t.Helper()+ metadata, err := driver.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, metadata.Schemas, 1)+ return metadata.Schemas[0]+}++func tableByName(t *testing.T, schemaMetadata *metadatapb.SchemaMetadata, name string) *metadatapb.TableMetadata {+ t.Helper()+ index := slices.IndexFunc(schemaMetadata.Tables, func(table *metadatapb.TableMetadata) bool { return table.Name == name })+ require.NotEqual(t, -1, index, "table %s", name)+ return schemaMetadata.Tables[index]+}++func names[T interface{ GetName() string }](items []T) []string {+ var result []string+ for _, item := range items {+ result = append(result, item.GetName())+ }+ return result+}++// syncedColumn is the part of a column these tests assert on.+type syncedColumn struct {+ Name string+ Position int32+ Invisible bool+ Nullable bool+ Default string+ Expression string+}++func syncedColumns(columns []*metadatapb.ColumnMetadata) []syncedColumn {+ var result []syncedColumn+ for _, column := range columns {+ result = append(result, syncedColumn{+ Name: column.Name,+ Position: column.Position,+ Invisible: column.IsInvisible,+ Nullable: column.Nullable,+ Default: column.Default,+ Expression: column.GetGeneration().GetExpression(),+ })+ }+ return result+}++func TestSyncListsObjectAndXMLTypeTables(t *testing.T) {+ t.Parallel()+ ctx := syncTestContext(t)+ driver, admin, s := newSyncTarget(ctx, t)+ execAll(ctx, t, admin,+ fmt.Sprintf(`CREATE TYPE %s.TY AS OBJECT (X NUMBER, Y VARCHAR2(10))`, s),+ fmt.Sprintf(`CREATE TABLE %[1]s.OT OF %[1]s.TY (CONSTRAINT OT_PK PRIMARY KEY (X))`, s),+ fmt.Sprintf(`CREATE INDEX %[1]s.OT_Y ON %[1]s.OT (Y)`, s),+ fmt.Sprintf(`CREATE TABLE %s.XT OF XMLTYPE`, s),+ fmt.Sprintf(`CREATE TABLE %s.REL (A NUMBER)`, s),+ )++ synced := syncedSchema(ctx, t, driver)+ require.ElementsMatch(t, []string{"OT", "REL", "XT"}, names(synced.Tables))+ require.False(t, tableByName(t, synced, "REL").SkipDump)++ objectTable := tableByName(t, synced, "OT")+ require.True(t, objectTable.SkipDump)+ require.Equal(t, []string{"X", "Y"}, names(objectTable.Columns))+ // The rest is the SYS_C unique constraint on the hidden object identifier.+ require.Subset(t, names(objectTable.Indexes), []string{"OT_PK", "OT_Y"})++ xmlTable := tableByName(t, synced, "XT")+ require.True(t, xmlTable.SkipDump)+ require.Equal(t, []string{"SYS_NC_ROWINFO$"}, names(xmlTable.Columns))+}++func TestSyncListsInvisibleColumns(t *testing.T) {+ t.Parallel()+ ctx := syncTestContext(t)+ driver, admin, s := newSyncTarget(ctx, t)+ execAll(ctx, t, admin,+ fmt.Sprintf(`CREATE TABLE %s.T (+ A NUMBER,+ H NUMBER INVISIBLE,+ B VARCHAR2(10),+ HV NUMBER INVISIBLE DEFAULT 7 NOT NULL,+ VC AS (A+1) VIRTUAL,+ IHV INVISIBLE AS (A*2) VIRTUAL,+ U NUMBER+ )`, s),+ // Each of these adds a hidden system column that must stay out.+ fmt.Sprintf(`CREATE INDEX %[1]s.T_FBI ON %[1]s.T (UPPER(B))`, s),+ fmt.Sprintf(`ALTER TABLE %s.T SET UNUSED COLUMN U`, s),+ fmt.Sprintf(`BEGIN DBMS_OUTPUT.PUT_LINE(DBMS_STATS.CREATE_EXTENDED_STATS('%s', 'T', '(A, B)')); END;`, s),+ fmt.Sprintf(`CREATE VIEW %[1]s.V (A, B INVISIBLE) AS SELECT A, B FROM %[1]s.T`, s),+ )+ want := []syncedColumn{+ {Name: "A", Position: 1, Nullable: true},+ {Name: "B", Position: 2, Nullable: true},+ {Name: "VC", Position: 3, Nullable: true, Expression: `"A"+1`},+ {Name: "H", Position: 4, Invisible: true, Nullable: true},+ {Name: "HV", Position: 5, Invisible: true, Default: "7"},+ {Name: "IHV", Position: 6, Invisible: true, Nullable: true, Expression: `"A"*2`},+ }++ synced := syncedSchema(ctx, t, driver)+ require.Equal(t, want, syncedColumns(tableByName(t, synced, "T").Columns))+ require.Len(t, synced.Views, 1)+ require.Equal(t, []syncedColumn{+ {Name: "A", Position: 1, Nullable: true},+ {Name: "B", Position: 2, Invisible: true, Nullable: true},+ }, syncedColumns(synced.Views[0].Columns))++ // Before 12.2 the columns come from ALL_TAB_COLUMNS and the virtual ones+ // from a query of their own.+ txn, err := driver.db.BeginTx(ctx, nil)+ require.NoError(t, err)+ defer txn.Rollback()+ columnMap, err := getTableColumns(txn, s, &plsql.Version{First: 12, Second: 1})+ require.NoError(t, err)+ require.Equal(t, want, syncedColumns(columnMap[db.TableKey{Schema: s, Table: "T"}]))+}++func TestSyncAttachesMaterializedViewIndexesAndTriggers(t *testing.T) {+ t.Parallel()+ ctx := syncTestContext(t)+ driver, admin, s := newSyncTarget(ctx, t)+ execAll(ctx, t, admin,+ fmt.Sprintf(`CREATE TABLE %s.BASE (ID NUMBER CONSTRAINT BASE_PK PRIMARY KEY, V NUMBER)`, s),+ fmt.Sprintf(`CREATE MATERIALIZED VIEW LOG ON %s.BASE WITH PRIMARY KEY, ROWID`, s),+ // A fast-refreshed ROWID materialized view gets the index I_SNAP$_MV.+ fmt.Sprintf(`CREATE MATERIALIZED VIEW %[1]s.MV REFRESH FAST WITH ROWID AS SELECT ID, V FROM %[1]s.BASE`, s),+ fmt.Sprintf(`CREATE INDEX %[1]s.MV_V ON %[1]s.MV (V)`, s),+ fmt.Sprintf(`CREATE TRIGGER %[1]s.MV_TRG BEFORE INSERT ON %[1]s.MV FOR EACH ROW BEGIN NULL; END;`, s),+ )++ synced := syncedSchema(ctx, t, driver)+ require.NotContains(t, names(synced.Tables), "MV")+ require.Len(t, synced.MaterializedViews, 1)+ materializedView := synced.MaterializedViews[0]+ require.ElementsMatch(t, []string{"I_SNAP$_MV", "MV_V"}, names(materializedView.Indexes))+ require.Equal(t, []string{"MV_TRG"}, names(materializedView.Triggers))+}++func TestSyncLeavesOutTriggersOnOtherSchemasTables(t *testing.T) {+ t.Parallel()+ ctx := syncTestContext(t)+ driver, admin, s := newSyncTarget(ctx, t)+ _, other := testcontainer.NewOracleSchema(t)+ execAll(ctx, t, admin,+ fmt.Sprintf(`CREATE TABLE %s.T (A NUMBER)`, s),+ fmt.Sprintf(`CREATE TABLE %s.T (A NUMBER)`, other),+ fmt.Sprintf(`CREATE TRIGGER %[1]s.ON_OWN BEFORE INSERT ON %[1]s.T FOR EACH ROW BEGIN NULL; END;`, s),+ fmt.Sprintf(`CREATE TRIGGER %s.ON_OTHER BEFORE INSERT ON %s.T FOR EACH ROW BEGIN NULL; END;`, s, other),+ )++ synced := syncedSchema(ctx, t, driver)+ require.Equal(t, []string{"ON_OWN"}, names(tableByName(t, synced, "T").Triggers))+}diff --git a/backend/plugin/parser/base/span.go b/backend/plugin/parser/base/span.goindex 9860631826..30efb4ff27 100644--- a/backend/plugin/parser/base/span.go+++ b/backend/plugin/parser/base/span.go@@ -127,6 +127,9 @@ type QuerySpanResult struct {SourceFieldPaths map[string][]*PathAST// SelectAsterisk indicates whether the field is selected by asterisk, used by Cosmos DB.SelectAsterisk bool+ // Invisible marks a table column that SELECT * leaves out, such as an+ // Oracle INVISIBLE column. A reference by name still resolves it.+ Invisible bool}// ColumnResource is the resource key for a column.@@ -279,6 +282,8 @@ type PhysicalTable struct {Name string// Columns are the columns of the table.Columns []string+ // InvisibleColumns are the Columns that SELECT * leaves out.+ InvisibleColumns map[string]bool}func (p *PhysicalTable) GetTableName() string {@@ -312,6 +317,7 @@ func (p *PhysicalTable) GetQuerySpanResult() []QuerySpanResult {Name: column,SourceColumns: sourceColumnSet,IsPlainField: true,+ Invisible: p.InvisibleColumns[column],})}return resultdiff --git a/backend/plugin/parser/plsql/query_span_extractor.go b/backend/plugin/parser/plsql/query_span_extractor.goindex ec703334d6..53ee3b9810 100644--- a/backend/plugin/parser/plsql/query_span_extractor.go+++ b/backend/plugin/parser/plsql/query_span_extractor.go@@ -3,6 +3,7 @@ package plsqlimport ("context""reflect"+ "slices""strings""unicode"@@ -218,14 +219,19 @@ func (q *querySpanExtractor) findTableSchemaInMetadata(instanceID string, dbMetaif table != nil {var columns []string+ invisibleColumns := make(map[string]bool)for _, column := range table.GetProto().GetColumns() {columns = append(columns, column.Name)+ if column.IsInvisible {+ invisibleColumns[column.Name] = true+ }}return &base.PhysicalTable{- Server: "",- Database: databaseName,- Name: table.GetProto().Name,- Columns: columns,+ Server: "",+ Database: databaseName,+ Name: table.GetProto().Name,+ Columns: columns,+ InvisibleColumns: invisibleColumns,}, nil}@@ -1503,7 +1509,18 @@ func (q *omniQuerySpanExtractor) getOmniFieldColumnSource(schemaName, tableName,return base.SourceColumnSet{}}+// expandOmniAsterisk returns the columns * stands for. It leaves out invisible+// columns, as Oracle does; including one would shift every result after it,+// and the masking applied to each.func (q *omniQuerySpanExtractor) expandOmniAsterisk(schemaName, tableName string) ([]base.QuerySpanResult, error) {+ results, err := q.expandOmniAsteriskWithInvisible(schemaName, tableName)+ if err != nil {+ return nil, err+ }+ return slices.DeleteFunc(results, func(result base.QuerySpanResult) bool { return result.Invisible }), nil+}++func (q *omniQuerySpanExtractor) expandOmniAsteriskWithInvisible(schemaName, tableName string) ([]base.QuerySpanResult, error) {findInTableSource := func(tableSource base.TableSource) ([]base.QuerySpanResult, bool) {if schemaName != "" && schemaName != tableSource.GetDatabaseName() {return nil, false@@ -1662,6 +1679,7 @@ func cloneQuerySpanResult(result base.QuerySpanResult) base.QuerySpanResult {Name: result.Name,SourceColumns: cloneSourceColumnSet(result.SourceColumns),IsPlainField: result.IsPlainField,+ Invisible: result.Invisible,}}diff --git a/backend/plugin/parser/plsql/restore.go b/backend/plugin/parser/plsql/restore.goindex d963068b95..710a707471 100644--- a/backend/plugin/parser/plsql/restore.go+++ b/backend/plugin/parser/plsql/restore.go@@ -186,7 +186,8 @@ func (g *generator) generateDelete() {func (g *generator) restorableColumns() []string {var columns []stringfor _, column := range g.table.GetProto().GetColumns() {- if column.GetGeneration() != nil {+ // The backup is copied with SELECT t.*, which has no invisible columns.+ if column.GetGeneration() != nil || column.IsInvisible {continue}columns = append(columns, column.Name)diff --git a/backend/plugin/parser/plsql/restore_test.go b/backend/plugin/parser/plsql/restore_test.goindex 621f3076f6..3e568a5c83 100644--- a/backend/plugin/parser/plsql/restore_test.go+++ b/backend/plugin/parser/plsql/restore_test.go@@ -215,6 +215,15 @@ func fixedMockDatabaseMetadataGetter(_ context.Context, _ string, database strin{Name: "",Tables: []*metadatapb.TableMetadata{+ {+ Name: "T_HIDDEN",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "A"},+ {Name: "H", IsInvisible: true},+ {Name: "V", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_VIRTUAL, Expression: `"A"+1`}},+ {Name: "B"},+ },+ },{Name: "T_GENERATED",Columns: []*metadatapb.ColumnMetadata{diff --git a/backend/plugin/parser/plsql/test-data/query_span.yaml b/backend/plugin/parser/plsql/test-data/query_span.yamlindex 77043cb476..58665a5416 100644--- a/backend/plugin/parser/plsql/test-data/query_span.yaml+++ b/backend/plugin/parser/plsql/test-data/query_span.yaml@@ -2351,3 +2351,170 @@table: Tcolumn: ""predicatecolumns: []+- description: SELECT * leaves out invisible columns+ statement: SELECT * FROM T_HIDDEN;+ defaultDatabase: PUBLIC+ metadata: |-+ {+ "name": "PUBLIC",+ "schemas": [+ {+ "name": "",+ "tables": [+ {+ "name": "T_HIDDEN",+ "columns": [+ {+ "name": "A"+ },+ {+ "name": "H",+ "isInvisible": true+ },+ {+ "name": "B"+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 1+ results:+ - name: A+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: A+ isplainfield: true+ sourcefieldpaths: []+ selectasterisk: false+ - name: B+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: B+ isplainfield: true+ sourcefieldpaths: []+ selectasterisk: false+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: ""+ predicatecolumns: []+- description: A reference by name reaches an invisible column+ statement: SELECT H FROM T_HIDDEN;+ defaultDatabase: PUBLIC+ metadata: |-+ {+ "name": "PUBLIC",+ "schemas": [+ {+ "name": "",+ "tables": [+ {+ "name": "T_HIDDEN",+ "columns": [+ {+ "name": "A"+ },+ {+ "name": "H",+ "isInvisible": true+ },+ {+ "name": "B"+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 1+ results:+ - name: H+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: H+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: false+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: ""+ predicatecolumns: []+- description: SELECT * keeps an invisible column a subquery selects+ statement: SELECT * FROM (SELECT A, H FROM T_HIDDEN);+ defaultDatabase: PUBLIC+ metadata: |-+ {+ "name": "PUBLIC",+ "schemas": [+ {+ "name": "",+ "tables": [+ {+ "name": "T_HIDDEN",+ "columns": [+ {+ "name": "A"+ },+ {+ "name": "H",+ "isInvisible": true+ },+ {+ "name": "B"+ }+ ]+ }+ ]+ }+ ]+ }+ querySpan:+ type: 1+ results:+ - name: A+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: A+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: false+ - name: H+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: H+ isplainfield: false+ sourcefieldpaths: []+ selectasterisk: false+ sourcecolumns:+ - server: ""+ database: PUBLIC+ schema: ""+ table: T_HIDDEN+ column: ""+ predicatecolumns: []diff --git a/backend/plugin/parser/plsql/test-data/test_restore.yaml b/backend/plugin/parser/plsql/test-data/test_restore.yamlindex 9884fdc3ce..9db0e16bcb 100644--- a/backend/plugin/parser/plsql/test-data/test_restore.yaml+++ b/backend/plugin/parser/plsql/test-data/test_restore.yaml@@ -120,3 +120,14 @@UPDATE SET t."C1" = b."C1"WHEN NOT MATCHED THENINSERT ("A", "B", "C") VALUES (b."A", b."B", b."C");+- input: DELETE FROM T_HIDDEN WHERE a = 1;+ backupdatabase: bbarchive+ backuptable: prefix_T_HIDDEN+ originaldatabase: DB+ originaltable: T_HIDDEN+ result: |-+ /*+ Original SQL:+ DELETE FROM T_HIDDEN WHERE a = 1+ */+ INSERT INTO "DB"."T_HIDDEN" ("A", "B") SELECT "A", "B" FROM "bbarchive"."prefix_T_HIDDEN";diff --git a/backend/plugin/schema/differ.go b/backend/plugin/schema/differ.goindex 52c0d717c7..e406ca39af 100644--- a/backend/plugin/schema/differ.go+++ b/backend/plugin/schema/differ.go@@ -25,6 +25,12 @@ type MetadataDiff struct {// Database level changesDatabaseName string+ // OldDatabase and NewDatabase are the metadata the diff was taken between.+ // A generator reads them for objects the diff leaves alone, such as another+ // table's foreign key on a key it must rebuild.+ OldDatabase *model.DatabaseMetadata+ NewDatabase *model.DatabaseMetadata+// Schema changesSchemaChanges []*SchemaDiff@@ -317,6 +323,8 @@ func GetDatabaseSchemaDiff(engine storepb.Engine, oldSchema, newSchema *model.Dadiff := &MetadataDiff{DatabaseName: newMetadata.Name,+ OldDatabase: oldSchema,+ NewDatabase: newSchema,}// Use the internal DatabaseMetadata structures for efficient access@@ -2261,6 +2269,8 @@ func FilterPostgresArchiveSchema(diff *MetadataDiff) *MetadataDiff {// Create a new diff object with filtered changesfiltered := &MetadataDiff{DatabaseName: diff.DatabaseName,+ OldDatabase: diff.OldDatabase,+ NewDatabase: diff.NewDatabase,}// Filter schema changesdiff --git a/backend/plugin/schema/oracle/generate_migration.go b/backend/plugin/schema/oracle/generate_migration.goindex a50fef2be9..6cd1be329c 100644--- a/backend/plugin/schema/oracle/generate_migration.go+++ b/backend/plugin/schema/oracle/generate_migration.go@@ -1,15 +1,18 @@package oracleimport (+ "cmp""fmt""slices""strings"metadatapb "github.com/bytebase/omni/metadata"+ "google.golang.org/protobuf/proto"storepb "github.com/bytebase/bytebase/backend/generated-go/store"parserbase "github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/plugin/schema"+ "github.com/bytebase/bytebase/backend/store/model")func init() {@@ -612,7 +615,9 @@ func createObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) {for _, tableDiff := range tableMap {if tableDiff.Action == schema.MetadataDiffActionCreate && tableDiff.NewTable != nil {for _, fk := range tableDiff.NewTable.ForeignKeys {- writeMigrationForeignKey(buf, tableDiff.SchemaName, tableDiff.TableName, fk)+ if !waitsForRebuild(diff, tableDiff.SchemaName, fk) {+ writeMigrationForeignKey(buf, tableDiff.SchemaName, tableDiff.TableName, fk)+ }}}}@@ -666,7 +671,9 @@ func createObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) {for _, tableDiff := range tableMap {if tableDiff.Action == schema.MetadataDiffActionCreate && tableDiff.NewTable != nil {for _, fk := range tableDiff.NewTable.ForeignKeys {- writeMigrationForeignKey(buf, tableDiff.SchemaName, tableDiff.TableName, fk)+ if !waitsForRebuild(diff, tableDiff.SchemaName, fk) {+ writeMigrationForeignKey(buf, tableDiff.SchemaName, tableDiff.TableName, fk)+ }}}}@@ -679,12 +686,7 @@ func createObjectsInOrder(diff *schema.MetadataDiff, buf *strings.Builder) {// Handle other ALTER operations// Alter columns (type changes, nullability, etc.)- for _, colDiff := range tableDiff.ColumnChanges {- if colDiff.Action == schema.MetadataDiffActionAlter {- alterColSQL := generateAlterColumn(tableDiff.SchemaName, tableDiff.TableName, colDiff)- _, _ = buf.WriteString(alterColSQL)- }- }+ writeAlterColumns(buf, diff, tableDiff)// Add indexesfor _, indexDiff := range tableDiff.IndexChanges {@@ -793,18 +795,177 @@ func generateCreateTable(schemaName, tableName string, table *metadatapb.TableMereturn buf.String(), nil}-func generateAlterColumn(schemaName, tableName string, colDiff *schema.ColumnDiff) string {+// writeAlterColumns writes one table's column alterations. A rebuilt column+// cannot change while an index or a constraint is on it (ORA-12991, ORA-54022,+// ORA-54023), so those of every rebuilt column are dropped before any column+// changes and made again once all have, each once: a constraint shared by two+// rebuilt columns is checked only against the finished table.+func writeAlterColumns(buf *strings.Builder, diff *schema.MetadataDiff, tableDiff *schema.TableDiff) {+ var oldColumns, newColumns []string+ for _, colDiff := range tableDiff.ColumnChanges {+ if colDiff.Action == schema.MetadataDiffActionAlter && rebuildsColumn(colDiff) {+ oldColumns = append(oldColumns, colDiff.OldColumn.Name)+ newColumns = append(newColumns, colDiff.NewColumn.Name)+ }+ }+ schemaName, tableName := tableDiff.SchemaName, tableDiff.TableName+ created := diffObjectNames(tableDiff, schema.MetadataDiffActionCreate)+ // Other tables' foreign keys on a rebuilt key go first, or dropping the key+ // fails (ORA-02273), and come back after it.+ for _, incoming := range findIncomingForeignKeys(diff, diff.OldDatabase, tableDiff, oldColumns, schema.MetadataDiffActionDrop) {+ writeDropForeignKey(buf, schemaName, incoming.table, incoming.foreignKey.Name)+ }+ findColumnDependents(tableDiff.OldTable, oldColumns, diffObjectNames(tableDiff, schema.MetadataDiffActionDrop)).writeDrops(buf, schemaName, tableName)+ for _, colDiff := range tableDiff.ColumnChanges {+ if colDiff.Action != schema.MetadataDiffActionAlter {+ continue+ }+ // A named NOT NULL went with the dependents. One the diff creates is+ // left to it.+ notNull := columnNotNull{dropped: notNullConstraintName(tableDiff.OldTable, colDiff.OldColumn.Name) != ""}+ if name := notNullConstraintName(tableDiff.NewTable, colDiff.NewColumn.Name); !created[name] {+ notNull.name = name+ }+ _, _ = buf.WriteString(generateAlterColumn(schemaName, tableName, colDiff, notNull))+ }+ findColumnDependents(tableDiff.NewTable, newColumns, created).writeCreates(buf, schemaName, tableName)+ for _, incoming := range findIncomingForeignKeys(diff, diff.NewDatabase, tableDiff, newColumns, schema.MetadataDiffActionCreate) {+ writeMigrationForeignKey(buf, schemaName, incoming.table, incoming.foreignKey)+ }+}++// incomingForeignKey is another table's foreign key on a rebuilt column.+type incomingForeignKey struct {+ table string+ foreignKey *metadatapb.ForeignKeyMetadata+}++// findIncomingForeignKeys lists the foreign keys of the database's other tables+// that reference any of the columns of tableDiff's table. It leaves out those+// the diff handles itself: on a table it creates or drops, or that it creates+// or drops, as action says.+func findIncomingForeignKeys(diff *schema.MetadataDiff, database *model.DatabaseMetadata, tableDiff *schema.TableDiff, columns []string, action schema.MetadataDiffAction) []incomingForeignKey {+ if len(columns) == 0 || database == nil {+ return nil+ }+ schemaMetadata := database.GetSchemaMetadata(tableDiff.SchemaName)+ if schemaMetadata == nil {+ return nil+ }+ tableDiffs := make(map[string]*schema.TableDiff)+ for _, other := range diff.TableChanges {+ if other.SchemaName == tableDiff.SchemaName {+ tableDiffs[other.TableName] = other+ }+ }+ var result []incomingForeignKey+ for _, table := range schemaMetadata.GetProto().GetTables() {+ if table.Name == tableDiff.TableName {+ continue+ }+ var handled map[string]bool+ switch other := tableDiffs[table.Name]; {+ case other == nil:+ case other.Action == schema.MetadataDiffActionCreate && action == schema.MetadataDiffActionCreate:+ // A new table's foreign keys on a rebuilt key waited for it.+ case other.Action != schema.MetadataDiffActionAlter:+ continue+ default:+ handled = diffObjectNames(other, action)+ }+ for _, fk := range table.GetForeignKeys() {+ if !handled[fk.Name] && referencesColumns(fk, tableDiff.SchemaName, diff.DatabaseName, tableDiff.TableName, columns) {+ result = append(result, incomingForeignKey{table: table.Name, foreignKey: fk})+ }+ }+ }+ return result+}++// waitsForRebuild reports whether a new table's foreign key references a column+// the diff rebuilds. writeAlterColumns adds it after the rebuild: added with+// the table, it would hold the key the rebuild drops (ORA-02273).+func waitsForRebuild(diff *schema.MetadataDiff, schemaName string, fk *metadatapb.ForeignKeyMetadata) bool {+ for _, tableDiff := range diff.TableChanges {+ if tableDiff.Action != schema.MetadataDiffActionAlter || tableDiff.SchemaName != schemaName {+ continue+ }+ for _, colDiff := range tableDiff.ColumnChanges {+ if colDiff.Action == schema.MetadataDiffActionAlter && rebuildsColumn(colDiff) &&+ referencesColumns(fk, schemaName, diff.DatabaseName, tableDiff.TableName, []string{colDiff.NewColumn.Name}) {+ return true+ }+ }+ }+ return false+}++// referencesColumns reports whether the foreign key references any of the+// columns of the table, which the sync names without a schema when it is the+// foreign key's own.+func referencesColumns(fk *metadatapb.ForeignKeyMetadata, schemaName, databaseName, tableName string, columns []string) bool {+ if fk.ReferencedTable != tableName || (fk.ReferencedSchema != "" && fk.ReferencedSchema != schemaName && fk.ReferencedSchema != databaseName) {+ return false+ }+ return slices.ContainsFunc(fk.ReferencedColumns, func(column string) bool { return slices.Contains(columns, column) })+}++// rebuildsColumn reports whether Oracle must drop and add the column again, or+// give it a new virtual type or expression.+func rebuildsColumn(colDiff *schema.ColumnDiff) bool {+ oldGeneration, newGeneration := colDiff.OldColumn.Generation, colDiff.NewColumn.Generation+ if (oldGeneration == nil) != (newGeneration == nil) {+ return true+ }+ return newGeneration != nil && (oldGeneration.Expression != newGeneration.Expression || colDiff.OldColumn.Type != colDiff.NewColumn.Type)+}++// columnNotNull is how a rebuilt column's NOT NULL is handled: dropped means a+// named one was dropped with the dependents, and name is the one to restore.+type columnNotNull struct {+ dropped bool+ name string+}++func generateAlterColumn(schemaName, tableName string, colDiff *schema.ColumnDiff, notNull columnNotNull) string {var buf strings.Builder+ // Oracle cannot turn a stored column into a virtual one or back, so the+ // column is dropped and added again, comment included.+ oldGeneration, newGeneration := colDiff.OldColumn.Generation, colDiff.NewColumn.Generation+ if (oldGeneration == nil) != (newGeneration == nil) {+ writeDropColumn(&buf, schemaName, tableName, colDiff.OldColumn.Name)+ if notNull.name == "" {+ writeAddColumn(&buf, schemaName, tableName, colDiff.NewColumn)+ } else {+ // Added nullable, so that NOT NULL comes back under its name.+ column := proto.CloneOf(colDiff.NewColumn)+ column.Nullable = true+ writeAddColumn(&buf, schemaName, tableName, column)+ writeSetNotNull(&buf, schemaName, tableName, colDiff.NewColumn, notNull.name)+ }+ if colDiff.NewColumn.Comment != "" {+ writeColumnComment(&buf, schemaName, tableName, colDiff.NewColumn.Name, colDiff.NewColumn.Comment)+ }+ return buf.String()+ }++ // Oracle changes a virtual column's type only with its expression+ // (ORA-54027), and NOT NULL blocks both (ORA-54023).+ virtualChanged := newGeneration != nil && (oldGeneration.Expression != newGeneration.Expression || colDiff.OldColumn.Type != colDiff.NewColumn.Type)+ if virtualChanged && !colDiff.OldColumn.Nullable && !notNull.dropped {+ writeAlterColumnDropNotNull(&buf, schemaName, tableName, colDiff.OldColumn.Name)+ }+// In Oracle, we need to handle different aspects of column changes separately// If type changed, alter the column type- if colDiff.OldColumn.Type != colDiff.NewColumn.Type {+ if !virtualChanged && colDiff.OldColumn.Type != colDiff.NewColumn.Type {writeAlterColumnType(&buf, schemaName, tableName, colDiff.NewColumn.Name, colDiff.NewColumn.Type)}// If nullability changed- if colDiff.OldColumn.Nullable != colDiff.NewColumn.Nullable {+ if !virtualChanged && colDiff.OldColumn.Nullable != colDiff.NewColumn.Nullable {if colDiff.NewColumn.Nullable {writeAlterColumnDropNotNull(&buf, schemaName, tableName, colDiff.NewColumn.Name)} else {@@ -830,9 +991,190 @@ func generateAlterColumn(schemaName, tableName string, colDiff *schema.ColumnDif}}+ if virtualChanged {+ writeModifyColumn(&buf, schemaName, tableName, colDiff.NewColumn.Name, colDiff.NewColumn.Type+" AS ("+newGeneration.Expression+")")+ writeSetNotNull(&buf, schemaName, tableName, colDiff.NewColumn, notNull.name)+ }++ if colDiff.OldColumn.IsInvisible != colDiff.NewColumn.IsInvisible {+ visibility := "VISIBLE"+ if colDiff.NewColumn.IsInvisible {+ visibility = "INVISIBLE"+ }+ writeModifyColumn(&buf, schemaName, tableName, colDiff.NewColumn.Name, visibility)+ }+return buf.String()}+// writeSetNotNull restores a rebuilt column's NOT NULL, under its name when it+// had one.+func writeSetNotNull(out *strings.Builder, schemaName, tableName string, column *metadatapb.ColumnMetadata, name string) {+ switch {+ case column.Nullable:+ case name != "":+ writeModifyColumn(out, schemaName, tableName, column.Name, `CONSTRAINT "`+name+`" NOT NULL`)+ default:+ writeAlterColumnSetNotNull(out, schemaName, tableName, column.Name)+ }+}++// notNullConstraintName returns the name of the column's named NOT NULL+// constraint, which the sync lists as a check on "<column>" IS NOT NULL.+func notNullConstraintName(table *metadatapb.TableMetadata, column string) string {+ want := `"` + column + `" IS NOT NULL`+ for _, check := range table.GetCheckConstraints() {+ if strings.Join(strings.Fields(check.Expression), " ") == want {+ return check.Name+ }+ }+ return ""+}++// columnDependents are a table's indexes and constraints on some of its+// columns. notNulls are named NOT NULL constraints: they are dropped with the+// rest, but restored by the column's own MODIFY, since a CHECK does not make+// a column NOT NULL.+type columnDependents struct {+ indexes []*metadatapb.IndexMetadata+ checks []*metadatapb.CheckConstraintMetadata+ notNulls []*metadatapb.CheckConstraintMetadata+ foreignKeys []*metadatapb.ForeignKeyMetadata+}++// findColumnDependents lists the table's indexes and constraints on any of the+// columns, each once, less those named in skip.+func findColumnDependents(table *metadatapb.TableMetadata, columns []string, skip map[string]bool) columnDependents {+ var dependents columnDependents+ if len(columns) == 0 {+ return dependents+ }+ references := func(expression string) bool {+ return slices.ContainsFunc(columns, func(column string) bool { return expressionReferencesColumn(expression, column) })+ }+ // A primary or unique key lists column names as Oracle stores them, not+ // SQL, so they compare as they are.+ isColumn := func(name string) bool { return slices.Contains(columns, strings.Trim(name, `"`)) }+ for _, index := range table.GetIndexes() {+ matches := references+ if index.IsConstraint {+ matches = isColumn+ }+ if !skip[index.Name] && slices.ContainsFunc(index.Expressions, matches) {+ dependents.indexes = append(dependents.indexes, index)+ }+ }+ for _, check := range table.GetCheckConstraints() {+ switch {+ case skip[check.Name] || !references(check.Expression):+ case slices.ContainsFunc(columns, func(column string) bool { return notNullConstraintName(table, column) == check.Name }):+ dependents.notNulls = append(dependents.notNulls, check)+ default:+ dependents.checks = append(dependents.checks, check)+ }+ }+ for _, fk := range table.GetForeignKeys() {+ onColumn := slices.ContainsFunc(fk.Columns, func(column string) bool { return slices.Contains(columns, column) })+ // A foreign key of the table on its own rebuilt key goes with the key.+ if !skip[fk.Name] && (onColumn || referencesColumns(fk, "", "", table.Name, columns)) {+ dependents.foreignKeys = append(dependents.foreignKeys, fk)+ }+ }+ return dependents+}++// diffObjectNames returns the names of the indexes and constraints the diff+// drops, or creates, on the table itself.+func diffObjectNames(tableDiff *schema.TableDiff, action schema.MetadataDiffAction) map[string]bool {+ names := make(map[string]bool)+ for _, indexDiff := range tableDiff.IndexChanges {+ if indexDiff.Action == action {+ names[cmp.Or(indexDiff.NewIndex, indexDiff.OldIndex).GetName()] = true+ }+ }+ for _, checkDiff := range tableDiff.CheckConstraintChanges {+ if checkDiff.Action == action {+ names[cmp.Or(checkDiff.NewCheckConstraint, checkDiff.OldCheckConstraint).GetName()] = true+ }+ }+ for _, fkDiff := range tableDiff.ForeignKeyChanges {+ if fkDiff.Action == action {+ names[cmp.Or(fkDiff.NewForeignKey, fkDiff.OldForeignKey).GetName()] = true+ }+ }+ return names+}++func (d columnDependents) writeDrops(out *strings.Builder, schemaName, tableName string) {+ for _, fk := range d.foreignKeys {+ writeDropForeignKey(out, schemaName, tableName, fk.Name)+ }+ for _, check := range slices.Concat(d.checks, d.notNulls) {+ writeDropCheckConstraint(out, schemaName, tableName, check.Name)+ }+ for _, index := range d.indexes {+ if index.IsConstraint {+ writeDropConstraint(out, schemaName, tableName, index.Name)+ } else {+ writeDropIndex(out, schemaName, index.Name)+ }+ }+}++func (d columnDependents) writeCreates(out *strings.Builder, schemaName, tableName string) {+ for _, index := range d.indexes {+ switch {+ case index.Primary:+ writeMigrationPrimaryKey(out, schemaName, tableName, index)+ case index.IsConstraint:+ writeMigrationUniqueKey(out, schemaName, tableName, index)+ default:+ writeMigrationIndex(out, schemaName, tableName, index)+ }+ }+ for _, check := range d.checks {+ writeAddCheckConstraint(out, schemaName, tableName, check)+ }+ for _, fk := range d.foreignKeys {+ writeMigrationForeignKey(out, schemaName, tableName, fk)+ }+}++// expressionReferencesColumn reports whether an index or check expression+// names the column, quoted or as an unquoted identifier. String literals are+// skipped.+func expressionReferencesColumn(expression, column string) bool {+ isIdentifierChar := func(c byte) bool {+ return c == '_' || c == '$' || c == '#' || ('0' <= c && c <= '9') || ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z')+ }+ for i := 0; i < len(expression); {+ switch c := expression[i]; {+ case c == '\'' || c == '"':+ end := strings.IndexByte(expression[i+1:], c)+ if end < 0 {+ return false+ }+ if c == '"' && expression[i+1:i+1+end] == column {+ return true+ }+ i += end + 2+ case isIdentifierChar(c):+ j := i + 1+ for j < len(expression) && isIdentifierChar(expression[j]) {+ j+++ }+ // A run that starts with a digit is a number, such as 1E5.+ if (c < '0' || c > '9') && strings.ToUpper(expression[i:j]) == column {+ return true+ }+ i = j+ default:+ i+++ }+ }+ return false+}+// hasDefaultValue checks if a column has any default valuefunc hasDefaultValue(column *metadatapb.ColumnMetadata) bool {if column == nil {@@ -1046,17 +1388,44 @@ func writeAddColumn(out *strings.Builder, schema, table string, column *metadata_, _ = out.WriteString(column.Name)_, _ = out.WriteString(`" `)_, _ = out.WriteString(column.Type)+ writeColumnAttributes(out, column)++ _, _ = out.WriteString(`;`)+ _, _ = out.WriteString("\n")+}- defaultExpr := getDefaultExpression(column)- if defaultExpr != "" {+// writeColumnAttributes writes what follows a column's type in CREATE TABLE+// and ADD: visibility, then the virtual expression or the default, then NOT NULL.+func writeColumnAttributes(out *strings.Builder, column *metadatapb.ColumnMetadata) {+ if column.IsInvisible {+ _, _ = out.WriteString(` INVISIBLE`)+ }+ if column.Generation != nil {+ _, _ = out.WriteString(virtualColumnClause(column.Generation))+ } else if defaultExpr := getDefaultExpression(column); defaultExpr != "" {_, _ = out.WriteString(` DEFAULT `)_, _ = out.WriteString(defaultExpr)}if !column.Nullable {_, _ = out.WriteString(` NOT NULL`)}+}- _, _ = out.WriteString(`;`)+// writeModifyColumn writes ALTER TABLE ... MODIFY ("column" clause).+func writeModifyColumn(out *strings.Builder, schema, table, column, clause string) {+ _, _ = out.WriteString(`ALTER TABLE `)+ if schema != "" {+ _, _ = out.WriteString(`"`)+ _, _ = out.WriteString(schema)+ _, _ = out.WriteString(`".`)+ }+ _, _ = out.WriteString(`"`)+ _, _ = out.WriteString(table)+ _, _ = out.WriteString(`" MODIFY ("`)+ _, _ = out.WriteString(column)+ _, _ = out.WriteString(`" `)+ _, _ = out.WriteString(clause)+ _, _ = out.WriteString(`);`)_, _ = out.WriteString("\n")}@@ -1461,16 +1830,7 @@ func writeMigrationCreateTable(out *strings.Builder, schema, table string, colum_, _ = out.WriteString(col.Name)_, _ = out.WriteString("\" ")_, _ = out.WriteString(col.Type)-- defaultExpr := getDefaultExpression(col)- if defaultExpr != "" {- _, _ = out.WriteString(" DEFAULT ")- _, _ = out.WriteString(defaultExpr)- }-- if !col.Nullable {- _, _ = out.WriteString(" NOT NULL")- }+ writeColumnAttributes(out, col)}// Write check constraintsdiff --git a/backend/plugin/schema/oracle/generate_migration_test.go b/backend/plugin/schema/oracle/generate_migration_test.goindex 4f51b70cf2..ee218cad3f 100644--- a/backend/plugin/schema/oracle/generate_migration_test.go+++ b/backend/plugin/schema/oracle/generate_migration_test.go@@ -2,6 +2,7 @@ package oracleimport ("os"+ "slices""strings""testing"@@ -456,3 +457,47 @@ func findStatementIndex(statements []string, stmtType, objectName string) int {}return -1}++func TestExpressionReferencesColumn(t *testing.T) {+ tests := []struct {+ expression string+ column string+ want bool+ }{+ {expression: `"S"`, column: "S", want: true},+ {expression: `UPPER("S")`, column: "S", want: true},+ {expression: `S+1`, column: "S", want: true},+ {expression: `s > 0`, column: "S", want: true},+ {expression: `"S2"`, column: "S", want: false},+ {expression: `S2`, column: "S", want: false},+ {expression: `'S'`, column: "S", want: false},+ {expression: `"s"`, column: "S", want: false},+ {expression: `1E5`, column: "E5", want: false},+ }+ for _, tc := range tests {+ require.Equal(t, tc.want, expressionReferencesColumn(tc.expression, tc.column), tc.expression)+ }+}++func TestFindColumnDependentsMatchesKeyColumnsByName(t *testing.T) {+ table := &metadatapb.TableMetadata{+ Indexes: []*metadatapb.IndexMetadata{+ {Name: "LOWER_PK", Expressions: []string{"v"}, Primary: true, Unique: true, IsConstraint: true},+ {Name: "MIXED_UK", Expressions: []string{"A", "Vv"}, Unique: true, IsConstraint: true},+ {Name: "UPPER_UK", Expressions: []string{"V"}, Unique: true, IsConstraint: true},+ {Name: "LOWER_IX", Expressions: []string{`"v"`}},+ },+ }+ require.Equal(t, []string{"LOWER_IX", "LOWER_PK"}, sortedIndexNames(findColumnDependents(table, []string{"v"}, nil)))+ require.Equal(t, []string{"MIXED_UK"}, sortedIndexNames(findColumnDependents(table, []string{"Vv"}, nil)))+ require.Equal(t, []string{"UPPER_UK"}, sortedIndexNames(findColumnDependents(table, []string{"V"}, nil)))+}++func sortedIndexNames(dependents columnDependents) []string {+ var names []string+ for _, index := range dependents.indexes {+ names = append(names, index.Name)+ }+ slices.Sort(names)+ return names+}diff --git a/backend/plugin/schema/oracle/get_database_definition.go b/backend/plugin/schema/oracle/get_database_definition.goindex 8e1f61da21..5b66669517 100644--- a/backend/plugin/schema/oracle/get_database_definition.go+++ b/backend/plugin/schema/oracle/get_database_definition.go@@ -58,8 +58,17 @@ func GetDatabaseDefinition(_ schema.GetDefinitionContext, to *metadatapb.DatabasfunctionMap := make(map[string]*metadatapb.FunctionMetadata)procedureMap := make(map[string]*metadatapb.ProcedureMetadata)- // Add tables to graph+ // SkipDump marks object and XMLType tables: CREATE TABLE ... OF needs the+ // object type, which the metadata does not carry.+ var tables []*metadatapb.TableMetadatafor _, table := range schema.Tables {+ if !table.SkipDump {+ tables = append(tables, table)+ }+ }++ // Add tables to graph+ for _, table := range tables {tableID := getObjectID(databaseName, table.Name)tableMap[tableID] = tablegraph.AddNode(tableID)@@ -111,7 +120,7 @@ func GetDatabaseDefinition(_ schema.GetDefinitionContext, to *metadatapb.Databas}// Add foreign key dependencies between tables- for _, table := range schema.Tables {+ for _, table := range tables {tableID := getObjectID(databaseName, table.Name)for _, fk := range table.ForeignKeys {if fk.ReferencedTable != table.Name { // Avoid self-references@@ -125,7 +134,7 @@ func GetDatabaseDefinition(_ schema.GetDefinitionContext, to *metadatapb.DatabasorderedList, err := graph.TopologicalSort()if err != nil {// If there are cycles, fall back to original order- for _, table := range schema.Tables {+ for _, table := range tables {if err := writeTable(&buf, table); err != nil {return "", err}@@ -186,6 +195,9 @@ func GetDatabaseDefinition(_ schema.GetDefinitionContext, to *metadatapb.Databas}func GetTableDefinition(_ string, table *metadatapb.TableMetadata, _ []*metadatapb.SequenceMetadata) (string, error) {+ if table.SkipDump {+ return "", nil+ }var buf strings.Builderif err := writeTable(&buf, table); err != nil {return "", err@@ -584,6 +596,22 @@ func writeColumn(buf *strings.Builder, column *metadatapb.ColumnMetadata) errorif _, err := buf.WriteString(column.Type); err != nil {return err}+ if column.IsInvisible {+ if _, err := buf.WriteString(` INVISIBLE`); err != nil {+ return err+ }+ }+ if column.Generation != nil {+ if _, err := buf.WriteString(virtualColumnClause(column.Generation)); err != nil {+ return err+ }+ if !column.Nullable {+ if _, err := buf.WriteString(` NOT NULL`); err != nil {+ return err+ }+ }+ return nil+ }// Skip collation for Oracle as it causes issues with standard string size settings// if column.Collation != "" {// if _, err := buf.WriteString(` COLLATE "`); err != nil {@@ -718,6 +746,12 @@ func writeView(buf *strings.Builder, view *metadatapb.ViewMetadata) error {return writeTriggers(buf, view.Triggers)}+// virtualColumnClause writes the clause that follows a virtual column's type+// and visibility.+func virtualColumnClause(generation *metadatapb.GenerationMetadata) string {+ return " GENERATED ALWAYS AS (" + generation.Expression + ") VIRTUAL"+}+// writeMaterializedView writes a CREATE MATERIALIZED VIEW statementfunc writeMaterializedView(buf *strings.Builder, view *metadatapb.MaterializedViewMetadata) error {if _, err := buf.WriteString(`CREATE MATERIALIZED VIEW "`); err != nil {diff --git a/backend/plugin/schema/oracle/get_database_definition_test.go b/backend/plugin/schema/oracle/get_database_definition_test.goindex 570f520544..9b0c4cf03a 100644--- a/backend/plugin/schema/oracle/get_database_definition_test.go+++ b/backend/plugin/schema/oracle/get_database_definition_test.go@@ -107,6 +107,15 @@ func TestGetTableDefinition(t *testing.T) {`,},+ {+ name: "object table has no definition",+ table: &metadatapb.TableMetadata{+ Name: "OT",+ SkipDump: true,+ Columns: []*metadatapb.ColumnMetadata{{Name: "X", Type: "NUMBER", Nullable: true}},+ },+ want: "",+ },}for _, tt := range tests {diff --git a/backend/plugin/schema/oracle/get_database_metadata.go b/backend/plugin/schema/oracle/get_database_metadata.goindex b24c7b3657..66c2efce98 100644--- a/backend/plugin/schema/oracle/get_database_metadata.go+++ b/backend/plugin/schema/oracle/get_database_metadata.go@@ -266,8 +266,12 @@ func (e *oracleOmniMetadataExtractor) extractColumn(n *ast.ColumnDef, table *metif n.TypeName == nil && n.Domain == nil {column.Type = "NUMBER"}- column.Default = e.exprText(n.Virtual)+ column.Generation = &metadatapb.GenerationMetadata{+ Type: metadatapb.GenerationMetadata_TYPE_VIRTUAL,+ Expression: e.exprText(n.Virtual),+ }}+ column.IsInvisible = n.Invisiblecolumn.DefaultOnNull = n.DefaultOnNullif n.Collation != "" {column.Collation = n.Collationdiff --git a/backend/plugin/schema/oracle/get_database_metadata_test.go b/backend/plugin/schema/oracle/get_database_metadata_test.goindex 61988a2bdd..8d3b00d6ec 100644--- a/backend/plugin/schema/oracle/get_database_metadata_test.go+++ b/backend/plugin/schema/oracle/get_database_metadata_test.go@@ -63,6 +63,25 @@ func TestGetDatabaseMetadataObjectKinds(t *testing.T) {ddl stringverify func(*testing.T, *metadatapb.DatabaseSchemaMetadata)}{+ {+ name: "invisible_and_virtual_columns",+ ddl: `+CREATE TABLE T (+ A NUMBER,+ H NUMBER INVISIBLE DEFAULT 7,+ IHV INVISIBLE AS (A * 2) VIRTUAL+);+`,+ verify: func(t *testing.T, metadata *metadatapb.DatabaseSchemaMetadata) {+ table := requireTable(t, requireSingleSchema(t, metadata), "T")+ require.Len(t, table.Columns, 3)+ require.False(t, table.Columns[0].IsInvisible)+ require.True(t, table.Columns[1].IsInvisible)+ require.Equal(t, "7", table.Columns[1].Default)+ require.True(t, table.Columns[2].IsInvisible)+ require.Equal(t, "A*2", table.Columns[2].GetGeneration().GetExpression())+ },+ },{name: "table_constraints_indexes_view_sequence",ddl: `@@ -334,7 +353,9 @@ CREATE TABLE ORDER_ITEMS (require.Len(t, orderItems.Columns, 3)require.Equal(t, "TOTAL", orderItems.Columns[2].Name)require.Equal(t, "NUMBER", orderItems.Columns[2].Type)- require.Equal(t, "QTY*PRICE", orderItems.Columns[2].Default)+ require.Empty(t, orderItems.Columns[2].Default)+ require.Equal(t, metadatapb.GenerationMetadata_TYPE_VIRTUAL, orderItems.Columns[2].GetGeneration().GetType())+ require.Equal(t, "QTY*PRICE", orderItems.Columns[2].GetGeneration().GetExpression())},},{@@ -349,7 +370,7 @@ CREATE TABLE AUTHORS (verify: func(t *testing.T, metadata *metadatapb.DatabaseSchemaMetadata) {authors := requireTable(t, requireSingleSchema(t, metadata), "AUTHORS")require.Len(t, authors.Columns, 2)- require.Equal(t, "CASEWHENNAME='O''Reilly'THEN1ELSE0END", authors.Columns[1].Default)+ require.Equal(t, "CASEWHENNAME='O''Reilly'THEN1ELSE0END", authors.Columns[1].GetGeneration().GetExpression())requireCheckConstraint(t, authors, "CHK_AUTHOR_NAME", "NAME<>'O''Reilly'")},},diff --git a/backend/plugin/schema/oracle/testdata/generate_migration.yaml b/backend/plugin/schema/oracle/testdata/generate_migration.yamlindex eaa6c652db..9b2896cfed 100644--- a/backend/plugin/schema/oracle/testdata/generate_migration.yaml+++ b/backend/plugin/schema/oracle/testdata/generate_migration.yaml@@ -8247,3 +8247,491 @@"serviceName": "FREEPDB1"}expected: "CREATE TABLE \"COMPLEX_TYPES\" (\n \"ID\" NUMBER NOT NULL,\n \"XML_DATA\" XMLTYPE,\n \"TIMESTAMP_TZ\" TIMESTAMP(6) WITH TIME ZONE DEFAULT SYSTIMESTAMP,\n \"INTERVAL_DS\" INTERVAL DAY(2) TO SECOND(6),\n \"RAW_DATA\" RAW(100),\n \"LONG_RAW_DATA\" LONG RAW\n);\nALTER TABLE \"COMPLEX_TYPES\" ADD CONSTRAINT \"SYS_C0000001\" PRIMARY KEY (ID);\nCOMMENT ON TABLE \"COMPLEX_TYPES\" IS 'Advanced data types testing table\nSupports: XMLType, Timestamps with timezone, Intervals, RAW data\nPerformance: Optimized for OLTP workloads @ 10K+ TPS\nSecurity: Row-level security enabled, audit trails maintained';\nCOMMENT ON COLUMN \"COMPLEX_TYPES\".\"XML_DATA\" IS 'XMLType column for structured XML storage\nExample: <?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<root><item id=\"1\" name=\"test\">Content</item></root>';\nCOMMENT ON COLUMN \"COMPLEX_TYPES\".\"TIMESTAMP_TZ\" IS 'Timestamp with timezone: YYYY-MM-DD HH24:MI:SS.FF TZR\nDefault: SYSTIMESTAMP (system timestamp with session timezone)';\nCOMMENT ON COLUMN \"COMPLEX_TYPES\".\"INTERVAL_DS\" IS 'Interval Day to Second: +/-DD HH:MI:SS.FFFFFF\nRange: +/-999999999 days, precision up to nanoseconds';\n\nCOMMENT ON TABLE \"MULTILANG_CONTENT\" IS 'Multilingual content: English, Español, 中文, العربية, Русский, 日本語';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TITLE_EN\" IS 'English title field';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TITLE_ES\" IS 'Título en español';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TITLE_ZH\" IS '中文标题字段 - 支持简体和繁体中文';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"CONTENT_TEXT\" IS 'Content field supporting:\n1. HTML tags: <b>bold</b>, <i>italic</i>, <a href=\"http://example.com\">links</a>\n2. Markdown: **bold**, *italic*, [link](url), backtick-code-backtick\n3. Special symbols: © ® ™ § ¶ † ‡ • … ‰ ′ ″\n4. Emojis: \U0001F600 \U0001F603 \U0001F604 \U0001F601 \U0001F680 ⭐ \U0001F389 \U0001F4AF\n5. Mathematical: ∀x∈ℝ: x² ≥ 0, ∑(i=1 to n) i = n(n+1)/2';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TAGS\" IS 'XML tags: <tag type=\"category\" value=\"news\"/><tag type=\"priority\" value=\"high\"/>End of XML';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TITLE_EN\" IS 'English title field';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TITLE_ES\" IS 'Título en español';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TITLE_ZH\" IS '中文标题字段 - 支持简体和繁体中文';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"CONTENT_TEXT\" IS 'Content field supporting:\n1. HTML tags: <b>bold</b>, <i>italic</i>, <a href=\"http://example.com\">links</a>\n2. Markdown: **bold**, *italic*, [link](url), backtick-code-backtick\n3. Special symbols: © ® ™ § ¶ † ‡ • … ‰ ′ ″\n4. Emojis: \U0001F600 \U0001F603 \U0001F604 \U0001F601 \U0001F680 ⭐ \U0001F389 \U0001F4AF\n5. Mathematical: ∀x∈ℝ: x² ≥ 0, ∑(i=1 to n) i = n(n+1)/2';\nCOMMENT ON COLUMN \"MULTILANG_CONTENT\".\"TAGS\" IS 'XML tags: <tag type=\"category\" value=\"news\"/><tag type=\"priority\" value=\"high\"/>End of XML';\nCOMMENT ON TABLE \"SPECIAL_DATA\" IS 'Table containing \"special\" data with various formats & types';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"DATA_VALUE\" IS 'Data value field - supports NULL, empty strings, and ''quoted'' content';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"METADATA_JSON\" IS 'JSON metadata: {\"key\": \"value\", \"array\": [1,2,3]}';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"UNICODE_TEXT\" IS 'Unicode text support:\nLine 1: Basic ASCII text\nLine 2: Special chars !@#$%^&*()_+-={}[]|;:,.<>?\nLine 3: Math symbols ≤ ≥ ± × ÷ ∑ ∏ ∆ π\nLine 4: Currencies $ € £ ¥ ₹';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"BINARY_DATA\" IS 'Binary data storage\\nSupports: images, documents, archives\\tTab-separated info\\r\\nWindows line endings';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"CATEGORY\" IS 'Category field: use % for wildcard searches, _ for single char matching';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"STATUS\" IS 'Status field - values: ''ACTIVE'', ''INACTIVE'', ''PENDING''; DROP TABLE users; --';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"DATA_VALUE\" IS 'Data value field - supports NULL, empty strings, and ''quoted'' content';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"METADATA_JSON\" IS 'JSON metadata: {\"key\": \"value\", \"array\": [1,2,3]}';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"UNICODE_TEXT\" IS 'Unicode text support:\nLine 1: Basic ASCII text\nLine 2: Special chars !@#$%^&*()_+-={}[]|;:,.<>?\nLine 3: Math symbols ≤ ≥ ± × ÷ ∑ ∏ ∆ π\nLine 4: Currencies $ € £ ¥ ₹';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"BINARY_DATA\" IS 'Binary data storage\\nSupports: images, documents, archives\\tTab-separated info\\r\\nWindows line endings';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"CATEGORY\" IS 'Category field: use % for wildcard searches, _ for single char matching';\nCOMMENT ON COLUMN \"SPECIAL_DATA\".\"STATUS\" IS 'Status field - values: ''ACTIVE'', ''INACTIVE'', ''PENDING''; DROP TABLE users; --';\n"+- description: Invisible and virtual columns, and object tables the migration leaves alone+ oldMetadata: |+ {+ "name": "TESTSCHEMA",+ "schemas": [+ {+ "tables": [+ {+ "name": "CHILD",+ "columns": [+ {+ "name": "ID",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "S_REF",+ "position": 2,+ "type": "NUMBER",+ "nullable": true+ },+ {+ "name": "A_REF",+ "position": 3,+ "type": "NUMBER",+ "nullable": true+ }+ ],+ "foreign_keys": [+ {+ "name": "CHILD_FK",+ "columns": [+ "S_REF",+ "A_REF"+ ],+ "referenced_table": "T",+ "referenced_columns": [+ "S",+ "A"+ ]+ }+ ]+ },+ {+ "name": "OT",+ "skip_dump": true,+ "columns": [+ {+ "name": "X",+ "position": 1,+ "type": "NUMBER",+ "nullable": true+ }+ ]+ },+ {+ "name": "T",+ "columns": [+ {+ "name": "A",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "VC",+ "position": 2,+ "type": "NUMBER",+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"+1"+ }+ },+ {+ "name": "H",+ "position": 3,+ "type": "NUMBER",+ "nullable": true,+ "is_invisible": true+ },+ {+ "name": "S",+ "position": 4,+ "type": "NUMBER",+ "nullable": true,+ "comment": "S comment"+ },+ {+ "name": "VT",+ "position": 5,+ "type": "NUMBER",+ "nullable": true,+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"+5"+ }+ },+ {+ "name": "R",+ "position": 6,+ "type": "NUMBER"+ }+ ],+ "indexes": [+ {+ "name": "T_A",+ "expressions": [+ "\"A\""+ ],+ "type": "NORMAL",+ "visible": true+ },+ {+ "name": "T_S",+ "expressions": [+ "\"S\""+ ],+ "type": "NORMAL",+ "visible": true+ },+ {+ "name": "T_VC",+ "expressions": [+ "\"VC\""+ ],+ "type": "NORMAL",+ "visible": true+ },+ {+ "name": "T_SA_UK",+ "expressions": [+ "S",+ "A"+ ],+ "type": "NORMAL",+ "unique": true,+ "visible": true,+ "is_constraint": true+ }+ ],+ "check_constraints": [+ {+ "name": "T_S_CK",+ "expression": "\"S\" > 0"+ },+ {+ "name": "T_VC_CK",+ "expression": "\"VC\" > 0"+ },+ {+ "name": "T_SA_CK",+ "expression": "\"S\" >= \"A\""+ },+ {+ "name": "T_VCS_CK",+ "expression": "\"VC\" <= \"S\" + 100"+ },+ {+ "name": "VC_NN",+ "expression": "\"VC\" IS NOT NULL"+ },+ {+ "name": "R_NN",+ "expression": "\"R\" IS NOT NULL"+ }+ ]+ }+ ]+ }+ ]+ }+ newMetadata: |+ {+ "name": "TESTSCHEMA",+ "schemas": [+ {+ "tables": [+ {+ "name": "CHILD",+ "columns": [+ {+ "name": "ID",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "S_REF",+ "position": 2,+ "type": "NUMBER",+ "nullable": true+ },+ {+ "name": "A_REF",+ "position": 3,+ "type": "NUMBER",+ "nullable": true+ }+ ],+ "foreign_keys": [+ {+ "name": "CHILD_FK",+ "columns": [+ "S_REF",+ "A_REF"+ ],+ "referenced_table": "T",+ "referenced_columns": [+ "S",+ "A"+ ]+ }+ ]+ },+ {+ "name": "OT",+ "skip_dump": true,+ "columns": [+ {+ "name": "X",+ "position": 1,+ "type": "NUMBER",+ "nullable": true+ },+ {+ "name": "Y",+ "position": 2,+ "type": "NUMBER",+ "nullable": true+ }+ ]+ },+ {+ "name": "OT2",+ "skip_dump": true,+ "columns": [+ {+ "name": "X",+ "position": 1,+ "type": "NUMBER",+ "nullable": true+ }+ ]+ },+ {+ "name": "T",+ "columns": [+ {+ "name": "A",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "VC",+ "position": 2,+ "type": "NUMBER",+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"+2"+ }+ },+ {+ "name": "H",+ "position": 3,+ "type": "NUMBER",+ "nullable": true+ },+ {+ "name": "S",+ "position": 4,+ "type": "NUMBER",+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"*3"+ },+ "comment": "S comment"+ },+ {+ "name": "N",+ "position": 5,+ "type": "NUMBER",+ "is_invisible": true,+ "default": "5"+ },+ {+ "name": "NV",+ "position": 6,+ "type": "NUMBER",+ "nullable": true,+ "is_invisible": true,+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"-1"+ }+ },+ {+ "name": "VT",+ "position": 7,+ "type": "VARCHAR2(40 BYTE)",+ "nullable": true,+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "TO_CHAR(\"A\")"+ }+ },+ {+ "name": "R",+ "position": 8,+ "type": "NUMBER",+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"*7"+ }+ }+ ],+ "indexes": [+ {+ "name": "T_A",+ "expressions": [+ "\"A\""+ ],+ "type": "NORMAL",+ "visible": true+ },+ {+ "name": "T_S",+ "expressions": [+ "\"S\""+ ],+ "type": "NORMAL",+ "visible": true+ },+ {+ "name": "T_VC",+ "expressions": [+ "\"VC\""+ ],+ "type": "NORMAL",+ "visible": true+ },+ {+ "name": "T_SA_UK",+ "expressions": [+ "S",+ "A"+ ],+ "type": "NORMAL",+ "unique": true,+ "visible": true,+ "is_constraint": true+ }+ ],+ "check_constraints": [+ {+ "name": "T_S_CK",+ "expression": "\"S\" > 0"+ },+ {+ "name": "T_VC_CK",+ "expression": "\"VC\" > 0"+ },+ {+ "name": "T_SA_CK",+ "expression": "\"S\" >= \"A\""+ },+ {+ "name": "T_VCS_CK",+ "expression": "\"VC\" <= \"S\" + 100"+ },+ {+ "name": "VC_NN",+ "expression": "\"VC\" IS NOT NULL"+ },+ {+ "name": "R_NN",+ "expression": "\"R\" IS NOT NULL"+ }+ ]+ },+ {+ "name": "T2",+ "columns": [+ {+ "name": "A",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "H",+ "position": 2,+ "type": "VARCHAR2(10 BYTE)",+ "nullable": true,+ "is_invisible": true+ },+ {+ "name": "VC",+ "position": 3,+ "type": "NUMBER",+ "nullable": true,+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"+1"+ }+ }+ ]+ },+ {+ "name": "NEWCHILD",+ "columns": [+ {+ "name": "ID",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "S_REF",+ "position": 2,+ "type": "NUMBER",+ "nullable": true+ },+ {+ "name": "A_REF",+ "position": 3,+ "type": "NUMBER",+ "nullable": true+ }+ ],+ "foreign_keys": [+ {+ "name": "NEWCHILD_FK",+ "columns": [+ "S_REF",+ "A_REF"+ ],+ "referenced_table": "T",+ "referenced_columns": [+ "S",+ "A"+ ]+ }+ ]+ }+ ]+ }+ ]+ }+ expected: |+ ALTER TABLE "T" ADD "N" NUMBER INVISIBLE DEFAULT 5 NOT NULL;+ ALTER TABLE "T" ADD "NV" NUMBER INVISIBLE GENERATED ALWAYS AS ("A"-1) VIRTUAL;+ CREATE TABLE "T2" (+ "A" NUMBER NOT NULL,+ "H" VARCHAR2(10 BYTE) INVISIBLE,+ "VC" NUMBER GENERATED ALWAYS AS ("A"+1) VIRTUAL+ );++ CREATE TABLE "NEWCHILD" (+ "ID" NUMBER NOT NULL,+ "S_REF" NUMBER,+ "A_REF" NUMBER+ );++ ALTER TABLE "CHILD" DROP CONSTRAINT "CHILD_FK";+ ALTER TABLE "T" DROP CONSTRAINT "T_S_CK";+ ALTER TABLE "T" DROP CONSTRAINT "T_VC_CK";+ ALTER TABLE "T" DROP CONSTRAINT "T_SA_CK";+ ALTER TABLE "T" DROP CONSTRAINT "T_VCS_CK";+ ALTER TABLE "T" DROP CONSTRAINT "VC_NN";+ ALTER TABLE "T" DROP CONSTRAINT "R_NN";+ DROP INDEX "T_S";+ DROP INDEX "T_VC";+ ALTER TABLE "T" DROP CONSTRAINT "T_SA_UK";+ ALTER TABLE "T" MODIFY ("VC" NUMBER AS ("A"+2));+ ALTER TABLE "T" MODIFY ("VC" CONSTRAINT "VC_NN" NOT NULL);+ ALTER TABLE "T" MODIFY ("H" VISIBLE);+ ALTER TABLE "T" DROP COLUMN "S";+ ALTER TABLE "T" ADD "S" NUMBER GENERATED ALWAYS AS ("A"*3) VIRTUAL NOT NULL;+ COMMENT ON COLUMN "T"."S" IS 'S comment';+ ALTER TABLE "T" MODIFY ("VT" VARCHAR2(40 BYTE) AS (TO_CHAR("A")));+ ALTER TABLE "T" DROP COLUMN "R";+ ALTER TABLE "T" ADD "R" NUMBER GENERATED ALWAYS AS ("A"*7) VIRTUAL;+ ALTER TABLE "T" MODIFY ("R" CONSTRAINT "R_NN" NOT NULL);+ CREATE INDEX "T_S" ON "T" ("S" ASC);+ CREATE INDEX "T_VC" ON "T" ("VC" ASC);+ ALTER TABLE "T" ADD CONSTRAINT "T_SA_UK" UNIQUE (S, A);+ ALTER TABLE "T" ADD CONSTRAINT "T_S_CK" CHECK ("S" > 0);+ ALTER TABLE "T" ADD CONSTRAINT "T_VC_CK" CHECK ("VC" > 0);+ ALTER TABLE "T" ADD CONSTRAINT "T_SA_CK" CHECK ("S" >= "A");+ ALTER TABLE "T" ADD CONSTRAINT "T_VCS_CK" CHECK ("VC" <= "S" + 100);+ ALTER TABLE "CHILD" ADD CONSTRAINT "CHILD_FK" FOREIGN KEY ("S_REF", "A_REF") REFERENCES "T" ("S", "A");+ ALTER TABLE "NEWCHILD" ADD CONSTRAINT "NEWCHILD_FK" FOREIGN KEY ("S_REF", "A_REF") REFERENCES "T" ("S", "A");diff --git a/backend/plugin/schema/oracle/testdata/get_database_definition.yaml b/backend/plugin/schema/oracle/testdata/get_database_definition.yamlindex 2ffb8b15ab..81a4eb4546 100644--- a/backend/plugin/schema/oracle/testdata/get_database_definition.yaml+++ b/backend/plugin/schema/oracle/testdata/get_database_definition.yaml@@ -249,3 +249,80 @@INSERT INTO EMPLOYEES (EMP_ID, EMAIL) VALUES (:NEW.EMP_ID, :NEW.EMAIL);END;+- description: An object table is left out, and invisible and virtual columns keep their clauses+ metadata: |+ {+ "name": "TESTSCHEMA",+ "schemas": [+ {+ "tables": [+ {+ "name": "OT",+ "skip_dump": true,+ "columns": [+ {+ "name": "X",+ "position": 1,+ "type": "NUMBER",+ "nullable": true+ }+ ]+ },+ {+ "name": "T",+ "columns": [+ {+ "name": "A",+ "position": 1,+ "type": "NUMBER"+ },+ {+ "name": "VC",+ "position": 2,+ "type": "NUMBER",+ "nullable": true,+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"+1"+ }+ },+ {+ "name": "H",+ "position": 3,+ "type": "NUMBER",+ "nullable": true,+ "is_invisible": true+ },+ {+ "name": "HV",+ "position": 4,+ "type": "NUMBER",+ "is_invisible": true,+ "default": "7"+ },+ {+ "name": "IHV",+ "position": 5,+ "type": "NUMBER",+ "nullable": true,+ "is_invisible": true,+ "generation": {+ "type": "TYPE_VIRTUAL",+ "expression": "\"A\"*2"+ }+ }+ ]+ }+ ]+ }+ ]+ }+ expected: |++ CREATE TABLE "T" (+ "A" NUMBER NOT NULL,+ "VC" NUMBER GENERATED ALWAYS AS ("A"+1) VIRTUAL,+ "H" NUMBER INVISIBLE,+ "HV" NUMBER INVISIBLE DEFAULT 7 NOT NULL,+ "IHV" NUMBER INVISIBLE GENERATED ALWAYS AS ("A"*2) VIRTUAL+ );+