Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: oracle sync (#20786)
ecmadao666419745ae1
diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.goindex ac936a3451..87e26c9ea1 100644--- a/backend/plugin/db/oracle/sync.go+++ b/backend/plugin/db/oracle/sync.go@@ -7,6 +7,7 @@ import ("log/slog""regexp""strings"+ "unicode/utf8""github.com/pkg/errors"@@ -248,7 +249,13 @@ func getTriggers(txn *sql.Tx, schemaName string) (map[db.TableKey][]*storepb.Trikey := db.TableKey{Schema: schemaName, Table: tableName.String}trigger := &storepb.TriggerMetadata{Name: triggerName.String,- Body: constructTriggerBody(description.String, triggerBody.String),+ Body: sanitizeOracleMetadataString(+ schemaName,+ "TRIGGER",+ triggerName.String,+ "body",+ constructTriggerBody(description.String, triggerBody.String),+ ),}if triggerType.Valid {trigger.Timing = triggerType.String@@ -977,8 +984,7 @@ func getViews(txn *sql.Tx, schemaName string, columnMap map[db.TableKey][]*storeif err := rows.Scan(&schemaName, &view.Name, &view.Definition); err != nil {return nil, err}- // Oracle may include a trailing C-string null terminator via the go-ora driver.- view.Definition = strings.TrimRight(view.Definition, "\x00")+ view.Definition = sanitizeOracleDefinition(schemaName, "VIEW", view.Name, view.Definition)key := db.TableKey{Schema: schemaName, Table: view.Name}view.Columns = columnMap[key]view.Triggers = triggerMap[key]@@ -1124,8 +1130,12 @@ func getMaterializedViews(txn *sql.Tx, schemaName string, _ map[db.TableKey][]*sif err := rows.Scan(&schemaName, &materializedView.Name, &materializedView.Definition); err != nil {return nil, err}- // Oracle may include a trailing C-string null terminator via the go-ora driver.- materializedView.Definition = strings.TrimRight(materializedView.Definition, "\x00")+ materializedView.Definition = sanitizeOracleDefinition(+ schemaName,+ "MATERIALIZED VIEW",+ materializedView.Name,+ materializedView.Definition,+ )// Ensure the definition ends with a newline to match expected formatif materializedView.Definition != "" && !strings.HasSuffix(materializedView.Definition, "\n") {@@ -1256,7 +1266,7 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [case "FUNCTION":function := &storepb.FunctionMetadata{Name: currentName,- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),+ Definition: sanitizeOracleDefinition(schemaName, "FUNCTION", currentName, strings.Join(defText, "")),}key := db.TableKey{Schema: schemaName, Table: currentName}if comment, ok := functionCommentMap[key]; ok {@@ -1266,12 +1276,12 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [case "PROCEDURE":procedures = append(procedures, &storepb.ProcedureMetadata{Name: currentName,- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),+ Definition: sanitizeOracleDefinition(schemaName, "PROCEDURE", currentName, strings.Join(defText, "")),})case "PACKAGE":packages = append(packages, &storepb.PackageMetadata{Name: currentName,- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),+ Definition: sanitizeOracleDefinition(schemaName, "PACKAGE", currentName, strings.Join(defText, "")),})default:// Ignore other types@@ -1286,7 +1296,7 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [case "FUNCTION":function := &storepb.FunctionMetadata{Name: currentName,- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),+ Definition: sanitizeOracleDefinition(schemaName, "FUNCTION", currentName, strings.Join(defText, "")),}key := db.TableKey{Schema: schemaName, Table: currentName}if comment, ok := functionCommentMap[key]; ok {@@ -1296,12 +1306,12 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [case "PROCEDURE":procedures = append(procedures, &storepb.ProcedureMetadata{Name: currentName,- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),+ Definition: sanitizeOracleDefinition(schemaName, "PROCEDURE", currentName, strings.Join(defText, "")),})case "PACKAGE":packages = append(packages, &storepb.PackageMetadata{Name: currentName,- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),+ Definition: sanitizeOracleDefinition(schemaName, "PACKAGE", currentName, strings.Join(defText, "")),})default:// Ignore other types@@ -1316,6 +1326,38 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [return functions, procedures, packages, nil}+func sanitizeOracleDefinition(schemaName, objectType, objectName, definition string) string {+ return sanitizeOracleMetadataString(schemaName, objectType, objectName, "definition", definition)+}++func sanitizeOracleMetadataString(schemaName, objectType, objectName, field, value string) string {+ // Oracle may include a trailing C-string null terminator via the go-ora driver.+ value = strings.TrimRight(value, "\x00")+ if utf8.ValidString(value) {+ return value+ }+ slog.Warn("sanitized invalid UTF-8 in Oracle metadata",+ slog.String("schema", schemaName),+ slog.String("object_type", objectType),+ slog.String("object_name", objectName),+ slog.String("field", field),+ )++ var sanitized strings.Builder+ sanitized.Grow(len(value))+ for i := 0; i < len(value); {+ r, width := utf8.DecodeRuneInString(value[i:])+ if r == utf8.RuneError && width == 1 {+ _, _ = fmt.Fprintf(&sanitized, "\\x%02x", value[i])+ i+++ continue+ }+ _, _ = sanitized.WriteString(value[i : i+width])+ i += width+ }+ return sanitized.String()+}+// cleanDefaultExpression cleans up Oracle default expressions to make them more portable// and fix common syntax issues like schema-qualified sequence NEXTVAL callsfunc cleanDefaultExpression(defaultExpr, currentSchema string) string {diff --git a/backend/plugin/db/oracle/sync_utf8_test.go b/backend/plugin/db/oracle/sync_utf8_test.goindex ab1f46630c..1783708b3d 100644--- a/backend/plugin/db/oracle/sync_utf8_test.go+++ b/backend/plugin/db/oracle/sync_utf8_test.go@@ -1,6 +1,8 @@package oracleimport (+ "bytes"+ "log/slog""testing""unicode/utf8"@@ -139,3 +141,160 @@ func TestOracleCommentSanitizedUTF8MarshalSuccess(t *testing.T) {_, err := protojson.Marshal(metadata)require.NoError(t, err, "marshal must succeed after sanitization")}++func TestOracleDefinitionInvalidUTF8MarshalFailure(t *testing.T) {+ require.False(t, utf8.ValidString(corruptedVietnamese),+ "test precondition: input must be invalid UTF-8")++ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "TESTDB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "TEST_SCHEMA",+ Views: []*storepb.ViewMetadata{+ {+ Name: "V_CUSTOMER",+ Definition: corruptedVietnamese,+ },+ },+ },+ },+ }++ _, err := protojson.Marshal(metadata)+ require.Error(t, err)+ require.Contains(t, err.Error(), "ViewMetadata.definition")+ require.Contains(t, err.Error(), "invalid UTF-8")+}++func TestOracleDefinitionSanitizedUTF8MarshalSuccess(t *testing.T) {+ var logBuf bytes.Buffer+ originalLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})))+ defer slog.SetDefault(originalLogger)+ corruptedDefinition := corruptedVietnamese + "\nSELECT 1 FROM DUAL"++ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "TESTDB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "TEST_SCHEMA",+ Views: []*storepb.ViewMetadata{+ {+ Name: "V_CUSTOMER",+ Definition: sanitizeOracleDefinition(+ "TEST_SCHEMA",+ "VIEW",+ "V_CUSTOMER",+ corruptedDefinition,+ ),+ },+ },+ MaterializedViews: []*storepb.MaterializedViewMetadata{+ {+ Name: "MV_CUSTOMER",+ Definition: sanitizeOracleDefinition(+ "TEST_SCHEMA",+ "MATERIALIZED VIEW",+ "MV_CUSTOMER",+ corruptedDefinition,+ ),+ },+ },+ Functions: []*storepb.FunctionMetadata{+ {+ Name: "FN_VALIDATE",+ Definition: sanitizeOracleDefinition(+ "TEST_SCHEMA",+ "FUNCTION",+ "FN_VALIDATE",+ corruptedDefinition,+ ),+ },+ },+ Procedures: []*storepb.ProcedureMetadata{+ {+ Name: "PR_VALIDATE",+ Definition: sanitizeOracleDefinition(+ "TEST_SCHEMA",+ "PROCEDURE",+ "PR_VALIDATE",+ corruptedDefinition,+ ),+ },+ },+ Packages: []*storepb.PackageMetadata{+ {+ Name: "PKG_VALIDATE",+ Definition: sanitizeOracleDefinition(+ "TEST_SCHEMA",+ "PACKAGE",+ "PKG_VALIDATE",+ corruptedDefinition,+ ),+ },+ },+ },+ },+ }++ _, err := protojson.Marshal(metadata)+ require.NoError(t, err, "marshal must succeed after definition sanitization")+ require.Contains(t, metadata.Schemas[0].Views[0].Definition, "\\xe1")+ require.Contains(t, metadata.Schemas[0].Views[0].Definition, "\nSELECT 1 FROM DUAL")++ logText := logBuf.String()+ require.Contains(t, logText, "sanitized invalid UTF-8 in Oracle metadata")+ require.Contains(t, logText, "schema=TEST_SCHEMA")+ require.Contains(t, logText, "object_type=VIEW")+ require.Contains(t, logText, "object_name=V_CUSTOMER")+ require.Contains(t, logText, "field=definition")+}++func TestOracleTriggerBodySanitizedUTF8MarshalSuccess(t *testing.T) {+ var logBuf bytes.Buffer+ originalLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})))+ defer slog.SetDefault(originalLogger)++ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "TESTDB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "TEST_SCHEMA",+ Tables: []*storepb.TableMetadata{+ {+ Name: "CUSTOMER",+ Triggers: []*storepb.TriggerMetadata{+ {+ Name: "TRG_CUSTOMER",+ Body: sanitizeOracleMetadataString(+ "TEST_SCHEMA",+ "TRIGGER",+ "TRG_CUSTOMER",+ "body",+ constructTriggerBody(+ "TRG_CUSTOMER BEFORE INSERT ON CUSTOMER\n",+ corruptedVietnamese+"\nBEGIN NULL; END;",+ ),+ ),+ },+ },+ },+ },+ },+ },+ }++ _, err := protojson.Marshal(metadata)+ require.NoError(t, err, "marshal must succeed after trigger body sanitization")+ require.Contains(t, metadata.Schemas[0].Tables[0].Triggers[0].Body, "\\xe1")+ require.Contains(t, metadata.Schemas[0].Tables[0].Triggers[0].Body, "\nBEGIN NULL; END;")++ logText := logBuf.String()+ require.Contains(t, logText, "sanitized invalid UTF-8 in Oracle metadata")+ require.Contains(t, logText, "schema=TEST_SCHEMA")+ require.Contains(t, logText, "object_type=TRIGGER")+ require.Contains(t, logText, "object_name=TRG_CUSTOMER")+ require.Contains(t, logText, "field=body")+}