Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: oracle sync (#20786)

ecmadao666419745ae1
diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.go
index ac936a3451..87e26c9ea1 100644
--- a/backend/plugin/db/oracle/sync.go
+++ b/backend/plugin/db/oracle/sync.go
@@ -7,6 +7,7 @@ import (
"log/slog"
"regexp"
"strings"
+ "unicode/utf8"
"github.com/pkg/errors"
@@ -248,7 +249,13 @@ func getTriggers(txn *sql.Tx, schemaName string) (map[db.TableKey][]*storepb.Tri
key := db.TableKey{Schema: schemaName, Table: tableName.String}
trigger := &storepb.TriggerMetadata{
Name: triggerName.String,
- Body: constructTriggerBody(description.String, triggerBody.String),
+ Body: sanitizeOracleMetadataString(
+ schemaName,
+ "TRIGGER",
+ triggerName.String,
+ "body",
+ constructTriggerBody(description.String, triggerBody.String),
+ ),
}
if triggerType.Valid {
trigger.Timing = triggerType.String
@@ -977,8 +984,7 @@ func getViews(txn *sql.Tx, schemaName string, columnMap map[db.TableKey][]*store
if err := rows.Scan(&schemaName, &view.Name, &view.Definition); err != nil {
return nil, err
}
- // Oracle may include a trailing C-string null terminator via the go-ora driver.
- view.Definition = strings.TrimRight(view.Definition, "\x00")
+ view.Definition = sanitizeOracleDefinition(schemaName, "VIEW", view.Name, view.Definition)
key := db.TableKey{Schema: schemaName, Table: view.Name}
view.Columns = columnMap[key]
view.Triggers = triggerMap[key]
@@ -1124,8 +1130,12 @@ func getMaterializedViews(txn *sql.Tx, schemaName string, _ map[db.TableKey][]*s
if err := rows.Scan(&schemaName, &materializedView.Name, &materializedView.Definition); err != nil {
return nil, err
}
- // Oracle may include a trailing C-string null terminator via the go-ora driver.
- materializedView.Definition = strings.TrimRight(materializedView.Definition, "\x00")
+ materializedView.Definition = sanitizeOracleDefinition(
+ schemaName,
+ "MATERIALIZED VIEW",
+ materializedView.Name,
+ materializedView.Definition,
+ )
// Ensure the definition ends with a newline to match expected format
if materializedView.Definition != "" && !strings.HasSuffix(materializedView.Definition, "\n") {
@@ -1256,7 +1266,7 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [
case "FUNCTION":
function := &storepb.FunctionMetadata{
Name: currentName,
- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),
+ Definition: sanitizeOracleDefinition(schemaName, "FUNCTION", currentName, strings.Join(defText, "")),
}
key := db.TableKey{Schema: schemaName, Table: currentName}
if comment, ok := functionCommentMap[key]; ok {
@@ -1266,12 +1276,12 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [
case "PROCEDURE":
procedures = append(procedures, &storepb.ProcedureMetadata{
Name: currentName,
- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),
+ Definition: sanitizeOracleDefinition(schemaName, "PROCEDURE", currentName, strings.Join(defText, "")),
})
case "PACKAGE":
packages = append(packages, &storepb.PackageMetadata{
Name: currentName,
- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),
+ Definition: sanitizeOracleDefinition(schemaName, "PACKAGE", currentName, strings.Join(defText, "")),
})
default:
// Ignore other types
@@ -1286,7 +1296,7 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [
case "FUNCTION":
function := &storepb.FunctionMetadata{
Name: currentName,
- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),
+ Definition: sanitizeOracleDefinition(schemaName, "FUNCTION", currentName, strings.Join(defText, "")),
}
key := db.TableKey{Schema: schemaName, Table: currentName}
if comment, ok := functionCommentMap[key]; ok {
@@ -1296,12 +1306,12 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [
case "PROCEDURE":
procedures = append(procedures, &storepb.ProcedureMetadata{
Name: currentName,
- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),
+ Definition: sanitizeOracleDefinition(schemaName, "PROCEDURE", currentName, strings.Join(defText, "")),
})
case "PACKAGE":
packages = append(packages, &storepb.PackageMetadata{
Name: currentName,
- Definition: strings.TrimRight(strings.Join(defText, ""), "\x00"),
+ Definition: sanitizeOracleDefinition(schemaName, "PACKAGE", currentName, strings.Join(defText, "")),
})
default:
// Ignore other types
@@ -1316,6 +1326,38 @@ func getRoutines(txn *sql.Tx, schemaName string) ([]*storepb.FunctionMetadata, [
return functions, procedures, packages, nil
}
+func sanitizeOracleDefinition(schemaName, objectType, objectName, definition string) string {
+ return sanitizeOracleMetadataString(schemaName, objectType, objectName, "definition", definition)
+}
+
+func sanitizeOracleMetadataString(schemaName, objectType, objectName, field, value string) string {
+ // Oracle may include a trailing C-string null terminator via the go-ora driver.
+ value = strings.TrimRight(value, "\x00")
+ if utf8.ValidString(value) {
+ return value
+ }
+ slog.Warn("sanitized invalid UTF-8 in Oracle metadata",
+ slog.String("schema", schemaName),
+ slog.String("object_type", objectType),
+ slog.String("object_name", objectName),
+ slog.String("field", field),
+ )
+
+ var sanitized strings.Builder
+ sanitized.Grow(len(value))
+ for i := 0; i < len(value); {
+ r, width := utf8.DecodeRuneInString(value[i:])
+ if r == utf8.RuneError && width == 1 {
+ _, _ = fmt.Fprintf(&sanitized, "\\x%02x", value[i])
+ i++
+ continue
+ }
+ _, _ = sanitized.WriteString(value[i : i+width])
+ i += width
+ }
+ return sanitized.String()
+}
+
// cleanDefaultExpression cleans up Oracle default expressions to make them more portable
// and fix common syntax issues like schema-qualified sequence NEXTVAL calls
func cleanDefaultExpression(defaultExpr, currentSchema string) string {
diff --git a/backend/plugin/db/oracle/sync_utf8_test.go b/backend/plugin/db/oracle/sync_utf8_test.go
index ab1f46630c..1783708b3d 100644
--- a/backend/plugin/db/oracle/sync_utf8_test.go
+++ b/backend/plugin/db/oracle/sync_utf8_test.go
@@ -1,6 +1,8 @@
package oracle
import (
+ "bytes"
+ "log/slog"
"testing"
"unicode/utf8"
@@ -139,3 +141,160 @@ func TestOracleCommentSanitizedUTF8MarshalSuccess(t *testing.T) {
_, err := protojson.Marshal(metadata)
require.NoError(t, err, "marshal must succeed after sanitization")
}
+
+func TestOracleDefinitionInvalidUTF8MarshalFailure(t *testing.T) {
+ require.False(t, utf8.ValidString(corruptedVietnamese),
+ "test precondition: input must be invalid UTF-8")
+
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: "TESTDB",
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "TEST_SCHEMA",
+ Views: []*storepb.ViewMetadata{
+ {
+ Name: "V_CUSTOMER",
+ Definition: corruptedVietnamese,
+ },
+ },
+ },
+ },
+ }
+
+ _, err := protojson.Marshal(metadata)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "ViewMetadata.definition")
+ require.Contains(t, err.Error(), "invalid UTF-8")
+}
+
+func TestOracleDefinitionSanitizedUTF8MarshalSuccess(t *testing.T) {
+ var logBuf bytes.Buffer
+ originalLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})))
+ defer slog.SetDefault(originalLogger)
+ corruptedDefinition := corruptedVietnamese + "\nSELECT 1 FROM DUAL"
+
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: "TESTDB",
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "TEST_SCHEMA",
+ Views: []*storepb.ViewMetadata{
+ {
+ Name: "V_CUSTOMER",
+ Definition: sanitizeOracleDefinition(
+ "TEST_SCHEMA",
+ "VIEW",
+ "V_CUSTOMER",
+ corruptedDefinition,
+ ),
+ },
+ },
+ MaterializedViews: []*storepb.MaterializedViewMetadata{
+ {
+ Name: "MV_CUSTOMER",
+ Definition: sanitizeOracleDefinition(
+ "TEST_SCHEMA",
+ "MATERIALIZED VIEW",
+ "MV_CUSTOMER",
+ corruptedDefinition,
+ ),
+ },
+ },
+ Functions: []*storepb.FunctionMetadata{
+ {
+ Name: "FN_VALIDATE",
+ Definition: sanitizeOracleDefinition(
+ "TEST_SCHEMA",
+ "FUNCTION",
+ "FN_VALIDATE",
+ corruptedDefinition,
+ ),
+ },
+ },
+ Procedures: []*storepb.ProcedureMetadata{
+ {
+ Name: "PR_VALIDATE",
+ Definition: sanitizeOracleDefinition(
+ "TEST_SCHEMA",
+ "PROCEDURE",
+ "PR_VALIDATE",
+ corruptedDefinition,
+ ),
+ },
+ },
+ Packages: []*storepb.PackageMetadata{
+ {
+ Name: "PKG_VALIDATE",
+ Definition: sanitizeOracleDefinition(
+ "TEST_SCHEMA",
+ "PACKAGE",
+ "PKG_VALIDATE",
+ corruptedDefinition,
+ ),
+ },
+ },
+ },
+ },
+ }
+
+ _, err := protojson.Marshal(metadata)
+ require.NoError(t, err, "marshal must succeed after definition sanitization")
+ require.Contains(t, metadata.Schemas[0].Views[0].Definition, "\\xe1")
+ require.Contains(t, metadata.Schemas[0].Views[0].Definition, "\nSELECT 1 FROM DUAL")
+
+ logText := logBuf.String()
+ require.Contains(t, logText, "sanitized invalid UTF-8 in Oracle metadata")
+ require.Contains(t, logText, "schema=TEST_SCHEMA")
+ require.Contains(t, logText, "object_type=VIEW")
+ require.Contains(t, logText, "object_name=V_CUSTOMER")
+ require.Contains(t, logText, "field=definition")
+}
+
+func TestOracleTriggerBodySanitizedUTF8MarshalSuccess(t *testing.T) {
+ var logBuf bytes.Buffer
+ originalLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelWarn})))
+ defer slog.SetDefault(originalLogger)
+
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: "TESTDB",
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "TEST_SCHEMA",
+ Tables: []*storepb.TableMetadata{
+ {
+ Name: "CUSTOMER",
+ Triggers: []*storepb.TriggerMetadata{
+ {
+ Name: "TRG_CUSTOMER",
+ Body: sanitizeOracleMetadataString(
+ "TEST_SCHEMA",
+ "TRIGGER",
+ "TRG_CUSTOMER",
+ "body",
+ constructTriggerBody(
+ "TRG_CUSTOMER BEFORE INSERT ON CUSTOMER\n",
+ corruptedVietnamese+"\nBEGIN NULL; END;",
+ ),
+ ),
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ _, err := protojson.Marshal(metadata)
+ require.NoError(t, err, "marshal must succeed after trigger body sanitization")
+ require.Contains(t, metadata.Schemas[0].Tables[0].Triggers[0].Body, "\\xe1")
+ require.Contains(t, metadata.Schemas[0].Tables[0].Triggers[0].Body, "\nBEGIN NULL; END;")
+
+ logText := logBuf.String()
+ require.Contains(t, logText, "sanitized invalid UTF-8 in Oracle metadata")
+ require.Contains(t, logText, "schema=TEST_SCHEMA")
+ require.Contains(t, logText, "object_type=TRIGGER")
+ require.Contains(t, logText, "object_name=TRG_CUSTOMER")
+ require.Contains(t, logText, "field=body")
+}