Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(oracle): sanitize UTF-8 in all comment fields during schema sync (#19509)

Vincent Huang513f6411a3a9
diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.go
index 12caa42fc2..8c4d684794 100644
--- a/backend/plugin/db/oracle/sync.go
+++ b/backend/plugin/db/oracle/sync.go
@@ -10,6 +10,7 @@ import (
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
@@ -385,7 +386,7 @@ func getTableComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string, e
return nil, err
}
key := db.TableKey{Schema: schemaName, Table: tableName}
- tableCommentMap[key] = comment
+ tableCommentMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
@@ -426,7 +427,7 @@ func getTableColumnComments(txn *sql.Tx, schemaName string) (map[db.ColumnKey]st
return nil, err
}
key := db.ColumnKey{Schema: schemaName, Table: tableName, Column: columnName}
- columnCommentsMap[key] = comment
+ columnCommentsMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
@@ -1370,7 +1371,7 @@ func getViewComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string, er
return nil, err
}
key := db.TableKey{Schema: schemaName, Table: viewName}
- viewCommentMap[key] = comment
+ viewCommentMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
@@ -1420,7 +1421,7 @@ func getSequenceComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string
return nil, err
}
key := db.TableKey{Schema: schemaName, Table: sequenceName}
- sequenceCommentMap[key] = comment
+ sequenceCommentMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
@@ -1471,7 +1472,7 @@ func getFunctionComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string
return nil, err
}
key := db.TableKey{Schema: schemaName, Table: functionName}
- functionCommentMap[key] = comment
+ functionCommentMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
@@ -1529,7 +1530,7 @@ func getMaterializedViewComments(txn *sql.Tx, schemaName string) (map[db.TableKe
return nil, err
}
key := db.TableKey{Schema: schemaName, Table: mvName}
- materializedViewCommentMap[key] = comment
+ materializedViewCommentMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
@@ -1580,7 +1581,7 @@ func getTriggerComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string,
return nil, err
}
key := db.TableKey{Schema: schemaName, Table: triggerName}
- triggerCommentMap[key] = comment
+ triggerCommentMap[key] = common.SanitizeUTF8String(comment)
}
if err := rows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, query)
diff --git a/backend/plugin/db/oracle/sync_utf8_test.go b/backend/plugin/db/oracle/sync_utf8_test.go
new file mode 100644
index 0000000000..ab1f46630c
--- /dev/null
+++ b/backend/plugin/db/oracle/sync_utf8_test.go
@@ -0,0 +1,141 @@
+package oracle
+
+import (
+ "testing"
+ "unicode/utf8"
+
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/encoding/protojson"
+
+ "github.com/bytebase/bytebase/backend/common"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// corruptedVietnamese simulates Vietnamese text encoded in Windows-1258
+// that was inserted into an AL32UTF8 Oracle database via a misconfigured client.
+// 0xe1 is 'á' in Win-1258 but an invalid UTF-8 lead byte without proper continuation.
+var corruptedVietnamese = "Tr\xe1ng th\xe1i"
+
+func TestOracleCommentInvalidUTF8MarshalFailure(t *testing.T) {
+ require.False(t, utf8.ValidString(corruptedVietnamese),
+ "test precondition: input must be invalid UTF-8")
+
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: "TESTDB",
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "TEST_SCHEMA",
+ Tables: []*storepb.TableMetadata{
+ {
+ Name: "CUSTOMER",
+ Comment: corruptedVietnamese,
+ Columns: []*storepb.ColumnMetadata{
+ {
+ Name: "STATUS",
+ Comment: corruptedVietnamese,
+ },
+ },
+ Triggers: []*storepb.TriggerMetadata{
+ {
+ Name: "TRG_AUDIT",
+ Comment: corruptedVietnamese,
+ },
+ },
+ },
+ },
+ Views: []*storepb.ViewMetadata{
+ {
+ Name: "V_CUSTOMER",
+ Comment: corruptedVietnamese,
+ },
+ },
+ MaterializedViews: []*storepb.MaterializedViewMetadata{
+ {
+ Name: "MV_CUSTOMER",
+ Comment: corruptedVietnamese,
+ },
+ },
+ Sequences: []*storepb.SequenceMetadata{
+ {
+ Name: "SEQ_CUSTOMER",
+ Comment: corruptedVietnamese,
+ },
+ },
+ Functions: []*storepb.FunctionMetadata{
+ {
+ Name: "FN_VALIDATE",
+ Comment: corruptedVietnamese,
+ },
+ },
+ },
+ },
+ }
+
+ // Prove this is the exact failure the customer sees.
+ _, err := protojson.Marshal(metadata)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "invalid UTF-8")
+}
+
+func TestOracleCommentSanitizedUTF8MarshalSuccess(t *testing.T) {
+ sanitized := common.SanitizeUTF8String(corruptedVietnamese)
+
+ require.True(t, utf8.ValidString(sanitized),
+ "sanitized string must be valid UTF-8")
+ require.Contains(t, sanitized, "\\xe1",
+ "sanitized string should preserve original bytes as hex")
+
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: "TESTDB",
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Name: "TEST_SCHEMA",
+ Tables: []*storepb.TableMetadata{
+ {
+ Name: "CUSTOMER",
+ Comment: sanitized,
+ Columns: []*storepb.ColumnMetadata{
+ {
+ Name: "STATUS",
+ Comment: sanitized,
+ },
+ },
+ Triggers: []*storepb.TriggerMetadata{
+ {
+ Name: "TRG_AUDIT",
+ Comment: sanitized,
+ },
+ },
+ },
+ },
+ Views: []*storepb.ViewMetadata{
+ {
+ Name: "V_CUSTOMER",
+ Comment: sanitized,
+ },
+ },
+ MaterializedViews: []*storepb.MaterializedViewMetadata{
+ {
+ Name: "MV_CUSTOMER",
+ Comment: sanitized,
+ },
+ },
+ Sequences: []*storepb.SequenceMetadata{
+ {
+ Name: "SEQ_CUSTOMER",
+ Comment: sanitized,
+ },
+ },
+ Functions: []*storepb.FunctionMetadata{
+ {
+ Name: "FN_VALIDATE",
+ Comment: sanitized,
+ },
+ },
+ },
+ },
+ }
+
+ _, err := protojson.Marshal(metadata)
+ require.NoError(t, err, "marshal must succeed after sanitization")
+}