Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(oracle): sanitize UTF-8 in all comment fields during schema sync (#19509)
Vincent Huang513f6411a3a9
diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.goindex 12caa42fc2..8c4d684794 100644--- a/backend/plugin/db/oracle/sync.go+++ b/backend/plugin/db/oracle/sync.go@@ -10,6 +10,7 @@ import ("github.com/pkg/errors"+ "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util"@@ -385,7 +386,7 @@ func getTableComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string, ereturn nil, err}key := db.TableKey{Schema: schemaName, Table: tableName}- tableCommentMap[key] = comment+ tableCommentMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)@@ -426,7 +427,7 @@ func getTableColumnComments(txn *sql.Tx, schemaName string) (map[db.ColumnKey]streturn nil, err}key := db.ColumnKey{Schema: schemaName, Table: tableName, Column: columnName}- columnCommentsMap[key] = comment+ columnCommentsMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)@@ -1370,7 +1371,7 @@ func getViewComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string, erreturn nil, err}key := db.TableKey{Schema: schemaName, Table: viewName}- viewCommentMap[key] = comment+ viewCommentMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)@@ -1420,7 +1421,7 @@ func getSequenceComments(txn *sql.Tx, schemaName string) (map[db.TableKey]stringreturn nil, err}key := db.TableKey{Schema: schemaName, Table: sequenceName}- sequenceCommentMap[key] = comment+ sequenceCommentMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)@@ -1471,7 +1472,7 @@ func getFunctionComments(txn *sql.Tx, schemaName string) (map[db.TableKey]stringreturn nil, err}key := db.TableKey{Schema: schemaName, Table: functionName}- functionCommentMap[key] = comment+ functionCommentMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)@@ -1529,7 +1530,7 @@ func getMaterializedViewComments(txn *sql.Tx, schemaName string) (map[db.TableKereturn nil, err}key := db.TableKey{Schema: schemaName, Table: mvName}- materializedViewCommentMap[key] = comment+ materializedViewCommentMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)@@ -1580,7 +1581,7 @@ func getTriggerComments(txn *sql.Tx, schemaName string) (map[db.TableKey]string,return nil, err}key := db.TableKey{Schema: schemaName, Table: triggerName}- triggerCommentMap[key] = comment+ triggerCommentMap[key] = common.SanitizeUTF8String(comment)}if err := rows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, query)diff --git a/backend/plugin/db/oracle/sync_utf8_test.go b/backend/plugin/db/oracle/sync_utf8_test.gonew file mode 100644index 0000000000..ab1f46630c--- /dev/null+++ b/backend/plugin/db/oracle/sync_utf8_test.go@@ -0,0 +1,141 @@+package oracle++import (+ "testing"+ "unicode/utf8"++ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/encoding/protojson"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// corruptedVietnamese simulates Vietnamese text encoded in Windows-1258+// that was inserted into an AL32UTF8 Oracle database via a misconfigured client.+// 0xe1 is 'á' in Win-1258 but an invalid UTF-8 lead byte without proper continuation.+var corruptedVietnamese = "Tr\xe1ng th\xe1i"++func TestOracleCommentInvalidUTF8MarshalFailure(t *testing.T) {+ require.False(t, utf8.ValidString(corruptedVietnamese),+ "test precondition: input must be invalid UTF-8")++ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "TESTDB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "TEST_SCHEMA",+ Tables: []*storepb.TableMetadata{+ {+ Name: "CUSTOMER",+ Comment: corruptedVietnamese,+ Columns: []*storepb.ColumnMetadata{+ {+ Name: "STATUS",+ Comment: corruptedVietnamese,+ },+ },+ Triggers: []*storepb.TriggerMetadata{+ {+ Name: "TRG_AUDIT",+ Comment: corruptedVietnamese,+ },+ },+ },+ },+ Views: []*storepb.ViewMetadata{+ {+ Name: "V_CUSTOMER",+ Comment: corruptedVietnamese,+ },+ },+ MaterializedViews: []*storepb.MaterializedViewMetadata{+ {+ Name: "MV_CUSTOMER",+ Comment: corruptedVietnamese,+ },+ },+ Sequences: []*storepb.SequenceMetadata{+ {+ Name: "SEQ_CUSTOMER",+ Comment: corruptedVietnamese,+ },+ },+ Functions: []*storepb.FunctionMetadata{+ {+ Name: "FN_VALIDATE",+ Comment: corruptedVietnamese,+ },+ },+ },+ },+ }++ // Prove this is the exact failure the customer sees.+ _, err := protojson.Marshal(metadata)+ require.Error(t, err)+ require.Contains(t, err.Error(), "invalid UTF-8")+}++func TestOracleCommentSanitizedUTF8MarshalSuccess(t *testing.T) {+ sanitized := common.SanitizeUTF8String(corruptedVietnamese)++ require.True(t, utf8.ValidString(sanitized),+ "sanitized string must be valid UTF-8")+ require.Contains(t, sanitized, "\\xe1",+ "sanitized string should preserve original bytes as hex")++ metadata := &storepb.DatabaseSchemaMetadata{+ Name: "TESTDB",+ Schemas: []*storepb.SchemaMetadata{+ {+ Name: "TEST_SCHEMA",+ Tables: []*storepb.TableMetadata{+ {+ Name: "CUSTOMER",+ Comment: sanitized,+ Columns: []*storepb.ColumnMetadata{+ {+ Name: "STATUS",+ Comment: sanitized,+ },+ },+ Triggers: []*storepb.TriggerMetadata{+ {+ Name: "TRG_AUDIT",+ Comment: sanitized,+ },+ },+ },+ },+ Views: []*storepb.ViewMetadata{+ {+ Name: "V_CUSTOMER",+ Comment: sanitized,+ },+ },+ MaterializedViews: []*storepb.MaterializedViewMetadata{+ {+ Name: "MV_CUSTOMER",+ Comment: sanitized,+ },+ },+ Sequences: []*storepb.SequenceMetadata{+ {+ Name: "SEQ_CUSTOMER",+ Comment: sanitized,+ },+ },+ Functions: []*storepb.FunctionMetadata{+ {+ Name: "FN_VALIDATE",+ Comment: sanitized,+ },+ },+ },+ },+ }++ _, err := protojson.Marshal(metadata)+ require.NoError(t, err, "marshal must succeed after sanitization")+}