Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(oracle): deep-sanitize UTF-8 across all schema sync string fields (#21005)
rebeliceba1c0900a757
diff --git a/backend/common/util.go b/backend/common/util.goindex ba69471c58..c653bbbe8a 100644--- a/backend/common/util.go+++ b/backend/common/util.go@@ -15,6 +15,8 @@ import ("github.com/nyaruka/phonenumbers/v2""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protoreflect")const (@@ -258,3 +260,88 @@ func Uniq[T comparable](array []T) []T {return res}++// SanitizeUTF8Message replaces invalid UTF-8 byte sequences in every+// populated string field of m, recursing into nested messages, repeated+// fields, and maps (both keys and values). proto3 requires string fields to+// hold valid UTF-8, so a single raw byte sequence smuggled in by an external+// system (e.g. a database driver passing through unconverted bytes) makes+// proto.Marshal fail for the entire message. Callers that assemble metadata+// from such sources sanitize the finished message once here instead of+// chasing every scan site.+func SanitizeUTF8Message(m proto.Message) {+ if m == nil {+ return+ }+ sanitizeUTF8Value(m.ProtoReflect())+}++func sanitizeUTF8Value(m protoreflect.Message) {+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {+ switch {+ case fd.IsMap():+ sanitizeUTF8Map(fd, v.Map())+ case fd.IsList():+ list := v.List()+ switch fd.Kind() {+ case protoreflect.StringKind:+ for i := 0; i < list.Len(); i++ {+ s := list.Get(i).String()+ if !utf8.ValidString(s) {+ list.Set(i, protoreflect.ValueOfString(SanitizeUTF8String(s)))+ }+ }+ case protoreflect.MessageKind, protoreflect.GroupKind:+ for i := 0; i < list.Len(); i++ {+ sanitizeUTF8Value(list.Get(i).Message())+ }+ default:+ }+ case fd.Kind() == protoreflect.StringKind:+ if s := v.String(); !utf8.ValidString(s) {+ m.Set(fd, protoreflect.ValueOfString(SanitizeUTF8String(s)))+ }+ case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind:+ sanitizeUTF8Value(v.Message())+ default:+ }+ return true+ })+}++func sanitizeUTF8Map(fd protoreflect.FieldDescriptor, mp protoreflect.Map) {+ valueKind := fd.MapValue().Kind()+ keyIsString := fd.MapKey().Kind() == protoreflect.StringKind++ type rekey struct {+ oldKey protoreflect.MapKey+ newKey protoreflect.MapKey+ value protoreflect.Value+ }+ var rekeys []rekey+ mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {+ switch valueKind {+ case protoreflect.StringKind:+ if s := v.String(); !utf8.ValidString(s) {+ mp.Set(k, protoreflect.ValueOfString(SanitizeUTF8String(s)))+ }+ case protoreflect.MessageKind, protoreflect.GroupKind:+ sanitizeUTF8Value(v.Message())+ default:+ }+ if keyIsString {+ if s := k.String(); !utf8.ValidString(s) {+ rekeys = append(rekeys, rekey{+ oldKey: k,+ newKey: protoreflect.ValueOfString(SanitizeUTF8String(s)).MapKey(),+ value: mp.Get(k),+ })+ }+ }+ return true+ })+ for _, r := range rekeys {+ mp.Clear(r.oldKey)+ mp.Set(r.newKey, r.value)+ }+}diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.goindex 87e26c9ea1..766f4352a2 100644--- a/backend/plugin/db/oracle/sync.go+++ b/backend/plugin/db/oracle/sync.go@@ -61,7 +61,11 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)var databases []*storepb.DatabaseSchemaMetadatafor _, schema := range schemas {databases = append(databases, &storepb.DatabaseSchemaMetadata{- Name: schema,+ // The driver can hand back raw non-UTF-8 bytes for dictionary+ // strings on non-UTF-8 charsets (e.g. ZHS16GBK values ending in a+ // truncated multi-byte character); proto string fields reject them+ // at marshal time, so sanitize at the source.+ Name: common.SanitizeUTF8String(schema),ServiceName: "",})}@@ -141,6 +145,12 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetadProcedures: procedures,Packages: packages,})+ // Deep-sanitize every string field (names, definitions, defaults, …), not+ // just comments: the go-ora driver returns wholly-unconverted raw bytes+ // for any GBK/SJIS/Big5 value that ends in a dangling multi-byte lead+ // byte, and one invalid string anywhere fails proto marshaling for the+ // entire database metadata (BYT-9916).+ common.SanitizeUTF8Message(databaseMetadata)return databaseMetadata, nil}diff --git a/backend/plugin/db/oracle/sync_utf8_test.go b/backend/plugin/db/oracle/sync_utf8_test.goindex 1783708b3d..2d43b360cb 100644--- a/backend/plugin/db/oracle/sync_utf8_test.go+++ b/backend/plugin/db/oracle/sync_utf8_test.go@@ -8,6 +8,8 @@ import ("github.com/stretchr/testify/require""google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protoreflect""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -298,3 +300,160 @@ func TestOracleTriggerBodySanitizedUTF8MarshalSuccess(t *testing.T) {require.Contains(t, logText, "object_name=TRG_CUSTOMER")require.Contains(t, logText, "field=body")}++// fillAllStringFields recursively sets every string field reachable from m+// (including nested messages, repeated fields, and map keys/values) to the+// given value, instantiating one element for each nested message, list, and+// map so that no string field in the schema shape stays unvisited.+func fillAllStringFields(m protoreflect.Message, value string, depth int) {+ if depth <= 0 {+ return+ }+ fields := m.Descriptor().Fields()+ for i := 0; i < fields.Len(); i++ {+ fd := fields.Get(i)+ switch {+ case fd.IsMap():+ mp := m.Mutable(fd).Map()+ var key protoreflect.MapKey+ if fd.MapKey().Kind() == protoreflect.StringKind {+ key = protoreflect.ValueOfString(value).MapKey()+ } else {+ key = protoreflect.ValueOfInt64(1).MapKey()+ }+ switch fd.MapValue().Kind() {+ case protoreflect.StringKind:+ mp.Set(key, protoreflect.ValueOfString(value))+ case protoreflect.MessageKind:+ nv := mp.NewValue()+ fillAllStringFields(nv.Message(), value, depth-1)+ mp.Set(key, nv)+ default:+ mp.Set(key, mp.NewValue())+ }+ case fd.IsList():+ list := m.Mutable(fd).List()+ switch fd.Kind() {+ case protoreflect.StringKind:+ list.Append(protoreflect.ValueOfString(value))+ case protoreflect.MessageKind:+ nv := list.NewElement()+ fillAllStringFields(nv.Message(), value, depth-1)+ list.Append(nv)+ default:+ }+ case fd.Kind() == protoreflect.StringKind:+ m.Set(fd, protoreflect.ValueOfString(value))+ case fd.Kind() == protoreflect.MessageKind:+ fillAllStringFields(m.Mutable(fd).Message(), value, depth-1)+ default:+ }+ }+}++// countInvalidStringFields walks m and returns how many string values+// (fields, list elements, map keys and values) hold invalid UTF-8.+func countInvalidStringFields(m protoreflect.Message) int {+ count := 0+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {+ switch {+ case fd.IsMap():+ v.Map().Range(func(k protoreflect.MapKey, mv protoreflect.Value) bool {+ if fd.MapKey().Kind() == protoreflect.StringKind && !utf8.ValidString(k.String()) {+ count+++ }+ switch fd.MapValue().Kind() {+ case protoreflect.StringKind:+ if !utf8.ValidString(mv.String()) {+ count+++ }+ case protoreflect.MessageKind:+ count += countInvalidStringFields(mv.Message())+ default:+ }+ return true+ })+ case fd.IsList():+ list := v.List()+ for i := 0; i < list.Len(); i++ {+ switch fd.Kind() {+ case protoreflect.StringKind:+ if !utf8.ValidString(list.Get(i).String()) {+ count+++ }+ case protoreflect.MessageKind:+ count += countInvalidStringFields(list.Get(i).Message())+ default:+ }+ }+ case fd.Kind() == protoreflect.StringKind:+ if !utf8.ValidString(v.String()) {+ count+++ }+ case fd.Kind() == protoreflect.MessageKind:+ count += countInvalidStringFields(v.Message())+ default:+ }+ return true+ })+ return count+}++// TestSanitizeUTF8MessageCoversEveryStringField proves the BYT-9916 fix+// covers every string field of the database metadata — names, types,+// defaults, definitions — not just comments: fill each one with the raw-GBK+// shape go-ora leaks for values ending in a dangling lead byte, then assert+// sanitization leaves zero invalid strings and proto marshaling succeeds.+func TestSanitizeUTF8MessageCoversEveryStringField(t *testing.T) {+ // 测试 + dangling lead byte 0xb1: the exact wholly-unconverted raw GBK+ // shape go-ora returns (engine-verified against Oracle 11gR2/ZHS16GBK).+ rawGBK := "\xb2\xe2\xca\xd4\xb1"+ require.False(t, utf8.ValidString(rawGBK))++ metadata := &storepb.DatabaseSchemaMetadata{}+ fillAllStringFields(metadata.ProtoReflect(), rawGBK, 8)++ invalidBefore := countInvalidStringFields(metadata.ProtoReflect())+ require.Greater(t, invalidBefore, 100,+ "filler must reach the full schema shape; got only %d string fields", invalidBefore)+ _, err := proto.Marshal(metadata)+ require.Error(t, err, "pre-sanitize marshal must fail — otherwise this test proves nothing")++ common.SanitizeUTF8Message(metadata)++ require.Equal(t, 0, countInvalidStringFields(metadata.ProtoReflect()),+ "sanitization must leave zero invalid string values anywhere in the message")+ _, err = proto.Marshal(metadata)+ require.NoError(t, err)+ _, err = protojson.Marshal(metadata)+ require.NoError(t, err)+}++// TestSanitizeUTF8MessageNameFields pins the customer-visible BYT-9916 shape:+// object names (schema/table/column) carrying raw GBK bytes must marshal+// after sanitization, and valid names must pass through untouched.+func TestSanitizeUTF8MessageNameFields(t *testing.T) {+ rawGBK := "AB\xe6"+ metadata := &storepb.DatabaseSchemaMetadata{+ Name: rawGBK,+ Schemas: []*storepb.SchemaMetadata{{+ Name: rawGBK,+ Tables: []*storepb.TableMetadata{{+ Name: rawGBK,+ Columns: []*storepb.ColumnMetadata{+ {Name: rawGBK, Type: rawGBK, Default: rawGBK},+ {Name: "测试列", Type: "VARCHAR2(50)"},+ },+ }},+ }},+ }+ _, err := proto.Marshal(metadata)+ require.Error(t, err)++ common.SanitizeUTF8Message(metadata)+ _, err = proto.Marshal(metadata)+ require.NoError(t, err)+ require.Equal(t, "测试列", metadata.Schemas[0].Tables[0].Columns[1].Name,+ "valid UTF-8 must pass through unchanged")+ require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))+}