Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(oracle): deep-sanitize UTF-8 across all schema sync string fields (#21005)

rebeliceba1c0900a757
diff --git a/backend/common/util.go b/backend/common/util.go
index ba69471c58..c653bbbe8a 100644
--- a/backend/common/util.go
+++ b/backend/common/util.go
@@ -15,6 +15,8 @@ import (
"github.com/nyaruka/phonenumbers/v2"
"github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
)
const (
@@ -258,3 +260,88 @@ func Uniq[T comparable](array []T) []T {
return res
}
+
+// SanitizeUTF8Message replaces invalid UTF-8 byte sequences in every
+// populated string field of m, recursing into nested messages, repeated
+// fields, and maps (both keys and values). proto3 requires string fields to
+// hold valid UTF-8, so a single raw byte sequence smuggled in by an external
+// system (e.g. a database driver passing through unconverted bytes) makes
+// proto.Marshal fail for the entire message. Callers that assemble metadata
+// from such sources sanitize the finished message once here instead of
+// chasing every scan site.
+func SanitizeUTF8Message(m proto.Message) {
+ if m == nil {
+ return
+ }
+ sanitizeUTF8Value(m.ProtoReflect())
+}
+
+func sanitizeUTF8Value(m protoreflect.Message) {
+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+ switch {
+ case fd.IsMap():
+ sanitizeUTF8Map(fd, v.Map())
+ case fd.IsList():
+ list := v.List()
+ switch fd.Kind() {
+ case protoreflect.StringKind:
+ for i := 0; i < list.Len(); i++ {
+ s := list.Get(i).String()
+ if !utf8.ValidString(s) {
+ list.Set(i, protoreflect.ValueOfString(SanitizeUTF8String(s)))
+ }
+ }
+ case protoreflect.MessageKind, protoreflect.GroupKind:
+ for i := 0; i < list.Len(); i++ {
+ sanitizeUTF8Value(list.Get(i).Message())
+ }
+ default:
+ }
+ case fd.Kind() == protoreflect.StringKind:
+ if s := v.String(); !utf8.ValidString(s) {
+ m.Set(fd, protoreflect.ValueOfString(SanitizeUTF8String(s)))
+ }
+ case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind:
+ sanitizeUTF8Value(v.Message())
+ default:
+ }
+ return true
+ })
+}
+
+func sanitizeUTF8Map(fd protoreflect.FieldDescriptor, mp protoreflect.Map) {
+ valueKind := fd.MapValue().Kind()
+ keyIsString := fd.MapKey().Kind() == protoreflect.StringKind
+
+ type rekey struct {
+ oldKey protoreflect.MapKey
+ newKey protoreflect.MapKey
+ value protoreflect.Value
+ }
+ var rekeys []rekey
+ mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
+ switch valueKind {
+ case protoreflect.StringKind:
+ if s := v.String(); !utf8.ValidString(s) {
+ mp.Set(k, protoreflect.ValueOfString(SanitizeUTF8String(s)))
+ }
+ case protoreflect.MessageKind, protoreflect.GroupKind:
+ sanitizeUTF8Value(v.Message())
+ default:
+ }
+ if keyIsString {
+ if s := k.String(); !utf8.ValidString(s) {
+ rekeys = append(rekeys, rekey{
+ oldKey: k,
+ newKey: protoreflect.ValueOfString(SanitizeUTF8String(s)).MapKey(),
+ value: mp.Get(k),
+ })
+ }
+ }
+ return true
+ })
+ for _, r := range rekeys {
+ mp.Clear(r.oldKey)
+ mp.Set(r.newKey, r.value)
+ }
+}
diff --git a/backend/plugin/db/oracle/sync.go b/backend/plugin/db/oracle/sync.go
index 87e26c9ea1..766f4352a2 100644
--- a/backend/plugin/db/oracle/sync.go
+++ b/backend/plugin/db/oracle/sync.go
@@ -61,7 +61,11 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)
var databases []*storepb.DatabaseSchemaMetadata
for _, schema := range schemas {
databases = append(databases, &storepb.DatabaseSchemaMetadata{
- Name: schema,
+ // The driver can hand back raw non-UTF-8 bytes for dictionary
+ // strings on non-UTF-8 charsets (e.g. ZHS16GBK values ending in a
+ // truncated multi-byte character); proto string fields reject them
+ // at marshal time, so sanitize at the source.
+ Name: common.SanitizeUTF8String(schema),
ServiceName: "",
})
}
@@ -141,6 +145,12 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad
Procedures: procedures,
Packages: packages,
})
+ // Deep-sanitize every string field (names, definitions, defaults, …), not
+ // just comments: the go-ora driver returns wholly-unconverted raw bytes
+ // for any GBK/SJIS/Big5 value that ends in a dangling multi-byte lead
+ // byte, and one invalid string anywhere fails proto marshaling for the
+ // entire database metadata (BYT-9916).
+ common.SanitizeUTF8Message(databaseMetadata)
return databaseMetadata, nil
}
diff --git a/backend/plugin/db/oracle/sync_utf8_test.go b/backend/plugin/db/oracle/sync_utf8_test.go
index 1783708b3d..2d43b360cb 100644
--- a/backend/plugin/db/oracle/sync_utf8_test.go
+++ b/backend/plugin/db/oracle/sync_utf8_test.go
@@ -8,6 +8,8 @@ import (
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -298,3 +300,160 @@ func TestOracleTriggerBodySanitizedUTF8MarshalSuccess(t *testing.T) {
require.Contains(t, logText, "object_name=TRG_CUSTOMER")
require.Contains(t, logText, "field=body")
}
+
+// fillAllStringFields recursively sets every string field reachable from m
+// (including nested messages, repeated fields, and map keys/values) to the
+// given value, instantiating one element for each nested message, list, and
+// map so that no string field in the schema shape stays unvisited.
+func fillAllStringFields(m protoreflect.Message, value string, depth int) {
+ if depth <= 0 {
+ return
+ }
+ fields := m.Descriptor().Fields()
+ for i := 0; i < fields.Len(); i++ {
+ fd := fields.Get(i)
+ switch {
+ case fd.IsMap():
+ mp := m.Mutable(fd).Map()
+ var key protoreflect.MapKey
+ if fd.MapKey().Kind() == protoreflect.StringKind {
+ key = protoreflect.ValueOfString(value).MapKey()
+ } else {
+ key = protoreflect.ValueOfInt64(1).MapKey()
+ }
+ switch fd.MapValue().Kind() {
+ case protoreflect.StringKind:
+ mp.Set(key, protoreflect.ValueOfString(value))
+ case protoreflect.MessageKind:
+ nv := mp.NewValue()
+ fillAllStringFields(nv.Message(), value, depth-1)
+ mp.Set(key, nv)
+ default:
+ mp.Set(key, mp.NewValue())
+ }
+ case fd.IsList():
+ list := m.Mutable(fd).List()
+ switch fd.Kind() {
+ case protoreflect.StringKind:
+ list.Append(protoreflect.ValueOfString(value))
+ case protoreflect.MessageKind:
+ nv := list.NewElement()
+ fillAllStringFields(nv.Message(), value, depth-1)
+ list.Append(nv)
+ default:
+ }
+ case fd.Kind() == protoreflect.StringKind:
+ m.Set(fd, protoreflect.ValueOfString(value))
+ case fd.Kind() == protoreflect.MessageKind:
+ fillAllStringFields(m.Mutable(fd).Message(), value, depth-1)
+ default:
+ }
+ }
+}
+
+// countInvalidStringFields walks m and returns how many string values
+// (fields, list elements, map keys and values) hold invalid UTF-8.
+func countInvalidStringFields(m protoreflect.Message) int {
+ count := 0
+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+ switch {
+ case fd.IsMap():
+ v.Map().Range(func(k protoreflect.MapKey, mv protoreflect.Value) bool {
+ if fd.MapKey().Kind() == protoreflect.StringKind && !utf8.ValidString(k.String()) {
+ count++
+ }
+ switch fd.MapValue().Kind() {
+ case protoreflect.StringKind:
+ if !utf8.ValidString(mv.String()) {
+ count++
+ }
+ case protoreflect.MessageKind:
+ count += countInvalidStringFields(mv.Message())
+ default:
+ }
+ return true
+ })
+ case fd.IsList():
+ list := v.List()
+ for i := 0; i < list.Len(); i++ {
+ switch fd.Kind() {
+ case protoreflect.StringKind:
+ if !utf8.ValidString(list.Get(i).String()) {
+ count++
+ }
+ case protoreflect.MessageKind:
+ count += countInvalidStringFields(list.Get(i).Message())
+ default:
+ }
+ }
+ case fd.Kind() == protoreflect.StringKind:
+ if !utf8.ValidString(v.String()) {
+ count++
+ }
+ case fd.Kind() == protoreflect.MessageKind:
+ count += countInvalidStringFields(v.Message())
+ default:
+ }
+ return true
+ })
+ return count
+}
+
+// TestSanitizeUTF8MessageCoversEveryStringField proves the BYT-9916 fix
+// covers every string field of the database metadata — names, types,
+// defaults, definitions — not just comments: fill each one with the raw-GBK
+// shape go-ora leaks for values ending in a dangling lead byte, then assert
+// sanitization leaves zero invalid strings and proto marshaling succeeds.
+func TestSanitizeUTF8MessageCoversEveryStringField(t *testing.T) {
+ // 测试 + dangling lead byte 0xb1: the exact wholly-unconverted raw GBK
+ // shape go-ora returns (engine-verified against Oracle 11gR2/ZHS16GBK).
+ rawGBK := "\xb2\xe2\xca\xd4\xb1"
+ require.False(t, utf8.ValidString(rawGBK))
+
+ metadata := &storepb.DatabaseSchemaMetadata{}
+ fillAllStringFields(metadata.ProtoReflect(), rawGBK, 8)
+
+ invalidBefore := countInvalidStringFields(metadata.ProtoReflect())
+ require.Greater(t, invalidBefore, 100,
+ "filler must reach the full schema shape; got only %d string fields", invalidBefore)
+ _, err := proto.Marshal(metadata)
+ require.Error(t, err, "pre-sanitize marshal must fail — otherwise this test proves nothing")
+
+ common.SanitizeUTF8Message(metadata)
+
+ require.Equal(t, 0, countInvalidStringFields(metadata.ProtoReflect()),
+ "sanitization must leave zero invalid string values anywhere in the message")
+ _, err = proto.Marshal(metadata)
+ require.NoError(t, err)
+ _, err = protojson.Marshal(metadata)
+ require.NoError(t, err)
+}
+
+// TestSanitizeUTF8MessageNameFields pins the customer-visible BYT-9916 shape:
+// object names (schema/table/column) carrying raw GBK bytes must marshal
+// after sanitization, and valid names must pass through untouched.
+func TestSanitizeUTF8MessageNameFields(t *testing.T) {
+ rawGBK := "AB\xe6"
+ metadata := &storepb.DatabaseSchemaMetadata{
+ Name: rawGBK,
+ Schemas: []*storepb.SchemaMetadata{{
+ Name: rawGBK,
+ Tables: []*storepb.TableMetadata{{
+ Name: rawGBK,
+ Columns: []*storepb.ColumnMetadata{
+ {Name: rawGBK, Type: rawGBK, Default: rawGBK},
+ {Name: "测试列", Type: "VARCHAR2(50)"},
+ },
+ }},
+ }},
+ }
+ _, err := proto.Marshal(metadata)
+ require.Error(t, err)
+
+ common.SanitizeUTF8Message(metadata)
+ _, err = proto.Marshal(metadata)
+ require.NoError(t, err)
+ require.Equal(t, "测试列", metadata.Schemas[0].Tables[0].Columns[1].Name,
+ "valid UTF-8 must pass through unchanged")
+ require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))
+}