Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: move backend common helpers to their owners (#21509)
p0nye71e9a32d67c
diff --git a/backend/api/oauth2/resource.go b/backend/api/oauth2/resource.goindex 52a87127b5..123cb5f363 100644--- a/backend/api/oauth2/resource.go+++ b/backend/api/oauth2/resource.go@@ -9,7 +9,7 @@ import ("github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/component/config""github.com/bytebase/bytebase/backend/utils")@@ -112,7 +112,7 @@ func checkConsentedResource(values url.Values, consented string) *oauth2Failureif requested == "" {return nil}- canonical, err := common.NormalizeExternalURL(requested)+ canonical, err := config.NormalizeExternalURL(requested)if err != nil {return &oauth2Failure{code: "invalid_target", description: err.Error()}}@@ -200,7 +200,7 @@ func singleValue(values url.Values, name string) (string, error) {// long-lived. Normalizing at the door keeps the audience single-valued.func validateResource(resource, externalURL string) (string, error) {mcpResource := externalURL + mcpResourcePath- canonical, err := common.NormalizeExternalURL(resource)+ canonical, err := config.NormalizeExternalURL(resource)if err != nil {return "", err}diff --git a/backend/api/v1/release_service_check.go b/backend/api/v1/release_service_check.goindex 125975f2ae..f6147dc504 100644--- a/backend/api/v1/release_service_check.go+++ b/backend/api/v1/release_service_check.go@@ -16,6 +16,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/component/parsercontext"+ "github.com/bytebase/bytebase/backend/component/review"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/advisor"@@ -816,7 +817,7 @@ func getRiskLevelFromStatementTypes(statementTypes []storepb.StatementType) v1pbfor _, statementType := range statementTypes {statementTypeStrings = append(statementTypeStrings, statementType.String())}- switch common.GetRiskLevelFromStatementTypes(statementTypeStrings) {+ switch review.GetRiskLevelFromStatementTypes(statementTypeStrings) {case storepb.RiskLevel_LOW:return v1pb.RiskLevel_LOWcase storepb.RiskLevel_MODERATE:diff --git a/backend/api/v1/setting_service.go b/backend/api/v1/setting_service.goindex f281a4fd83..6916d5f989 100644--- a/backend/api/v1/setting_service.go+++ b/backend/api/v1/setting_service.go@@ -862,7 +862,7 @@ func (s *SettingService) preflightWorkspaceProfilePaths(ctx context.Context, worreturn connect.NewError(connect.CodeInvalidArgument, errors.Errorf("external URL is managed via --external-url command-line flag and cannot be changed through the UI"))}if payload.ExternalUrl != "" {- externalURL, err := common.NormalizeExternalURL(payload.ExternalUrl)+ externalURL, err := config.NormalizeExternalURL(payload.ExternalUrl)if err != nil {return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid external url: %v", err))}diff --git a/backend/bin/server/cmd/root.go b/backend/bin/server/cmd/root.goindex a59b2beaa2..654cabc4a3 100644--- a/backend/bin/server/cmd/root.go+++ b/backend/bin/server/cmd/root.go@@ -15,8 +15,8 @@ import ("github.com/pkg/errors""github.com/spf13/cobra"- "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"+ "github.com/bytebase/bytebase/backend/component/config""github.com/bytebase/bytebase/backend/server")@@ -157,7 +157,7 @@ func start() {var err errorif flags.externalURL != "" {- flags.externalURL, err = common.NormalizeExternalURL(flags.externalURL)+ flags.externalURL, err = config.NormalizeExternalURL(flags.externalURL)if err != nil {slog.Error("invalid --external-url", log.BBError(err))returndiff --git a/backend/common/cel.go b/backend/common/cel.goindex 02b8a46ea5..38ce2dbcf0 100644--- a/backend/common/cel.go+++ b/backend/common/cel.go@@ -232,61 +232,6 @@ func validateCELExpr(expression *expr.Expr, conditionCELAttributes []cel.EnvOptireturn prog, nil}-// QueryExportFactors is the factors for query and export.-type QueryExportFactors struct {- Databases []string-}--// GetQueryExportFactors is used to get risk factors from query and export expressions.-func GetQueryExportFactors(expression string) (*QueryExportFactors, error) {- // If the expression is empty, return an empty struct.- if expression == "" {- return &QueryExportFactors{}, nil- }-- factors := &QueryExportFactors{}- e, err := cel.NewEnv(IAMPolicyConditionCELAttributes...)- if err != nil {- return nil, err- }- ast, issues := e.Compile(expression)- if issues != nil {- return nil, errors.Errorf("found issue %v", issues)- }- parsedExpr, err := cel.AstToParsedExpr(ast)- if err != nil {- return nil, err- }- callExpr := parsedExpr.Expr.GetCallExpr()- findField(callExpr, factors)- return factors, nil-}--func findField(callExpr *exprproto.Expr_Call, factors *QueryExportFactors) {- if callExpr == nil {- return- }- if len(callExpr.Args) == 2 {- idExpr := callExpr.Args[0].GetIdentExpr()- if idExpr != nil {- if idExpr.Name == CELAttributeResourceDatabase && callExpr.Function == "_==_" {- factors.Databases = append(factors.Databases, callExpr.Args[1].GetConstExpr().GetStringValue())- }- if idExpr.Name == CELAttributeResourceDatabase && callExpr.Function == "@in" {- list := callExpr.Args[1].GetListExpr()- for _, element := range list.Elements {- factors.Databases = append(factors.Databases, element.GetConstExpr().GetStringValue())- }- }- return- }- }- for _, arg := range callExpr.Args {- callExpr := arg.GetCallExpr()- findField(callExpr, factors)- }-}-func EvalBindingCondition(expr string, requestTime time.Time) (bool, error) {input := map[string]any{CELAttributeRequestTime: requestTime,diff --git a/backend/common/cel_test.go b/backend/common/cel_test.goindex 742dfb4d1d..fc687d8e97 100644--- a/backend/common/cel_test.go+++ b/backend/common/cel_test.go@@ -6,7 +6,6 @@ import ("time""github.com/google/cel-go/cel"- "github.com/stretchr/testify/assert""github.com/stretchr/testify/require"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -75,48 +74,6 @@ func TestPartialEval(t *testing.T) {}}-func TestGetQueryExportFactors(t *testing.T) {- a := assert.New(t)- tests := []struct {- expression string- want QueryExportFactors- }{- {- expression: "request.time < timestamp(\"2023-07-04T06:09:03.384Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_manager\"])",- want: QueryExportFactors{- Databases: []string{"instances/postgres-sample/databases/employee"},- },- },- {- expression: "request.time < timestamp(\"2023-07-04T07:40:05.658Z\") && (resource.database in [\"instances/postgres-sample/databases/employee\"])",- want: QueryExportFactors{- Databases: []string{"instances/postgres-sample/databases/employee"},- },- },- {- expression: "request.time < timestamp(\"2023-08-02T07:33:45.686Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_emp\",\"department\"])",- want: QueryExportFactors{- Databases: []string{"instances/postgres-sample/databases/employee"},- },- },- {- expression: "request.time < timestamp(\"2023-07-10T08:14:34.788Z\")",- want: QueryExportFactors{},- },- {- expression: "request.time < timestamp(\"2023-07-10T08:15:46.773Z\") && ((resource.database in [\"instances/postgres-sample/databases/blog\"]) || (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name in [\"public\"]))",- want: QueryExportFactors{- Databases: []string{"instances/postgres-sample/databases/blog", "instances/postgres-sample/databases/employee"},- },- },- }- for _, tt := range tests {- factors, err := GetQueryExportFactors(tt.expression)- a.NoError(err)- a.Equal(tt.want, *factors)- }-}-func TestFallbackApprovalFactors(t *testing.T) {a := require.New(t)diff --git a/backend/common/position.go b/backend/common/position.godeleted file mode 100644index 687dcc2a10..0000000000--- a/backend/common/position.go+++ /dev/null@@ -1,67 +0,0 @@-//nolint:revive-package common--import (- storepb "github.com/bytebase/bytebase/backend/generated-go/store"-)--// ANTLRPosition is a position in a text expressed as one-based line and-// zero-based column character (code point) offset integrated with ANTLR4.-type ANTLRPosition struct {- // Line position in a text (one-based).- Line int32- // Column position in a text (zero-based), equivalent to character offset.- Column int32-}--// ConvertANTLRPositionToPosition converts an ANTLRPosition to a Position in a given text.-// ANTLRPosition uses 1-based line and 0-based character column.-// Returns a Position with 1-based line and 1-based character column.-// Returns the end Position if the ANTLRPosition is out of the end of text,-// returns the previous Position if the ANTLRPosition is out of the end of a line.-func ConvertANTLRPositionToPosition(a *ANTLRPosition, text string) *storepb.Position {- runes := []rune(text)- line := int32(1) // Start at 1 for 1-based line numbering- charOffsetInLine := int32(0)- globalCharOffset := int32(0)-- for globalCharOffset < int32(len(runes)) {- // Skip lines before the target line- if line < a.Line {- if runes[globalCharOffset] == '\n' {- line++- }- globalCharOffset++- continue- }-- // Stop if we've reached the target column or hit a newline- if charOffsetInLine >= a.Column || runes[globalCharOffset] == '\n' {- break- }-- charOffsetInLine++- globalCharOffset++- }-- // Convert from 0-based to 1-based column- return &storepb.Position{- Line: line,- Column: charOffsetInLine + 1,- }-}--// ConvertANTLRTokenToExclusiveEndPosition converts an ANTLR token position to an exclusive end Position.-// The end token's line and column (0-based) plus its text are used to calculate where the position-// should point to (after the last character of the token).-// This is used for Statement.End which should use exclusive semantics (pointing after the last character).-func ConvertANTLRTokenToExclusiveEndPosition(line int32, column int32, tokenText string) *storepb.Position {- tokenCharLength := int32(len([]rune(tokenText)))- // For exclusive end, we simply add the token length to the column.- // The column is 0-based in ANTLR, so we add 1 to convert to 1-based.- // The result points to the position AFTER the last character of the token.- return &storepb.Position{- Line: line,- Column: column + tokenCharLength + 1,- }-}diff --git a/backend/common/position_test.go b/backend/common/position_test.godeleted file mode 100644index 0ab14d23e6..0000000000--- a/backend/common/position_test.go+++ /dev/null@@ -1,45 +0,0 @@-//nolint:revive-package common--import (- "testing"-- "github.com/stretchr/testify/require"-- storepb "github.com/bytebase/bytebase/backend/generated-go/store"-)--func TestConvertANTLRPositionToPosition(t *testing.T) {- testCases := []struct {- description string- text string- antlrPosition *ANTLRPosition- want *storepb.Position- }{- {- description: "first line first column",- text: "hello, world",- antlrPosition: &ANTLRPosition{Line: 1, Column: 0},- want: &storepb.Position{Line: 1, Column: 1},- },- {- description: "ASCII text",- text: "hello, world",- antlrPosition: &ANTLRPosition{Line: 1, Column: 6},- want: &storepb.Position{Line: 1, Column: 7}, // ANTLR column 6 (0-based) -> column 7 (1-based)- },- {- description: "multi-byte characters",- text: "你好\n世界",- antlrPosition: &ANTLRPosition{Line: 2, Column: 1},- want: &storepb.Position{Line: 2, Column: 2}, // Character-based, not byte-based- },- }-- for _, tc := range testCases {- t.Run(tc.description, func(t *testing.T) {- got := ConvertANTLRPositionToPosition(tc.antlrPosition, tc.text)- require.Equal(t, tc.want, got)- })- }-}diff --git a/backend/common/proto.go b/backend/common/proto.gonew file mode 100644index 0000000000..a124c7e8b1--- /dev/null+++ b/backend/common/proto.go@@ -0,0 +1,94 @@+//nolint:revive+package common++import (+ "unicode/utf8"++ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protoreflect"+)++// SanitizeUTF8Message replaces invalid UTF-8 byte sequences in every+// populated string field of m, recursing into nested messages, repeated+// fields, and maps (both keys and values). proto3 requires string fields to+// hold valid UTF-8, so a single raw byte sequence smuggled in by an external+// system (e.g. a database driver passing through unconverted bytes) makes+// proto.Marshal fail for the entire message. Callers that assemble metadata+// from such sources sanitize the finished message once here instead of+// chasing every scan site.+func SanitizeUTF8Message(m proto.Message) {+ if m == nil {+ return+ }+ sanitizeUTF8Value(m.ProtoReflect())+}++func sanitizeUTF8Value(m protoreflect.Message) {+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {+ switch {+ case fd.IsMap():+ sanitizeUTF8Map(fd, v.Map())+ case fd.IsList():+ list := v.List()+ switch fd.Kind() {+ case protoreflect.StringKind:+ for i := 0; i < list.Len(); i++ {+ s := list.Get(i).String()+ if !utf8.ValidString(s) {+ list.Set(i, protoreflect.ValueOfString(SanitizeUTF8String(s)))+ }+ }+ case protoreflect.MessageKind, protoreflect.GroupKind:+ for i := 0; i < list.Len(); i++ {+ sanitizeUTF8Value(list.Get(i).Message())+ }+ default:+ }+ case fd.Kind() == protoreflect.StringKind:+ if s := v.String(); !utf8.ValidString(s) {+ m.Set(fd, protoreflect.ValueOfString(SanitizeUTF8String(s)))+ }+ case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind:+ sanitizeUTF8Value(v.Message())+ default:+ }+ return true+ })+}++func sanitizeUTF8Map(fd protoreflect.FieldDescriptor, mp protoreflect.Map) {+ valueKind := fd.MapValue().Kind()+ keyIsString := fd.MapKey().Kind() == protoreflect.StringKind++ type rekey struct {+ oldKey protoreflect.MapKey+ newKey protoreflect.MapKey+ value protoreflect.Value+ }+ var rekeys []rekey+ mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {+ switch valueKind {+ case protoreflect.StringKind:+ if s := v.String(); !utf8.ValidString(s) {+ mp.Set(k, protoreflect.ValueOfString(SanitizeUTF8String(s)))+ }+ case protoreflect.MessageKind, protoreflect.GroupKind:+ sanitizeUTF8Value(v.Message())+ default:+ }+ if keyIsString {+ if s := k.String(); !utf8.ValidString(s) {+ rekeys = append(rekeys, rekey{+ oldKey: k,+ newKey: protoreflect.ValueOfString(SanitizeUTF8String(s)).MapKey(),+ value: mp.Get(k),+ })+ }+ }+ return true+ })+ for _, r := range rekeys {+ mp.Clear(r.oldKey)+ mp.Set(r.newKey, r.value)+ }+}diff --git a/backend/common/proto_test.go b/backend/common/proto_test.gonew file mode 100644index 0000000000..69de1fdb03--- /dev/null+++ b/backend/common/proto_test.go@@ -0,0 +1,170 @@+//nolint:revive+package common++import (+ "testing"+ "unicode/utf8"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/encoding/protojson"+ "google.golang.org/protobuf/proto"+ "google.golang.org/protobuf/reflect/protoreflect"+)++// fillAllStringFields recursively sets every string field reachable from m+// (including nested messages, repeated fields, and map keys/values) to the+// given value, instantiating one element for each nested message, list, and+// map so that no string field in the schema shape stays unvisited.+func fillAllStringFields(m protoreflect.Message, value string, depth int) {+ if depth <= 0 {+ return+ }+ fields := m.Descriptor().Fields()+ for i := 0; i < fields.Len(); i++ {+ fd := fields.Get(i)+ switch {+ case fd.IsMap():+ mp := m.Mutable(fd).Map()+ var key protoreflect.MapKey+ if fd.MapKey().Kind() == protoreflect.StringKind {+ key = protoreflect.ValueOfString(value).MapKey()+ } else {+ key = protoreflect.ValueOfInt64(1).MapKey()+ }+ switch fd.MapValue().Kind() {+ case protoreflect.StringKind:+ mp.Set(key, protoreflect.ValueOfString(value))+ case protoreflect.MessageKind:+ nv := mp.NewValue()+ fillAllStringFields(nv.Message(), value, depth-1)+ mp.Set(key, nv)+ default:+ mp.Set(key, mp.NewValue())+ }+ case fd.IsList():+ list := m.Mutable(fd).List()+ switch fd.Kind() {+ case protoreflect.StringKind:+ list.Append(protoreflect.ValueOfString(value))+ case protoreflect.MessageKind:+ nv := list.NewElement()+ fillAllStringFields(nv.Message(), value, depth-1)+ list.Append(nv)+ default:+ }+ case fd.Kind() == protoreflect.StringKind:+ m.Set(fd, protoreflect.ValueOfString(value))+ case fd.Kind() == protoreflect.MessageKind:+ fillAllStringFields(m.Mutable(fd).Message(), value, depth-1)+ default:+ }+ }+}++// countInvalidStringFields walks m and returns how many string values+// (fields, list elements, map keys and values) hold invalid UTF-8.+func countInvalidStringFields(m protoreflect.Message) int {+ count := 0+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {+ switch {+ case fd.IsMap():+ v.Map().Range(func(k protoreflect.MapKey, mv protoreflect.Value) bool {+ if fd.MapKey().Kind() == protoreflect.StringKind && !utf8.ValidString(k.String()) {+ count+++ }+ switch fd.MapValue().Kind() {+ case protoreflect.StringKind:+ if !utf8.ValidString(mv.String()) {+ count+++ }+ case protoreflect.MessageKind:+ count += countInvalidStringFields(mv.Message())+ default:+ }+ return true+ })+ case fd.IsList():+ list := v.List()+ for i := 0; i < list.Len(); i++ {+ switch fd.Kind() {+ case protoreflect.StringKind:+ if !utf8.ValidString(list.Get(i).String()) {+ count+++ }+ case protoreflect.MessageKind:+ count += countInvalidStringFields(list.Get(i).Message())+ default:+ }+ }+ case fd.Kind() == protoreflect.StringKind:+ if !utf8.ValidString(v.String()) {+ count+++ }+ case fd.Kind() == protoreflect.MessageKind:+ count += countInvalidStringFields(v.Message())+ default:+ }+ return true+ })+ return count+}++// TestSanitizeUTF8MessageCoversEveryStringField proves the BYT-9916 fix+// covers every string field of the database metadata — names, types,+// defaults, definitions — not just comments: fill each one with the raw-GBK+// shape go-ora leaks for values ending in a dangling lead byte, then assert+// sanitization leaves zero invalid strings and proto marshaling succeeds.+func TestSanitizeUTF8MessageCoversEveryStringField(t *testing.T) {+ // 测试 + dangling lead byte 0xb1: the exact wholly-unconverted raw GBK+ // shape go-ora returns (engine-verified against Oracle 11gR2/ZHS16GBK).+ rawGBK := "\xb2\xe2\xca\xd4\xb1"+ require.False(t, utf8.ValidString(rawGBK))++ metadata := &metadatapb.DatabaseSchemaMetadata{}+ fillAllStringFields(metadata.ProtoReflect(), rawGBK, 8)++ invalidBefore := countInvalidStringFields(metadata.ProtoReflect())+ require.Greater(t, invalidBefore, 100,+ "filler must reach the full schema shape; got only %d string fields", invalidBefore)+ _, err := proto.Marshal(metadata)+ require.Error(t, err, "pre-sanitize marshal must fail — otherwise this test proves nothing")++ SanitizeUTF8Message(metadata)++ require.Equal(t, 0, countInvalidStringFields(metadata.ProtoReflect()),+ "sanitization must leave zero invalid string values anywhere in the message")+ _, err = proto.Marshal(metadata)+ require.NoError(t, err)+ _, err = protojson.Marshal(metadata)+ require.NoError(t, err)+}++// TestSanitizeUTF8MessageNameFields pins the customer-visible BYT-9916 shape:+// object names (schema/table/column) carrying raw GBK bytes must marshal+// after sanitization, and valid names must pass through untouched.+func TestSanitizeUTF8MessageNameFields(t *testing.T) {+ rawGBK := "AB\xe6"+ metadata := &metadatapb.DatabaseSchemaMetadata{+ Name: rawGBK,+ Schemas: []*metadatapb.SchemaMetadata{{+ Name: rawGBK,+ Tables: []*metadatapb.TableMetadata{{+ Name: rawGBK,+ Columns: []*metadatapb.ColumnMetadata{+ {Name: rawGBK, Type: rawGBK, Default: rawGBK},+ {Name: "测试列", Type: "VARCHAR2(50)"},+ },+ }},+ }},+ }+ _, err := proto.Marshal(metadata)+ require.Error(t, err)++ SanitizeUTF8Message(metadata)+ _, err = proto.Marshal(metadata)+ require.NoError(t, err)+ require.Equal(t, "测试列", metadata.Schemas[0].Tables[0].Columns[1].Name,+ "valid UTF-8 must pass through unchanged")+ require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))+}diff --git a/backend/common/util.go b/backend/common/util.goindex d4f43c146f..0d943a254e 100644--- a/backend/common/util.go+++ b/backend/common/util.go@@ -6,7 +6,6 @@ import ("fmt""math""math/big"- "net/url""reflect""regexp""strings"@@ -15,8 +14,6 @@ import ("github.com/nyaruka/phonenumbers/v2""github.com/pkg/errors""google.golang.org/protobuf/encoding/protojson"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/reflect/protoreflect")const (@@ -90,48 +87,6 @@ func TruncateString(str string, limit int) (string, bool) {return str, false}-// NormalizeExternalURL will format the external url.-func NormalizeExternalURL(externalURL string) (string, error) {- r := strings.TrimSpace(externalURL)- r = strings.TrimSuffix(r, "/")- u, err := url.Parse(r)- if err != nil {- return "", errors.Wrapf(err, "%s malformed", externalURL)- }- scheme := strings.ToLower(u.Scheme)- if scheme != "http" && scheme != "https" {- return "", errors.Errorf("%s must start with http:// or https://", externalURL)- }- if u.Host == "" {- return "", errors.Errorf("%s must name a host", externalURL)- }- if u.User != nil {- return "", errors.Errorf("%s must not carry userinfo", externalURL)- }- if u.RawQuery != "" || u.ForceQuery {- return "", errors.Errorf("%s must not carry a query string", externalURL)- }- if u.Fragment != "" || u.RawFragment != "" {- return "", errors.Errorf("%s must not carry a fragment", externalURL)- }-- host := strings.ToLower(u.Host)- port := u.Port()- if port != "" {- // The external URL is used as the redirectURL in the get token process of OAuth, and the- // RedirectURL needs to be consistent with the RedirectURL in the get code process.- // The frontend gets it through window.location.origin in the get code- // process, so port 80/443 need to be cropped.- if (scheme == "http" && port == "80") || (scheme == "https" && port == "443") {- host = strings.ToLower(u.Hostname())- if strings.Contains(host, ":") {- host = "[" + host + "]"- }- }- }- return scheme + "://" + host + strings.TrimSuffix(u.EscapedPath(), "/"), nil-}-// ValidatePhone validates the phone number.func ValidatePhone(phone string) error {phoneNumber, err := phonenumbers.Parse(phone, "")@@ -208,10 +163,6 @@ func SanitizeUTF8String(s string) string {return b.String()}-func FormatMaximumSQLResultSizeMessage(limit int64) string {- return fmt.Sprintf("Output of query exceeds max allowed output size of %dMB", limit/1024/1024)-}-func IsNil(val any) bool {if val == nil {return true@@ -245,88 +196,3 @@ func Uniq[T comparable](array []T) []T {return res}--// SanitizeUTF8Message replaces invalid UTF-8 byte sequences in every-// populated string field of m, recursing into nested messages, repeated-// fields, and maps (both keys and values). proto3 requires string fields to-// hold valid UTF-8, so a single raw byte sequence smuggled in by an external-// system (e.g. a database driver passing through unconverted bytes) makes-// proto.Marshal fail for the entire message. Callers that assemble metadata-// from such sources sanitize the finished message once here instead of-// chasing every scan site.-func SanitizeUTF8Message(m proto.Message) {- if m == nil {- return- }- sanitizeUTF8Value(m.ProtoReflect())-}--func sanitizeUTF8Value(m protoreflect.Message) {- m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {- switch {- case fd.IsMap():- sanitizeUTF8Map(fd, v.Map())- case fd.IsList():- list := v.List()- switch fd.Kind() {- case protoreflect.StringKind:- for i := 0; i < list.Len(); i++ {- s := list.Get(i).String()- if !utf8.ValidString(s) {- list.Set(i, protoreflect.ValueOfString(SanitizeUTF8String(s)))- }- }- case protoreflect.MessageKind, protoreflect.GroupKind:- for i := 0; i < list.Len(); i++ {- sanitizeUTF8Value(list.Get(i).Message())- }- default:- }- case fd.Kind() == protoreflect.StringKind:- if s := v.String(); !utf8.ValidString(s) {- m.Set(fd, protoreflect.ValueOfString(SanitizeUTF8String(s)))- }- case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind:- sanitizeUTF8Value(v.Message())- default:- }- return true- })-}--func sanitizeUTF8Map(fd protoreflect.FieldDescriptor, mp protoreflect.Map) {- valueKind := fd.MapValue().Kind()- keyIsString := fd.MapKey().Kind() == protoreflect.StringKind-- type rekey struct {- oldKey protoreflect.MapKey- newKey protoreflect.MapKey- value protoreflect.Value- }- var rekeys []rekey- mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {- switch valueKind {- case protoreflect.StringKind:- if s := v.String(); !utf8.ValidString(s) {- mp.Set(k, protoreflect.ValueOfString(SanitizeUTF8String(s)))- }- case protoreflect.MessageKind, protoreflect.GroupKind:- sanitizeUTF8Value(v.Message())- default:- }- if keyIsString {- if s := k.String(); !utf8.ValidString(s) {- rekeys = append(rekeys, rekey{- oldKey: k,- newKey: protoreflect.ValueOfString(SanitizeUTF8String(s)).MapKey(),- value: mp.Get(k),- })- }- }- return true- })- for _, r := range rekeys {- mp.Clear(r.oldKey)- mp.Set(r.newKey, r.value)- }-}diff --git a/backend/common/util_test.go b/backend/common/util_test.goindex 19d98b9343..f2085c4818 100644--- a/backend/common/util_test.go+++ b/backend/common/util_test.go@@ -6,12 +6,8 @@ import ("testing""unicode/utf8"- metadatapb "github.com/bytebase/omni/metadata""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"- "google.golang.org/protobuf/encoding/protojson"- "google.golang.org/protobuf/proto"- "google.golang.org/protobuf/reflect/protoreflect")func TestTruncateString(t *testing.T) {@@ -90,93 +86,6 @@ func TestTruncateString(t *testing.T) {}}-func TestNormalizeExternalURL(t *testing.T) {- tests := []struct {- url string- want string- wantErr bool- }{- {- url: "http://localhost:3000",- want: "http://localhost:3000",- wantErr: false,- },- {- url: "https://localhost:3000",- want: "https://localhost:3000",- wantErr: false,- },- {- url: "https://localhost",- want: "https://localhost",- wantErr: false,- },- {- url: "http://localhost:80",- want: "http://localhost",- wantErr: false,- },- {- url: "https://localhost:443",- want: "https://localhost",- wantErr: false,- },- {- url: " https://localhost:3000/ ",- want: "https://localhost:3000",- wantErr: false,- },- {- url: "HTTPS://LOCALHOST:443/mcp/",- want: "https://localhost/mcp",- wantErr: false,- },- // Missing http:// or https://- {- url: "localhost:3000",- want: "",- wantErr: true,- },- // Invalid port- {- url: "http://localhost:xxx",- want: "",- wantErr: true,- },- {- url: "https://user@localhost",- want: "",- wantErr: true,- },- {- url: "https://localhost?x=1",- want: "",- wantErr: true,- },- {- url: "https://localhost#fragment",- want: "",- wantErr: true,- },- }- for _, tt := range tests {- t.Run(tt.url, func(t *testing.T) {- g, err := NormalizeExternalURL(tt.url)- if err != nil {- if !tt.wantErr {- t.Errorf("expect no error, got %s", err.Error())- }- } else {- if tt.wantErr {- t.Error("expect error")- } else if tt.want != g {- t.Errorf("expect %s, got %s", tt.want, g)- }- }- })- }-}-func TestValidatePhone(t *testing.T) {tests := []struct {phone string@@ -217,163 +126,6 @@ func TestSanitizeUTF8String(t *testing.T) {require.Equal(t, "\u6d4b\u8bd5", SanitizeUTF8String("\u6d4b\u8bd5"), "valid UTF-8 must pass through unchanged")}-// fillAllStringFields recursively sets every string field reachable from m-// (including nested messages, repeated fields, and map keys/values) to the-// given value, instantiating one element for each nested message, list, and-// map so that no string field in the schema shape stays unvisited.-func fillAllStringFields(m protoreflect.Message, value string, depth int) {- if depth <= 0 {- return- }- fields := m.Descriptor().Fields()- for i := 0; i < fields.Len(); i++ {- fd := fields.Get(i)- switch {- case fd.IsMap():- mp := m.Mutable(fd).Map()- var key protoreflect.MapKey- if fd.MapKey().Kind() == protoreflect.StringKind {- key = protoreflect.ValueOfString(value).MapKey()- } else {- key = protoreflect.ValueOfInt64(1).MapKey()- }- switch fd.MapValue().Kind() {- case protoreflect.StringKind:- mp.Set(key, protoreflect.ValueOfString(value))- case protoreflect.MessageKind:- nv := mp.NewValue()- fillAllStringFields(nv.Message(), value, depth-1)- mp.Set(key, nv)- default:- mp.Set(key, mp.NewValue())- }- case fd.IsList():- list := m.Mutable(fd).List()- switch fd.Kind() {- case protoreflect.StringKind:- list.Append(protoreflect.ValueOfString(value))- case protoreflect.MessageKind:- nv := list.NewElement()- fillAllStringFields(nv.Message(), value, depth-1)- list.Append(nv)- default:- }- case fd.Kind() == protoreflect.StringKind:- m.Set(fd, protoreflect.ValueOfString(value))- case fd.Kind() == protoreflect.MessageKind:- fillAllStringFields(m.Mutable(fd).Message(), value, depth-1)- default:- }- }-}--// countInvalidStringFields walks m and returns how many string values-// (fields, list elements, map keys and values) hold invalid UTF-8.-func countInvalidStringFields(m protoreflect.Message) int {- count := 0- m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {- switch {- case fd.IsMap():- v.Map().Range(func(k protoreflect.MapKey, mv protoreflect.Value) bool {- if fd.MapKey().Kind() == protoreflect.StringKind && !utf8.ValidString(k.String()) {- count++- }- switch fd.MapValue().Kind() {- case protoreflect.StringKind:- if !utf8.ValidString(mv.String()) {- count++- }- case protoreflect.MessageKind:- count += countInvalidStringFields(mv.Message())- default:- }- return true- })- case fd.IsList():- list := v.List()- for i := 0; i < list.Len(); i++ {- switch fd.Kind() {- case protoreflect.StringKind:- if !utf8.ValidString(list.Get(i).String()) {- count++- }- case protoreflect.MessageKind:- count += countInvalidStringFields(list.Get(i).Message())- default:- }- }- case fd.Kind() == protoreflect.StringKind:- if !utf8.ValidString(v.String()) {- count++- }- case fd.Kind() == protoreflect.MessageKind:- count += countInvalidStringFields(v.Message())- default:- }- return true- })- return count-}--// TestSanitizeUTF8MessageCoversEveryStringField proves the BYT-9916 fix-// covers every string field of the database metadata — names, types,-// defaults, definitions — not just comments: fill each one with the raw-GBK-// shape go-ora leaks for values ending in a dangling lead byte, then assert-// sanitization leaves zero invalid strings and proto marshaling succeeds.-func TestSanitizeUTF8MessageCoversEveryStringField(t *testing.T) {- // 测试 + dangling lead byte 0xb1: the exact wholly-unconverted raw GBK- // shape go-ora returns (engine-verified against Oracle 11gR2/ZHS16GBK).- rawGBK := "\xb2\xe2\xca\xd4\xb1"- require.False(t, utf8.ValidString(rawGBK))-- metadata := &metadatapb.DatabaseSchemaMetadata{}- fillAllStringFields(metadata.ProtoReflect(), rawGBK, 8)-- invalidBefore := countInvalidStringFields(metadata.ProtoReflect())- require.Greater(t, invalidBefore, 100,- "filler must reach the full schema shape; got only %d string fields", invalidBefore)- _, err := proto.Marshal(metadata)- require.Error(t, err, "pre-sanitize marshal must fail — otherwise this test proves nothing")-- SanitizeUTF8Message(metadata)-- require.Equal(t, 0, countInvalidStringFields(metadata.ProtoReflect()),- "sanitization must leave zero invalid string values anywhere in the message")- _, err = proto.Marshal(metadata)- require.NoError(t, err)- _, err = protojson.Marshal(metadata)- require.NoError(t, err)-}--// TestSanitizeUTF8MessageNameFields pins the customer-visible BYT-9916 shape:-// object names (schema/table/column) carrying raw GBK bytes must marshal-// after sanitization, and valid names must pass through untouched.-func TestSanitizeUTF8MessageNameFields(t *testing.T) {- rawGBK := "AB\xe6"- metadata := &metadatapb.DatabaseSchemaMetadata{- Name: rawGBK,- Schemas: []*metadatapb.SchemaMetadata{{- Name: rawGBK,- Tables: []*metadatapb.TableMetadata{{- Name: rawGBK,- Columns: []*metadatapb.ColumnMetadata{- {Name: rawGBK, Type: rawGBK, Default: rawGBK},- {Name: "测试列", Type: "VARCHAR2(50)"},- },- }},- }},- }- _, err := proto.Marshal(metadata)- require.Error(t, err)-- SanitizeUTF8Message(metadata)- _, err = proto.Marshal(metadata)- require.NoError(t, err)- require.Equal(t, "测试列", metadata.Schemas[0].Tables[0].Columns[1].Name,- "valid UTF-8 must pass through unchanged")- require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))-}-func TestRoundRows(t *testing.T) {require.Equal(t, int64(3), RoundRows(2.5))require.Equal(t, int64(1000), RoundRows(999.6))diff --git a/backend/component/config/url.go b/backend/component/config/url.gonew file mode 100644index 0000000000..bce6a841c3--- /dev/null+++ b/backend/component/config/url.go@@ -0,0 +1,50 @@+package config++import (+ "net/url"+ "strings"++ "github.com/pkg/errors"+)++// NormalizeExternalURL will format the external url.+func NormalizeExternalURL(externalURL string) (string, error) {+ r := strings.TrimSpace(externalURL)+ r = strings.TrimSuffix(r, "/")+ u, err := url.Parse(r)+ if err != nil {+ return "", errors.Wrapf(err, "%s malformed", externalURL)+ }+ scheme := strings.ToLower(u.Scheme)+ if scheme != "http" && scheme != "https" {+ return "", errors.Errorf("%s must start with http:// or https://", externalURL)+ }+ if u.Host == "" {+ return "", errors.Errorf("%s must name a host", externalURL)+ }+ if u.User != nil {+ return "", errors.Errorf("%s must not carry userinfo", externalURL)+ }+ if u.RawQuery != "" || u.ForceQuery {+ return "", errors.Errorf("%s must not carry a query string", externalURL)+ }+ if u.Fragment != "" || u.RawFragment != "" {+ return "", errors.Errorf("%s must not carry a fragment", externalURL)+ }++ host := strings.ToLower(u.Host)+ port := u.Port()+ if port != "" {+ // The external URL is used as the redirectURL in the get token process of OAuth, and the+ // RedirectURL needs to be consistent with the RedirectURL in the get code process.+ // The frontend gets it through window.location.origin in the get code+ // process, so port 80/443 need to be cropped.+ if (scheme == "http" && port == "80") || (scheme == "https" && port == "443") {+ host = strings.ToLower(u.Hostname())+ if strings.Contains(host, ":") {+ host = "[" + host + "]"+ }+ }+ }+ return scheme + "://" + host + strings.TrimSuffix(u.EscapedPath(), "/"), nil+}diff --git a/backend/component/config/url_test.go b/backend/component/config/url_test.gonew file mode 100644index 0000000000..320774338e--- /dev/null+++ b/backend/component/config/url_test.go@@ -0,0 +1,90 @@+package config++import "testing"++func TestNormalizeExternalURL(t *testing.T) {+ tests := []struct {+ url string+ want string+ wantErr bool+ }{+ {+ url: "http://localhost:3000",+ want: "http://localhost:3000",+ wantErr: false,+ },+ {+ url: "https://localhost:3000",+ want: "https://localhost:3000",+ wantErr: false,+ },+ {+ url: "https://localhost",+ want: "https://localhost",+ wantErr: false,+ },+ {+ url: "http://localhost:80",+ want: "http://localhost",+ wantErr: false,+ },+ {+ url: "https://localhost:443",+ want: "https://localhost",+ wantErr: false,+ },+ {+ url: " https://localhost:3000/ ",+ want: "https://localhost:3000",+ wantErr: false,+ },+ {+ url: "HTTPS://LOCALHOST:443/mcp/",+ want: "https://localhost/mcp",+ wantErr: false,+ },+ // Missing http:// or https://+ {+ url: "localhost:3000",+ want: "",+ wantErr: true,+ },+ // Invalid port+ {+ url: "http://localhost:xxx",+ want: "",+ wantErr: true,+ },+ {+ url: "https://user@localhost",+ want: "",+ wantErr: true,+ },+ {+ url: "https://localhost?x=1",+ want: "",+ wantErr: true,+ },+ {+ url: "https://localhost#fragment",+ want: "",+ wantErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.url, func(t *testing.T) {+ g, err := NormalizeExternalURL(tt.url)+ if err != nil {+ if !tt.wantErr {+ t.Errorf("expect no error, got %s", err.Error())+ }+ } else {+ if tt.wantErr {+ t.Error("expect error")+ } else if tt.want != g {+ t.Errorf("expect %s, got %s", tt.want, g)+ }+ }+ })+ }+}diff --git a/backend/component/review/evaluator.go b/backend/component/review/evaluator.goindex 5bb9df5ed8..614ab0e870 100644--- a/backend/component/review/evaluator.go+++ b/backend/component/review/evaluator.go@@ -182,7 +182,7 @@ func calculateRiskLevelFromCELVars(celVarsList []map[string]any) storepb.RiskLevreturn storepb.RiskLevel_LOW}statementTypes := collectStatementTypes(celVarsList)- return common.GetRiskLevelFromStatementTypes(statementTypes)+ return GetRiskLevelFromStatementTypes(statementTypes)}// injectRiskLevelIntoCELVars adds the risk level to all CEL variable maps.@@ -797,7 +797,7 @@ func buildCELVariablesForRoleGrant(ctx context.Context, stores *store.Store, issreturn nil, false, errors.New("role grant payload not found")}- factors, err := common.GetQueryExportFactors(payload.GetRoleGrant().GetCondition().GetExpression())+ factors, err := getQueryExportFactors(payload.GetRoleGrant().GetCondition().GetExpression())if err != nil {return nil, false, errors.Wrap(err, "failed to get query export factors")}diff --git a/backend/component/review/query_export_factors.go b/backend/component/review/query_export_factors.gonew file mode 100644index 0000000000..2d5027e7cb--- /dev/null+++ b/backend/component/review/query_export_factors.go@@ -0,0 +1,61 @@+package review++import (+ "github.com/google/cel-go/cel"+ "github.com/pkg/errors"+ exprproto "google.golang.org/genproto/googleapis/api/expr/v1alpha1"++ "github.com/bytebase/bytebase/backend/common"+)++type queryExportFactors struct {+ Databases []string+}++func getQueryExportFactors(expression string) (*queryExportFactors, error) {+ if expression == "" {+ return &queryExportFactors{}, nil+ }++ factors := &queryExportFactors{}+ e, err := cel.NewEnv(common.IAMPolicyConditionCELAttributes...)+ if err != nil {+ return nil, err+ }+ ast, issues := e.Compile(expression)+ if issues != nil {+ return nil, errors.Errorf("found issue %v", issues)+ }+ parsedExpr, err := cel.AstToParsedExpr(ast)+ if err != nil {+ return nil, err+ }+ callExpr := parsedExpr.Expr.GetCallExpr()+ findField(callExpr, factors)+ return factors, nil+}++func findField(callExpr *exprproto.Expr_Call, factors *queryExportFactors) {+ if callExpr == nil {+ return+ }+ if len(callExpr.Args) == 2 {+ idExpr := callExpr.Args[0].GetIdentExpr()+ if idExpr != nil {+ if idExpr.Name == common.CELAttributeResourceDatabase && callExpr.Function == "_==_" {+ factors.Databases = append(factors.Databases, callExpr.Args[1].GetConstExpr().GetStringValue())+ }+ if idExpr.Name == common.CELAttributeResourceDatabase && callExpr.Function == "@in" {+ list := callExpr.Args[1].GetListExpr()+ for _, element := range list.Elements {+ factors.Databases = append(factors.Databases, element.GetConstExpr().GetStringValue())+ }+ }+ return+ }+ }+ for _, arg := range callExpr.Args {+ callExpr := arg.GetCallExpr()+ findField(callExpr, factors)+ }+}diff --git a/backend/component/review/query_export_factors_test.go b/backend/component/review/query_export_factors_test.gonew file mode 100644index 0000000000..113539cd37--- /dev/null+++ b/backend/component/review/query_export_factors_test.go@@ -0,0 +1,49 @@+package review++import (+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestGetQueryExportFactors(t *testing.T) {+ a := assert.New(t)+ tests := []struct {+ expression string+ want queryExportFactors+ }{+ {+ expression: "request.time < timestamp(\"2023-07-04T06:09:03.384Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_manager\"])",+ want: queryExportFactors{+ Databases: []string{"instances/postgres-sample/databases/employee"},+ },+ },+ {+ expression: "request.time < timestamp(\"2023-07-04T07:40:05.658Z\") && (resource.database in [\"instances/postgres-sample/databases/employee\"])",+ want: queryExportFactors{+ Databases: []string{"instances/postgres-sample/databases/employee"},+ },+ },+ {+ expression: "request.time < timestamp(\"2023-08-02T07:33:45.686Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_emp\",\"department\"])",+ want: queryExportFactors{+ Databases: []string{"instances/postgres-sample/databases/employee"},+ },+ },+ {+ expression: "request.time < timestamp(\"2023-07-10T08:14:34.788Z\")",+ want: queryExportFactors{},+ },+ {+ expression: "request.time < timestamp(\"2023-07-10T08:15:46.773Z\") && ((resource.database in [\"instances/postgres-sample/databases/blog\"]) || (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name in [\"public\"]))",+ want: queryExportFactors{+ Databases: []string{"instances/postgres-sample/databases/blog", "instances/postgres-sample/databases/employee"},+ },+ },+ }+ for _, tt := range tests {+ factors, err := getQueryExportFactors(tt.expression)+ a.NoError(err)+ a.Equal(tt.want, *factors)+ }+}diff --git a/backend/common/risk.go b/backend/component/review/risk.gosimilarity index 96%rename from backend/common/risk.gorename to backend/component/review/risk.goindex 407d897640..28b473a22c 100644--- a/backend/common/risk.go+++ b/backend/component/review/risk.go@@ -1,5 +1,4 @@-//nolint:revive-package common+package reviewimport (storepb "github.com/bytebase/bytebase/backend/generated-go/store"diff --git a/backend/common/risk_test.go b/backend/component/review/risk_test.gosimilarity index 98%rename from backend/common/risk_test.gorename to backend/component/review/risk_test.goindex d8c5f1e767..5858c787c4 100644--- a/backend/common/risk_test.go+++ b/backend/component/review/risk_test.go@@ -1,4 +1,4 @@-package common // nolint:revive+package reviewimport ("testing"diff --git a/backend/plugin/db/bigquery/bigquery.go b/backend/plugin/db/bigquery/bigquery.goindex b38d6c57fa..13d5a9672f 100644--- a/backend/plugin/db/bigquery/bigquery.go+++ b/backend/plugin/db/bigquery/bigquery.go@@ -17,7 +17,6 @@ import ("google.golang.org/protobuf/types/known/durationpb""google.golang.org/protobuf/types/known/structpb"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db"@@ -213,7 +212,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, qresult.Rows = append(result.Rows, row)n := len(result.Rows)if (n&(n-1) == 0) && int64(proto.Size(result)) > queryContext.MaximumSQLResultSize {- result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)+ result.Error = util.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)break}}diff --git a/backend/plugin/db/cassandra/cassandra.go b/backend/plugin/db/cassandra/cassandra.goindex 1c0737c5a3..9df900cac2 100644--- a/backend/plugin/db/cassandra/cassandra.go+++ b/backend/plugin/db/cassandra/cassandra.go@@ -18,7 +18,6 @@ import ("google.golang.org/protobuf/types/known/timestamppb""gopkg.in/inf.v0"- "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"@@ -158,7 +157,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, rawStatement stringresult.Rows = append(result.Rows, row)n := len(result.Rows)if (n&(n-1) == 0) && int64(proto.Size(result)) > queryContext.MaximumSQLResultSize {- result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)+ result.Error = util.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)break}diff --git a/backend/plugin/db/hive/hive.go b/backend/plugin/db/hive/hive.goindex 7735c1d38a..fdd74ad507 100644--- a/backend/plugin/db/hive/hive.go+++ b/backend/plugin/db/hive/hive.go@@ -15,7 +15,6 @@ import ("google.golang.org/protobuf/types/known/durationpb""google.golang.org/protobuf/types/known/structpb"- "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/plugin/db/util""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -317,7 +316,7 @@ func (d *Driver) queryStatementWithLimit(ctx context.Context, statement string,// Check size limitn := len(result.Rows)if (n&(n-1) == 0) && limit > 0 && int64(proto.Size(result)) > limit {- result.Error = common.FormatMaximumSQLResultSizeMessage(limit)+ result.Error = util.FormatMaximumSQLResultSizeMessage(limit)break}}diff --git a/backend/plugin/db/redis/redis.go b/backend/plugin/db/redis/redis.goindex 05509d3f0f..4643e1c09d 100644--- a/backend/plugin/db/redis/redis.go+++ b/backend/plugin/db/redis/redis.go@@ -20,7 +20,6 @@ import ("google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/durationpb"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db"@@ -347,7 +346,7 @@ func setQueryResultRows(result *v1pb.QueryResult, cmd *redis.Cmd, limit int64) {result.Rows = append(result.Rows, getResultRow(i+1, v))n := len(result.Rows)if (n&(n-1) == 0) && int64(proto.Size(result)) > limit {- result.Error = common.FormatMaximumSQLResultSizeMessage(limit)+ result.Error = util.FormatMaximumSQLResultSizeMessage(limit)return}}diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.goindex 5b3e395e9e..b989170850 100644--- a/backend/plugin/db/spanner/spanner.go+++ b/backend/plugin/db/spanner/spanner.go@@ -26,7 +26,6 @@ import ("google.golang.org/protobuf/types/known/structpb""google.golang.org/protobuf/types/known/timestamppb"- "github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"@@ -434,7 +433,7 @@ func (d *Driver) queryStatement(ctx context.Context, statement string, queryContresult.Rows = append(result.Rows, rowData)n := len(result.Rows)if (n&(n-1) == 0) && int64(proto.Size(result)) > queryContext.MaximumSQLResultSize {- result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)+ result.Error = util.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)break}if queryContext.Limit > 0 && n >= queryContext.Limit {diff --git a/backend/plugin/db/util/driverutil.go b/backend/plugin/db/util/driverutil.goindex 6a7f37cc70..58cc9d4da3 100644--- a/backend/plugin/db/util/driverutil.go+++ b/backend/plugin/db/util/driverutil.go@@ -15,7 +15,6 @@ import ("google.golang.org/protobuf/types/known/structpb""google.golang.org/protobuf/types/known/timestamppb"- "github.com/bytebase/bytebase/backend/common"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/utils")@@ -136,7 +135,7 @@ func RowsToLimitedQueryResult(rows *sql.Rows, valueMaker func(string, *sql.Columresult.Rows = append(result.Rows, row)n := len(result.Rows)if (n&(n-1) == 0) && int64(proto.Size(result)) > limit {- result.Error = common.FormatMaximumSQLResultSizeMessage(limit)+ result.Error = FormatMaximumSQLResultSizeMessage(limit)break}}@@ -242,3 +241,7 @@ func GetColumnIndex(columns []string, name string) (int, bool) {}return 0, false}++func FormatMaximumSQLResultSizeMessage(limit int64) string {+ return fmt.Sprintf("Output of query exceeds max allowed output size of %dMB", limit/1024/1024)+}diff --git a/docs/backend-common-audit.md b/docs/backend-common-audit.mdindex 04d39514bb..845995524e 100644--- a/docs/backend-common-audit.md+++ b/docs/backend-common-audit.md@@ -9,9 +9,12 @@ conversion to `plugin/parser/base`, and TiDB error-position conversion to aprivate TiDB parser helper. A subsequent migration moves the Connect-specificparser-engine conversion into a private v1 helper. This migration movestransaction configuration and SQL isolation conversion to `plugin/db/transaction`, and audit recording,-stdout formatting, and HTTP metadata helpers to `component/audit`. Protobuf-sanitization remains unchanged pending a separate ownership decision. The-remaining entries are proposals.+stdout formatting, and HTTP metadata helpers to `component/audit`.+The latest migration moves query/export factors and risk classification to+`component/review`, external-URL normalization to `component/config`, and the SQL+result-size message to `plugin/db/util`. Protobuf sanitization remains shared in+`common/proto.go`, with its tests in `common/proto_test.go`. The unused ANTLR+position helpers are removed. The remaining entries are proposals.The root package mixes unrelated ownership: resource names, identity, requestcontext, audit transport, policy expressions, SQL execution, and generic helpers.@@ -41,8 +44,8 @@ necessarily existing files or packages.| Transaction types and default mode in `common/engine.go` | `plugin/db/transaction` (implemented) | Execution settings and SQL isolation conversion are driver-owned. The directive parser imports this leaf package without depending on the full driver package. || `ConvertToParserEngine` (completed) | Private helper in `api/v1` | Only v1 calls it, and it returns a Connect error. The move preserves the mapping and error behavior. || Audit callback keys, setters, getters, `PermissionDeniedError` | Private helpers in `api/v1` | These coordinate v1 interceptors and handlers; no production callers outside v1 were found. |-| `GetQueryExportFactors` and its traversal | Private helpers in `component/review` | The only production caller is review evaluation. It should consume the shared IAM expression definitions described below. |-| `SanitizeUTF8Message` and reflection traversal | Deferred | Oracle metadata sync is the current caller, but the implementation handles arbitrary protobuf messages and can serve other engines. Decide shared ownership separately. |+| Query/export factors and their traversal (implemented) | Private helpers in `component/review/query_export_factors.go` | The only production caller is review evaluation. It should consume the shared IAM expression definitions described below. |+| `SanitizeUTF8Message` and reflection traversal (implemented) | `common/proto.go` | Oracle metadata sync is the current caller, but the implementation handles arbitrary protobuf messages and remains shared. || `GetPostgresSocketDir` | `resources/postgres` | Callers are embedded PostgreSQL setup, server startup, and the self-hosted sample manager. || `ReleaseMode` | `component/config` | `Profile.Mode` is the central configuration field; the package currently imports common only for this type. || Build-tagged `IsDev` | Local build-tagged files in `plugin/db/cosmosdb` | Cosmos DB is its only production consumer. Keep build-time mode distinct from runtime `Profile.Mode`. |@@ -56,7 +59,7 @@ necessarily existing files or packages.| `AuthContext`, `DelegatedGrant`, authorization resource types | `api/auth` | They describe transport authentication and authorization. These can live with the existing auth implementation, unlike the cross-component workspace context. || Workspace/user context keys and workspace accessor | A leaf `common/requestcontext` package | Review and parser-context components also consume request identity. Keep it independent of `api/auth`, store, and Connect. Use typed accessors and private keys as a separate interface cleanup. || `audit.go` | `component/audit` (implemented) | Shared by v1, MCP, and OAuth2. Preserves the narrow writer interface, detached bounded write, denial severity, stdout behavior, and caller-IP semantics. HTTP metadata helpers remain with the audit module. |-| Approval CEL definitions/validation and `risk.go` | `component/review` | This is review policy. Keep validation and runtime evaluation on the same definitions; map validation errors to Connect at the v1 caller. |+| Approval CEL definitions/validation; risk classification already moved | `component/review` | This is review policy. Keep validation and runtime evaluation on the same definitions; map validation errors to Connect at the v1 caller. || IAM CEL definitions, member validation, `EvalBindingCondition` | A leaf `component/iam/condition` package | Store and utils call the evaluator. Moving it into the parent IAM package creates a cycle because IAM imports store and utils. Preserve partial-evaluation behavior. || Masking CEL definitions/validation | Initially private files in `api/v1` | All current external consumers are v1. The existing `component/masker` implements value masking, which is a different responsibility from policy-expression evaluation. || Database-group CEL definitions/validation | A leaf `component/databasegroup` package | Shared by v1 and the group evaluation in `utils`; move that evaluation into this owner too. Keep the expression core independent of store. |@@ -77,12 +80,13 @@ not imply shared ownership. Split their tests along the same lines.| --- | --- || `RoundRows`, `AddRows` | `plugin/parser/base`, alongside row-estimate handling; retain saturation tests. || `RandomString` | The shared identity leaf, alongside its credential consumers; retain cryptographic randomness. |-| `NormalizeExternalURL` | `component/config`, shared by startup and API configuration validation. |+| `NormalizeExternalURL` (implemented) | `component/config/url.go`, shared by startup, API configuration validation, and OAuth resource validation. || `TruncateString`, `SanitizeUTF8String` | A focused `common/text` package; preserve rune-count and invalid-byte behavior. || `ProtojsonUnmarshaler` | `common/protoutil`; it is used beyond store. Preserve `DiscardUnknown` for persisted protobuf JSON compatibility. || `Uniq` | A focused collection helper if retained; preserve order. `slices.Compact` is not an equivalent replacement. || `IsNil` | Inspect its v1 and T-SQL callers for typed nil checks before retaining a shared reflection helper. |-| `MaximumCommands`, default SQL result size and result-size message | `plugin/db`, which owns the execution interface and is already consumed by the relevant drivers. |+| `MaximumCommands`, default SQL result size | `plugin/db`, which owns the execution interface and is already consumed by the relevant drivers. |+| Result-size message (implemented) | `plugin/db/util`, shared by database drivers. || `MaximumAdvicePerStatus`, `MaximumLintExplainSize` | `plugin/advisor`, alongside review and EXPLAIN-budget enforcement. || `MaxSheetSize`, `MaxSheetCheckSize` | A leaf `component/sheet/limits` package; the current sheet component and driver dependencies need to remain separate. Revisit whether display, parsing, and prior-backup limits truly share one policy in a separate behavior change. |@@ -95,8 +99,8 @@ test infrastructure, not a production foundation.## Delete before relocating+The unused ANTLR position helpers and their tests have been removed.Repository-wide Go searches found no production callers for-`ConvertANTLRPositionToPosition`, `ConvertANTLRTokenToExclusiveEndPosition`,`TrimSuffixAndGetInstanceDatabaseID`, `GetProjectIDPlanIDPlanCheckRunID`,`FormatRevision`, `FormatProjectRevision`, or `FormatSpec`.Some have tests only. The default test/prod environment constants also have no