Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: move backend common helpers to their owners (#21509)

p0nye71e9a32d67c
diff --git a/backend/api/oauth2/resource.go b/backend/api/oauth2/resource.go
index 52a87127b5..123cb5f363 100644
--- a/backend/api/oauth2/resource.go
+++ b/backend/api/oauth2/resource.go
@@ -9,7 +9,7 @@ import (
"github.com/pkg/errors"
- "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/component/config"
"github.com/bytebase/bytebase/backend/utils"
)
@@ -112,7 +112,7 @@ func checkConsentedResource(values url.Values, consented string) *oauth2Failure
if requested == "" {
return nil
}
- canonical, err := common.NormalizeExternalURL(requested)
+ canonical, err := config.NormalizeExternalURL(requested)
if err != nil {
return &oauth2Failure{code: "invalid_target", description: err.Error()}
}
@@ -200,7 +200,7 @@ func singleValue(values url.Values, name string) (string, error) {
// long-lived. Normalizing at the door keeps the audience single-valued.
func validateResource(resource, externalURL string) (string, error) {
mcpResource := externalURL + mcpResourcePath
- canonical, err := common.NormalizeExternalURL(resource)
+ canonical, err := config.NormalizeExternalURL(resource)
if err != nil {
return "", err
}
diff --git a/backend/api/v1/release_service_check.go b/backend/api/v1/release_service_check.go
index 125975f2ae..f6147dc504 100644
--- a/backend/api/v1/release_service_check.go
+++ b/backend/api/v1/release_service_check.go
@@ -16,6 +16,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/component/parsercontext"
+ "github.com/bytebase/bytebase/backend/component/review"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/advisor"
@@ -816,7 +817,7 @@ func getRiskLevelFromStatementTypes(statementTypes []storepb.StatementType) v1pb
for _, statementType := range statementTypes {
statementTypeStrings = append(statementTypeStrings, statementType.String())
}
- switch common.GetRiskLevelFromStatementTypes(statementTypeStrings) {
+ switch review.GetRiskLevelFromStatementTypes(statementTypeStrings) {
case storepb.RiskLevel_LOW:
return v1pb.RiskLevel_LOW
case storepb.RiskLevel_MODERATE:
diff --git a/backend/api/v1/setting_service.go b/backend/api/v1/setting_service.go
index f281a4fd83..6916d5f989 100644
--- a/backend/api/v1/setting_service.go
+++ b/backend/api/v1/setting_service.go
@@ -862,7 +862,7 @@ func (s *SettingService) preflightWorkspaceProfilePaths(ctx context.Context, wor
return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("external URL is managed via --external-url command-line flag and cannot be changed through the UI"))
}
if payload.ExternalUrl != "" {
- externalURL, err := common.NormalizeExternalURL(payload.ExternalUrl)
+ externalURL, err := config.NormalizeExternalURL(payload.ExternalUrl)
if err != nil {
return connect.NewError(connect.CodeInvalidArgument, errors.Errorf("invalid external url: %v", err))
}
diff --git a/backend/bin/server/cmd/root.go b/backend/bin/server/cmd/root.go
index a59b2beaa2..654cabc4a3 100644
--- a/backend/bin/server/cmd/root.go
+++ b/backend/bin/server/cmd/root.go
@@ -15,8 +15,8 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
- "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
+ "github.com/bytebase/bytebase/backend/component/config"
"github.com/bytebase/bytebase/backend/server"
)
@@ -157,7 +157,7 @@ func start() {
var err error
if flags.externalURL != "" {
- flags.externalURL, err = common.NormalizeExternalURL(flags.externalURL)
+ flags.externalURL, err = config.NormalizeExternalURL(flags.externalURL)
if err != nil {
slog.Error("invalid --external-url", log.BBError(err))
return
diff --git a/backend/common/cel.go b/backend/common/cel.go
index 02b8a46ea5..38ce2dbcf0 100644
--- a/backend/common/cel.go
+++ b/backend/common/cel.go
@@ -232,61 +232,6 @@ func validateCELExpr(expression *expr.Expr, conditionCELAttributes []cel.EnvOpti
return prog, nil
}
-// QueryExportFactors is the factors for query and export.
-type QueryExportFactors struct {
- Databases []string
-}
-
-// GetQueryExportFactors is used to get risk factors from query and export expressions.
-func GetQueryExportFactors(expression string) (*QueryExportFactors, error) {
- // If the expression is empty, return an empty struct.
- if expression == "" {
- return &QueryExportFactors{}, nil
- }
-
- factors := &QueryExportFactors{}
- e, err := cel.NewEnv(IAMPolicyConditionCELAttributes...)
- if err != nil {
- return nil, err
- }
- ast, issues := e.Compile(expression)
- if issues != nil {
- return nil, errors.Errorf("found issue %v", issues)
- }
- parsedExpr, err := cel.AstToParsedExpr(ast)
- if err != nil {
- return nil, err
- }
- callExpr := parsedExpr.Expr.GetCallExpr()
- findField(callExpr, factors)
- return factors, nil
-}
-
-func findField(callExpr *exprproto.Expr_Call, factors *QueryExportFactors) {
- if callExpr == nil {
- return
- }
- if len(callExpr.Args) == 2 {
- idExpr := callExpr.Args[0].GetIdentExpr()
- if idExpr != nil {
- if idExpr.Name == CELAttributeResourceDatabase && callExpr.Function == "_==_" {
- factors.Databases = append(factors.Databases, callExpr.Args[1].GetConstExpr().GetStringValue())
- }
- if idExpr.Name == CELAttributeResourceDatabase && callExpr.Function == "@in" {
- list := callExpr.Args[1].GetListExpr()
- for _, element := range list.Elements {
- factors.Databases = append(factors.Databases, element.GetConstExpr().GetStringValue())
- }
- }
- return
- }
- }
- for _, arg := range callExpr.Args {
- callExpr := arg.GetCallExpr()
- findField(callExpr, factors)
- }
-}
-
func EvalBindingCondition(expr string, requestTime time.Time) (bool, error) {
input := map[string]any{
CELAttributeRequestTime: requestTime,
diff --git a/backend/common/cel_test.go b/backend/common/cel_test.go
index 742dfb4d1d..fc687d8e97 100644
--- a/backend/common/cel_test.go
+++ b/backend/common/cel_test.go
@@ -6,7 +6,6 @@ import (
"time"
"github.com/google/cel-go/cel"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -75,48 +74,6 @@ func TestPartialEval(t *testing.T) {
}
}
-func TestGetQueryExportFactors(t *testing.T) {
- a := assert.New(t)
- tests := []struct {
- expression string
- want QueryExportFactors
- }{
- {
- expression: "request.time < timestamp(\"2023-07-04T06:09:03.384Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_manager\"])",
- want: QueryExportFactors{
- Databases: []string{"instances/postgres-sample/databases/employee"},
- },
- },
- {
- expression: "request.time < timestamp(\"2023-07-04T07:40:05.658Z\") && (resource.database in [\"instances/postgres-sample/databases/employee\"])",
- want: QueryExportFactors{
- Databases: []string{"instances/postgres-sample/databases/employee"},
- },
- },
- {
- expression: "request.time < timestamp(\"2023-08-02T07:33:45.686Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_emp\",\"department\"])",
- want: QueryExportFactors{
- Databases: []string{"instances/postgres-sample/databases/employee"},
- },
- },
- {
- expression: "request.time < timestamp(\"2023-07-10T08:14:34.788Z\")",
- want: QueryExportFactors{},
- },
- {
- expression: "request.time < timestamp(\"2023-07-10T08:15:46.773Z\") && ((resource.database in [\"instances/postgres-sample/databases/blog\"]) || (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name in [\"public\"]))",
- want: QueryExportFactors{
- Databases: []string{"instances/postgres-sample/databases/blog", "instances/postgres-sample/databases/employee"},
- },
- },
- }
- for _, tt := range tests {
- factors, err := GetQueryExportFactors(tt.expression)
- a.NoError(err)
- a.Equal(tt.want, *factors)
- }
-}
-
func TestFallbackApprovalFactors(t *testing.T) {
a := require.New(t)
diff --git a/backend/common/position.go b/backend/common/position.go
deleted file mode 100644
index 687dcc2a10..0000000000
--- a/backend/common/position.go
+++ /dev/null
@@ -1,67 +0,0 @@
-//nolint:revive
-package common
-
-import (
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
-)
-
-// ANTLRPosition is a position in a text expressed as one-based line and
-// zero-based column character (code point) offset integrated with ANTLR4.
-type ANTLRPosition struct {
- // Line position in a text (one-based).
- Line int32
- // Column position in a text (zero-based), equivalent to character offset.
- Column int32
-}
-
-// ConvertANTLRPositionToPosition converts an ANTLRPosition to a Position in a given text.
-// ANTLRPosition uses 1-based line and 0-based character column.
-// Returns a Position with 1-based line and 1-based character column.
-// Returns the end Position if the ANTLRPosition is out of the end of text,
-// returns the previous Position if the ANTLRPosition is out of the end of a line.
-func ConvertANTLRPositionToPosition(a *ANTLRPosition, text string) *storepb.Position {
- runes := []rune(text)
- line := int32(1) // Start at 1 for 1-based line numbering
- charOffsetInLine := int32(0)
- globalCharOffset := int32(0)
-
- for globalCharOffset < int32(len(runes)) {
- // Skip lines before the target line
- if line < a.Line {
- if runes[globalCharOffset] == '\n' {
- line++
- }
- globalCharOffset++
- continue
- }
-
- // Stop if we've reached the target column or hit a newline
- if charOffsetInLine >= a.Column || runes[globalCharOffset] == '\n' {
- break
- }
-
- charOffsetInLine++
- globalCharOffset++
- }
-
- // Convert from 0-based to 1-based column
- return &storepb.Position{
- Line: line,
- Column: charOffsetInLine + 1,
- }
-}
-
-// ConvertANTLRTokenToExclusiveEndPosition converts an ANTLR token position to an exclusive end Position.
-// The end token's line and column (0-based) plus its text are used to calculate where the position
-// should point to (after the last character of the token).
-// This is used for Statement.End which should use exclusive semantics (pointing after the last character).
-func ConvertANTLRTokenToExclusiveEndPosition(line int32, column int32, tokenText string) *storepb.Position {
- tokenCharLength := int32(len([]rune(tokenText)))
- // For exclusive end, we simply add the token length to the column.
- // The column is 0-based in ANTLR, so we add 1 to convert to 1-based.
- // The result points to the position AFTER the last character of the token.
- return &storepb.Position{
- Line: line,
- Column: column + tokenCharLength + 1,
- }
-}
diff --git a/backend/common/position_test.go b/backend/common/position_test.go
deleted file mode 100644
index 0ab14d23e6..0000000000
--- a/backend/common/position_test.go
+++ /dev/null
@@ -1,45 +0,0 @@
-//nolint:revive
-package common
-
-import (
- "testing"
-
- "github.com/stretchr/testify/require"
-
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
-)
-
-func TestConvertANTLRPositionToPosition(t *testing.T) {
- testCases := []struct {
- description string
- text string
- antlrPosition *ANTLRPosition
- want *storepb.Position
- }{
- {
- description: "first line first column",
- text: "hello, world",
- antlrPosition: &ANTLRPosition{Line: 1, Column: 0},
- want: &storepb.Position{Line: 1, Column: 1},
- },
- {
- description: "ASCII text",
- text: "hello, world",
- antlrPosition: &ANTLRPosition{Line: 1, Column: 6},
- want: &storepb.Position{Line: 1, Column: 7}, // ANTLR column 6 (0-based) -> column 7 (1-based)
- },
- {
- description: "multi-byte characters",
- text: "你好\n世界",
- antlrPosition: &ANTLRPosition{Line: 2, Column: 1},
- want: &storepb.Position{Line: 2, Column: 2}, // Character-based, not byte-based
- },
- }
-
- for _, tc := range testCases {
- t.Run(tc.description, func(t *testing.T) {
- got := ConvertANTLRPositionToPosition(tc.antlrPosition, tc.text)
- require.Equal(t, tc.want, got)
- })
- }
-}
diff --git a/backend/common/proto.go b/backend/common/proto.go
new file mode 100644
index 0000000000..a124c7e8b1
--- /dev/null
+++ b/backend/common/proto.go
@@ -0,0 +1,94 @@
+//nolint:revive
+package common
+
+import (
+ "unicode/utf8"
+
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// SanitizeUTF8Message replaces invalid UTF-8 byte sequences in every
+// populated string field of m, recursing into nested messages, repeated
+// fields, and maps (both keys and values). proto3 requires string fields to
+// hold valid UTF-8, so a single raw byte sequence smuggled in by an external
+// system (e.g. a database driver passing through unconverted bytes) makes
+// proto.Marshal fail for the entire message. Callers that assemble metadata
+// from such sources sanitize the finished message once here instead of
+// chasing every scan site.
+func SanitizeUTF8Message(m proto.Message) {
+ if m == nil {
+ return
+ }
+ sanitizeUTF8Value(m.ProtoReflect())
+}
+
+func sanitizeUTF8Value(m protoreflect.Message) {
+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+ switch {
+ case fd.IsMap():
+ sanitizeUTF8Map(fd, v.Map())
+ case fd.IsList():
+ list := v.List()
+ switch fd.Kind() {
+ case protoreflect.StringKind:
+ for i := 0; i < list.Len(); i++ {
+ s := list.Get(i).String()
+ if !utf8.ValidString(s) {
+ list.Set(i, protoreflect.ValueOfString(SanitizeUTF8String(s)))
+ }
+ }
+ case protoreflect.MessageKind, protoreflect.GroupKind:
+ for i := 0; i < list.Len(); i++ {
+ sanitizeUTF8Value(list.Get(i).Message())
+ }
+ default:
+ }
+ case fd.Kind() == protoreflect.StringKind:
+ if s := v.String(); !utf8.ValidString(s) {
+ m.Set(fd, protoreflect.ValueOfString(SanitizeUTF8String(s)))
+ }
+ case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind:
+ sanitizeUTF8Value(v.Message())
+ default:
+ }
+ return true
+ })
+}
+
+func sanitizeUTF8Map(fd protoreflect.FieldDescriptor, mp protoreflect.Map) {
+ valueKind := fd.MapValue().Kind()
+ keyIsString := fd.MapKey().Kind() == protoreflect.StringKind
+
+ type rekey struct {
+ oldKey protoreflect.MapKey
+ newKey protoreflect.MapKey
+ value protoreflect.Value
+ }
+ var rekeys []rekey
+ mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
+ switch valueKind {
+ case protoreflect.StringKind:
+ if s := v.String(); !utf8.ValidString(s) {
+ mp.Set(k, protoreflect.ValueOfString(SanitizeUTF8String(s)))
+ }
+ case protoreflect.MessageKind, protoreflect.GroupKind:
+ sanitizeUTF8Value(v.Message())
+ default:
+ }
+ if keyIsString {
+ if s := k.String(); !utf8.ValidString(s) {
+ rekeys = append(rekeys, rekey{
+ oldKey: k,
+ newKey: protoreflect.ValueOfString(SanitizeUTF8String(s)).MapKey(),
+ value: mp.Get(k),
+ })
+ }
+ }
+ return true
+ })
+ for _, r := range rekeys {
+ mp.Clear(r.oldKey)
+ mp.Set(r.newKey, r.value)
+ }
+}
diff --git a/backend/common/proto_test.go b/backend/common/proto_test.go
new file mode 100644
index 0000000000..69de1fdb03
--- /dev/null
+++ b/backend/common/proto_test.go
@@ -0,0 +1,170 @@
+//nolint:revive
+package common
+
+import (
+ "testing"
+ "unicode/utf8"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+ "google.golang.org/protobuf/reflect/protoreflect"
+)
+
+// fillAllStringFields recursively sets every string field reachable from m
+// (including nested messages, repeated fields, and map keys/values) to the
+// given value, instantiating one element for each nested message, list, and
+// map so that no string field in the schema shape stays unvisited.
+func fillAllStringFields(m protoreflect.Message, value string, depth int) {
+ if depth <= 0 {
+ return
+ }
+ fields := m.Descriptor().Fields()
+ for i := 0; i < fields.Len(); i++ {
+ fd := fields.Get(i)
+ switch {
+ case fd.IsMap():
+ mp := m.Mutable(fd).Map()
+ var key protoreflect.MapKey
+ if fd.MapKey().Kind() == protoreflect.StringKind {
+ key = protoreflect.ValueOfString(value).MapKey()
+ } else {
+ key = protoreflect.ValueOfInt64(1).MapKey()
+ }
+ switch fd.MapValue().Kind() {
+ case protoreflect.StringKind:
+ mp.Set(key, protoreflect.ValueOfString(value))
+ case protoreflect.MessageKind:
+ nv := mp.NewValue()
+ fillAllStringFields(nv.Message(), value, depth-1)
+ mp.Set(key, nv)
+ default:
+ mp.Set(key, mp.NewValue())
+ }
+ case fd.IsList():
+ list := m.Mutable(fd).List()
+ switch fd.Kind() {
+ case protoreflect.StringKind:
+ list.Append(protoreflect.ValueOfString(value))
+ case protoreflect.MessageKind:
+ nv := list.NewElement()
+ fillAllStringFields(nv.Message(), value, depth-1)
+ list.Append(nv)
+ default:
+ }
+ case fd.Kind() == protoreflect.StringKind:
+ m.Set(fd, protoreflect.ValueOfString(value))
+ case fd.Kind() == protoreflect.MessageKind:
+ fillAllStringFields(m.Mutable(fd).Message(), value, depth-1)
+ default:
+ }
+ }
+}
+
+// countInvalidStringFields walks m and returns how many string values
+// (fields, list elements, map keys and values) hold invalid UTF-8.
+func countInvalidStringFields(m protoreflect.Message) int {
+ count := 0
+ m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+ switch {
+ case fd.IsMap():
+ v.Map().Range(func(k protoreflect.MapKey, mv protoreflect.Value) bool {
+ if fd.MapKey().Kind() == protoreflect.StringKind && !utf8.ValidString(k.String()) {
+ count++
+ }
+ switch fd.MapValue().Kind() {
+ case protoreflect.StringKind:
+ if !utf8.ValidString(mv.String()) {
+ count++
+ }
+ case protoreflect.MessageKind:
+ count += countInvalidStringFields(mv.Message())
+ default:
+ }
+ return true
+ })
+ case fd.IsList():
+ list := v.List()
+ for i := 0; i < list.Len(); i++ {
+ switch fd.Kind() {
+ case protoreflect.StringKind:
+ if !utf8.ValidString(list.Get(i).String()) {
+ count++
+ }
+ case protoreflect.MessageKind:
+ count += countInvalidStringFields(list.Get(i).Message())
+ default:
+ }
+ }
+ case fd.Kind() == protoreflect.StringKind:
+ if !utf8.ValidString(v.String()) {
+ count++
+ }
+ case fd.Kind() == protoreflect.MessageKind:
+ count += countInvalidStringFields(v.Message())
+ default:
+ }
+ return true
+ })
+ return count
+}
+
+// TestSanitizeUTF8MessageCoversEveryStringField proves the BYT-9916 fix
+// covers every string field of the database metadata — names, types,
+// defaults, definitions — not just comments: fill each one with the raw-GBK
+// shape go-ora leaks for values ending in a dangling lead byte, then assert
+// sanitization leaves zero invalid strings and proto marshaling succeeds.
+func TestSanitizeUTF8MessageCoversEveryStringField(t *testing.T) {
+ // 测试 + dangling lead byte 0xb1: the exact wholly-unconverted raw GBK
+ // shape go-ora returns (engine-verified against Oracle 11gR2/ZHS16GBK).
+ rawGBK := "\xb2\xe2\xca\xd4\xb1"
+ require.False(t, utf8.ValidString(rawGBK))
+
+ metadata := &metadatapb.DatabaseSchemaMetadata{}
+ fillAllStringFields(metadata.ProtoReflect(), rawGBK, 8)
+
+ invalidBefore := countInvalidStringFields(metadata.ProtoReflect())
+ require.Greater(t, invalidBefore, 100,
+ "filler must reach the full schema shape; got only %d string fields", invalidBefore)
+ _, err := proto.Marshal(metadata)
+ require.Error(t, err, "pre-sanitize marshal must fail — otherwise this test proves nothing")
+
+ SanitizeUTF8Message(metadata)
+
+ require.Equal(t, 0, countInvalidStringFields(metadata.ProtoReflect()),
+ "sanitization must leave zero invalid string values anywhere in the message")
+ _, err = proto.Marshal(metadata)
+ require.NoError(t, err)
+ _, err = protojson.Marshal(metadata)
+ require.NoError(t, err)
+}
+
+// TestSanitizeUTF8MessageNameFields pins the customer-visible BYT-9916 shape:
+// object names (schema/table/column) carrying raw GBK bytes must marshal
+// after sanitization, and valid names must pass through untouched.
+func TestSanitizeUTF8MessageNameFields(t *testing.T) {
+ rawGBK := "AB\xe6"
+ metadata := &metadatapb.DatabaseSchemaMetadata{
+ Name: rawGBK,
+ Schemas: []*metadatapb.SchemaMetadata{{
+ Name: rawGBK,
+ Tables: []*metadatapb.TableMetadata{{
+ Name: rawGBK,
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: rawGBK, Type: rawGBK, Default: rawGBK},
+ {Name: "测试列", Type: "VARCHAR2(50)"},
+ },
+ }},
+ }},
+ }
+ _, err := proto.Marshal(metadata)
+ require.Error(t, err)
+
+ SanitizeUTF8Message(metadata)
+ _, err = proto.Marshal(metadata)
+ require.NoError(t, err)
+ require.Equal(t, "测试列", metadata.Schemas[0].Tables[0].Columns[1].Name,
+ "valid UTF-8 must pass through unchanged")
+ require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))
+}
diff --git a/backend/common/util.go b/backend/common/util.go
index d4f43c146f..0d943a254e 100644
--- a/backend/common/util.go
+++ b/backend/common/util.go
@@ -6,7 +6,6 @@ import (
"fmt"
"math"
"math/big"
- "net/url"
"reflect"
"regexp"
"strings"
@@ -15,8 +14,6 @@ import (
"github.com/nyaruka/phonenumbers/v2"
"github.com/pkg/errors"
"google.golang.org/protobuf/encoding/protojson"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/reflect/protoreflect"
)
const (
@@ -90,48 +87,6 @@ func TruncateString(str string, limit int) (string, bool) {
return str, false
}
-// NormalizeExternalURL will format the external url.
-func NormalizeExternalURL(externalURL string) (string, error) {
- r := strings.TrimSpace(externalURL)
- r = strings.TrimSuffix(r, "/")
- u, err := url.Parse(r)
- if err != nil {
- return "", errors.Wrapf(err, "%s malformed", externalURL)
- }
- scheme := strings.ToLower(u.Scheme)
- if scheme != "http" && scheme != "https" {
- return "", errors.Errorf("%s must start with http:// or https://", externalURL)
- }
- if u.Host == "" {
- return "", errors.Errorf("%s must name a host", externalURL)
- }
- if u.User != nil {
- return "", errors.Errorf("%s must not carry userinfo", externalURL)
- }
- if u.RawQuery != "" || u.ForceQuery {
- return "", errors.Errorf("%s must not carry a query string", externalURL)
- }
- if u.Fragment != "" || u.RawFragment != "" {
- return "", errors.Errorf("%s must not carry a fragment", externalURL)
- }
-
- host := strings.ToLower(u.Host)
- port := u.Port()
- if port != "" {
- // The external URL is used as the redirectURL in the get token process of OAuth, and the
- // RedirectURL needs to be consistent with the RedirectURL in the get code process.
- // The frontend gets it through window.location.origin in the get code
- // process, so port 80/443 need to be cropped.
- if (scheme == "http" && port == "80") || (scheme == "https" && port == "443") {
- host = strings.ToLower(u.Hostname())
- if strings.Contains(host, ":") {
- host = "[" + host + "]"
- }
- }
- }
- return scheme + "://" + host + strings.TrimSuffix(u.EscapedPath(), "/"), nil
-}
-
// ValidatePhone validates the phone number.
func ValidatePhone(phone string) error {
phoneNumber, err := phonenumbers.Parse(phone, "")
@@ -208,10 +163,6 @@ func SanitizeUTF8String(s string) string {
return b.String()
}
-func FormatMaximumSQLResultSizeMessage(limit int64) string {
- return fmt.Sprintf("Output of query exceeds max allowed output size of %dMB", limit/1024/1024)
-}
-
func IsNil(val any) bool {
if val == nil {
return true
@@ -245,88 +196,3 @@ func Uniq[T comparable](array []T) []T {
return res
}
-
-// SanitizeUTF8Message replaces invalid UTF-8 byte sequences in every
-// populated string field of m, recursing into nested messages, repeated
-// fields, and maps (both keys and values). proto3 requires string fields to
-// hold valid UTF-8, so a single raw byte sequence smuggled in by an external
-// system (e.g. a database driver passing through unconverted bytes) makes
-// proto.Marshal fail for the entire message. Callers that assemble metadata
-// from such sources sanitize the finished message once here instead of
-// chasing every scan site.
-func SanitizeUTF8Message(m proto.Message) {
- if m == nil {
- return
- }
- sanitizeUTF8Value(m.ProtoReflect())
-}
-
-func sanitizeUTF8Value(m protoreflect.Message) {
- m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
- switch {
- case fd.IsMap():
- sanitizeUTF8Map(fd, v.Map())
- case fd.IsList():
- list := v.List()
- switch fd.Kind() {
- case protoreflect.StringKind:
- for i := 0; i < list.Len(); i++ {
- s := list.Get(i).String()
- if !utf8.ValidString(s) {
- list.Set(i, protoreflect.ValueOfString(SanitizeUTF8String(s)))
- }
- }
- case protoreflect.MessageKind, protoreflect.GroupKind:
- for i := 0; i < list.Len(); i++ {
- sanitizeUTF8Value(list.Get(i).Message())
- }
- default:
- }
- case fd.Kind() == protoreflect.StringKind:
- if s := v.String(); !utf8.ValidString(s) {
- m.Set(fd, protoreflect.ValueOfString(SanitizeUTF8String(s)))
- }
- case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind:
- sanitizeUTF8Value(v.Message())
- default:
- }
- return true
- })
-}
-
-func sanitizeUTF8Map(fd protoreflect.FieldDescriptor, mp protoreflect.Map) {
- valueKind := fd.MapValue().Kind()
- keyIsString := fd.MapKey().Kind() == protoreflect.StringKind
-
- type rekey struct {
- oldKey protoreflect.MapKey
- newKey protoreflect.MapKey
- value protoreflect.Value
- }
- var rekeys []rekey
- mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool {
- switch valueKind {
- case protoreflect.StringKind:
- if s := v.String(); !utf8.ValidString(s) {
- mp.Set(k, protoreflect.ValueOfString(SanitizeUTF8String(s)))
- }
- case protoreflect.MessageKind, protoreflect.GroupKind:
- sanitizeUTF8Value(v.Message())
- default:
- }
- if keyIsString {
- if s := k.String(); !utf8.ValidString(s) {
- rekeys = append(rekeys, rekey{
- oldKey: k,
- newKey: protoreflect.ValueOfString(SanitizeUTF8String(s)).MapKey(),
- value: mp.Get(k),
- })
- }
- }
- return true
- })
- for _, r := range rekeys {
- mp.Clear(r.oldKey)
- mp.Set(r.newKey, r.value)
- }
-}
diff --git a/backend/common/util_test.go b/backend/common/util_test.go
index 19d98b9343..f2085c4818 100644
--- a/backend/common/util_test.go
+++ b/backend/common/util_test.go
@@ -6,12 +6,8 @@ import (
"testing"
"unicode/utf8"
- metadatapb "github.com/bytebase/omni/metadata"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "google.golang.org/protobuf/encoding/protojson"
- "google.golang.org/protobuf/proto"
- "google.golang.org/protobuf/reflect/protoreflect"
)
func TestTruncateString(t *testing.T) {
@@ -90,93 +86,6 @@ func TestTruncateString(t *testing.T) {
}
}
-func TestNormalizeExternalURL(t *testing.T) {
- tests := []struct {
- url string
- want string
- wantErr bool
- }{
- {
- url: "http://localhost:3000",
- want: "http://localhost:3000",
- wantErr: false,
- },
- {
- url: "https://localhost:3000",
- want: "https://localhost:3000",
- wantErr: false,
- },
- {
- url: "https://localhost",
- want: "https://localhost",
- wantErr: false,
- },
- {
- url: "http://localhost:80",
- want: "http://localhost",
- wantErr: false,
- },
- {
- url: "https://localhost:443",
- want: "https://localhost",
- wantErr: false,
- },
- {
- url: " https://localhost:3000/ ",
- want: "https://localhost:3000",
- wantErr: false,
- },
- {
- url: "HTTPS://LOCALHOST:443/mcp/",
- want: "https://localhost/mcp",
- wantErr: false,
- },
- // Missing http:// or https://
- {
- url: "localhost:3000",
- want: "",
- wantErr: true,
- },
- // Invalid port
- {
- url: "http://localhost:xxx",
- want: "",
- wantErr: true,
- },
- {
- url: "https://user@localhost",
- want: "",
- wantErr: true,
- },
- {
- url: "https://localhost?x=1",
- want: "",
- wantErr: true,
- },
- {
- url: "https://localhost#fragment",
- want: "",
- wantErr: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.url, func(t *testing.T) {
- g, err := NormalizeExternalURL(tt.url)
- if err != nil {
- if !tt.wantErr {
- t.Errorf("expect no error, got %s", err.Error())
- }
- } else {
- if tt.wantErr {
- t.Error("expect error")
- } else if tt.want != g {
- t.Errorf("expect %s, got %s", tt.want, g)
- }
- }
- })
- }
-}
-
func TestValidatePhone(t *testing.T) {
tests := []struct {
phone string
@@ -217,163 +126,6 @@ func TestSanitizeUTF8String(t *testing.T) {
require.Equal(t, "\u6d4b\u8bd5", SanitizeUTF8String("\u6d4b\u8bd5"), "valid UTF-8 must pass through unchanged")
}
-// fillAllStringFields recursively sets every string field reachable from m
-// (including nested messages, repeated fields, and map keys/values) to the
-// given value, instantiating one element for each nested message, list, and
-// map so that no string field in the schema shape stays unvisited.
-func fillAllStringFields(m protoreflect.Message, value string, depth int) {
- if depth <= 0 {
- return
- }
- fields := m.Descriptor().Fields()
- for i := 0; i < fields.Len(); i++ {
- fd := fields.Get(i)
- switch {
- case fd.IsMap():
- mp := m.Mutable(fd).Map()
- var key protoreflect.MapKey
- if fd.MapKey().Kind() == protoreflect.StringKind {
- key = protoreflect.ValueOfString(value).MapKey()
- } else {
- key = protoreflect.ValueOfInt64(1).MapKey()
- }
- switch fd.MapValue().Kind() {
- case protoreflect.StringKind:
- mp.Set(key, protoreflect.ValueOfString(value))
- case protoreflect.MessageKind:
- nv := mp.NewValue()
- fillAllStringFields(nv.Message(), value, depth-1)
- mp.Set(key, nv)
- default:
- mp.Set(key, mp.NewValue())
- }
- case fd.IsList():
- list := m.Mutable(fd).List()
- switch fd.Kind() {
- case protoreflect.StringKind:
- list.Append(protoreflect.ValueOfString(value))
- case protoreflect.MessageKind:
- nv := list.NewElement()
- fillAllStringFields(nv.Message(), value, depth-1)
- list.Append(nv)
- default:
- }
- case fd.Kind() == protoreflect.StringKind:
- m.Set(fd, protoreflect.ValueOfString(value))
- case fd.Kind() == protoreflect.MessageKind:
- fillAllStringFields(m.Mutable(fd).Message(), value, depth-1)
- default:
- }
- }
-}
-
-// countInvalidStringFields walks m and returns how many string values
-// (fields, list elements, map keys and values) hold invalid UTF-8.
-func countInvalidStringFields(m protoreflect.Message) int {
- count := 0
- m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
- switch {
- case fd.IsMap():
- v.Map().Range(func(k protoreflect.MapKey, mv protoreflect.Value) bool {
- if fd.MapKey().Kind() == protoreflect.StringKind && !utf8.ValidString(k.String()) {
- count++
- }
- switch fd.MapValue().Kind() {
- case protoreflect.StringKind:
- if !utf8.ValidString(mv.String()) {
- count++
- }
- case protoreflect.MessageKind:
- count += countInvalidStringFields(mv.Message())
- default:
- }
- return true
- })
- case fd.IsList():
- list := v.List()
- for i := 0; i < list.Len(); i++ {
- switch fd.Kind() {
- case protoreflect.StringKind:
- if !utf8.ValidString(list.Get(i).String()) {
- count++
- }
- case protoreflect.MessageKind:
- count += countInvalidStringFields(list.Get(i).Message())
- default:
- }
- }
- case fd.Kind() == protoreflect.StringKind:
- if !utf8.ValidString(v.String()) {
- count++
- }
- case fd.Kind() == protoreflect.MessageKind:
- count += countInvalidStringFields(v.Message())
- default:
- }
- return true
- })
- return count
-}
-
-// TestSanitizeUTF8MessageCoversEveryStringField proves the BYT-9916 fix
-// covers every string field of the database metadata — names, types,
-// defaults, definitions — not just comments: fill each one with the raw-GBK
-// shape go-ora leaks for values ending in a dangling lead byte, then assert
-// sanitization leaves zero invalid strings and proto marshaling succeeds.
-func TestSanitizeUTF8MessageCoversEveryStringField(t *testing.T) {
- // 测试 + dangling lead byte 0xb1: the exact wholly-unconverted raw GBK
- // shape go-ora returns (engine-verified against Oracle 11gR2/ZHS16GBK).
- rawGBK := "\xb2\xe2\xca\xd4\xb1"
- require.False(t, utf8.ValidString(rawGBK))
-
- metadata := &metadatapb.DatabaseSchemaMetadata{}
- fillAllStringFields(metadata.ProtoReflect(), rawGBK, 8)
-
- invalidBefore := countInvalidStringFields(metadata.ProtoReflect())
- require.Greater(t, invalidBefore, 100,
- "filler must reach the full schema shape; got only %d string fields", invalidBefore)
- _, err := proto.Marshal(metadata)
- require.Error(t, err, "pre-sanitize marshal must fail — otherwise this test proves nothing")
-
- SanitizeUTF8Message(metadata)
-
- require.Equal(t, 0, countInvalidStringFields(metadata.ProtoReflect()),
- "sanitization must leave zero invalid string values anywhere in the message")
- _, err = proto.Marshal(metadata)
- require.NoError(t, err)
- _, err = protojson.Marshal(metadata)
- require.NoError(t, err)
-}
-
-// TestSanitizeUTF8MessageNameFields pins the customer-visible BYT-9916 shape:
-// object names (schema/table/column) carrying raw GBK bytes must marshal
-// after sanitization, and valid names must pass through untouched.
-func TestSanitizeUTF8MessageNameFields(t *testing.T) {
- rawGBK := "AB\xe6"
- metadata := &metadatapb.DatabaseSchemaMetadata{
- Name: rawGBK,
- Schemas: []*metadatapb.SchemaMetadata{{
- Name: rawGBK,
- Tables: []*metadatapb.TableMetadata{{
- Name: rawGBK,
- Columns: []*metadatapb.ColumnMetadata{
- {Name: rawGBK, Type: rawGBK, Default: rawGBK},
- {Name: "测试列", Type: "VARCHAR2(50)"},
- },
- }},
- }},
- }
- _, err := proto.Marshal(metadata)
- require.Error(t, err)
-
- SanitizeUTF8Message(metadata)
- _, err = proto.Marshal(metadata)
- require.NoError(t, err)
- require.Equal(t, "测试列", metadata.Schemas[0].Tables[0].Columns[1].Name,
- "valid UTF-8 must pass through unchanged")
- require.True(t, utf8.ValidString(metadata.Schemas[0].Tables[0].Name))
-}
-
func TestRoundRows(t *testing.T) {
require.Equal(t, int64(3), RoundRows(2.5))
require.Equal(t, int64(1000), RoundRows(999.6))
diff --git a/backend/component/config/url.go b/backend/component/config/url.go
new file mode 100644
index 0000000000..bce6a841c3
--- /dev/null
+++ b/backend/component/config/url.go
@@ -0,0 +1,50 @@
+package config
+
+import (
+ "net/url"
+ "strings"
+
+ "github.com/pkg/errors"
+)
+
+// NormalizeExternalURL will format the external url.
+func NormalizeExternalURL(externalURL string) (string, error) {
+ r := strings.TrimSpace(externalURL)
+ r = strings.TrimSuffix(r, "/")
+ u, err := url.Parse(r)
+ if err != nil {
+ return "", errors.Wrapf(err, "%s malformed", externalURL)
+ }
+ scheme := strings.ToLower(u.Scheme)
+ if scheme != "http" && scheme != "https" {
+ return "", errors.Errorf("%s must start with http:// or https://", externalURL)
+ }
+ if u.Host == "" {
+ return "", errors.Errorf("%s must name a host", externalURL)
+ }
+ if u.User != nil {
+ return "", errors.Errorf("%s must not carry userinfo", externalURL)
+ }
+ if u.RawQuery != "" || u.ForceQuery {
+ return "", errors.Errorf("%s must not carry a query string", externalURL)
+ }
+ if u.Fragment != "" || u.RawFragment != "" {
+ return "", errors.Errorf("%s must not carry a fragment", externalURL)
+ }
+
+ host := strings.ToLower(u.Host)
+ port := u.Port()
+ if port != "" {
+ // The external URL is used as the redirectURL in the get token process of OAuth, and the
+ // RedirectURL needs to be consistent with the RedirectURL in the get code process.
+ // The frontend gets it through window.location.origin in the get code
+ // process, so port 80/443 need to be cropped.
+ if (scheme == "http" && port == "80") || (scheme == "https" && port == "443") {
+ host = strings.ToLower(u.Hostname())
+ if strings.Contains(host, ":") {
+ host = "[" + host + "]"
+ }
+ }
+ }
+ return scheme + "://" + host + strings.TrimSuffix(u.EscapedPath(), "/"), nil
+}
diff --git a/backend/component/config/url_test.go b/backend/component/config/url_test.go
new file mode 100644
index 0000000000..320774338e
--- /dev/null
+++ b/backend/component/config/url_test.go
@@ -0,0 +1,90 @@
+package config
+
+import "testing"
+
+func TestNormalizeExternalURL(t *testing.T) {
+ tests := []struct {
+ url string
+ want string
+ wantErr bool
+ }{
+ {
+ url: "http://localhost:3000",
+ want: "http://localhost:3000",
+ wantErr: false,
+ },
+ {
+ url: "https://localhost:3000",
+ want: "https://localhost:3000",
+ wantErr: false,
+ },
+ {
+ url: "https://localhost",
+ want: "https://localhost",
+ wantErr: false,
+ },
+ {
+ url: "http://localhost:80",
+ want: "http://localhost",
+ wantErr: false,
+ },
+ {
+ url: "https://localhost:443",
+ want: "https://localhost",
+ wantErr: false,
+ },
+ {
+ url: " https://localhost:3000/ ",
+ want: "https://localhost:3000",
+ wantErr: false,
+ },
+ {
+ url: "HTTPS://LOCALHOST:443/mcp/",
+ want: "https://localhost/mcp",
+ wantErr: false,
+ },
+ // Missing http:// or https://
+ {
+ url: "localhost:3000",
+ want: "",
+ wantErr: true,
+ },
+ // Invalid port
+ {
+ url: "http://localhost:xxx",
+ want: "",
+ wantErr: true,
+ },
+ {
+ url: "https://user@localhost",
+ want: "",
+ wantErr: true,
+ },
+ {
+ url: "https://localhost?x=1",
+ want: "",
+ wantErr: true,
+ },
+ {
+ url: "https://localhost#fragment",
+ want: "",
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.url, func(t *testing.T) {
+ g, err := NormalizeExternalURL(tt.url)
+ if err != nil {
+ if !tt.wantErr {
+ t.Errorf("expect no error, got %s", err.Error())
+ }
+ } else {
+ if tt.wantErr {
+ t.Error("expect error")
+ } else if tt.want != g {
+ t.Errorf("expect %s, got %s", tt.want, g)
+ }
+ }
+ })
+ }
+}
diff --git a/backend/component/review/evaluator.go b/backend/component/review/evaluator.go
index 5bb9df5ed8..614ab0e870 100644
--- a/backend/component/review/evaluator.go
+++ b/backend/component/review/evaluator.go
@@ -182,7 +182,7 @@ func calculateRiskLevelFromCELVars(celVarsList []map[string]any) storepb.RiskLev
return storepb.RiskLevel_LOW
}
statementTypes := collectStatementTypes(celVarsList)
- return common.GetRiskLevelFromStatementTypes(statementTypes)
+ return GetRiskLevelFromStatementTypes(statementTypes)
}
// injectRiskLevelIntoCELVars adds the risk level to all CEL variable maps.
@@ -797,7 +797,7 @@ func buildCELVariablesForRoleGrant(ctx context.Context, stores *store.Store, iss
return nil, false, errors.New("role grant payload not found")
}
- factors, err := common.GetQueryExportFactors(payload.GetRoleGrant().GetCondition().GetExpression())
+ factors, err := getQueryExportFactors(payload.GetRoleGrant().GetCondition().GetExpression())
if err != nil {
return nil, false, errors.Wrap(err, "failed to get query export factors")
}
diff --git a/backend/component/review/query_export_factors.go b/backend/component/review/query_export_factors.go
new file mode 100644
index 0000000000..2d5027e7cb
--- /dev/null
+++ b/backend/component/review/query_export_factors.go
@@ -0,0 +1,61 @@
+package review
+
+import (
+ "github.com/google/cel-go/cel"
+ "github.com/pkg/errors"
+ exprproto "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
+
+ "github.com/bytebase/bytebase/backend/common"
+)
+
+type queryExportFactors struct {
+ Databases []string
+}
+
+func getQueryExportFactors(expression string) (*queryExportFactors, error) {
+ if expression == "" {
+ return &queryExportFactors{}, nil
+ }
+
+ factors := &queryExportFactors{}
+ e, err := cel.NewEnv(common.IAMPolicyConditionCELAttributes...)
+ if err != nil {
+ return nil, err
+ }
+ ast, issues := e.Compile(expression)
+ if issues != nil {
+ return nil, errors.Errorf("found issue %v", issues)
+ }
+ parsedExpr, err := cel.AstToParsedExpr(ast)
+ if err != nil {
+ return nil, err
+ }
+ callExpr := parsedExpr.Expr.GetCallExpr()
+ findField(callExpr, factors)
+ return factors, nil
+}
+
+func findField(callExpr *exprproto.Expr_Call, factors *queryExportFactors) {
+ if callExpr == nil {
+ return
+ }
+ if len(callExpr.Args) == 2 {
+ idExpr := callExpr.Args[0].GetIdentExpr()
+ if idExpr != nil {
+ if idExpr.Name == common.CELAttributeResourceDatabase && callExpr.Function == "_==_" {
+ factors.Databases = append(factors.Databases, callExpr.Args[1].GetConstExpr().GetStringValue())
+ }
+ if idExpr.Name == common.CELAttributeResourceDatabase && callExpr.Function == "@in" {
+ list := callExpr.Args[1].GetListExpr()
+ for _, element := range list.Elements {
+ factors.Databases = append(factors.Databases, element.GetConstExpr().GetStringValue())
+ }
+ }
+ return
+ }
+ }
+ for _, arg := range callExpr.Args {
+ callExpr := arg.GetCallExpr()
+ findField(callExpr, factors)
+ }
+}
diff --git a/backend/component/review/query_export_factors_test.go b/backend/component/review/query_export_factors_test.go
new file mode 100644
index 0000000000..113539cd37
--- /dev/null
+++ b/backend/component/review/query_export_factors_test.go
@@ -0,0 +1,49 @@
+package review
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetQueryExportFactors(t *testing.T) {
+ a := assert.New(t)
+ tests := []struct {
+ expression string
+ want queryExportFactors
+ }{
+ {
+ expression: "request.time < timestamp(\"2023-07-04T06:09:03.384Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_manager\"])",
+ want: queryExportFactors{
+ Databases: []string{"instances/postgres-sample/databases/employee"},
+ },
+ },
+ {
+ expression: "request.time < timestamp(\"2023-07-04T07:40:05.658Z\") && (resource.database in [\"instances/postgres-sample/databases/employee\"])",
+ want: queryExportFactors{
+ Databases: []string{"instances/postgres-sample/databases/employee"},
+ },
+ },
+ {
+ expression: "request.time < timestamp(\"2023-08-02T07:33:45.686Z\") && (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name == \"public\" && resource.table_name in [\"dept_emp\",\"department\"])",
+ want: queryExportFactors{
+ Databases: []string{"instances/postgres-sample/databases/employee"},
+ },
+ },
+ {
+ expression: "request.time < timestamp(\"2023-07-10T08:14:34.788Z\")",
+ want: queryExportFactors{},
+ },
+ {
+ expression: "request.time < timestamp(\"2023-07-10T08:15:46.773Z\") && ((resource.database in [\"instances/postgres-sample/databases/blog\"]) || (resource.database == \"instances/postgres-sample/databases/employee\" && resource.schema_name in [\"public\"]))",
+ want: queryExportFactors{
+ Databases: []string{"instances/postgres-sample/databases/blog", "instances/postgres-sample/databases/employee"},
+ },
+ },
+ }
+ for _, tt := range tests {
+ factors, err := getQueryExportFactors(tt.expression)
+ a.NoError(err)
+ a.Equal(tt.want, *factors)
+ }
+}
diff --git a/backend/common/risk.go b/backend/component/review/risk.go
similarity index 96%
rename from backend/common/risk.go
rename to backend/component/review/risk.go
index 407d897640..28b473a22c 100644
--- a/backend/common/risk.go
+++ b/backend/component/review/risk.go
@@ -1,5 +1,4 @@
-//nolint:revive
-package common
+package review
import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
diff --git a/backend/common/risk_test.go b/backend/component/review/risk_test.go
similarity index 98%
rename from backend/common/risk_test.go
rename to backend/component/review/risk_test.go
index d8c5f1e767..5858c787c4 100644
--- a/backend/common/risk_test.go
+++ b/backend/component/review/risk_test.go
@@ -1,4 +1,4 @@
-package common // nolint:revive
+package review
import (
"testing"
diff --git a/backend/plugin/db/bigquery/bigquery.go b/backend/plugin/db/bigquery/bigquery.go
index b38d6c57fa..13d5a9672f 100644
--- a/backend/plugin/db/bigquery/bigquery.go
+++ b/backend/plugin/db/bigquery/bigquery.go
@@ -17,7 +17,6 @@ import (
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/structpb"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
@@ -213,7 +212,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q
result.Rows = append(result.Rows, row)
n := len(result.Rows)
if (n&(n-1) == 0) && int64(proto.Size(result)) > queryContext.MaximumSQLResultSize {
- result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
+ result.Error = util.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
break
}
}
diff --git a/backend/plugin/db/cassandra/cassandra.go b/backend/plugin/db/cassandra/cassandra.go
index 1c0737c5a3..9df900cac2 100644
--- a/backend/plugin/db/cassandra/cassandra.go
+++ b/backend/plugin/db/cassandra/cassandra.go
@@ -18,7 +18,6 @@ import (
"google.golang.org/protobuf/types/known/timestamppb"
"gopkg.in/inf.v0"
- "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
@@ -158,7 +157,7 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, rawStatement string
result.Rows = append(result.Rows, row)
n := len(result.Rows)
if (n&(n-1) == 0) && int64(proto.Size(result)) > queryContext.MaximumSQLResultSize {
- result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
+ result.Error = util.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
break
}
diff --git a/backend/plugin/db/hive/hive.go b/backend/plugin/db/hive/hive.go
index 7735c1d38a..fdd74ad507 100644
--- a/backend/plugin/db/hive/hive.go
+++ b/backend/plugin/db/hive/hive.go
@@ -15,7 +15,6 @@ import (
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/structpb"
- "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -317,7 +316,7 @@ func (d *Driver) queryStatementWithLimit(ctx context.Context, statement string,
// Check size limit
n := len(result.Rows)
if (n&(n-1) == 0) && limit > 0 && int64(proto.Size(result)) > limit {
- result.Error = common.FormatMaximumSQLResultSizeMessage(limit)
+ result.Error = util.FormatMaximumSQLResultSizeMessage(limit)
break
}
}
diff --git a/backend/plugin/db/redis/redis.go b/backend/plugin/db/redis/redis.go
index 05509d3f0f..4643e1c09d 100644
--- a/backend/plugin/db/redis/redis.go
+++ b/backend/plugin/db/redis/redis.go
@@ -20,7 +20,6 @@ import (
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
@@ -347,7 +346,7 @@ func setQueryResultRows(result *v1pb.QueryResult, cmd *redis.Cmd, limit int64) {
result.Rows = append(result.Rows, getResultRow(i+1, v))
n := len(result.Rows)
if (n&(n-1) == 0) && int64(proto.Size(result)) > limit {
- result.Error = common.FormatMaximumSQLResultSizeMessage(limit)
+ result.Error = util.FormatMaximumSQLResultSizeMessage(limit)
return
}
}
diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.go
index 5b3e395e9e..b989170850 100644
--- a/backend/plugin/db/spanner/spanner.go
+++ b/backend/plugin/db/spanner/spanner.go
@@ -26,7 +26,6 @@ import (
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/timestamppb"
- "github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
@@ -434,7 +433,7 @@ func (d *Driver) queryStatement(ctx context.Context, statement string, queryCont
result.Rows = append(result.Rows, rowData)
n := len(result.Rows)
if (n&(n-1) == 0) && int64(proto.Size(result)) > queryContext.MaximumSQLResultSize {
- result.Error = common.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
+ result.Error = util.FormatMaximumSQLResultSizeMessage(queryContext.MaximumSQLResultSize)
break
}
if queryContext.Limit > 0 && n >= queryContext.Limit {
diff --git a/backend/plugin/db/util/driverutil.go b/backend/plugin/db/util/driverutil.go
index 6a7f37cc70..58cc9d4da3 100644
--- a/backend/plugin/db/util/driverutil.go
+++ b/backend/plugin/db/util/driverutil.go
@@ -15,7 +15,6 @@ import (
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/timestamppb"
- "github.com/bytebase/bytebase/backend/common"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/utils"
)
@@ -136,7 +135,7 @@ func RowsToLimitedQueryResult(rows *sql.Rows, valueMaker func(string, *sql.Colum
result.Rows = append(result.Rows, row)
n := len(result.Rows)
if (n&(n-1) == 0) && int64(proto.Size(result)) > limit {
- result.Error = common.FormatMaximumSQLResultSizeMessage(limit)
+ result.Error = FormatMaximumSQLResultSizeMessage(limit)
break
}
}
@@ -242,3 +241,7 @@ func GetColumnIndex(columns []string, name string) (int, bool) {
}
return 0, false
}
+
+func FormatMaximumSQLResultSizeMessage(limit int64) string {
+ return fmt.Sprintf("Output of query exceeds max allowed output size of %dMB", limit/1024/1024)
+}
diff --git a/docs/backend-common-audit.md b/docs/backend-common-audit.md
index 04d39514bb..845995524e 100644
--- a/docs/backend-common-audit.md
+++ b/docs/backend-common-audit.md
@@ -9,9 +9,12 @@ conversion to `plugin/parser/base`, and TiDB error-position conversion to a
private TiDB parser helper. A subsequent migration moves the Connect-specific
parser-engine conversion into a private v1 helper. This migration moves
transaction configuration and SQL isolation conversion to `plugin/db/transaction`, and audit recording,
-stdout formatting, and HTTP metadata helpers to `component/audit`. Protobuf
-sanitization remains unchanged pending a separate ownership decision. The
-remaining entries are proposals.
+stdout formatting, and HTTP metadata helpers to `component/audit`.
+The latest migration moves query/export factors and risk classification to
+`component/review`, external-URL normalization to `component/config`, and the SQL
+result-size message to `plugin/db/util`. Protobuf sanitization remains shared in
+`common/proto.go`, with its tests in `common/proto_test.go`. The unused ANTLR
+position helpers are removed. The remaining entries are proposals.
The root package mixes unrelated ownership: resource names, identity, request
context, audit transport, policy expressions, SQL execution, and generic helpers.
@@ -41,8 +44,8 @@ necessarily existing files or packages.
| Transaction types and default mode in `common/engine.go` | `plugin/db/transaction` (implemented) | Execution settings and SQL isolation conversion are driver-owned. The directive parser imports this leaf package without depending on the full driver package. |
| `ConvertToParserEngine` (completed) | Private helper in `api/v1` | Only v1 calls it, and it returns a Connect error. The move preserves the mapping and error behavior. |
| Audit callback keys, setters, getters, `PermissionDeniedError` | Private helpers in `api/v1` | These coordinate v1 interceptors and handlers; no production callers outside v1 were found. |
-| `GetQueryExportFactors` and its traversal | Private helpers in `component/review` | The only production caller is review evaluation. It should consume the shared IAM expression definitions described below. |
-| `SanitizeUTF8Message` and reflection traversal | Deferred | Oracle metadata sync is the current caller, but the implementation handles arbitrary protobuf messages and can serve other engines. Decide shared ownership separately. |
+| Query/export factors and their traversal (implemented) | Private helpers in `component/review/query_export_factors.go` | The only production caller is review evaluation. It should consume the shared IAM expression definitions described below. |
+| `SanitizeUTF8Message` and reflection traversal (implemented) | `common/proto.go` | Oracle metadata sync is the current caller, but the implementation handles arbitrary protobuf messages and remains shared. |
| `GetPostgresSocketDir` | `resources/postgres` | Callers are embedded PostgreSQL setup, server startup, and the self-hosted sample manager. |
| `ReleaseMode` | `component/config` | `Profile.Mode` is the central configuration field; the package currently imports common only for this type. |
| Build-tagged `IsDev` | Local build-tagged files in `plugin/db/cosmosdb` | Cosmos DB is its only production consumer. Keep build-time mode distinct from runtime `Profile.Mode`. |
@@ -56,7 +59,7 @@ necessarily existing files or packages.
| `AuthContext`, `DelegatedGrant`, authorization resource types | `api/auth` | They describe transport authentication and authorization. These can live with the existing auth implementation, unlike the cross-component workspace context. |
| Workspace/user context keys and workspace accessor | A leaf `common/requestcontext` package | Review and parser-context components also consume request identity. Keep it independent of `api/auth`, store, and Connect. Use typed accessors and private keys as a separate interface cleanup. |
| `audit.go` | `component/audit` (implemented) | Shared by v1, MCP, and OAuth2. Preserves the narrow writer interface, detached bounded write, denial severity, stdout behavior, and caller-IP semantics. HTTP metadata helpers remain with the audit module. |
-| Approval CEL definitions/validation and `risk.go` | `component/review` | This is review policy. Keep validation and runtime evaluation on the same definitions; map validation errors to Connect at the v1 caller. |
+| Approval CEL definitions/validation; risk classification already moved | `component/review` | This is review policy. Keep validation and runtime evaluation on the same definitions; map validation errors to Connect at the v1 caller. |
| IAM CEL definitions, member validation, `EvalBindingCondition` | A leaf `component/iam/condition` package | Store and utils call the evaluator. Moving it into the parent IAM package creates a cycle because IAM imports store and utils. Preserve partial-evaluation behavior. |
| Masking CEL definitions/validation | Initially private files in `api/v1` | All current external consumers are v1. The existing `component/masker` implements value masking, which is a different responsibility from policy-expression evaluation. |
| Database-group CEL definitions/validation | A leaf `component/databasegroup` package | Shared by v1 and the group evaluation in `utils`; move that evaluation into this owner too. Keep the expression core independent of store. |
@@ -77,12 +80,13 @@ not imply shared ownership. Split their tests along the same lines.
| --- | --- |
| `RoundRows`, `AddRows` | `plugin/parser/base`, alongside row-estimate handling; retain saturation tests. |
| `RandomString` | The shared identity leaf, alongside its credential consumers; retain cryptographic randomness. |
-| `NormalizeExternalURL` | `component/config`, shared by startup and API configuration validation. |
+| `NormalizeExternalURL` (implemented) | `component/config/url.go`, shared by startup, API configuration validation, and OAuth resource validation. |
| `TruncateString`, `SanitizeUTF8String` | A focused `common/text` package; preserve rune-count and invalid-byte behavior. |
| `ProtojsonUnmarshaler` | `common/protoutil`; it is used beyond store. Preserve `DiscardUnknown` for persisted protobuf JSON compatibility. |
| `Uniq` | A focused collection helper if retained; preserve order. `slices.Compact` is not an equivalent replacement. |
| `IsNil` | Inspect its v1 and T-SQL callers for typed nil checks before retaining a shared reflection helper. |
-| `MaximumCommands`, default SQL result size and result-size message | `plugin/db`, which owns the execution interface and is already consumed by the relevant drivers. |
+| `MaximumCommands`, default SQL result size | `plugin/db`, which owns the execution interface and is already consumed by the relevant drivers. |
+| Result-size message (implemented) | `plugin/db/util`, shared by database drivers. |
| `MaximumAdvicePerStatus`, `MaximumLintExplainSize` | `plugin/advisor`, alongside review and EXPLAIN-budget enforcement. |
| `MaxSheetSize`, `MaxSheetCheckSize` | A leaf `component/sheet/limits` package; the current sheet component and driver dependencies need to remain separate. Revisit whether display, parsing, and prior-backup limits truly share one policy in a separate behavior change. |
@@ -95,8 +99,8 @@ test infrastructure, not a production foundation.
## Delete before relocating
+The unused ANTLR position helpers and their tests have been removed.
Repository-wide Go searches found no production callers for
-`ConvertANTLRPositionToPosition`, `ConvertANTLRTokenToExclusiveEndPosition`,
`TrimSuffixAndGetInstanceDatabaseID`, `GetProjectIDPlanIDPlanCheckRunID`,
`FormatRevision`, `FormatProjectRevision`, or `FormatSpec`.
Some have tests only. The default test/prod environment constants also have no