Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: render MongoDB doubles in mongosh notation instead of scientific notation (#20905)
h3n4ldf43697a55a7
diff --git a/backend/plugin/db/mongodb/extjson.go b/backend/plugin/db/mongodb/extjson.gonew file mode 100644index 0000000000..5e7f488dfa--- /dev/null+++ b/backend/plugin/db/mongodb/extjson.go@@ -0,0 +1,163 @@+package mongodb++import (+ "bytes"+ "encoding/json"+ "io"+ "strconv"+ "strings"+)++// The MongoDB Go driver formats relaxed Extended JSON doubles with Go's %G+// verb, which switches to scientific notation at 1e6 (e.g.+// 1.779696815227E+12) — notation no MongoDB tool uses.+// normalizeExtJSONNumbers rewrites every double token into the notation+// mongosh displays (ECMAScript Number::toString): plain decimal when the+// value's decimal exponent is within [-6, 21), scientific otherwise,+// shortest round-trip digits, and no ".0" suffix on integral doubles.+//+// A number token is a double if and only if it contains '.', 'e', or 'E':+// the driver suffixes integral doubles with ".0", while int32/int64+// serialize as bare digits. Non-finite doubles and Decimal128 arrive as+// {"$numberDouble": "..."} / {"$numberDecimal": "..."} strings, which a+// number-token rewrite never touches. On any parse anomaly the input is+// returned unchanged.+func normalizeExtJSONNumbers(data []byte) []byte {+ dec := json.NewDecoder(bytes.NewReader(data))+ dec.UseNumber()++ var out bytes.Buffer+ out.Grow(len(data))+ // Container stack: true for object, false for array.+ var stack []bool+ needComma := false+ expectKey := false++ completeValue := func() {+ needComma = true+ expectKey = len(stack) > 0 && stack[len(stack)-1]+ }++ for {+ token, err := dec.Token()+ if err == io.EOF {+ break+ }+ if err != nil {+ return data+ }++ if delim, ok := token.(json.Delim); ok && (delim == '}' || delim == ']') {+ out.WriteByte(byte(delim))+ stack = stack[:len(stack)-1]+ completeValue()+ continue+ }++ if needComma {+ out.WriteByte(',')+ needComma = false+ }++ switch v := token.(type) {+ case json.Delim:+ // '{' or '['; closing delims are handled above.+ out.WriteByte(byte(v))+ stack = append(stack, v == '{')+ expectKey = v == '{'+ case string:+ if err := writeJSONString(&out, v); err != nil {+ return data+ }+ if expectKey {+ out.WriteByte(':')+ expectKey = false+ } else {+ completeValue()+ }+ case json.Number:+ raw := v.String()+ if strings.ContainsAny(raw, ".eE") {+ if f, err := strconv.ParseFloat(raw, 64); err == nil {+ raw = formatDoubleJS(f)+ }+ }+ out.WriteString(raw)+ completeValue()+ case bool:+ out.WriteString(strconv.FormatBool(v))+ completeValue()+ case nil:+ out.WriteString("null")+ completeValue()+ default:+ return data+ }+ }++ if len(stack) != 0 {+ return data+ }+ return out.Bytes()+}++// writeJSONString encodes s as a JSON string without HTML escaping, matching+// how the driver writes strings ("<" stays "<", not "\u003c").+func writeJSONString(buf *bytes.Buffer, s string) error {+ enc := json.NewEncoder(buf)+ enc.SetEscapeHTML(false)+ if err := enc.Encode(s); err != nil {+ return err+ }+ // Encode appends a newline; drop it.+ buf.Truncate(buf.Len() - 1)+ return nil+}++// formatDoubleJS renders a finite float64 the way ECMAScript+// Number::toString does — and therefore the way mongosh, Compass, and+// DataGrip display it. The one deviation is negative zero, rendered "-0" to+// keep the sign actually stored in the BSON double (String(-0) is "0" in+// ECMAScript).+func formatDoubleJS(f float64) string {+ // Shortest round-trip digits in the form d[.ddd]e±XX.+ s := strconv.FormatFloat(f, 'e', -1, 64)+ mantissa, expStr, found := strings.Cut(s, "e")+ if !found {+ // ±Inf/NaN; unreachable from JSON number tokens.+ return s+ }+ exp, err := strconv.Atoi(expStr)+ if err != nil {+ return s+ }+ sign := ""+ if strings.HasPrefix(mantissa, "-") {+ sign = "-"+ mantissa = mantissa[1:]+ }+ digits := strings.Replace(mantissa, ".", "", 1)+ // ECMAScript Number::toString(10): the value is digits × 10^(n-k) with+ // k significant digits; plain decimal notation applies for -6 < n <= 21.+ n := exp + 1+ k := len(digits)+ switch {+ case k <= n && n <= 21:+ return sign + digits + strings.Repeat("0", n-k)+ case 0 < n && n <= 21:+ return sign + digits[:n] + "." + digits[n:]+ case -6 < n && n <= 0:+ return sign + "0." + strings.Repeat("0", -n) + digits+ default:+ e := n - 1+ expPart := "e+"+ if e < 0 {+ expPart = "e-"+ e = -e+ }+ if k == 1 {+ return sign + digits + expPart + strconv.Itoa(e)+ }+ return sign + digits[:1] + "." + digits[1:] + expPart + strconv.Itoa(e)+ }+}diff --git a/backend/plugin/db/mongodb/extjson_test.go b/backend/plugin/db/mongodb/extjson_test.gonew file mode 100644index 0000000000..70b18c6113--- /dev/null+++ b/backend/plugin/db/mongodb/extjson_test.go@@ -0,0 +1,182 @@+package mongodb++import (+ "math"+ "testing"++ "github.com/stretchr/testify/require"+ "go.mongodb.org/mongo-driver/v2/bson"+)++func TestFormatDoubleJS(t *testing.T) {+ tests := []struct {+ input float64+ want string+ }{+ {0, "0"},+ // Deviates from ECMAScript String(-0) == "0" to keep the sign+ // actually stored in the BSON double.+ {math.Copysign(0, -1), "-0"},+ {1, "1"},+ {-1, "-1"},+ {42, "42"},+ {999999, "999999"},+ {1000000, "1000000"},+ {1234567.5, "1234567.5"},+ {-1234567.5, "-1234567.5"},+ {123456.789, "123456.789"},+ {0.5, "0.5"},+ {0.1, "0.1"},+ {123.25, "123.25"},+ {0.000001, "0.000001"},+ {0.0000001, "1e-7"},+ {-0.0000001, "-1e-7"},+ {1e-10, "1e-10"},+ {2.5e-10, "2.5e-10"},+ {5e-324, "5e-324"},+ {0.30000000000000004, "0.30000000000000004"},+ {1779696815227, "1779696815227"},+ {585723378473606, "585723378473606"},+ {9007199254740992, "9007199254740992"},+ {123456789123456789, "123456789123456780"},+ {36893488147419103232, "36893488147419103000"},+ {1e20, "100000000000000000000"},+ {999999999999999900000, "999999999999999900000"},+ {1e21, "1e+21"},+ {-1e21, "-1e+21"},+ {1.5e21, "1.5e+21"},+ {math.MaxFloat64, "1.7976931348623157e+308"},+ }+ for _, tt := range tests {+ t.Run(tt.want, func(t *testing.T) {+ require.Equal(t, tt.want, formatDoubleJS(tt.input))+ })+ }+}++func TestNormalizeExtJSONNumbers(t *testing.T) {+ tests := []struct {+ name string+ input string+ want string+ }{+ {+ name: "issue 20895 document",+ input: `{"_id":1129063441,"name":"repro user 1","created_at":1.779696815227E+12,"updated_at":1.779803079861E+12,"current_workspace_id":5.85723378473606E+14}`,+ want: `{"_id":1129063441,"name":"repro user 1","created_at":1779696815227,"updated_at":1779803079861,"current_workspace_id":585723378473606}`,+ },+ {+ name: "integral double drops .0",+ input: `{"score":42.0}`,+ want: `{"score":42}`,+ },+ {+ name: "integer tokens unchanged",+ input: `{"a":42,"b":9223372036854775807,"c":-15}`,+ want: `{"a":42,"b":9223372036854775807,"c":-15}`,+ },+ {+ name: "small scientific uses unpadded exponent",+ input: `{"x":1E-07}`,+ want: `{"x":1e-7}`,+ },+ {+ name: "huge doubles stay scientific",+ input: `{"x":1.5E+21}`,+ want: `{"x":1.5e+21}`,+ },+ {+ name: "numeric-looking strings and $numberDecimal untouched",+ input: `{"s":"1.5E+10","d":{"$numberDecimal":"1.5E+100"}}`,+ want: `{"s":"1.5E+10","d":{"$numberDecimal":"1.5E+100"}}`,+ },+ {+ name: "$numberDouble special values untouched",+ input: `{"nan":{"$numberDouble":"NaN"},"inf":{"$numberDouble":"Infinity"}}`,+ want: `{"nan":{"$numberDouble":"NaN"},"inf":{"$numberDouble":"Infinity"}}`,+ },+ {+ name: "nested arrays and objects",+ input: `{"arr":[1.0E+07,2,"x",{"n":9.9E-08}],"b":true,"z":null}`,+ want: `{"arr":[10000000,2,"x",{"n":9.9e-8}],"b":true,"z":null}`,+ },+ {+ name: "negative zero keeps sign",+ input: `{"z":-0.0}`,+ want: `{"z":-0}`,+ },+ {+ name: "string escapes survive",+ input: `{"s":"a\"b\\c\né"}`,+ want: `{"s":"a\"b\\c\né"}`,+ },+ {+ name: "html characters not escaped",+ input: `{"s":"<a>&"}`,+ want: `{"s":"<a>&"}`,+ },+ {+ name: "top-level array",+ input: `[1.0E+06,{"k":2.5}]`,+ want: `[1000000,{"k":2.5}]`,+ },+ {+ name: "invalid json returned unchanged",+ input: `{"a":`,+ want: `{"a":`,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ require.Equal(t, tt.want, string(normalizeExtJSONNumbers([]byte(tt.input))))+ })+ }+}++func TestMarshalValueToExtJSONNumberNotation(t *testing.T) {+ decimal, err := bson.ParseDecimal128("1.5E+100")+ require.NoError(t, err)++ tests := []struct {+ name string+ input any+ want string+ }{+ {+ name: "doubles render like mongosh",+ input: bson.D{+ {Key: "_id", Value: int32(1129063441)},+ {Key: "created_at", Value: float64(1779696815227)},+ {Key: "current_workspace_id", Value: float64(585723378473606)},+ {Key: "score", Value: 42.5},+ {Key: "whole", Value: float64(42)},+ },+ want: `{"_id":1129063441,"created_at":1779696815227,"current_workspace_id":585723378473606,"score":42.5,"whole":42}`,+ },+ {+ name: "int64 precision preserved",+ input: bson.D{{Key: "big", Value: int64(9223372036854775807)}},+ want: `{"big":9223372036854775807}`,+ },+ {+ name: "special doubles and decimal128 keep extended json form",+ input: bson.D{+ {Key: "nan", Value: math.NaN()},+ {Key: "dec", Value: decimal},+ },+ want: `{"nan":{"$numberDouble":"NaN"},"dec":{"$numberDecimal":"1.5E+100"}}`,+ },+ {+ name: "primitive value wrapped",+ input: int64(5),+ want: `{"value":5}`,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got, err := marshalValueToExtJSON(tt.input)+ require.NoError(t, err)+ require.Equal(t, tt.want, got)+ })+ }+}diff --git a/backend/plugin/db/mongodb/mongodb.go b/backend/plugin/db/mongodb/mongodb.goindex d1f434f12b..d89546a137 100644--- a/backend/plugin/db/mongodb/mongodb.go+++ b/backend/plugin/db/mongodb/mongodb.go@@ -237,5 +237,5 @@ func marshalValueToExtJSON(v any) (string, error) {if err != nil {return "", err}- return string(jsonBytes), nil+ return string(normalizeExtJSONNumbers(jsonBytes)), nil}diff --git a/backend/plugin/db/mongodb/mongodb_testcontainer_test.go b/backend/plugin/db/mongodb/mongodb_testcontainer_test.goindex 7b7f0fec7f..f79746bf55 100644--- a/backend/plugin/db/mongodb/mongodb_testcontainer_test.go+++ b/backend/plugin/db/mongodb/mongodb_testcontainer_test.go@@ -7,6 +7,7 @@ import ("github.com/google/go-cmp/cmp""github.com/stretchr/testify/require"+ "go.mongodb.org/mongo-driver/v2/bson""google.golang.org/protobuf/testing/protocmp""github.com/bytebase/bytebase/backend/common/testcontainer"@@ -213,3 +214,64 @@ func TestQueryWithBracketNotationStructure(t *testing.T) {require.Contains(t, jsonStr, `"age"`)require.Contains(t, jsonStr, "Test User")}++// TestQueryDoubleNotation reproduces GitHub issue #20895: BSON doubles holding+// large integers must render in mongosh notation (plain decimal), not the Go+// driver's scientific notation (e.g. 1.779696815227E+12).+func TestQueryDoubleNotation(t *testing.T) {+ if testing.Short() {+ t.Skip("Skipping MongoDB testcontainer test in short mode")+ }++ ctx := context.Background()++ container := testcontainer.GetTestMongoDBContainer(ctx, t)+ defer container.Close(ctx)++ driver := &Driver{}+ connConfig := db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Host: container.GetHost(),+ Port: container.GetPort(),+ Username: container.GetUsername(),+ },+ ConnectionContext: db.ConnectionContext{+ DatabaseName: "testdb",+ },+ Password: container.GetPassword(),+ }++ openedDriver, err := driver.Open(ctx, storepb.Engine_MONGODB, connConfig)+ require.NoError(t, err)+ defer openedDriver.Close(ctx)++ err = openedDriver.Ping(ctx)+ require.NoError(t, err, "Failed to ping MongoDB")++ // Insert through the official driver so the field types are guaranteed+ // BSON doubles, matching the issue's data shape.+ mongoDriver, ok := openedDriver.(*Driver)+ require.True(t, ok)+ _, err = mongoDriver.client.Database("testdb").Collection("users").InsertOne(ctx, bson.D{+ {Key: "_id", Value: int32(1129063441)},+ {Key: "name", Value: "repro user 1"},+ {Key: "created_at", Value: float64(1779696815227)},+ {Key: "updated_at", Value: float64(1779803079861)},+ {Key: "current_workspace_id", Value: float64(585723378473606)},+ })+ require.NoError(t, err, "Failed to insert test data")++ results, err := openedDriver.QueryConn(ctx, nil, `db.users.find()`, db.QueryContext{+ Limit: 50,+ MaximumSQLResultSize: 10 * 1024 * 1024,+ })+ require.NoError(t, err)+ require.Len(t, results, 1)+ require.Empty(t, results[0].Error)+ require.Len(t, results[0].Rows, 1)++ jsonStr := results[0].Rows[0].Values[0].GetStringValue()+ require.Equal(t,+ `{"_id":1129063441,"name":"repro user 1","created_at":1779696815227,"updated_at":1779803079861,"current_workspace_id":585723378473606}`,+ jsonStr)+}