Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: render MongoDB doubles in mongosh notation instead of scientific notation (#20905)

h3n4ldf43697a55a7
diff --git a/backend/plugin/db/mongodb/extjson.go b/backend/plugin/db/mongodb/extjson.go
new file mode 100644
index 0000000000..5e7f488dfa
--- /dev/null
+++ b/backend/plugin/db/mongodb/extjson.go
@@ -0,0 +1,163 @@
+package mongodb
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "strconv"
+ "strings"
+)
+
+// The MongoDB Go driver formats relaxed Extended JSON doubles with Go's %G
+// verb, which switches to scientific notation at 1e6 (e.g.
+// 1.779696815227E+12) — notation no MongoDB tool uses.
+// normalizeExtJSONNumbers rewrites every double token into the notation
+// mongosh displays (ECMAScript Number::toString): plain decimal when the
+// value's decimal exponent is within [-6, 21), scientific otherwise,
+// shortest round-trip digits, and no ".0" suffix on integral doubles.
+//
+// A number token is a double if and only if it contains '.', 'e', or 'E':
+// the driver suffixes integral doubles with ".0", while int32/int64
+// serialize as bare digits. Non-finite doubles and Decimal128 arrive as
+// {"$numberDouble": "..."} / {"$numberDecimal": "..."} strings, which a
+// number-token rewrite never touches. On any parse anomaly the input is
+// returned unchanged.
+func normalizeExtJSONNumbers(data []byte) []byte {
+ dec := json.NewDecoder(bytes.NewReader(data))
+ dec.UseNumber()
+
+ var out bytes.Buffer
+ out.Grow(len(data))
+ // Container stack: true for object, false for array.
+ var stack []bool
+ needComma := false
+ expectKey := false
+
+ completeValue := func() {
+ needComma = true
+ expectKey = len(stack) > 0 && stack[len(stack)-1]
+ }
+
+ for {
+ token, err := dec.Token()
+ if err == io.EOF {
+ break
+ }
+ if err != nil {
+ return data
+ }
+
+ if delim, ok := token.(json.Delim); ok && (delim == '}' || delim == ']') {
+ out.WriteByte(byte(delim))
+ stack = stack[:len(stack)-1]
+ completeValue()
+ continue
+ }
+
+ if needComma {
+ out.WriteByte(',')
+ needComma = false
+ }
+
+ switch v := token.(type) {
+ case json.Delim:
+ // '{' or '['; closing delims are handled above.
+ out.WriteByte(byte(v))
+ stack = append(stack, v == '{')
+ expectKey = v == '{'
+ case string:
+ if err := writeJSONString(&out, v); err != nil {
+ return data
+ }
+ if expectKey {
+ out.WriteByte(':')
+ expectKey = false
+ } else {
+ completeValue()
+ }
+ case json.Number:
+ raw := v.String()
+ if strings.ContainsAny(raw, ".eE") {
+ if f, err := strconv.ParseFloat(raw, 64); err == nil {
+ raw = formatDoubleJS(f)
+ }
+ }
+ out.WriteString(raw)
+ completeValue()
+ case bool:
+ out.WriteString(strconv.FormatBool(v))
+ completeValue()
+ case nil:
+ out.WriteString("null")
+ completeValue()
+ default:
+ return data
+ }
+ }
+
+ if len(stack) != 0 {
+ return data
+ }
+ return out.Bytes()
+}
+
+// writeJSONString encodes s as a JSON string without HTML escaping, matching
+// how the driver writes strings ("<" stays "<", not "\u003c").
+func writeJSONString(buf *bytes.Buffer, s string) error {
+ enc := json.NewEncoder(buf)
+ enc.SetEscapeHTML(false)
+ if err := enc.Encode(s); err != nil {
+ return err
+ }
+ // Encode appends a newline; drop it.
+ buf.Truncate(buf.Len() - 1)
+ return nil
+}
+
+// formatDoubleJS renders a finite float64 the way ECMAScript
+// Number::toString does — and therefore the way mongosh, Compass, and
+// DataGrip display it. The one deviation is negative zero, rendered "-0" to
+// keep the sign actually stored in the BSON double (String(-0) is "0" in
+// ECMAScript).
+func formatDoubleJS(f float64) string {
+ // Shortest round-trip digits in the form d[.ddd]e±XX.
+ s := strconv.FormatFloat(f, 'e', -1, 64)
+ mantissa, expStr, found := strings.Cut(s, "e")
+ if !found {
+ // ±Inf/NaN; unreachable from JSON number tokens.
+ return s
+ }
+ exp, err := strconv.Atoi(expStr)
+ if err != nil {
+ return s
+ }
+ sign := ""
+ if strings.HasPrefix(mantissa, "-") {
+ sign = "-"
+ mantissa = mantissa[1:]
+ }
+ digits := strings.Replace(mantissa, ".", "", 1)
+ // ECMAScript Number::toString(10): the value is digits × 10^(n-k) with
+ // k significant digits; plain decimal notation applies for -6 < n <= 21.
+ n := exp + 1
+ k := len(digits)
+ switch {
+ case k <= n && n <= 21:
+ return sign + digits + strings.Repeat("0", n-k)
+ case 0 < n && n <= 21:
+ return sign + digits[:n] + "." + digits[n:]
+ case -6 < n && n <= 0:
+ return sign + "0." + strings.Repeat("0", -n) + digits
+ default:
+ e := n - 1
+ expPart := "e+"
+ if e < 0 {
+ expPart = "e-"
+ e = -e
+ }
+ if k == 1 {
+ return sign + digits + expPart + strconv.Itoa(e)
+ }
+ return sign + digits[:1] + "." + digits[1:] + expPart + strconv.Itoa(e)
+ }
+}
diff --git a/backend/plugin/db/mongodb/extjson_test.go b/backend/plugin/db/mongodb/extjson_test.go
new file mode 100644
index 0000000000..70b18c6113
--- /dev/null
+++ b/backend/plugin/db/mongodb/extjson_test.go
@@ -0,0 +1,182 @@
+package mongodb
+
+import (
+ "math"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "go.mongodb.org/mongo-driver/v2/bson"
+)
+
+func TestFormatDoubleJS(t *testing.T) {
+ tests := []struct {
+ input float64
+ want string
+ }{
+ {0, "0"},
+ // Deviates from ECMAScript String(-0) == "0" to keep the sign
+ // actually stored in the BSON double.
+ {math.Copysign(0, -1), "-0"},
+ {1, "1"},
+ {-1, "-1"},
+ {42, "42"},
+ {999999, "999999"},
+ {1000000, "1000000"},
+ {1234567.5, "1234567.5"},
+ {-1234567.5, "-1234567.5"},
+ {123456.789, "123456.789"},
+ {0.5, "0.5"},
+ {0.1, "0.1"},
+ {123.25, "123.25"},
+ {0.000001, "0.000001"},
+ {0.0000001, "1e-7"},
+ {-0.0000001, "-1e-7"},
+ {1e-10, "1e-10"},
+ {2.5e-10, "2.5e-10"},
+ {5e-324, "5e-324"},
+ {0.30000000000000004, "0.30000000000000004"},
+ {1779696815227, "1779696815227"},
+ {585723378473606, "585723378473606"},
+ {9007199254740992, "9007199254740992"},
+ {123456789123456789, "123456789123456780"},
+ {36893488147419103232, "36893488147419103000"},
+ {1e20, "100000000000000000000"},
+ {999999999999999900000, "999999999999999900000"},
+ {1e21, "1e+21"},
+ {-1e21, "-1e+21"},
+ {1.5e21, "1.5e+21"},
+ {math.MaxFloat64, "1.7976931348623157e+308"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.want, func(t *testing.T) {
+ require.Equal(t, tt.want, formatDoubleJS(tt.input))
+ })
+ }
+}
+
+func TestNormalizeExtJSONNumbers(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ want string
+ }{
+ {
+ name: "issue 20895 document",
+ input: `{"_id":1129063441,"name":"repro user 1","created_at":1.779696815227E+12,"updated_at":1.779803079861E+12,"current_workspace_id":5.85723378473606E+14}`,
+ want: `{"_id":1129063441,"name":"repro user 1","created_at":1779696815227,"updated_at":1779803079861,"current_workspace_id":585723378473606}`,
+ },
+ {
+ name: "integral double drops .0",
+ input: `{"score":42.0}`,
+ want: `{"score":42}`,
+ },
+ {
+ name: "integer tokens unchanged",
+ input: `{"a":42,"b":9223372036854775807,"c":-15}`,
+ want: `{"a":42,"b":9223372036854775807,"c":-15}`,
+ },
+ {
+ name: "small scientific uses unpadded exponent",
+ input: `{"x":1E-07}`,
+ want: `{"x":1e-7}`,
+ },
+ {
+ name: "huge doubles stay scientific",
+ input: `{"x":1.5E+21}`,
+ want: `{"x":1.5e+21}`,
+ },
+ {
+ name: "numeric-looking strings and $numberDecimal untouched",
+ input: `{"s":"1.5E+10","d":{"$numberDecimal":"1.5E+100"}}`,
+ want: `{"s":"1.5E+10","d":{"$numberDecimal":"1.5E+100"}}`,
+ },
+ {
+ name: "$numberDouble special values untouched",
+ input: `{"nan":{"$numberDouble":"NaN"},"inf":{"$numberDouble":"Infinity"}}`,
+ want: `{"nan":{"$numberDouble":"NaN"},"inf":{"$numberDouble":"Infinity"}}`,
+ },
+ {
+ name: "nested arrays and objects",
+ input: `{"arr":[1.0E+07,2,"x",{"n":9.9E-08}],"b":true,"z":null}`,
+ want: `{"arr":[10000000,2,"x",{"n":9.9e-8}],"b":true,"z":null}`,
+ },
+ {
+ name: "negative zero keeps sign",
+ input: `{"z":-0.0}`,
+ want: `{"z":-0}`,
+ },
+ {
+ name: "string escapes survive",
+ input: `{"s":"a\"b\\c\né"}`,
+ want: `{"s":"a\"b\\c\né"}`,
+ },
+ {
+ name: "html characters not escaped",
+ input: `{"s":"<a>&"}`,
+ want: `{"s":"<a>&"}`,
+ },
+ {
+ name: "top-level array",
+ input: `[1.0E+06,{"k":2.5}]`,
+ want: `[1000000,{"k":2.5}]`,
+ },
+ {
+ name: "invalid json returned unchanged",
+ input: `{"a":`,
+ want: `{"a":`,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ require.Equal(t, tt.want, string(normalizeExtJSONNumbers([]byte(tt.input))))
+ })
+ }
+}
+
+func TestMarshalValueToExtJSONNumberNotation(t *testing.T) {
+ decimal, err := bson.ParseDecimal128("1.5E+100")
+ require.NoError(t, err)
+
+ tests := []struct {
+ name string
+ input any
+ want string
+ }{
+ {
+ name: "doubles render like mongosh",
+ input: bson.D{
+ {Key: "_id", Value: int32(1129063441)},
+ {Key: "created_at", Value: float64(1779696815227)},
+ {Key: "current_workspace_id", Value: float64(585723378473606)},
+ {Key: "score", Value: 42.5},
+ {Key: "whole", Value: float64(42)},
+ },
+ want: `{"_id":1129063441,"created_at":1779696815227,"current_workspace_id":585723378473606,"score":42.5,"whole":42}`,
+ },
+ {
+ name: "int64 precision preserved",
+ input: bson.D{{Key: "big", Value: int64(9223372036854775807)}},
+ want: `{"big":9223372036854775807}`,
+ },
+ {
+ name: "special doubles and decimal128 keep extended json form",
+ input: bson.D{
+ {Key: "nan", Value: math.NaN()},
+ {Key: "dec", Value: decimal},
+ },
+ want: `{"nan":{"$numberDouble":"NaN"},"dec":{"$numberDecimal":"1.5E+100"}}`,
+ },
+ {
+ name: "primitive value wrapped",
+ input: int64(5),
+ want: `{"value":5}`,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := marshalValueToExtJSON(tt.input)
+ require.NoError(t, err)
+ require.Equal(t, tt.want, got)
+ })
+ }
+}
diff --git a/backend/plugin/db/mongodb/mongodb.go b/backend/plugin/db/mongodb/mongodb.go
index d1f434f12b..d89546a137 100644
--- a/backend/plugin/db/mongodb/mongodb.go
+++ b/backend/plugin/db/mongodb/mongodb.go
@@ -237,5 +237,5 @@ func marshalValueToExtJSON(v any) (string, error) {
if err != nil {
return "", err
}
- return string(jsonBytes), nil
+ return string(normalizeExtJSONNumbers(jsonBytes)), nil
}
diff --git a/backend/plugin/db/mongodb/mongodb_testcontainer_test.go b/backend/plugin/db/mongodb/mongodb_testcontainer_test.go
index 7b7f0fec7f..f79746bf55 100644
--- a/backend/plugin/db/mongodb/mongodb_testcontainer_test.go
+++ b/backend/plugin/db/mongodb/mongodb_testcontainer_test.go
@@ -7,6 +7,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
+ "go.mongodb.org/mongo-driver/v2/bson"
"google.golang.org/protobuf/testing/protocmp"
"github.com/bytebase/bytebase/backend/common/testcontainer"
@@ -213,3 +214,64 @@ func TestQueryWithBracketNotationStructure(t *testing.T) {
require.Contains(t, jsonStr, `"age"`)
require.Contains(t, jsonStr, "Test User")
}
+
+// TestQueryDoubleNotation reproduces GitHub issue #20895: BSON doubles holding
+// large integers must render in mongosh notation (plain decimal), not the Go
+// driver's scientific notation (e.g. 1.779696815227E+12).
+func TestQueryDoubleNotation(t *testing.T) {
+ if testing.Short() {
+ t.Skip("Skipping MongoDB testcontainer test in short mode")
+ }
+
+ ctx := context.Background()
+
+ container := testcontainer.GetTestMongoDBContainer(ctx, t)
+ defer container.Close(ctx)
+
+ driver := &Driver{}
+ connConfig := db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Host: container.GetHost(),
+ Port: container.GetPort(),
+ Username: container.GetUsername(),
+ },
+ ConnectionContext: db.ConnectionContext{
+ DatabaseName: "testdb",
+ },
+ Password: container.GetPassword(),
+ }
+
+ openedDriver, err := driver.Open(ctx, storepb.Engine_MONGODB, connConfig)
+ require.NoError(t, err)
+ defer openedDriver.Close(ctx)
+
+ err = openedDriver.Ping(ctx)
+ require.NoError(t, err, "Failed to ping MongoDB")
+
+ // Insert through the official driver so the field types are guaranteed
+ // BSON doubles, matching the issue's data shape.
+ mongoDriver, ok := openedDriver.(*Driver)
+ require.True(t, ok)
+ _, err = mongoDriver.client.Database("testdb").Collection("users").InsertOne(ctx, bson.D{
+ {Key: "_id", Value: int32(1129063441)},
+ {Key: "name", Value: "repro user 1"},
+ {Key: "created_at", Value: float64(1779696815227)},
+ {Key: "updated_at", Value: float64(1779803079861)},
+ {Key: "current_workspace_id", Value: float64(585723378473606)},
+ })
+ require.NoError(t, err, "Failed to insert test data")
+
+ results, err := openedDriver.QueryConn(ctx, nil, `db.users.find()`, db.QueryContext{
+ Limit: 50,
+ MaximumSQLResultSize: 10 * 1024 * 1024,
+ })
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ require.Empty(t, results[0].Error)
+ require.Len(t, results[0].Rows, 1)
+
+ jsonStr := results[0].Rows[0].Values[0].GetStringValue()
+ require.Equal(t,
+ `{"_id":1129063441,"name":"repro user 1","created_at":1779696815227,"updated_at":1779803079861,"current_workspace_id":585723378473606}`,
+ jsonStr)
+}