Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: properly display Spanner/ClickHouse/Cassandra query results (#18204)

Vincent Huang3600fb05d1f5
diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.go
index 10a9900410..dfc8a389ea 100644
--- a/backend/plugin/db/spanner/spanner.go
+++ b/backend/plugin/db/spanner/spanner.go
@@ -4,8 +4,12 @@ package spanner
import (
"context"
"database/sql"
+ "encoding/base64"
+ "encoding/json"
"fmt"
+ "log/slog"
"regexp"
+ "strconv"
"strings"
"time"
@@ -18,8 +22,11 @@ import (
"google.golang.org/api/option"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/durationpb"
+ "google.golang.org/protobuf/types/known/structpb"
+ "google.golang.org/protobuf/types/known/timestamppb"
"github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
@@ -414,6 +421,114 @@ func (d *Driver) querySingleSQL(ctx context.Context, statement string, queryCont
return result, nil
}
+// convertSpannerValue converts google.protobuf.Value to RowValue.
+// Uses type metadata to preserve INT64 precision and properly handle all Spanner types.
+func convertSpannerValue(colType *sppb.Type, v *structpb.Value) *v1pb.RowValue {
+ if v == nil || v.Kind == nil {
+ return util.NullRowValue
+ }
+
+ switch v.Kind.(type) {
+ case *structpb.Value_NullValue:
+ return util.NullRowValue
+ case *structpb.Value_StringValue:
+ stringValue := v.GetStringValue()
+ if colType == nil {
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: stringValue,
+ }}
+ }
+
+ switch colType.Code {
+ case sppb.TypeCode_INT64:
+ // Spanner encodes INT64 as strings to preserve precision
+ val, err := strconv.ParseInt(stringValue, 10, 64)
+ if err != nil {
+ slog.Error("failed to parse INT64 string value", log.BBError(err))
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: stringValue,
+ }}
+ }
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: val,
+ }}
+ case sppb.TypeCode_TIMESTAMP:
+ // Spanner encodes TIMESTAMP as RFC3339 string with nanosecond precision
+ t, err := time.Parse(time.RFC3339Nano, stringValue)
+ if err != nil {
+ slog.Error("failed to parse TIMESTAMP string value", log.BBError(err))
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: stringValue,
+ }}
+ }
+ // Determine accuracy from the string
+ accuracy := int32(6) // Default to microsecond precision
+ if dotIndex := strings.Index(stringValue, "."); dotIndex >= 0 {
+ // Find the end of fractional seconds (before 'Z' or timezone)
+ endIndex := strings.IndexAny(stringValue[dotIndex:], "Z+-")
+ if endIndex > 0 {
+ accuracy = int32(endIndex - 1)
+ if accuracy > 9 {
+ accuracy = 9 // Cap at nanosecond precision
+ }
+ }
+ }
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_TimestampValue{
+ TimestampValue: &v1pb.RowValue_Timestamp{
+ GoogleTimestamp: timestamppb.New(t),
+ Accuracy: accuracy,
+ },
+ }}
+ case sppb.TypeCode_BYTES:
+ // Spanner encodes BYTES as base64 string
+ bytes, err := base64.StdEncoding.DecodeString(stringValue)
+ if err != nil {
+ slog.Error("failed to decode BYTES base64 string value", log.BBError(err))
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: stringValue,
+ }}
+ }
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_BytesValue{
+ BytesValue: bytes,
+ }}
+ default:
+ // DATE, JSON, STRING all stay as StringValue
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: stringValue,
+ }}
+ }
+ case *structpb.Value_NumberValue:
+ // Check if this is INT64 to preserve precision
+ if colType != nil && colType.Code == sppb.TypeCode_INT64 {
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: int64(v.GetNumberValue()),
+ }}
+ }
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_DoubleValue{
+ DoubleValue: v.GetNumberValue(),
+ }}
+ case *structpb.Value_BoolValue:
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_BoolValue{
+ BoolValue: v.GetBoolValue(),
+ }}
+ case *structpb.Value_ListValue, *structpb.Value_StructValue:
+ // Flatten complex types to JSON strings (no array_value/struct_value in RowValue proto).
+ goValue := v.AsInterface()
+ jsonBytes, err := json.Marshal(goValue)
+ if err != nil {
+ slog.Error("failed to marshal Spanner complex value", log.BBError(err))
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: fmt.Sprintf("%v", goValue),
+ }}
+ }
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: string(jsonBytes),
+ }}
+ default:
+ return util.NullRowValue
+ }
+}
+
func readRow(row *spanner.Row) (*v1pb.QueryRow, error) {
result := &v1pb.QueryRow{}
for i := 0; i < row.Size(); i++ {
@@ -421,7 +536,7 @@ func readRow(row *spanner.Row) (*v1pb.QueryRow, error) {
if err := row.Column(i, &col); err != nil {
return nil, err
}
- result.Values = append(result.Values, &v1pb.RowValue{Kind: &v1pb.RowValue_ValueValue{ValueValue: col.Value}})
+ result.Values = append(result.Values, convertSpannerValue(col.Type, col.Value))
}
return result, nil
diff --git a/backend/plugin/db/spanner/spanner_test.go b/backend/plugin/db/spanner/spanner_test.go
index 2e8485574f..5c64e0c6ec 100644
--- a/backend/plugin/db/spanner/spanner_test.go
+++ b/backend/plugin/db/spanner/spanner_test.go
@@ -1,10 +1,17 @@
package spanner
import (
+ "math"
"testing"
+ "time"
- "cloud.google.com/go/spanner/apiv1/spannerpb"
+ sppb "cloud.google.com/go/spanner/apiv1/spannerpb"
"github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/types/known/structpb"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/db/util"
)
func TestGetDatabaseFromDSN(t *testing.T) {
@@ -39,32 +46,32 @@ func TestGetDatabaseFromDSN(t *testing.T) {
func TestGetColumnTypeName(t *testing.T) {
tests := []struct {
- spannerType spannerpb.Type
+ spannerType sppb.Type
want string
}{
{
- spannerType: spannerpb.Type{
- Code: spannerpb.TypeCode_JSON,
+ spannerType: sppb.Type{
+ Code: sppb.TypeCode_JSON,
},
want: "JSON",
},
{
- spannerType: spannerpb.Type{
- Code: spannerpb.TypeCode_DATE,
+ spannerType: sppb.Type{
+ Code: sppb.TypeCode_DATE,
},
want: "DATE",
},
{
- spannerType: spannerpb.Type{
- Code: spannerpb.TypeCode_TIMESTAMP,
+ spannerType: sppb.Type{
+ Code: sppb.TypeCode_TIMESTAMP,
},
want: "TIMESTAMP",
},
{
- spannerType: spannerpb.Type{
- Code: spannerpb.TypeCode_ARRAY,
- ArrayElementType: &spannerpb.Type{
- Code: spannerpb.TypeCode_BYTES,
+ spannerType: sppb.Type{
+ Code: sppb.TypeCode_ARRAY,
+ ArrayElementType: &sppb.Type{
+ Code: sppb.TypeCode_BYTES,
},
},
want: "[]BYTES",
@@ -77,3 +84,247 @@ func TestGetColumnTypeName(t *testing.T) {
a.Equal(tests[i].want, got)
}
}
+
+// Helper function to create expected timestamp RowValue for tests
+func timestampRowValue(rfc3339 string, accuracy int32) *v1pb.RowValue {
+ t, _ := time.Parse(time.RFC3339Nano, rfc3339)
+ return &v1pb.RowValue{Kind: &v1pb.RowValue_TimestampValue{
+ TimestampValue: &v1pb.RowValue_Timestamp{
+ GoogleTimestamp: timestamppb.New(t),
+ Accuracy: accuracy,
+ },
+ }}
+}
+
+func TestConvertSpannerValue(t *testing.T) {
+ tests := []struct {
+ name string
+ colType *sppb.Type
+ value *structpb.Value
+ expected *v1pb.RowValue
+ }{
+ {
+ name: "nil value",
+ colType: nil,
+ value: nil,
+ expected: util.NullRowValue,
+ },
+ {
+ name: "null value",
+ colType: &sppb.Type{Code: sppb.TypeCode_STRING},
+ value: structpb.NewNullValue(),
+ expected: util.NullRowValue,
+ },
+ {
+ name: "string value",
+ colType: &sppb.Type{Code: sppb.TypeCode_STRING},
+ value: structpb.NewStringValue("hello"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "hello",
+ }},
+ },
+ {
+ name: "bool value - true",
+ colType: &sppb.Type{Code: sppb.TypeCode_BOOL},
+ value: structpb.NewBoolValue(true),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_BoolValue{
+ BoolValue: true,
+ }},
+ },
+ {
+ name: "bool value - false",
+ colType: &sppb.Type{Code: sppb.TypeCode_BOOL},
+ value: structpb.NewBoolValue(false),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_BoolValue{
+ BoolValue: false,
+ }},
+ },
+ {
+ name: "float64 value",
+ colType: &sppb.Type{Code: sppb.TypeCode_FLOAT64},
+ value: structpb.NewNumberValue(3.14159),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_DoubleValue{
+ DoubleValue: 3.14159,
+ }},
+ },
+ {
+ name: "int64 small value",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewNumberValue(42),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: 42,
+ }},
+ },
+ {
+ name: "int64 large value - near 2^53",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewNumberValue(9007199254740992), // 2^53
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: 9007199254740992,
+ }},
+ },
+ {
+ name: "int64 large value - within float64 safe range",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewNumberValue(9007199254740991), // 2^53 - 1 (max safe int in float64)
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: 9007199254740991,
+ }},
+ },
+ {
+ name: "int64 negative large value - within float64 safe range",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewNumberValue(-9007199254740991), // -(2^53 - 1)
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: -9007199254740991,
+ }},
+ },
+ {
+ name: "int64 as string - small value",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewStringValue("42"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: 42,
+ }},
+ },
+ {
+ name: "int64 as string - max value",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewStringValue("9223372036854775807"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: math.MaxInt64,
+ }},
+ },
+ {
+ name: "int64 as string - min value",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewStringValue("-9223372036854775808"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: math.MinInt64,
+ }},
+ },
+ {
+ name: "int64 as string - invalid falls back to string",
+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},
+ value: structpb.NewStringValue("not-a-number"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "not-a-number",
+ }},
+ },
+ {
+ name: "number without type metadata defaults to double",
+ colType: nil,
+ value: structpb.NewNumberValue(42),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_DoubleValue{
+ DoubleValue: 42,
+ }},
+ },
+ {
+ name: "array value",
+ colType: &sppb.Type{Code: sppb.TypeCode_ARRAY},
+ value: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewNumberValue(1), structpb.NewNumberValue(2), structpb.NewNumberValue(3)}}),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "[1,2,3]",
+ }},
+ },
+ {
+ name: "struct value",
+ colType: &sppb.Type{Code: sppb.TypeCode_STRUCT},
+ value: structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{
+ "name": structpb.NewStringValue("Alice"),
+ "age": structpb.NewNumberValue(30),
+ }}),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: `{"age":30,"name":"Alice"}`,
+ }},
+ },
+ {
+ name: "nested array",
+ colType: &sppb.Type{Code: sppb.TypeCode_ARRAY},
+ value: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{
+ structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewNumberValue(1), structpb.NewNumberValue(2)}}),
+ structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewNumberValue(3), structpb.NewNumberValue(4)}}),
+ }}),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "[[1,2],[3,4]]",
+ }},
+ },
+ {
+ name: "array of structs",
+ colType: &sppb.Type{Code: sppb.TypeCode_ARRAY},
+ value: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{
+ structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{"id": structpb.NewNumberValue(1)}}),
+ structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{"id": structpb.NewNumberValue(2)}}),
+ }}),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: `[{"id":1},{"id":2}]`,
+ }},
+ },
+ {
+ name: "timestamp value",
+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},
+ value: structpb.NewStringValue("2024-01-15T10:30:45.123456Z"),
+ expected: timestampRowValue("2024-01-15T10:30:45.123456Z", 6),
+ },
+ {
+ name: "timestamp with nanosecond precision",
+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},
+ value: structpb.NewStringValue("2024-01-15T10:30:45.123456789Z"),
+ expected: timestampRowValue("2024-01-15T10:30:45.123456789Z", 9),
+ },
+ {
+ name: "timestamp without fractional seconds",
+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},
+ value: structpb.NewStringValue("2024-01-15T10:30:45Z"),
+ expected: timestampRowValue("2024-01-15T10:30:45Z", 6),
+ },
+ {
+ name: "timestamp invalid format falls back to string",
+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},
+ value: structpb.NewStringValue("not-a-timestamp"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "not-a-timestamp",
+ }},
+ },
+ {
+ name: "bytes value",
+ colType: &sppb.Type{Code: sppb.TypeCode_BYTES},
+ value: structpb.NewStringValue("SGVsbG8gV29ybGQ="), // "Hello World" in base64
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_BytesValue{
+ BytesValue: []byte("Hello World"),
+ }},
+ },
+ {
+ name: "bytes invalid base64 falls back to string",
+ colType: &sppb.Type{Code: sppb.TypeCode_BYTES},
+ value: structpb.NewStringValue("not-valid-base64!"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "not-valid-base64!",
+ }},
+ },
+ {
+ name: "date value",
+ colType: &sppb.Type{Code: sppb.TypeCode_DATE},
+ value: structpb.NewStringValue("2024-01-15"),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: "2024-01-15",
+ }},
+ },
+ {
+ name: "json value",
+ colType: &sppb.Type{Code: sppb.TypeCode_JSON},
+ value: structpb.NewStringValue(`{"name":"Alice","age":30}`),
+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{
+ StringValue: `{"name":"Alice","age":30}`,
+ }},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ a := require.New(t)
+ result := convertSpannerValue(tt.colType, tt.value)
+ a.Equal(tt.expected, result)
+ })
+ }
+}