Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: properly display Spanner/ClickHouse/Cassandra query results (#18204)
Vincent Huang3600fb05d1f5
diff --git a/backend/plugin/db/spanner/spanner.go b/backend/plugin/db/spanner/spanner.goindex 10a9900410..dfc8a389ea 100644--- a/backend/plugin/db/spanner/spanner.go+++ b/backend/plugin/db/spanner/spanner.go@@ -4,8 +4,12 @@ package spannerimport ("context""database/sql"+ "encoding/base64"+ "encoding/json""fmt"+ "log/slog""regexp"+ "strconv""strings""time"@@ -18,8 +22,11 @@ import ("google.golang.org/api/option""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/durationpb"+ "google.golang.org/protobuf/types/known/structpb"+ "google.golang.org/protobuf/types/known/timestamppb""github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db"@@ -414,6 +421,114 @@ func (d *Driver) querySingleSQL(ctx context.Context, statement string, queryContreturn result, nil}+// convertSpannerValue converts google.protobuf.Value to RowValue.+// Uses type metadata to preserve INT64 precision and properly handle all Spanner types.+func convertSpannerValue(colType *sppb.Type, v *structpb.Value) *v1pb.RowValue {+ if v == nil || v.Kind == nil {+ return util.NullRowValue+ }++ switch v.Kind.(type) {+ case *structpb.Value_NullValue:+ return util.NullRowValue+ case *structpb.Value_StringValue:+ stringValue := v.GetStringValue()+ if colType == nil {+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: stringValue,+ }}+ }++ switch colType.Code {+ case sppb.TypeCode_INT64:+ // Spanner encodes INT64 as strings to preserve precision+ val, err := strconv.ParseInt(stringValue, 10, 64)+ if err != nil {+ slog.Error("failed to parse INT64 string value", log.BBError(err))+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: stringValue,+ }}+ }+ return &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: val,+ }}+ case sppb.TypeCode_TIMESTAMP:+ // Spanner encodes TIMESTAMP as RFC3339 string with nanosecond precision+ t, err := time.Parse(time.RFC3339Nano, stringValue)+ if err != nil {+ slog.Error("failed to parse TIMESTAMP string value", log.BBError(err))+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: stringValue,+ }}+ }+ // Determine accuracy from the string+ accuracy := int32(6) // Default to microsecond precision+ if dotIndex := strings.Index(stringValue, "."); dotIndex >= 0 {+ // Find the end of fractional seconds (before 'Z' or timezone)+ endIndex := strings.IndexAny(stringValue[dotIndex:], "Z+-")+ if endIndex > 0 {+ accuracy = int32(endIndex - 1)+ if accuracy > 9 {+ accuracy = 9 // Cap at nanosecond precision+ }+ }+ }+ return &v1pb.RowValue{Kind: &v1pb.RowValue_TimestampValue{+ TimestampValue: &v1pb.RowValue_Timestamp{+ GoogleTimestamp: timestamppb.New(t),+ Accuracy: accuracy,+ },+ }}+ case sppb.TypeCode_BYTES:+ // Spanner encodes BYTES as base64 string+ bytes, err := base64.StdEncoding.DecodeString(stringValue)+ if err != nil {+ slog.Error("failed to decode BYTES base64 string value", log.BBError(err))+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: stringValue,+ }}+ }+ return &v1pb.RowValue{Kind: &v1pb.RowValue_BytesValue{+ BytesValue: bytes,+ }}+ default:+ // DATE, JSON, STRING all stay as StringValue+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: stringValue,+ }}+ }+ case *structpb.Value_NumberValue:+ // Check if this is INT64 to preserve precision+ if colType != nil && colType.Code == sppb.TypeCode_INT64 {+ return &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: int64(v.GetNumberValue()),+ }}+ }+ return &v1pb.RowValue{Kind: &v1pb.RowValue_DoubleValue{+ DoubleValue: v.GetNumberValue(),+ }}+ case *structpb.Value_BoolValue:+ return &v1pb.RowValue{Kind: &v1pb.RowValue_BoolValue{+ BoolValue: v.GetBoolValue(),+ }}+ case *structpb.Value_ListValue, *structpb.Value_StructValue:+ // Flatten complex types to JSON strings (no array_value/struct_value in RowValue proto).+ goValue := v.AsInterface()+ jsonBytes, err := json.Marshal(goValue)+ if err != nil {+ slog.Error("failed to marshal Spanner complex value", log.BBError(err))+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: fmt.Sprintf("%v", goValue),+ }}+ }+ return &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: string(jsonBytes),+ }}+ default:+ return util.NullRowValue+ }+}+func readRow(row *spanner.Row) (*v1pb.QueryRow, error) {result := &v1pb.QueryRow{}for i := 0; i < row.Size(); i++ {@@ -421,7 +536,7 @@ func readRow(row *spanner.Row) (*v1pb.QueryRow, error) {if err := row.Column(i, &col); err != nil {return nil, err}- result.Values = append(result.Values, &v1pb.RowValue{Kind: &v1pb.RowValue_ValueValue{ValueValue: col.Value}})+ result.Values = append(result.Values, convertSpannerValue(col.Type, col.Value))}return result, nildiff --git a/backend/plugin/db/spanner/spanner_test.go b/backend/plugin/db/spanner/spanner_test.goindex 2e8485574f..5c64e0c6ec 100644--- a/backend/plugin/db/spanner/spanner_test.go+++ b/backend/plugin/db/spanner/spanner_test.go@@ -1,10 +1,17 @@package spannerimport (+ "math""testing"+ "time"- "cloud.google.com/go/spanner/apiv1/spannerpb"+ sppb "cloud.google.com/go/spanner/apiv1/spannerpb""github.com/stretchr/testify/require"+ "google.golang.org/protobuf/types/known/structpb"+ "google.golang.org/protobuf/types/known/timestamppb"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/db/util")func TestGetDatabaseFromDSN(t *testing.T) {@@ -39,32 +46,32 @@ func TestGetDatabaseFromDSN(t *testing.T) {func TestGetColumnTypeName(t *testing.T) {tests := []struct {- spannerType spannerpb.Type+ spannerType sppb.Typewant string}{{- spannerType: spannerpb.Type{- Code: spannerpb.TypeCode_JSON,+ spannerType: sppb.Type{+ Code: sppb.TypeCode_JSON,},want: "JSON",},{- spannerType: spannerpb.Type{- Code: spannerpb.TypeCode_DATE,+ spannerType: sppb.Type{+ Code: sppb.TypeCode_DATE,},want: "DATE",},{- spannerType: spannerpb.Type{- Code: spannerpb.TypeCode_TIMESTAMP,+ spannerType: sppb.Type{+ Code: sppb.TypeCode_TIMESTAMP,},want: "TIMESTAMP",},{- spannerType: spannerpb.Type{- Code: spannerpb.TypeCode_ARRAY,- ArrayElementType: &spannerpb.Type{- Code: spannerpb.TypeCode_BYTES,+ spannerType: sppb.Type{+ Code: sppb.TypeCode_ARRAY,+ ArrayElementType: &sppb.Type{+ Code: sppb.TypeCode_BYTES,},},want: "[]BYTES",@@ -77,3 +84,247 @@ func TestGetColumnTypeName(t *testing.T) {a.Equal(tests[i].want, got)}}++// Helper function to create expected timestamp RowValue for tests+func timestampRowValue(rfc3339 string, accuracy int32) *v1pb.RowValue {+ t, _ := time.Parse(time.RFC3339Nano, rfc3339)+ return &v1pb.RowValue{Kind: &v1pb.RowValue_TimestampValue{+ TimestampValue: &v1pb.RowValue_Timestamp{+ GoogleTimestamp: timestamppb.New(t),+ Accuracy: accuracy,+ },+ }}+}++func TestConvertSpannerValue(t *testing.T) {+ tests := []struct {+ name string+ colType *sppb.Type+ value *structpb.Value+ expected *v1pb.RowValue+ }{+ {+ name: "nil value",+ colType: nil,+ value: nil,+ expected: util.NullRowValue,+ },+ {+ name: "null value",+ colType: &sppb.Type{Code: sppb.TypeCode_STRING},+ value: structpb.NewNullValue(),+ expected: util.NullRowValue,+ },+ {+ name: "string value",+ colType: &sppb.Type{Code: sppb.TypeCode_STRING},+ value: structpb.NewStringValue("hello"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "hello",+ }},+ },+ {+ name: "bool value - true",+ colType: &sppb.Type{Code: sppb.TypeCode_BOOL},+ value: structpb.NewBoolValue(true),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_BoolValue{+ BoolValue: true,+ }},+ },+ {+ name: "bool value - false",+ colType: &sppb.Type{Code: sppb.TypeCode_BOOL},+ value: structpb.NewBoolValue(false),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_BoolValue{+ BoolValue: false,+ }},+ },+ {+ name: "float64 value",+ colType: &sppb.Type{Code: sppb.TypeCode_FLOAT64},+ value: structpb.NewNumberValue(3.14159),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_DoubleValue{+ DoubleValue: 3.14159,+ }},+ },+ {+ name: "int64 small value",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewNumberValue(42),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: 42,+ }},+ },+ {+ name: "int64 large value - near 2^53",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewNumberValue(9007199254740992), // 2^53+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: 9007199254740992,+ }},+ },+ {+ name: "int64 large value - within float64 safe range",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewNumberValue(9007199254740991), // 2^53 - 1 (max safe int in float64)+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: 9007199254740991,+ }},+ },+ {+ name: "int64 negative large value - within float64 safe range",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewNumberValue(-9007199254740991), // -(2^53 - 1)+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: -9007199254740991,+ }},+ },+ {+ name: "int64 as string - small value",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewStringValue("42"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: 42,+ }},+ },+ {+ name: "int64 as string - max value",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewStringValue("9223372036854775807"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: math.MaxInt64,+ }},+ },+ {+ name: "int64 as string - min value",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewStringValue("-9223372036854775808"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_Int64Value{+ Int64Value: math.MinInt64,+ }},+ },+ {+ name: "int64 as string - invalid falls back to string",+ colType: &sppb.Type{Code: sppb.TypeCode_INT64},+ value: structpb.NewStringValue("not-a-number"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "not-a-number",+ }},+ },+ {+ name: "number without type metadata defaults to double",+ colType: nil,+ value: structpb.NewNumberValue(42),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_DoubleValue{+ DoubleValue: 42,+ }},+ },+ {+ name: "array value",+ colType: &sppb.Type{Code: sppb.TypeCode_ARRAY},+ value: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewNumberValue(1), structpb.NewNumberValue(2), structpb.NewNumberValue(3)}}),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "[1,2,3]",+ }},+ },+ {+ name: "struct value",+ colType: &sppb.Type{Code: sppb.TypeCode_STRUCT},+ value: structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{+ "name": structpb.NewStringValue("Alice"),+ "age": structpb.NewNumberValue(30),+ }}),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: `{"age":30,"name":"Alice"}`,+ }},+ },+ {+ name: "nested array",+ colType: &sppb.Type{Code: sppb.TypeCode_ARRAY},+ value: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{+ structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewNumberValue(1), structpb.NewNumberValue(2)}}),+ structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{structpb.NewNumberValue(3), structpb.NewNumberValue(4)}}),+ }}),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "[[1,2],[3,4]]",+ }},+ },+ {+ name: "array of structs",+ colType: &sppb.Type{Code: sppb.TypeCode_ARRAY},+ value: structpb.NewListValue(&structpb.ListValue{Values: []*structpb.Value{+ structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{"id": structpb.NewNumberValue(1)}}),+ structpb.NewStructValue(&structpb.Struct{Fields: map[string]*structpb.Value{"id": structpb.NewNumberValue(2)}}),+ }}),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: `[{"id":1},{"id":2}]`,+ }},+ },+ {+ name: "timestamp value",+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},+ value: structpb.NewStringValue("2024-01-15T10:30:45.123456Z"),+ expected: timestampRowValue("2024-01-15T10:30:45.123456Z", 6),+ },+ {+ name: "timestamp with nanosecond precision",+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},+ value: structpb.NewStringValue("2024-01-15T10:30:45.123456789Z"),+ expected: timestampRowValue("2024-01-15T10:30:45.123456789Z", 9),+ },+ {+ name: "timestamp without fractional seconds",+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},+ value: structpb.NewStringValue("2024-01-15T10:30:45Z"),+ expected: timestampRowValue("2024-01-15T10:30:45Z", 6),+ },+ {+ name: "timestamp invalid format falls back to string",+ colType: &sppb.Type{Code: sppb.TypeCode_TIMESTAMP},+ value: structpb.NewStringValue("not-a-timestamp"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "not-a-timestamp",+ }},+ },+ {+ name: "bytes value",+ colType: &sppb.Type{Code: sppb.TypeCode_BYTES},+ value: structpb.NewStringValue("SGVsbG8gV29ybGQ="), // "Hello World" in base64+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_BytesValue{+ BytesValue: []byte("Hello World"),+ }},+ },+ {+ name: "bytes invalid base64 falls back to string",+ colType: &sppb.Type{Code: sppb.TypeCode_BYTES},+ value: structpb.NewStringValue("not-valid-base64!"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "not-valid-base64!",+ }},+ },+ {+ name: "date value",+ colType: &sppb.Type{Code: sppb.TypeCode_DATE},+ value: structpb.NewStringValue("2024-01-15"),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: "2024-01-15",+ }},+ },+ {+ name: "json value",+ colType: &sppb.Type{Code: sppb.TypeCode_JSON},+ value: structpb.NewStringValue(`{"name":"Alice","age":30}`),+ expected: &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{+ StringValue: `{"name":"Alice","age":30}`,+ }},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ a := require.New(t)+ result := convertSpannerValue(tt.colType, tt.value)+ a.Equal(tt.expected, result)+ })+ }+}