Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: standardize binary data export encoding across formats (#18233)
Vincent Huang4ea3f7ddae83
diff --git a/backend/component/export/csv.go b/backend/component/export/csv.goindex e6240ab91b..6b903e10a8 100644--- a/backend/component/export/csv.go+++ b/backend/component/export/csv.go@@ -1,10 +1,10 @@package exportimport (+ "encoding/hex""io""strconv""strings"- "time""google.golang.org/protobuf/types/known/structpb"@@ -67,15 +67,14 @@ func convertValueToBytesInCSV(value *v1pb.RowValue) []byte {case *v1pb.RowValue_BoolValue:return []byte(strconv.FormatBool(value.GetBoolValue()))case *v1pb.RowValue_BytesValue:- return quoteCSVString(string(value.GetBytesValue()))+ hexStr := "0x" + hex.EncodeToString(value.GetBytesValue())+ return quoteCSVString(hexStr)case *v1pb.RowValue_NullValue:return []byte("")case *v1pb.RowValue_TimestampValue:- return quoteCSVString(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000"))+ return quoteCSVString(formatTimestamp(value.GetTimestampValue()))case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- return quoteCSVString(t.In(z).Format(time.RFC3339Nano))+ return quoteCSVString(formatTimestampTz(value.GetTimestampTzValue()))case *v1pb.RowValue_ValueValue:return convertValueValueToBytes(value.GetValueValue())default:diff --git a/backend/component/export/export.go b/backend/component/export/export.goindex 6e1231ec88..1554e27ca2 100644--- a/backend/component/export/export.go+++ b/backend/component/export/export.go@@ -5,10 +5,16 @@ package exportimport ("bytes""io"+ "time"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")+const (+ timestampFormat = "2006-01-02 15:04:05.000000"+ timestampTzFormat = time.RFC3339Nano+)+// Writer is a function type that writes query results to a writer.type Writer func(w io.Writer, result *v1pb.QueryResult) error@@ -20,3 +26,13 @@ func exportToBytes(result *v1pb.QueryResult, writerFunc Writer) ([]byte, error)}return buf.Bytes(), nil}++func formatTimestamp(ts *v1pb.RowValue_Timestamp) string {+ return ts.GoogleTimestamp.AsTime().Format(timestampFormat)+}++func formatTimestampTz(ts *v1pb.RowValue_TimestampTZ) string {+ t := ts.GoogleTimestamp.AsTime()+ z := time.FixedZone(ts.GetZone(), int(ts.GetOffset()))+ return t.In(z).Format(timestampTzFormat)+}diff --git a/backend/component/export/export_test.go b/backend/component/export/export_test.goindex f44ef8638f..4b4c4b29d5 100644--- a/backend/component/export/export_test.go+++ b/backend/component/export/export_test.go@@ -193,7 +193,7 @@ func TestExportSQL(t *testing.T) {},},},- want: "INSERT INTO `<table_name>` (`a`) VALUES (B'101');",+ want: "INSERT INTO `<table_name>` (`a`) VALUES (0x05);",},}a := assert.New(t)@@ -265,7 +265,7 @@ func TestExportJSON(t *testing.T) {},want: `[{- "a": "0b00000101"+ "a": "BQ=="}]`,},diff --git a/backend/component/export/json.go b/backend/component/export/json.goindex d2b35a247a..0928762a8b 100644--- a/backend/component/export/json.go+++ b/backend/component/export/json.go@@ -1,11 +1,9 @@package exportimport (- "bytes"+ "encoding/base64""encoding/json"- "fmt""io"- "time""github.com/pkg/errors"@@ -61,35 +59,16 @@ func convertValueToJSONValue(value *v1pb.RowValue) any {case *v1pb.RowValue_BoolValue:return value.GetBoolValue()case *v1pb.RowValue_BytesValue:- binaryStr, err := convertBytesToBinaryString(value.GetBytesValue())- if err != nil {- return nil- }- return binaryStr+ return base64.StdEncoding.EncodeToString(value.GetBytesValue())case *v1pb.RowValue_NullValue:return nilcase *v1pb.RowValue_TimestampValue:- return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")+ return formatTimestamp(value.GetTimestampValue())case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- return t.In(z).Format(time.RFC3339Nano)+ return formatTimestampTz(value.GetTimestampTzValue())case *v1pb.RowValue_ValueValue:return value.GetValueValue().String()default:return nil}}--func convertBytesToBinaryString(bs []byte) (string, error) {- var buf bytes.Buffer- if _, err := buf.WriteString("0b"); err != nil {- return "", err- }- for _, b := range bs {- if _, err := buf.WriteString(fmt.Sprintf("%08b", b)); err != nil {- return "", err- }- }- return buf.String(), nil-}diff --git a/backend/component/export/sql.go b/backend/component/export/sql.goindex ebd32e9390..5ce02d4eea 100644--- a/backend/component/export/sql.go+++ b/backend/component/export/sql.go@@ -1,11 +1,11 @@package exportimport (+ "encoding/hex""fmt""io""strconv""strings"- "time""github.com/lib/pq""github.com/pkg/errors"@@ -107,12 +107,9 @@ func convertValueToBytesInSQL(engine storepb.Engine, value *v1pb.RowValue) []bytcase *v1pb.RowValue_NullValue:return []byte("NULL")case *v1pb.RowValue_TimestampValue:- return escapeSQLString(engine, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")))+ return escapeSQLString(engine, []byte(formatTimestamp(value.GetTimestampValue())))case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- s := t.In(z).Format(time.RFC3339Nano)- return escapeSQLString(engine, []byte(s))+ return escapeSQLString(engine, []byte(formatTimestampTz(value.GetTimestampTzValue())))case *v1pb.RowValue_ValueValue:return convertValueValueToBytes(value.GetValueValue())default:@@ -136,16 +133,10 @@ func escapeSQLString(engine storepb.Engine, v []byte) []byte {}}-func escapeSQLBytes(engine storepb.Engine, v []byte) []byte {- switch engine {- case storepb.Engine_MYSQL, storepb.Engine_MARIADB:- result := []byte("B'")- s := fmt.Sprintf("%b", v)- s = s[1 : len(s)-1]- result = append(result, []byte(s)...)- result = append(result, '\'')- return result- default:- return escapeSQLString(engine, v)- }+func escapeSQLBytes(_ storepb.Engine, v []byte) []byte {+ // Use hex encoding for all engines (SQL standard)+ result := make([]byte, 2+hex.EncodedLen(len(v)))+ copy(result, "0x")+ hex.Encode(result[2:], v)+ return result}diff --git a/backend/component/export/xlsx.go b/backend/component/export/xlsx.goindex 4edf9d619c..e02506288d 100644--- a/backend/component/export/xlsx.go+++ b/backend/component/export/xlsx.go@@ -5,7 +5,6 @@ import ("fmt""io""strconv"- "time""github.com/pkg/errors""github.com/xuri/excelize/v2"@@ -111,12 +110,9 @@ func convertValueToStringInXLSX(value *v1pb.RowValue) string {case *v1pb.RowValue_NullValue:return ""case *v1pb.RowValue_TimestampValue:- return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")+ return formatTimestamp(value.GetTimestampValue())case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- s := t.In(z).Format(time.RFC3339Nano)- return s+ return formatTimestampTz(value.GetTimestampTzValue())case *v1pb.RowValue_ValueValue:return value.GetValueValue().String()default:diff --git a/backend/tests/sql_export_test.go b/backend/tests/sql_export_test.goindex 4b8f450d72..1761e61617 100644--- a/backend/tests/sql_export_test.go+++ b/backend/tests/sql_export_test.go@@ -61,8 +61,8 @@ func TestSQLExport(t *testing.T) {statement: "SELECT * FROM Test1.tbl;",content: `[{- "gender": "0b00000000",- "height": "0b01111111",+ "gender": "AA==",+ "height": "fw==","id": 1,"name": "Alice"}@@ -79,7 +79,7 @@ func TestSQLExport(t *testing.T) {}{{statement: "SELECT * FROM Test1.tbl;",- content: "id,name,gender,height\n1,\"Alice\",\"\x00\",\"\x7f\"",+ content: "id,name,gender,height\n1,\"Alice\",\"0x00\",\"0x7f\"",},},},