Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: standardize binary data export encoding across formats (#18233)

Vincent Huang4ea3f7ddae83
diff --git a/backend/component/export/csv.go b/backend/component/export/csv.go
index e6240ab91b..6b903e10a8 100644
--- a/backend/component/export/csv.go
+++ b/backend/component/export/csv.go
@@ -1,10 +1,10 @@
package export
import (
+ "encoding/hex"
"io"
"strconv"
"strings"
- "time"
"google.golang.org/protobuf/types/known/structpb"
@@ -67,15 +67,14 @@ func convertValueToBytesInCSV(value *v1pb.RowValue) []byte {
case *v1pb.RowValue_BoolValue:
return []byte(strconv.FormatBool(value.GetBoolValue()))
case *v1pb.RowValue_BytesValue:
- return quoteCSVString(string(value.GetBytesValue()))
+ hexStr := "0x" + hex.EncodeToString(value.GetBytesValue())
+ return quoteCSVString(hexStr)
case *v1pb.RowValue_NullValue:
return []byte("")
case *v1pb.RowValue_TimestampValue:
- return quoteCSVString(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000"))
+ return quoteCSVString(formatTimestamp(value.GetTimestampValue()))
case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- return quoteCSVString(t.In(z).Format(time.RFC3339Nano))
+ return quoteCSVString(formatTimestampTz(value.GetTimestampTzValue()))
case *v1pb.RowValue_ValueValue:
return convertValueValueToBytes(value.GetValueValue())
default:
diff --git a/backend/component/export/export.go b/backend/component/export/export.go
index 6e1231ec88..1554e27ca2 100644
--- a/backend/component/export/export.go
+++ b/backend/component/export/export.go
@@ -5,10 +5,16 @@ package export
import (
"bytes"
"io"
+ "time"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
+const (
+ timestampFormat = "2006-01-02 15:04:05.000000"
+ timestampTzFormat = time.RFC3339Nano
+)
+
// Writer is a function type that writes query results to a writer.
type Writer func(w io.Writer, result *v1pb.QueryResult) error
@@ -20,3 +26,13 @@ func exportToBytes(result *v1pb.QueryResult, writerFunc Writer) ([]byte, error)
}
return buf.Bytes(), nil
}
+
+func formatTimestamp(ts *v1pb.RowValue_Timestamp) string {
+ return ts.GoogleTimestamp.AsTime().Format(timestampFormat)
+}
+
+func formatTimestampTz(ts *v1pb.RowValue_TimestampTZ) string {
+ t := ts.GoogleTimestamp.AsTime()
+ z := time.FixedZone(ts.GetZone(), int(ts.GetOffset()))
+ return t.In(z).Format(timestampTzFormat)
+}
diff --git a/backend/component/export/export_test.go b/backend/component/export/export_test.go
index f44ef8638f..4b4c4b29d5 100644
--- a/backend/component/export/export_test.go
+++ b/backend/component/export/export_test.go
@@ -193,7 +193,7 @@ func TestExportSQL(t *testing.T) {
},
},
},
- want: "INSERT INTO `<table_name>` (`a`) VALUES (B'101');",
+ want: "INSERT INTO `<table_name>` (`a`) VALUES (0x05);",
},
}
a := assert.New(t)
@@ -265,7 +265,7 @@ func TestExportJSON(t *testing.T) {
},
want: `[
{
- "a": "0b00000101"
+ "a": "BQ=="
}
]`,
},
diff --git a/backend/component/export/json.go b/backend/component/export/json.go
index d2b35a247a..0928762a8b 100644
--- a/backend/component/export/json.go
+++ b/backend/component/export/json.go
@@ -1,11 +1,9 @@
package export
import (
- "bytes"
+ "encoding/base64"
"encoding/json"
- "fmt"
"io"
- "time"
"github.com/pkg/errors"
@@ -61,35 +59,16 @@ func convertValueToJSONValue(value *v1pb.RowValue) any {
case *v1pb.RowValue_BoolValue:
return value.GetBoolValue()
case *v1pb.RowValue_BytesValue:
- binaryStr, err := convertBytesToBinaryString(value.GetBytesValue())
- if err != nil {
- return nil
- }
- return binaryStr
+ return base64.StdEncoding.EncodeToString(value.GetBytesValue())
case *v1pb.RowValue_NullValue:
return nil
case *v1pb.RowValue_TimestampValue:
- return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")
+ return formatTimestamp(value.GetTimestampValue())
case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- return t.In(z).Format(time.RFC3339Nano)
+ return formatTimestampTz(value.GetTimestampTzValue())
case *v1pb.RowValue_ValueValue:
return value.GetValueValue().String()
default:
return nil
}
}
-
-func convertBytesToBinaryString(bs []byte) (string, error) {
- var buf bytes.Buffer
- if _, err := buf.WriteString("0b"); err != nil {
- return "", err
- }
- for _, b := range bs {
- if _, err := buf.WriteString(fmt.Sprintf("%08b", b)); err != nil {
- return "", err
- }
- }
- return buf.String(), nil
-}
diff --git a/backend/component/export/sql.go b/backend/component/export/sql.go
index ebd32e9390..5ce02d4eea 100644
--- a/backend/component/export/sql.go
+++ b/backend/component/export/sql.go
@@ -1,11 +1,11 @@
package export
import (
+ "encoding/hex"
"fmt"
"io"
"strconv"
"strings"
- "time"
"github.com/lib/pq"
"github.com/pkg/errors"
@@ -107,12 +107,9 @@ func convertValueToBytesInSQL(engine storepb.Engine, value *v1pb.RowValue) []byt
case *v1pb.RowValue_NullValue:
return []byte("NULL")
case *v1pb.RowValue_TimestampValue:
- return escapeSQLString(engine, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")))
+ return escapeSQLString(engine, []byte(formatTimestamp(value.GetTimestampValue())))
case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- s := t.In(z).Format(time.RFC3339Nano)
- return escapeSQLString(engine, []byte(s))
+ return escapeSQLString(engine, []byte(formatTimestampTz(value.GetTimestampTzValue())))
case *v1pb.RowValue_ValueValue:
return convertValueValueToBytes(value.GetValueValue())
default:
@@ -136,16 +133,10 @@ func escapeSQLString(engine storepb.Engine, v []byte) []byte {
}
}
-func escapeSQLBytes(engine storepb.Engine, v []byte) []byte {
- switch engine {
- case storepb.Engine_MYSQL, storepb.Engine_MARIADB:
- result := []byte("B'")
- s := fmt.Sprintf("%b", v)
- s = s[1 : len(s)-1]
- result = append(result, []byte(s)...)
- result = append(result, '\'')
- return result
- default:
- return escapeSQLString(engine, v)
- }
+func escapeSQLBytes(_ storepb.Engine, v []byte) []byte {
+ // Use hex encoding for all engines (SQL standard)
+ result := make([]byte, 2+hex.EncodedLen(len(v)))
+ copy(result, "0x")
+ hex.Encode(result[2:], v)
+ return result
}
diff --git a/backend/component/export/xlsx.go b/backend/component/export/xlsx.go
index 4edf9d619c..e02506288d 100644
--- a/backend/component/export/xlsx.go
+++ b/backend/component/export/xlsx.go
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"strconv"
- "time"
"github.com/pkg/errors"
"github.com/xuri/excelize/v2"
@@ -111,12 +110,9 @@ func convertValueToStringInXLSX(value *v1pb.RowValue) string {
case *v1pb.RowValue_NullValue:
return ""
case *v1pb.RowValue_TimestampValue:
- return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")
+ return formatTimestamp(value.GetTimestampValue())
case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- s := t.In(z).Format(time.RFC3339Nano)
- return s
+ return formatTimestampTz(value.GetTimestampTzValue())
case *v1pb.RowValue_ValueValue:
return value.GetValueValue().String()
default:
diff --git a/backend/tests/sql_export_test.go b/backend/tests/sql_export_test.go
index 4b8f450d72..1761e61617 100644
--- a/backend/tests/sql_export_test.go
+++ b/backend/tests/sql_export_test.go
@@ -61,8 +61,8 @@ func TestSQLExport(t *testing.T) {
statement: "SELECT * FROM Test1.tbl;",
content: `[
{
- "gender": "0b00000000",
- "height": "0b01111111",
+ "gender": "AA==",
+ "height": "fw==",
"id": 1,
"name": "Alice"
}
@@ -79,7 +79,7 @@ func TestSQLExport(t *testing.T) {
}{
{
statement: "SELECT * FROM Test1.tbl;",
- content: "id,name,gender,height\n1,\"Alice\",\"\x00\",\"\x7f\"",
+ content: "id,name,gender,height\n1,\"Alice\",\"0x00\",\"0x7f\"",
},
},
},