Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor: move export utilities to dedicated package (#17952)

Danny Xu426f90a17270
diff --git a/backend/api/v1/audit_log_service.go b/backend/api/v1/audit_log_service.go
index 97bc7a3874..a8dcd70e59 100644
--- a/backend/api/v1/audit_log_service.go
+++ b/backend/api/v1/audit_log_service.go
@@ -11,6 +11,7 @@ import (
"github.com/bytebase/bytebase/backend/common"
"github.com/bytebase/bytebase/backend/common/qb"
+ "github.com/bytebase/bytebase/backend/component/export"
"github.com/bytebase/bytebase/backend/component/iam"
"github.com/bytebase/bytebase/backend/enterprise"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -140,15 +141,15 @@ func (s *AuditLogService) ExportAuditLogs(ctx context.Context, request *connect.
var content []byte
switch request.Msg.Format {
case v1pb.ExportFormat_CSV:
- if content, err = exportCSV(result); err != nil {
+ if content, err = export.CSV(result); err != nil {
return nil, err
}
case v1pb.ExportFormat_JSON:
- if content, err = exportJSON(result); err != nil {
+ if content, err = export.JSON(result); err != nil {
return nil, err
}
case v1pb.ExportFormat_XLSX:
- if content, err = exportXLSX(result); err != nil {
+ if content, err = export.XLSX(result); err != nil {
return nil, err
}
default:
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index dac2d45fe4..a65cd4d62d 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -22,6 +22,7 @@ import (
"github.com/bytebase/bytebase/backend/common/log"
"github.com/bytebase/bytebase/backend/component/config"
"github.com/bytebase/bytebase/backend/component/dbfactory"
+ "github.com/bytebase/bytebase/backend/component/export"
"github.com/bytebase/bytebase/backend/component/iam"
"github.com/bytebase/bytebase/backend/component/sheet"
"github.com/bytebase/bytebase/backend/enterprise"
@@ -945,127 +946,175 @@ func DoExport(
}
}
- pendingEncryptContents := []*encryptContent{}
+ var buf bytes.Buffer
+ zipw := zip.NewWriter(&buf)
+
+ exportCount := 0
for i, result := range results {
if result.GetError() != "" {
- slog.Error("failed to query result",
- slog.String("instance", database.InstanceID),
- slog.String("database", database.DatabaseName),
- slog.String("project", database.ProjectID),
- slog.String("query_error", result.GetError()),
- )
+ logExportError(database, "failed to query result", errors.New(result.GetError()))
continue
}
- content, err := formatExport(ctx, stores, instance, database, result, request)
- if err != nil {
- slog.Error("failed to format export",
- log.BBError(err),
- slog.String("instance", database.InstanceID),
- slog.String("database", database.DatabaseName),
- slog.String("project", database.ProjectID),
- )
+ if err := exportResultToZip(ctx, zipw, stores, instance, database, result, request, i+1); err != nil {
+ logExportError(database, "failed to export result to zip", err)
continue
}
- filename := fmt.Sprintf("%s/%s/statement-%d", database.InstanceID, database.DatabaseName, i+1)
- pendingEncryptContents = append(
- pendingEncryptContents,
- &encryptContent{
- Name: fmt.Sprintf("%s.sql", filename),
- Content: []byte(result.Statement),
- },
- &encryptContent{
- Name: fmt.Sprintf("%s.result.%s", filename, strings.ToLower(request.Format.String())),
- Content: content,
- },
- )
+
+ exportCount++
+ // Help GC by clearing the result data we've already processed
+ result.Rows = nil
}
- if len(pendingEncryptContents) == 0 {
+ if exportCount == 0 {
return nil, duration, errors.Errorf("empty export data for database %s", database.DatabaseName)
}
- // we will save both query result and statement into the result, so we always zip files.
- encryptedBytes, err := doEncrypt(pendingEncryptContents, request.GetPassword())
- if err != nil {
- return nil, duration, errors.Wrap(err, "failed to encrypt data")
+ if err := zipw.Close(); err != nil {
+ return nil, duration, errors.Wrap(err, "failed to close zip writer")
}
- return encryptedBytes, duration, nil
+
+ return buf.Bytes(), duration, nil
}
-type encryptContent struct {
- Name string
- Content []byte
+// logExportError logs export-related errors with consistent database context.
+func logExportError(database *store.DatabaseMessage, message string, err error) {
+ slog.Error(message,
+ log.BBError(err),
+ slog.String("instance", database.InstanceID),
+ slog.String("database", database.DatabaseName),
+ slog.String("project", database.ProjectID),
+ )
}
-func doEncrypt(exports []*encryptContent, password string) ([]byte, error) {
- var b bytes.Buffer
- fzip := io.Writer(&b)
+// exportResultToZip exports a single query result to the ZIP archive.
+// It writes both the SQL statement and the formatted result data.
+func exportResultToZip(
+ ctx context.Context,
+ zipw *zip.Writer,
+ stores *store.Store,
+ instance *store.InstanceMessage,
+ database *store.DatabaseMessage,
+ result *v1pb.QueryResult,
+ request *v1pb.ExportRequest,
+ statementNumber int,
+) error {
+ baseFilename := fmt.Sprintf("%s/%s/statement-%d", database.InstanceID, database.DatabaseName, statementNumber)
- zipw := zip.NewWriter(fzip)
- defer zipw.Close()
+ // Write statement file
+ statementFilename := fmt.Sprintf("%s.sql", baseFilename)
+ if err := export.WriteZipEntry(zipw, statementFilename, []byte(result.Statement), request.GetPassword()); err != nil {
+ return errors.Wrap(err, "failed to write statement")
+ }
- for _, export := range exports {
- fh := &zip.FileHeader{
- Name: export.Name,
- Method: zip.Deflate,
- }
- fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(time.Now())
- if password != "" {
- fh.SetPassword(password)
- }
- writer, err := zipw.CreateHeader(fh)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to create encrypt export file")
- }
- if _, err := io.Copy(writer, bytes.NewReader(export.Content)); err != nil {
- return nil, errors.Wrapf(err, "failed to write export file")
- }
+ // Write result file by streaming directly to ZIP
+ resultExt := strings.ToLower(request.Format.String())
+ resultFilename := fmt.Sprintf("%s.result.%s", baseFilename, resultExt)
+ if err := formatExportToZip(ctx, zipw, resultFilename, stores, instance, database, result, request); err != nil {
+ return errors.Wrap(err, "failed to write formatted result")
}
- if err := zipw.Close(); err != nil {
- return nil, errors.Wrap(err, "failed to close zip writer")
+ return nil
+}
+
+// formatExportToZip formats query results and writes them directly to a ZIP entry.
+// This function streams the formatted data to minimize memory usage.
+func formatExportToZip(
+ ctx context.Context,
+ zipw *zip.Writer,
+ filename string,
+ stores *store.Store,
+ instance *store.InstanceMessage,
+ database *store.DatabaseMessage,
+ result *v1pb.QueryResult,
+ request *v1pb.ExportRequest,
+) error {
+ writer, err := export.CreateZipWriter(zipw, filename, request.GetPassword())
+ if err != nil {
+ return err
}
- return b.Bytes(), nil
+ return writeFormattedResult(ctx, writer, stores, instance, database, result, request)
}
-func formatExport(
+// writeFormattedResult writes the query result in the requested format to the writer.
+func writeFormattedResult(
ctx context.Context,
+ w io.Writer,
stores *store.Store,
instance *store.InstanceMessage,
database *store.DatabaseMessage,
result *v1pb.QueryResult,
request *v1pb.ExportRequest,
-) ([]byte, error) {
+) error {
switch request.Format {
case v1pb.ExportFormat_CSV:
- return exportCSV(result)
+ return export.CSVToWriter(w, result)
case v1pb.ExportFormat_JSON:
- return exportJSON(result)
+ return export.JSONToWriter(w, result)
case v1pb.ExportFormat_SQL:
- resourceList, err := getResources(ctx, stores, instance.Metadata.GetEngine(), database.DatabaseName, request.Statement, instance)
- if err != nil {
- return nil, errors.Errorf("failed to extract resource list: %v", err)
- }
- statementPrefix, err := getSQLStatementPrefix(instance.Metadata.GetEngine(), resourceList, result.ColumnNames)
- if err != nil {
- return nil, err
- }
- return exportSQL(instance.Metadata.GetEngine(), statementPrefix, result)
+ return exportSQLWithContext(ctx, w, stores, instance, database, result, request)
case v1pb.ExportFormat_XLSX:
- return exportXLSX(result)
+ return export.XLSXToWriter(w, result)
default:
- return nil, errors.Errorf("unsupported export format: %s", request.Format.String())
+ return errors.Errorf("unsupported export format: %s", request.Format.String())
}
}
-// timeToMsDosTime converts a time.Time to an MS-DOS date and time.
-// this is a modified copy for gihub.com/alexmullins/zip/struct.go cause the package has a bug, it will convert the time to UTC time and drop the timezone.
-func timeToMsDosTime(t time.Time) (uint16, uint16) {
- fDate := uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
- fTime := uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
- return fDate, fTime
+// exportSQLWithContext exports SQL INSERT statements with proper context.
+func exportSQLWithContext(
+ ctx context.Context,
+ w io.Writer,
+ stores *store.Store,
+ instance *store.InstanceMessage,
+ database *store.DatabaseMessage,
+ result *v1pb.QueryResult,
+ request *v1pb.ExportRequest,
+) error {
+ resourceList, err := export.GetResources(
+ ctx,
+ stores,
+ instance.Metadata.GetEngine(),
+ database.DatabaseName,
+ request.Statement,
+ instance,
+ BuildGetDatabaseMetadataFunc(stores),
+ BuildListDatabaseNamesFunc(stores),
+ BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),
+ )
+ if err != nil {
+ return errors.Wrapf(err, "failed to extract resource list")
+ }
+ statementPrefix, err := export.SQLStatementPrefix(instance.Metadata.GetEngine(), resourceList, result.ColumnNames)
+ if err != nil {
+ return err
+ }
+ return export.SQLToWriter(w, instance.Metadata.GetEngine(), statementPrefix, result)
+}
+
+type encryptContent struct {
+ Name string
+ Content []byte
+}
+
+func doEncrypt(exports []*encryptContent, password string) ([]byte, error) {
+ var b bytes.Buffer
+ fzip := io.Writer(&b)
+
+ zipw := zip.NewWriter(fzip)
+ defer zipw.Close()
+
+ for _, exportContent := range exports {
+ if err := export.WriteZipEntry(zipw, exportContent.Name, exportContent.Content, password); err != nil {
+ return nil, err
+ }
+ }
+
+ if err := zipw.Close(); err != nil {
+ return nil, errors.Wrap(err, "failed to close zip writer")
+ }
+
+ return b.Bytes(), nil
}
func (s *SQLService) createQueryHistory(database *store.DatabaseMessage, queryType store.QueryHistoryType, statement string, userUID int, duration time.Duration, queryErr error) {
diff --git a/backend/api/v1/sql_service_export_utils.go b/backend/api/v1/sql_service_export_utils.go
deleted file mode 100644
index 5fb40a07fb..0000000000
--- a/backend/api/v1/sql_service_export_utils.go
+++ /dev/null
@@ -1,941 +0,0 @@
-package v1
-
-import (
- "bytes"
- "context"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "strconv"
- "strings"
- "time"
-
- "connectrpc.com/connect"
- "github.com/labstack/echo/v4"
- "github.com/lib/pq"
- "github.com/pkg/errors"
- "github.com/xuri/excelize/v2"
- "google.golang.org/protobuf/types/known/structpb"
-
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
- "github.com/bytebase/bytebase/backend/plugin/parser/base"
- "github.com/bytebase/bytebase/backend/store"
- "github.com/bytebase/bytebase/backend/store/model"
- "github.com/bytebase/bytebase/backend/utils"
-)
-
-func exportCSV(result *v1pb.QueryResult) ([]byte, error) {
- var buf bytes.Buffer
- if _, err := buf.WriteString(strings.Join(result.ColumnNames, ",")); err != nil {
- return nil, err
- }
- if err := buf.WriteByte('\n'); err != nil {
- return nil, err
- }
- for i, row := range result.Rows {
- for i, value := range row.Values {
- if i != 0 {
- if err := buf.WriteByte(','); err != nil {
- return nil, err
- }
- }
- if _, err := buf.Write(convertValueToBytesInCSV(value)); err != nil {
- return nil, err
- }
- }
- if i != len(result.Rows)-1 {
- if err := buf.WriteByte('\n'); err != nil {
- return nil, err
- }
- }
- }
- return buf.Bytes(), nil
-}
-
-func convertValueToBytesInCSV(value *v1pb.RowValue) []byte {
- if value == nil || value.Kind == nil {
- return []byte("")
- }
- switch value.Kind.(type) {
- case *v1pb.RowValue_StringValue:
- var result []byte
- result = append(result, '"')
- result = append(result, []byte(escapeCSVString(value.GetStringValue()))...)
- result = append(result, '"')
- return result
- case *v1pb.RowValue_Int32Value:
- return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))
- case *v1pb.RowValue_Int64Value:
- return []byte(strconv.FormatInt(value.GetInt64Value(), 10))
- case *v1pb.RowValue_Uint32Value:
- return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))
- case *v1pb.RowValue_Uint64Value:
- return []byte(strconv.FormatUint(value.GetUint64Value(), 10))
- case *v1pb.RowValue_FloatValue:
- return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))
- case *v1pb.RowValue_DoubleValue:
- return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))
- case *v1pb.RowValue_BoolValue:
- return []byte(strconv.FormatBool(value.GetBoolValue()))
- case *v1pb.RowValue_BytesValue:
- var result []byte
- result = append(result, '"')
- result = append(result, []byte(escapeCSVString(string(value.GetBytesValue())))...)
- result = append(result, '"')
- return result
- case *v1pb.RowValue_NullValue:
- return []byte("")
- case *v1pb.RowValue_TimestampValue:
- var result []byte
- result = append(result, '"')
- result = append(result, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000"))...)
- result = append(result, '"')
- return result
- case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- s := t.In(z).Format(time.RFC3339Nano)
- var result []byte
- result = append(result, '"')
- result = append(result, []byte(s)...)
- result = append(result, '"')
- return result
- case *v1pb.RowValue_ValueValue:
- // This is used by ClickHouse and Spanner only.
- return convertValueValueToBytes(value.GetValueValue())
- default:
- return []byte("")
- }
-}
-
-func escapeCSVString(str string) string {
- escapedStr := strings.ReplaceAll(str, `"`, `""`)
- return escapedStr
-}
-
-func getSQLStatementPrefix(engine storepb.Engine, resourceList []base.SchemaResource, columnNames []string) (string, error) {
- var escapeQuote string
- switch engine {
- case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_TIDB, storepb.Engine_OCEANBASE, storepb.Engine_SPANNER:
- escapeQuote = "`"
- case storepb.Engine_CLICKHOUSE, storepb.Engine_MSSQL, storepb.Engine_ORACLE, storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT, storepb.Engine_SQLITE, storepb.Engine_SNOWFLAKE:
- // ClickHouse takes both double-quotes or backticks.
- escapeQuote = "\""
- default:
- // storepb.Engine_MONGODB, storepb.Engine_REDIS
- return "", errors.Errorf("unsupported engine %v for exporting as SQL", engine)
- }
-
- s := "INSERT INTO "
- if len(resourceList) == 1 {
- resource := resourceList[0]
- if resource.Schema != "" {
- s = fmt.Sprintf("%s%s%s%s%s", s, escapeQuote, resource.Schema, escapeQuote, ".")
- }
- s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, resource.Table, escapeQuote)
- } else {
- s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, "<table_name>", escapeQuote)
- }
- var columnTokens []string
- for _, columnName := range columnNames {
- columnTokens = append(columnTokens, fmt.Sprintf("%s%s%s", escapeQuote, columnName, escapeQuote))
- }
- s = fmt.Sprintf("%s (%s) VALUES (", s, strings.Join(columnTokens, ","))
- return s, nil
-}
-
-func exportSQL(engine storepb.Engine, statementPrefix string, result *v1pb.QueryResult) ([]byte, error) {
- var buf bytes.Buffer
- for i, row := range result.Rows {
- if _, err := buf.WriteString(statementPrefix); err != nil {
- return nil, err
- }
- for i, value := range row.Values {
- if i != 0 {
- if err := buf.WriteByte(','); err != nil {
- return nil, err
- }
- }
- if _, err := buf.Write(convertValueToBytesInSQL(engine, value)); err != nil {
- return nil, err
- }
- }
- if i != len(result.Rows)-1 {
- if _, err := buf.WriteString(");\n"); err != nil {
- return nil, err
- }
- } else {
- if _, err := buf.WriteString(");"); err != nil {
- return nil, err
- }
- }
- }
- return buf.Bytes(), nil
-}
-
-func convertValueToBytesInSQL(engine storepb.Engine, value *v1pb.RowValue) []byte {
- if value == nil || value.Kind == nil {
- return []byte("")
- }
- switch value.Kind.(type) {
- case *v1pb.RowValue_StringValue:
- return escapeSQLString(engine, []byte(value.GetStringValue()))
- case *v1pb.RowValue_Int32Value:
- return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))
- case *v1pb.RowValue_Int64Value:
- return []byte(strconv.FormatInt(value.GetInt64Value(), 10))
- case *v1pb.RowValue_Uint32Value:
- return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))
- case *v1pb.RowValue_Uint64Value:
- return []byte(strconv.FormatUint(value.GetUint64Value(), 10))
- case *v1pb.RowValue_FloatValue:
- return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))
- case *v1pb.RowValue_DoubleValue:
- return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))
- case *v1pb.RowValue_BoolValue:
- return []byte(strconv.FormatBool(value.GetBoolValue()))
- case *v1pb.RowValue_BytesValue:
- return escapeSQLBytes(engine, value.GetBytesValue())
- case *v1pb.RowValue_NullValue:
- return []byte("NULL")
- case *v1pb.RowValue_TimestampValue:
- return escapeSQLString(engine, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")))
- case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- s := t.In(z).Format(time.RFC3339Nano)
- return escapeSQLString(engine, []byte(s))
- case *v1pb.RowValue_ValueValue:
- // This is used by ClickHouse and Spanner only.
- return convertValueValueToBytes(value.GetValueValue())
- default:
- return []byte("")
- }
-}
-
-func escapeSQLString(engine storepb.Engine, v []byte) []byte {
- switch engine {
- case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:
- escapedStr := pq.QuoteLiteral(string(v))
- return []byte(escapedStr)
- default:
- result := []byte("'")
- s := strconv.Quote(string(v))
- s = s[1 : len(s)-1]
- s = strings.ReplaceAll(s, `'`, `''`)
- result = append(result, []byte(s)...)
- result = append(result, '\'')
- return result
- }
-}
-
-func escapeSQLBytes(engine storepb.Engine, v []byte) []byte {
- switch engine {
- case storepb.Engine_MYSQL, storepb.Engine_MARIADB:
- result := []byte("B'")
- s := fmt.Sprintf("%b", v)
- s = s[1 : len(s)-1]
- result = append(result, []byte(s)...)
- result = append(result, '\'')
- return result
- default:
- return escapeSQLString(engine, v)
- }
-}
-
-func convertValueValueToBytes(value *structpb.Value) []byte {
- if value == nil || value.Kind == nil {
- return []byte("")
- }
- switch value.Kind.(type) {
- case *structpb.Value_NullValue:
- return []byte("")
- case *structpb.Value_StringValue:
- var result []byte
- result = append(result, '"')
- result = append(result, []byte(value.GetStringValue())...)
- result = append(result, '"')
- return result
- case *structpb.Value_NumberValue:
- return []byte(strconv.FormatFloat(value.GetNumberValue(), 'f', -1, 64))
- case *structpb.Value_BoolValue:
- return []byte(strconv.FormatBool(value.GetBoolValue()))
- case *structpb.Value_ListValue:
- var buf [][]byte
- for _, v := range value.GetListValue().Values {
- buf = append(buf, convertValueValueToBytes(v))
- }
- var result []byte
- result = append(result, '"')
- result = append(result, '[')
- result = append(result, bytes.Join(buf, []byte(","))...)
- result = append(result, ']')
- result = append(result, '"')
- return result
- case *structpb.Value_StructValue:
- first := true
- var buf []byte
- buf = append(buf, '"')
- for k, v := range value.GetStructValue().Fields {
- if first {
- first = false
- } else {
- buf = append(buf, ',')
- }
- buf = append(buf, []byte(k)...)
- buf = append(buf, ':')
- buf = append(buf, convertValueValueToBytes(v)...)
- }
- buf = append(buf, '"')
- return buf
- default:
- return []byte("")
- }
-}
-
-// getResources extracts the resource list from the statement for exporting results as SQL.
-func getResources(ctx context.Context, storeInstance *store.Store, engine storepb.Engine, databaseName string, statement string, instance *store.InstanceMessage) ([]base.SchemaResource, error) {
- switch engine {
- case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_OCEANBASE:
- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),
- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),
- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),
- }, engine, statement, databaseName, "", !store.IsObjectCaseSensitive(instance))
- if err != nil {
- return nil, err
- } else if databaseName == "" {
- var list []base.SchemaResource
- for _, span := range spans {
- for sourceColumn := range span.SourceColumns {
- list = append(list, base.SchemaResource{
- Database: sourceColumn.Database,
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- LinkedServer: sourceColumn.Server,
- })
- }
- }
- return list, nil
- }
-
- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &databaseName,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {
- // If database not found, skip.
- return nil, nil
- }
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
- }
- if database == nil {
- return nil, nil
- }
-
- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
- }
-
- var result []base.SchemaResource
- for _, span := range spans {
- for sourceColumn := range span.SourceColumns {
- sr := base.SchemaResource{
- Database: sourceColumn.Database,
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- LinkedServer: sourceColumn.Server,
- }
- if sourceColumn.Database != dbSchema.GetMetadata().Name {
- // MySQL allows cross-database query, we should check the corresponding database.
- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &sourceColumn.Database,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sourceColumn.Database, instance.ResourceID))
- }
- if resourceDB == nil {
- continue
- }
- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: resourceDB.InstanceID,
- DatabaseName: resourceDB.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sourceColumn.Database, instance.ResourceID))
- }
- if !resourceExists(resourceDBSchema, sr) {
- // If table not found, we regard it as a CTE/alias/... and skip.
- continue
- }
- result = append(result, sr)
- continue
- }
- if !resourceExists(dbSchema, sr) {
- // If table not found, skip.
- continue
- }
- result = append(result, sr)
- }
- }
- return result, nil
- case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:
- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),
- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),
- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),
- }, engine, statement, databaseName, "public", !store.IsObjectCaseSensitive(instance))
- if err != nil {
- return nil, err
- }
-
- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &databaseName,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
- }
- if database == nil {
- return nil, nil
- }
-
- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
- }
-
- var result []base.SchemaResource
- for _, span := range spans {
- for sourceColumn := range span.SourceColumns {
- sr := base.SchemaResource{
- Database: sourceColumn.Database,
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- LinkedServer: sourceColumn.Server,
- }
-
- if sourceColumn.Database != dbSchema.GetMetadata().Name {
- // Should not happen.
- continue
- }
-
- if !resourceExists(dbSchema, sr) {
- // If table not found, skip.
- continue
- }
-
- result = append(result, sr)
- }
- }
-
- return result, nil
- case storepb.Engine_ORACLE:
- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),
- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),
- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),
- }, engine, statement, databaseName, databaseName, !store.IsObjectCaseSensitive(instance))
- if err != nil {
- return nil, err
- }
-
- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &databaseName,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
- }
- if database == nil {
- return nil, nil
- }
-
- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
- }
-
- var result []base.SchemaResource
- for _, span := range spans {
- for sourceColumn := range span.SourceColumns {
- sr := base.SchemaResource{
- Database: sourceColumn.Database,
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- LinkedServer: sourceColumn.Server,
- }
- if sr.Database != dbSchema.GetMetadata().Name {
- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &sr.Database,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sr.Database, instance.ResourceID))
- }
- if resourceDB == nil {
- continue
- }
- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: resourceDB.InstanceID,
- DatabaseName: resourceDB.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sr.Database, instance.ResourceID))
- }
- if !resourceExists(resourceDBSchema, sr) {
- // If table not found, we regard it as a CTE/alias/... and skip.
- continue
- }
- result = append(result, sr)
- continue
- }
-
- if !resourceExists(dbSchema, sr) {
- // If table not found, skip.
- continue
- }
-
- result = append(result, sr)
- }
- }
-
- return result, nil
- case storepb.Engine_SNOWFLAKE:
- dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)
- adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)
- // If there are no read-only data source, fall back to admin data source.
- if dataSource == nil {
- dataSource = adminDataSource
- }
- if dataSource == nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))
- }
- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),
- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),
- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),
- }, engine, statement, databaseName, "PUBLIC", !store.IsObjectCaseSensitive(instance))
- if err != nil {
- return nil, err
- }
- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &databaseName,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {
- // If database not found, skip.
- return nil, nil
- }
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
- }
- if database == nil {
- return nil, nil
- }
-
- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
- }
-
- var result []base.SchemaResource
- for _, span := range spans {
- for sourceColumn := range span.SourceColumns {
- sr := base.SchemaResource{
- Database: sourceColumn.Database,
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- LinkedServer: sourceColumn.Server,
- }
- if sr.Database != dbSchema.GetMetadata().Name {
- // Snowflake allows cross-database query, we should check the corresponding database.
- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &sr.Database,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sr.Database, instance.ResourceID))
- }
- if resourceDB == nil {
- continue
- }
- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: resourceDB.InstanceID,
- DatabaseName: resourceDB.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sr.Database, instance.ResourceID))
- }
- if !resourceExists(resourceDBSchema, sr) {
- // If table not found, we regard it as a CTE/alias/... and skip.
- continue
- }
- result = append(result, sr)
- continue
- }
- if !resourceExists(dbSchema, sr) {
- // If table not found, skip.
- continue
- }
- result = append(result, sr)
- }
- }
-
- return result, nil
- case storepb.Engine_MSSQL:
- dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)
- adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)
- // If there are no read-only data source, fall back to admin data source.
- if dataSource == nil {
- dataSource = adminDataSource
- }
- if dataSource == nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))
- }
- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
- InstanceID: instance.ResourceID,
- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),
- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),
- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),
- }, engine, statement, databaseName, "dbo", !store.IsObjectCaseSensitive(instance))
- if err != nil {
- return nil, err
- }
- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &databaseName,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {
- // If database not found, skip.
- return nil, nil
- }
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
- }
- if database == nil {
- return nil, nil
- }
-
- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: database.InstanceID,
- DatabaseName: database.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
- }
-
- var result []base.SchemaResource
- for _, span := range spans {
- for sourceColumn := range span.SourceColumns {
- sr := base.SchemaResource{
- Database: sourceColumn.Database,
- Schema: sourceColumn.Schema,
- Table: sourceColumn.Table,
- LinkedServer: sourceColumn.Server,
- }
-
- if sr.LinkedServer != "" {
- continue
- }
- if sr.Database != dbSchema.GetMetadata().Name {
- // MSSQL allows cross-database query, we should check the corresponding database.
- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
- InstanceID: &instance.ResourceID,
- DatabaseName: &sr.Database,
- IsCaseSensitive: store.IsObjectCaseSensitive(instance),
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sr.Database, instance.ResourceID))
- }
- if resourceDB == nil {
- continue
- }
- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
- InstanceID: resourceDB.InstanceID,
- DatabaseName: resourceDB.DatabaseName,
- })
- if err != nil {
- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sr.Database, instance.ResourceID))
- }
- if !resourceExists(resourceDBSchema, sr) {
- // If table not found, we regard it as a CTE/alias/... and skip.
- continue
- }
- result = append(result, sr)
- continue
- }
- if !resourceExists(dbSchema, sr) {
- // If table not found, skip.
- continue
- }
- result = append(result, sr)
- }
- }
-
- return result, nil
- default:
- if databaseName == "" {
- return nil, errors.Errorf("database must be specified")
- }
- return []base.SchemaResource{{Database: databaseName}}, nil
- }
-}
-
-func exportJSON(result *v1pb.QueryResult) ([]byte, error) {
- var buf bytes.Buffer
- if _, err := buf.WriteString("["); err != nil {
- return nil, err
- }
-
- for rowIndex, row := range result.Rows {
- if _, err := buf.WriteString("{"); err != nil {
- return nil, err
- }
- for i, value := range row.Values {
- if _, err := buf.WriteString(fmt.Sprintf(`"%s":`, result.ColumnNames[i])); err != nil {
- return nil, err
- }
- if _, err := buf.WriteString(convertValueToStringInJSON(value)); err != nil {
- return nil, err
- }
- if i != len(row.Values)-1 {
- if _, err := buf.WriteString(","); err != nil {
- return nil, err
- }
- }
- }
- if _, err := buf.WriteString("}"); err != nil {
- return nil, err
- }
-
- if rowIndex != len(result.Rows)-1 {
- if _, err := buf.WriteString(","); err != nil {
- return nil, err
- }
- }
- }
-
- if _, err := buf.WriteString("]"); err != nil {
- return nil, err
- }
-
- var data any
- err := json.Unmarshal(buf.Bytes(), &data)
- if err != nil {
- return nil, errors.Errorf("failed to unmarshalling JSON with error: %v", err)
- }
-
- // Marshal the data back into a pretty-printed JSON string
- // The second argument "" is the prefix for each line, and " " is the indentation string.
- formattedJSONBytes, err := json.MarshalIndent(data, "", " ")
- if err != nil {
- return nil, errors.Errorf("failed to marshalling JSON with error: %v", err)
- }
- return formattedJSONBytes, nil
-}
-
-func convertValueToStringInJSON(value *v1pb.RowValue) string {
- if value == nil || value.Kind == nil {
- return ""
- }
-
- switch value.Kind.(type) {
- case *v1pb.RowValue_StringValue:
- return `"` + escapeJSONString(value.GetStringValue()) + `"`
- case *v1pb.RowValue_Int32Value:
- return strconv.FormatInt(int64(value.GetInt32Value()), 10)
- case *v1pb.RowValue_Int64Value:
- return strconv.FormatInt(value.GetInt64Value(), 10)
- case *v1pb.RowValue_Uint32Value:
- return strconv.FormatUint(uint64(value.GetUint32Value()), 10)
- case *v1pb.RowValue_Uint64Value:
- return strconv.FormatUint(value.GetUint64Value(), 10)
- case *v1pb.RowValue_FloatValue:
- return strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32)
- case *v1pb.RowValue_DoubleValue:
- return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)
- case *v1pb.RowValue_BoolValue:
- return strconv.FormatBool(value.GetBoolValue())
- case *v1pb.RowValue_BytesValue:
- value, err := convertBytesToBinaryString(value.GetBytesValue())
- if err != nil {
- return ""
- }
- return `"` + value + `"`
- case *v1pb.RowValue_NullValue:
- return "null"
- case *v1pb.RowValue_TimestampValue:
- return `"` + value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000") + `"`
- case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- s := t.In(z).Format(time.RFC3339Nano)
- return `"` + s + `"`
- case *v1pb.RowValue_ValueValue:
- // This is used by ClickHouse and Spanner only.
- return value.GetValueValue().String()
- default:
- return ""
- }
-}
-
-func convertBytesToBinaryString(bs []byte) (string, error) {
- var buf bytes.Buffer
- if _, err := buf.WriteString("0b"); err != nil {
- return "", err
- }
- for _, b := range bs {
- if _, err := buf.WriteString(fmt.Sprintf("%08b", b)); err != nil {
- return "", err
- }
- }
- return buf.String(), nil
-}
-
-func escapeJSONString(str string) string {
- s := strconv.Quote(str)
- return s[1 : len(s)-1]
-}
-
-const (
- excelLetters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
- sheet1Name = "Sheet1"
- excelMaxColumn = 18278
-)
-
-func exportXLSX(result *v1pb.QueryResult) ([]byte, error) {
- f := excelize.NewFile()
- defer f.Close()
- index, err := f.NewSheet("Sheet1")
- if err != nil {
- return nil, err
- }
- var columnPrefixes []string
- for i, columnName := range result.ColumnNames {
- columnPrefix, err := getExcelColumnName(i)
- if err != nil {
- return nil, err
- }
- columnPrefixes = append(columnPrefixes, columnPrefix)
- if err := f.SetCellValue(sheet1Name, fmt.Sprintf("%s1", columnPrefix), columnName); err != nil {
- return nil, err
- }
- }
- for i, row := range result.Rows {
- for j, value := range row.Values {
- columnName := fmt.Sprintf("%s%d", columnPrefixes[j], i+2)
- if err := f.SetCellValue("Sheet1", columnName, convertValueToStringInXLSX(value)); err != nil {
- return nil, err
- }
- }
- }
- f.SetActiveSheet(index)
- excelBytes, err := f.WriteToBuffer()
- if err != nil {
- return nil, err
- }
- return excelBytes.Bytes(), nil
-}
-
-func getExcelColumnName(index int) (string, error) {
- if index >= excelMaxColumn {
- return "", errors.Errorf("index cannot be greater than %v (column ZZZ)", excelMaxColumn)
- }
-
- var s string
- for {
- remain := index % 26
- s = string(excelLetters[remain]) + s
- index = index/26 - 1
- if index < 0 {
- break
- }
- }
- return s, nil
-}
-
-func convertValueToStringInXLSX(value *v1pb.RowValue) string {
- if value == nil || value.Kind == nil {
- return ""
- }
- switch value.Kind.(type) {
- case *v1pb.RowValue_StringValue:
- return value.GetStringValue()
- case *v1pb.RowValue_Int32Value:
- return strconv.FormatInt(int64(value.GetInt32Value()), 10)
- case *v1pb.RowValue_Int64Value:
- return strconv.FormatInt(value.GetInt64Value(), 10)
- case *v1pb.RowValue_Uint32Value:
- return strconv.FormatUint(uint64(value.GetUint32Value()), 10)
- case *v1pb.RowValue_Uint64Value:
- return strconv.FormatUint(value.GetUint64Value(), 10)
- case *v1pb.RowValue_FloatValue:
- return strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32)
- case *v1pb.RowValue_DoubleValue:
- return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)
- case *v1pb.RowValue_BoolValue:
- return strconv.FormatBool(value.GetBoolValue())
- case *v1pb.RowValue_BytesValue:
- return base64.StdEncoding.EncodeToString(value.GetBytesValue())
- case *v1pb.RowValue_NullValue:
- return ""
- case *v1pb.RowValue_TimestampValue:
- return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")
- case *v1pb.RowValue_TimestampTzValue:
- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
- s := t.In(z).Format(time.RFC3339Nano)
- return s
- case *v1pb.RowValue_ValueValue:
- // This is used by ClickHouse and Spanner only.
- return value.GetValueValue().String()
- default:
- return ""
- }
-}
-
-func resourceExists(dbSchema *model.DatabaseSchema, resource base.SchemaResource) bool {
- schema := dbSchema.GetDatabaseMetadata().GetSchema(resource.Schema)
- if schema == nil {
- return false
- }
- if schema.GetTable(resource.Table) != nil {
- return true
- }
- if schema.GetView(resource.Table) != nil {
- return true
- }
- if schema.GetMaterializedView(resource.Table) != nil {
- return true
- }
- return false
-}
diff --git a/backend/component/export/csv.go b/backend/component/export/csv.go
new file mode 100644
index 0000000000..e6240ab91b
--- /dev/null
+++ b/backend/component/export/csv.go
@@ -0,0 +1,156 @@
+package export
+
+import (
+ "io"
+ "strconv"
+ "strings"
+ "time"
+
+ "google.golang.org/protobuf/types/known/structpb"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// CSV exports query results as CSV format (legacy wrapper).
+func CSV(result *v1pb.QueryResult) ([]byte, error) {
+ return exportToBytes(result, CSVToWriter)
+}
+
+// CSVToWriter streams query results as CSV directly to the writer.
+// This minimizes memory usage by avoiding intermediate buffering.
+func CSVToWriter(w io.Writer, result *v1pb.QueryResult) error {
+ if _, err := w.Write([]byte(strings.Join(result.ColumnNames, ","))); err != nil {
+ return err
+ }
+ if _, err := w.Write([]byte{'\n'}); err != nil {
+ return err
+ }
+ for i, row := range result.Rows {
+ for j, value := range row.Values {
+ if j != 0 {
+ if _, err := w.Write([]byte{','}); err != nil {
+ return err
+ }
+ }
+ if _, err := w.Write(convertValueToBytesInCSV(value)); err != nil {
+ return err
+ }
+ }
+ if i != len(result.Rows)-1 {
+ if _, err := w.Write([]byte{'\n'}); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func convertValueToBytesInCSV(value *v1pb.RowValue) []byte {
+ if value == nil || value.Kind == nil {
+ return []byte("")
+ }
+ switch value.Kind.(type) {
+ case *v1pb.RowValue_StringValue:
+ return quoteCSVString(value.GetStringValue())
+ case *v1pb.RowValue_Int32Value:
+ return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))
+ case *v1pb.RowValue_Int64Value:
+ return []byte(strconv.FormatInt(value.GetInt64Value(), 10))
+ case *v1pb.RowValue_Uint32Value:
+ return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))
+ case *v1pb.RowValue_Uint64Value:
+ return []byte(strconv.FormatUint(value.GetUint64Value(), 10))
+ case *v1pb.RowValue_FloatValue:
+ return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))
+ case *v1pb.RowValue_DoubleValue:
+ return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))
+ case *v1pb.RowValue_BoolValue:
+ return []byte(strconv.FormatBool(value.GetBoolValue()))
+ case *v1pb.RowValue_BytesValue:
+ return quoteCSVString(string(value.GetBytesValue()))
+ case *v1pb.RowValue_NullValue:
+ return []byte("")
+ case *v1pb.RowValue_TimestampValue:
+ return quoteCSVString(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000"))
+ case *v1pb.RowValue_TimestampTzValue:
+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
+ return quoteCSVString(t.In(z).Format(time.RFC3339Nano))
+ case *v1pb.RowValue_ValueValue:
+ return convertValueValueToBytes(value.GetValueValue())
+ default:
+ return []byte("")
+ }
+}
+
+// quoteCSVString wraps a string in double quotes and escapes internal quotes.
+func quoteCSVString(s string) []byte {
+ escaped := escapeCSVString(s)
+ result := make([]byte, 0, len(escaped)+2)
+ result = append(result, '"')
+ result = append(result, []byte(escaped)...)
+ result = append(result, '"')
+ return result
+}
+
+func escapeCSVString(str string) string {
+ return strings.ReplaceAll(str, `"`, `""`)
+}
+
+func convertValueValueToBytes(value *structpb.Value) []byte {
+ if value == nil || value.Kind == nil {
+ return []byte("")
+ }
+ switch value.Kind.(type) {
+ case *structpb.Value_NullValue:
+ return []byte("")
+ case *structpb.Value_StringValue:
+ var result []byte
+ result = append(result, '"')
+ result = append(result, []byte(value.GetStringValue())...)
+ result = append(result, '"')
+ return result
+ case *structpb.Value_NumberValue:
+ return []byte(strconv.FormatFloat(value.GetNumberValue(), 'f', -1, 64))
+ case *structpb.Value_BoolValue:
+ return []byte(strconv.FormatBool(value.GetBoolValue()))
+ case *structpb.Value_ListValue:
+ var buf [][]byte
+ for _, v := range value.GetListValue().Values {
+ buf = append(buf, convertValueValueToBytes(v))
+ }
+ var result []byte
+ result = append(result, '"')
+ result = append(result, '[')
+ result = append(result, []byte(strings.Join(toStringSlice(buf), ","))...)
+ result = append(result, ']')
+ result = append(result, '"')
+ return result
+ case *structpb.Value_StructValue:
+ first := true
+ var buf []byte
+ buf = append(buf, '"')
+ for k, v := range value.GetStructValue().Fields {
+ if first {
+ first = false
+ } else {
+ buf = append(buf, ',')
+ }
+ buf = append(buf, []byte(k)...)
+ buf = append(buf, ':')
+ buf = append(buf, convertValueValueToBytes(v)...)
+ }
+ buf = append(buf, '"')
+ return buf
+ default:
+ return []byte("")
+ }
+}
+
+func toStringSlice(bufs [][]byte) []string {
+ result := make([]string, len(bufs))
+ for i, buf := range bufs {
+ result[i] = string(buf)
+ }
+ return result
+}
diff --git a/backend/component/export/export.go b/backend/component/export/export.go
new file mode 100644
index 0000000000..6e1231ec88
--- /dev/null
+++ b/backend/component/export/export.go
@@ -0,0 +1,22 @@
+// Package export provides data export functionality for various formats (CSV, JSON, SQL, XLSX).
+// It implements streaming export to minimize memory usage for large datasets.
+package export
+
+import (
+ "bytes"
+ "io"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// Writer is a function type that writes query results to a writer.
+type Writer func(w io.Writer, result *v1pb.QueryResult) error
+
+// exportToBytes is a helper function that exports to a byte slice using a writer function.
+func exportToBytes(result *v1pb.QueryResult, writerFunc Writer) ([]byte, error) {
+ var buf bytes.Buffer
+ if err := writerFunc(&buf, result); err != nil {
+ return nil, err
+ }
+ return buf.Bytes(), nil
+}
diff --git a/backend/api/v1/sql_service_export_utils_test.go b/backend/component/export/export_test.go
similarity index 95%
rename from backend/api/v1/sql_service_export_utils_test.go
rename to backend/component/export/export_test.go
index 2c511db69c..f44ef8638f 100644
--- a/backend/api/v1/sql_service_export_utils_test.go
+++ b/backend/component/export/export_test.go
@@ -1,4 +1,4 @@
-package v1
+package export
import (
"testing"
@@ -51,7 +51,7 @@ func TestGetSQLStatementPrefix(t *testing.T) {
a := assert.New(t)
for _, test := range tests {
- got, err := getSQLStatementPrefix(test.engine, test.resourceList, test.columnNames)
+ got, err := SQLStatementPrefix(test.engine, test.resourceList, test.columnNames)
a.NoError(err)
a.Equal(test.want, got)
}
@@ -199,7 +199,7 @@ func TestExportSQL(t *testing.T) {
a := assert.New(t)
for _, test := range tests {
- got, err := exportSQL(test.engine, test.statementPrefix, test.result)
+ got, err := SQL(test.engine, test.statementPrefix, test.result)
a.NoError(err)
a.Equal(test.want, string(got))
}
@@ -233,13 +233,13 @@ func TestGetExcelColumnName(t *testing.T) {
want: "AB",
},
{
- index: excelMaxColumn - 1,
+ index: ExcelMaxColumn - 1,
want: "ZZZ",
},
}
for _, test := range tests {
- got, err := getExcelColumnName(test.index)
+ got, err := ExcelColumnName(test.index)
a.NoError(err)
a.Equal(test.want, got)
}
@@ -304,7 +304,7 @@ func TestExportJSON(t *testing.T) {
a := assert.New(t)
for _, test := range tests {
- got, err := exportJSON(test.result)
+ got, err := JSON(test.result)
a.NoError(err)
a.Equal(test.want, string(got))
}
diff --git a/backend/component/export/json.go b/backend/component/export/json.go
new file mode 100644
index 0000000000..d2b35a247a
--- /dev/null
+++ b/backend/component/export/json.go
@@ -0,0 +1,95 @@
+package export
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "time"
+
+ "github.com/pkg/errors"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+// JSON exports query results as JSON format (legacy wrapper).
+func JSON(result *v1pb.QueryResult) ([]byte, error) {
+ return exportToBytes(result, JSONToWriter)
+}
+
+// JSONToWriter streams query results as pretty-printed JSON directly to the writer.
+func JSONToWriter(w io.Writer, result *v1pb.QueryResult) error {
+ records := make([]map[string]any, 0, len(result.Rows))
+ for _, row := range result.Rows {
+ record := make(map[string]any, len(result.ColumnNames))
+ for i, value := range row.Values {
+ record[result.ColumnNames[i]] = convertValueToJSONValue(value)
+ }
+ records = append(records, record)
+ }
+
+ jsonBytes, err := json.MarshalIndent(records, "", " ")
+ if err != nil {
+ return errors.Errorf("failed to encode JSON: %v", err)
+ }
+ if _, err := w.Write(jsonBytes); err != nil {
+ return err
+ }
+ return nil
+}
+
+func convertValueToJSONValue(value *v1pb.RowValue) any {
+ if value == nil || value.Kind == nil {
+ return nil
+ }
+
+ switch value.Kind.(type) {
+ case *v1pb.RowValue_StringValue:
+ return value.GetStringValue()
+ case *v1pb.RowValue_Int32Value:
+ return value.GetInt32Value()
+ case *v1pb.RowValue_Int64Value:
+ return value.GetInt64Value()
+ case *v1pb.RowValue_Uint32Value:
+ return value.GetUint32Value()
+ case *v1pb.RowValue_Uint64Value:
+ return value.GetUint64Value()
+ case *v1pb.RowValue_FloatValue:
+ return value.GetFloatValue()
+ case *v1pb.RowValue_DoubleValue:
+ return value.GetDoubleValue()
+ case *v1pb.RowValue_BoolValue:
+ return value.GetBoolValue()
+ case *v1pb.RowValue_BytesValue:
+ binaryStr, err := convertBytesToBinaryString(value.GetBytesValue())
+ if err != nil {
+ return nil
+ }
+ return binaryStr
+ case *v1pb.RowValue_NullValue:
+ return nil
+ case *v1pb.RowValue_TimestampValue:
+ return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")
+ case *v1pb.RowValue_TimestampTzValue:
+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
+ return t.In(z).Format(time.RFC3339Nano)
+ case *v1pb.RowValue_ValueValue:
+ return value.GetValueValue().String()
+ default:
+ return nil
+ }
+}
+
+func convertBytesToBinaryString(bs []byte) (string, error) {
+ var buf bytes.Buffer
+ if _, err := buf.WriteString("0b"); err != nil {
+ return "", err
+ }
+ for _, b := range bs {
+ if _, err := buf.WriteString(fmt.Sprintf("%08b", b)); err != nil {
+ return "", err
+ }
+ }
+ return buf.String(), nil
+}
diff --git a/backend/component/export/resources.go b/backend/component/export/resources.go
new file mode 100644
index 0000000000..8323600f9b
--- /dev/null
+++ b/backend/component/export/resources.go
@@ -0,0 +1,289 @@
+package export
+
+import (
+ "context"
+
+ "connectrpc.com/connect"
+ "github.com/labstack/echo/v4"
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store"
+ "github.com/bytebase/bytebase/backend/store/model"
+ "github.com/bytebase/bytebase/backend/utils"
+)
+
+// GetResources extracts the resource list from the statement for exporting results as SQL.
+// It analyzes the SQL query to determine which database tables/views are being queried.
+func GetResources(
+ ctx context.Context,
+ storeInstance *store.Store,
+ engine storepb.Engine,
+ databaseName string,
+ statement string,
+ instance *store.InstanceMessage,
+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,
+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,
+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,
+) ([]base.SchemaResource, error) {
+ switch engine {
+ case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_OCEANBASE:
+ return getResourcesForMySQL(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+ case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:
+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, "public", getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+ case storepb.Engine_ORACLE:
+ return getResourcesForOracle(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+ case storepb.Engine_SNOWFLAKE:
+ return getResourcesForSnowflake(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+ case storepb.Engine_MSSQL:
+ return getResourcesForMSSQL(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+ default:
+ if databaseName == "" {
+ return nil, errors.Errorf("database must be specified")
+ }
+ return []base.SchemaResource{{Database: databaseName}}, nil
+ }
+}
+
+func getResourcesForMySQL(
+ ctx context.Context,
+ storeInstance *store.Store,
+ engine storepb.Engine,
+ databaseName string,
+ statement string,
+ instance *store.InstanceMessage,
+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,
+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,
+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,
+) ([]base.SchemaResource, error) {
+ spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
+ InstanceID: instance.ResourceID,
+ GetDatabaseMetadataFunc: getDatabaseMetadataFunc,
+ ListDatabaseNamesFunc: listDatabaseNamesFunc,
+ GetLinkedDatabaseMetadataFunc: getLinkedDatabaseMetadataFunc,
+ }, engine, statement, databaseName, "", !store.IsObjectCaseSensitive(instance))
+ if err != nil {
+ return nil, err
+ } else if databaseName == "" {
+ var list []base.SchemaResource
+ for _, span := range spans {
+ for sourceColumn := range span.SourceColumns {
+ list = append(list, base.SchemaResource{
+ Database: sourceColumn.Database,
+ Schema: sourceColumn.Schema,
+ Table: sourceColumn.Table,
+ LinkedServer: sourceColumn.Server,
+ })
+ }
+ }
+ return list, nil
+ }
+
+ database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
+ InstanceID: &instance.ResourceID,
+ DatabaseName: &databaseName,
+ IsCaseSensitive: store.IsObjectCaseSensitive(instance),
+ })
+ if err != nil {
+ if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {
+ return nil, nil
+ }
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
+ }
+ if database == nil {
+ return nil, nil
+ }
+
+ dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
+ InstanceID: database.InstanceID,
+ DatabaseName: database.DatabaseName,
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
+ }
+
+ var result []base.SchemaResource
+ for _, span := range spans {
+ for sourceColumn := range span.SourceColumns {
+ sr := base.SchemaResource{
+ Database: sourceColumn.Database,
+ Schema: sourceColumn.Schema,
+ Table: sourceColumn.Table,
+ LinkedServer: sourceColumn.Server,
+ }
+ if sourceColumn.Database != dbSchema.GetMetadata().Name {
+ resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
+ InstanceID: &instance.ResourceID,
+ DatabaseName: &sourceColumn.Database,
+ IsCaseSensitive: store.IsObjectCaseSensitive(instance),
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sourceColumn.Database, instance.ResourceID))
+ }
+ if resourceDB == nil {
+ continue
+ }
+ resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
+ InstanceID: resourceDB.InstanceID,
+ DatabaseName: resourceDB.DatabaseName,
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sourceColumn.Database, instance.ResourceID))
+ }
+ if !resourceExists(resourceDBSchema, sr) {
+ continue
+ }
+ result = append(result, sr)
+ continue
+ }
+ if !resourceExists(dbSchema, sr) {
+ continue
+ }
+ result = append(result, sr)
+ }
+ }
+ return result, nil
+}
+
+func getResourcesForPostgres(
+ ctx context.Context,
+ storeInstance *store.Store,
+ engine storepb.Engine,
+ databaseName string,
+ statement string,
+ instance *store.InstanceMessage,
+ defaultSchema string,
+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,
+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,
+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,
+) ([]base.SchemaResource, error) {
+ spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{
+ InstanceID: instance.ResourceID,
+ GetDatabaseMetadataFunc: getDatabaseMetadataFunc,
+ ListDatabaseNamesFunc: listDatabaseNamesFunc,
+ GetLinkedDatabaseMetadataFunc: getLinkedDatabaseMetadataFunc,
+ }, engine, statement, databaseName, defaultSchema, !store.IsObjectCaseSensitive(instance))
+ if err != nil {
+ return nil, err
+ }
+
+ database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{
+ InstanceID: &instance.ResourceID,
+ DatabaseName: &databaseName,
+ IsCaseSensitive: store.IsObjectCaseSensitive(instance),
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))
+ }
+ if database == nil {
+ return nil, nil
+ }
+
+ dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{
+ InstanceID: database.InstanceID,
+ DatabaseName: database.DatabaseName,
+ })
+ if err != nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))
+ }
+
+ var result []base.SchemaResource
+ for _, span := range spans {
+ for sourceColumn := range span.SourceColumns {
+ sr := base.SchemaResource{
+ Database: sourceColumn.Database,
+ Schema: sourceColumn.Schema,
+ Table: sourceColumn.Table,
+ LinkedServer: sourceColumn.Server,
+ }
+
+ if sourceColumn.Database != dbSchema.GetMetadata().Name {
+ continue
+ }
+
+ if !resourceExists(dbSchema, sr) {
+ continue
+ }
+
+ result = append(result, sr)
+ }
+ }
+
+ return result, nil
+}
+
+func getResourcesForOracle(
+ ctx context.Context,
+ storeInstance *store.Store,
+ engine storepb.Engine,
+ databaseName string,
+ statement string,
+ instance *store.InstanceMessage,
+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,
+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,
+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,
+) ([]base.SchemaResource, error) {
+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, databaseName, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+}
+
+func getResourcesForSnowflake(
+ ctx context.Context,
+ storeInstance *store.Store,
+ engine storepb.Engine,
+ databaseName string,
+ statement string,
+ instance *store.InstanceMessage,
+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,
+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,
+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,
+) ([]base.SchemaResource, error) {
+ dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)
+ adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)
+ if dataSource == nil {
+ dataSource = adminDataSource
+ }
+ if dataSource == nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))
+ }
+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, "PUBLIC", getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+}
+
+func getResourcesForMSSQL(
+ ctx context.Context,
+ storeInstance *store.Store,
+ engine storepb.Engine,
+ databaseName string,
+ statement string,
+ instance *store.InstanceMessage,
+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,
+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,
+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,
+) ([]base.SchemaResource, error) {
+ dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)
+ adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)
+ if dataSource == nil {
+ dataSource = adminDataSource
+ }
+ if dataSource == nil {
+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))
+ }
+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, "dbo", getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)
+}
+
+func resourceExists(dbSchema *model.DatabaseSchema, resource base.SchemaResource) bool {
+ schema := dbSchema.GetDatabaseMetadata().GetSchema(resource.Schema)
+ if schema == nil {
+ return false
+ }
+ if schema.GetTable(resource.Table) != nil {
+ return true
+ }
+ if schema.GetView(resource.Table) != nil {
+ return true
+ }
+ if schema.GetMaterializedView(resource.Table) != nil {
+ return true
+ }
+ return false
+}
diff --git a/backend/component/export/sql.go b/backend/component/export/sql.go
new file mode 100644
index 0000000000..ebd32e9390
--- /dev/null
+++ b/backend/component/export/sql.go
@@ -0,0 +1,151 @@
+package export
+
+import (
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/lib/pq"
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// SQL exports query results as SQL INSERT statements (legacy wrapper).
+func SQL(engine storepb.Engine, statementPrefix string, result *v1pb.QueryResult) ([]byte, error) {
+ return exportToBytes(result, func(w io.Writer, r *v1pb.QueryResult) error {
+ return SQLToWriter(w, engine, statementPrefix, r)
+ })
+}
+
+// SQLToWriter streams SQL INSERT statements directly to the writer.
+func SQLToWriter(w io.Writer, engine storepb.Engine, statementPrefix string, result *v1pb.QueryResult) error {
+ for i, row := range result.Rows {
+ if _, err := w.Write([]byte(statementPrefix)); err != nil {
+ return err
+ }
+ for j, value := range row.Values {
+ if j != 0 {
+ if _, err := w.Write([]byte{','}); err != nil {
+ return err
+ }
+ }
+ if _, err := w.Write(convertValueToBytesInSQL(engine, value)); err != nil {
+ return err
+ }
+ }
+ if i != len(result.Rows)-1 {
+ if _, err := w.Write([]byte(");\n")); err != nil {
+ return err
+ }
+ } else {
+ if _, err := w.Write([]byte(");")); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// SQLStatementPrefix generates the INSERT INTO statement prefix.
+func SQLStatementPrefix(engine storepb.Engine, resourceList []base.SchemaResource, columnNames []string) (string, error) {
+ var escapeQuote string
+ switch engine {
+ case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_TIDB, storepb.Engine_OCEANBASE, storepb.Engine_SPANNER:
+ escapeQuote = "`"
+ case storepb.Engine_CLICKHOUSE, storepb.Engine_MSSQL, storepb.Engine_ORACLE, storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT, storepb.Engine_SQLITE, storepb.Engine_SNOWFLAKE:
+ escapeQuote = "\""
+ default:
+ return "", errors.Errorf("unsupported engine %v for exporting as SQL", engine)
+ }
+
+ s := "INSERT INTO "
+ if len(resourceList) == 1 {
+ resource := resourceList[0]
+ if resource.Schema != "" {
+ s = fmt.Sprintf("%s%s%s%s%s", s, escapeQuote, resource.Schema, escapeQuote, ".")
+ }
+ s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, resource.Table, escapeQuote)
+ } else {
+ s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, "<table_name>", escapeQuote)
+ }
+ var columnTokens []string
+ for _, columnName := range columnNames {
+ columnTokens = append(columnTokens, fmt.Sprintf("%s%s%s", escapeQuote, columnName, escapeQuote))
+ }
+ s = fmt.Sprintf("%s (%s) VALUES (", s, strings.Join(columnTokens, ","))
+ return s, nil
+}
+
+func convertValueToBytesInSQL(engine storepb.Engine, value *v1pb.RowValue) []byte {
+ if value == nil || value.Kind == nil {
+ return []byte("")
+ }
+ switch value.Kind.(type) {
+ case *v1pb.RowValue_StringValue:
+ return escapeSQLString(engine, []byte(value.GetStringValue()))
+ case *v1pb.RowValue_Int32Value:
+ return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))
+ case *v1pb.RowValue_Int64Value:
+ return []byte(strconv.FormatInt(value.GetInt64Value(), 10))
+ case *v1pb.RowValue_Uint32Value:
+ return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))
+ case *v1pb.RowValue_Uint64Value:
+ return []byte(strconv.FormatUint(value.GetUint64Value(), 10))
+ case *v1pb.RowValue_FloatValue:
+ return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))
+ case *v1pb.RowValue_DoubleValue:
+ return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))
+ case *v1pb.RowValue_BoolValue:
+ return []byte(strconv.FormatBool(value.GetBoolValue()))
+ case *v1pb.RowValue_BytesValue:
+ return escapeSQLBytes(engine, value.GetBytesValue())
+ case *v1pb.RowValue_NullValue:
+ return []byte("NULL")
+ case *v1pb.RowValue_TimestampValue:
+ return escapeSQLString(engine, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")))
+ case *v1pb.RowValue_TimestampTzValue:
+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
+ s := t.In(z).Format(time.RFC3339Nano)
+ return escapeSQLString(engine, []byte(s))
+ case *v1pb.RowValue_ValueValue:
+ return convertValueValueToBytes(value.GetValueValue())
+ default:
+ return []byte("")
+ }
+}
+
+func escapeSQLString(engine storepb.Engine, v []byte) []byte {
+ switch engine {
+ case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:
+ escapedStr := pq.QuoteLiteral(string(v))
+ return []byte(escapedStr)
+ default:
+ result := []byte("'")
+ s := strconv.Quote(string(v))
+ s = s[1 : len(s)-1]
+ s = strings.ReplaceAll(s, `'`, `''`)
+ result = append(result, []byte(s)...)
+ result = append(result, '\'')
+ return result
+ }
+}
+
+func escapeSQLBytes(engine storepb.Engine, v []byte) []byte {
+ switch engine {
+ case storepb.Engine_MYSQL, storepb.Engine_MARIADB:
+ result := []byte("B'")
+ s := fmt.Sprintf("%b", v)
+ s = s[1 : len(s)-1]
+ result = append(result, []byte(s)...)
+ result = append(result, '\'')
+ return result
+ default:
+ return escapeSQLString(engine, v)
+ }
+}
diff --git a/backend/component/export/xlsx.go b/backend/component/export/xlsx.go
new file mode 100644
index 0000000000..4edf9d619c
--- /dev/null
+++ b/backend/component/export/xlsx.go
@@ -0,0 +1,125 @@
+package export
+
+import (
+ "encoding/base64"
+ "fmt"
+ "io"
+ "strconv"
+ "time"
+
+ "github.com/pkg/errors"
+ "github.com/xuri/excelize/v2"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+const (
+ excelLetters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ sheet1Name = "Sheet1"
+ // ExcelMaxColumn is the maximum number of columns supported by Excel.
+ ExcelMaxColumn = 18278
+)
+
+// XLSX exports query results as XLSX format.
+// Note: XLSX still materializes in memory due to excelize library limitations.
+func XLSX(result *v1pb.QueryResult) ([]byte, error) {
+ f := excelize.NewFile()
+ defer f.Close()
+ index, err := f.NewSheet("Sheet1")
+ if err != nil {
+ return nil, err
+ }
+ var columnPrefixes []string
+ for i, columnName := range result.ColumnNames {
+ columnPrefix, err := ExcelColumnName(i)
+ if err != nil {
+ return nil, err
+ }
+ columnPrefixes = append(columnPrefixes, columnPrefix)
+ if err := f.SetCellValue(sheet1Name, fmt.Sprintf("%s1", columnPrefix), columnName); err != nil {
+ return nil, err
+ }
+ }
+ for i, row := range result.Rows {
+ for j, value := range row.Values {
+ columnName := fmt.Sprintf("%s%d", columnPrefixes[j], i+2)
+ if err := f.SetCellValue("Sheet1", columnName, convertValueToStringInXLSX(value)); err != nil {
+ return nil, err
+ }
+ }
+ }
+ f.SetActiveSheet(index)
+ excelBytes, err := f.WriteToBuffer()
+ if err != nil {
+ return nil, err
+ }
+ return excelBytes.Bytes(), nil
+}
+
+// XLSXToWriter exports XLSX format to a writer.
+// Note: XLSX still materializes in memory due to excelize library limitations.
+func XLSXToWriter(w io.Writer, result *v1pb.QueryResult) error {
+ content, err := XLSX(result)
+ if err != nil {
+ return err
+ }
+ _, err = w.Write(content)
+ return err
+}
+
+// ExcelColumnName converts a column index to Excel column name (A, B, ..., Z, AA, AB, ..., ZZZ).
+func ExcelColumnName(index int) (string, error) {
+ if index >= ExcelMaxColumn {
+ return "", errors.Errorf("index cannot be greater than %v (column ZZZ)", ExcelMaxColumn)
+ }
+
+ var s string
+ for {
+ remain := index % 26
+ s = string(excelLetters[remain]) + s
+ index = index/26 - 1
+ if index < 0 {
+ break
+ }
+ }
+ return s, nil
+}
+
+func convertValueToStringInXLSX(value *v1pb.RowValue) string {
+ if value == nil || value.Kind == nil {
+ return ""
+ }
+ switch value.Kind.(type) {
+ case *v1pb.RowValue_StringValue:
+ return value.GetStringValue()
+ case *v1pb.RowValue_Int32Value:
+ return strconv.FormatInt(int64(value.GetInt32Value()), 10)
+ case *v1pb.RowValue_Int64Value:
+ return strconv.FormatInt(value.GetInt64Value(), 10)
+ case *v1pb.RowValue_Uint32Value:
+ return strconv.FormatUint(uint64(value.GetUint32Value()), 10)
+ case *v1pb.RowValue_Uint64Value:
+ return strconv.FormatUint(value.GetUint64Value(), 10)
+ case *v1pb.RowValue_FloatValue:
+ return strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32)
+ case *v1pb.RowValue_DoubleValue:
+ return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)
+ case *v1pb.RowValue_BoolValue:
+ return strconv.FormatBool(value.GetBoolValue())
+ case *v1pb.RowValue_BytesValue:
+ return base64.StdEncoding.EncodeToString(value.GetBytesValue())
+ case *v1pb.RowValue_NullValue:
+ return ""
+ case *v1pb.RowValue_TimestampValue:
+ return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")
+ case *v1pb.RowValue_TimestampTzValue:
+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()
+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))
+ s := t.In(z).Format(time.RFC3339Nano)
+ return s
+ case *v1pb.RowValue_ValueValue:
+ return value.GetValueValue().String()
+ default:
+ return ""
+ }
+}
diff --git a/backend/component/export/zip.go b/backend/component/export/zip.go
new file mode 100644
index 0000000000..75e832ca4c
--- /dev/null
+++ b/backend/component/export/zip.go
@@ -0,0 +1,54 @@
+package export
+
+import (
+ "io"
+ "time"
+
+ "github.com/alexmullins/zip"
+ "github.com/pkg/errors"
+)
+
+// CreateZipFileHeader creates a zip file header with optional password protection.
+func CreateZipFileHeader(filename string, password string) *zip.FileHeader {
+ fh := &zip.FileHeader{
+ Name: filename,
+ Method: zip.Deflate,
+ }
+ fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(time.Now())
+ if password != "" {
+ fh.SetPassword(password)
+ }
+ return fh
+}
+
+// WriteZipEntry writes content to a zip file entry.
+func WriteZipEntry(zipw *zip.Writer, filename string, content []byte, password string) error {
+ fh := CreateZipFileHeader(filename, password)
+ writer, err := zipw.CreateHeader(fh)
+ if err != nil {
+ return errors.Wrapf(err, "failed to create zip entry for %s", filename)
+ }
+ if _, err := writer.Write(content); err != nil {
+ return errors.Wrapf(err, "failed to write zip entry for %s", filename)
+ }
+ return nil
+}
+
+// CreateZipWriter creates a zip writer for the given writer with a file header.
+func CreateZipWriter(zipw *zip.Writer, filename string, password string) (io.Writer, error) {
+ fh := CreateZipFileHeader(filename, password)
+ writer, err := zipw.CreateHeader(fh)
+ if err != nil {
+ return nil, errors.Wrapf(err, "failed to create zip entry for %s", filename)
+ }
+ return writer, nil
+}
+
+// timeToMsDosTime converts a time.Time to an MS-DOS date and time.
+// This is a modified copy for github.com/alexmullins/zip/struct.go because the package has a bug,
+// it will convert the time to UTC time and drop the timezone.
+func timeToMsDosTime(t time.Time) (uint16, uint16) {
+ fDate := uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)
+ fTime := uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)
+ return fDate, fTime
+}