Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor: move export utilities to dedicated package (#17952)
Danny Xu426f90a17270
diff --git a/backend/api/v1/audit_log_service.go b/backend/api/v1/audit_log_service.goindex 97bc7a3874..a8dcd70e59 100644--- a/backend/api/v1/audit_log_service.go+++ b/backend/api/v1/audit_log_service.go@@ -11,6 +11,7 @@ import ("github.com/bytebase/bytebase/backend/common""github.com/bytebase/bytebase/backend/common/qb"+ "github.com/bytebase/bytebase/backend/component/export""github.com/bytebase/bytebase/backend/component/iam""github.com/bytebase/bytebase/backend/enterprise"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -140,15 +141,15 @@ func (s *AuditLogService) ExportAuditLogs(ctx context.Context, request *connect.var content []byteswitch request.Msg.Format {case v1pb.ExportFormat_CSV:- if content, err = exportCSV(result); err != nil {+ if content, err = export.CSV(result); err != nil {return nil, err}case v1pb.ExportFormat_JSON:- if content, err = exportJSON(result); err != nil {+ if content, err = export.JSON(result); err != nil {return nil, err}case v1pb.ExportFormat_XLSX:- if content, err = exportXLSX(result); err != nil {+ if content, err = export.XLSX(result); err != nil {return nil, err}default:diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex dac2d45fe4..a65cd4d62d 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -22,6 +22,7 @@ import ("github.com/bytebase/bytebase/backend/common/log""github.com/bytebase/bytebase/backend/component/config""github.com/bytebase/bytebase/backend/component/dbfactory"+ "github.com/bytebase/bytebase/backend/component/export""github.com/bytebase/bytebase/backend/component/iam""github.com/bytebase/bytebase/backend/component/sheet""github.com/bytebase/bytebase/backend/enterprise"@@ -945,127 +946,175 @@ func DoExport(}}- pendingEncryptContents := []*encryptContent{}+ var buf bytes.Buffer+ zipw := zip.NewWriter(&buf)++ exportCount := 0for i, result := range results {if result.GetError() != "" {- slog.Error("failed to query result",- slog.String("instance", database.InstanceID),- slog.String("database", database.DatabaseName),- slog.String("project", database.ProjectID),- slog.String("query_error", result.GetError()),- )+ logExportError(database, "failed to query result", errors.New(result.GetError()))continue}- content, err := formatExport(ctx, stores, instance, database, result, request)- if err != nil {- slog.Error("failed to format export",- log.BBError(err),- slog.String("instance", database.InstanceID),- slog.String("database", database.DatabaseName),- slog.String("project", database.ProjectID),- )+ if err := exportResultToZip(ctx, zipw, stores, instance, database, result, request, i+1); err != nil {+ logExportError(database, "failed to export result to zip", err)continue}- filename := fmt.Sprintf("%s/%s/statement-%d", database.InstanceID, database.DatabaseName, i+1)- pendingEncryptContents = append(- pendingEncryptContents,- &encryptContent{- Name: fmt.Sprintf("%s.sql", filename),- Content: []byte(result.Statement),- },- &encryptContent{- Name: fmt.Sprintf("%s.result.%s", filename, strings.ToLower(request.Format.String())),- Content: content,- },- )++ exportCount+++ // Help GC by clearing the result data we've already processed+ result.Rows = nil}- if len(pendingEncryptContents) == 0 {+ if exportCount == 0 {return nil, duration, errors.Errorf("empty export data for database %s", database.DatabaseName)}- // we will save both query result and statement into the result, so we always zip files.- encryptedBytes, err := doEncrypt(pendingEncryptContents, request.GetPassword())- if err != nil {- return nil, duration, errors.Wrap(err, "failed to encrypt data")+ if err := zipw.Close(); err != nil {+ return nil, duration, errors.Wrap(err, "failed to close zip writer")}- return encryptedBytes, duration, nil++ return buf.Bytes(), duration, nil}-type encryptContent struct {- Name string- Content []byte+// logExportError logs export-related errors with consistent database context.+func logExportError(database *store.DatabaseMessage, message string, err error) {+ slog.Error(message,+ log.BBError(err),+ slog.String("instance", database.InstanceID),+ slog.String("database", database.DatabaseName),+ slog.String("project", database.ProjectID),+ )}-func doEncrypt(exports []*encryptContent, password string) ([]byte, error) {- var b bytes.Buffer- fzip := io.Writer(&b)+// exportResultToZip exports a single query result to the ZIP archive.+// It writes both the SQL statement and the formatted result data.+func exportResultToZip(+ ctx context.Context,+ zipw *zip.Writer,+ stores *store.Store,+ instance *store.InstanceMessage,+ database *store.DatabaseMessage,+ result *v1pb.QueryResult,+ request *v1pb.ExportRequest,+ statementNumber int,+) error {+ baseFilename := fmt.Sprintf("%s/%s/statement-%d", database.InstanceID, database.DatabaseName, statementNumber)- zipw := zip.NewWriter(fzip)- defer zipw.Close()+ // Write statement file+ statementFilename := fmt.Sprintf("%s.sql", baseFilename)+ if err := export.WriteZipEntry(zipw, statementFilename, []byte(result.Statement), request.GetPassword()); err != nil {+ return errors.Wrap(err, "failed to write statement")+ }- for _, export := range exports {- fh := &zip.FileHeader{- Name: export.Name,- Method: zip.Deflate,- }- fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(time.Now())- if password != "" {- fh.SetPassword(password)- }- writer, err := zipw.CreateHeader(fh)- if err != nil {- return nil, errors.Wrapf(err, "failed to create encrypt export file")- }- if _, err := io.Copy(writer, bytes.NewReader(export.Content)); err != nil {- return nil, errors.Wrapf(err, "failed to write export file")- }+ // Write result file by streaming directly to ZIP+ resultExt := strings.ToLower(request.Format.String())+ resultFilename := fmt.Sprintf("%s.result.%s", baseFilename, resultExt)+ if err := formatExportToZip(ctx, zipw, resultFilename, stores, instance, database, result, request); err != nil {+ return errors.Wrap(err, "failed to write formatted result")}- if err := zipw.Close(); err != nil {- return nil, errors.Wrap(err, "failed to close zip writer")+ return nil+}++// formatExportToZip formats query results and writes them directly to a ZIP entry.+// This function streams the formatted data to minimize memory usage.+func formatExportToZip(+ ctx context.Context,+ zipw *zip.Writer,+ filename string,+ stores *store.Store,+ instance *store.InstanceMessage,+ database *store.DatabaseMessage,+ result *v1pb.QueryResult,+ request *v1pb.ExportRequest,+) error {+ writer, err := export.CreateZipWriter(zipw, filename, request.GetPassword())+ if err != nil {+ return err}- return b.Bytes(), nil+ return writeFormattedResult(ctx, writer, stores, instance, database, result, request)}-func formatExport(+// writeFormattedResult writes the query result in the requested format to the writer.+func writeFormattedResult(ctx context.Context,+ w io.Writer,stores *store.Store,instance *store.InstanceMessage,database *store.DatabaseMessage,result *v1pb.QueryResult,request *v1pb.ExportRequest,-) ([]byte, error) {+) error {switch request.Format {case v1pb.ExportFormat_CSV:- return exportCSV(result)+ return export.CSVToWriter(w, result)case v1pb.ExportFormat_JSON:- return exportJSON(result)+ return export.JSONToWriter(w, result)case v1pb.ExportFormat_SQL:- resourceList, err := getResources(ctx, stores, instance.Metadata.GetEngine(), database.DatabaseName, request.Statement, instance)- if err != nil {- return nil, errors.Errorf("failed to extract resource list: %v", err)- }- statementPrefix, err := getSQLStatementPrefix(instance.Metadata.GetEngine(), resourceList, result.ColumnNames)- if err != nil {- return nil, err- }- return exportSQL(instance.Metadata.GetEngine(), statementPrefix, result)+ return exportSQLWithContext(ctx, w, stores, instance, database, result, request)case v1pb.ExportFormat_XLSX:- return exportXLSX(result)+ return export.XLSXToWriter(w, result)default:- return nil, errors.Errorf("unsupported export format: %s", request.Format.String())+ return errors.Errorf("unsupported export format: %s", request.Format.String())}}-// timeToMsDosTime converts a time.Time to an MS-DOS date and time.-// this is a modified copy for gihub.com/alexmullins/zip/struct.go cause the package has a bug, it will convert the time to UTC time and drop the timezone.-func timeToMsDosTime(t time.Time) (uint16, uint16) {- fDate := uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)- fTime := uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)- return fDate, fTime+// exportSQLWithContext exports SQL INSERT statements with proper context.+func exportSQLWithContext(+ ctx context.Context,+ w io.Writer,+ stores *store.Store,+ instance *store.InstanceMessage,+ database *store.DatabaseMessage,+ result *v1pb.QueryResult,+ request *v1pb.ExportRequest,+) error {+ resourceList, err := export.GetResources(+ ctx,+ stores,+ instance.Metadata.GetEngine(),+ database.DatabaseName,+ request.Statement,+ instance,+ BuildGetDatabaseMetadataFunc(stores),+ BuildListDatabaseNamesFunc(stores),+ BuildGetLinkedDatabaseMetadataFunc(stores, instance.Metadata.GetEngine()),+ )+ if err != nil {+ return errors.Wrapf(err, "failed to extract resource list")+ }+ statementPrefix, err := export.SQLStatementPrefix(instance.Metadata.GetEngine(), resourceList, result.ColumnNames)+ if err != nil {+ return err+ }+ return export.SQLToWriter(w, instance.Metadata.GetEngine(), statementPrefix, result)+}++type encryptContent struct {+ Name string+ Content []byte+}++func doEncrypt(exports []*encryptContent, password string) ([]byte, error) {+ var b bytes.Buffer+ fzip := io.Writer(&b)++ zipw := zip.NewWriter(fzip)+ defer zipw.Close()++ for _, exportContent := range exports {+ if err := export.WriteZipEntry(zipw, exportContent.Name, exportContent.Content, password); err != nil {+ return nil, err+ }+ }++ if err := zipw.Close(); err != nil {+ return nil, errors.Wrap(err, "failed to close zip writer")+ }++ return b.Bytes(), nil}func (s *SQLService) createQueryHistory(database *store.DatabaseMessage, queryType store.QueryHistoryType, statement string, userUID int, duration time.Duration, queryErr error) {diff --git a/backend/api/v1/sql_service_export_utils.go b/backend/api/v1/sql_service_export_utils.godeleted file mode 100644index 5fb40a07fb..0000000000--- a/backend/api/v1/sql_service_export_utils.go+++ /dev/null@@ -1,941 +0,0 @@-package v1--import (- "bytes"- "context"- "encoding/base64"- "encoding/json"- "fmt"- "strconv"- "strings"- "time"-- "connectrpc.com/connect"- "github.com/labstack/echo/v4"- "github.com/lib/pq"- "github.com/pkg/errors"- "github.com/xuri/excelize/v2"- "google.golang.org/protobuf/types/known/structpb"-- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"- "github.com/bytebase/bytebase/backend/plugin/parser/base"- "github.com/bytebase/bytebase/backend/store"- "github.com/bytebase/bytebase/backend/store/model"- "github.com/bytebase/bytebase/backend/utils"-)--func exportCSV(result *v1pb.QueryResult) ([]byte, error) {- var buf bytes.Buffer- if _, err := buf.WriteString(strings.Join(result.ColumnNames, ",")); err != nil {- return nil, err- }- if err := buf.WriteByte('\n'); err != nil {- return nil, err- }- for i, row := range result.Rows {- for i, value := range row.Values {- if i != 0 {- if err := buf.WriteByte(','); err != nil {- return nil, err- }- }- if _, err := buf.Write(convertValueToBytesInCSV(value)); err != nil {- return nil, err- }- }- if i != len(result.Rows)-1 {- if err := buf.WriteByte('\n'); err != nil {- return nil, err- }- }- }- return buf.Bytes(), nil-}--func convertValueToBytesInCSV(value *v1pb.RowValue) []byte {- if value == nil || value.Kind == nil {- return []byte("")- }- switch value.Kind.(type) {- case *v1pb.RowValue_StringValue:- var result []byte- result = append(result, '"')- result = append(result, []byte(escapeCSVString(value.GetStringValue()))...)- result = append(result, '"')- return result- case *v1pb.RowValue_Int32Value:- return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))- case *v1pb.RowValue_Int64Value:- return []byte(strconv.FormatInt(value.GetInt64Value(), 10))- case *v1pb.RowValue_Uint32Value:- return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))- case *v1pb.RowValue_Uint64Value:- return []byte(strconv.FormatUint(value.GetUint64Value(), 10))- case *v1pb.RowValue_FloatValue:- return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))- case *v1pb.RowValue_DoubleValue:- return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))- case *v1pb.RowValue_BoolValue:- return []byte(strconv.FormatBool(value.GetBoolValue()))- case *v1pb.RowValue_BytesValue:- var result []byte- result = append(result, '"')- result = append(result, []byte(escapeCSVString(string(value.GetBytesValue())))...)- result = append(result, '"')- return result- case *v1pb.RowValue_NullValue:- return []byte("")- case *v1pb.RowValue_TimestampValue:- var result []byte- result = append(result, '"')- result = append(result, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000"))...)- result = append(result, '"')- return result- case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- s := t.In(z).Format(time.RFC3339Nano)- var result []byte- result = append(result, '"')- result = append(result, []byte(s)...)- result = append(result, '"')- return result- case *v1pb.RowValue_ValueValue:- // This is used by ClickHouse and Spanner only.- return convertValueValueToBytes(value.GetValueValue())- default:- return []byte("")- }-}--func escapeCSVString(str string) string {- escapedStr := strings.ReplaceAll(str, `"`, `""`)- return escapedStr-}--func getSQLStatementPrefix(engine storepb.Engine, resourceList []base.SchemaResource, columnNames []string) (string, error) {- var escapeQuote string- switch engine {- case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_TIDB, storepb.Engine_OCEANBASE, storepb.Engine_SPANNER:- escapeQuote = "`"- case storepb.Engine_CLICKHOUSE, storepb.Engine_MSSQL, storepb.Engine_ORACLE, storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT, storepb.Engine_SQLITE, storepb.Engine_SNOWFLAKE:- // ClickHouse takes both double-quotes or backticks.- escapeQuote = "\""- default:- // storepb.Engine_MONGODB, storepb.Engine_REDIS- return "", errors.Errorf("unsupported engine %v for exporting as SQL", engine)- }-- s := "INSERT INTO "- if len(resourceList) == 1 {- resource := resourceList[0]- if resource.Schema != "" {- s = fmt.Sprintf("%s%s%s%s%s", s, escapeQuote, resource.Schema, escapeQuote, ".")- }- s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, resource.Table, escapeQuote)- } else {- s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, "<table_name>", escapeQuote)- }- var columnTokens []string- for _, columnName := range columnNames {- columnTokens = append(columnTokens, fmt.Sprintf("%s%s%s", escapeQuote, columnName, escapeQuote))- }- s = fmt.Sprintf("%s (%s) VALUES (", s, strings.Join(columnTokens, ","))- return s, nil-}--func exportSQL(engine storepb.Engine, statementPrefix string, result *v1pb.QueryResult) ([]byte, error) {- var buf bytes.Buffer- for i, row := range result.Rows {- if _, err := buf.WriteString(statementPrefix); err != nil {- return nil, err- }- for i, value := range row.Values {- if i != 0 {- if err := buf.WriteByte(','); err != nil {- return nil, err- }- }- if _, err := buf.Write(convertValueToBytesInSQL(engine, value)); err != nil {- return nil, err- }- }- if i != len(result.Rows)-1 {- if _, err := buf.WriteString(");\n"); err != nil {- return nil, err- }- } else {- if _, err := buf.WriteString(");"); err != nil {- return nil, err- }- }- }- return buf.Bytes(), nil-}--func convertValueToBytesInSQL(engine storepb.Engine, value *v1pb.RowValue) []byte {- if value == nil || value.Kind == nil {- return []byte("")- }- switch value.Kind.(type) {- case *v1pb.RowValue_StringValue:- return escapeSQLString(engine, []byte(value.GetStringValue()))- case *v1pb.RowValue_Int32Value:- return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))- case *v1pb.RowValue_Int64Value:- return []byte(strconv.FormatInt(value.GetInt64Value(), 10))- case *v1pb.RowValue_Uint32Value:- return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))- case *v1pb.RowValue_Uint64Value:- return []byte(strconv.FormatUint(value.GetUint64Value(), 10))- case *v1pb.RowValue_FloatValue:- return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))- case *v1pb.RowValue_DoubleValue:- return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))- case *v1pb.RowValue_BoolValue:- return []byte(strconv.FormatBool(value.GetBoolValue()))- case *v1pb.RowValue_BytesValue:- return escapeSQLBytes(engine, value.GetBytesValue())- case *v1pb.RowValue_NullValue:- return []byte("NULL")- case *v1pb.RowValue_TimestampValue:- return escapeSQLString(engine, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")))- case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- s := t.In(z).Format(time.RFC3339Nano)- return escapeSQLString(engine, []byte(s))- case *v1pb.RowValue_ValueValue:- // This is used by ClickHouse and Spanner only.- return convertValueValueToBytes(value.GetValueValue())- default:- return []byte("")- }-}--func escapeSQLString(engine storepb.Engine, v []byte) []byte {- switch engine {- case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:- escapedStr := pq.QuoteLiteral(string(v))- return []byte(escapedStr)- default:- result := []byte("'")- s := strconv.Quote(string(v))- s = s[1 : len(s)-1]- s = strings.ReplaceAll(s, `'`, `''`)- result = append(result, []byte(s)...)- result = append(result, '\'')- return result- }-}--func escapeSQLBytes(engine storepb.Engine, v []byte) []byte {- switch engine {- case storepb.Engine_MYSQL, storepb.Engine_MARIADB:- result := []byte("B'")- s := fmt.Sprintf("%b", v)- s = s[1 : len(s)-1]- result = append(result, []byte(s)...)- result = append(result, '\'')- return result- default:- return escapeSQLString(engine, v)- }-}--func convertValueValueToBytes(value *structpb.Value) []byte {- if value == nil || value.Kind == nil {- return []byte("")- }- switch value.Kind.(type) {- case *structpb.Value_NullValue:- return []byte("")- case *structpb.Value_StringValue:- var result []byte- result = append(result, '"')- result = append(result, []byte(value.GetStringValue())...)- result = append(result, '"')- return result- case *structpb.Value_NumberValue:- return []byte(strconv.FormatFloat(value.GetNumberValue(), 'f', -1, 64))- case *structpb.Value_BoolValue:- return []byte(strconv.FormatBool(value.GetBoolValue()))- case *structpb.Value_ListValue:- var buf [][]byte- for _, v := range value.GetListValue().Values {- buf = append(buf, convertValueValueToBytes(v))- }- var result []byte- result = append(result, '"')- result = append(result, '[')- result = append(result, bytes.Join(buf, []byte(","))...)- result = append(result, ']')- result = append(result, '"')- return result- case *structpb.Value_StructValue:- first := true- var buf []byte- buf = append(buf, '"')- for k, v := range value.GetStructValue().Fields {- if first {- first = false- } else {- buf = append(buf, ',')- }- buf = append(buf, []byte(k)...)- buf = append(buf, ':')- buf = append(buf, convertValueValueToBytes(v)...)- }- buf = append(buf, '"')- return buf- default:- return []byte("")- }-}--// getResources extracts the resource list from the statement for exporting results as SQL.-func getResources(ctx context.Context, storeInstance *store.Store, engine storepb.Engine, databaseName string, statement string, instance *store.InstanceMessage) ([]base.SchemaResource, error) {- switch engine {- case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_OCEANBASE:- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),- }, engine, statement, databaseName, "", !store.IsObjectCaseSensitive(instance))- if err != nil {- return nil, err- } else if databaseName == "" {- var list []base.SchemaResource- for _, span := range spans {- for sourceColumn := range span.SourceColumns {- list = append(list, base.SchemaResource{- Database: sourceColumn.Database,- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- LinkedServer: sourceColumn.Server,- })- }- }- return list, nil- }-- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &databaseName,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {- // If database not found, skip.- return nil, nil- }- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))- }- if database == nil {- return nil, nil- }-- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))- }-- var result []base.SchemaResource- for _, span := range spans {- for sourceColumn := range span.SourceColumns {- sr := base.SchemaResource{- Database: sourceColumn.Database,- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- LinkedServer: sourceColumn.Server,- }- if sourceColumn.Database != dbSchema.GetMetadata().Name {- // MySQL allows cross-database query, we should check the corresponding database.- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &sourceColumn.Database,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sourceColumn.Database, instance.ResourceID))- }- if resourceDB == nil {- continue- }- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: resourceDB.InstanceID,- DatabaseName: resourceDB.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sourceColumn.Database, instance.ResourceID))- }- if !resourceExists(resourceDBSchema, sr) {- // If table not found, we regard it as a CTE/alias/... and skip.- continue- }- result = append(result, sr)- continue- }- if !resourceExists(dbSchema, sr) {- // If table not found, skip.- continue- }- result = append(result, sr)- }- }- return result, nil- case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),- }, engine, statement, databaseName, "public", !store.IsObjectCaseSensitive(instance))- if err != nil {- return nil, err- }-- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &databaseName,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))- }- if database == nil {- return nil, nil- }-- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))- }-- var result []base.SchemaResource- for _, span := range spans {- for sourceColumn := range span.SourceColumns {- sr := base.SchemaResource{- Database: sourceColumn.Database,- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- LinkedServer: sourceColumn.Server,- }-- if sourceColumn.Database != dbSchema.GetMetadata().Name {- // Should not happen.- continue- }-- if !resourceExists(dbSchema, sr) {- // If table not found, skip.- continue- }-- result = append(result, sr)- }- }-- return result, nil- case storepb.Engine_ORACLE:- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),- }, engine, statement, databaseName, databaseName, !store.IsObjectCaseSensitive(instance))- if err != nil {- return nil, err- }-- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &databaseName,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))- }- if database == nil {- return nil, nil- }-- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))- }-- var result []base.SchemaResource- for _, span := range spans {- for sourceColumn := range span.SourceColumns {- sr := base.SchemaResource{- Database: sourceColumn.Database,- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- LinkedServer: sourceColumn.Server,- }- if sr.Database != dbSchema.GetMetadata().Name {- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &sr.Database,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sr.Database, instance.ResourceID))- }- if resourceDB == nil {- continue- }- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: resourceDB.InstanceID,- DatabaseName: resourceDB.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sr.Database, instance.ResourceID))- }- if !resourceExists(resourceDBSchema, sr) {- // If table not found, we regard it as a CTE/alias/... and skip.- continue- }- result = append(result, sr)- continue- }-- if !resourceExists(dbSchema, sr) {- // If table not found, skip.- continue- }-- result = append(result, sr)- }- }-- return result, nil- case storepb.Engine_SNOWFLAKE:- dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)- adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)- // If there are no read-only data source, fall back to admin data source.- if dataSource == nil {- dataSource = adminDataSource- }- if dataSource == nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))- }- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),- }, engine, statement, databaseName, "PUBLIC", !store.IsObjectCaseSensitive(instance))- if err != nil {- return nil, err- }- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &databaseName,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {- // If database not found, skip.- return nil, nil- }- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))- }- if database == nil {- return nil, nil- }-- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))- }-- var result []base.SchemaResource- for _, span := range spans {- for sourceColumn := range span.SourceColumns {- sr := base.SchemaResource{- Database: sourceColumn.Database,- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- LinkedServer: sourceColumn.Server,- }- if sr.Database != dbSchema.GetMetadata().Name {- // Snowflake allows cross-database query, we should check the corresponding database.- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &sr.Database,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sr.Database, instance.ResourceID))- }- if resourceDB == nil {- continue- }- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: resourceDB.InstanceID,- DatabaseName: resourceDB.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sr.Database, instance.ResourceID))- }- if !resourceExists(resourceDBSchema, sr) {- // If table not found, we regard it as a CTE/alias/... and skip.- continue- }- result = append(result, sr)- continue- }- if !resourceExists(dbSchema, sr) {- // If table not found, skip.- continue- }- result = append(result, sr)- }- }-- return result, nil- case storepb.Engine_MSSQL:- dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)- adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)- // If there are no read-only data source, fall back to admin data source.- if dataSource == nil {- dataSource = adminDataSource- }- if dataSource == nil {- return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))- }- spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{- InstanceID: instance.ResourceID,- GetDatabaseMetadataFunc: BuildGetDatabaseMetadataFunc(storeInstance),- ListDatabaseNamesFunc: BuildListDatabaseNamesFunc(storeInstance),- GetLinkedDatabaseMetadataFunc: BuildGetLinkedDatabaseMetadataFunc(storeInstance, instance.Metadata.GetEngine()),- }, engine, statement, databaseName, "dbo", !store.IsObjectCaseSensitive(instance))- if err != nil {- return nil, err- }- database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &databaseName,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {- // If database not found, skip.- return nil, nil- }- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))- }- if database == nil {- return nil, nil- }-- dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))- }-- var result []base.SchemaResource- for _, span := range spans {- for sourceColumn := range span.SourceColumns {- sr := base.SchemaResource{- Database: sourceColumn.Database,- Schema: sourceColumn.Schema,- Table: sourceColumn.Table,- LinkedServer: sourceColumn.Server,- }-- if sr.LinkedServer != "" {- continue- }- if sr.Database != dbSchema.GetMetadata().Name {- // MSSQL allows cross-database query, we should check the corresponding database.- resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{- InstanceID: &instance.ResourceID,- DatabaseName: &sr.Database,- IsCaseSensitive: store.IsObjectCaseSensitive(instance),- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sr.Database, instance.ResourceID))- }- if resourceDB == nil {- continue- }- resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{- InstanceID: resourceDB.InstanceID,- DatabaseName: resourceDB.DatabaseName,- })- if err != nil {- return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sr.Database, instance.ResourceID))- }- if !resourceExists(resourceDBSchema, sr) {- // If table not found, we regard it as a CTE/alias/... and skip.- continue- }- result = append(result, sr)- continue- }- if !resourceExists(dbSchema, sr) {- // If table not found, skip.- continue- }- result = append(result, sr)- }- }-- return result, nil- default:- if databaseName == "" {- return nil, errors.Errorf("database must be specified")- }- return []base.SchemaResource{{Database: databaseName}}, nil- }-}--func exportJSON(result *v1pb.QueryResult) ([]byte, error) {- var buf bytes.Buffer- if _, err := buf.WriteString("["); err != nil {- return nil, err- }-- for rowIndex, row := range result.Rows {- if _, err := buf.WriteString("{"); err != nil {- return nil, err- }- for i, value := range row.Values {- if _, err := buf.WriteString(fmt.Sprintf(`"%s":`, result.ColumnNames[i])); err != nil {- return nil, err- }- if _, err := buf.WriteString(convertValueToStringInJSON(value)); err != nil {- return nil, err- }- if i != len(row.Values)-1 {- if _, err := buf.WriteString(","); err != nil {- return nil, err- }- }- }- if _, err := buf.WriteString("}"); err != nil {- return nil, err- }-- if rowIndex != len(result.Rows)-1 {- if _, err := buf.WriteString(","); err != nil {- return nil, err- }- }- }-- if _, err := buf.WriteString("]"); err != nil {- return nil, err- }-- var data any- err := json.Unmarshal(buf.Bytes(), &data)- if err != nil {- return nil, errors.Errorf("failed to unmarshalling JSON with error: %v", err)- }-- // Marshal the data back into a pretty-printed JSON string- // The second argument "" is the prefix for each line, and " " is the indentation string.- formattedJSONBytes, err := json.MarshalIndent(data, "", " ")- if err != nil {- return nil, errors.Errorf("failed to marshalling JSON with error: %v", err)- }- return formattedJSONBytes, nil-}--func convertValueToStringInJSON(value *v1pb.RowValue) string {- if value == nil || value.Kind == nil {- return ""- }-- switch value.Kind.(type) {- case *v1pb.RowValue_StringValue:- return `"` + escapeJSONString(value.GetStringValue()) + `"`- case *v1pb.RowValue_Int32Value:- return strconv.FormatInt(int64(value.GetInt32Value()), 10)- case *v1pb.RowValue_Int64Value:- return strconv.FormatInt(value.GetInt64Value(), 10)- case *v1pb.RowValue_Uint32Value:- return strconv.FormatUint(uint64(value.GetUint32Value()), 10)- case *v1pb.RowValue_Uint64Value:- return strconv.FormatUint(value.GetUint64Value(), 10)- case *v1pb.RowValue_FloatValue:- return strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32)- case *v1pb.RowValue_DoubleValue:- return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)- case *v1pb.RowValue_BoolValue:- return strconv.FormatBool(value.GetBoolValue())- case *v1pb.RowValue_BytesValue:- value, err := convertBytesToBinaryString(value.GetBytesValue())- if err != nil {- return ""- }- return `"` + value + `"`- case *v1pb.RowValue_NullValue:- return "null"- case *v1pb.RowValue_TimestampValue:- return `"` + value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000") + `"`- case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- s := t.In(z).Format(time.RFC3339Nano)- return `"` + s + `"`- case *v1pb.RowValue_ValueValue:- // This is used by ClickHouse and Spanner only.- return value.GetValueValue().String()- default:- return ""- }-}--func convertBytesToBinaryString(bs []byte) (string, error) {- var buf bytes.Buffer- if _, err := buf.WriteString("0b"); err != nil {- return "", err- }- for _, b := range bs {- if _, err := buf.WriteString(fmt.Sprintf("%08b", b)); err != nil {- return "", err- }- }- return buf.String(), nil-}--func escapeJSONString(str string) string {- s := strconv.Quote(str)- return s[1 : len(s)-1]-}--const (- excelLetters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"- sheet1Name = "Sheet1"- excelMaxColumn = 18278-)--func exportXLSX(result *v1pb.QueryResult) ([]byte, error) {- f := excelize.NewFile()- defer f.Close()- index, err := f.NewSheet("Sheet1")- if err != nil {- return nil, err- }- var columnPrefixes []string- for i, columnName := range result.ColumnNames {- columnPrefix, err := getExcelColumnName(i)- if err != nil {- return nil, err- }- columnPrefixes = append(columnPrefixes, columnPrefix)- if err := f.SetCellValue(sheet1Name, fmt.Sprintf("%s1", columnPrefix), columnName); err != nil {- return nil, err- }- }- for i, row := range result.Rows {- for j, value := range row.Values {- columnName := fmt.Sprintf("%s%d", columnPrefixes[j], i+2)- if err := f.SetCellValue("Sheet1", columnName, convertValueToStringInXLSX(value)); err != nil {- return nil, err- }- }- }- f.SetActiveSheet(index)- excelBytes, err := f.WriteToBuffer()- if err != nil {- return nil, err- }- return excelBytes.Bytes(), nil-}--func getExcelColumnName(index int) (string, error) {- if index >= excelMaxColumn {- return "", errors.Errorf("index cannot be greater than %v (column ZZZ)", excelMaxColumn)- }-- var s string- for {- remain := index % 26- s = string(excelLetters[remain]) + s- index = index/26 - 1- if index < 0 {- break- }- }- return s, nil-}--func convertValueToStringInXLSX(value *v1pb.RowValue) string {- if value == nil || value.Kind == nil {- return ""- }- switch value.Kind.(type) {- case *v1pb.RowValue_StringValue:- return value.GetStringValue()- case *v1pb.RowValue_Int32Value:- return strconv.FormatInt(int64(value.GetInt32Value()), 10)- case *v1pb.RowValue_Int64Value:- return strconv.FormatInt(value.GetInt64Value(), 10)- case *v1pb.RowValue_Uint32Value:- return strconv.FormatUint(uint64(value.GetUint32Value()), 10)- case *v1pb.RowValue_Uint64Value:- return strconv.FormatUint(value.GetUint64Value(), 10)- case *v1pb.RowValue_FloatValue:- return strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32)- case *v1pb.RowValue_DoubleValue:- return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)- case *v1pb.RowValue_BoolValue:- return strconv.FormatBool(value.GetBoolValue())- case *v1pb.RowValue_BytesValue:- return base64.StdEncoding.EncodeToString(value.GetBytesValue())- case *v1pb.RowValue_NullValue:- return ""- case *v1pb.RowValue_TimestampValue:- return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")- case *v1pb.RowValue_TimestampTzValue:- t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()- z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))- s := t.In(z).Format(time.RFC3339Nano)- return s- case *v1pb.RowValue_ValueValue:- // This is used by ClickHouse and Spanner only.- return value.GetValueValue().String()- default:- return ""- }-}--func resourceExists(dbSchema *model.DatabaseSchema, resource base.SchemaResource) bool {- schema := dbSchema.GetDatabaseMetadata().GetSchema(resource.Schema)- if schema == nil {- return false- }- if schema.GetTable(resource.Table) != nil {- return true- }- if schema.GetView(resource.Table) != nil {- return true- }- if schema.GetMaterializedView(resource.Table) != nil {- return true- }- return false-}diff --git a/backend/component/export/csv.go b/backend/component/export/csv.gonew file mode 100644index 0000000000..e6240ab91b--- /dev/null+++ b/backend/component/export/csv.go@@ -0,0 +1,156 @@+package export++import (+ "io"+ "strconv"+ "strings"+ "time"++ "google.golang.org/protobuf/types/known/structpb"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// CSV exports query results as CSV format (legacy wrapper).+func CSV(result *v1pb.QueryResult) ([]byte, error) {+ return exportToBytes(result, CSVToWriter)+}++// CSVToWriter streams query results as CSV directly to the writer.+// This minimizes memory usage by avoiding intermediate buffering.+func CSVToWriter(w io.Writer, result *v1pb.QueryResult) error {+ if _, err := w.Write([]byte(strings.Join(result.ColumnNames, ","))); err != nil {+ return err+ }+ if _, err := w.Write([]byte{'\n'}); err != nil {+ return err+ }+ for i, row := range result.Rows {+ for j, value := range row.Values {+ if j != 0 {+ if _, err := w.Write([]byte{','}); err != nil {+ return err+ }+ }+ if _, err := w.Write(convertValueToBytesInCSV(value)); err != nil {+ return err+ }+ }+ if i != len(result.Rows)-1 {+ if _, err := w.Write([]byte{'\n'}); err != nil {+ return err+ }+ }+ }+ return nil+}++func convertValueToBytesInCSV(value *v1pb.RowValue) []byte {+ if value == nil || value.Kind == nil {+ return []byte("")+ }+ switch value.Kind.(type) {+ case *v1pb.RowValue_StringValue:+ return quoteCSVString(value.GetStringValue())+ case *v1pb.RowValue_Int32Value:+ return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))+ case *v1pb.RowValue_Int64Value:+ return []byte(strconv.FormatInt(value.GetInt64Value(), 10))+ case *v1pb.RowValue_Uint32Value:+ return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))+ case *v1pb.RowValue_Uint64Value:+ return []byte(strconv.FormatUint(value.GetUint64Value(), 10))+ case *v1pb.RowValue_FloatValue:+ return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))+ case *v1pb.RowValue_DoubleValue:+ return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))+ case *v1pb.RowValue_BoolValue:+ return []byte(strconv.FormatBool(value.GetBoolValue()))+ case *v1pb.RowValue_BytesValue:+ return quoteCSVString(string(value.GetBytesValue()))+ case *v1pb.RowValue_NullValue:+ return []byte("")+ case *v1pb.RowValue_TimestampValue:+ return quoteCSVString(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000"))+ case *v1pb.RowValue_TimestampTzValue:+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))+ return quoteCSVString(t.In(z).Format(time.RFC3339Nano))+ case *v1pb.RowValue_ValueValue:+ return convertValueValueToBytes(value.GetValueValue())+ default:+ return []byte("")+ }+}++// quoteCSVString wraps a string in double quotes and escapes internal quotes.+func quoteCSVString(s string) []byte {+ escaped := escapeCSVString(s)+ result := make([]byte, 0, len(escaped)+2)+ result = append(result, '"')+ result = append(result, []byte(escaped)...)+ result = append(result, '"')+ return result+}++func escapeCSVString(str string) string {+ return strings.ReplaceAll(str, `"`, `""`)+}++func convertValueValueToBytes(value *structpb.Value) []byte {+ if value == nil || value.Kind == nil {+ return []byte("")+ }+ switch value.Kind.(type) {+ case *structpb.Value_NullValue:+ return []byte("")+ case *structpb.Value_StringValue:+ var result []byte+ result = append(result, '"')+ result = append(result, []byte(value.GetStringValue())...)+ result = append(result, '"')+ return result+ case *structpb.Value_NumberValue:+ return []byte(strconv.FormatFloat(value.GetNumberValue(), 'f', -1, 64))+ case *structpb.Value_BoolValue:+ return []byte(strconv.FormatBool(value.GetBoolValue()))+ case *structpb.Value_ListValue:+ var buf [][]byte+ for _, v := range value.GetListValue().Values {+ buf = append(buf, convertValueValueToBytes(v))+ }+ var result []byte+ result = append(result, '"')+ result = append(result, '[')+ result = append(result, []byte(strings.Join(toStringSlice(buf), ","))...)+ result = append(result, ']')+ result = append(result, '"')+ return result+ case *structpb.Value_StructValue:+ first := true+ var buf []byte+ buf = append(buf, '"')+ for k, v := range value.GetStructValue().Fields {+ if first {+ first = false+ } else {+ buf = append(buf, ',')+ }+ buf = append(buf, []byte(k)...)+ buf = append(buf, ':')+ buf = append(buf, convertValueValueToBytes(v)...)+ }+ buf = append(buf, '"')+ return buf+ default:+ return []byte("")+ }+}++func toStringSlice(bufs [][]byte) []string {+ result := make([]string, len(bufs))+ for i, buf := range bufs {+ result[i] = string(buf)+ }+ return result+}diff --git a/backend/component/export/export.go b/backend/component/export/export.gonew file mode 100644index 0000000000..6e1231ec88--- /dev/null+++ b/backend/component/export/export.go@@ -0,0 +1,22 @@+// Package export provides data export functionality for various formats (CSV, JSON, SQL, XLSX).+// It implements streaming export to minimize memory usage for large datasets.+package export++import (+ "bytes"+ "io"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// Writer is a function type that writes query results to a writer.+type Writer func(w io.Writer, result *v1pb.QueryResult) error++// exportToBytes is a helper function that exports to a byte slice using a writer function.+func exportToBytes(result *v1pb.QueryResult, writerFunc Writer) ([]byte, error) {+ var buf bytes.Buffer+ if err := writerFunc(&buf, result); err != nil {+ return nil, err+ }+ return buf.Bytes(), nil+}diff --git a/backend/api/v1/sql_service_export_utils_test.go b/backend/component/export/export_test.gosimilarity index 95%rename from backend/api/v1/sql_service_export_utils_test.gorename to backend/component/export/export_test.goindex 2c511db69c..f44ef8638f 100644--- a/backend/api/v1/sql_service_export_utils_test.go+++ b/backend/component/export/export_test.go@@ -1,4 +1,4 @@-package v1+package exportimport ("testing"@@ -51,7 +51,7 @@ func TestGetSQLStatementPrefix(t *testing.T) {a := assert.New(t)for _, test := range tests {- got, err := getSQLStatementPrefix(test.engine, test.resourceList, test.columnNames)+ got, err := SQLStatementPrefix(test.engine, test.resourceList, test.columnNames)a.NoError(err)a.Equal(test.want, got)}@@ -199,7 +199,7 @@ func TestExportSQL(t *testing.T) {a := assert.New(t)for _, test := range tests {- got, err := exportSQL(test.engine, test.statementPrefix, test.result)+ got, err := SQL(test.engine, test.statementPrefix, test.result)a.NoError(err)a.Equal(test.want, string(got))}@@ -233,13 +233,13 @@ func TestGetExcelColumnName(t *testing.T) {want: "AB",},{- index: excelMaxColumn - 1,+ index: ExcelMaxColumn - 1,want: "ZZZ",},}for _, test := range tests {- got, err := getExcelColumnName(test.index)+ got, err := ExcelColumnName(test.index)a.NoError(err)a.Equal(test.want, got)}@@ -304,7 +304,7 @@ func TestExportJSON(t *testing.T) {a := assert.New(t)for _, test := range tests {- got, err := exportJSON(test.result)+ got, err := JSON(test.result)a.NoError(err)a.Equal(test.want, string(got))}diff --git a/backend/component/export/json.go b/backend/component/export/json.gonew file mode 100644index 0000000000..d2b35a247a--- /dev/null+++ b/backend/component/export/json.go@@ -0,0 +1,95 @@+package export++import (+ "bytes"+ "encoding/json"+ "fmt"+ "io"+ "time"++ "github.com/pkg/errors"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++// JSON exports query results as JSON format (legacy wrapper).+func JSON(result *v1pb.QueryResult) ([]byte, error) {+ return exportToBytes(result, JSONToWriter)+}++// JSONToWriter streams query results as pretty-printed JSON directly to the writer.+func JSONToWriter(w io.Writer, result *v1pb.QueryResult) error {+ records := make([]map[string]any, 0, len(result.Rows))+ for _, row := range result.Rows {+ record := make(map[string]any, len(result.ColumnNames))+ for i, value := range row.Values {+ record[result.ColumnNames[i]] = convertValueToJSONValue(value)+ }+ records = append(records, record)+ }++ jsonBytes, err := json.MarshalIndent(records, "", " ")+ if err != nil {+ return errors.Errorf("failed to encode JSON: %v", err)+ }+ if _, err := w.Write(jsonBytes); err != nil {+ return err+ }+ return nil+}++func convertValueToJSONValue(value *v1pb.RowValue) any {+ if value == nil || value.Kind == nil {+ return nil+ }++ switch value.Kind.(type) {+ case *v1pb.RowValue_StringValue:+ return value.GetStringValue()+ case *v1pb.RowValue_Int32Value:+ return value.GetInt32Value()+ case *v1pb.RowValue_Int64Value:+ return value.GetInt64Value()+ case *v1pb.RowValue_Uint32Value:+ return value.GetUint32Value()+ case *v1pb.RowValue_Uint64Value:+ return value.GetUint64Value()+ case *v1pb.RowValue_FloatValue:+ return value.GetFloatValue()+ case *v1pb.RowValue_DoubleValue:+ return value.GetDoubleValue()+ case *v1pb.RowValue_BoolValue:+ return value.GetBoolValue()+ case *v1pb.RowValue_BytesValue:+ binaryStr, err := convertBytesToBinaryString(value.GetBytesValue())+ if err != nil {+ return nil+ }+ return binaryStr+ case *v1pb.RowValue_NullValue:+ return nil+ case *v1pb.RowValue_TimestampValue:+ return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")+ case *v1pb.RowValue_TimestampTzValue:+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))+ return t.In(z).Format(time.RFC3339Nano)+ case *v1pb.RowValue_ValueValue:+ return value.GetValueValue().String()+ default:+ return nil+ }+}++func convertBytesToBinaryString(bs []byte) (string, error) {+ var buf bytes.Buffer+ if _, err := buf.WriteString("0b"); err != nil {+ return "", err+ }+ for _, b := range bs {+ if _, err := buf.WriteString(fmt.Sprintf("%08b", b)); err != nil {+ return "", err+ }+ }+ return buf.String(), nil+}diff --git a/backend/component/export/resources.go b/backend/component/export/resources.gonew file mode 100644index 0000000000..8323600f9b--- /dev/null+++ b/backend/component/export/resources.go@@ -0,0 +1,289 @@+package export++import (+ "context"++ "connectrpc.com/connect"+ "github.com/labstack/echo/v4"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store"+ "github.com/bytebase/bytebase/backend/store/model"+ "github.com/bytebase/bytebase/backend/utils"+)++// GetResources extracts the resource list from the statement for exporting results as SQL.+// It analyzes the SQL query to determine which database tables/views are being queried.+func GetResources(+ ctx context.Context,+ storeInstance *store.Store,+ engine storepb.Engine,+ databaseName string,+ statement string,+ instance *store.InstanceMessage,+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,+) ([]base.SchemaResource, error) {+ switch engine {+ case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_OCEANBASE:+ return getResourcesForMySQL(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+ case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, "public", getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+ case storepb.Engine_ORACLE:+ return getResourcesForOracle(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+ case storepb.Engine_SNOWFLAKE:+ return getResourcesForSnowflake(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+ case storepb.Engine_MSSQL:+ return getResourcesForMSSQL(ctx, storeInstance, engine, databaseName, statement, instance, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+ default:+ if databaseName == "" {+ return nil, errors.Errorf("database must be specified")+ }+ return []base.SchemaResource{{Database: databaseName}}, nil+ }+}++func getResourcesForMySQL(+ ctx context.Context,+ storeInstance *store.Store,+ engine storepb.Engine,+ databaseName string,+ statement string,+ instance *store.InstanceMessage,+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,+) ([]base.SchemaResource, error) {+ spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{+ InstanceID: instance.ResourceID,+ GetDatabaseMetadataFunc: getDatabaseMetadataFunc,+ ListDatabaseNamesFunc: listDatabaseNamesFunc,+ GetLinkedDatabaseMetadataFunc: getLinkedDatabaseMetadataFunc,+ }, engine, statement, databaseName, "", !store.IsObjectCaseSensitive(instance))+ if err != nil {+ return nil, err+ } else if databaseName == "" {+ var list []base.SchemaResource+ for _, span := range spans {+ for sourceColumn := range span.SourceColumns {+ list = append(list, base.SchemaResource{+ Database: sourceColumn.Database,+ Schema: sourceColumn.Schema,+ Table: sourceColumn.Table,+ LinkedServer: sourceColumn.Server,+ })+ }+ }+ return list, nil+ }++ database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{+ InstanceID: &instance.ResourceID,+ DatabaseName: &databaseName,+ IsCaseSensitive: store.IsObjectCaseSensitive(instance),+ })+ if err != nil {+ if httpErr, ok := err.(*echo.HTTPError); ok && httpErr.Code == echo.ErrNotFound.Code {+ return nil, nil+ }+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))+ }+ if database == nil {+ return nil, nil+ }++ dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))+ }++ var result []base.SchemaResource+ for _, span := range spans {+ for sourceColumn := range span.SourceColumns {+ sr := base.SchemaResource{+ Database: sourceColumn.Database,+ Schema: sourceColumn.Schema,+ Table: sourceColumn.Table,+ LinkedServer: sourceColumn.Server,+ }+ if sourceColumn.Database != dbSchema.GetMetadata().Name {+ resourceDB, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{+ InstanceID: &instance.ResourceID,+ DatabaseName: &sourceColumn.Database,+ IsCaseSensitive: store.IsObjectCaseSensitive(instance),+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database %v in instance %v", sourceColumn.Database, instance.ResourceID))+ }+ if resourceDB == nil {+ continue+ }+ resourceDBSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{+ InstanceID: resourceDB.InstanceID,+ DatabaseName: resourceDB.DatabaseName,+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to get database schema %v in instance %v", sourceColumn.Database, instance.ResourceID))+ }+ if !resourceExists(resourceDBSchema, sr) {+ continue+ }+ result = append(result, sr)+ continue+ }+ if !resourceExists(dbSchema, sr) {+ continue+ }+ result = append(result, sr)+ }+ }+ return result, nil+}++func getResourcesForPostgres(+ ctx context.Context,+ storeInstance *store.Store,+ engine storepb.Engine,+ databaseName string,+ statement string,+ instance *store.InstanceMessage,+ defaultSchema string,+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,+) ([]base.SchemaResource, error) {+ spans, err := base.GetQuerySpan(ctx, base.GetQuerySpanContext{+ InstanceID: instance.ResourceID,+ GetDatabaseMetadataFunc: getDatabaseMetadataFunc,+ ListDatabaseNamesFunc: listDatabaseNamesFunc,+ GetLinkedDatabaseMetadataFunc: getLinkedDatabaseMetadataFunc,+ }, engine, statement, databaseName, defaultSchema, !store.IsObjectCaseSensitive(instance))+ if err != nil {+ return nil, err+ }++ database, err := storeInstance.GetDatabaseV2(ctx, &store.FindDatabaseMessage{+ InstanceID: &instance.ResourceID,+ DatabaseName: &databaseName,+ IsCaseSensitive: store.IsObjectCaseSensitive(instance),+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database"))+ }+ if database == nil {+ return nil, nil+ }++ dbSchema, err := storeInstance.GetDBSchema(ctx, &store.FindDBSchemaMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ })+ if err != nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Wrap(err, "failed to fetch database schema"))+ }++ var result []base.SchemaResource+ for _, span := range spans {+ for sourceColumn := range span.SourceColumns {+ sr := base.SchemaResource{+ Database: sourceColumn.Database,+ Schema: sourceColumn.Schema,+ Table: sourceColumn.Table,+ LinkedServer: sourceColumn.Server,+ }++ if sourceColumn.Database != dbSchema.GetMetadata().Name {+ continue+ }++ if !resourceExists(dbSchema, sr) {+ continue+ }++ result = append(result, sr)+ }+ }++ return result, nil+}++func getResourcesForOracle(+ ctx context.Context,+ storeInstance *store.Store,+ engine storepb.Engine,+ databaseName string,+ statement string,+ instance *store.InstanceMessage,+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,+) ([]base.SchemaResource, error) {+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, databaseName, getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+}++func getResourcesForSnowflake(+ ctx context.Context,+ storeInstance *store.Store,+ engine storepb.Engine,+ databaseName string,+ statement string,+ instance *store.InstanceMessage,+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,+) ([]base.SchemaResource, error) {+ dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)+ adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)+ if dataSource == nil {+ dataSource = adminDataSource+ }+ if dataSource == nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))+ }+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, "PUBLIC", getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+}++func getResourcesForMSSQL(+ ctx context.Context,+ storeInstance *store.Store,+ engine storepb.Engine,+ databaseName string,+ statement string,+ instance *store.InstanceMessage,+ getDatabaseMetadataFunc base.GetDatabaseMetadataFunc,+ listDatabaseNamesFunc base.ListDatabaseNamesFunc,+ getLinkedDatabaseMetadataFunc base.GetLinkedDatabaseMetadataFunc,+) ([]base.SchemaResource, error) {+ dataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_READ_ONLY)+ adminDataSource := utils.DataSourceFromInstanceWithType(instance, storepb.DataSourceType_ADMIN)+ if dataSource == nil {+ dataSource = adminDataSource+ }+ if dataSource == nil {+ return nil, connect.NewError(connect.CodeInternal, errors.Errorf("failed to find data source for instance: %s", instance.ResourceID))+ }+ return getResourcesForPostgres(ctx, storeInstance, engine, databaseName, statement, instance, "dbo", getDatabaseMetadataFunc, listDatabaseNamesFunc, getLinkedDatabaseMetadataFunc)+}++func resourceExists(dbSchema *model.DatabaseSchema, resource base.SchemaResource) bool {+ schema := dbSchema.GetDatabaseMetadata().GetSchema(resource.Schema)+ if schema == nil {+ return false+ }+ if schema.GetTable(resource.Table) != nil {+ return true+ }+ if schema.GetView(resource.Table) != nil {+ return true+ }+ if schema.GetMaterializedView(resource.Table) != nil {+ return true+ }+ return false+}diff --git a/backend/component/export/sql.go b/backend/component/export/sql.gonew file mode 100644index 0000000000..ebd32e9390--- /dev/null+++ b/backend/component/export/sql.go@@ -0,0 +1,151 @@+package export++import (+ "fmt"+ "io"+ "strconv"+ "strings"+ "time"++ "github.com/lib/pq"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// SQL exports query results as SQL INSERT statements (legacy wrapper).+func SQL(engine storepb.Engine, statementPrefix string, result *v1pb.QueryResult) ([]byte, error) {+ return exportToBytes(result, func(w io.Writer, r *v1pb.QueryResult) error {+ return SQLToWriter(w, engine, statementPrefix, r)+ })+}++// SQLToWriter streams SQL INSERT statements directly to the writer.+func SQLToWriter(w io.Writer, engine storepb.Engine, statementPrefix string, result *v1pb.QueryResult) error {+ for i, row := range result.Rows {+ if _, err := w.Write([]byte(statementPrefix)); err != nil {+ return err+ }+ for j, value := range row.Values {+ if j != 0 {+ if _, err := w.Write([]byte{','}); err != nil {+ return err+ }+ }+ if _, err := w.Write(convertValueToBytesInSQL(engine, value)); err != nil {+ return err+ }+ }+ if i != len(result.Rows)-1 {+ if _, err := w.Write([]byte(");\n")); err != nil {+ return err+ }+ } else {+ if _, err := w.Write([]byte(");")); err != nil {+ return err+ }+ }+ }+ return nil+}++// SQLStatementPrefix generates the INSERT INTO statement prefix.+func SQLStatementPrefix(engine storepb.Engine, resourceList []base.SchemaResource, columnNames []string) (string, error) {+ var escapeQuote string+ switch engine {+ case storepb.Engine_MYSQL, storepb.Engine_MARIADB, storepb.Engine_TIDB, storepb.Engine_OCEANBASE, storepb.Engine_SPANNER:+ escapeQuote = "`"+ case storepb.Engine_CLICKHOUSE, storepb.Engine_MSSQL, storepb.Engine_ORACLE, storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT, storepb.Engine_SQLITE, storepb.Engine_SNOWFLAKE:+ escapeQuote = "\""+ default:+ return "", errors.Errorf("unsupported engine %v for exporting as SQL", engine)+ }++ s := "INSERT INTO "+ if len(resourceList) == 1 {+ resource := resourceList[0]+ if resource.Schema != "" {+ s = fmt.Sprintf("%s%s%s%s%s", s, escapeQuote, resource.Schema, escapeQuote, ".")+ }+ s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, resource.Table, escapeQuote)+ } else {+ s = fmt.Sprintf("%s%s%s%s", s, escapeQuote, "<table_name>", escapeQuote)+ }+ var columnTokens []string+ for _, columnName := range columnNames {+ columnTokens = append(columnTokens, fmt.Sprintf("%s%s%s", escapeQuote, columnName, escapeQuote))+ }+ s = fmt.Sprintf("%s (%s) VALUES (", s, strings.Join(columnTokens, ","))+ return s, nil+}++func convertValueToBytesInSQL(engine storepb.Engine, value *v1pb.RowValue) []byte {+ if value == nil || value.Kind == nil {+ return []byte("")+ }+ switch value.Kind.(type) {+ case *v1pb.RowValue_StringValue:+ return escapeSQLString(engine, []byte(value.GetStringValue()))+ case *v1pb.RowValue_Int32Value:+ return []byte(strconv.FormatInt(int64(value.GetInt32Value()), 10))+ case *v1pb.RowValue_Int64Value:+ return []byte(strconv.FormatInt(value.GetInt64Value(), 10))+ case *v1pb.RowValue_Uint32Value:+ return []byte(strconv.FormatUint(uint64(value.GetUint32Value()), 10))+ case *v1pb.RowValue_Uint64Value:+ return []byte(strconv.FormatUint(value.GetUint64Value(), 10))+ case *v1pb.RowValue_FloatValue:+ return []byte(strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32))+ case *v1pb.RowValue_DoubleValue:+ return []byte(strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64))+ case *v1pb.RowValue_BoolValue:+ return []byte(strconv.FormatBool(value.GetBoolValue()))+ case *v1pb.RowValue_BytesValue:+ return escapeSQLBytes(engine, value.GetBytesValue())+ case *v1pb.RowValue_NullValue:+ return []byte("NULL")+ case *v1pb.RowValue_TimestampValue:+ return escapeSQLString(engine, []byte(value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")))+ case *v1pb.RowValue_TimestampTzValue:+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))+ s := t.In(z).Format(time.RFC3339Nano)+ return escapeSQLString(engine, []byte(s))+ case *v1pb.RowValue_ValueValue:+ return convertValueValueToBytes(value.GetValueValue())+ default:+ return []byte("")+ }+}++func escapeSQLString(engine storepb.Engine, v []byte) []byte {+ switch engine {+ case storepb.Engine_POSTGRES, storepb.Engine_REDSHIFT:+ escapedStr := pq.QuoteLiteral(string(v))+ return []byte(escapedStr)+ default:+ result := []byte("'")+ s := strconv.Quote(string(v))+ s = s[1 : len(s)-1]+ s = strings.ReplaceAll(s, `'`, `''`)+ result = append(result, []byte(s)...)+ result = append(result, '\'')+ return result+ }+}++func escapeSQLBytes(engine storepb.Engine, v []byte) []byte {+ switch engine {+ case storepb.Engine_MYSQL, storepb.Engine_MARIADB:+ result := []byte("B'")+ s := fmt.Sprintf("%b", v)+ s = s[1 : len(s)-1]+ result = append(result, []byte(s)...)+ result = append(result, '\'')+ return result+ default:+ return escapeSQLString(engine, v)+ }+}diff --git a/backend/component/export/xlsx.go b/backend/component/export/xlsx.gonew file mode 100644index 0000000000..4edf9d619c--- /dev/null+++ b/backend/component/export/xlsx.go@@ -0,0 +1,125 @@+package export++import (+ "encoding/base64"+ "fmt"+ "io"+ "strconv"+ "time"++ "github.com/pkg/errors"+ "github.com/xuri/excelize/v2"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++const (+ excelLetters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ sheet1Name = "Sheet1"+ // ExcelMaxColumn is the maximum number of columns supported by Excel.+ ExcelMaxColumn = 18278+)++// XLSX exports query results as XLSX format.+// Note: XLSX still materializes in memory due to excelize library limitations.+func XLSX(result *v1pb.QueryResult) ([]byte, error) {+ f := excelize.NewFile()+ defer f.Close()+ index, err := f.NewSheet("Sheet1")+ if err != nil {+ return nil, err+ }+ var columnPrefixes []string+ for i, columnName := range result.ColumnNames {+ columnPrefix, err := ExcelColumnName(i)+ if err != nil {+ return nil, err+ }+ columnPrefixes = append(columnPrefixes, columnPrefix)+ if err := f.SetCellValue(sheet1Name, fmt.Sprintf("%s1", columnPrefix), columnName); err != nil {+ return nil, err+ }+ }+ for i, row := range result.Rows {+ for j, value := range row.Values {+ columnName := fmt.Sprintf("%s%d", columnPrefixes[j], i+2)+ if err := f.SetCellValue("Sheet1", columnName, convertValueToStringInXLSX(value)); err != nil {+ return nil, err+ }+ }+ }+ f.SetActiveSheet(index)+ excelBytes, err := f.WriteToBuffer()+ if err != nil {+ return nil, err+ }+ return excelBytes.Bytes(), nil+}++// XLSXToWriter exports XLSX format to a writer.+// Note: XLSX still materializes in memory due to excelize library limitations.+func XLSXToWriter(w io.Writer, result *v1pb.QueryResult) error {+ content, err := XLSX(result)+ if err != nil {+ return err+ }+ _, err = w.Write(content)+ return err+}++// ExcelColumnName converts a column index to Excel column name (A, B, ..., Z, AA, AB, ..., ZZZ).+func ExcelColumnName(index int) (string, error) {+ if index >= ExcelMaxColumn {+ return "", errors.Errorf("index cannot be greater than %v (column ZZZ)", ExcelMaxColumn)+ }++ var s string+ for {+ remain := index % 26+ s = string(excelLetters[remain]) + s+ index = index/26 - 1+ if index < 0 {+ break+ }+ }+ return s, nil+}++func convertValueToStringInXLSX(value *v1pb.RowValue) string {+ if value == nil || value.Kind == nil {+ return ""+ }+ switch value.Kind.(type) {+ case *v1pb.RowValue_StringValue:+ return value.GetStringValue()+ case *v1pb.RowValue_Int32Value:+ return strconv.FormatInt(int64(value.GetInt32Value()), 10)+ case *v1pb.RowValue_Int64Value:+ return strconv.FormatInt(value.GetInt64Value(), 10)+ case *v1pb.RowValue_Uint32Value:+ return strconv.FormatUint(uint64(value.GetUint32Value()), 10)+ case *v1pb.RowValue_Uint64Value:+ return strconv.FormatUint(value.GetUint64Value(), 10)+ case *v1pb.RowValue_FloatValue:+ return strconv.FormatFloat(float64(value.GetFloatValue()), 'f', -1, 32)+ case *v1pb.RowValue_DoubleValue:+ return strconv.FormatFloat(value.GetDoubleValue(), 'f', -1, 64)+ case *v1pb.RowValue_BoolValue:+ return strconv.FormatBool(value.GetBoolValue())+ case *v1pb.RowValue_BytesValue:+ return base64.StdEncoding.EncodeToString(value.GetBytesValue())+ case *v1pb.RowValue_NullValue:+ return ""+ case *v1pb.RowValue_TimestampValue:+ return value.GetTimestampValue().GoogleTimestamp.AsTime().Format("2006-01-02 15:04:05.000000")+ case *v1pb.RowValue_TimestampTzValue:+ t := value.GetTimestampTzValue().GoogleTimestamp.AsTime()+ z := time.FixedZone(value.GetTimestampTzValue().GetZone(), int(value.GetTimestampTzValue().GetOffset()))+ s := t.In(z).Format(time.RFC3339Nano)+ return s+ case *v1pb.RowValue_ValueValue:+ return value.GetValueValue().String()+ default:+ return ""+ }+}diff --git a/backend/component/export/zip.go b/backend/component/export/zip.gonew file mode 100644index 0000000000..75e832ca4c--- /dev/null+++ b/backend/component/export/zip.go@@ -0,0 +1,54 @@+package export++import (+ "io"+ "time"++ "github.com/alexmullins/zip"+ "github.com/pkg/errors"+)++// CreateZipFileHeader creates a zip file header with optional password protection.+func CreateZipFileHeader(filename string, password string) *zip.FileHeader {+ fh := &zip.FileHeader{+ Name: filename,+ Method: zip.Deflate,+ }+ fh.ModifiedDate, fh.ModifiedTime = timeToMsDosTime(time.Now())+ if password != "" {+ fh.SetPassword(password)+ }+ return fh+}++// WriteZipEntry writes content to a zip file entry.+func WriteZipEntry(zipw *zip.Writer, filename string, content []byte, password string) error {+ fh := CreateZipFileHeader(filename, password)+ writer, err := zipw.CreateHeader(fh)+ if err != nil {+ return errors.Wrapf(err, "failed to create zip entry for %s", filename)+ }+ if _, err := writer.Write(content); err != nil {+ return errors.Wrapf(err, "failed to write zip entry for %s", filename)+ }+ return nil+}++// CreateZipWriter creates a zip writer for the given writer with a file header.+func CreateZipWriter(zipw *zip.Writer, filename string, password string) (io.Writer, error) {+ fh := CreateZipFileHeader(filename, password)+ writer, err := zipw.CreateHeader(fh)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to create zip entry for %s", filename)+ }+ return writer, nil+}++// timeToMsDosTime converts a time.Time to an MS-DOS date and time.+// This is a modified copy for github.com/alexmullins/zip/struct.go because the package has a bug,+// it will convert the time to UTC time and drop the timezone.+func timeToMsDosTime(t time.Time) (uint16, uint16) {+ fDate := uint16(t.Day() + int(t.Month())<<5 + (t.Year()-1980)<<9)+ fTime := uint16(t.Second()/2 + t.Minute()<<5 + t.Hour()<<11)+ return fDate, fTime+}