Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

chore: implement tests for export (#17721)

ecmadao72144d3e1a36
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.go
index adf50911f0..0b13e02757 100644
--- a/backend/api/v1/sql_service.go
+++ b/backend/api/v1/sql_service.go
@@ -974,14 +974,14 @@ func DoExport(
filename := fmt.Sprintf("%s/%s/statement-%d", database.InstanceID, database.DatabaseName, i+1)
pendingEncryptContents = append(
pendingEncryptContents,
- &encryptContent{
- Name: fmt.Sprintf("%s.result.%s", filename, strings.ToLower(request.Format.String())),
- Content: content,
- },
&encryptContent{
Name: fmt.Sprintf("%s.sql", filename),
Content: []byte(result.Statement),
},
+ &encryptContent{
+ Name: fmt.Sprintf("%s.result.%s", filename, strings.ToLower(request.Format.String())),
+ Content: content,
+ },
)
}
diff --git a/backend/api/v1/sql_service_export_utils.go b/backend/api/v1/sql_service_export_utils.go
index 746033b254..5fb40a07fb 100644
--- a/backend/api/v1/sql_service_export_utils.go
+++ b/backend/api/v1/sql_service_export_utils.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"encoding/base64"
+ "encoding/json"
"fmt"
"strconv"
"strings"
@@ -746,7 +747,20 @@ func exportJSON(result *v1pb.QueryResult) ([]byte, error) {
if _, err := buf.WriteString("]"); err != nil {
return nil, err
}
- return buf.Bytes(), nil
+
+ var data any
+ err := json.Unmarshal(buf.Bytes(), &data)
+ if err != nil {
+ return nil, errors.Errorf("failed to unmarshalling JSON with error: %v", err)
+ }
+
+ // Marshal the data back into a pretty-printed JSON string
+ // The second argument "" is the prefix for each line, and " " is the indentation string.
+ formattedJSONBytes, err := json.MarshalIndent(data, "", " ")
+ if err != nil {
+ return nil, errors.Errorf("failed to marshalling JSON with error: %v", err)
+ }
+ return formattedJSONBytes, nil
}
func convertValueToStringInJSON(value *v1pb.RowValue) string {
diff --git a/backend/api/v1/sql_service_export_utils_test.go b/backend/api/v1/sql_service_export_utils_test.go
index df077bdd05..2c511db69c 100644
--- a/backend/api/v1/sql_service_export_utils_test.go
+++ b/backend/api/v1/sql_service_export_utils_test.go
@@ -263,7 +263,11 @@ func TestExportJSON(t *testing.T) {
},
},
},
- want: "[{\"a\":\"0b00000101\"}]",
+ want: `[
+ {
+ "a": "0b00000101"
+ }
+]`,
},
{
result: &v1pb.QueryResult{
@@ -287,7 +291,14 @@ func TestExportJSON(t *testing.T) {
},
},
},
- want: "[{\"id\":1,\"name\":\"Alice\",\"email\":\"[email protected]\",\"age\":20}]",
+ want: `[
+ {
+ "age": 20,
+ "email": "[email protected]",
+ "id": 1,
+ "name": "Alice"
+ }
+]`,
},
}
diff --git a/backend/tests/sql_export_test.go b/backend/tests/sql_export_test.go
index 7a193f9e65..4b8f450d72 100644
--- a/backend/tests/sql_export_test.go
+++ b/backend/tests/sql_export_test.go
@@ -4,112 +4,143 @@ import (
"bytes"
"context"
"fmt"
+ "io"
+ "strings"
"testing"
"connectrpc.com/connect"
"github.com/alexmullins/zip"
- "github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
- "google.golang.org/protobuf/testing/protocmp"
- "google.golang.org/protobuf/types/known/durationpb"
+ "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
func TestSQLExport(t *testing.T) {
tests := []struct {
- databaseName string
- dbType storepb.Engine
- prepareStatements string
- query string
- password string
- reset string
- export string
- queryResult []*v1pb.QueryResult
- resetResult []*v1pb.QueryResult
+ databaseName string
+ dbType storepb.Engine
+ prepareStatement string
+ exportTests []struct {
+ format v1pb.ExportFormat
+ statement string
+ password string
+ results []struct {
+ statement string
+ content string
+ }
+ }
}{
{
- databaseName: "Test1",
- dbType: storepb.Engine_MYSQL,
- prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",
- query: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",
- reset: "DELETE FROM tbl;",
- export: "SELECT * FROM Test1.tbl;",
- password: "123",
- queryResult: []*v1pb.QueryResult{
+ databaseName: "Test1",
+ dbType: storepb.Engine_MYSQL,
+ prepareStatement: `
+ CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));
+ INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');
+ `,
+ exportTests: []struct {
+ format v1pb.ExportFormat
+ statement string
+ password string
+ results []struct {
+ statement string
+ content string
+ }
+ }{
{
- ColumnNames: []string{"Affected Rows"},
- ColumnTypeNames: []string{"INT"},
- Rows: []*v1pb.QueryRow{
+ format: v1pb.ExportFormat_JSON,
+ password: "123",
+ statement: "SELECT * FROM Test1.tbl;",
+ results: []struct {
+ statement string
+ content string
+ }{
{
- Values: []*v1pb.RowValue{
- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
- },
+ statement: "SELECT * FROM Test1.tbl;",
+ content: `[
+ {
+ "gender": "0b00000000",
+ "height": "0b01111111",
+ "id": 1,
+ "name": "Alice"
+ }
+]`,
},
},
- Statement: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",
- RowsCount: 1,
- AllowExport: true,
},
- },
- resetResult: []*v1pb.QueryResult{
{
- ColumnNames: []string{"Affected Rows"},
- ColumnTypeNames: []string{"INT"},
- Rows: []*v1pb.QueryRow{
+ format: v1pb.ExportFormat_CSV,
+ statement: "SELECT * FROM Test1.tbl;",
+ results: []struct {
+ statement string
+ content string
+ }{
{
- Values: []*v1pb.RowValue{
- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
- },
+ statement: "SELECT * FROM Test1.tbl;",
+ content: "id,name,gender,height\n1,\"Alice\",\"\x00\",\"\x7f\"",
},
},
- RowsCount: 1,
- AllowExport: true,
},
},
},
{
- databaseName: "Test2",
- dbType: storepb.Engine_POSTGRES,
- prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",
- query: "INSERT INTO tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",
- reset: "DELETE FROM tbl;",
- export: "SELECT * FROM tbl;",
- password: "",
- queryResult: []*v1pb.QueryResult{
+ databaseName: "Test2",
+ dbType: storepb.Engine_POSTGRES,
+ prepareStatement: "CREATE TABLE tbl(id INT PRIMARY KEY);",
+ exportTests: []struct {
+ format v1pb.ExportFormat
+ statement string
+ password string
+ results []struct {
+ statement string
+ content string
+ }
+ }{
{
- ColumnNames: []string{"Affected Rows"},
- ColumnTypeNames: []string{"INT"},
- Rows: []*v1pb.QueryRow{
+ format: v1pb.ExportFormat_JSON,
+ statement: `
+ SELECT 1;
+ SELECT 2;
+ `,
+ results: []struct {
+ statement string
+ content string
+ }{
+ {
+ statement: "SELECT 1;",
+ content: `[
+ {
+ "?column?": 1
+ }
+]`,
+ },
{
- Values: []*v1pb.RowValue{
- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
- },
+ statement: "SELECT 2;",
+ content: `[
+ {
+ "?column?": 2
+ }
+]`,
},
},
- RowsCount: 1,
- AllowExport: true,
},
- },
- resetResult: []*v1pb.QueryResult{
{
- ColumnNames: []string{"Affected Rows"},
- ColumnTypeNames: []string{"INT"},
- Rows: []*v1pb.QueryRow{
+ format: v1pb.ExportFormat_CSV,
+ statement: "SELECT 1;",
+ results: []struct {
+ statement string
+ content string
+ }{
{
- Values: []*v1pb.RowValue{
- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
- },
+ statement: "SELECT 1;",
+ content: "?column?\n1",
},
},
- RowsCount: 1,
- AllowExport: true,
},
},
},
- // TODO(ed): test multiple queries.
}
t.Parallel()
@@ -173,6 +204,9 @@ func TestSQLExport(t *testing.T) {
err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, tt.databaseName, databaseOwner)
a.NoError(err)
+ instanceID, err := common.GetInstanceID(instance.Name)
+ a.NoError(err)
+
databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
Name: fmt.Sprintf("%s/databases/%s", instance.Name, tt.databaseName),
}))
@@ -183,7 +217,7 @@ func TestSQLExport(t *testing.T) {
Parent: ctl.project.Name,
Sheet: &v1pb.Sheet{
Title: "prepareStatements",
- Content: []byte(tt.prepareStatements),
+ Content: []byte(tt.prepareStatement),
},
}))
a.NoError(err)
@@ -196,44 +230,44 @@ func TestSQLExport(t *testing.T) {
err = ctl.changeDatabase(ctx, ctl.project, database, sheet, v1pb.MigrationType_DDL)
a.NoError(err)
- statement := tt.query
- results, err := ctl.adminQuery(ctx, database, statement)
- a.NoError(err)
- checkResults(a, tt.databaseName, statement, tt.queryResult, results)
-
- request := &v1pb.ExportRequest{
- Name: database.Name,
- Format: v1pb.ExportFormat_SQL,
- Limit: 1,
- Statement: tt.export,
- Password: tt.password,
- DataSourceId: dataSource.Id,
- }
- exportResp, err := ctl.sqlServiceClient.Export(ctx, connect.NewRequest(request))
- a.NoError(err)
- export := exportResp.Msg
+ for _, exportTest := range tt.exportTests {
+ request := &v1pb.ExportRequest{
+ Name: database.Name,
+ Format: exportTest.format,
+ Statement: exportTest.statement,
+ Password: exportTest.password,
+ DataSourceId: dataSource.Id,
+ }
+ exportResp, err := ctl.sqlServiceClient.Export(ctx, connect.NewRequest(request))
+ a.NoError(err)
+ export := exportResp.Msg
- statement = tt.reset
- results, err = ctl.adminQuery(ctx, database, statement)
- a.NoError(err)
- checkResults(a, tt.databaseName, statement, tt.resetResult, results)
+ reader := bytes.NewReader(export.Content)
+ zipReader, err := zip.NewReader(reader, int64(len(export.Content)))
+ a.NoError(err)
- reader := bytes.NewReader(export.Content)
- zipReader, err := zip.NewReader(reader, int64(len(export.Content)))
- a.NoError(err)
- a.Equal(len(tt.queryResult)*2, len(zipReader.File))
+ a.Equal(len(exportTest.results)*2, len(zipReader.File))
- // TODO(ed): test file content.
- }
-}
+ for i, compressedFile := range zipReader.File {
+ if exportTest.password != "" {
+ compressedFile.SetPassword(exportTest.password)
+ }
-func checkResults(a *require.Assertions, databaseName string, query string, affectedRows []*v1pb.QueryResult, results []*v1pb.QueryResult) {
- a.Equal(len(affectedRows), len(results))
- for idx, result := range results {
- a.Equal("", result.Error, "database %s: %s", databaseName, query)
- result.Latency = nil
- affectedRows[idx].Statement = query
- diff := cmp.Diff(affectedRows[idx], result, protocmp.Transform(), protocmp.IgnoreMessages(&durationpb.Duration{}))
- a.Empty(diff, "database %s: %s", databaseName, query)
+ expectFilenamePrefix := fmt.Sprintf("%s/%s/statement-%d", instanceID, tt.databaseName, (i/2)+1)
+ a.True(strings.HasPrefix(compressedFile.Name, expectFilenamePrefix))
+ expectResult := exportTest.results[i/2]
+
+ file, err := compressedFile.Open()
+ a.NoError(err)
+ content, err := io.ReadAll(file)
+ a.NoError(err)
+ if compressedFile.Name == fmt.Sprintf("%s.sql", expectFilenamePrefix) {
+ a.Equal(strings.TrimSpace(expectResult.statement), strings.TrimSpace(string(content)))
+ } else {
+ a.Equal(fmt.Sprintf("%s.result.%s", expectFilenamePrefix, strings.ToLower(request.Format.String())), compressedFile.Name)
+ a.Equal(expectResult.content, string(content))
+ }
+ }
+ }
}
}
diff --git a/backend/tests/sql_query_test.go b/backend/tests/sql_query_test.go
new file mode 100644
index 0000000000..772f4a035b
--- /dev/null
+++ b/backend/tests/sql_query_test.go
@@ -0,0 +1,209 @@
+package tests
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "connectrpc.com/connect"
+ "github.com/google/go-cmp/cmp"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/testing/protocmp"
+ "google.golang.org/protobuf/types/known/durationpb"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+)
+
+func TestSQLAdminQuery(t *testing.T) {
+ tests := []struct {
+ databaseName string
+ dbType storepb.Engine
+ prepareStatements string
+ query string
+ reset string
+ queryResult []*v1pb.QueryResult
+ resetResult []*v1pb.QueryResult
+ }{
+ {
+ databaseName: "Test1",
+ dbType: storepb.Engine_MYSQL,
+ prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",
+ query: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",
+ reset: "DELETE FROM tbl;",
+ queryResult: []*v1pb.QueryResult{
+ {
+ ColumnNames: []string{"Affected Rows"},
+ ColumnTypeNames: []string{"INT"},
+ Rows: []*v1pb.QueryRow{
+ {
+ Values: []*v1pb.RowValue{
+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
+ },
+ },
+ },
+ Statement: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",
+ RowsCount: 1,
+ AllowExport: true,
+ },
+ },
+ resetResult: []*v1pb.QueryResult{
+ {
+ ColumnNames: []string{"Affected Rows"},
+ ColumnTypeNames: []string{"INT"},
+ Rows: []*v1pb.QueryRow{
+ {
+ Values: []*v1pb.RowValue{
+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
+ },
+ },
+ },
+ RowsCount: 1,
+ AllowExport: true,
+ },
+ },
+ },
+ {
+ databaseName: "Test2",
+ dbType: storepb.Engine_POSTGRES,
+ prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",
+ query: "INSERT INTO tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",
+ reset: "DELETE FROM tbl;",
+ queryResult: []*v1pb.QueryResult{
+ {
+ ColumnNames: []string{"Affected Rows"},
+ ColumnTypeNames: []string{"INT"},
+ Rows: []*v1pb.QueryRow{
+ {
+ Values: []*v1pb.RowValue{
+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
+ },
+ },
+ },
+ RowsCount: 1,
+ AllowExport: true,
+ },
+ },
+ resetResult: []*v1pb.QueryResult{
+ {
+ ColumnNames: []string{"Affected Rows"},
+ ColumnTypeNames: []string{"INT"},
+ Rows: []*v1pb.QueryRow{
+ {
+ Values: []*v1pb.RowValue{
+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},
+ },
+ },
+ },
+ RowsCount: 1,
+ AllowExport: true,
+ },
+ },
+ },
+ }
+
+ t.Parallel()
+ a := require.New(t)
+ ctx := context.Background()
+ ctl := &controller{}
+ ctx, err := ctl.StartServerWithExternalPg(ctx)
+ a.NoError(err)
+ defer ctl.Close(ctx)
+
+ mysqlContainer, err := getMySQLContainer(ctx)
+ defer func() {
+ mysqlContainer.Close(ctx)
+ }()
+ a.NoError(err)
+
+ pgContainer, err := getPgContainer(ctx)
+ defer func() {
+ pgContainer.Close(ctx)
+ }()
+ a.NoError(err)
+
+ mysqlInstanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
+ InstanceId: generateRandomString("instance"),
+ Instance: &v1pb.Instance{
+ Title: "mysqlInstance",
+ Engine: v1pb.Engine_MYSQL,
+ Environment: stringPtr("environments/prod"),
+ Activation: true,
+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: mysqlContainer.host, Port: mysqlContainer.port, Username: "root", Password: "root-password", Id: "admin"}},
+ },
+ }))
+ a.NoError(err)
+ mysqlInstance := mysqlInstanceResp.Msg
+
+ pgInstanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{
+ InstanceId: generateRandomString("instance"),
+ Instance: &v1pb.Instance{
+ Title: "pgInstance",
+ Engine: v1pb.Engine_POSTGRES,
+ Environment: stringPtr("environments/prod"),
+ Activation: true,
+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: pgContainer.host, Port: pgContainer.port, Username: "postgres", Password: "root-password", Id: "admin"}},
+ },
+ }))
+ a.NoError(err)
+ pgInstance := pgInstanceResp.Msg
+
+ for _, tt := range tests {
+ var instance *v1pb.Instance
+ databaseOwner := ""
+ switch tt.dbType {
+ case storepb.Engine_MYSQL:
+ instance = mysqlInstance
+ case storepb.Engine_POSTGRES:
+ instance = pgInstance
+ databaseOwner = "postgres"
+ default:
+ a.FailNow("unsupported db type")
+ }
+ err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, tt.databaseName, databaseOwner)
+ a.NoError(err)
+
+ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{
+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, tt.databaseName),
+ }))
+ a.NoError(err)
+ database := databaseResp.Msg
+
+ sheetResp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{
+ Parent: ctl.project.Name,
+ Sheet: &v1pb.Sheet{
+ Title: "prepareStatements",
+ Content: []byte(tt.prepareStatements),
+ },
+ }))
+ a.NoError(err)
+ sheet := sheetResp.Msg
+
+ a.NotNil(database.InstanceResource)
+ a.Equal(1, len(database.InstanceResource.DataSources))
+
+ err = ctl.changeDatabase(ctx, ctl.project, database, sheet, v1pb.MigrationType_DDL)
+ a.NoError(err)
+
+ statement := tt.query
+ results, err := ctl.adminQuery(ctx, database, statement)
+ a.NoError(err)
+ checkResults(a, tt.databaseName, statement, tt.queryResult, results)
+
+ statement = tt.reset
+ results, err = ctl.adminQuery(ctx, database, statement)
+ a.NoError(err)
+ checkResults(a, tt.databaseName, statement, tt.resetResult, results)
+ }
+}
+
+func checkResults(a *require.Assertions, databaseName string, query string, affectedRows []*v1pb.QueryResult, results []*v1pb.QueryResult) {
+ a.Equal(len(affectedRows), len(results))
+ for idx, result := range results {
+ a.Equal("", result.Error, "database %s: %s", databaseName, query)
+ result.Latency = nil
+ affectedRows[idx].Statement = query
+ diff := cmp.Diff(affectedRows[idx], result, protocmp.Transform(), protocmp.IgnoreMessages(&durationpb.Duration{}))
+ a.Empty(diff, "database %s: %s", databaseName, query)
+ }
+}