Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: implement tests for export (#17721)
ecmadao72144d3e1a36
diff --git a/backend/api/v1/sql_service.go b/backend/api/v1/sql_service.goindex adf50911f0..0b13e02757 100644--- a/backend/api/v1/sql_service.go+++ b/backend/api/v1/sql_service.go@@ -974,14 +974,14 @@ func DoExport(filename := fmt.Sprintf("%s/%s/statement-%d", database.InstanceID, database.DatabaseName, i+1)pendingEncryptContents = append(pendingEncryptContents,- &encryptContent{- Name: fmt.Sprintf("%s.result.%s", filename, strings.ToLower(request.Format.String())),- Content: content,- },&encryptContent{Name: fmt.Sprintf("%s.sql", filename),Content: []byte(result.Statement),},+ &encryptContent{+ Name: fmt.Sprintf("%s.result.%s", filename, strings.ToLower(request.Format.String())),+ Content: content,+ },)}diff --git a/backend/api/v1/sql_service_export_utils.go b/backend/api/v1/sql_service_export_utils.goindex 746033b254..5fb40a07fb 100644--- a/backend/api/v1/sql_service_export_utils.go+++ b/backend/api/v1/sql_service_export_utils.go@@ -4,6 +4,7 @@ import ("bytes""context""encoding/base64"+ "encoding/json""fmt""strconv""strings"@@ -746,7 +747,20 @@ func exportJSON(result *v1pb.QueryResult) ([]byte, error) {if _, err := buf.WriteString("]"); err != nil {return nil, err}- return buf.Bytes(), nil++ var data any+ err := json.Unmarshal(buf.Bytes(), &data)+ if err != nil {+ return nil, errors.Errorf("failed to unmarshalling JSON with error: %v", err)+ }++ // Marshal the data back into a pretty-printed JSON string+ // The second argument "" is the prefix for each line, and " " is the indentation string.+ formattedJSONBytes, err := json.MarshalIndent(data, "", " ")+ if err != nil {+ return nil, errors.Errorf("failed to marshalling JSON with error: %v", err)+ }+ return formattedJSONBytes, nil}func convertValueToStringInJSON(value *v1pb.RowValue) string {diff --git a/backend/api/v1/sql_service_export_utils_test.go b/backend/api/v1/sql_service_export_utils_test.goindex df077bdd05..2c511db69c 100644--- a/backend/api/v1/sql_service_export_utils_test.go+++ b/backend/api/v1/sql_service_export_utils_test.go@@ -263,7 +263,11 @@ func TestExportJSON(t *testing.T) {},},},- want: "[{\"a\":\"0b00000101\"}]",+ want: `[+ {+ "a": "0b00000101"+ }+]`,},{result: &v1pb.QueryResult{@@ -287,7 +291,14 @@ func TestExportJSON(t *testing.T) {},},},- want: "[{\"id\":1,\"name\":\"Alice\",\"email\":\"[email protected]\",\"age\":20}]",+ want: `[+ {+ "age": 20,+ "email": "[email protected]",+ "id": 1,+ "name": "Alice"+ }+]`,},}diff --git a/backend/tests/sql_export_test.go b/backend/tests/sql_export_test.goindex 7a193f9e65..4b8f450d72 100644--- a/backend/tests/sql_export_test.go+++ b/backend/tests/sql_export_test.go@@ -4,112 +4,143 @@ import ("bytes""context""fmt"+ "io"+ "strings""testing""connectrpc.com/connect""github.com/alexmullins/zip"- "github.com/google/go-cmp/cmp""github.com/stretchr/testify/require"- "google.golang.org/protobuf/testing/protocmp"- "google.golang.org/protobuf/types/known/durationpb"+ "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")func TestSQLExport(t *testing.T) {tests := []struct {- databaseName string- dbType storepb.Engine- prepareStatements string- query string- password string- reset string- export string- queryResult []*v1pb.QueryResult- resetResult []*v1pb.QueryResult+ databaseName string+ dbType storepb.Engine+ prepareStatement string+ exportTests []struct {+ format v1pb.ExportFormat+ statement string+ password string+ results []struct {+ statement string+ content string+ }+ }}{{- databaseName: "Test1",- dbType: storepb.Engine_MYSQL,- prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",- query: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",- reset: "DELETE FROM tbl;",- export: "SELECT * FROM Test1.tbl;",- password: "123",- queryResult: []*v1pb.QueryResult{+ databaseName: "Test1",+ dbType: storepb.Engine_MYSQL,+ prepareStatement: `+ CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));+ INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');+ `,+ exportTests: []struct {+ format v1pb.ExportFormat+ statement string+ password string+ results []struct {+ statement string+ content string+ }+ }{{- ColumnNames: []string{"Affected Rows"},- ColumnTypeNames: []string{"INT"},- Rows: []*v1pb.QueryRow{+ format: v1pb.ExportFormat_JSON,+ password: "123",+ statement: "SELECT * FROM Test1.tbl;",+ results: []struct {+ statement string+ content string+ }{{- Values: []*v1pb.RowValue{- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},- },+ statement: "SELECT * FROM Test1.tbl;",+ content: `[+ {+ "gender": "0b00000000",+ "height": "0b01111111",+ "id": 1,+ "name": "Alice"+ }+]`,},},- Statement: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",- RowsCount: 1,- AllowExport: true,},- },- resetResult: []*v1pb.QueryResult{{- ColumnNames: []string{"Affected Rows"},- ColumnTypeNames: []string{"INT"},- Rows: []*v1pb.QueryRow{+ format: v1pb.ExportFormat_CSV,+ statement: "SELECT * FROM Test1.tbl;",+ results: []struct {+ statement string+ content string+ }{{- Values: []*v1pb.RowValue{- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},- },+ statement: "SELECT * FROM Test1.tbl;",+ content: "id,name,gender,height\n1,\"Alice\",\"\x00\",\"\x7f\"",},},- RowsCount: 1,- AllowExport: true,},},},{- databaseName: "Test2",- dbType: storepb.Engine_POSTGRES,- prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",- query: "INSERT INTO tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",- reset: "DELETE FROM tbl;",- export: "SELECT * FROM tbl;",- password: "",- queryResult: []*v1pb.QueryResult{+ databaseName: "Test2",+ dbType: storepb.Engine_POSTGRES,+ prepareStatement: "CREATE TABLE tbl(id INT PRIMARY KEY);",+ exportTests: []struct {+ format v1pb.ExportFormat+ statement string+ password string+ results []struct {+ statement string+ content string+ }+ }{{- ColumnNames: []string{"Affected Rows"},- ColumnTypeNames: []string{"INT"},- Rows: []*v1pb.QueryRow{+ format: v1pb.ExportFormat_JSON,+ statement: `+ SELECT 1;+ SELECT 2;+ `,+ results: []struct {+ statement string+ content string+ }{+ {+ statement: "SELECT 1;",+ content: `[+ {+ "?column?": 1+ }+]`,+ },{- Values: []*v1pb.RowValue{- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},- },+ statement: "SELECT 2;",+ content: `[+ {+ "?column?": 2+ }+]`,},},- RowsCount: 1,- AllowExport: true,},- },- resetResult: []*v1pb.QueryResult{{- ColumnNames: []string{"Affected Rows"},- ColumnTypeNames: []string{"INT"},- Rows: []*v1pb.QueryRow{+ format: v1pb.ExportFormat_CSV,+ statement: "SELECT 1;",+ results: []struct {+ statement string+ content string+ }{{- Values: []*v1pb.RowValue{- {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},- },+ statement: "SELECT 1;",+ content: "?column?\n1",},},- RowsCount: 1,- AllowExport: true,},},},- // TODO(ed): test multiple queries.}t.Parallel()@@ -173,6 +204,9 @@ func TestSQLExport(t *testing.T) {err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, tt.databaseName, databaseOwner)a.NoError(err)+ instanceID, err := common.GetInstanceID(instance.Name)+ a.NoError(err)+databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{Name: fmt.Sprintf("%s/databases/%s", instance.Name, tt.databaseName),}))@@ -183,7 +217,7 @@ func TestSQLExport(t *testing.T) {Parent: ctl.project.Name,Sheet: &v1pb.Sheet{Title: "prepareStatements",- Content: []byte(tt.prepareStatements),+ Content: []byte(tt.prepareStatement),},}))a.NoError(err)@@ -196,44 +230,44 @@ func TestSQLExport(t *testing.T) {err = ctl.changeDatabase(ctx, ctl.project, database, sheet, v1pb.MigrationType_DDL)a.NoError(err)- statement := tt.query- results, err := ctl.adminQuery(ctx, database, statement)- a.NoError(err)- checkResults(a, tt.databaseName, statement, tt.queryResult, results)-- request := &v1pb.ExportRequest{- Name: database.Name,- Format: v1pb.ExportFormat_SQL,- Limit: 1,- Statement: tt.export,- Password: tt.password,- DataSourceId: dataSource.Id,- }- exportResp, err := ctl.sqlServiceClient.Export(ctx, connect.NewRequest(request))- a.NoError(err)- export := exportResp.Msg+ for _, exportTest := range tt.exportTests {+ request := &v1pb.ExportRequest{+ Name: database.Name,+ Format: exportTest.format,+ Statement: exportTest.statement,+ Password: exportTest.password,+ DataSourceId: dataSource.Id,+ }+ exportResp, err := ctl.sqlServiceClient.Export(ctx, connect.NewRequest(request))+ a.NoError(err)+ export := exportResp.Msg- statement = tt.reset- results, err = ctl.adminQuery(ctx, database, statement)- a.NoError(err)- checkResults(a, tt.databaseName, statement, tt.resetResult, results)+ reader := bytes.NewReader(export.Content)+ zipReader, err := zip.NewReader(reader, int64(len(export.Content)))+ a.NoError(err)- reader := bytes.NewReader(export.Content)- zipReader, err := zip.NewReader(reader, int64(len(export.Content)))- a.NoError(err)- a.Equal(len(tt.queryResult)*2, len(zipReader.File))+ a.Equal(len(exportTest.results)*2, len(zipReader.File))- // TODO(ed): test file content.- }-}+ for i, compressedFile := range zipReader.File {+ if exportTest.password != "" {+ compressedFile.SetPassword(exportTest.password)+ }-func checkResults(a *require.Assertions, databaseName string, query string, affectedRows []*v1pb.QueryResult, results []*v1pb.QueryResult) {- a.Equal(len(affectedRows), len(results))- for idx, result := range results {- a.Equal("", result.Error, "database %s: %s", databaseName, query)- result.Latency = nil- affectedRows[idx].Statement = query- diff := cmp.Diff(affectedRows[idx], result, protocmp.Transform(), protocmp.IgnoreMessages(&durationpb.Duration{}))- a.Empty(diff, "database %s: %s", databaseName, query)+ expectFilenamePrefix := fmt.Sprintf("%s/%s/statement-%d", instanceID, tt.databaseName, (i/2)+1)+ a.True(strings.HasPrefix(compressedFile.Name, expectFilenamePrefix))+ expectResult := exportTest.results[i/2]++ file, err := compressedFile.Open()+ a.NoError(err)+ content, err := io.ReadAll(file)+ a.NoError(err)+ if compressedFile.Name == fmt.Sprintf("%s.sql", expectFilenamePrefix) {+ a.Equal(strings.TrimSpace(expectResult.statement), strings.TrimSpace(string(content)))+ } else {+ a.Equal(fmt.Sprintf("%s.result.%s", expectFilenamePrefix, strings.ToLower(request.Format.String())), compressedFile.Name)+ a.Equal(expectResult.content, string(content))+ }+ }+ }}}diff --git a/backend/tests/sql_query_test.go b/backend/tests/sql_query_test.gonew file mode 100644index 0000000000..772f4a035b--- /dev/null+++ b/backend/tests/sql_query_test.go@@ -0,0 +1,209 @@+package tests++import (+ "context"+ "fmt"+ "testing"++ "connectrpc.com/connect"+ "github.com/google/go-cmp/cmp"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/testing/protocmp"+ "google.golang.org/protobuf/types/known/durationpb"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+)++func TestSQLAdminQuery(t *testing.T) {+ tests := []struct {+ databaseName string+ dbType storepb.Engine+ prepareStatements string+ query string+ reset string+ queryResult []*v1pb.QueryResult+ resetResult []*v1pb.QueryResult+ }{+ {+ databaseName: "Test1",+ dbType: storepb.Engine_MYSQL,+ prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",+ query: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",+ reset: "DELETE FROM tbl;",+ queryResult: []*v1pb.QueryResult{+ {+ ColumnNames: []string{"Affected Rows"},+ ColumnTypeNames: []string{"INT"},+ Rows: []*v1pb.QueryRow{+ {+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},+ },+ },+ },+ Statement: "INSERT INTO Test1.tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",+ RowsCount: 1,+ AllowExport: true,+ },+ },+ resetResult: []*v1pb.QueryResult{+ {+ ColumnNames: []string{"Affected Rows"},+ ColumnTypeNames: []string{"INT"},+ Rows: []*v1pb.QueryRow{+ {+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},+ },+ },+ },+ RowsCount: 1,+ AllowExport: true,+ },+ },+ },+ {+ databaseName: "Test2",+ dbType: storepb.Engine_POSTGRES,+ prepareStatements: "CREATE TABLE tbl(id INT PRIMARY KEY, name VARCHAR(64), gender BIT(1), height BIT(8));",+ query: "INSERT INTO tbl (id, name, gender, height) VALUES(1, 'Alice', B'0', B'01111111');",+ reset: "DELETE FROM tbl;",+ queryResult: []*v1pb.QueryResult{+ {+ ColumnNames: []string{"Affected Rows"},+ ColumnTypeNames: []string{"INT"},+ Rows: []*v1pb.QueryRow{+ {+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},+ },+ },+ },+ RowsCount: 1,+ AllowExport: true,+ },+ },+ resetResult: []*v1pb.QueryResult{+ {+ ColumnNames: []string{"Affected Rows"},+ ColumnTypeNames: []string{"INT"},+ Rows: []*v1pb.QueryRow{+ {+ Values: []*v1pb.RowValue{+ {Kind: &v1pb.RowValue_Int64Value{Int64Value: 1}},+ },+ },+ },+ RowsCount: 1,+ AllowExport: true,+ },+ },+ },+ }++ t.Parallel()+ a := require.New(t)+ ctx := context.Background()+ ctl := &controller{}+ ctx, err := ctl.StartServerWithExternalPg(ctx)+ a.NoError(err)+ defer ctl.Close(ctx)++ mysqlContainer, err := getMySQLContainer(ctx)+ defer func() {+ mysqlContainer.Close(ctx)+ }()+ a.NoError(err)++ pgContainer, err := getPgContainer(ctx)+ defer func() {+ pgContainer.Close(ctx)+ }()+ a.NoError(err)++ mysqlInstanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("instance"),+ Instance: &v1pb.Instance{+ Title: "mysqlInstance",+ Engine: v1pb.Engine_MYSQL,+ Environment: stringPtr("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: mysqlContainer.host, Port: mysqlContainer.port, Username: "root", Password: "root-password", Id: "admin"}},+ },+ }))+ a.NoError(err)+ mysqlInstance := mysqlInstanceResp.Msg++ pgInstanceResp, err := ctl.instanceServiceClient.CreateInstance(ctx, connect.NewRequest(&v1pb.CreateInstanceRequest{+ InstanceId: generateRandomString("instance"),+ Instance: &v1pb.Instance{+ Title: "pgInstance",+ Engine: v1pb.Engine_POSTGRES,+ Environment: stringPtr("environments/prod"),+ Activation: true,+ DataSources: []*v1pb.DataSource{{Type: v1pb.DataSourceType_ADMIN, Host: pgContainer.host, Port: pgContainer.port, Username: "postgres", Password: "root-password", Id: "admin"}},+ },+ }))+ a.NoError(err)+ pgInstance := pgInstanceResp.Msg++ for _, tt := range tests {+ var instance *v1pb.Instance+ databaseOwner := ""+ switch tt.dbType {+ case storepb.Engine_MYSQL:+ instance = mysqlInstance+ case storepb.Engine_POSTGRES:+ instance = pgInstance+ databaseOwner = "postgres"+ default:+ a.FailNow("unsupported db type")+ }+ err = ctl.createDatabaseV2(ctx, ctl.project, instance, nil /* environment */, tt.databaseName, databaseOwner)+ a.NoError(err)++ databaseResp, err := ctl.databaseServiceClient.GetDatabase(ctx, connect.NewRequest(&v1pb.GetDatabaseRequest{+ Name: fmt.Sprintf("%s/databases/%s", instance.Name, tt.databaseName),+ }))+ a.NoError(err)+ database := databaseResp.Msg++ sheetResp, err := ctl.sheetServiceClient.CreateSheet(ctx, connect.NewRequest(&v1pb.CreateSheetRequest{+ Parent: ctl.project.Name,+ Sheet: &v1pb.Sheet{+ Title: "prepareStatements",+ Content: []byte(tt.prepareStatements),+ },+ }))+ a.NoError(err)+ sheet := sheetResp.Msg++ a.NotNil(database.InstanceResource)+ a.Equal(1, len(database.InstanceResource.DataSources))++ err = ctl.changeDatabase(ctx, ctl.project, database, sheet, v1pb.MigrationType_DDL)+ a.NoError(err)++ statement := tt.query+ results, err := ctl.adminQuery(ctx, database, statement)+ a.NoError(err)+ checkResults(a, tt.databaseName, statement, tt.queryResult, results)++ statement = tt.reset+ results, err = ctl.adminQuery(ctx, database, statement)+ a.NoError(err)+ checkResults(a, tt.databaseName, statement, tt.resetResult, results)+ }+}++func checkResults(a *require.Assertions, databaseName string, query string, affectedRows []*v1pb.QueryResult, results []*v1pb.QueryResult) {+ a.Equal(len(affectedRows), len(results))+ for idx, result := range results {+ a.Equal("", result.Error, "database %s: %s", databaseName, query)+ result.Latency = nil+ affectedRows[idx].Statement = query+ diff := cmp.Diff(affectedRows[idx], result, protocmp.Transform(), protocmp.IgnoreMessages(&durationpb.Duration{}))+ a.Empty(diff, "database %s: %s", databaseName, query)+ }+}