Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: Security and performance optimizations (#18808)
Danny Xu4b11d95789f4
diff --git a/backend/plugin/db/clickhouse/dump.go b/backend/plugin/db/clickhouse/dump.goindex ece7dda34b..4e298a3a22 100644--- a/backend/plugin/db/clickhouse/dump.go+++ b/backend/plugin/db/clickhouse/dump.go@@ -9,18 +9,12 @@ import ("github.com/pkg/errors"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store")// Dump and restore.const (- databaseHeaderFmt = "" +- "--\n" +- "-- ClickHouse database structure for `%s`\n" +- "--\n"- useDatabaseFmt = "USE `%s`;\n\n"- tableStmtFmt = "" ++ tableStmtFmt = "" +"--\n" +"-- Table structure for `%s`\n" +"--\n" +@@ -47,108 +41,22 @@ func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSchreturn txn.Commit()}-// getDatabases gets all databases of an instance.-func getDatabases(ctx context.Context, txn *sql.Tx) ([]string, error) {- var dbNames []string- rows, err := txn.QueryContext(ctx, "SELECT name FROM system.databases")- if err != nil {- return nil, err- }- defer rows.Close()-- for rows.Next() {- var name string- if err := rows.Scan(&name); err != nil {- return nil, err- }- dbNames = append(dbNames, name)- }- if err := rows.Err(); err != nil {- return nil, err- }- return dbNames, nil-}-// dumpTxn will dump the input database. schemaOnly isn't supported yet and true by default.func dumpTxn(ctx context.Context, txn *sql.Tx, database string, out io.Writer) error {- // Find all dumpable databases- dbNames, err := getDatabases(ctx, txn)+ // Table and view statement.+ tables, err := getTables(ctx, txn, database)if err != nil {- return errors.Wrap(err, "failed to get databases")+ return errors.Wrapf(err, "failed to get tables of database %q", database)}-- var dumpableDBNames []string- if database != "" {- exist := false- for _, n := range dbNames {- if n == database {- exist = true- break- }- }- if !exist {- return common.Errorf(common.NotFound, "database %s not found", database)- }- dumpableDBNames = []string{database}- } else {- for _, dbName := range dbNames {- if systemDatabases[dbName] {- continue- }- dumpableDBNames = append(dumpableDBNames, dbName)- }- }-- for _, dbName := range dumpableDBNames {- // Include "USE DATABASE xxx" if dumping multiple databases.- if len(dumpableDBNames) > 1 {- // Database header.- header := fmt.Sprintf(databaseHeaderFmt, dbName)- if _, err := io.WriteString(out, header); err != nil {- return err- }- dbStmt, err := getDatabaseStmt(ctx, txn, dbName)- if err != nil {- return errors.Wrapf(err, "failed to get database %q", dbName)- }- if _, err := io.WriteString(out, dbStmt); err != nil {- return err- }- // Use database statement.- useStmt := fmt.Sprintf(useDatabaseFmt, dbName)- if _, err := io.WriteString(out, useStmt); err != nil {- return err- }- }-- // Table and view statement.- tables, err := getTables(ctx, txn, dbName)- if err != nil {- return errors.Wrapf(err, "failed to get tables of database %q", dbName)- }- for _, tbl := range tables {- if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.statement)); err != nil {- return err- }+ for _, tbl := range tables {+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.statement)); err != nil {+ return err}}return nil}-// getDatabaseStmt gets the create statement of a database.-func getDatabaseStmt(ctx context.Context, txn *sql.Tx, dbName string) (string, error) {- query := fmt.Sprintf("SHOW CREATE DATABASE IF NOT EXISTS %s;", dbName)- var stmt, unused string- if err := txn.QueryRowContext(ctx, query).Scan(&unused, &stmt); err != nil {- if err == sql.ErrNoRows {- return "", common.FormatDBErrorEmptyRowWithQuery(query)- }- return "", err- }- return fmt.Sprintf("%s;\n", stmt), nil-}-// tableSchema describes the schema of a table or view.type tableSchema struct {name string@@ -159,8 +67,8 @@ type tableSchema struct {// getTables gets all tables of a database.func getTables(ctx context.Context, txn *sql.Tx, dbName string) ([]*tableSchema, error) {var tables []*tableSchema- query := fmt.Sprintf("SELECT name, engine, create_table_query FROM system.tables WHERE database='%s';", dbName)- rows, err := txn.QueryContext(ctx, query)+ query := "SELECT name, engine, create_table_query FROM system.tables WHERE database=?"+ rows, err := txn.QueryContext(ctx, query, dbName)if err != nil {return nil, err}diff --git a/backend/plugin/db/snowflake/dump.go b/backend/plugin/db/snowflake/dump.goindex 5a9e4263fb..29a24c9923 100644--- a/backend/plugin/db/snowflake/dump.go+++ b/backend/plugin/db/snowflake/dump.go@@ -7,21 +7,11 @@ import ("io""strings"- "github.com/pkg/errors"-storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db/util")-// Dump and restore.-const (- databaseHeaderFmt = "" +- "--\n" +- "-- Snowflake database structure for %s\n" +- "--\n"-)--// Dump dumps the database.+// Dump dumps database.func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSchemaMetadata) error {txn, err := d.db.BeginTx(ctx, &sql.TxOptions{})if err != nil {@@ -29,51 +19,7 @@ func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSch}defer txn.Rollback()- if err := dumpTxn(ctx, txn, d.databaseName, out); err != nil {- return err- }-- err = txn.Commit()- return err-}--// dumpTxn will dump the input database. schemaOnly isn't supported yet and true by default.-func dumpTxn(ctx context.Context, txn *sql.Tx, database string, out io.Writer) error {- // Find all dumpable databases- var dumpableDBNames []string- if database != "" {- dumpableDBNames = []string{database}- } else {- var err error- dumpableDBNames, err = getDatabasesTxn(ctx, txn)- if err != nil {- return errors.Wrap(err, "failed to get databases")- }- }-- for _, dbName := range dumpableDBNames {- // includeCreateDatabaseStmt should be false if dumping a single database.- dumpSingleDatabase := len(dumpableDBNames) == 1- if err := dumpOneDatabase(ctx, txn, dbName, out, dumpSingleDatabase); err != nil {- return err- }- }-- return nil-}--// dumpOneDatabase will dump the database DDL schema for a database.-// Note: this operation is not supported on shared databases, e.g. SNOWFLAKE_SAMPLE_DATA.-func dumpOneDatabase(ctx context.Context, txn *sql.Tx, database string, out io.Writer, dumpSingleDatabase bool) error {- if !dumpSingleDatabase {- // Database header.- header := fmt.Sprintf(databaseHeaderFmt, database)- if _, err := io.WriteString(out, header); err != nil {- return err- }- }-- query := fmt.Sprintf(`SELECT GET_DDL('DATABASE', '"%s"', true)`, database)+ query := fmt.Sprintf(`SELECT GET_DDL('DATABASE', '"%s"', true)`, d.databaseName)rows, err := txn.QueryContext(ctx, query)if err != nil {return util.FormatErrorWithQuery(err, query)@@ -82,9 +28,7 @@ func dumpOneDatabase(ctx context.Context, txn *sql.Tx, database string, out io.Wvar databaseDDL stringfor rows.Next() {- if err := rows.Scan(- &databaseDDL,- ); err != nil {+ if err := rows.Scan(&databaseDDL); err != nil {return err}}@@ -92,37 +36,35 @@ func dumpOneDatabase(ctx context.Context, txn *sql.Tx, database string, out io.Wreturn err}- // Transform1: if dumpSingleDatabase, we should remove `create or replace database` statement.- if dumpSingleDatabase {- lines := strings.Split(databaseDDL, "\n")- if len(lines) >= 2 {- lines = lines[2:]- }- databaseDDL = strings.Join(lines, "\n")+ // Transform1: remove `create or replace database` statement.+ lines := strings.Split(databaseDDL, "\n")+ if len(lines) >= 2 {+ lines = lines[2:]}+ databaseDDL = strings.Join(lines, "\n")// Transform2: remove "create or replace schema PUBLIC;\n\n" because it's created by default.- schemaStmt := fmt.Sprintf("create or replace schema %s.PUBLIC;", database)+ schemaStmt := fmt.Sprintf("create or replace schema %s.PUBLIC;", d.databaseName)databaseDDL = strings.ReplaceAll(databaseDDL, schemaStmt+"\n\n", "")// If this is the last statement.databaseDDL = strings.ReplaceAll(databaseDDL, schemaStmt, "")- var lines []string+ var transformedLines []stringfor _, line := range strings.Split(databaseDDL, "\n") {if strings.HasPrefix(strings.ToLower(line), "create ") {// Transform3: Remove "DEMO_DB." quantifier.- line = strings.ReplaceAll(line, fmt.Sprintf(" %s.", database), " ")+ line = strings.ReplaceAll(line, fmt.Sprintf(" %s.", d.databaseName), " ")// Transform4 (Important!): replace all `create or replace ` with `create ` to not break existing schema by any chance.line = strings.ReplaceAll(line, "create or replace ", "create ")}- lines = append(lines, line)+ transformedLines = append(transformedLines, line)}- databaseDDL = strings.Join(lines, "\n")+ databaseDDL = strings.Join(transformedLines, "\n")if _, err := io.WriteString(out, databaseDDL); err != nil {return err}- return nil+ return txn.Commit()}diff --git a/backend/plugin/db/spanner/dump.go b/backend/plugin/db/spanner/dump.goindex 65300a9534..6c630610e6 100644--- a/backend/plugin/db/spanner/dump.go+++ b/backend/plugin/db/spanner/dump.go@@ -6,47 +6,23 @@ import ("io""cloud.google.com/go/spanner/admin/database/apiv1/databasepb"- "github.com/pkg/errors"storepb "github.com/bytebase/bytebase/backend/generated-go/store")-// Dump dumps the database.+// Dump dumps database.func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSchemaMetadata) error {- instance, err := d.SyncInstance(ctx)+ resp, err := d.dbClient.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{+ Database: getDSN(d.config.DataSource.Host, d.databaseName),+ })if err != nil {return err}- var dumpableDBNames []string- if d.databaseName != "" {- exist := false- for _, db := range instance.Databases {- if db.Name == d.databaseName {- exist = true- break- }- }- if !exist {- return errors.Errorf("database %q not found", d.databaseName)- }- dumpableDBNames = []string{d.databaseName}- } else {- for _, db := range instance.Databases {- dumpableDBNames = append(dumpableDBNames, db.Name)- }- }- for _, db := range dumpableDBNames {- resp, err := d.dbClient.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{- Database: getDSN(d.config.DataSource.Host, db),- })- if err != nil {++ for _, stmt := range resp.Statements {+ if _, err := io.WriteString(out, fmt.Sprintf("%s;\n", stmt)); err != nil {return err}- for _, stmt := range resp.Statements {- if _, err := io.WriteString(out, fmt.Sprintf("%s;\n", stmt)); err != nil {- return err- }- }}return nildiff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.goindex 90bfaa3571..996ce7e8a0 100644--- a/backend/plugin/db/starrocks/dump.go+++ b/backend/plugin/db/starrocks/dump.go@@ -239,8 +239,8 @@ type eventSchema struct {// getTables gets all tables of a database using the provided database instance.func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema, error) {var tables []*TableSchema- query := fmt.Sprintf("SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '%s';", dbName)- rows, err := db.Query(query)+ query := "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?"+ rows, err := db.Query(query, dbName)if err != nil {return nil, err}diff --git a/backend/plugin/parser/standard/split_test.go b/backend/plugin/parser/standard/split_test.goindex fca3982d71..7564e7c65d 100644--- a/backend/plugin/parser/standard/split_test.go+++ b/backend/plugin/parser/standard/split_test.go@@ -1,11 +1,11 @@package standardimport (+ "crypto/rand""fmt"- "math/rand"+ "math/big""strings""testing"- "time""github.com/stretchr/testify/require"@@ -19,11 +19,17 @@ func TestStandardSplitSQL(t *testing.T) {}func generateOneMBInsert() string {- var rand = rand.New(rand.NewSource(time.Now().UnixNano()))letterList := []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")b := make([]byte, 1024*1024)for i := range b {- b[i] = letterList[rand.Intn(len(letterList))]+ // Use crypto/rand for secure random generation+ randomIndex, err := rand.Int(rand.Reader, big.NewInt(int64(len(letterList))))+ if err != nil {+ // Fallback to a simple pattern if crypto/rand fails+ b[i] = letterList[i%26]+ } else {+ b[i] = letterList[randomIndex.Int64()]+ }}return fmt.Sprintf("INSERT INTO t values('%s')", string(b))}diff --git a/backend/runner/plancheck/ghost_sync_executor.go b/backend/runner/plancheck/ghost_sync_executor.goindex 121117d664..780fde4f69 100644--- a/backend/runner/plancheck/ghost_sync_executor.go+++ b/backend/runner/plancheck/ghost_sync_executor.go@@ -2,9 +2,10 @@ package plancheckimport ("context"+ "crypto/rand""fmt""log/slog"- "math/rand"+ "math/big""strings""time"@@ -136,7 +137,12 @@ func (e *GhostSyncExecutor) RunForTarget(ctx context.Context, target *CheckTarge}, nil}- migrationContext, err := ghost.NewMigrationContext(ctx, rand.Intn(10000000), database, adminDataSource, tableName, fmt.Sprintf("_dryrun_%d", time.Now().Unix()), statement, true, target.GhostFlags, 20000000)+ // Generate secure random number for migration context+ randomInt, err := rand.Int(rand.Reader, big.NewInt(10000000))+ if err != nil {+ return nil, common.Wrapf(err, common.Internal, "failed to generate secure random number")+ }+ migrationContext, err := ghost.NewMigrationContext(ctx, int(randomInt.Int64()), database, adminDataSource, tableName, fmt.Sprintf("_dryrun_%d", time.Now().Unix()), statement, true, target.GhostFlags, 20000000)if err != nil {return nil, common.Wrapf(err, common.Internal, "failed to create migration context")}