Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: Security and performance optimizations (#18808)

Danny Xu4b11d95789f4
diff --git a/backend/plugin/db/clickhouse/dump.go b/backend/plugin/db/clickhouse/dump.go
index ece7dda34b..4e298a3a22 100644
--- a/backend/plugin/db/clickhouse/dump.go
+++ b/backend/plugin/db/clickhouse/dump.go
@@ -9,18 +9,12 @@ import (
"github.com/pkg/errors"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
// Dump and restore.
const (
- databaseHeaderFmt = "" +
- "--\n" +
- "-- ClickHouse database structure for `%s`\n" +
- "--\n"
- useDatabaseFmt = "USE `%s`;\n\n"
- tableStmtFmt = "" +
+ tableStmtFmt = "" +
"--\n" +
"-- Table structure for `%s`\n" +
"--\n" +
@@ -47,108 +41,22 @@ func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSch
return txn.Commit()
}
-// getDatabases gets all databases of an instance.
-func getDatabases(ctx context.Context, txn *sql.Tx) ([]string, error) {
- var dbNames []string
- rows, err := txn.QueryContext(ctx, "SELECT name FROM system.databases")
- if err != nil {
- return nil, err
- }
- defer rows.Close()
-
- for rows.Next() {
- var name string
- if err := rows.Scan(&name); err != nil {
- return nil, err
- }
- dbNames = append(dbNames, name)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
- return dbNames, nil
-}
-
// dumpTxn will dump the input database. schemaOnly isn't supported yet and true by default.
func dumpTxn(ctx context.Context, txn *sql.Tx, database string, out io.Writer) error {
- // Find all dumpable databases
- dbNames, err := getDatabases(ctx, txn)
+ // Table and view statement.
+ tables, err := getTables(ctx, txn, database)
if err != nil {
- return errors.Wrap(err, "failed to get databases")
+ return errors.Wrapf(err, "failed to get tables of database %q", database)
}
-
- var dumpableDBNames []string
- if database != "" {
- exist := false
- for _, n := range dbNames {
- if n == database {
- exist = true
- break
- }
- }
- if !exist {
- return common.Errorf(common.NotFound, "database %s not found", database)
- }
- dumpableDBNames = []string{database}
- } else {
- for _, dbName := range dbNames {
- if systemDatabases[dbName] {
- continue
- }
- dumpableDBNames = append(dumpableDBNames, dbName)
- }
- }
-
- for _, dbName := range dumpableDBNames {
- // Include "USE DATABASE xxx" if dumping multiple databases.
- if len(dumpableDBNames) > 1 {
- // Database header.
- header := fmt.Sprintf(databaseHeaderFmt, dbName)
- if _, err := io.WriteString(out, header); err != nil {
- return err
- }
- dbStmt, err := getDatabaseStmt(ctx, txn, dbName)
- if err != nil {
- return errors.Wrapf(err, "failed to get database %q", dbName)
- }
- if _, err := io.WriteString(out, dbStmt); err != nil {
- return err
- }
- // Use database statement.
- useStmt := fmt.Sprintf(useDatabaseFmt, dbName)
- if _, err := io.WriteString(out, useStmt); err != nil {
- return err
- }
- }
-
- // Table and view statement.
- tables, err := getTables(ctx, txn, dbName)
- if err != nil {
- return errors.Wrapf(err, "failed to get tables of database %q", dbName)
- }
- for _, tbl := range tables {
- if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.statement)); err != nil {
- return err
- }
+ for _, tbl := range tables {
+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.statement)); err != nil {
+ return err
}
}
return nil
}
-// getDatabaseStmt gets the create statement of a database.
-func getDatabaseStmt(ctx context.Context, txn *sql.Tx, dbName string) (string, error) {
- query := fmt.Sprintf("SHOW CREATE DATABASE IF NOT EXISTS %s;", dbName)
- var stmt, unused string
- if err := txn.QueryRowContext(ctx, query).Scan(&unused, &stmt); err != nil {
- if err == sql.ErrNoRows {
- return "", common.FormatDBErrorEmptyRowWithQuery(query)
- }
- return "", err
- }
- return fmt.Sprintf("%s;\n", stmt), nil
-}
-
// tableSchema describes the schema of a table or view.
type tableSchema struct {
name string
@@ -159,8 +67,8 @@ type tableSchema struct {
// getTables gets all tables of a database.
func getTables(ctx context.Context, txn *sql.Tx, dbName string) ([]*tableSchema, error) {
var tables []*tableSchema
- query := fmt.Sprintf("SELECT name, engine, create_table_query FROM system.tables WHERE database='%s';", dbName)
- rows, err := txn.QueryContext(ctx, query)
+ query := "SELECT name, engine, create_table_query FROM system.tables WHERE database=?"
+ rows, err := txn.QueryContext(ctx, query, dbName)
if err != nil {
return nil, err
}
diff --git a/backend/plugin/db/snowflake/dump.go b/backend/plugin/db/snowflake/dump.go
index 5a9e4263fb..29a24c9923 100644
--- a/backend/plugin/db/snowflake/dump.go
+++ b/backend/plugin/db/snowflake/dump.go
@@ -7,21 +7,11 @@ import (
"io"
"strings"
- "github.com/pkg/errors"
-
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db/util"
)
-// Dump and restore.
-const (
- databaseHeaderFmt = "" +
- "--\n" +
- "-- Snowflake database structure for %s\n" +
- "--\n"
-)
-
-// Dump dumps the database.
+// Dump dumps database.
func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSchemaMetadata) error {
txn, err := d.db.BeginTx(ctx, &sql.TxOptions{})
if err != nil {
@@ -29,51 +19,7 @@ func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSch
}
defer txn.Rollback()
- if err := dumpTxn(ctx, txn, d.databaseName, out); err != nil {
- return err
- }
-
- err = txn.Commit()
- return err
-}
-
-// dumpTxn will dump the input database. schemaOnly isn't supported yet and true by default.
-func dumpTxn(ctx context.Context, txn *sql.Tx, database string, out io.Writer) error {
- // Find all dumpable databases
- var dumpableDBNames []string
- if database != "" {
- dumpableDBNames = []string{database}
- } else {
- var err error
- dumpableDBNames, err = getDatabasesTxn(ctx, txn)
- if err != nil {
- return errors.Wrap(err, "failed to get databases")
- }
- }
-
- for _, dbName := range dumpableDBNames {
- // includeCreateDatabaseStmt should be false if dumping a single database.
- dumpSingleDatabase := len(dumpableDBNames) == 1
- if err := dumpOneDatabase(ctx, txn, dbName, out, dumpSingleDatabase); err != nil {
- return err
- }
- }
-
- return nil
-}
-
-// dumpOneDatabase will dump the database DDL schema for a database.
-// Note: this operation is not supported on shared databases, e.g. SNOWFLAKE_SAMPLE_DATA.
-func dumpOneDatabase(ctx context.Context, txn *sql.Tx, database string, out io.Writer, dumpSingleDatabase bool) error {
- if !dumpSingleDatabase {
- // Database header.
- header := fmt.Sprintf(databaseHeaderFmt, database)
- if _, err := io.WriteString(out, header); err != nil {
- return err
- }
- }
-
- query := fmt.Sprintf(`SELECT GET_DDL('DATABASE', '"%s"', true)`, database)
+ query := fmt.Sprintf(`SELECT GET_DDL('DATABASE', '"%s"', true)`, d.databaseName)
rows, err := txn.QueryContext(ctx, query)
if err != nil {
return util.FormatErrorWithQuery(err, query)
@@ -82,9 +28,7 @@ func dumpOneDatabase(ctx context.Context, txn *sql.Tx, database string, out io.W
var databaseDDL string
for rows.Next() {
- if err := rows.Scan(
- &databaseDDL,
- ); err != nil {
+ if err := rows.Scan(&databaseDDL); err != nil {
return err
}
}
@@ -92,37 +36,35 @@ func dumpOneDatabase(ctx context.Context, txn *sql.Tx, database string, out io.W
return err
}
- // Transform1: if dumpSingleDatabase, we should remove `create or replace database` statement.
- if dumpSingleDatabase {
- lines := strings.Split(databaseDDL, "\n")
- if len(lines) >= 2 {
- lines = lines[2:]
- }
- databaseDDL = strings.Join(lines, "\n")
+ // Transform1: remove `create or replace database` statement.
+ lines := strings.Split(databaseDDL, "\n")
+ if len(lines) >= 2 {
+ lines = lines[2:]
}
+ databaseDDL = strings.Join(lines, "\n")
// Transform2: remove "create or replace schema PUBLIC;\n\n" because it's created by default.
- schemaStmt := fmt.Sprintf("create or replace schema %s.PUBLIC;", database)
+ schemaStmt := fmt.Sprintf("create or replace schema %s.PUBLIC;", d.databaseName)
databaseDDL = strings.ReplaceAll(databaseDDL, schemaStmt+"\n\n", "")
// If this is the last statement.
databaseDDL = strings.ReplaceAll(databaseDDL, schemaStmt, "")
- var lines []string
+ var transformedLines []string
for _, line := range strings.Split(databaseDDL, "\n") {
if strings.HasPrefix(strings.ToLower(line), "create ") {
// Transform3: Remove "DEMO_DB." quantifier.
- line = strings.ReplaceAll(line, fmt.Sprintf(" %s.", database), " ")
+ line = strings.ReplaceAll(line, fmt.Sprintf(" %s.", d.databaseName), " ")
// Transform4 (Important!): replace all `create or replace ` with `create ` to not break existing schema by any chance.
line = strings.ReplaceAll(line, "create or replace ", "create ")
}
- lines = append(lines, line)
+ transformedLines = append(transformedLines, line)
}
- databaseDDL = strings.Join(lines, "\n")
+ databaseDDL = strings.Join(transformedLines, "\n")
if _, err := io.WriteString(out, databaseDDL); err != nil {
return err
}
- return nil
+ return txn.Commit()
}
diff --git a/backend/plugin/db/spanner/dump.go b/backend/plugin/db/spanner/dump.go
index 65300a9534..6c630610e6 100644
--- a/backend/plugin/db/spanner/dump.go
+++ b/backend/plugin/db/spanner/dump.go
@@ -6,47 +6,23 @@ import (
"io"
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
- "github.com/pkg/errors"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
-// Dump dumps the database.
+// Dump dumps database.
func (d *Driver) Dump(ctx context.Context, out io.Writer, _ *storepb.DatabaseSchemaMetadata) error {
- instance, err := d.SyncInstance(ctx)
+ resp, err := d.dbClient.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{
+ Database: getDSN(d.config.DataSource.Host, d.databaseName),
+ })
if err != nil {
return err
}
- var dumpableDBNames []string
- if d.databaseName != "" {
- exist := false
- for _, db := range instance.Databases {
- if db.Name == d.databaseName {
- exist = true
- break
- }
- }
- if !exist {
- return errors.Errorf("database %q not found", d.databaseName)
- }
- dumpableDBNames = []string{d.databaseName}
- } else {
- for _, db := range instance.Databases {
- dumpableDBNames = append(dumpableDBNames, db.Name)
- }
- }
- for _, db := range dumpableDBNames {
- resp, err := d.dbClient.GetDatabaseDdl(ctx, &databasepb.GetDatabaseDdlRequest{
- Database: getDSN(d.config.DataSource.Host, db),
- })
- if err != nil {
+
+ for _, stmt := range resp.Statements {
+ if _, err := io.WriteString(out, fmt.Sprintf("%s;\n", stmt)); err != nil {
return err
}
- for _, stmt := range resp.Statements {
- if _, err := io.WriteString(out, fmt.Sprintf("%s;\n", stmt)); err != nil {
- return err
- }
- }
}
return nil
diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.go
index 90bfaa3571..996ce7e8a0 100644
--- a/backend/plugin/db/starrocks/dump.go
+++ b/backend/plugin/db/starrocks/dump.go
@@ -239,8 +239,8 @@ type eventSchema struct {
// getTables gets all tables of a database using the provided database instance.
func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema, error) {
var tables []*TableSchema
- query := fmt.Sprintf("SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = '%s';", dbName)
- rows, err := db.Query(query)
+ query := "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?"
+ rows, err := db.Query(query, dbName)
if err != nil {
return nil, err
}
diff --git a/backend/plugin/parser/standard/split_test.go b/backend/plugin/parser/standard/split_test.go
index fca3982d71..7564e7c65d 100644
--- a/backend/plugin/parser/standard/split_test.go
+++ b/backend/plugin/parser/standard/split_test.go
@@ -1,11 +1,11 @@
package standard
import (
+ "crypto/rand"
"fmt"
- "math/rand"
+ "math/big"
"strings"
"testing"
- "time"
"github.com/stretchr/testify/require"
@@ -19,11 +19,17 @@ func TestStandardSplitSQL(t *testing.T) {
}
func generateOneMBInsert() string {
- var rand = rand.New(rand.NewSource(time.Now().UnixNano()))
letterList := []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
b := make([]byte, 1024*1024)
for i := range b {
- b[i] = letterList[rand.Intn(len(letterList))]
+ // Use crypto/rand for secure random generation
+ randomIndex, err := rand.Int(rand.Reader, big.NewInt(int64(len(letterList))))
+ if err != nil {
+ // Fallback to a simple pattern if crypto/rand fails
+ b[i] = letterList[i%26]
+ } else {
+ b[i] = letterList[randomIndex.Int64()]
+ }
}
return fmt.Sprintf("INSERT INTO t values('%s')", string(b))
}
diff --git a/backend/runner/plancheck/ghost_sync_executor.go b/backend/runner/plancheck/ghost_sync_executor.go
index 121117d664..780fde4f69 100644
--- a/backend/runner/plancheck/ghost_sync_executor.go
+++ b/backend/runner/plancheck/ghost_sync_executor.go
@@ -2,9 +2,10 @@ package plancheck
import (
"context"
+ "crypto/rand"
"fmt"
"log/slog"
- "math/rand"
+ "math/big"
"strings"
"time"
@@ -136,7 +137,12 @@ func (e *GhostSyncExecutor) RunForTarget(ctx context.Context, target *CheckTarge
}, nil
}
- migrationContext, err := ghost.NewMigrationContext(ctx, rand.Intn(10000000), database, adminDataSource, tableName, fmt.Sprintf("_dryrun_%d", time.Now().Unix()), statement, true, target.GhostFlags, 20000000)
+ // Generate secure random number for migration context
+ randomInt, err := rand.Int(rand.Reader, big.NewInt(10000000))
+ if err != nil {
+ return nil, common.Wrapf(err, common.Internal, "failed to generate secure random number")
+ }
+ migrationContext, err := ghost.NewMigrationContext(ctx, int(randomInt.Int64()), database, adminDataSource, tableName, fmt.Sprintf("_dryrun_%d", time.Now().Unix()), statement, true, target.GhostFlags, 20000000)
if err != nil {
return nil, common.Wrapf(err, common.Internal, "failed to create migration context")
}