Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(db): release the Cloud SQL connector with the driver (#21726)
Danny Xu554b25fc2c61
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.goindex 39e984a158..e8d68cfb7f 100644--- a/backend/plugin/db/mysql/mysql.go+++ b/backend/plugin/db/mysql/mysql.go@@ -17,6 +17,7 @@ import ("sync/atomic""time"+ "cloud.google.com/go/cloudsqlconn""github.com/aws/aws-sdk-go-v2/feature/rds/auth""github.com/go-sql-driver/mysql""github.com/google/uuid"@@ -72,6 +73,12 @@ type Driver struct {databaseName stringsshClient *ssh.Client+ // cloudSQLDialer refreshes its ephemeral certificate in the background until+ // closed, so it lives and dies with the driver. It is reached through the+ // cloudSQLProtocol network name, set with it.+ cloudSQLDialer *cloudsqlconn.Dialer+ cloudSQLProtocol string+// Called upon driver.Open() finishes.openCleanUp []func()}@@ -82,17 +89,23 @@ func newDriver() db.Driver {// Open opens a MySQL driver.func (d *Driver) Open(ctx context.Context, dbType storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {+ opened := falsedefer func() {for _, f := range d.openCleanUp {f()}+ // A driver whose Open fails is discarded without a Close, so Open+ // releases what it acquired itself.+ if !opened {+ _ = d.Close(ctx)+ }}()var dsn stringvar err errorswitch connCfg.DataSource.GetAuthenticationType() {case storepb.DataSource_GOOGLE_CLOUD_SQL_IAM:- dsn, err = getCloudSQLConnection(ctx, connCfg)+ dsn, err = d.getCloudSQLConnection(ctx, connCfg)case storepb.DataSource_AWS_RDS_IAM:dsn, err = d.getRDSConnection(ctx, connCfg)default:@@ -114,6 +127,7 @@ func (d *Driver) Open(ctx context.Context, dbType storepb.Engine, connCfg db.Condb.SetMaxIdleConns(15)d.databaseName = connCfg.ConnectionContext.DatabaseName+ opened = truereturn d, nil}@@ -330,31 +344,47 @@ func (d *Driver) getRDSConnection(ctx context.Context, connCfg db.ConnectionConf), nil}-func getCloudSQLConnection(ctx context.Context, connCfg db.ConnectionConfig) (string, error) {- d, err := util.GetGCPConnectionConfig(ctx, connCfg)+func (d *Driver) getCloudSQLConnection(ctx context.Context, connCfg db.ConnectionConfig) (string, error) {+ dialer, err := util.GetGCPConnectionConfig(ctx, connCfg)if err != nil {return "", errors.Wrap(err, "load gcp config failed")}+ d.cloudSQLDialer = dialer- mysql.RegisterDialContext("cloudsqlconn",+ // The network name keys a process-wide registry, so it has to be unique per+ // driver: a shared name would let the next Cloud SQL driver take over this+ // driver's pool and dial its own instance instead.+ d.cloudSQLProtocol = "cloudsqlconn-" + uuid.NewString()+ mysql.RegisterDialContext(d.cloudSQLProtocol,func(ctx context.Context, _ string) (net.Conn, error) {- return d.Dial(ctx, connCfg.DataSource.Host)+ return dialer.Dial(ctx, connCfg.DataSource.Host)})- return fmt.Sprintf("%s:empty@cloudsqlconn(localhost:3306)/%s?parseTime=true&multiStatements=true&maxAllowedPacket=0",- connCfg.DataSource.Username, connCfg.ConnectionContext.DatabaseName), nil+ return fmt.Sprintf("%s:empty@%s(localhost:3306)/%s?parseTime=true&multiStatements=true&maxAllowedPacket=0",+ connCfg.DataSource.Username, d.cloudSQLProtocol, connCfg.ConnectionContext.DatabaseName), nil}// Close closes the driver.func (d *Driver) Close(context.Context) error {var err error- err = multierr.Append(err, d.db.Close())+ if d.db != nil {+ err = multierr.Append(err, d.db.Close())+ }if d.sshClient != nil {err = multierr.Append(err, d.sshClient.Close())}+ err = multierr.Append(err, d.closeCloudSQL())return err}+func (d *Driver) closeCloudSQL() error {+ if d.cloudSQLDialer == nil {+ return nil+ }+ mysql.DeregisterDialContext(d.cloudSQLProtocol)+ return d.cloudSQLDialer.Close()+}+// Ping pings the database.func (d *Driver) Ping(ctx context.Context) error {return d.db.PingContext(ctx)diff --git a/backend/plugin/db/mysql/mysql_test.go b/backend/plugin/db/mysql/mysql_test.goindex 7294c78ac6..d511d289a0 100644--- a/backend/plugin/db/mysql/mysql_test.go+++ b/backend/plugin/db/mysql/mysql_test.go@@ -8,6 +8,7 @@ import ("crypto/rand""crypto/x509""crypto/x509/pkix"+ "database/sql""encoding/json""encoding/pem""math/big"@@ -19,6 +20,8 @@ import ("testing""time"+ "cloud.google.com/go/cloudsqlconn"+ "github.com/go-sql-driver/mysql""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common"@@ -585,3 +588,109 @@ func planRoot(t *testing.T, result *v1pb.QueryResult) string {}return ""}++// cloudSQLIAMConnectionConfig returns a Google Cloud SQL IAM data source that+// cloudsqlconn accepts offline. The credential is an authorized_user document so+// that building the connector needs neither a key nor a network round trip; it+// is never good enough to dial with.+const cloudSQLTestInstance = "example-project:us-central1:example-instance"++func cloudSQLIAMConnectionConfig(instance string) db.ConnectionConfig {+ return db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Host: instance,+ Username: "[email protected]",+ AuthenticationType: storepb.DataSource_GOOGLE_CLOUD_SQL_IAM,+ IamExtension: &storepb.DataSource_GcpCredential{+ GcpCredential: &storepb.DataSource_GCPCredential{+ Content: `{"type":"authorized_user","client_id":"not-a-real-client","client_secret":"not-a-real-secret","refresh_token":"not-a-real-token"}`,+ },+ },+ },+ ConnectionContext: db.ConnectionContext{DatabaseName: "example"},+ }+}++func TestCloudSQLIAMDialerClosesWithDriver(t *testing.T) {+ ctx := context.Background()++ driver := &Driver{}+ opened, err := driver.Open(ctx, storepb.Engine_MYSQL, cloudSQLIAMConnectionConfig(cloudSQLTestInstance))+ require.NoError(t, err)+ require.Same(t, driver, opened)+ require.NotNil(t, driver.cloudSQLDialer)+ require.NotEmpty(t, driver.cloudSQLProtocol)++ dialer := driver.cloudSQLDialer+ require.NoError(t, driver.Close(ctx))++ // The connector keeps refreshing its ephemeral certificate until it is+ // closed, so a driver that leaves it open leaks it for the life of the+ // process, once per connection opened.+ _, err = dialer.Dial(ctx, cloudSQLTestInstance)+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)+}++func TestCloudSQLIAMDialerClosesWhenOpenFails(t *testing.T) {+ ctx := context.Background()++ config := cloudSQLIAMConnectionConfig(cloudSQLTestInstance)+ // A database name that does not survive the DSN fails Open after the+ // connector has been built.+ config.ConnectionContext.DatabaseName = "example/db"++ driver := &Driver{}+ _, err := driver.Open(ctx, storepb.Engine_MYSQL, config)+ require.Error(t, err)++ // A driver whose Open fails is discarded without a Close, so Open has to have+ // closed the connector itself.+ require.NotNil(t, driver.cloudSQLDialer)+ _, err = driver.cloudSQLDialer.Dial(ctx, cloudSQLTestInstance)+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)+}++func TestCloudSQLIAMNetworkNameIsPerDriver(t *testing.T) {+ ctx := context.Background()++ first := &Driver{}+ firstDSN, err := first.getCloudSQLConnection(ctx, cloudSQLIAMConnectionConfig("example-project:us-central1:first"))+ require.NoError(t, err)+ defer func() { require.NoError(t, first.closeCloudSQL()) }()++ second := &Driver{}+ secondDSN, err := second.getCloudSQLConnection(ctx, cloudSQLIAMConnectionConfig("example-project:us-central1:second"))+ require.NoError(t, err)+ defer func() { require.NoError(t, second.closeCloudSQL()) }()++ // The network name keys a process-wide registry. Sharing one name let the+ // second driver take over the first driver's pool and dial its own instance.+ require.NotEqual(t, first.cloudSQLProtocol, second.cloudSQLProtocol)++ firstConfig, err := mysql.ParseDSN(firstDSN)+ require.NoError(t, err)+ require.Equal(t, first.cloudSQLProtocol, firstConfig.Net)++ secondConfig, err := mysql.ParseDSN(secondDSN)+ require.NoError(t, err)+ require.Equal(t, second.cloudSQLProtocol, secondConfig.Net)+}++func TestCloudSQLIAMNetworkNameDeregisteredOnClose(t *testing.T) {+ ctx := context.Background()++ driver := &Driver{}+ dsn, err := driver.getCloudSQLConnection(ctx, cloudSQLIAMConnectionConfig(cloudSQLTestInstance))+ require.NoError(t, err)+ require.NoError(t, driver.closeCloudSQL())++ // The registration would otherwise pin the closed connector in the MySQL+ // driver's process-wide map, one entry per driver ever opened.+ pool, err := sql.Open("mysql", dsn)+ require.NoError(t, err)+ defer pool.Close()+ err = pool.PingContext(ctx)+ require.Error(t, err)+ require.NotErrorIs(t, err, cloudsqlconn.ErrDialerClosed)+}diff --git a/backend/plugin/db/pg/pg.go b/backend/plugin/db/pg/pg.goindex 6da77df8d4..d46ab27408 100644--- a/backend/plugin/db/pg/pg.go+++ b/backend/plugin/db/pg/pg.go@@ -13,6 +13,7 @@ import ("time""unicode"+ "cloud.google.com/go/cloudsqlconn""github.com/aws/aws-sdk-go-v2/feature/rds/auth"metadatapb "github.com/bytebase/omni/metadata""github.com/jackc/pgx/v5"@@ -51,6 +52,9 @@ type Driver struct {db *sql.DBsshClient *ssh.Client+ // cloudSQLDialer refreshes its ephemeral certificate in the background until+ // closed, so it lives and dies with the driver.+ cloudSQLDialer *cloudsqlconn.Dialer// connectionString is the connection string registered by pgx.// Unregister connectionString if we don't need it.connectionString string@@ -64,12 +68,21 @@ func newDriver() db.Driver {// Open opens a Postgres driver.func (d *Driver) Open(ctx context.Context, _ storepb.Engine, config db.ConnectionConfig) (db.Driver, error) {+ // A driver whose Open fails is discarded without a Close, so Open releases+ // what it acquired itself.+ opened := false+ defer func() {+ if !opened {+ _ = d.Close(ctx)+ }+ }()+var pgxConnConfig *pgx.ConnConfigvar err errorswitch config.DataSource.GetAuthenticationType() {case storepb.DataSource_GOOGLE_CLOUD_SQL_IAM:- pgxConnConfig, err = getCloudSQLConnectionConfig(ctx, config)+ pgxConnConfig, err = d.getCloudSQLConnectionConfig(ctx, config)case storepb.DataSource_AWS_RDS_IAM:pgxConnConfig, err = getRDSConnectionConfig(ctx, config)default:@@ -127,6 +140,7 @@ func (d *Driver) Open(ctx context.Context, _ storepb.Engine, config db.Connectio}}d.connectionCtx = config.ConnectionContext+ opened = truereturn d, nil}@@ -271,11 +285,12 @@ func getRDSConnectionConfig(ctx context.Context, conf db.ConnectionConfig) (*pgx// refs:// https://cloud.google.com/sql/docs/postgres/connect-connectors// https://github.com/GoogleCloudPlatform/golang-samples/blob/main/cloudsql/postgres/database-sql/cloudsql.go-func getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig) (*pgx.ConnConfig, error) {- d, err := util.GetGCPConnectionConfig(ctx, conf)+func (d *Driver) getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig) (*pgx.ConnConfig, error) {+ dialer, err := util.GetGCPConnectionConfig(ctx, conf)if err != nil {return nil, errors.Wrap(err, "load gcp config failed")}+ d.cloudSQLDialer = dialerdsn := fmt.Sprintf("user=%s", conf.DataSource.Username)config, err := pgx.ParseConfig(dsn)@@ -283,7 +298,7 @@ func getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig)return nil, err}config.DialFunc = func(ctx context.Context, _, _ string) (net.Conn, error) {- return d.Dial(ctx, conf.DataSource.Host)+ return dialer.Dial(ctx, conf.DataSource.Host)}config.DefaultQueryExecMode = pgx.QueryExecModeExec@@ -295,10 +310,15 @@ func getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig)func (d *Driver) Close(context.Context) error {stdlib.UnregisterConnConfig(d.connectionString)var err error- err = multierr.Append(err, d.db.Close())+ if d.db != nil {+ err = multierr.Append(err, d.db.Close())+ }if d.sshClient != nil {err = multierr.Append(err, d.sshClient.Close())}+ if d.cloudSQLDialer != nil {+ err = multierr.Append(err, d.cloudSQLDialer.Close())+ }return err}diff --git a/backend/plugin/db/pg/pg_test.go b/backend/plugin/db/pg/pg_test.goindex 7a6c85d138..e17e3cfc8a 100644--- a/backend/plugin/db/pg/pg_test.go+++ b/backend/plugin/db/pg/pg_test.go@@ -13,6 +13,7 @@ import ("testing""time"+ "cloud.google.com/go/cloudsqlconn""github.com/jackc/pgx/v5""github.com/stretchr/testify/require"@@ -532,3 +533,87 @@ func resultsContainValue(results []*v1pb.QueryResult, want string) bool {}return false}++// cloudSQLIAMConnectionConfig returns a Google Cloud SQL IAM data source that+// cloudsqlconn accepts offline. The credential is an authorized_user document so+// that building the connector needs neither a key nor a network round trip; it+// is never good enough to dial with.+const cloudSQLTestInstance = "example-project:us-central1:example-instance"++func cloudSQLIAMConnectionConfig(instance string) db.ConnectionConfig {+ return db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Host: instance,+ Username: "[email protected]",+ AuthenticationType: storepb.DataSource_GOOGLE_CLOUD_SQL_IAM,+ IamExtension: &storepb.DataSource_GcpCredential{+ GcpCredential: &storepb.DataSource_GCPCredential{+ Content: `{"type":"authorized_user","client_id":"not-a-real-client","client_secret":"not-a-real-secret","refresh_token":"not-a-real-token"}`,+ },+ },+ },+ ConnectionContext: db.ConnectionContext{DatabaseName: "postgres"},+ }+}++func TestCloudSQLIAMDialerClosesWithDriver(t *testing.T) {+ ctx := context.Background()++ driver := &Driver{}+ opened, err := driver.Open(ctx, storepb.Engine_POSTGRES, cloudSQLIAMConnectionConfig(cloudSQLTestInstance))+ require.NoError(t, err)+ require.Same(t, driver, opened)+ require.NotNil(t, driver.cloudSQLDialer)++ dialer := driver.cloudSQLDialer+ require.NoError(t, driver.Close(ctx))++ // The connector keeps refreshing its ephemeral certificate until it is+ // closed, so a driver that leaves it open leaks it for the life of the+ // process, once per connection opened.+ _, err = dialer.Dial(ctx, cloudSQLTestInstance)+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)+}++func TestCloudSQLIAMDialerClosesWhenOpenFails(t *testing.T) {+ ctx := context.Background()++ config := cloudSQLIAMConnectionConfig(cloudSQLTestInstance)+ // A username that does not survive the keyword/value DSN fails Open after the+ // connector has been built.+ config.DataSource.Username = "bytebase service account"++ driver := &Driver{}+ _, err := driver.Open(ctx, storepb.Engine_POSTGRES, config)+ require.Error(t, err)++ // A driver whose Open fails is discarded without a Close, so Open has to have+ // closed the connector itself.+ require.NotNil(t, driver.cloudSQLDialer)+ _, err = driver.cloudSQLDialer.Dial(ctx, cloudSQLTestInstance)+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)+}++func TestOpenFailureReleasesConnectionPool(t *testing.T) {+ ctx := context.Background()++ driver := &Driver{}+ // Tenant mode looks up the database owner, so Open fails against a dead port+ // only after it has built the connection pool.+ _, err := driver.Open(ctx, storepb.Engine_POSTGRES, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Host: "127.0.0.1",+ Port: "1",+ Username: "bytebase",+ },+ ConnectionContext: db.ConnectionContext{DatabaseName: "postgres", TenantMode: true},+ })+ require.Error(t, err)++ // A driver whose Open fails is discarded without a Close, so the pool has to+ // be closed already. database/sql reports that only by message.+ require.NotNil(t, driver.db)+ require.ErrorContains(t, driver.db.PingContext(ctx), "database is closed")+}