Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(db): release the Cloud SQL connector with the driver (#21726)

Danny Xu554b25fc2c61
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.go
index 39e984a158..e8d68cfb7f 100644
--- a/backend/plugin/db/mysql/mysql.go
+++ b/backend/plugin/db/mysql/mysql.go
@@ -17,6 +17,7 @@ import (
"sync/atomic"
"time"
+ "cloud.google.com/go/cloudsqlconn"
"github.com/aws/aws-sdk-go-v2/feature/rds/auth"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
@@ -72,6 +73,12 @@ type Driver struct {
databaseName string
sshClient *ssh.Client
+ // cloudSQLDialer refreshes its ephemeral certificate in the background until
+ // closed, so it lives and dies with the driver. It is reached through the
+ // cloudSQLProtocol network name, set with it.
+ cloudSQLDialer *cloudsqlconn.Dialer
+ cloudSQLProtocol string
+
// Called upon driver.Open() finishes.
openCleanUp []func()
}
@@ -82,17 +89,23 @@ func newDriver() db.Driver {
// Open opens a MySQL driver.
func (d *Driver) Open(ctx context.Context, dbType storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {
+ opened := false
defer func() {
for _, f := range d.openCleanUp {
f()
}
+ // A driver whose Open fails is discarded without a Close, so Open
+ // releases what it acquired itself.
+ if !opened {
+ _ = d.Close(ctx)
+ }
}()
var dsn string
var err error
switch connCfg.DataSource.GetAuthenticationType() {
case storepb.DataSource_GOOGLE_CLOUD_SQL_IAM:
- dsn, err = getCloudSQLConnection(ctx, connCfg)
+ dsn, err = d.getCloudSQLConnection(ctx, connCfg)
case storepb.DataSource_AWS_RDS_IAM:
dsn, err = d.getRDSConnection(ctx, connCfg)
default:
@@ -114,6 +127,7 @@ func (d *Driver) Open(ctx context.Context, dbType storepb.Engine, connCfg db.Con
db.SetMaxIdleConns(15)
d.databaseName = connCfg.ConnectionContext.DatabaseName
+ opened = true
return d, nil
}
@@ -330,31 +344,47 @@ func (d *Driver) getRDSConnection(ctx context.Context, connCfg db.ConnectionConf
), nil
}
-func getCloudSQLConnection(ctx context.Context, connCfg db.ConnectionConfig) (string, error) {
- d, err := util.GetGCPConnectionConfig(ctx, connCfg)
+func (d *Driver) getCloudSQLConnection(ctx context.Context, connCfg db.ConnectionConfig) (string, error) {
+ dialer, err := util.GetGCPConnectionConfig(ctx, connCfg)
if err != nil {
return "", errors.Wrap(err, "load gcp config failed")
}
+ d.cloudSQLDialer = dialer
- mysql.RegisterDialContext("cloudsqlconn",
+ // The network name keys a process-wide registry, so it has to be unique per
+ // driver: a shared name would let the next Cloud SQL driver take over this
+ // driver's pool and dial its own instance instead.
+ d.cloudSQLProtocol = "cloudsqlconn-" + uuid.NewString()
+ mysql.RegisterDialContext(d.cloudSQLProtocol,
func(ctx context.Context, _ string) (net.Conn, error) {
- return d.Dial(ctx, connCfg.DataSource.Host)
+ return dialer.Dial(ctx, connCfg.DataSource.Host)
})
- return fmt.Sprintf("%s:empty@cloudsqlconn(localhost:3306)/%s?parseTime=true&multiStatements=true&maxAllowedPacket=0",
- connCfg.DataSource.Username, connCfg.ConnectionContext.DatabaseName), nil
+ return fmt.Sprintf("%s:empty@%s(localhost:3306)/%s?parseTime=true&multiStatements=true&maxAllowedPacket=0",
+ connCfg.DataSource.Username, d.cloudSQLProtocol, connCfg.ConnectionContext.DatabaseName), nil
}
// Close closes the driver.
func (d *Driver) Close(context.Context) error {
var err error
- err = multierr.Append(err, d.db.Close())
+ if d.db != nil {
+ err = multierr.Append(err, d.db.Close())
+ }
if d.sshClient != nil {
err = multierr.Append(err, d.sshClient.Close())
}
+ err = multierr.Append(err, d.closeCloudSQL())
return err
}
+func (d *Driver) closeCloudSQL() error {
+ if d.cloudSQLDialer == nil {
+ return nil
+ }
+ mysql.DeregisterDialContext(d.cloudSQLProtocol)
+ return d.cloudSQLDialer.Close()
+}
+
// Ping pings the database.
func (d *Driver) Ping(ctx context.Context) error {
return d.db.PingContext(ctx)
diff --git a/backend/plugin/db/mysql/mysql_test.go b/backend/plugin/db/mysql/mysql_test.go
index 7294c78ac6..d511d289a0 100644
--- a/backend/plugin/db/mysql/mysql_test.go
+++ b/backend/plugin/db/mysql/mysql_test.go
@@ -8,6 +8,7 @@ import (
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
+ "database/sql"
"encoding/json"
"encoding/pem"
"math/big"
@@ -19,6 +20,8 @@ import (
"testing"
"time"
+ "cloud.google.com/go/cloudsqlconn"
+ "github.com/go-sql-driver/mysql"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/common"
@@ -585,3 +588,109 @@ func planRoot(t *testing.T, result *v1pb.QueryResult) string {
}
return ""
}
+
+// cloudSQLIAMConnectionConfig returns a Google Cloud SQL IAM data source that
+// cloudsqlconn accepts offline. The credential is an authorized_user document so
+// that building the connector needs neither a key nor a network round trip; it
+// is never good enough to dial with.
+const cloudSQLTestInstance = "example-project:us-central1:example-instance"
+
+func cloudSQLIAMConnectionConfig(instance string) db.ConnectionConfig {
+ return db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Host: instance,
+ Username: "[email protected]",
+ AuthenticationType: storepb.DataSource_GOOGLE_CLOUD_SQL_IAM,
+ IamExtension: &storepb.DataSource_GcpCredential{
+ GcpCredential: &storepb.DataSource_GCPCredential{
+ Content: `{"type":"authorized_user","client_id":"not-a-real-client","client_secret":"not-a-real-secret","refresh_token":"not-a-real-token"}`,
+ },
+ },
+ },
+ ConnectionContext: db.ConnectionContext{DatabaseName: "example"},
+ }
+}
+
+func TestCloudSQLIAMDialerClosesWithDriver(t *testing.T) {
+ ctx := context.Background()
+
+ driver := &Driver{}
+ opened, err := driver.Open(ctx, storepb.Engine_MYSQL, cloudSQLIAMConnectionConfig(cloudSQLTestInstance))
+ require.NoError(t, err)
+ require.Same(t, driver, opened)
+ require.NotNil(t, driver.cloudSQLDialer)
+ require.NotEmpty(t, driver.cloudSQLProtocol)
+
+ dialer := driver.cloudSQLDialer
+ require.NoError(t, driver.Close(ctx))
+
+ // The connector keeps refreshing its ephemeral certificate until it is
+ // closed, so a driver that leaves it open leaks it for the life of the
+ // process, once per connection opened.
+ _, err = dialer.Dial(ctx, cloudSQLTestInstance)
+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)
+}
+
+func TestCloudSQLIAMDialerClosesWhenOpenFails(t *testing.T) {
+ ctx := context.Background()
+
+ config := cloudSQLIAMConnectionConfig(cloudSQLTestInstance)
+ // A database name that does not survive the DSN fails Open after the
+ // connector has been built.
+ config.ConnectionContext.DatabaseName = "example/db"
+
+ driver := &Driver{}
+ _, err := driver.Open(ctx, storepb.Engine_MYSQL, config)
+ require.Error(t, err)
+
+ // A driver whose Open fails is discarded without a Close, so Open has to have
+ // closed the connector itself.
+ require.NotNil(t, driver.cloudSQLDialer)
+ _, err = driver.cloudSQLDialer.Dial(ctx, cloudSQLTestInstance)
+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)
+}
+
+func TestCloudSQLIAMNetworkNameIsPerDriver(t *testing.T) {
+ ctx := context.Background()
+
+ first := &Driver{}
+ firstDSN, err := first.getCloudSQLConnection(ctx, cloudSQLIAMConnectionConfig("example-project:us-central1:first"))
+ require.NoError(t, err)
+ defer func() { require.NoError(t, first.closeCloudSQL()) }()
+
+ second := &Driver{}
+ secondDSN, err := second.getCloudSQLConnection(ctx, cloudSQLIAMConnectionConfig("example-project:us-central1:second"))
+ require.NoError(t, err)
+ defer func() { require.NoError(t, second.closeCloudSQL()) }()
+
+ // The network name keys a process-wide registry. Sharing one name let the
+ // second driver take over the first driver's pool and dial its own instance.
+ require.NotEqual(t, first.cloudSQLProtocol, second.cloudSQLProtocol)
+
+ firstConfig, err := mysql.ParseDSN(firstDSN)
+ require.NoError(t, err)
+ require.Equal(t, first.cloudSQLProtocol, firstConfig.Net)
+
+ secondConfig, err := mysql.ParseDSN(secondDSN)
+ require.NoError(t, err)
+ require.Equal(t, second.cloudSQLProtocol, secondConfig.Net)
+}
+
+func TestCloudSQLIAMNetworkNameDeregisteredOnClose(t *testing.T) {
+ ctx := context.Background()
+
+ driver := &Driver{}
+ dsn, err := driver.getCloudSQLConnection(ctx, cloudSQLIAMConnectionConfig(cloudSQLTestInstance))
+ require.NoError(t, err)
+ require.NoError(t, driver.closeCloudSQL())
+
+ // The registration would otherwise pin the closed connector in the MySQL
+ // driver's process-wide map, one entry per driver ever opened.
+ pool, err := sql.Open("mysql", dsn)
+ require.NoError(t, err)
+ defer pool.Close()
+ err = pool.PingContext(ctx)
+ require.Error(t, err)
+ require.NotErrorIs(t, err, cloudsqlconn.ErrDialerClosed)
+}
diff --git a/backend/plugin/db/pg/pg.go b/backend/plugin/db/pg/pg.go
index 6da77df8d4..d46ab27408 100644
--- a/backend/plugin/db/pg/pg.go
+++ b/backend/plugin/db/pg/pg.go
@@ -13,6 +13,7 @@ import (
"time"
"unicode"
+ "cloud.google.com/go/cloudsqlconn"
"github.com/aws/aws-sdk-go-v2/feature/rds/auth"
metadatapb "github.com/bytebase/omni/metadata"
"github.com/jackc/pgx/v5"
@@ -51,6 +52,9 @@ type Driver struct {
db *sql.DB
sshClient *ssh.Client
+ // cloudSQLDialer refreshes its ephemeral certificate in the background until
+ // closed, so it lives and dies with the driver.
+ cloudSQLDialer *cloudsqlconn.Dialer
// connectionString is the connection string registered by pgx.
// Unregister connectionString if we don't need it.
connectionString string
@@ -64,12 +68,21 @@ func newDriver() db.Driver {
// Open opens a Postgres driver.
func (d *Driver) Open(ctx context.Context, _ storepb.Engine, config db.ConnectionConfig) (db.Driver, error) {
+ // A driver whose Open fails is discarded without a Close, so Open releases
+ // what it acquired itself.
+ opened := false
+ defer func() {
+ if !opened {
+ _ = d.Close(ctx)
+ }
+ }()
+
var pgxConnConfig *pgx.ConnConfig
var err error
switch config.DataSource.GetAuthenticationType() {
case storepb.DataSource_GOOGLE_CLOUD_SQL_IAM:
- pgxConnConfig, err = getCloudSQLConnectionConfig(ctx, config)
+ pgxConnConfig, err = d.getCloudSQLConnectionConfig(ctx, config)
case storepb.DataSource_AWS_RDS_IAM:
pgxConnConfig, err = getRDSConnectionConfig(ctx, config)
default:
@@ -127,6 +140,7 @@ func (d *Driver) Open(ctx context.Context, _ storepb.Engine, config db.Connectio
}
}
d.connectionCtx = config.ConnectionContext
+ opened = true
return d, nil
}
@@ -271,11 +285,12 @@ func getRDSConnectionConfig(ctx context.Context, conf db.ConnectionConfig) (*pgx
// refs:
// https://cloud.google.com/sql/docs/postgres/connect-connectors
// https://github.com/GoogleCloudPlatform/golang-samples/blob/main/cloudsql/postgres/database-sql/cloudsql.go
-func getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig) (*pgx.ConnConfig, error) {
- d, err := util.GetGCPConnectionConfig(ctx, conf)
+func (d *Driver) getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig) (*pgx.ConnConfig, error) {
+ dialer, err := util.GetGCPConnectionConfig(ctx, conf)
if err != nil {
return nil, errors.Wrap(err, "load gcp config failed")
}
+ d.cloudSQLDialer = dialer
dsn := fmt.Sprintf("user=%s", conf.DataSource.Username)
config, err := pgx.ParseConfig(dsn)
@@ -283,7 +298,7 @@ func getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig)
return nil, err
}
config.DialFunc = func(ctx context.Context, _, _ string) (net.Conn, error) {
- return d.Dial(ctx, conf.DataSource.Host)
+ return dialer.Dial(ctx, conf.DataSource.Host)
}
config.DefaultQueryExecMode = pgx.QueryExecModeExec
@@ -295,10 +310,15 @@ func getCloudSQLConnectionConfig(ctx context.Context, conf db.ConnectionConfig)
func (d *Driver) Close(context.Context) error {
stdlib.UnregisterConnConfig(d.connectionString)
var err error
- err = multierr.Append(err, d.db.Close())
+ if d.db != nil {
+ err = multierr.Append(err, d.db.Close())
+ }
if d.sshClient != nil {
err = multierr.Append(err, d.sshClient.Close())
}
+ if d.cloudSQLDialer != nil {
+ err = multierr.Append(err, d.cloudSQLDialer.Close())
+ }
return err
}
diff --git a/backend/plugin/db/pg/pg_test.go b/backend/plugin/db/pg/pg_test.go
index 7a6c85d138..e17e3cfc8a 100644
--- a/backend/plugin/db/pg/pg_test.go
+++ b/backend/plugin/db/pg/pg_test.go
@@ -13,6 +13,7 @@ import (
"testing"
"time"
+ "cloud.google.com/go/cloudsqlconn"
"github.com/jackc/pgx/v5"
"github.com/stretchr/testify/require"
@@ -532,3 +533,87 @@ func resultsContainValue(results []*v1pb.QueryResult, want string) bool {
}
return false
}
+
+// cloudSQLIAMConnectionConfig returns a Google Cloud SQL IAM data source that
+// cloudsqlconn accepts offline. The credential is an authorized_user document so
+// that building the connector needs neither a key nor a network round trip; it
+// is never good enough to dial with.
+const cloudSQLTestInstance = "example-project:us-central1:example-instance"
+
+func cloudSQLIAMConnectionConfig(instance string) db.ConnectionConfig {
+ return db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Host: instance,
+ Username: "[email protected]",
+ AuthenticationType: storepb.DataSource_GOOGLE_CLOUD_SQL_IAM,
+ IamExtension: &storepb.DataSource_GcpCredential{
+ GcpCredential: &storepb.DataSource_GCPCredential{
+ Content: `{"type":"authorized_user","client_id":"not-a-real-client","client_secret":"not-a-real-secret","refresh_token":"not-a-real-token"}`,
+ },
+ },
+ },
+ ConnectionContext: db.ConnectionContext{DatabaseName: "postgres"},
+ }
+}
+
+func TestCloudSQLIAMDialerClosesWithDriver(t *testing.T) {
+ ctx := context.Background()
+
+ driver := &Driver{}
+ opened, err := driver.Open(ctx, storepb.Engine_POSTGRES, cloudSQLIAMConnectionConfig(cloudSQLTestInstance))
+ require.NoError(t, err)
+ require.Same(t, driver, opened)
+ require.NotNil(t, driver.cloudSQLDialer)
+
+ dialer := driver.cloudSQLDialer
+ require.NoError(t, driver.Close(ctx))
+
+ // The connector keeps refreshing its ephemeral certificate until it is
+ // closed, so a driver that leaves it open leaks it for the life of the
+ // process, once per connection opened.
+ _, err = dialer.Dial(ctx, cloudSQLTestInstance)
+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)
+}
+
+func TestCloudSQLIAMDialerClosesWhenOpenFails(t *testing.T) {
+ ctx := context.Background()
+
+ config := cloudSQLIAMConnectionConfig(cloudSQLTestInstance)
+ // A username that does not survive the keyword/value DSN fails Open after the
+ // connector has been built.
+ config.DataSource.Username = "bytebase service account"
+
+ driver := &Driver{}
+ _, err := driver.Open(ctx, storepb.Engine_POSTGRES, config)
+ require.Error(t, err)
+
+ // A driver whose Open fails is discarded without a Close, so Open has to have
+ // closed the connector itself.
+ require.NotNil(t, driver.cloudSQLDialer)
+ _, err = driver.cloudSQLDialer.Dial(ctx, cloudSQLTestInstance)
+ require.ErrorIs(t, err, cloudsqlconn.ErrDialerClosed)
+}
+
+func TestOpenFailureReleasesConnectionPool(t *testing.T) {
+ ctx := context.Background()
+
+ driver := &Driver{}
+ // Tenant mode looks up the database owner, so Open fails against a dead port
+ // only after it has built the connection pool.
+ _, err := driver.Open(ctx, storepb.Engine_POSTGRES, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Host: "127.0.0.1",
+ Port: "1",
+ Username: "bytebase",
+ },
+ ConnectionContext: db.ConnectionContext{DatabaseName: "postgres", TenantMode: true},
+ })
+ require.Error(t, err)
+
+ // A driver whose Open fails is discarded without a Close, so the pool has to
+ // be closed already. database/sql reports that only by message.
+ require.NotNil(t, driver.db)
+ require.ErrorContains(t, driver.db.PingContext(ctx), "database is closed")
+}