Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(mysql): stop downloading the RDS CA bundle on every connection (#21186)
Dmitry Makarenkoa6b87a904236
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.goindex a8842045ce..e1cbe20347 100644--- a/backend/plugin/db/mysql/mysql.go+++ b/backend/plugin/db/mysql/mysql.go@@ -14,6 +14,7 @@ import ("net/http""regexp""strings"+ "sync/atomic""time""github.com/aws/aws-sdk-go-v2/feature/rds/auth"@@ -22,6 +23,7 @@ import ("github.com/pkg/errors""go.uber.org/multierr""golang.org/x/crypto/ssh"+ "golang.org/x/sync/singleflight""google.golang.org/protobuf/types/known/durationpb""github.com/bytebase/bytebase/backend/common"@@ -154,6 +156,23 @@ func (d *Driver) getMySQLConnection(connCfg db.ConnectionConfig) (string, error)return fmt.Sprintf("%s:%s@%s(%s:%s)/%s?%s", connCfg.DataSource.Username, connCfg.Password, protocol, connCfg.DataSource.Host, connCfg.DataSource.Port, connCfg.ConnectionContext.DatabaseName, strings.Join(params, "&")), nil}+// rdsCertBundleURL is a variable so tests can point it at a local server.+var rdsCertBundleURL = "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem"++// rdsCertFetchTimeout bounds the download so a stalled fetch cannot hold a+// connection attempt open indefinitely.+const rdsCertFetchTimeout = 30 * time.Second++// maxRDSCertBundleSize bounds the downloaded bundle so a misbehaving endpoint+// cannot stream unbounded data into memory (the real global bundle is ~165 KB).+const maxRDSCertBundleSize int64 = 4 << 20++// rdsCertPool caches the downloaded bundle for the process lifetime.+var rdsCertPool atomic.Pointer[x509.CertPool]++// rdsCertGroup collapses concurrent cold-start downloads into one request.+var rdsCertGroup singleflight.Group+// getRDSCertPool downloads and returns the RDS CA certificate pool.// AWS RDS connection with IAM require TLS connection.//@@ -161,25 +180,30 @@ func (d *Driver) getMySQLConnection(connCfg db.ConnectionConfig) (string, error)// https://github.com/aws/aws-sdk-go/issues/1248// https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/mysql-ssl-connections.htmlfunc getRDSCertPool(ctx context.Context) (*x509.CertPool, error) {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem", nil)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rdsCertBundleURL, nil)if err != nil {return nil, errors.Wrapf(err, "failed to build request for rds cert")}- client := &http.Client{}+ client := &http.Client{Timeout: rdsCertFetchTimeout}resp, err := client.Do(req)if err != nil {return nil, err}defer resp.Body.Close()- pem, err := io.ReadAll(resp.Body)+ // Without this an error page would reach AppendCertsFromPEM and surface as+ // "failed to parse RDS CA certificates", hiding the real cause.+ if resp.StatusCode != http.StatusOK {+ return nil, errors.Errorf("failed to download RDS CA certificates: %s", resp.Status)+ }++ pem, err := io.ReadAll(io.LimitReader(resp.Body, maxRDSCertBundleSize+1))if err != nil {return nil, err}-- if err := resp.Body.Close(); err != nil {- return nil, errors.Wrapf(err, "failed to close response")+ if int64(len(pem)) > maxRDSCertBundleSize {+ return nil, errors.Errorf("RDS CA bundle exceeds %d bytes", maxRDSCertBundleSize)}rootCertPool := x509.NewCertPool()@@ -190,6 +214,75 @@ func getRDSCertPool(ctx context.Context) (*x509.CertPool, error) {return rootCertPool, nil}+// cachedRDSCertPool returns the AWS RDS CA bundle, downloading it once per+// process. Only successes are cached, so a transient failure can still recover.+func cachedRDSCertPool(ctx context.Context) (*x509.CertPool, error) {+ if pool := rdsCertPool.Load(); pool != nil {+ return pool, nil+ }+ // Collapse a cold-start burst into one download. The shared fetch does not+ // inherit any one caller's context, so the caller that starts it cannot+ // cancel it for everybody else; each caller waits on its own context below.+ ch := rdsCertGroup.DoChan("", func() (any, error) {+ // Double check after entering singleflight.+ if pool := rdsCertPool.Load(); pool != nil {+ return pool, nil+ }+ fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rdsCertFetchTimeout)+ defer cancel()+ pool, err := getRDSCertPool(fetchCtx)+ if err != nil {+ return nil, err+ }+ rdsCertPool.Store(pool)+ return pool, nil+ })+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case res := <-ch:+ if res.Err != nil {+ return nil, res.Err+ }+ pool, ok := res.Val.(*x509.CertPool)+ if !ok {+ return nil, errors.Errorf("unexpected RDS cert pool type %T", res.Val)+ }+ return pool, nil+ }+}++// rdsRootCAs prefers an operator-supplied CA, so a deployment can serve RDS IAM+// connections without any outbound network access.+func rdsRootCAs(ctx context.Context, dataSource *storepb.DataSource) (*x509.CertPool, error) {+ if ca := dataSource.GetSslCa(); ca != "" {+ pool := x509.NewCertPool()+ if !pool.AppendCertsFromPEM([]byte(ca)) {+ return nil, errors.Errorf("failed to parse ssl_ca certificates")+ }+ return pool, nil+ }+ return cachedRDSCertPool(ctx)+}++// rdsTLSConfig builds the TLS config for an RDS IAM connection. IAM auth always+// requires TLS, so this always returns a config.+func rdsTLSConfig(ctx context.Context, dataSource *storepb.DataSource) (*tls.Config, error) {+ if !dataSource.GetVerifyTlsCertificate() {+ // No verifier runs in this mode, so a root pool would never be consulted.+ return &tls.Config{InsecureSkipVerify: true}, nil // NOSONAR(go:S4830,go:S5527) operator disabled verification (verify_tls_certificate=false)+ }+ rootCertPool, err := rdsRootCAs(ctx, dataSource)+ if err != nil {+ return nil, errors.Wrap(err, "failed to get RDS cert pool")+ }+ return &tls.Config{ // NOSONAR(go:S4830) VerifyPeerCertificate below verifies chain and hostname+ RootCAs: rootCertPool,+ InsecureSkipVerify: true, // superseded by VerifyPeerCertificate below+ VerifyPeerCertificate: util.CreateCertificateVerifier(rootCertPool, dataSource.GetHost()),+ }, nil+}+// getRDSConnection returns the connection string with IAM for AWS RDS.//// refs:@@ -213,31 +306,14 @@ func (d *Driver) getRDSConnection(ctx context.Context, connCfg db.ConnectionConfreturn "", errors.Wrap(err, "failed to create authentication token")}- // Get RDS CA certificate pool- rootCertPool, err := getRDSCertPool(ctx)+ tlsConfig, err := rdsTLSConfig(ctx, connCfg.DataSource)if err != nil {- return "", errors.Wrap(err, "failed to get RDS cert pool")+ return "", err}// Create TLS config with unique name for this connectiontlsKey := uuid.NewString()- var tlsConfig *tls.Config- if connCfg.DataSource.GetVerifyTlsCertificate() {- // Secure config with certificate verification- tlsConfig = &tls.Config{- RootCAs: rootCertPool,- InsecureSkipVerify: true, // We use custom verification- }- tlsConfig.VerifyPeerCertificate = util.CreateCertificateVerifier(rootCertPool, connCfg.DataSource.Host)- } else {- // Backward compatible config without verification- tlsConfig = &tls.Config{- RootCAs: rootCertPool,- InsecureSkipVerify: true,- }- }-// Register the TLS config with unique nameif err := mysql.RegisterTLSConfig(tlsKey, tlsConfig); err != nil {return "", errors.Wrap(err, "failed to register RDS TLS config")diff --git a/backend/plugin/db/mysql/mysql_test.go b/backend/plugin/db/mysql/mysql_test.goindex 32d326bdf5..4c525b8cbd 100644--- a/backend/plugin/db/mysql/mysql_test.go+++ b/backend/plugin/db/mysql/mysql_test.go@@ -1,12 +1,28 @@package mysqlimport (+ "bytes"+ "context"+ "crypto/ecdsa"+ "crypto/elliptic"+ "crypto/rand"+ "crypto/x509"+ "crypto/x509/pkix"+ "encoding/pem"+ "math/big"+ "net/http"+ "net/http/httptest""strings"+ "sync"+ "sync/atomic""testing"+ "time""github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db/util")func TestValidateMySQLExtraConnectionParameters(t *testing.T) {@@ -152,3 +168,330 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.Trequire.Len(t, commands, 1)require.Equal(t, statement, commands[0].Text)}++// selfSignedCAPEM builds a throwaway CA so the tests can assert that a supplied+// ssl_ca is parsed and that a served bundle is cached, without shipping a+// certificate that eventually expires.+func selfSignedCAPEM(t *testing.T) string {+ t.Helper()+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)+ require.NoError(t, err)+ template := &x509.Certificate{+ SerialNumber: big.NewInt(1),+ Subject: pkix.Name{CommonName: "bytebase-test-ca"},+ NotBefore: time.Now().Add(-time.Hour),+ NotAfter: time.Now().Add(time.Hour),+ IsCA: true,+ BasicConstraintsValid: true,+ KeyUsage: x509.KeyUsageCertSign,+ }+ der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)+ require.NoError(t, err)+ return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))+}++// serveRDSCertBundle points rdsCertBundleURL at a local server and returns the+// request counter, so a test can assert that no download happened at all.+func serveRDSCertBundle(t *testing.T, handler http.HandlerFunc) *atomic.Int32 {+ t.Helper()+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ handler(w, r)+ }))+ t.Cleanup(server.Close)++ previousURL := rdsCertBundleURL+ rdsCertBundleURL = server.URL+ rdsCertPool.Store(nil)+ t.Cleanup(func() {+ rdsCertBundleURL = previousURL+ rdsCertPool.Store(nil)+ })+ return &requests+}++func TestRDSTLSConfigSkipsDownloadWhenVerificationDisabled(t *testing.T) {+ ca := selfSignedCAPEM(t)+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(ca))+ })++ cfg, err := rdsTLSConfig(context.Background(), &storepb.DataSource{+ Host: "db.example.com",+ VerifyTlsCertificate: false,+ })+ require.NoError(t, err)+ require.True(t, cfg.InsecureSkipVerify)+ require.Nil(t, cfg.RootCAs)+ require.Nil(t, cfg.VerifyPeerCertificate)+ require.Equal(t, int32(0), requests.Load())+}++func TestRDSTLSConfigSkipsDownloadWhenSslCaConfigured(t *testing.T) {+ ca := selfSignedCAPEM(t)+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(ca))+ })++ cfg, err := rdsTLSConfig(context.Background(), &storepb.DataSource{+ Host: "db.example.com",+ VerifyTlsCertificate: true,+ SslCa: ca,+ })+ require.NoError(t, err)+ require.NotNil(t, cfg.RootCAs)+ require.NotNil(t, cfg.VerifyPeerCertificate)+ require.Equal(t, int32(0), requests.Load())+}++func TestRDSTLSConfigDownloadsBundleOnlyOnce(t *testing.T) {+ ca := selfSignedCAPEM(t)+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write([]byte(ca))+ })++ dataSource := &storepb.DataSource{Host: "db.example.com", VerifyTlsCertificate: true}+ for range 5 {+ cfg, err := rdsTLSConfig(context.Background(), dataSource)+ require.NoError(t, err)+ require.NotNil(t, cfg.RootCAs)+ }+ require.Equal(t, int32(1), requests.Load())+}++func TestRDSCertPoolDoesNotCacheFailures(t *testing.T) {+ ca := selfSignedCAPEM(t)+ var fail atomic.Bool+ fail.Store(true)+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ if fail.Load() {+ w.WriteHeader(http.StatusServiceUnavailable)+ return+ }+ _, _ = w.Write([]byte(ca))+ })++ _, err := cachedRDSCertPool(context.Background())+ require.Error(t, err)++ fail.Store(false)+ pool, err := cachedRDSCertPool(context.Background())+ require.NoError(t, err)+ require.NotNil(t, pool)+ require.Equal(t, int32(2), requests.Load())+}++func TestRDSCertPoolReportsHTTPStatus(t *testing.T) {+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusForbidden)+ _, _ = w.Write([]byte("<Error>AccessDenied</Error>"))+ })++ _, err := getRDSCertPool(context.Background())+ require.ErrorContains(t, err, "403")+}++func TestRDSRootCAsRejectsUnparseableSslCa(t *testing.T) {+ _, err := rdsRootCAs(context.Background(), &storepb.DataSource{SslCa: "not a certificate"})+ require.ErrorContains(t, err, "ssl_ca")+}++func TestRDSCertPoolRejectsOversizedResponse(t *testing.T) {+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ _, _ = w.Write(bytes.Repeat([]byte("x"), int(maxRDSCertBundleSize)+1024))+ })+ _, err := getRDSCertPool(context.Background())+ require.Error(t, err)+ require.Contains(t, err.Error(), "exceeds")+}++func TestCreateCertificateVerifierValidatesChainAndHostname(t *testing.T) {+ rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)+ require.NoError(t, err)+ rootTemplate := &x509.Certificate{+ SerialNumber: big.NewInt(1),+ Subject: pkix.Name{CommonName: "bytebase-test-root"},+ NotBefore: time.Now().Add(-time.Hour),+ NotAfter: time.Now().Add(time.Hour),+ IsCA: true,+ BasicConstraintsValid: true,+ KeyUsage: x509.KeyUsageCertSign,+ }+ rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, &rootKey.PublicKey, rootKey)+ require.NoError(t, err)+ rootCert, err := x509.ParseCertificate(rootDER)+ require.NoError(t, err)+ rootPool := x509.NewCertPool()+ rootPool.AddCert(rootCert)++ foreignKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)+ require.NoError(t, err)+ foreignTemplate := &x509.Certificate{+ SerialNumber: big.NewInt(2),+ Subject: pkix.Name{CommonName: "bytebase-test-foreign-root"},+ NotBefore: time.Now().Add(-time.Hour),+ NotAfter: time.Now().Add(time.Hour),+ IsCA: true,+ BasicConstraintsValid: true,+ KeyUsage: x509.KeyUsageCertSign,+ }+ foreignDER, err := x509.CreateCertificate(rand.Reader, foreignTemplate, foreignTemplate, &foreignKey.PublicKey, foreignKey)+ require.NoError(t, err)+ foreignCert, err := x509.ParseCertificate(foreignDER)+ require.NoError(t, err)++ // leafFor returns a DER-encoded leaf signed by parent, valid for dnsName.+ leafFor := func(parent *x509.Certificate, parentKey *ecdsa.PrivateKey, dnsName string) []byte {+ leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)+ require.NoError(t, err)+ leaf := &x509.Certificate{+ SerialNumber: big.NewInt(100),+ Subject: pkix.Name{CommonName: dnsName},+ NotBefore: time.Now().Add(-time.Hour),+ NotAfter: time.Now().Add(time.Hour),+ KeyUsage: x509.KeyUsageDigitalSignature,+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},+ DNSNames: []string{dnsName},+ }+ der, err := x509.CreateCertificate(rand.Reader, leaf, parent, &leafKey.PublicKey, parentKey)+ require.NoError(t, err)+ return der+ }++ const host = "db.example.com"+ verifier := util.CreateCertificateVerifier(rootPool, host)++ require.NoError(t, verifier([][]byte{leafFor(rootCert, rootKey, host)}, nil))+ require.Error(t, verifier([][]byte{leafFor(rootCert, rootKey, "other.example.com")}, nil))+ require.Error(t, verifier([][]byte{leafFor(foreignCert, foreignKey, host)}, nil))+}++func TestRDSCertPoolCollapsesConcurrentDownloads(t *testing.T) {+ const callers = 32+ ca := selfSignedCAPEM(t)+ // release blocks the first response until every caller is inside the fetch,+ // so a per-caller download cannot pass by finishing before the others start.+ release := make(chan struct{})+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ <-release+ _, _ = w.Write([]byte(ca))+ })++ dataSource := &storepb.DataSource{Host: "db.example.com", VerifyTlsCertificate: true}+ var ready, done sync.WaitGroup+ ready.Add(callers)+ done.Add(callers)+ pools := make([]*x509.CertPool, callers)+ errs := make([]error, callers)+ for i := range callers {+ go func() {+ defer done.Done()+ ready.Done()+ cfg, err := rdsTLSConfig(context.Background(), dataSource)+ errs[i] = err+ if err == nil {+ pools[i] = cfg.RootCAs+ }+ }()+ }+ ready.Wait()+ close(release)+ done.Wait()++ for i := range callers {+ require.NoError(t, errs[i])+ require.NotNil(t, pools[i])+ require.Same(t, pools[0], pools[i])+ }+ require.Equal(t, int32(1), requests.Load())+}++func TestRDSCertPoolWaiterObservesItsOwnCancellation(t *testing.T) {+ ca := selfSignedCAPEM(t)+ release := make(chan struct{})+ inFlight := make(chan struct{}, 1)+ var leader sync.WaitGroup+ // Registered before serveRDSCertBundle, so it runs last: the leader must finish+ // before the helper resets rdsCertPool, or it re-stores this test's pool+ // afterwards and a later cache test never contacts its own server.+ t.Cleanup(leader.Wait)+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ inFlight <- struct{}{}+ <-release+ _, _ = w.Write([]byte(ca))+ })+ // Registered after serveRDSCertBundle so this runs before its server.Close,+ // which would otherwise wait forever on the blocked handler.+ t.Cleanup(func() { close(release) })++ dataSource := &storepb.DataSource{Host: "db.example.com", VerifyTlsCertificate: true}+ // Leader occupies the flight and stays blocked in the handler.+ leader.Add(1)+ go func() {+ defer leader.Done()+ _, _ = cachedRDSCertPool(context.Background())+ }()+ <-inFlight++ // The waiter joins the in-flight download, then gives up on its own context.+ ctx, cancel := context.WithCancel(context.Background())+ waiterDone := make(chan error, 1)+ leader.Add(1)+ go func() {+ defer leader.Done()+ _, err := rdsTLSConfig(ctx, dataSource)+ waiterDone <- err+ }()+ cancel()++ select {+ case err := <-waiterDone:+ require.ErrorIs(t, err, context.Canceled)+ case <-time.After(5 * time.Second):+ require.Fail(t, "canceled waiter blocked on the shared download")+ }+}++func TestRDSCertPoolLeaderCancellationDoesNotFailWaiter(t *testing.T) {+ ca := selfSignedCAPEM(t)+ release := make(chan struct{})+ inFlight := make(chan struct{}, 1)+ var leader sync.WaitGroup+ // See the note in TestRDSCertPoolWaiterObservesItsOwnCancellation: this runs+ // last, so no background store lands after the cache reset.+ t.Cleanup(leader.Wait)+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {+ inFlight <- struct{}{}+ <-release+ _, _ = w.Write([]byte(ca))+ })++ leaderCtx, cancelLeader := context.WithCancel(context.Background())+ leader.Add(1)+ go func() {+ defer leader.Done()+ _, _ = cachedRDSCertPool(leaderCtx)+ }()+ <-inFlight++ // A second caller with a live context is waiting on the same flight.+ waiterDone := make(chan error, 1)+ leader.Add(1)+ go func() {+ defer leader.Done()+ _, err := cachedRDSCertPool(context.Background())+ waiterDone <- err+ }()++ // The leader walks away; the shared fetch must still complete for the waiter.+ cancelLeader()+ close(release)++ select {+ case err := <-waiterDone:+ require.NoError(t, err)+ case <-time.After(5 * time.Second):+ require.Fail(t, "waiter never completed after leader cancellation")+ }+}