Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(mysql): stop downloading the RDS CA bundle on every connection (#21186)

Dmitry Makarenkoa6b87a904236
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.go
index a8842045ce..e1cbe20347 100644
--- a/backend/plugin/db/mysql/mysql.go
+++ b/backend/plugin/db/mysql/mysql.go
@@ -14,6 +14,7 @@ import (
"net/http"
"regexp"
"strings"
+ "sync/atomic"
"time"
"github.com/aws/aws-sdk-go-v2/feature/rds/auth"
@@ -22,6 +23,7 @@ import (
"github.com/pkg/errors"
"go.uber.org/multierr"
"golang.org/x/crypto/ssh"
+ "golang.org/x/sync/singleflight"
"google.golang.org/protobuf/types/known/durationpb"
"github.com/bytebase/bytebase/backend/common"
@@ -154,6 +156,23 @@ func (d *Driver) getMySQLConnection(connCfg db.ConnectionConfig) (string, error)
return fmt.Sprintf("%s:%s@%s(%s:%s)/%s?%s", connCfg.DataSource.Username, connCfg.Password, protocol, connCfg.DataSource.Host, connCfg.DataSource.Port, connCfg.ConnectionContext.DatabaseName, strings.Join(params, "&")), nil
}
+// rdsCertBundleURL is a variable so tests can point it at a local server.
+var rdsCertBundleURL = "https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem"
+
+// rdsCertFetchTimeout bounds the download so a stalled fetch cannot hold a
+// connection attempt open indefinitely.
+const rdsCertFetchTimeout = 30 * time.Second
+
+// maxRDSCertBundleSize bounds the downloaded bundle so a misbehaving endpoint
+// cannot stream unbounded data into memory (the real global bundle is ~165 KB).
+const maxRDSCertBundleSize int64 = 4 << 20
+
+// rdsCertPool caches the downloaded bundle for the process lifetime.
+var rdsCertPool atomic.Pointer[x509.CertPool]
+
+// rdsCertGroup collapses concurrent cold-start downloads into one request.
+var rdsCertGroup singleflight.Group
+
// getRDSCertPool downloads and returns the RDS CA certificate pool.
// AWS RDS connection with IAM require TLS connection.
//
@@ -161,25 +180,30 @@ func (d *Driver) getMySQLConnection(connCfg db.ConnectionConfig) (string, error)
// https://github.com/aws/aws-sdk-go/issues/1248
// https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/mysql-ssl-connections.html
func getRDSCertPool(ctx context.Context) (*x509.CertPool, error) {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem", nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, rdsCertBundleURL, nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to build request for rds cert")
}
- client := &http.Client{}
+ client := &http.Client{Timeout: rdsCertFetchTimeout}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
- pem, err := io.ReadAll(resp.Body)
+ // Without this an error page would reach AppendCertsFromPEM and surface as
+ // "failed to parse RDS CA certificates", hiding the real cause.
+ if resp.StatusCode != http.StatusOK {
+ return nil, errors.Errorf("failed to download RDS CA certificates: %s", resp.Status)
+ }
+
+ pem, err := io.ReadAll(io.LimitReader(resp.Body, maxRDSCertBundleSize+1))
if err != nil {
return nil, err
}
-
- if err := resp.Body.Close(); err != nil {
- return nil, errors.Wrapf(err, "failed to close response")
+ if int64(len(pem)) > maxRDSCertBundleSize {
+ return nil, errors.Errorf("RDS CA bundle exceeds %d bytes", maxRDSCertBundleSize)
}
rootCertPool := x509.NewCertPool()
@@ -190,6 +214,75 @@ func getRDSCertPool(ctx context.Context) (*x509.CertPool, error) {
return rootCertPool, nil
}
+// cachedRDSCertPool returns the AWS RDS CA bundle, downloading it once per
+// process. Only successes are cached, so a transient failure can still recover.
+func cachedRDSCertPool(ctx context.Context) (*x509.CertPool, error) {
+ if pool := rdsCertPool.Load(); pool != nil {
+ return pool, nil
+ }
+ // Collapse a cold-start burst into one download. The shared fetch does not
+ // inherit any one caller's context, so the caller that starts it cannot
+ // cancel it for everybody else; each caller waits on its own context below.
+ ch := rdsCertGroup.DoChan("", func() (any, error) {
+ // Double check after entering singleflight.
+ if pool := rdsCertPool.Load(); pool != nil {
+ return pool, nil
+ }
+ fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rdsCertFetchTimeout)
+ defer cancel()
+ pool, err := getRDSCertPool(fetchCtx)
+ if err != nil {
+ return nil, err
+ }
+ rdsCertPool.Store(pool)
+ return pool, nil
+ })
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case res := <-ch:
+ if res.Err != nil {
+ return nil, res.Err
+ }
+ pool, ok := res.Val.(*x509.CertPool)
+ if !ok {
+ return nil, errors.Errorf("unexpected RDS cert pool type %T", res.Val)
+ }
+ return pool, nil
+ }
+}
+
+// rdsRootCAs prefers an operator-supplied CA, so a deployment can serve RDS IAM
+// connections without any outbound network access.
+func rdsRootCAs(ctx context.Context, dataSource *storepb.DataSource) (*x509.CertPool, error) {
+ if ca := dataSource.GetSslCa(); ca != "" {
+ pool := x509.NewCertPool()
+ if !pool.AppendCertsFromPEM([]byte(ca)) {
+ return nil, errors.Errorf("failed to parse ssl_ca certificates")
+ }
+ return pool, nil
+ }
+ return cachedRDSCertPool(ctx)
+}
+
+// rdsTLSConfig builds the TLS config for an RDS IAM connection. IAM auth always
+// requires TLS, so this always returns a config.
+func rdsTLSConfig(ctx context.Context, dataSource *storepb.DataSource) (*tls.Config, error) {
+ if !dataSource.GetVerifyTlsCertificate() {
+ // No verifier runs in this mode, so a root pool would never be consulted.
+ return &tls.Config{InsecureSkipVerify: true}, nil // NOSONAR(go:S4830,go:S5527) operator disabled verification (verify_tls_certificate=false)
+ }
+ rootCertPool, err := rdsRootCAs(ctx, dataSource)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to get RDS cert pool")
+ }
+ return &tls.Config{ // NOSONAR(go:S4830) VerifyPeerCertificate below verifies chain and hostname
+ RootCAs: rootCertPool,
+ InsecureSkipVerify: true, // superseded by VerifyPeerCertificate below
+ VerifyPeerCertificate: util.CreateCertificateVerifier(rootCertPool, dataSource.GetHost()),
+ }, nil
+}
+
// getRDSConnection returns the connection string with IAM for AWS RDS.
//
// refs:
@@ -213,31 +306,14 @@ func (d *Driver) getRDSConnection(ctx context.Context, connCfg db.ConnectionConf
return "", errors.Wrap(err, "failed to create authentication token")
}
- // Get RDS CA certificate pool
- rootCertPool, err := getRDSCertPool(ctx)
+ tlsConfig, err := rdsTLSConfig(ctx, connCfg.DataSource)
if err != nil {
- return "", errors.Wrap(err, "failed to get RDS cert pool")
+ return "", err
}
// Create TLS config with unique name for this connection
tlsKey := uuid.NewString()
- var tlsConfig *tls.Config
- if connCfg.DataSource.GetVerifyTlsCertificate() {
- // Secure config with certificate verification
- tlsConfig = &tls.Config{
- RootCAs: rootCertPool,
- InsecureSkipVerify: true, // We use custom verification
- }
- tlsConfig.VerifyPeerCertificate = util.CreateCertificateVerifier(rootCertPool, connCfg.DataSource.Host)
- } else {
- // Backward compatible config without verification
- tlsConfig = &tls.Config{
- RootCAs: rootCertPool,
- InsecureSkipVerify: true,
- }
- }
-
// Register the TLS config with unique name
if err := mysql.RegisterTLSConfig(tlsKey, tlsConfig); err != nil {
return "", errors.Wrap(err, "failed to register RDS TLS config")
diff --git a/backend/plugin/db/mysql/mysql_test.go b/backend/plugin/db/mysql/mysql_test.go
index 32d326bdf5..4c525b8cbd 100644
--- a/backend/plugin/db/mysql/mysql_test.go
+++ b/backend/plugin/db/mysql/mysql_test.go
@@ -1,12 +1,28 @@
package mysql
import (
+ "bytes"
+ "context"
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/pem"
+ "math/big"
+ "net/http"
+ "net/http/httptest"
"strings"
+ "sync"
+ "sync/atomic"
"testing"
+ "time"
"github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/common"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db/util"
)
func TestValidateMySQLExtraConnectionParameters(t *testing.T) {
@@ -152,3 +168,330 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.T
require.Len(t, commands, 1)
require.Equal(t, statement, commands[0].Text)
}
+
+// selfSignedCAPEM builds a throwaway CA so the tests can assert that a supplied
+// ssl_ca is parsed and that a served bundle is cached, without shipping a
+// certificate that eventually expires.
+func selfSignedCAPEM(t *testing.T) string {
+ t.Helper()
+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ require.NoError(t, err)
+ template := &x509.Certificate{
+ SerialNumber: big.NewInt(1),
+ Subject: pkix.Name{CommonName: "bytebase-test-ca"},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(time.Hour),
+ IsCA: true,
+ BasicConstraintsValid: true,
+ KeyUsage: x509.KeyUsageCertSign,
+ }
+ der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+ require.NoError(t, err)
+ return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
+}
+
+// serveRDSCertBundle points rdsCertBundleURL at a local server and returns the
+// request counter, so a test can assert that no download happened at all.
+func serveRDSCertBundle(t *testing.T, handler http.HandlerFunc) *atomic.Int32 {
+ t.Helper()
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ handler(w, r)
+ }))
+ t.Cleanup(server.Close)
+
+ previousURL := rdsCertBundleURL
+ rdsCertBundleURL = server.URL
+ rdsCertPool.Store(nil)
+ t.Cleanup(func() {
+ rdsCertBundleURL = previousURL
+ rdsCertPool.Store(nil)
+ })
+ return &requests
+}
+
+func TestRDSTLSConfigSkipsDownloadWhenVerificationDisabled(t *testing.T) {
+ ca := selfSignedCAPEM(t)
+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(ca))
+ })
+
+ cfg, err := rdsTLSConfig(context.Background(), &storepb.DataSource{
+ Host: "db.example.com",
+ VerifyTlsCertificate: false,
+ })
+ require.NoError(t, err)
+ require.True(t, cfg.InsecureSkipVerify)
+ require.Nil(t, cfg.RootCAs)
+ require.Nil(t, cfg.VerifyPeerCertificate)
+ require.Equal(t, int32(0), requests.Load())
+}
+
+func TestRDSTLSConfigSkipsDownloadWhenSslCaConfigured(t *testing.T) {
+ ca := selfSignedCAPEM(t)
+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(ca))
+ })
+
+ cfg, err := rdsTLSConfig(context.Background(), &storepb.DataSource{
+ Host: "db.example.com",
+ VerifyTlsCertificate: true,
+ SslCa: ca,
+ })
+ require.NoError(t, err)
+ require.NotNil(t, cfg.RootCAs)
+ require.NotNil(t, cfg.VerifyPeerCertificate)
+ require.Equal(t, int32(0), requests.Load())
+}
+
+func TestRDSTLSConfigDownloadsBundleOnlyOnce(t *testing.T) {
+ ca := selfSignedCAPEM(t)
+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte(ca))
+ })
+
+ dataSource := &storepb.DataSource{Host: "db.example.com", VerifyTlsCertificate: true}
+ for range 5 {
+ cfg, err := rdsTLSConfig(context.Background(), dataSource)
+ require.NoError(t, err)
+ require.NotNil(t, cfg.RootCAs)
+ }
+ require.Equal(t, int32(1), requests.Load())
+}
+
+func TestRDSCertPoolDoesNotCacheFailures(t *testing.T) {
+ ca := selfSignedCAPEM(t)
+ var fail atomic.Bool
+ fail.Store(true)
+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ if fail.Load() {
+ w.WriteHeader(http.StatusServiceUnavailable)
+ return
+ }
+ _, _ = w.Write([]byte(ca))
+ })
+
+ _, err := cachedRDSCertPool(context.Background())
+ require.Error(t, err)
+
+ fail.Store(false)
+ pool, err := cachedRDSCertPool(context.Background())
+ require.NoError(t, err)
+ require.NotNil(t, pool)
+ require.Equal(t, int32(2), requests.Load())
+}
+
+func TestRDSCertPoolReportsHTTPStatus(t *testing.T) {
+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = w.Write([]byte("<Error>AccessDenied</Error>"))
+ })
+
+ _, err := getRDSCertPool(context.Background())
+ require.ErrorContains(t, err, "403")
+}
+
+func TestRDSRootCAsRejectsUnparseableSslCa(t *testing.T) {
+ _, err := rdsRootCAs(context.Background(), &storepb.DataSource{SslCa: "not a certificate"})
+ require.ErrorContains(t, err, "ssl_ca")
+}
+
+func TestRDSCertPoolRejectsOversizedResponse(t *testing.T) {
+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write(bytes.Repeat([]byte("x"), int(maxRDSCertBundleSize)+1024))
+ })
+ _, err := getRDSCertPool(context.Background())
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "exceeds")
+}
+
+func TestCreateCertificateVerifierValidatesChainAndHostname(t *testing.T) {
+ rootKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ require.NoError(t, err)
+ rootTemplate := &x509.Certificate{
+ SerialNumber: big.NewInt(1),
+ Subject: pkix.Name{CommonName: "bytebase-test-root"},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(time.Hour),
+ IsCA: true,
+ BasicConstraintsValid: true,
+ KeyUsage: x509.KeyUsageCertSign,
+ }
+ rootDER, err := x509.CreateCertificate(rand.Reader, rootTemplate, rootTemplate, &rootKey.PublicKey, rootKey)
+ require.NoError(t, err)
+ rootCert, err := x509.ParseCertificate(rootDER)
+ require.NoError(t, err)
+ rootPool := x509.NewCertPool()
+ rootPool.AddCert(rootCert)
+
+ foreignKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ require.NoError(t, err)
+ foreignTemplate := &x509.Certificate{
+ SerialNumber: big.NewInt(2),
+ Subject: pkix.Name{CommonName: "bytebase-test-foreign-root"},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(time.Hour),
+ IsCA: true,
+ BasicConstraintsValid: true,
+ KeyUsage: x509.KeyUsageCertSign,
+ }
+ foreignDER, err := x509.CreateCertificate(rand.Reader, foreignTemplate, foreignTemplate, &foreignKey.PublicKey, foreignKey)
+ require.NoError(t, err)
+ foreignCert, err := x509.ParseCertificate(foreignDER)
+ require.NoError(t, err)
+
+ // leafFor returns a DER-encoded leaf signed by parent, valid for dnsName.
+ leafFor := func(parent *x509.Certificate, parentKey *ecdsa.PrivateKey, dnsName string) []byte {
+ leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ require.NoError(t, err)
+ leaf := &x509.Certificate{
+ SerialNumber: big.NewInt(100),
+ Subject: pkix.Name{CommonName: dnsName},
+ NotBefore: time.Now().Add(-time.Hour),
+ NotAfter: time.Now().Add(time.Hour),
+ KeyUsage: x509.KeyUsageDigitalSignature,
+ ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+ DNSNames: []string{dnsName},
+ }
+ der, err := x509.CreateCertificate(rand.Reader, leaf, parent, &leafKey.PublicKey, parentKey)
+ require.NoError(t, err)
+ return der
+ }
+
+ const host = "db.example.com"
+ verifier := util.CreateCertificateVerifier(rootPool, host)
+
+ require.NoError(t, verifier([][]byte{leafFor(rootCert, rootKey, host)}, nil))
+ require.Error(t, verifier([][]byte{leafFor(rootCert, rootKey, "other.example.com")}, nil))
+ require.Error(t, verifier([][]byte{leafFor(foreignCert, foreignKey, host)}, nil))
+}
+
+func TestRDSCertPoolCollapsesConcurrentDownloads(t *testing.T) {
+ const callers = 32
+ ca := selfSignedCAPEM(t)
+ // release blocks the first response until every caller is inside the fetch,
+ // so a per-caller download cannot pass by finishing before the others start.
+ release := make(chan struct{})
+ requests := serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ <-release
+ _, _ = w.Write([]byte(ca))
+ })
+
+ dataSource := &storepb.DataSource{Host: "db.example.com", VerifyTlsCertificate: true}
+ var ready, done sync.WaitGroup
+ ready.Add(callers)
+ done.Add(callers)
+ pools := make([]*x509.CertPool, callers)
+ errs := make([]error, callers)
+ for i := range callers {
+ go func() {
+ defer done.Done()
+ ready.Done()
+ cfg, err := rdsTLSConfig(context.Background(), dataSource)
+ errs[i] = err
+ if err == nil {
+ pools[i] = cfg.RootCAs
+ }
+ }()
+ }
+ ready.Wait()
+ close(release)
+ done.Wait()
+
+ for i := range callers {
+ require.NoError(t, errs[i])
+ require.NotNil(t, pools[i])
+ require.Same(t, pools[0], pools[i])
+ }
+ require.Equal(t, int32(1), requests.Load())
+}
+
+func TestRDSCertPoolWaiterObservesItsOwnCancellation(t *testing.T) {
+ ca := selfSignedCAPEM(t)
+ release := make(chan struct{})
+ inFlight := make(chan struct{}, 1)
+ var leader sync.WaitGroup
+ // Registered before serveRDSCertBundle, so it runs last: the leader must finish
+ // before the helper resets rdsCertPool, or it re-stores this test's pool
+ // afterwards and a later cache test never contacts its own server.
+ t.Cleanup(leader.Wait)
+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ inFlight <- struct{}{}
+ <-release
+ _, _ = w.Write([]byte(ca))
+ })
+ // Registered after serveRDSCertBundle so this runs before its server.Close,
+ // which would otherwise wait forever on the blocked handler.
+ t.Cleanup(func() { close(release) })
+
+ dataSource := &storepb.DataSource{Host: "db.example.com", VerifyTlsCertificate: true}
+ // Leader occupies the flight and stays blocked in the handler.
+ leader.Add(1)
+ go func() {
+ defer leader.Done()
+ _, _ = cachedRDSCertPool(context.Background())
+ }()
+ <-inFlight
+
+ // The waiter joins the in-flight download, then gives up on its own context.
+ ctx, cancel := context.WithCancel(context.Background())
+ waiterDone := make(chan error, 1)
+ leader.Add(1)
+ go func() {
+ defer leader.Done()
+ _, err := rdsTLSConfig(ctx, dataSource)
+ waiterDone <- err
+ }()
+ cancel()
+
+ select {
+ case err := <-waiterDone:
+ require.ErrorIs(t, err, context.Canceled)
+ case <-time.After(5 * time.Second):
+ require.Fail(t, "canceled waiter blocked on the shared download")
+ }
+}
+
+func TestRDSCertPoolLeaderCancellationDoesNotFailWaiter(t *testing.T) {
+ ca := selfSignedCAPEM(t)
+ release := make(chan struct{})
+ inFlight := make(chan struct{}, 1)
+ var leader sync.WaitGroup
+ // See the note in TestRDSCertPoolWaiterObservesItsOwnCancellation: this runs
+ // last, so no background store lands after the cache reset.
+ t.Cleanup(leader.Wait)
+ serveRDSCertBundle(t, func(w http.ResponseWriter, _ *http.Request) {
+ inFlight <- struct{}{}
+ <-release
+ _, _ = w.Write([]byte(ca))
+ })
+
+ leaderCtx, cancelLeader := context.WithCancel(context.Background())
+ leader.Add(1)
+ go func() {
+ defer leader.Done()
+ _, _ = cachedRDSCertPool(leaderCtx)
+ }()
+ <-inFlight
+
+ // A second caller with a live context is waiting on the same flight.
+ waiterDone := make(chan error, 1)
+ leader.Add(1)
+ go func() {
+ defer leader.Done()
+ _, err := cachedRDSCertPool(context.Background())
+ waiterDone <- err
+ }()
+
+ // The leader walks away; the shared fetch must still complete for the waiter.
+ cancelLeader()
+ close(release)
+
+ select {
+ case err := <-waiterDone:
+ require.NoError(t, err)
+ case <-time.After(5 * time.Second):
+ require.Fail(t, "waiter never completed after leader cancellation")
+ }
+}