Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(deps): update go-redis to v9.23.0, keeping its new defaults off (#21858)
renovate[bot]0a2fb97c5c81
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.goindex 5f9004d3ea..e98bc98854 100644--- a/backend/common/testcontainer/shared.go+++ b/backend/common/testcontainer/shared.go@@ -29,7 +29,8 @@ var (sharedTrino = startOnce(getTrinoContainer)sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))// 10.7 is the oldest MariaDB release Bytebase supports.- sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))+ sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))+ sharedRedisCluster = startOnce(getRedisClusterContainer)startedMu sync.Mutexstarted []*Container@@ -131,6 +132,10 @@ func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMa// oldest supported release; tests take a database each with NewMariaDB107Database.func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, sharedMariaDB107) }+// SharedRedisClusterContainer is a one-node Redis cluster that owns every hash+// slot, for the driver paths that need a *redis.ClusterClient.+func SharedRedisClusterContainer(t testing.TB) *Container { return shared(t, sharedRedisCluster) }+// NewPgDatabase creates an empty database for the test on the shared// PostgreSQL container and returns its name with a superuser handle to it.// Anything cluster-wide the test creates, such as a role, should be nameddiff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.goindex 1b540a67e1..535ba83234 100644--- a/backend/common/testcontainer/testcontainer.go+++ b/backend/common/testcontainer/testcontainer.go@@ -10,6 +10,7 @@ import ("database/sql""encoding/pem""fmt"+ "io""log/slog""math/big""net"@@ -22,6 +23,7 @@ import ("github.com/pkg/errors"goora "github.com/sijms/go-ora/v2""github.com/testcontainers/testcontainers-go"+ tcexec "github.com/testcontainers/testcontainers-go/exec""github.com/testcontainers/testcontainers-go/wait")@@ -469,3 +471,79 @@ func createTLSMaterial(ips ...net.IP) (string, string, []byte, []byte, error) {}return tlsDir, caPath, certificate, key, nil}++// RedisClusterSlots is the number of hash slots a Redis cluster partitions keys+// into; the single node getRedisClusterContainer starts owns all of them.+const RedisClusterSlots = 16384++// getRedisClusterContainer starts a one-node Redis cluster. Cluster clients+// follow the addresses CLUSTER SLOTS reports, so the node is told to announce+// the host-mapped address rather than its address inside Docker, which a test+// outside Docker cannot reach.+func getRedisClusterContainer(ctx context.Context) (retC *Container, retErr error) {+ c, err := start(ctx, testcontainers.ContainerRequest{+ Image: "redis:7.2-alpine",+ Cmd: []string{+ "redis-server", "--cluster-enabled", "yes",+ "--cluster-config-file", "/tmp/nodes.conf", "--cluster-node-timeout", "5000",+ },+ WaitingFor: wait.ForLog("Ready to accept connections").WithStartupTimeout(3 * time.Minute),+ }, spec{port: "6379/tcp"})+ if err != nil {+ return nil, err+ }+ defer func() {+ if retErr != nil {+ c.Close(ctx)+ }+ }()++ if _, err := redisCLI(ctx, c, "config", "set", "cluster-announce-ip", c.host); err != nil {+ return nil, err+ }+ if _, err := redisCLI(ctx, c, "config", "set", "cluster-announce-port", c.port); err != nil {+ return nil, err+ }+ if _, err := redisCLI(ctx, c, "cluster", "addslotsrange", "0", strconv.Itoa(RedisClusterSlots-1)); err != nil {+ return nil, err+ }+ // Owning every slot is not enough on its own: the node reports state fail+ // until its cluster cron has taken the new slots into account.+ ticker := time.NewTicker(200 * time.Millisecond)+ defer ticker.Stop()+ timeout := time.After(2 * time.Minute)+ for {+ info, err := redisCLI(ctx, c, "cluster", "info")+ if err != nil {+ return nil, err+ }+ if strings.Contains(info, "cluster_state:ok") {+ return c, nil+ }+ select {+ case <-ticker.C:+ case <-timeout:+ return nil, errors.Errorf("redis cluster did not reach state ok: %s", info)+ case <-ctx.Done():+ return nil, ctx.Err()+ }+ }+}++// redisCLI runs redis-cli inside the container, which keeps the cluster+// bootstrap off the host and out of this package's dependencies. Exec output is+// a multiplexed Docker stream, so Multiplexed decodes it into plain text.+func redisCLI(ctx context.Context, c *Container, args ...string) (string, error) {+ code, reader, err := c.container.Exec(ctx, append([]string{"redis-cli"}, args...), tcexec.Multiplexed())+ if err != nil {+ return "", errors.Wrapf(err, "exec redis-cli %s", strings.Join(args, " "))+ }+ out, err := io.ReadAll(reader)+ if err != nil {+ return "", errors.Wrapf(err, "read redis-cli %s output", strings.Join(args, " "))+ }+ if code != 0 {+ return "", errors.Errorf("redis-cli %s exited %d: %s", strings.Join(args, " "), code, out)+ }+ return string(out), nil+}diff --git a/backend/plugin/db/redis/main_test.go b/backend/plugin/db/redis/main_test.gonew file mode 100644index 0000000000..6a7383ee4a--- /dev/null+++ b/backend/plugin/db/redis/main_test.go@@ -0,0 +1,9 @@+package redis++import (+ "testing"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+)++func TestMain(m *testing.M) { testcontainer.Main(m) }diff --git a/backend/plugin/db/redis/redis.go b/backend/plugin/db/redis/redis.goindex f3eecafabd..aab591c69b 100644--- a/backend/plugin/db/redis/redis.go+++ b/backend/plugin/db/redis/redis.go@@ -9,6 +9,7 @@ import ("net""strconv""strings"+ "sync""time"metadatapb "github.com/bytebase/omni/metadata"@@ -68,6 +69,10 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionCPassword: config.Password,TLSConfig: tlscfg,DB: db,+ // Pipelines run on the main pool. A driver is opened per request+ // and runs a single pipeline, so go-redis's dedicated pipeline pool+ // would only add a connection and its handshake to every request.+ PipelinePoolSize: -1,}if config.DataSource.GetSshHost() != "" {sshClient, err := util.GetSSHClient(config.DataSource)@@ -102,6 +107,7 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionCSentinelAddrs: sentinelAddrs,DB: db,TLSConfig: tlscfg,+ PipelinePoolSize: -1,}if config.DataSource.GetSshHost() != "" {sshClient, err := util.GetSSHClient(config.DataSource)@@ -128,10 +134,17 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionCaddrs = append(addrs, fmt.Sprintf("%s:%s", addr.Host, addr.Port))}options := &redis.ClusterOptions{- Addrs: addrs,- Username: config.DataSource.Username,- Password: config.Password,- TLSConfig: tlscfg,+ Addrs: addrs,+ Username: config.DataSource.Username,+ Password: config.Password,+ TLSConfig: tlscfg,+ PipelinePoolSize: -1,+ // go-redis 9.22.0 raised its own default to a minute. A node that+ // refuses connections does not trigger a reload, so a failover is+ // only noticed by this periodic one, and a driver that outlives it+ // keeps routing to the master it has lost.+ // https://github.com/redis/go-redis/issues/3979+ ClusterStateReloadInterval: 10 * time.Second,}if config.DataSource.GetSshHost() != "" {sshClient, err := util.GetSSHClient(config.DataSource)@@ -279,20 +292,10 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, qreturn nil, errors.Errorf("expect *redis.ClusterClient, but get %T", d.rdb)}+ // ForEachShard runs the callbacks concurrently and returns once they+ // have all finished, so the mutex is what orders the appends.+ var mu sync.Mutexvar cmdss [][]*redis.Cmd- cmdsChan := make(chan []*redis.Cmd)- stopChan := make(chan struct{})- go func() {- for {- select {- case cmds := <-cmdsChan:- cmdss = append(cmdss, cmds)- case <-stopChan:- return- }- }- }()-err := cluster.ForEachShard(ctx, func(ctx context.Context, client *redis.Client) error {var cmds []*redis.Cmd_, _ = client.Pipelined(ctx, func(p redis.Pipeliner) error {@@ -302,10 +305,11 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q}return nil})- cmdsChan <- cmds+ mu.Lock()+ defer mu.Unlock()+ cmdss = append(cmdss, cmds)return nil})- close(stopChan)if err != nil && err != redis.Nil {return nil, errors.Wrapf(err, "failed to query for each shard")}diff --git a/backend/plugin/db/redis/redis_test.go b/backend/plugin/db/redis/redis_test.goindex 64855c679f..952f8b01f6 100644--- a/backend/plugin/db/redis/redis_test.go+++ b/backend/plugin/db/redis/redis_test.go@@ -1,9 +1,15 @@package redisimport (+ "context""testing""github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/db")func TestSplitCommands(t *testing.T) {@@ -53,3 +59,39 @@ func TestSplitCommandsRejectsAnUnclosedQuote(t *testing.T) {_, _, err := splitCommands(`SET a "unclosed`)require.Error(t, err)}++// TestQueryConnOnAllNodes covers the ALL_NODES query option, whose per-shard+// replies used to be collected by a goroutine the caller never waited for, so+// the results were read before any of them had been recorded and every command+// reported no rows.+func TestQueryConnOnAllNodes(t *testing.T) {+ ctx := context.Background()+ container := testcontainer.SharedRedisClusterContainer(t)++ d, err := newDriver().Open(ctx, storepb.Engine_REDIS, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Host: container.GetHost(),+ Port: container.GetPort(),+ RedisType: storepb.DataSource_CLUSTER,+ },+ })+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, d.Close(ctx)) })++ _, err = d.Execute(ctx, "SET allnodes:key allnodes-value", db.ExecuteOptions{})+ require.NoError(t, err)++ results, err := d.QueryConn(ctx, nil, "GET allnodes:key\nDBSIZE", db.QueryContext{+ MaximumSQLResultSize: 1 << 20,+ Option: &v1pb.QueryOption{RedisRunCommandsOn: v1pb.QueryOption_ALL_NODES},+ })+ require.NoError(t, err)+ require.Len(t, results, 2)++ for i, want := range []string{"allnodes-value", "1"} {+ require.Len(t, results[i].Rows, 1, "statement %q returned no row", results[i].Statement)+ require.Equal(t, want, results[i].Rows[0].Values[1].GetStringValue())+ require.Equal(t, int64(1), results[i].RowsCount)+ }+}diff --git a/go.mod b/go.modindex 35defcd7ce..f79c866e6a 100644--- a/go.mod+++ b/go.mod@@ -83,7 +83,7 @@ require (github.com/prometheus/client_golang v1.25.0github.com/prometheus/client_model v0.6.3github.com/prometheus/common v0.73.1- github.com/redis/go-redis/v9 v9.21.0+ github.com/redis/go-redis/v9 v9.23.0github.com/shopspring/decimal v1.6.0github.com/sijms/go-ora/v2 v2.9.0github.com/snowflakedb/gosnowflake/v2 v2.3.0@@ -260,7 +260,7 @@ require (github.com/jcmturner/rpc/v2 v2.0.3 // indirectgithub.com/json-iterator/go v1.1.12 // indirectgithub.com/klauspost/compress v1.20.3 // indirect- github.com/klauspost/cpuid/v2 v2.2.11 // indirect+ github.com/klauspost/cpuid/v2 v2.4.0 // indirectgithub.com/kr/pretty v0.3.1 // indirectgithub.com/kr/text v0.2.0 // indirectgithub.com/kylelemons/godebug v1.1.0 // indirect@@ -350,7 +350,7 @@ require (go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirectgo.opentelemetry.io/otel/trace v1.47.0 // indirectgo.opentelemetry.io/proto/otlp v1.11.0 // indirect- go.uber.org/atomic v1.11.0 // indirect+ go.uber.org/atomic v1.12.0 // indirectgo.uber.org/zap v1.28.0 // indirectgo.yaml.in/yaml/v3 v3.0.5 // indirectgolang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e // indirectdiff --git a/go.sum b/go.sumindex 6257dbda5b..4f45a2f1de 100644--- a/go.sum+++ b/go.sum@@ -667,8 +667,8 @@ github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfgithub.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=-github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=-github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=+github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=@@ -860,8 +860,8 @@ github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJgithub.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64=github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0=github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI=-github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E=-github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=+github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=+github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=github.com/richardlehane/mscfb v1.0.9 h1:8xdd9auUvXbFoCw3L9h1spnQHZgjNsSX+ek46J6A9tE=@@ -1092,8 +1092,9 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=-go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=+go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=+go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=