Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(deps): update go-redis to v9.23.0, keeping its new defaults off (#21858)

renovate[bot]0a2fb97c5c81
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.go
index 5f9004d3ea..e98bc98854 100644
--- a/backend/common/testcontainer/shared.go
+++ b/backend/common/testcontainer/shared.go
@@ -29,7 +29,8 @@ var (
sharedTrino = startOnce(getTrinoContainer)
sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))
// 10.7 is the oldest MariaDB release Bytebase supports.
- sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))
+ sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))
+ sharedRedisCluster = startOnce(getRedisClusterContainer)
startedMu sync.Mutex
started []*Container
@@ -131,6 +132,10 @@ func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMa
// oldest supported release; tests take a database each with NewMariaDB107Database.
func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, sharedMariaDB107) }
+// SharedRedisClusterContainer is a one-node Redis cluster that owns every hash
+// slot, for the driver paths that need a *redis.ClusterClient.
+func SharedRedisClusterContainer(t testing.TB) *Container { return shared(t, sharedRedisCluster) }
+
// NewPgDatabase creates an empty database for the test on the shared
// PostgreSQL container and returns its name with a superuser handle to it.
// Anything cluster-wide the test creates, such as a role, should be named
diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.go
index 1b540a67e1..535ba83234 100644
--- a/backend/common/testcontainer/testcontainer.go
+++ b/backend/common/testcontainer/testcontainer.go
@@ -10,6 +10,7 @@ import (
"database/sql"
"encoding/pem"
"fmt"
+ "io"
"log/slog"
"math/big"
"net"
@@ -22,6 +23,7 @@ import (
"github.com/pkg/errors"
goora "github.com/sijms/go-ora/v2"
"github.com/testcontainers/testcontainers-go"
+ tcexec "github.com/testcontainers/testcontainers-go/exec"
"github.com/testcontainers/testcontainers-go/wait"
)
@@ -469,3 +471,79 @@ func createTLSMaterial(ips ...net.IP) (string, string, []byte, []byte, error) {
}
return tlsDir, caPath, certificate, key, nil
}
+
+// RedisClusterSlots is the number of hash slots a Redis cluster partitions keys
+// into; the single node getRedisClusterContainer starts owns all of them.
+const RedisClusterSlots = 16384
+
+// getRedisClusterContainer starts a one-node Redis cluster. Cluster clients
+// follow the addresses CLUSTER SLOTS reports, so the node is told to announce
+// the host-mapped address rather than its address inside Docker, which a test
+// outside Docker cannot reach.
+func getRedisClusterContainer(ctx context.Context) (retC *Container, retErr error) {
+ c, err := start(ctx, testcontainers.ContainerRequest{
+ Image: "redis:7.2-alpine",
+ Cmd: []string{
+ "redis-server", "--cluster-enabled", "yes",
+ "--cluster-config-file", "/tmp/nodes.conf", "--cluster-node-timeout", "5000",
+ },
+ WaitingFor: wait.ForLog("Ready to accept connections").WithStartupTimeout(3 * time.Minute),
+ }, spec{port: "6379/tcp"})
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if retErr != nil {
+ c.Close(ctx)
+ }
+ }()
+
+ if _, err := redisCLI(ctx, c, "config", "set", "cluster-announce-ip", c.host); err != nil {
+ return nil, err
+ }
+ if _, err := redisCLI(ctx, c, "config", "set", "cluster-announce-port", c.port); err != nil {
+ return nil, err
+ }
+ if _, err := redisCLI(ctx, c, "cluster", "addslotsrange", "0", strconv.Itoa(RedisClusterSlots-1)); err != nil {
+ return nil, err
+ }
+ // Owning every slot is not enough on its own: the node reports state fail
+ // until its cluster cron has taken the new slots into account.
+ ticker := time.NewTicker(200 * time.Millisecond)
+ defer ticker.Stop()
+ timeout := time.After(2 * time.Minute)
+ for {
+ info, err := redisCLI(ctx, c, "cluster", "info")
+ if err != nil {
+ return nil, err
+ }
+ if strings.Contains(info, "cluster_state:ok") {
+ return c, nil
+ }
+ select {
+ case <-ticker.C:
+ case <-timeout:
+ return nil, errors.Errorf("redis cluster did not reach state ok: %s", info)
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+ }
+}
+
+// redisCLI runs redis-cli inside the container, which keeps the cluster
+// bootstrap off the host and out of this package's dependencies. Exec output is
+// a multiplexed Docker stream, so Multiplexed decodes it into plain text.
+func redisCLI(ctx context.Context, c *Container, args ...string) (string, error) {
+ code, reader, err := c.container.Exec(ctx, append([]string{"redis-cli"}, args...), tcexec.Multiplexed())
+ if err != nil {
+ return "", errors.Wrapf(err, "exec redis-cli %s", strings.Join(args, " "))
+ }
+ out, err := io.ReadAll(reader)
+ if err != nil {
+ return "", errors.Wrapf(err, "read redis-cli %s output", strings.Join(args, " "))
+ }
+ if code != 0 {
+ return "", errors.Errorf("redis-cli %s exited %d: %s", strings.Join(args, " "), code, out)
+ }
+ return string(out), nil
+}
diff --git a/backend/plugin/db/redis/main_test.go b/backend/plugin/db/redis/main_test.go
new file mode 100644
index 0000000000..6a7383ee4a
--- /dev/null
+++ b/backend/plugin/db/redis/main_test.go
@@ -0,0 +1,9 @@
+package redis
+
+import (
+ "testing"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+)
+
+func TestMain(m *testing.M) { testcontainer.Main(m) }
diff --git a/backend/plugin/db/redis/redis.go b/backend/plugin/db/redis/redis.go
index f3eecafabd..aab591c69b 100644
--- a/backend/plugin/db/redis/redis.go
+++ b/backend/plugin/db/redis/redis.go
@@ -9,6 +9,7 @@ import (
"net"
"strconv"
"strings"
+ "sync"
"time"
metadatapb "github.com/bytebase/omni/metadata"
@@ -68,6 +69,10 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionC
Password: config.Password,
TLSConfig: tlscfg,
DB: db,
+ // Pipelines run on the main pool. A driver is opened per request
+ // and runs a single pipeline, so go-redis's dedicated pipeline pool
+ // would only add a connection and its handshake to every request.
+ PipelinePoolSize: -1,
}
if config.DataSource.GetSshHost() != "" {
sshClient, err := util.GetSSHClient(config.DataSource)
@@ -102,6 +107,7 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionC
SentinelAddrs: sentinelAddrs,
DB: db,
TLSConfig: tlscfg,
+ PipelinePoolSize: -1,
}
if config.DataSource.GetSshHost() != "" {
sshClient, err := util.GetSSHClient(config.DataSource)
@@ -128,10 +134,17 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionC
addrs = append(addrs, fmt.Sprintf("%s:%s", addr.Host, addr.Port))
}
options := &redis.ClusterOptions{
- Addrs: addrs,
- Username: config.DataSource.Username,
- Password: config.Password,
- TLSConfig: tlscfg,
+ Addrs: addrs,
+ Username: config.DataSource.Username,
+ Password: config.Password,
+ TLSConfig: tlscfg,
+ PipelinePoolSize: -1,
+ // go-redis 9.22.0 raised its own default to a minute. A node that
+ // refuses connections does not trigger a reload, so a failover is
+ // only noticed by this periodic one, and a driver that outlives it
+ // keeps routing to the master it has lost.
+ // https://github.com/redis/go-redis/issues/3979
+ ClusterStateReloadInterval: 10 * time.Second,
}
if config.DataSource.GetSshHost() != "" {
sshClient, err := util.GetSSHClient(config.DataSource)
@@ -279,20 +292,10 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q
return nil, errors.Errorf("expect *redis.ClusterClient, but get %T", d.rdb)
}
+ // ForEachShard runs the callbacks concurrently and returns once they
+ // have all finished, so the mutex is what orders the appends.
+ var mu sync.Mutex
var cmdss [][]*redis.Cmd
- cmdsChan := make(chan []*redis.Cmd)
- stopChan := make(chan struct{})
- go func() {
- for {
- select {
- case cmds := <-cmdsChan:
- cmdss = append(cmdss, cmds)
- case <-stopChan:
- return
- }
- }
- }()
-
err := cluster.ForEachShard(ctx, func(ctx context.Context, client *redis.Client) error {
var cmds []*redis.Cmd
_, _ = client.Pipelined(ctx, func(p redis.Pipeliner) error {
@@ -302,10 +305,11 @@ func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, q
}
return nil
})
- cmdsChan <- cmds
+ mu.Lock()
+ defer mu.Unlock()
+ cmdss = append(cmdss, cmds)
return nil
})
- close(stopChan)
if err != nil && err != redis.Nil {
return nil, errors.Wrapf(err, "failed to query for each shard")
}
diff --git a/backend/plugin/db/redis/redis_test.go b/backend/plugin/db/redis/redis_test.go
index 64855c679f..952f8b01f6 100644
--- a/backend/plugin/db/redis/redis_test.go
+++ b/backend/plugin/db/redis/redis_test.go
@@ -1,9 +1,15 @@
package redis
import (
+ "context"
"testing"
"github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/db"
)
func TestSplitCommands(t *testing.T) {
@@ -53,3 +59,39 @@ func TestSplitCommandsRejectsAnUnclosedQuote(t *testing.T) {
_, _, err := splitCommands(`SET a "unclosed`)
require.Error(t, err)
}
+
+// TestQueryConnOnAllNodes covers the ALL_NODES query option, whose per-shard
+// replies used to be collected by a goroutine the caller never waited for, so
+// the results were read before any of them had been recorded and every command
+// reported no rows.
+func TestQueryConnOnAllNodes(t *testing.T) {
+ ctx := context.Background()
+ container := testcontainer.SharedRedisClusterContainer(t)
+
+ d, err := newDriver().Open(ctx, storepb.Engine_REDIS, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Host: container.GetHost(),
+ Port: container.GetPort(),
+ RedisType: storepb.DataSource_CLUSTER,
+ },
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, d.Close(ctx)) })
+
+ _, err = d.Execute(ctx, "SET allnodes:key allnodes-value", db.ExecuteOptions{})
+ require.NoError(t, err)
+
+ results, err := d.QueryConn(ctx, nil, "GET allnodes:key\nDBSIZE", db.QueryContext{
+ MaximumSQLResultSize: 1 << 20,
+ Option: &v1pb.QueryOption{RedisRunCommandsOn: v1pb.QueryOption_ALL_NODES},
+ })
+ require.NoError(t, err)
+ require.Len(t, results, 2)
+
+ for i, want := range []string{"allnodes-value", "1"} {
+ require.Len(t, results[i].Rows, 1, "statement %q returned no row", results[i].Statement)
+ require.Equal(t, want, results[i].Rows[0].Values[1].GetStringValue())
+ require.Equal(t, int64(1), results[i].RowsCount)
+ }
+}
diff --git a/go.mod b/go.mod
index 35defcd7ce..f79c866e6a 100644
--- a/go.mod
+++ b/go.mod
@@ -83,7 +83,7 @@ require (
github.com/prometheus/client_golang v1.25.0
github.com/prometheus/client_model v0.6.3
github.com/prometheus/common v0.73.1
- github.com/redis/go-redis/v9 v9.21.0
+ github.com/redis/go-redis/v9 v9.23.0
github.com/shopspring/decimal v1.6.0
github.com/sijms/go-ora/v2 v2.9.0
github.com/snowflakedb/gosnowflake/v2 v2.3.0
@@ -260,7 +260,7 @@ require (
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.20.3 // indirect
- github.com/klauspost/cpuid/v2 v2.2.11 // indirect
+ github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
@@ -350,7 +350,7 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect
go.opentelemetry.io/otel/trace v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
- go.uber.org/atomic v1.11.0 // indirect
+ go.uber.org/atomic v1.12.0 // indirect
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/exp v0.0.0-20261014143641-f0251eb4aa3e // indirect
diff --git a/go.sum b/go.sum
index 6257dbda5b..4f45a2f1de 100644
--- a/go.sum
+++ b/go.sum
@@ -667,8 +667,8 @@ github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTf
github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
-github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
-github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
+github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -860,8 +860,8 @@ github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJ
github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64=
github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0=
github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI=
-github.com/redis/go-redis/v9 v9.21.0 h1:FPBE4hhbAke+TLmcY3WkpbDffJEomdqPn3HYiqAtL9E=
-github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA=
+github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s=
+github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/richardlehane/mscfb v1.0.9 h1:8xdd9auUvXbFoCw3L9h1spnQHZgjNsSX+ek46J6A9tE=
@@ -1092,8 +1092,9 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
-go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE=
+go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=