Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

11.2 KB
Raw Blame
// Package redis implements redis driver
package redis

import (
	"context"
	"database/sql"
	"fmt"
	"io"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	metadatapb "github.com/bytebase/omni/metadata"
	"github.com/google/shlex"
	"github.com/pkg/errors"
	"github.com/redis/go-redis/v9"
	"go.uber.org/multierr"
	"golang.org/x/crypto/ssh"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/durationpb"

	storepb "github.com/bytebase/bytebase/backend/generated-go/store"
	v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
	"github.com/bytebase/bytebase/backend/plugin/db"
	"github.com/bytebase/bytebase/backend/plugin/db/util"
)

var (
	_ db.Driver = (*Driver)(nil)
)

func init() {
	db.Register(storepb.Engine_REDIS, newDriver)
}

// Driver is the redis driver.
type Driver struct {
	rdb          redis.UniversalClient
	sshClient    *ssh.Client
	databaseName string
}

func newDriver() db.Driver {
	return &Driver{}
}

// Open opens the redis driver.
func (d *Driver) Open(_ context.Context, _ storepb.Engine, config db.ConnectionConfig) (db.Driver, error) {
	tlscfg, err := util.GetTLSConfig(config.DataSource)
	if err != nil {
		return nil, errors.Wrap(err, "redis: failed to get tls config")
	}
	db := 0
	if config.ConnectionContext.DatabaseName != "" {
		database, err := strconv.Atoi(config.ConnectionContext.DatabaseName)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to convert database %s to int", config.ConnectionContext.DatabaseName)
		}
		db = database
	}
	d.databaseName = fmt.Sprintf("%d", db)
	switch config.DataSource.GetRedisType() {
	case storepb.DataSource_REDIS_TYPE_UNSPECIFIED, storepb.DataSource_STANDALONE:
		options := &redis.Options{
			Addr:      fmt.Sprintf("%s:%s", config.DataSource.Host, config.DataSource.Port),
			Username:  config.DataSource.Username,
			Password:  config.Password,
			TLSConfig: tlscfg,
			DB:        db,
			// Pipelines run on the main pool. A driver is opened per request
			// and runs a single pipeline, so go-redis's dedicated pipeline pool
			// would only add a connection and its handshake to every request.
			PipelinePoolSize: -1,
		}
		if config.DataSource.GetSshHost() != "" {
			sshClient, err := util.GetSSHClient(config.DataSource)
			if err != nil {
				return nil, err
			}
			d.sshClient = sshClient

			options.Dialer = func(_ context.Context, network, addr string) (net.Conn, error) {
				conn, err := sshClient.Dial(network, addr)
				if err != nil {
					return nil, err
				}
				return &util.NoDeadlineConn{Conn: conn}, nil
			}
		}
		client := redis.NewClient(options)
		d.databaseName = fmt.Sprintf("%d", db)
		d.rdb = client
	case storepb.DataSource_SENTINEL:
		sentinelAddrs := make([]string, 0, 1+len(config.DataSource.GetAdditionalAddresses()))
		sentinelAddrs = append(sentinelAddrs, fmt.Sprintf("%s:%s", config.DataSource.Host, config.DataSource.Port))
		for _, sentinelAddr := range config.DataSource.GetAdditionalAddresses() {
			sentinelAddrs = append(sentinelAddrs, fmt.Sprintf("%s:%s", sentinelAddr.Host, sentinelAddr.Port))
		}
		options := &redis.FailoverOptions{
			MasterName:       config.DataSource.GetMasterName(),
			Username:         config.DataSource.GetMasterUsername(),
			Password:         config.DataSource.GetMasterPassword(),
			SentinelUsername: config.DataSource.Username,
			SentinelPassword: config.Password,
			SentinelAddrs:    sentinelAddrs,
			DB:               db,
			TLSConfig:        tlscfg,
			PipelinePoolSize: -1,
		}
		if config.DataSource.GetSshHost() != "" {
			sshClient, err := util.GetSSHClient(config.DataSource)
			if err != nil {
				return nil, err
			}
			d.sshClient = sshClient

			options.Dialer = func(_ context.Context, network, addr string) (net.Conn, error) {
				conn, err := sshClient.Dial(network, addr)
				if err != nil {
					return nil, err
				}
				return &util.NoDeadlineConn{Conn: conn}, nil
			}
		}
		d.databaseName = fmt.Sprintf("%d", db)
		client := redis.NewFailoverClient(options)
		d.rdb = client
	case storepb.DataSource_CLUSTER:
		addrs := make([]string, 0, 1+len(config.DataSource.GetAdditionalAddresses()))
		addrs = append(addrs, fmt.Sprintf("%s:%s", config.DataSource.Host, config.DataSource.Port))
		for _, addr := range config.DataSource.GetAdditionalAddresses() {
			addrs = append(addrs, fmt.Sprintf("%s:%s", addr.Host, addr.Port))
		}
		options := &redis.ClusterOptions{
			Addrs:            addrs,
			Username:         config.DataSource.Username,
			Password:         config.Password,
			TLSConfig:        tlscfg,
			PipelinePoolSize: -1,
			// go-redis 9.22.0 raised its own default to a minute. A node that
			// refuses connections does not trigger a reload, so a failover is
			// only noticed by this periodic one, and a driver that outlives it
			// keeps routing to the master it has lost.
			// https://github.com/redis/go-redis/issues/3979
			ClusterStateReloadInterval: 10 * time.Second,
		}
		if config.DataSource.GetSshHost() != "" {
			sshClient, err := util.GetSSHClient(config.DataSource)
			if err != nil {
				return nil, err
			}
			d.sshClient = sshClient

			options.Dialer = func(_ context.Context, network, addr string) (net.Conn, error) {
				conn, err := sshClient.Dial(network, addr)
				if err != nil {
					return nil, err
				}
				return &util.NoDeadlineConn{Conn: conn}, nil
			}
		}
		client := redis.NewClusterClient(options)
		d.rdb = client
	default:
		return nil, errors.Errorf("unsupported redis type %s", config.DataSource.GetRedisType().String())
	}
	return d, nil
}

// Close closes the redis driver.
func (d *Driver) Close(context.Context) error {
	var err error
	err = multierr.Append(err, d.rdb.Close())
	if d.sshClient != nil {
		err = multierr.Append(err, d.sshClient.Close())
	}
	return err
}

// Ping pings the redis server.
func (d *Driver) Ping(ctx context.Context) error {
	return d.rdb.Ping(ctx).Err()
}

// GetDB gets the database.
func (*Driver) GetDB() *sql.DB {
	return nil
}

// Execute will execute the statement. For CREATE DATABASE statement, some types of databases such as Postgres
// will not use transactions to execute the statement but will still use transactions to execute the rest of statements.
func (d *Driver) Execute(ctx context.Context, statement string, opts db.ExecuteOptions) (int64, error) {
	if opts.CreateDatabase {
		return 0, errors.New("redis: cannot create database")
	}

	inputs, _, err := splitCommands(statement)
	if err != nil {
		return 0, err
	}
	if _, err := d.rdb.Pipelined(ctx, func(p redis.Pipeliner) error {
		for _, input := range inputs {
			_ = p.Do(ctx, input...)
		}
		return nil
	}); err != nil && err != redis.Nil {
		return 0, err
	}

	return 0, nil
}

// splitCommands reads one command per line, skipping lines that hold none, and
// returns each command's arguments beside the line it came from. A result is
// reported per command rather than per line, so the two are built together to
// keep them aligned.
func splitCommands(statement string) ([][]any, []string, error) {
	var inputs [][]any
	var commands []string
	for _, line := range strings.Split(statement, "\n") {
		fields, err := shlex.Split(line)
		if err != nil {
			return nil, nil, errors.Wrapf(err, "failed to split command %s", line)
		}
		if len(fields) == 0 {
			continue
		}
		var input []any
		for _, v := range fields {
			input = append(input, v)
		}
		inputs = append(inputs, input)
		commands = append(commands, line)
	}
	return inputs, commands, nil
}

// Dump and restore
// Dump the database, if dbName is empty, then dump all databases.
// Redis is schemaless, we don't support dump Redis data currently.
func (*Driver) Dump(_ context.Context, _ io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
	return nil
}

// QueryConn queries a SQL statement in a given connection.
func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, queryContext db.QueryContext) ([]*v1pb.QueryResult, error) {
	if queryContext.Explain {
		return nil, errors.New("Redis does not support EXPLAIN")
	}

	startTime := time.Now()
	inputs, commands, err := splitCommands(statement)
	if err != nil {
		return nil, err
	}

	results := make([]*v1pb.QueryResult, len(inputs))
	for i := range results {
		results[i] = &v1pb.QueryResult{
			ColumnNames:     []string{"#", "Value"},
			ColumnTypeNames: []string{"INT", "TEXT"},
			Statement:       commands[i],
		}
	}

	switch queryContext.Option.GetRedisRunCommandsOn() {
	case
		v1pb.QueryOption_REDIS_RUN_COMMANDS_ON_UNSPECIFIED,
		v1pb.QueryOption_SINGLE_NODE:
		var cmds []*redis.Cmd
		if _, err := d.rdb.Pipelined(ctx, func(p redis.Pipeliner) error {
			for _, input := range inputs {
				cmd := p.Do(ctx, input...)
				cmds = append(cmds, cmd)
			}
			return nil
		}); err != nil && err != redis.Nil {
			return nil, err
		}

		for i, cmd := range cmds {
			setQueryResultRows(results[i], cmd, queryContext.MaximumSQLResultSize)
			results[i].Latency = durationpb.New(time.Since(startTime))
			results[i].RowsCount = int64(len(results[i].Rows))
		}

	case v1pb.QueryOption_ALL_NODES:
		cluster, ok := d.rdb.(*redis.ClusterClient)
		if !ok {
			return nil, errors.Errorf("expect *redis.ClusterClient, but get %T", d.rdb)
		}

		// ForEachShard runs the callbacks concurrently and returns once they
		// have all finished, so the mutex is what orders the appends.
		var mu sync.Mutex
		var cmdss [][]*redis.Cmd
		err := cluster.ForEachShard(ctx, func(ctx context.Context, client *redis.Client) error {
			var cmds []*redis.Cmd
			_, _ = client.Pipelined(ctx, func(p redis.Pipeliner) error {
				for _, input := range inputs {
					cmd := p.Do(ctx, input...)
					cmds = append(cmds, cmd)
				}
				return nil
			})
			mu.Lock()
			defer mu.Unlock()
			cmdss = append(cmdss, cmds)
			return nil
		})
		if err != nil && err != redis.Nil {
			return nil, errors.Wrapf(err, "failed to query for each shard")
		}

		for _, cmds := range cmdss {
			for i, cmd := range cmds {
				setQueryResultRows(results[i], cmd, queryContext.MaximumSQLResultSize)
				results[i].Latency = durationpb.New(time.Since(startTime))
				results[i].RowsCount = int64(len(results[i].Rows))
			}
		}
	default:
		return nil, errors.Errorf("unknown RedisRunCommandOn enum %v", queryContext.Option.GetRedisRunCommandsOn())
	}

	return results, nil
}

func setQueryResultRows(result *v1pb.QueryResult, cmd *redis.Cmd, limit int64) {
	if cmd.Err() == redis.Nil {
		result.Rows = append(result.Rows, &v1pb.QueryRow{
			Values: []*v1pb.RowValue{
				{Kind: &v1pb.RowValue_Int32Value{Int32Value: 1}},
				{Kind: &v1pb.RowValue_NullValue{}},
			},
		})
		return
	}
	if cmd.Err() != nil {
		result.Rows = append(result.Rows, &v1pb.QueryRow{
			Values: []*v1pb.RowValue{
				{Kind: &v1pb.RowValue_Int32Value{Int32Value: 1}},
				{Kind: &v1pb.RowValue_StringValue{StringValue: cmd.Err().Error()}},
			},
		})
		return
	}

	val := cmd.Val()
	l, ok := val.([]any)
	if ok {
		for i, v := range l {
			result.Rows = append(result.Rows, getResultRow(i+1, v))
			n := len(result.Rows)
			if (n&(n-1) == 0) && int64(proto.Size(result)) > limit {
				result.Error = util.FormatMaximumSQLResultSizeMessage(limit)
				return
			}
		}
	} else {
		result.Rows = append(result.Rows, getResultRow(1, val))
	}
}

func getResultRow(i int, v any) *v1pb.QueryRow {
	s := fmt.Sprintf("%v", v)
	return &v1pb.QueryRow{Values: []*v1pb.RowValue{
		{Kind: &v1pb.RowValue_Int32Value{Int32Value: int32(i)}},
		{Kind: &v1pb.RowValue_StringValue{StringValue: s}}},
	}
}