Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

8.2 KB
Raw Blame
// Package cosmosdb is the plugin for CosmosDB driver.
package cosmosdb

import (
	"context"
	"crypto/tls"
	"database/sql"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"connectrpc.com/connect"
	metadatapb "github.com/bytebase/omni/metadata"
	"google.golang.org/protobuf/types/known/durationpb"

	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
	"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"
	"github.com/pkg/errors"

	"github.com/bytebase/bytebase/backend/plugin/db/cosmosdb/queryengine/gonative"

	"github.com/bytebase/bytebase/backend/common"
	storepb "github.com/bytebase/bytebase/backend/generated-go/store"
	v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
	"github.com/bytebase/bytebase/backend/plugin/db"
	"github.com/bytebase/bytebase/backend/plugin/db/util"
	"github.com/bytebase/bytebase/backend/plugin/parser/base"
)

var _ db.Driver = (*Driver)(nil)

func init() {
	db.Register(storepb.Engine_COSMOSDB, newDriver)
}

// Driver is the CosmosDB driver.
type Driver struct {
	client       *azcosmos.Client
	connCfg      db.ConnectionConfig
	databaseName string
}

func newDriver() db.Driver {
	return &Driver{}
}

// cosmosDBEmulatorKey is the well-known master key for the CosmosDB emulator.
// https://learn.microsoft.com/en-us/azure/cosmos-db/emulator
const cosmosDBEmulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="

// Open opens a CosmosDB driver.
func (d *Driver) Open(_ context.Context, _ storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {
	endpoint := connCfg.DataSource.Host

	var client *azcosmos.Client
	var err error

	if common.IsDev() && isLocalhostEndpoint(endpoint) {
		cred, credErr := azcosmos.NewKeyCredential(cosmosDBEmulatorKey)
		if credErr != nil {
			return nil, errors.Wrapf(credErr, "failed to create emulator key credential")
		}
		// The emulator uses a self-signed certificate, skip TLS verification in dev mode.
		options := &azcosmos.ClientOptions{
			ClientOptions: azcore.ClientOptions{
				Transport: &http.Client{
					Transport: &http.Transport{
						TLSClientConfig: &tls.Config{
							MinVersion:         tls.VersionTLS12,
							InsecureSkipVerify: true, //nolint:gosec
						},
					},
				},
			},
		}
		client, err = azcosmos.NewClientWithKey(endpoint, cred, options)
	} else {
		credential, credErr := util.GetAzureConnectionConfig(connCfg)
		if credErr != nil {
			return nil, credErr
		}
		client, err = azcosmos.NewClient(endpoint, credential, nil)
	}

	if err != nil {
		return nil, errors.Wrapf(err, "failed to create CosmosDB client")
	}
	d.client = client
	d.databaseName = connCfg.ConnectionContext.DatabaseName
	d.connCfg = connCfg
	return d, nil
}

func isLocalhostEndpoint(endpoint string) bool {
	u, err := url.Parse(endpoint)
	if err != nil {
		return false
	}
	host := u.Hostname()
	return host == "localhost" || host == "127.0.0.1" || host == "host.docker.internal"
}

// Close closes the CosmosDB driver.
func (*Driver) Close(_ context.Context) error {
	return nil
}

// Ping pings the database.
func (d *Driver) Ping(ctx context.Context) error {
	endpoint := d.connCfg.DataSource.Host
	if common.IsDev() && isLocalhostEndpoint(endpoint) {
		client, err := newEmulatorRESTClient(endpoint)
		if err != nil {
			return errors.Wrap(err, "failed to create emulator REST client")
		}
		// List databases as a connectivity check.
		if _, err := client.listDatabases(); err != nil {
			return errors.Wrap(err, "failed to ping CosmosDB emulator")
		}
		return nil
	}

	queryPager := d.client.NewQueryDatabasesPager("select 1", nil)
	for queryPager.More() {
		_, err := queryPager.NextPage(ctx)
		if err != nil {
			return errors.Wrapf(err, "failed to ping CosmosDB")
		}
	}
	return nil
}

// GetDB gets the database.
func (*Driver) GetDB() *sql.DB {
	return nil
}

func (*Driver) Execute(_ context.Context, _ string, _ db.ExecuteOptions) (int64, error) {
	return 0, connect.NewError(connect.CodeUnimplemented, errors.New("method Execute unimplemented"))
}

// Dump dumps the database.
func (*Driver) Dump(_ context.Context, _ io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
	return nil
}

// QueryConn queries a SQL statement in a given connection.
func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, queryContext db.QueryContext) ([]*v1pb.QueryResult, error) {
	if queryContext.Container == "" {
		return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("container argument is required for CosmosDB"))
	}

	// Allow `SELECT * FROM {container} [alias]` only.
	_, _, err := base.ValidateSQLForEditor(storepb.Engine_COSMOSDB, statement)
	if err != nil {
		return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("support simple SELECT statement for Cosmos DB engine only, err: %s", err.Error()))
	}

	startTime := time.Now()
	container, err := d.client.NewContainer(d.databaseName, queryContext.Container)
	if err != nil {
		return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to create container"))
	}

	var queryOption *azcosmos.QueryOptions
	// TODO(zp): Rewrite limit while parser supported.
	if queryContext.Limit > 0 && queryContext.Limit < 1000 {
		// Set the page size to limit to avoid large page size.
		queryOption = &azcosmos.QueryOptions{
			PageSizeHint: int32(queryContext.Limit),
		}
	}

	items, err := QueryEveryPartition(ctx, container, statement, queryOption, queryContext.Limit)
	if err != nil {
		return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to read more items"))
	}

	result := &v1pb.QueryResult{
		ColumnNames:     []string{"result"},
		ColumnTypeNames: []string{"TEXT"},
		Statement:       statement,
	}
	for _, item := range items {
		result.Rows = append(result.Rows, &v1pb.QueryRow{
			Values: []*v1pb.RowValue{
				{Kind: &v1pb.RowValue_StringValue{StringValue: string(item)}},
			},
		})
	}

	result.Latency = durationpb.New(time.Since(startTime))
	result.RowsCount = int64(len(result.Rows))

	return []*v1pb.QueryResult{result}, nil
}

// QueryEveryPartition runs statement across every partition of the container.
// The gateway serves what it can on its own; an aggregate, DISTINCT, ORDER BY,
// TOP, OFFSET/LIMIT or GROUP BY that spans partitions it refuses with a
// BadRequest instead, and the query then runs again through the pure-Go engine,
// which merges the per-partition streams itself.
//
// Only a query that has returned nothing yet can switch: once the gateway has
// handed back a continuation token, the engine cannot resume from it.
func QueryEveryPartition(ctx context.Context, container *azcosmos.ContainerClient, statement string, o *azcosmos.QueryOptions, limit int) ([][]byte, error) {
	items, pages, err := drainQuery(ctx, container, statement, o, limit)
	if err == nil || pages > 0 || !isCrossPartitionUnsupported(err) {
		return items, err
	}

	withEngine := azcosmos.QueryOptions{}
	if o != nil {
		withEngine = *o
	}
	withEngine.QueryEngine = gonative.Default()
	items, _, err = drainQuery(ctx, container, statement, &withEngine, limit)
	return items, err
}

// drainQuery pages a query to exhaustion or to limit, reporting how many pages
// it read so the caller can tell a query that failed outright from one that
// failed partway through.
func drainQuery(ctx context.Context, container *azcosmos.ContainerClient, statement string, o *azcosmos.QueryOptions, limit int) ([][]byte, int, error) {
	pager := container.NewQueryItemsPager(statement, azcosmos.NewPartitionKey(), o)
	var items [][]byte
	pages := 0
	for pager.More() {
		response, err := pager.NextPage(ctx)
		if err != nil {
			return nil, pages, err
		}
		pages++
		for _, item := range response.Items {
			items = append(items, item)
			if limit > 0 && len(items) >= limit {
				return items, pages, nil
			}
		}
	}
	return items, pages, nil
}

// isCrossPartitionUnsupported reports whether err is the gateway saying it
// cannot serve this query across partitions by itself.
func isCrossPartitionUnsupported(err error) bool {
	var responseErr *azcore.ResponseError
	if !errors.As(err, &responseErr) || responseErr.StatusCode != http.StatusBadRequest {
		return false
	}
	msg := err.Error()
	return strings.Contains(msg, "cross partition query can not be directly served by the gateway") ||
		strings.Contains(msg, "Cross partition query only supports")
}