Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
// Package cosmosdb is the plugin for CosmosDB driver.
package cosmosdb
import (
"context"
"crypto/tls"
"database/sql"
"io"
"net/http"
"net/url"
"strings"
"time"
"connectrpc.com/connect"
metadatapb "github.com/bytebase/omni/metadata"
"google.golang.org/protobuf/types/known/durationpb"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/plugin/db/cosmosdb/queryengine/gonative"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
var _ db.Driver = (*Driver)(nil)
func init() {
db.Register(storepb.Engine_COSMOSDB, newDriver)
}
// Driver is the CosmosDB driver.
type Driver struct {
client *azcosmos.Client
connCfg db.ConnectionConfig
databaseName string
}
func newDriver() db.Driver {
return &Driver{}
}
// cosmosDBEmulatorKey is the well-known master key for the CosmosDB emulator.
// https://learn.microsoft.com/en-us/azure/cosmos-db/emulator
const cosmosDBEmulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
// Open opens a CosmosDB driver.
func (d *Driver) Open(_ context.Context, _ storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {
endpoint := connCfg.DataSource.Host
var client *azcosmos.Client
var err error
if common.IsDev() && isLocalhostEndpoint(endpoint) {
cred, credErr := azcosmos.NewKeyCredential(cosmosDBEmulatorKey)
if credErr != nil {
return nil, errors.Wrapf(credErr, "failed to create emulator key credential")
}
// The emulator uses a self-signed certificate, skip TLS verification in dev mode.
options := &azcosmos.ClientOptions{
ClientOptions: azcore.ClientOptions{
Transport: &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true, //nolint:gosec
},
},
},
},
}
client, err = azcosmos.NewClientWithKey(endpoint, cred, options)
} else {
credential, credErr := util.GetAzureConnectionConfig(connCfg)
if credErr != nil {
return nil, credErr
}
client, err = azcosmos.NewClient(endpoint, credential, nil)
}
if err != nil {
return nil, errors.Wrapf(err, "failed to create CosmosDB client")
}
d.client = client
d.databaseName = connCfg.ConnectionContext.DatabaseName
d.connCfg = connCfg
return d, nil
}
func isLocalhostEndpoint(endpoint string) bool {
u, err := url.Parse(endpoint)
if err != nil {
return false
}
host := u.Hostname()
return host == "localhost" || host == "127.0.0.1" || host == "host.docker.internal"
}
// Close closes the CosmosDB driver.
func (*Driver) Close(_ context.Context) error {
return nil
}
// Ping pings the database.
func (d *Driver) Ping(ctx context.Context) error {
endpoint := d.connCfg.DataSource.Host
if common.IsDev() && isLocalhostEndpoint(endpoint) {
client, err := newEmulatorRESTClient(endpoint)
if err != nil {
return errors.Wrap(err, "failed to create emulator REST client")
}
// List databases as a connectivity check.
if _, err := client.listDatabases(); err != nil {
return errors.Wrap(err, "failed to ping CosmosDB emulator")
}
return nil
}
queryPager := d.client.NewQueryDatabasesPager("select 1", nil)
for queryPager.More() {
_, err := queryPager.NextPage(ctx)
if err != nil {
return errors.Wrapf(err, "failed to ping CosmosDB")
}
}
return nil
}
// GetDB gets the database.
func (*Driver) GetDB() *sql.DB {
return nil
}
func (*Driver) Execute(_ context.Context, _ string, _ db.ExecuteOptions) (int64, error) {
return 0, connect.NewError(connect.CodeUnimplemented, errors.New("method Execute unimplemented"))
}
// Dump dumps the database.
func (*Driver) Dump(_ context.Context, _ io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
return nil
}
// QueryConn queries a SQL statement in a given connection.
func (d *Driver) QueryConn(ctx context.Context, _ *sql.Conn, statement string, queryContext db.QueryContext) ([]*v1pb.QueryResult, error) {
if queryContext.Container == "" {
return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("container argument is required for CosmosDB"))
}
// Allow `SELECT * FROM {container} [alias]` only.
_, _, err := base.ValidateSQLForEditor(storepb.Engine_COSMOSDB, statement)
if err != nil {
return nil, connect.NewError(connect.CodeInvalidArgument, errors.Errorf("support simple SELECT statement for Cosmos DB engine only, err: %s", err.Error()))
}
startTime := time.Now()
container, err := d.client.NewContainer(d.databaseName, queryContext.Container)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to create container"))
}
var queryOption *azcosmos.QueryOptions
// TODO(zp): Rewrite limit while parser supported.
if queryContext.Limit > 0 && queryContext.Limit < 1000 {
// Set the page size to limit to avoid large page size.
queryOption = &azcosmos.QueryOptions{
PageSizeHint: int32(queryContext.Limit),
}
}
items, err := QueryEveryPartition(ctx, container, statement, queryOption, queryContext.Limit)
if err != nil {
return nil, connect.NewError(connect.CodeInternal, errors.Wrapf(err, "failed to read more items"))
}
result := &v1pb.QueryResult{
ColumnNames: []string{"result"},
ColumnTypeNames: []string{"TEXT"},
Statement: statement,
}
for _, item := range items {
result.Rows = append(result.Rows, &v1pb.QueryRow{
Values: []*v1pb.RowValue{
{Kind: &v1pb.RowValue_StringValue{StringValue: string(item)}},
},
})
}
result.Latency = durationpb.New(time.Since(startTime))
result.RowsCount = int64(len(result.Rows))
return []*v1pb.QueryResult{result}, nil
}
// QueryEveryPartition runs statement across every partition of the container.
// The gateway serves what it can on its own; an aggregate, DISTINCT, ORDER BY,
// TOP, OFFSET/LIMIT or GROUP BY that spans partitions it refuses with a
// BadRequest instead, and the query then runs again through the pure-Go engine,
// which merges the per-partition streams itself.
//
// Only a query that has returned nothing yet can switch: once the gateway has
// handed back a continuation token, the engine cannot resume from it.
func QueryEveryPartition(ctx context.Context, container *azcosmos.ContainerClient, statement string, o *azcosmos.QueryOptions, limit int) ([][]byte, error) {
items, pages, err := drainQuery(ctx, container, statement, o, limit)
if err == nil || pages > 0 || !isCrossPartitionUnsupported(err) {
return items, err
}
withEngine := azcosmos.QueryOptions{}
if o != nil {
withEngine = *o
}
withEngine.QueryEngine = gonative.Default()
items, _, err = drainQuery(ctx, container, statement, &withEngine, limit)
return items, err
}
// drainQuery pages a query to exhaustion or to limit, reporting how many pages
// it read so the caller can tell a query that failed outright from one that
// failed partway through.
func drainQuery(ctx context.Context, container *azcosmos.ContainerClient, statement string, o *azcosmos.QueryOptions, limit int) ([][]byte, int, error) {
pager := container.NewQueryItemsPager(statement, azcosmos.NewPartitionKey(), o)
var items [][]byte
pages := 0
for pager.More() {
response, err := pager.NextPage(ctx)
if err != nil {
return nil, pages, err
}
pages++
for _, item := range response.Items {
items = append(items, item)
if limit > 0 && len(items) >= limit {
return items, pages, nil
}
}
}
return items, pages, nil
}
// isCrossPartitionUnsupported reports whether err is the gateway saying it
// cannot serve this query across partitions by itself.
func isCrossPartitionUnsupported(err error) bool {
var responseErr *azcore.ResponseError
if !errors.As(err, &responseErr) || responseErr.StatusCode != http.StatusBadRequest {
return false
}
msg := err.Error()
return strings.Contains(msg, "cross partition query can not be directly served by the gateway") ||
strings.Contains(msg, "Cross partition query only supports")
}