Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(cosmosdb): support local emulator connection (#19679)

h3n4lbbb9f402d8f9
diff --git a/backend/plugin/db/cosmosdb/cosmosdb.go b/backend/plugin/db/cosmosdb/cosmosdb.go
index 6998537d70..5d4d4d96ea 100644
--- a/backend/plugin/db/cosmosdb/cosmosdb.go
+++ b/backend/plugin/db/cosmosdb/cosmosdb.go
@@ -3,16 +3,21 @@ package cosmosdb
import (
"context"
+ "crypto/tls"
"database/sql"
"io"
+ "net/http"
+ "net/url"
"time"
"connectrpc.com/connect"
"google.golang.org/protobuf/types/known/durationpb"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos"
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
@@ -37,15 +42,44 @@ func newDriver() db.Driver {
return &Driver{}
}
+// cosmosDBEmulatorKey is the well-known master key for the CosmosDB emulator.
+// https://learn.microsoft.com/en-us/azure/cosmos-db/emulator
+const cosmosDBEmulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
+
// Open opens a CosmosDB driver.
func (d *Driver) Open(_ context.Context, _ storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {
endpoint := connCfg.DataSource.Host
- credential, err := util.GetAzureConnectionConfig(connCfg)
- if err != nil {
- return nil, err
+
+ var client *azcosmos.Client
+ var err error
+
+ if common.IsDev() && isLocalhostEndpoint(endpoint) {
+ cred, credErr := azcosmos.NewKeyCredential(cosmosDBEmulatorKey)
+ if credErr != nil {
+ return nil, errors.Wrapf(credErr, "failed to create emulator key credential")
+ }
+ // The emulator uses a self-signed certificate, skip TLS verification in dev mode.
+ options := &azcosmos.ClientOptions{
+ ClientOptions: azcore.ClientOptions{
+ Transport: &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ InsecureSkipVerify: true, //nolint:gosec
+ },
+ },
+ },
+ },
+ }
+ client, err = azcosmos.NewClientWithKey(endpoint, cred, options)
+ } else {
+ credential, credErr := util.GetAzureConnectionConfig(connCfg)
+ if credErr != nil {
+ return nil, credErr
+ }
+ client, err = azcosmos.NewClient(endpoint, credential, nil)
}
- client, err := azcosmos.NewClient(endpoint, credential, nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to create CosmosDB client")
}
@@ -55,6 +89,15 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, connCfg db.Connection
return d, nil
}
+func isLocalhostEndpoint(endpoint string) bool {
+ u, err := url.Parse(endpoint)
+ if err != nil {
+ return false
+ }
+ host := u.Hostname()
+ return host == "localhost" || host == "127.0.0.1" || host == "host.docker.internal"
+}
+
// Close closes the CosmosDB driver.
func (*Driver) Close(_ context.Context) error {
return nil
@@ -62,11 +105,23 @@ func (*Driver) Close(_ context.Context) error {
// Ping pings the database.
func (d *Driver) Ping(ctx context.Context) error {
+ endpoint := d.connCfg.DataSource.Host
+ if common.IsDev() && isLocalhostEndpoint(endpoint) {
+ client, err := newEmulatorRESTClient(endpoint)
+ if err != nil {
+ return errors.Wrap(err, "failed to create emulator REST client")
+ }
+ // List databases as a connectivity check.
+ if _, err := client.listDatabases(); err != nil {
+ return errors.Wrap(err, "failed to ping CosmosDB emulator")
+ }
+ return nil
+ }
+
queryPager := d.client.NewQueryDatabasesPager("select 1", nil)
for queryPager.More() {
_, err := queryPager.NextPage(ctx)
if err != nil {
- // TODO(zp): Deserialize the error into azcore.ResponseError
return errors.Wrapf(err, "failed to ping CosmosDB")
}
}
diff --git a/backend/plugin/db/cosmosdb/emulator.go b/backend/plugin/db/cosmosdb/emulator.go
new file mode 100644
index 0000000000..69e3a2ff56
--- /dev/null
+++ b/backend/plugin/db/cosmosdb/emulator.go
@@ -0,0 +1,139 @@
+package cosmosdb
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "crypto/tls"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "github.com/pkg/errors"
+)
+
+// emulatorRESTClient makes direct REST API calls to the CosmosDB emulator,
+// bypassing the SDK's query API which the vnext-preview emulator does not support.
+type emulatorRESTClient struct {
+ endpoint string
+ accountKey []byte
+ httpClient *http.Client
+}
+
+func newEmulatorRESTClient(endpoint string) (*emulatorRESTClient, error) {
+ key, err := base64.StdEncoding.DecodeString(cosmosDBEmulatorKey)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to decode emulator key")
+ }
+ return &emulatorRESTClient{
+ endpoint: strings.TrimRight(endpoint, "/"),
+ accountKey: key,
+ httpClient: &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ InsecureSkipVerify: true, //nolint:gosec
+ },
+ },
+ },
+ }, nil
+}
+
+func (c *emulatorRESTClient) generateAuthHeader(verb, resourceType, resourceLink string, date string) string {
+ stringToSign := strings.ToLower(verb) + "\n" +
+ strings.ToLower(resourceType) + "\n" +
+ resourceLink + "\n" +
+ strings.ToLower(date) + "\n" +
+ "" + "\n"
+
+ h := hmac.New(sha256.New, c.accountKey)
+ h.Write([]byte(stringToSign))
+ signature := base64.StdEncoding.EncodeToString(h.Sum(nil))
+
+ return url.QueryEscape("type=master&ver=1.0&sig=" + signature)
+}
+
+func (c *emulatorRESTClient) get(path, resourceType, resourceLink string) ([]byte, error) {
+ reqURL := c.endpoint + "/" + path
+ req, err := http.NewRequest(http.MethodGet, reqURL, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ date := time.Now().UTC().Format(http.TimeFormat)
+ authHeader := c.generateAuthHeader("GET", resourceType, resourceLink, date)
+
+ req.Header.Set("Authorization", authHeader)
+ req.Header.Set("x-ms-date", date)
+ req.Header.Set("x-ms-version", "2018-12-31")
+
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ body, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, errors.Errorf("HTTP %d: %s", resp.StatusCode, string(body))
+ }
+ return body, nil
+}
+
+type listDatabasesResponse struct {
+ Databases []struct {
+ ID string `json:"id"`
+ } `json:"Databases"`
+}
+
+type listContainersResponse struct {
+ DocumentCollections []struct {
+ ID string `json:"id"`
+ } `json:"DocumentCollections"`
+}
+
+func (c *emulatorRESTClient) listDatabases() ([]string, error) {
+ body, err := c.get("dbs", "dbs", "")
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to list databases")
+ }
+
+ var resp listDatabasesResponse
+ if err := json.Unmarshal(body, &resp); err != nil {
+ return nil, errors.Wrap(err, "failed to parse database list")
+ }
+
+ var names []string
+ for _, db := range resp.Databases {
+ names = append(names, db.ID)
+ }
+ return names, nil
+}
+
+func (c *emulatorRESTClient) listContainers(databaseName string) ([]string, error) {
+ path := fmt.Sprintf("dbs/%s/colls", databaseName)
+ resourceLink := fmt.Sprintf("dbs/%s", databaseName)
+ body, err := c.get(path, "colls", resourceLink)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to list containers")
+ }
+
+ var resp listContainersResponse
+ if err := json.Unmarshal(body, &resp); err != nil {
+ return nil, errors.Wrap(err, "failed to parse container list")
+ }
+
+ var names []string
+ for _, container := range resp.DocumentCollections {
+ names = append(names, container.ID)
+ }
+ return names, nil
+}
diff --git a/backend/plugin/db/cosmosdb/sync.go b/backend/plugin/db/cosmosdb/sync.go
index 4e4b92dcb3..98941421c5 100644
--- a/backend/plugin/db/cosmosdb/sync.go
+++ b/backend/plugin/db/cosmosdb/sync.go
@@ -5,15 +5,19 @@ import (
"github.com/pkg/errors"
+ "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db"
)
// SyncInstance syncs the instance meta.
func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {
+ endpoint := d.connCfg.DataSource.Host
+ if common.IsDev() && isLocalhostEndpoint(endpoint) {
+ return d.syncInstanceViaREST()
+ }
+
var databases []*storepb.DatabaseSchemaMetadata
- // List databases names.
- // https://github.com/Azure/azure-sdk-for-go/pull/19769
queryPager := d.client.NewQueryDatabasesPager("select * from dbs d", nil)
for queryPager.More() {
queryResponse, err := queryPager.NextPage(ctx)
@@ -31,11 +35,34 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)
}, nil
}
+func (d *Driver) syncInstanceViaREST() (*db.InstanceMetadata, error) {
+ client, err := newEmulatorRESTClient(d.connCfg.DataSource.Host)
+ if err != nil {
+ return nil, err
+ }
+ names, err := client.listDatabases()
+ if err != nil {
+ return nil, err
+ }
+ var databases []*storepb.DatabaseSchemaMetadata
+ for _, name := range names {
+ databases = append(databases, &storepb.DatabaseSchemaMetadata{
+ Name: name,
+ })
+ }
+ return &db.InstanceMetadata{
+ Databases: databases,
+ }, nil
+}
+
// SyncDBSchema syncs the database schema.
func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetadata, error) {
+ endpoint := d.connCfg.DataSource.Host
+ if common.IsDev() && isLocalhostEndpoint(endpoint) {
+ return d.syncDBSchemaViaREST()
+ }
+
var containers []*storepb.TableMetadata
- // List containers names.
- // https://github.com/Azure/azure-sdk-for-go/pull/19769
database, err := d.client.NewDatabase(d.databaseName)
if err != nil {
return nil, errors.Wrapf(err, "failed to get database %q", d.databaseName)
@@ -61,3 +88,28 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad
},
}, nil
}
+
+func (d *Driver) syncDBSchemaViaREST() (*storepb.DatabaseSchemaMetadata, error) {
+ client, err := newEmulatorRESTClient(d.connCfg.DataSource.Host)
+ if err != nil {
+ return nil, err
+ }
+ names, err := client.listContainers(d.databaseName)
+ if err != nil {
+ return nil, err
+ }
+ var containers []*storepb.TableMetadata
+ for _, name := range names {
+ containers = append(containers, &storepb.TableMetadata{
+ Name: name,
+ })
+ }
+ return &storepb.DatabaseSchemaMetadata{
+ Name: d.databaseName,
+ Schemas: []*storepb.SchemaMetadata{
+ {
+ Tables: containers,
+ },
+ },
+ }, nil
+}