Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(cosmosdb): support local emulator connection (#19679)
h3n4lbbb9f402d8f9
diff --git a/backend/plugin/db/cosmosdb/cosmosdb.go b/backend/plugin/db/cosmosdb/cosmosdb.goindex 6998537d70..5d4d4d96ea 100644--- a/backend/plugin/db/cosmosdb/cosmosdb.go+++ b/backend/plugin/db/cosmosdb/cosmosdb.go@@ -3,16 +3,21 @@ package cosmosdbimport ("context"+ "crypto/tls""database/sql""io"+ "net/http"+ "net/url""time""connectrpc.com/connect""google.golang.org/protobuf/types/known/durationpb"+ "github.com/Azure/azure-sdk-for-go/sdk/azcore""github.com/Azure/azure-sdk-for-go/sdk/data/azcosmos""github.com/pkg/errors"+ "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"v1pb "github.com/bytebase/bytebase/backend/generated-go/v1""github.com/bytebase/bytebase/backend/plugin/db"@@ -37,15 +42,44 @@ func newDriver() db.Driver {return &Driver{}}+// cosmosDBEmulatorKey is the well-known master key for the CosmosDB emulator.+// https://learn.microsoft.com/en-us/azure/cosmos-db/emulator+const cosmosDBEmulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="+// Open opens a CosmosDB driver.func (d *Driver) Open(_ context.Context, _ storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {endpoint := connCfg.DataSource.Host- credential, err := util.GetAzureConnectionConfig(connCfg)- if err != nil {- return nil, err++ var client *azcosmos.Client+ var err error++ if common.IsDev() && isLocalhostEndpoint(endpoint) {+ cred, credErr := azcosmos.NewKeyCredential(cosmosDBEmulatorKey)+ if credErr != nil {+ return nil, errors.Wrapf(credErr, "failed to create emulator key credential")+ }+ // The emulator uses a self-signed certificate, skip TLS verification in dev mode.+ options := &azcosmos.ClientOptions{+ ClientOptions: azcore.ClientOptions{+ Transport: &http.Client{+ Transport: &http.Transport{+ TLSClientConfig: &tls.Config{+ MinVersion: tls.VersionTLS12,+ InsecureSkipVerify: true, //nolint:gosec+ },+ },+ },+ },+ }+ client, err = azcosmos.NewClientWithKey(endpoint, cred, options)+ } else {+ credential, credErr := util.GetAzureConnectionConfig(connCfg)+ if credErr != nil {+ return nil, credErr+ }+ client, err = azcosmos.NewClient(endpoint, credential, nil)}- client, err := azcosmos.NewClient(endpoint, credential, nil)if err != nil {return nil, errors.Wrapf(err, "failed to create CosmosDB client")}@@ -55,6 +89,15 @@ func (d *Driver) Open(_ context.Context, _ storepb.Engine, connCfg db.Connectionreturn d, nil}+func isLocalhostEndpoint(endpoint string) bool {+ u, err := url.Parse(endpoint)+ if err != nil {+ return false+ }+ host := u.Hostname()+ return host == "localhost" || host == "127.0.0.1" || host == "host.docker.internal"+}+// Close closes the CosmosDB driver.func (*Driver) Close(_ context.Context) error {return nil@@ -62,11 +105,23 @@ func (*Driver) Close(_ context.Context) error {// Ping pings the database.func (d *Driver) Ping(ctx context.Context) error {+ endpoint := d.connCfg.DataSource.Host+ if common.IsDev() && isLocalhostEndpoint(endpoint) {+ client, err := newEmulatorRESTClient(endpoint)+ if err != nil {+ return errors.Wrap(err, "failed to create emulator REST client")+ }+ // List databases as a connectivity check.+ if _, err := client.listDatabases(); err != nil {+ return errors.Wrap(err, "failed to ping CosmosDB emulator")+ }+ return nil+ }+queryPager := d.client.NewQueryDatabasesPager("select 1", nil)for queryPager.More() {_, err := queryPager.NextPage(ctx)if err != nil {- // TODO(zp): Deserialize the error into azcore.ResponseErrorreturn errors.Wrapf(err, "failed to ping CosmosDB")}}diff --git a/backend/plugin/db/cosmosdb/emulator.go b/backend/plugin/db/cosmosdb/emulator.gonew file mode 100644index 0000000000..69e3a2ff56--- /dev/null+++ b/backend/plugin/db/cosmosdb/emulator.go@@ -0,0 +1,139 @@+package cosmosdb++import (+ "crypto/hmac"+ "crypto/sha256"+ "crypto/tls"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "strings"+ "time"++ "github.com/pkg/errors"+)++// emulatorRESTClient makes direct REST API calls to the CosmosDB emulator,+// bypassing the SDK's query API which the vnext-preview emulator does not support.+type emulatorRESTClient struct {+ endpoint string+ accountKey []byte+ httpClient *http.Client+}++func newEmulatorRESTClient(endpoint string) (*emulatorRESTClient, error) {+ key, err := base64.StdEncoding.DecodeString(cosmosDBEmulatorKey)+ if err != nil {+ return nil, errors.Wrap(err, "failed to decode emulator key")+ }+ return &emulatorRESTClient{+ endpoint: strings.TrimRight(endpoint, "/"),+ accountKey: key,+ httpClient: &http.Client{+ Transport: &http.Transport{+ TLSClientConfig: &tls.Config{+ MinVersion: tls.VersionTLS12,+ InsecureSkipVerify: true, //nolint:gosec+ },+ },+ },+ }, nil+}++func (c *emulatorRESTClient) generateAuthHeader(verb, resourceType, resourceLink string, date string) string {+ stringToSign := strings.ToLower(verb) + "\n" ++ strings.ToLower(resourceType) + "\n" ++ resourceLink + "\n" ++ strings.ToLower(date) + "\n" ++ "" + "\n"++ h := hmac.New(sha256.New, c.accountKey)+ h.Write([]byte(stringToSign))+ signature := base64.StdEncoding.EncodeToString(h.Sum(nil))++ return url.QueryEscape("type=master&ver=1.0&sig=" + signature)+}++func (c *emulatorRESTClient) get(path, resourceType, resourceLink string) ([]byte, error) {+ reqURL := c.endpoint + "/" + path+ req, err := http.NewRequest(http.MethodGet, reqURL, nil)+ if err != nil {+ return nil, err+ }++ date := time.Now().UTC().Format(http.TimeFormat)+ authHeader := c.generateAuthHeader("GET", resourceType, resourceLink, date)++ req.Header.Set("Authorization", authHeader)+ req.Header.Set("x-ms-date", date)+ req.Header.Set("x-ms-version", "2018-12-31")++ resp, err := c.httpClient.Do(req)+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()++ body, err := io.ReadAll(resp.Body)+ if err != nil {+ return nil, err+ }++ if resp.StatusCode != http.StatusOK {+ return nil, errors.Errorf("HTTP %d: %s", resp.StatusCode, string(body))+ }+ return body, nil+}++type listDatabasesResponse struct {+ Databases []struct {+ ID string `json:"id"`+ } `json:"Databases"`+}++type listContainersResponse struct {+ DocumentCollections []struct {+ ID string `json:"id"`+ } `json:"DocumentCollections"`+}++func (c *emulatorRESTClient) listDatabases() ([]string, error) {+ body, err := c.get("dbs", "dbs", "")+ if err != nil {+ return nil, errors.Wrap(err, "failed to list databases")+ }++ var resp listDatabasesResponse+ if err := json.Unmarshal(body, &resp); err != nil {+ return nil, errors.Wrap(err, "failed to parse database list")+ }++ var names []string+ for _, db := range resp.Databases {+ names = append(names, db.ID)+ }+ return names, nil+}++func (c *emulatorRESTClient) listContainers(databaseName string) ([]string, error) {+ path := fmt.Sprintf("dbs/%s/colls", databaseName)+ resourceLink := fmt.Sprintf("dbs/%s", databaseName)+ body, err := c.get(path, "colls", resourceLink)+ if err != nil {+ return nil, errors.Wrap(err, "failed to list containers")+ }++ var resp listContainersResponse+ if err := json.Unmarshal(body, &resp); err != nil {+ return nil, errors.Wrap(err, "failed to parse container list")+ }++ var names []string+ for _, container := range resp.DocumentCollections {+ names = append(names, container.ID)+ }+ return names, nil+}diff --git a/backend/plugin/db/cosmosdb/sync.go b/backend/plugin/db/cosmosdb/sync.goindex 4e4b92dcb3..98941421c5 100644--- a/backend/plugin/db/cosmosdb/sync.go+++ b/backend/plugin/db/cosmosdb/sync.go@@ -5,15 +5,19 @@ import ("github.com/pkg/errors"+ "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db")// SyncInstance syncs the instance meta.func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {+ endpoint := d.connCfg.DataSource.Host+ if common.IsDev() && isLocalhostEndpoint(endpoint) {+ return d.syncInstanceViaREST()+ }+var databases []*storepb.DatabaseSchemaMetadata- // List databases names.- // https://github.com/Azure/azure-sdk-for-go/pull/19769queryPager := d.client.NewQueryDatabasesPager("select * from dbs d", nil)for queryPager.More() {queryResponse, err := queryPager.NextPage(ctx)@@ -31,11 +35,34 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)}, nil}+func (d *Driver) syncInstanceViaREST() (*db.InstanceMetadata, error) {+ client, err := newEmulatorRESTClient(d.connCfg.DataSource.Host)+ if err != nil {+ return nil, err+ }+ names, err := client.listDatabases()+ if err != nil {+ return nil, err+ }+ var databases []*storepb.DatabaseSchemaMetadata+ for _, name := range names {+ databases = append(databases, &storepb.DatabaseSchemaMetadata{+ Name: name,+ })+ }+ return &db.InstanceMetadata{+ Databases: databases,+ }, nil+}+// SyncDBSchema syncs the database schema.func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetadata, error) {+ endpoint := d.connCfg.DataSource.Host+ if common.IsDev() && isLocalhostEndpoint(endpoint) {+ return d.syncDBSchemaViaREST()+ }+var containers []*storepb.TableMetadata- // List containers names.- // https://github.com/Azure/azure-sdk-for-go/pull/19769database, err := d.client.NewDatabase(d.databaseName)if err != nil {return nil, errors.Wrapf(err, "failed to get database %q", d.databaseName)@@ -61,3 +88,28 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad},}, nil}++func (d *Driver) syncDBSchemaViaREST() (*storepb.DatabaseSchemaMetadata, error) {+ client, err := newEmulatorRESTClient(d.connCfg.DataSource.Host)+ if err != nil {+ return nil, err+ }+ names, err := client.listContainers(d.databaseName)+ if err != nil {+ return nil, err+ }+ var containers []*storepb.TableMetadata+ for _, name := range names {+ containers = append(containers, &storepb.TableMetadata{+ Name: name,+ })+ }+ return &storepb.DatabaseSchemaMetadata{+ Name: d.databaseName,+ Schemas: []*storepb.SchemaMetadata{+ {+ Tables: containers,+ },+ },+ }, nil+}