Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
package elasticsearch
import (
"context"
"encoding/json"
"io"
"net/http"
"regexp"
"strconv"
"strings"
metadatapb "github.com/bytebase/omni/metadata"
"github.com/elastic/go-elasticsearch/v7/esapi"
"github.com/opensearch-project/opensearch-go/v5/opensearchapi"
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db"
)
func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {
version, err := d.getVersion()
if err != nil {
return nil, errors.Wrapf(err, "failed to fetch version from Elasticsearch server")
}
dbMetadata, err := d.SyncDBSchema(ctx)
if err != nil {
return nil, errors.Wrapf(err, "failed to fetch indices from Elasticsearch server")
}
instanceRoles, err := d.getInstanceRoles(ctx)
if err != nil {
return nil, err
}
return &db.InstanceMetadata{
Version: version,
Databases: []*metadatapb.DatabaseSchemaMetadata{dbMetadata},
Metadata: &storepb.Instance{
Roles: instanceRoles,
},
}, nil
}
// SyncDBSchema implements db.Driver.
func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMetadata, error) {
var dbMetadataProto metadatapb.DatabaseSchemaMetadata
// indices.
indices, err := d.getIndices(ctx)
if err != nil {
return nil, err
}
// TODO(tommy): database name?
dbMetadataProto.Name = "node"
dbMetadataProto.Schemas = append(dbMetadataProto.Schemas, &metadatapb.SchemaMetadata{Tables: indices})
return &dbMetadataProto, nil
}
// struct for getVersion().
type VersionResult struct {
Version struct {
Number string `json:"number"`
} `json:"version"`
}
func (d *Driver) getVersion() (string, error) {
if d.isOpenSearch && d.opensearchAPI != nil {
ctx := context.Background()
info, err := d.opensearchAPI.Info(ctx, &opensearchapi.InfoReq{})
if err != nil {
return "", err
}
return info.Version.Number, nil
}
resp, err := d.basicAuthClient.Do("GET", []byte("/"), nil)
if err != nil {
return "", err
}
defer resp.Body.Close()
bytes, err := io.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "failed to read response body")
}
// Check HTTP status code
if resp.StatusCode != http.StatusOK {
// Include response body for debugging
return "", errors.Errorf("unexpected status code %d: %s", resp.StatusCode, string(bytes))
}
var result VersionResult
err = json.Unmarshal(bytes, &result)
if err != nil {
// Include response body to help debug parsing issues
bodyPreview, truncated := common.TruncateString(string(bytes), 500)
if truncated {
bodyPreview += "..."
}
return "", errors.Wrapf(err, "failed to parse version response: %s", bodyPreview)
}
return result.Version.Number, nil
}
type IndicesResult struct {
IndexSize string `json:"store.size"`
DocsCount string `json:"docs.count"`
Index string `json:"index"`
}
// IndexSettingsResult represents the settings for an index.
type IndexSettingsResult struct {
Settings struct {
Index struct {
Hidden string `json:"hidden"`
} `json:"index"`
} `json:"settings"`
}
// getHiddenIndices returns a set of index names that are marked as hidden.
func (d *Driver) getHiddenIndices(ctx context.Context) (map[string]bool, error) {
hiddenIndices := make(map[string]bool)
if d.isOpenSearch && d.opensearchAPI != nil {
resp, err := d.opensearchAPI.Indices.GetSettings(ctx, &opensearchapi.IndicesGetSettingsReq{
Indices: []string{"*"},
Name: []string{"index.hidden"},
Params: &opensearchapi.IndicesGetSettingsParams{
ExpandWildcards: []string{"open", "hidden"},
},
})
if err != nil {
return nil, errors.Wrapf(err, "failed to get index settings")
}
for indexName, indexState := range resp.Entries {
if indexState.Settings == nil || indexState.Settings.Index == nil {
continue
}
if hidden := indexState.Settings.Index.Hidden; hidden != nil && *hidden == "true" {
hiddenIndices[indexName] = true
}
}
return hiddenIndices, nil
} else if d.typedClient != nil {
expandWildcards := "open,hidden"
res, err := esapi.IndicesGetSettingsRequest{
Index: []string{"*"},
Name: []string{"index.hidden"},
ExpandWildcards: expandWildcards,
FilterPath: []string{"**.hidden"},
}.Do(ctx, d.typedClient)
if err != nil {
return nil, errors.Wrapf(err, "failed to get index settings")
}
bytes, err := readBytesAndClose(res)
if err != nil {
return nil, errors.Wrap(err, "failed to read index settings response")
}
var results map[string]IndexSettingsResult
if err := json.Unmarshal(bytes, &results); err != nil {
return nil, errors.Wrapf(err, "failed to parse index settings response")
}
for indexName, indexData := range results {
if indexData.Settings.Index.Hidden == "true" {
hiddenIndices[indexName] = true
}
}
return hiddenIndices, nil
}
// Fallback to basic auth client
resp, err := d.basicAuthClient.Do("GET", []byte("/*/_settings?filter_path=**.hidden&expand_wildcards=open,hidden"), nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to get index settings")
}
defer resp.Body.Close()
bytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "failed to read index settings response body")
}
if resp.StatusCode != http.StatusOK {
return nil, errors.Errorf("failed to get index settings: unexpected status code %d: %s", resp.StatusCode, string(bytes))
}
var results map[string]IndexSettingsResult
if err := json.Unmarshal(bytes, &results); err != nil {
return nil, errors.Wrapf(err, "failed to parse index settings response")
}
for indexName, indexData := range results {
if indexData.Settings.Index.Hidden == "true" {
hiddenIndices[indexName] = true
}
}
return hiddenIndices, nil
}
func (d *Driver) getIndices(ctx context.Context) ([]*metadatapb.TableMetadata, error) {
var indicesMetadata []*metadatapb.TableMetadata
// Get hidden indices to filter them out.
hiddenIndices, err := d.getHiddenIndices(ctx)
if err != nil {
return nil, errors.Wrap(err, "failed to get hidden indices")
}
if d.isOpenSearch && d.opensearchAPI != nil {
resp, err := d.opensearchAPI.Cat.Indices(ctx, &opensearchapi.CatIndicesReq{})
if err != nil {
return nil, errors.Wrapf(err, "failed to list indices")
}
for _, idx := range resp.Records {
if idx.Index == nil || hiddenIndices[*idx.Index] {
continue
}
var datasize int64
if idx.StoreSize != nil {
datasize, err = unitConversion(*idx.StoreSize)
if err != nil {
return nil, err
}
}
var docCount int64
if idx.DocsCount != nil {
docCount, err = parseDocCount(*idx.DocsCount)
if err != nil {
return nil, err
}
}
indicesMetadata = append(indicesMetadata, &metadatapb.TableMetadata{
Name: *idx.Index,
DataSize: datasize,
RowCount: docCount,
})
}
return indicesMetadata, nil
} else if d.typedClient != nil {
res, err := esapi.CatIndicesRequest{Format: "json", Pretty: true}.Do(ctx, d.typedClient)
if err != nil {
return nil, errors.Wrapf(err, "failed to list indices")
}
bytes, err := readBytesAndClose(res)
if err != nil {
return nil, errors.Wrap(err, "failed to list Elasticsearch indices")
}
var results []IndicesResult
if err := json.Unmarshal(bytes, &results); err != nil {
// Include response body to help debug parsing issues
bodyPreview, truncated := common.TruncateString(string(bytes), 500)
if truncated {
bodyPreview += "..."
}
return nil, errors.Wrapf(err, "failed to parse Elasticsearch indices response: %s", bodyPreview)
}
for _, m := range results {
if hiddenIndices[m.Index] {
continue
}
datasize, err := unitConversion(m.IndexSize)
if err != nil {
return nil, err
}
docCount, err := parseDocCount(m.DocsCount)
if err != nil {
return nil, err
}
indicesMetadata = append(indicesMetadata, &metadatapb.TableMetadata{
Name: m.Index,
DataSize: datasize,
RowCount: docCount,
})
}
return indicesMetadata, nil
}
// Fallback to basic auth client
resp, err := d.basicAuthClient.Do("GET", []byte("/_cat/indices?format=json&pretty"), nil)
if err != nil {
return nil, errors.Wrapf(err, "failed to list indices")
}
defer resp.Body.Close()
bytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, errors.Wrap(err, "failed to read indices response body")
}
// Check HTTP status code
if resp.StatusCode != http.StatusOK {
// Include response body for debugging
return nil, errors.Errorf("failed to list indices: unexpected status code %d: %s", resp.StatusCode, string(bytes))
}
var results []IndicesResult
if err := json.Unmarshal(bytes, &results); err != nil {
// Include response body to help debug parsing issues
bodyPreview, truncated := common.TruncateString(string(bytes), 500)
if truncated {
bodyPreview += "..."
}
return nil, errors.Wrapf(err, "failed to parse indices response: %s", bodyPreview)
}
for _, m := range results {
if hiddenIndices[m.Index] {
continue
}
datasize, err := unitConversion(m.IndexSize)
if err != nil {
return nil, err
}
docCount, err := parseDocCount(m.DocsCount)
if err != nil {
return nil, err
}
indicesMetadata = append(indicesMetadata, &metadatapb.TableMetadata{
Name: m.Index,
DataSize: datasize,
RowCount: docCount,
})
}
return indicesMetadata, nil
}
// parseDocCount parses document count string, returning 0 for empty values (e.g., closed indices).
func parseDocCount(s string) (int64, error) {
if s == "" {
return 0, nil
}
count, err := strconv.Atoi(s)
return int64(count), err
}
func unitConversion(sizeWithUnit string) (int64, error) {
// Empty size is expected for closed indices (ES returns null → unmarshals to "")
if sizeWithUnit == "" {
return 0, nil
}
sizeWithUnit = strings.ToLower(sizeWithUnit)
sizeRe := regexp.MustCompile("([0-9.]+)([kmgtpe]?b)")
match := sizeRe.FindSubmatch([]byte(sizeWithUnit))
// Non-empty string that doesn't match expected format is unexpected
if len(match) < 3 {
return 0, errors.Errorf("unexpected size format: %q", sizeWithUnit)
}
unit := string(match[2])
size, err := strconv.ParseFloat(string(match[1]), 64)
if err != nil {
return 0, err
}
switch unit {
case "kb":
size *= 1024
case "mb":
size *= 1024 * 1024
case "gb":
size *= 1024 * 1024 * 1024
case "tb":
size *= 1024 * 1024 * 1024 * 1024
case "pb":
size *= 1024 * 1024 * 1024 * 1024 * 1024
case "eb":
size *= 1024 * 1024 * 1024 * 1024 * 1024 * 1024
default:
// For "b" (bytes) or any other unit, keep the size as-is
}
return int64(size), nil
}
func readBytesAndClose(anyResp any) ([]byte, error) {
var body io.ReadCloser
var statusCode int
var isError bool
// get body closer and status code.
switch resp := anyResp.(type) {
case *http.Response:
body = resp.Body
statusCode = resp.StatusCode
isError = resp.StatusCode >= 400
case *esapi.Response:
body = resp.Body
statusCode = resp.StatusCode
isError = resp.IsError()
default:
return nil, errors.New("not supported response type")
}
// read bytes.
bytes, err := io.ReadAll(body)
if err != nil {
return nil, errors.Wrap(err, "failed to read response body")
}
if err := body.Close(); err != nil {
return nil, errors.Wrap(err, "failed to close response body")
}
// Check for error status codes
if isError {
return bytes, errors.Errorf("request failed with status code %d: %s", statusCode, string(bytes))
}
return bytes, nil
}