Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
package elasticsearch
import (
"bytes"
"context"
"crypto/tls"
"database/sql"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"github.com/elastic/go-elasticsearch/v7"
opensearch "github.com/opensearch-project/opensearch-go/v5"
"github.com/opensearch-project/opensearch-go/v5/opensearchapi"
"github.com/opensearch-project/opensearch-go/v5/signer/awsv2"
"github.com/pkg/errors"
"google.golang.org/protobuf/types/known/durationpb"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
"github.com/bytebase/bytebase/backend/plugin/db"
"github.com/bytebase/bytebase/backend/plugin/db/util"
parser "github.com/bytebase/bytebase/backend/plugin/parser/elasticsearch"
)
func elasticsearchDriverFunc() db.Driver {
return &Driver{}
}
func init() {
db.Register(storepb.Engine_ELASTICSEARCH, elasticsearchDriverFunc)
}
var (
_ db.Driver = (*Driver)(nil)
)
type Driver struct {
// For Elasticsearch
typedClient *elasticsearch.Client
// For OpenSearch
opensearchClient *opensearch.Client
opensearchAPI *opensearchapi.Client
// Common basic auth client for REST operations
basicAuthClient *BasicAuthClient
config db.ConnectionConfig
isOpenSearch bool
}
type BasicAuthClient struct {
httpClient *http.Client
addrScheduler *AddressScheduler
basicAuthString string
}
func (client *BasicAuthClient) Do(method string, route []byte, queryString []byte) (*http.Response, error) {
address := client.addrScheduler.GetNewAddress()
// Parse base URL and join with route path
baseURL, err := url.Parse(address)
if err != nil {
return nil, errors.Wrapf(err, "failed to parse base URL")
}
// Split route into path and query parameters
// The route may contain query parameters (e.g., "/_mapping?pretty")
// We need to separate them to avoid URL-encoding the '?' character
routeStr := string(route)
pathPart, queryPart, hasQuery := strings.Cut(routeStr, "?")
// Join only the path part (without query parameters)
// This ensures JoinPath only processes the actual path
fullURL := baseURL.JoinPath(pathPart)
// Add query parameters if present in the route
// Assign directly to RawQuery to preserve the query string as-is
if hasQuery {
fullURL.RawQuery = queryPart
}
req, err := http.NewRequest(method, fullURL.String(), bytes.NewReader(queryString))
if err != nil {
return nil, errors.Wrapf(err, "failed to init a HTTP request")
}
// Only add basic auth header if it exists (not for AWS IAM auth)
if client.basicAuthString != "" {
req.Header.Add("Authorization", client.basicAuthString)
}
req.Header.Set("Content-Type", "application/json")
return client.httpClient.Do(req)
}
type AddressScheduler struct {
addresses []string
count int
}
// Get a new address using round-robin. No failover mechanisms temporarily.
func (scheduler *AddressScheduler) GetNewAddress() string {
address := scheduler.addresses[scheduler.count]
scheduler.count = (scheduler.count + 1) % len(scheduler.addresses)
return address
}
func (*Driver) Open(ctx context.Context, _ storepb.Engine, config db.ConnectionConfig) (db.Driver, error) {
address := fmt.Sprintf("%s:%s", config.DataSource.Host, config.DataSource.Port)
// Check if address already has a protocol
if !strings.HasPrefix(address, "http://") && !strings.HasPrefix(address, "https://") {
protocol := "http"
if config.DataSource.GetUseSsl() {
protocol = "https"
}
// AWS OpenSearch always requires HTTPS (port 443)
// Even if UseSsl is false, AWS managed services enforce HTTPS
if config.DataSource.GetAuthenticationType() == storepb.DataSource_AWS_RDS_IAM {
protocol = "https"
}
address = fmt.Sprintf("%s://%s", protocol, address)
}
if _, err := url.Parse(address); err != nil {
return nil, errors.Wrapf(err, "failed to parse address: %v", address)
}
if config.DataSource.GetAuthenticationType() == storepb.DataSource_AWS_RDS_IAM {
return openWithOpenSearchClient(ctx, config, address)
}
return openWithBasicAuth(ctx, config, address)
}
func openWithBasicAuth(_ context.Context, config db.ConnectionConfig, address string) (db.Driver, error) {
// Get TLS config that respects verify_tls_certificate setting
tlsConfig, err := util.GetTLSConfig(config.DataSource)
if err != nil {
return nil, errors.Wrap(err, "failed to get TLS config")
}
// Default to insecure if no SSL is configured
if tlsConfig == nil {
tlsConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
} else {
// Ensure minimum TLS version
tlsConfig.MinVersion = tls.VersionTLS12
}
esConfig := newElasticsearchConfig(config, address, tlsConfig)
// default http client.
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsConfig,
},
}
// typed elasticsearch client.
typedClient, err := elasticsearch.NewClient(esConfig)
if err != nil {
return nil, errors.Wrapf(err, "failed to create elasticsearch client")
}
// generate basic authentication string for http client.
encodedUsrAndPasswd := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", config.DataSource.Username, config.Password)))
basicAuthString := fmt.Sprintf("Basic %s", string(encodedUsrAndPasswd))
return &Driver{
typedClient: typedClient,
basicAuthClient: &BasicAuthClient{
httpClient: httpClient,
addrScheduler: &AddressScheduler{
addresses: []string{address},
count: 0,
},
basicAuthString: basicAuthString,
},
config: config,
}, nil
}
func newElasticsearchConfig(config db.ConnectionConfig, address string, tlsConfig *tls.Config) elasticsearch.Config {
esConfig := elasticsearch.Config{
Username: config.DataSource.Username,
Password: config.Password,
Addresses: []string{address},
Transport: &http.Transport{
MaxIdleConnsPerHost: 10,
ResponseHeaderTimeout: 5 * time.Second,
TLSClientConfig: tlsConfig,
},
}
if config.DataSource.GetSslCa() != "" {
esConfig.CACert = []byte(config.DataSource.GetSslCa())
}
return esConfig
}
func openWithOpenSearchClient(ctx context.Context, config db.ConnectionConfig, address string) (db.Driver, error) {
// Keep debugInfo for error messages
debugInfo := fmt.Sprintf("OpenSearch config - address: %s, authType: %v", address, config.DataSource.GetAuthenticationType())
// Get TLS config that respects verify_tls_certificate setting
tlsConfig, err := util.GetTLSConfig(config.DataSource)
if err != nil {
return nil, errors.Wrap(err, "failed to get TLS config")
}
// Default to insecure if no SSL is configured
if tlsConfig == nil {
tlsConfig = &tls.Config{
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
} else {
// Ensure minimum TLS version
tlsConfig.MinVersion = tls.VersionTLS12
}
baseTransport := &http.Transport{
MaxIdleConnsPerHost: 10,
ResponseHeaderTimeout: 5 * time.Second, // Increase timeout for AWS
TLSClientConfig: tlsConfig,
}
osConfig := opensearch.Config{
Addresses: []string{address},
Transport: baseTransport,
// A managed domain fronts its nodes behind one endpoint, so the
// addresses the cluster publishes are not reachable from here and
// SigV4 is bound to the host we were configured with. Left to its
// default the client discovers nodes and routes to those published
// addresses instead, abandoning the endpoint.
DiscoverNodesOnStart: new(false),
DiscoverNodesInterval: -1,
}
// Configure authentication
switch config.DataSource.GetAuthenticationType() {
case storepb.DataSource_AWS_RDS_IAM:
// Validate AWS-specific requirements
if config.DataSource.GetRegion() == "" {
return nil, errors.New("region is required for AWS IAM authentication")
}
// Load AWS configuration (uses specific credentials from UI if provided, otherwise default chain)
awsCfg, err := util.GetAWSConnectionConfig(ctx, config)
if err != nil {
return nil, errors.Wrap(err, "failed to load AWS config")
}
// Handle cross-account role assumption if configured
if err := util.AssumeRoleIfNeeded(ctx, &awsCfg, config.ConnectionContext, config.DataSource.GetAwsCredential()); err != nil {
return nil, err
}
// Verify credentials are available
_, err = awsCfg.Credentials.Retrieve(ctx)
if err != nil {
return nil, errors.Wrap(err, "failed to retrieve AWS credentials")
}
// Create AWS signer
signer, err := awsv2.NewSigner(awsCfg)
if err != nil {
return nil, errors.Wrap(err, "failed to create AWS signer")
}
osConfig.Signer = signer
default:
// Basic auth
osConfig.Username = config.DataSource.Username
osConfig.Password = config.Password
}
// Note: util.GetTLSConfig already handles CA cert configuration,
// so we don't need to manually handle it here anymore
// Create OpenSearch client
osClient, err := opensearch.NewClient(osConfig)
if err != nil {
return nil, errors.Wrapf(err, "failed to create OpenSearch client; %s", debugInfo)
}
// Create OpenSearch API client
apiClient, err := opensearchapi.NewClient(opensearchapi.Config{
Client: osConfig,
})
if err != nil {
return nil, errors.Wrapf(err, "failed to create OpenSearch API client; %s", debugInfo)
}
// For consistency, still create a basic auth client for REST operations
httpClient := &http.Client{
Transport: osConfig.Transport,
}
basicAuthString := ""
// For AWS IAM, we don't use basic auth - the AWS signer handles authentication
if config.DataSource.GetAuthenticationType() != storepb.DataSource_AWS_RDS_IAM &&
config.DataSource.Username != "" && config.Password != "" {
encodedUsrAndPasswd := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", config.DataSource.Username, config.Password)))
basicAuthString = fmt.Sprintf("Basic %s", string(encodedUsrAndPasswd))
}
return &Driver{
opensearchClient: osClient,
opensearchAPI: apiClient,
basicAuthClient: &BasicAuthClient{
httpClient: httpClient,
addrScheduler: &AddressScheduler{
addresses: []string{address},
count: 0,
},
basicAuthString: basicAuthString,
},
config: config,
isOpenSearch: true,
}, nil
}
// ElasticSearch doesn't keep a live connection as it uses stateless HTTP.
func (*Driver) Close(_ context.Context) error {
return nil
}
func (d *Driver) Ping(_ context.Context) error {
if d.isOpenSearch && d.opensearchAPI != nil {
ctx := context.Background()
info, err := d.opensearchAPI.Info(ctx, &opensearchapi.InfoReq{})
if err != nil {
// Check if it's an authentication or connection issue
errStr := err.Error()
if strings.Contains(errStr, "401") || strings.Contains(errStr, "403") {
// Check if role assumption was attempted
if d.config.DataSource.GetAwsCredential() != nil &&
d.config.DataSource.GetAwsCredential().RoleArn != "" {
return errors.Errorf("authentication failed: unable to assume role %s: %v",
d.config.DataSource.GetAwsCredential().RoleArn, err)
}
return errors.Errorf("authentication failed (consider using cross-account role if accessing different AWS account): %v", err)
}
if strings.Contains(errStr, "404") {
return errors.Errorf("endpoint not found (check if path is correct): %v", err)
}
// For any other error, return the full error
return errors.Wrapf(err, "failed to connect to OpenSearch")
}
if info == nil || info.Version.Number == "" {
return errors.New("invalid response from server")
}
return nil
}
// Use Elasticsearch client
if d.typedClient != nil {
res, err := d.typedClient.Ping()
if err != nil {
return errors.Wrapf(err, "failed to ping db")
}
defer res.Body.Close()
if res.IsError() {
return errors.Errorf("ping failed: %s", res.String())
}
return nil
}
// Fallback to basic auth client
if d.basicAuthClient != nil {
resp, err := d.basicAuthClient.Do("GET", []byte("/"), nil)
if err != nil {
return errors.Wrapf(err, "failed to ping db")
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
body, _ := io.ReadAll(resp.Body)
return errors.Errorf("ping failed with status %d: %s", resp.StatusCode, body)
}
return nil
}
return errors.New("no client available for ping")
}
func (d *Driver) Execute(ctx context.Context, statement string, _ db.ExecuteOptions) (int64, error) {
_, err := d.QueryConn(ctx, nil, statement, db.QueryContext{})
if err != nil {
return 0, err
}
return 0, nil
}
func (d *Driver) QueryConn(_ context.Context, _ *sql.Conn, statement string, queryContext db.QueryContext) ([]*v1pb.QueryResult, error) {
parseResult, err := parser.ParseElasticsearchREST(statement)
if err != nil {
return nil, err
}
if len(parseResult.Errors) > 0 {
return nil, parseResult.Errors[0]
}
if parseResult.Requests == nil {
return nil, nil
}
var results []*v1pb.QueryResult
for _, request := range parseResult.Requests {
if queryContext.Explain {
// The plan is what the search did, so the search runs with
// profiling on; anything that is not a search is refused rather
// than run.
profiled, err := profiledSearch(request)
if err != nil {
return nil, err
}
request = profiled
}
if err := func() error {
startTime := time.Now()
// send HTTP request.
// For NDJSON endpoints (_msearch, _bulk, etc.), each JSON object must
// be on a single line. Remove embedded newlines from each data item
// before joining with newline separators.
var data []byte
for _, item := range request.Data {
data = append(data, []byte(strings.ReplaceAll(item, "\n", ""))...)
data = append(data, '\n')
}
var resp *http.Response
// For AWS IAM auth, we need to use the OpenSearch client which has the signer
if d.isOpenSearch && d.opensearchClient != nil && d.config.DataSource.GetAuthenticationType() == storepb.DataSource_AWS_RDS_IAM {
// Create request for OpenSearch client
// Ensure the path starts with "/" for proper HTTP request
requestPath := strings.TrimSpace(request.URL)
if requestPath != "" && !strings.HasPrefix(requestPath, "/") {
requestPath = "/" + requestPath
}
req, err := http.NewRequest(request.Method, requestPath, bytes.NewReader(data))
if err != nil {
return errors.Wrapf(err, "failed to create HTTP request")
}
req.Header.Set("Content-Type", "application/json")
// Use OpenSearch client's Stream method which handles AWS signing
resp, err = d.opensearchClient.Stream(req)
if err != nil {
return errors.Wrapf(err, "failed to send HTTP request via OpenSearch client")
}
} else {
// For basic auth, use the basic auth client as before
resp, err = d.basicAuthClient.Do(request.Method, []byte(request.URL), data)
if err != nil {
return errors.Wrapf(err, "failed to send HTTP request")
}
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return errors.Wrap(err, "failed to read response body")
}
// Check HTTP status code for errors (4xx, 5xx)
if resp.StatusCode >= 400 {
return errors.Errorf("request failed with status code %d: %s", resp.StatusCode, string(respBytes))
}
// structure results.
var result v1pb.QueryResult
var row v1pb.QueryRow
contentType := resp.Header.Get("Content-Type")
// Debug HTML responses
if strings.Contains(contentType, "text/html") && resp.StatusCode >= 400 {
return errors.Errorf("received HTML error response (status %d): %s", resp.StatusCode, string(respBytes))
}
switch {
case strings.Contains(contentType, "application/json"):
// ElasticSearch/OpenSearch REST APIs return two types of JSON responses:
// 1. Objects (most APIs): {"took":5,"hits":{...}}
// 2. Arrays (_cat APIs with format=json): [{"index":"test",...}]
// They never return primitive types (strings, numbers, booleans, null).
// First unmarshal into any to determine the JSON type
var data any
if err := json.Unmarshal(respBytes, &data); err != nil {
return errors.Wrapf(err, "failed to parse json body")
}
// Handle based on the actual type
switch v := data.(type) {
case map[string]any:
if queryContext.Explain {
// The search ran so that it could be profiled; its hits
// are not what an explain asked for, and they leave
// here rather than reaching a caller that masking no
// longer covers.
profile, err := profileOnly(v)
if err != nil {
return err
}
result.ColumnNames = append(result.ColumnNames, profileField)
row.Values = append(row.Values, &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{StringValue: profile}})
result.QueryPlan = explainQueryPlan()
break
}
// Object response: create one column per key
for key, val := range v {
result.ColumnNames = append(result.ColumnNames, key)
bytes, err := json.Marshal(val)
if err != nil {
return err
}
row.Values = append(row.Values, &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{StringValue: string(bytes)}})
}
case []any:
if queryContext.Explain {
return errors.Errorf("the profiled search answered with a list, which is not a search response")
}
// Array response (e.g., _cat APIs with format=json): create a single column with array data
result.ColumnNames = append(result.ColumnNames, "result")
bytes, err := json.Marshal(v)
if err != nil {
return err
}
row.Values = append(row.Values, &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{StringValue: string(bytes)}})
default:
// ElasticSearch/OpenSearch APIs only return objects or arrays, never primitives.
// If we get here, something unexpected happened.
return errors.Errorf("unexpected JSON type: %T (expected object or array)", v)
}
case strings.Contains(contentType, "text/plain"):
result.ColumnNames = append(result.ColumnNames, "text/plain")
row.Values = append(row.Values, &v1pb.RowValue{Kind: &v1pb.RowValue_StringValue{StringValue: string(respBytes)}})
default:
return errors.Errorf("Content-Type not supported: %s", contentType)
}
result.Rows = append(result.Rows, &row)
result.Latency = durationpb.New(time.Since(startTime))
result.RowsCount = int64(len(result.Rows))
result.Statement = fmt.Sprintf("%s %s\n%s", request.Method, request.URL, string(data))
// TODO(d): handle max size.
results = append(results, &result)
return nil
}(); err != nil {
return nil, err
}
}
return results, nil
}
func (*Driver) GetDB() *sql.DB {
return nil
}