Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(elasticsearch): add statement type ACL support (#19251)
h3n4l75aa9af86b88
diff --git a/backend/common/engine.go b/backend/common/engine.goindex 8225fb4e57..f9c4183c4b 100644--- a/backend/common/engine.go+++ b/backend/common/engine.go@@ -58,7 +58,8 @@ func EngineSupportQueryNewACL(engine storepb.Engine) bool {storepb.Engine_SNOWFLAKE,storepb.Engine_SPANNER,storepb.Engine_BIGQUERY,- storepb.Engine_MONGODB:+ storepb.Engine_MONGODB,+ storepb.Engine_ELASTICSEARCH:return truecasestorepb.Engine_ENGINE_UNSPECIFIED,@@ -74,7 +75,6 @@ func EngineSupportQueryNewACL(engine storepb.Engine) bool {storepb.Engine_COCKROACHDB,storepb.Engine_DORIS,storepb.Engine_DYNAMODB,- storepb.Engine_ELASTICSEARCH,storepb.Engine_DATABRICKS,storepb.Engine_COSMOSDB,storepb.Engine_TRINO:diff --git a/backend/plugin/parser/elasticsearch/parser.go b/backend/plugin/parser/elasticsearch/parser.goindex c0dc02f324..d09d27c5cb 100644--- a/backend/plugin/parser/elasticsearch/parser.go+++ b/backend/plugin/parser/elasticsearch/parser.go@@ -1039,9 +1039,6 @@ func (p *parser) method() (string, error) {if err := p.nextOneOf([]rune{'E', 'e'}); err != nil {return "", err}- if err := p.nextOneOf([]rune{'S', 's'}); err != nil {- return "", err- }return "DELETE", nilcase "P":if err := p.nextOneOf([]rune{'P', 'p'}); err != nil {diff --git a/backend/plugin/parser/elasticsearch/query_span.go b/backend/plugin/parser/elasticsearch/query_span.gonew file mode 100644index 0000000000..bcd327374c--- /dev/null+++ b/backend/plugin/parser/elasticsearch/query_span.go@@ -0,0 +1,34 @@+package elasticsearch++import (+ "context"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterGetQuerySpan(storepb.Engine_ELASTICSEARCH, GetQuerySpan)+}++// GetQuerySpan returns the query span for an ElasticSearch REST API request.+func GetQuerySpan(+ _ context.Context,+ _ base.GetQuerySpanContext,+ stmt base.Statement,+ _, _ string,+ _ bool,+) (*base.QuerySpan, error) {+ parseResult, _ := ParseElasticsearchREST(stmt.Text)++ if parseResult == nil || len(parseResult.Requests) == 0 {+ return &base.QuerySpan{Type: base.QueryTypeUnknown}, nil+ }++ // After splitting, each statement should contain a single request.+ // Use the first request for classification.+ req := parseResult.Requests[0]+ queryType := ClassifyRequest(req.Method, req.URL)++ return &base.QuerySpan{Type: queryType}, nil+}diff --git a/backend/plugin/parser/elasticsearch/query_span_test.go b/backend/plugin/parser/elasticsearch/query_span_test.gonew file mode 100644index 0000000000..1371b84b8d--- /dev/null+++ b/backend/plugin/parser/elasticsearch/query_span_test.go@@ -0,0 +1,58 @@+package elasticsearch++import (+ "context"+ "io"+ "os"+ "testing"++ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestGetQuerySpan(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description,omitempty"`+ Statement string `yaml:"statement,omitempty"`+ QueryType base.QueryType `yaml:"queryType,omitempty"`+ }++ const (+ record = false+ testDataPath = "test-data/query_span.yaml"+ )++ a := require.New(t)++ yamlFile, err := os.Open(testDataPath)+ a.NoError(err)++ byteValue, err := io.ReadAll(yamlFile)+ a.NoError(err)+ a.NoError(yamlFile.Close())++ var testCases []testCase+ a.NoError(yaml.Unmarshal(byteValue, &testCases))++ for i, tc := range testCases {+ t.Run(tc.Description, func(_ *testing.T) {+ span, err := GetQuerySpan(context.Background(), base.GetQuerySpanContext{}, base.Statement{Text: tc.Statement}, "", "", false)+ a.NoError(err)+ a.NotNil(span)+ if record {+ testCases[i].QueryType = span.Type+ } else {+ a.Equalf(tc.QueryType, span.Type, "description: %s, statement: %s", tc.Description, tc.Statement)+ }+ })+ }++ if record {+ byteValue, err := yaml.Marshal(testCases)+ a.NoError(err)+ err = os.WriteFile(testDataPath, byteValue, 0644)+ a.NoError(err)+ }+}diff --git a/backend/plugin/parser/elasticsearch/query_type.go b/backend/plugin/parser/elasticsearch/query_type.gonew file mode 100644index 0000000000..289852fbb5--- /dev/null+++ b/backend/plugin/parser/elasticsearch/query_type.go@@ -0,0 +1,156 @@+package elasticsearch++import (+ "strings"++ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// readOnlyPostEndpoints lists URL patterns that are read-only despite using POST.+// These endpoints accept a request body for query parameters but do not modify data.+var readOnlyPostEndpoints = []string{+ "_search",+ "_msearch",+ "_count",+ "_field_caps",+ "_validate/query",+ "_search_shards",+ "_terms_enum",+ "_termvectors",+ "_mtermvectors",+ "_search_template",+ "_msearch_template",+ "_render/template",+ "_rank_eval",+ "_knn_search",+ "_search_mvt",+ "_sql",+ "_esql/query",+ "_eql/search",+ "_fleet/_search",+ "_fleet/_msearch",+ "_graph/explore",+ "rollup_search",+ "async_search",+}++// infoSchemaEndpoints lists URL patterns for cluster/node metadata queries.+var infoSchemaEndpoints = []string{+ "_cat/",+ "_cluster/",+ "_nodes/",+}++// dmlPostEndpoints lists URL patterns for document write operations via POST.+var dmlPostEndpoints = []string{+ "_doc",+ "_create/",+ "_update/",+ "_bulk",+ "_delete_by_query",+ "_update_by_query",+ "_reindex",+}++// ClassifyRequest determines the QueryType for an ElasticSearch REST API request.+func ClassifyRequest(method, url string) base.QueryType {+ method = strings.ToUpper(method)+ urlLower := strings.ToLower(url)++ switch method {+ case "HEAD":+ return base.Select++ case "GET":+ if isInfoSchemaURL(urlLower) {+ return base.SelectInfoSchema+ }+ return base.Select++ case "DELETE":+ if isDocumentURL(urlLower) {+ return base.DML+ }+ return base.DDL++ case "PUT":+ if isDocumentWriteURL(urlLower) {+ return base.DML+ }+ return base.DDL++ case "PATCH":+ return base.DML++ case "POST":+ return classifyPostRequest(urlLower)++ default:+ return base.QueryTypeUnknown+ }+}++func classifyPostRequest(url string) base.QueryType {+ // Explain+ if strings.Contains(url, "_explain/") || strings.HasSuffix(url, "_explain") {+ return base.Explain+ }++ // Info schema (metadata reads)+ if isInfoSchemaURL(url) {+ return base.SelectInfoSchema+ }++ // Read-only POST endpoints+ if isReadOnlyPostURL(url) {+ return base.Select+ }++ // Document writes (DML)+ if isDMLPostURL(url) {+ return base.DML+ }++ // Default: DDL for other POST operations (index management, admin, etc.)+ return base.DDL+}++func isInfoSchemaURL(url string) bool {+ for _, pattern := range infoSchemaEndpoints {+ if strings.Contains(url, pattern) {+ return true+ }+ }+ return false+}++func isReadOnlyPostURL(url string) bool {+ for _, pattern := range readOnlyPostEndpoints {+ if strings.Contains(url, pattern) {+ return true+ }+ }+ return false+}++func isDMLPostURL(url string) bool {+ for _, pattern := range dmlPostEndpoints {+ if strings.Contains(url, pattern) {+ return true+ }+ }+ return false+}++func isDocumentURL(url string) bool {+ // DELETE /{index}/_doc/{id} is DML (document deletion)+ return strings.Contains(url, "_doc/")+}++func isDocumentWriteURL(url string) bool {+ // PUT /{index}/_doc/{id}, PUT /{index}/_create/{id}, PUT /_bulk are DML+ return strings.Contains(url, "_doc/") ||+ strings.Contains(url, "_doc") ||+ strings.Contains(url, "_create/") ||+ strings.Contains(url, "_bulk")+}diff --git a/backend/plugin/parser/elasticsearch/splitter.go b/backend/plugin/parser/elasticsearch/splitter.gonew file mode 100644index 0000000000..df52a469d2--- /dev/null+++ b/backend/plugin/parser/elasticsearch/splitter.go@@ -0,0 +1,80 @@+package elasticsearch++import (+ "strings"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func init() {+ base.RegisterSplitterFunc(storepb.Engine_ELASTICSEARCH, SplitMultiSQL)+}++// SplitMultiSQL splits the input into individual ElasticSearch REST API requests.+func SplitMultiSQL(statement string) ([]base.Statement, error) {+ parseResult, err := ParseElasticsearchREST(statement)+ if err != nil {+ return nil, err+ }++ if parseResult == nil || len(parseResult.Requests) == 0 {+ if len(strings.TrimSpace(statement)) == 0 {+ return nil, nil+ }+ return []base.Statement{{Text: statement}}, nil+ }++ var statements []base.Statement+ for _, req := range parseResult.Requests {+ if req == nil {+ continue+ }+ text := statement[req.StartOffset:req.EndOffset]+ empty := len(strings.TrimSpace(text)) == 0++ // Calculate 1-based line and column positions+ startLine, startColumn := byteOffsetToPosition(statement, req.StartOffset)+ endLine, endColumn := byteOffsetToPosition(statement, req.EndOffset)++ statements = append(statements, base.Statement{+ Text: text,+ Empty: empty,+ Start: &storepb.Position{+ Line: int32(startLine),+ Column: int32(startColumn),+ },+ End: &storepb.Position{+ Line: int32(endLine),+ Column: int32(endColumn),+ },+ Range: &storepb.Range{+ Start: int32(req.StartOffset),+ End: int32(req.EndOffset),+ },+ })+ }+ return statements, nil+}++// byteOffsetToPosition converts a byte offset to 1-based line and column numbers.+// Column is measured in Unicode code points (runes), not bytes.+func byteOffsetToPosition(text string, byteOffset int) (line, column int) {+ line = 1+ column = 1+ currentByte := 0++ for _, r := range text {+ if currentByte >= byteOffset {+ break+ }+ if r == '\n' {+ line+++ column = 1+ } else {+ column+++ }+ currentByte += len(string(r))+ }+ return line, column+}diff --git a/backend/plugin/parser/elasticsearch/splitter_test.go b/backend/plugin/parser/elasticsearch/splitter_test.gonew file mode 100644index 0000000000..ac34afb55a--- /dev/null+++ b/backend/plugin/parser/elasticsearch/splitter_test.go@@ -0,0 +1,139 @@+package elasticsearch++import (+ "io"+ "os"+ "testing"++ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++type yamlPosition struct {+ Line int32 `yaml:"line,omitempty"`+ Column int32 `yaml:"column,omitempty"`+}++type yamlRange struct {+ Start int32 `yaml:"start,omitempty"`+ End int32 `yaml:"end,omitempty"`+}++type yamlStatement struct {+ Text string `yaml:"text,omitempty"`+ Start *yamlPosition `yaml:"start,omitempty"`+ End *yamlPosition `yaml:"end,omitempty"`+ Range *yamlRange `yaml:"range,omitempty"`+}++func TestSplitMultiSQL(t *testing.T) {+ type testCase struct {+ Description string `yaml:"description,omitempty"`+ Statement string `yaml:"statement,omitempty"`+ ExpectedCount int `yaml:"expectedCount,omitempty"`+ Statements []yamlStatement `yaml:"statements,omitempty"`+ }++ const (+ testDataPath = "test-data/splitter.yaml"+ )++ a := require.New(t)++ yamlFile, err := os.Open(testDataPath)+ a.NoError(err)++ byteValue, err := io.ReadAll(yamlFile)+ a.NoError(err)+ a.NoError(yamlFile.Close())++ var testCases []testCase+ a.NoError(yaml.Unmarshal(byteValue, &testCases))++ for _, tc := range testCases {+ t.Run(tc.Description, func(_ *testing.T) {+ statements, err := SplitMultiSQL(tc.Statement)+ a.NoError(err)+ a.Len(statements, tc.ExpectedCount)++ for i, expected := range tc.Statements {+ if i >= len(statements) {+ break+ }+ stmt := statements[i]++ if expected.Text != "" {+ a.Equal(expected.Text, stmt.Text, "statement %d text", i)+ }++ if expected.Start != nil {+ a.NotNil(stmt.Start, "statement %d should have Start position", i)+ a.Equal(expected.Start.Line, stmt.Start.Line, "statement %d start line", i)+ a.Equal(expected.Start.Column, stmt.Start.Column, "statement %d start column", i)+ }++ if expected.End != nil {+ a.NotNil(stmt.End, "statement %d should have End position", i)+ a.Equal(expected.End.Line, stmt.End.Line, "statement %d end line", i)+ a.Equal(expected.End.Column, stmt.End.Column, "statement %d end column", i)+ }++ if expected.Range != nil {+ a.NotNil(stmt.Range, "statement %d should have Range", i)+ a.Equal(expected.Range.Start, stmt.Range.Start, "statement %d range start", i)+ a.Equal(expected.Range.End, stmt.Range.End, "statement %d range end", i)+ }+ }++ // Verify all statements have required fields+ for i, stmt := range statements {+ a.NotNil(stmt.Start, "statement %d should have Start", i)+ a.NotNil(stmt.End, "statement %d should have End", i)+ a.NotNil(stmt.Range, "statement %d should have Range", i)+ a.False(stmt.Empty, "statement %d should not be empty", i)+ }+ })+ }+}++// Keep this for compatibility with existing tests that use the function directly+func TestSplitMultiSQLDirect(t *testing.T) {+ tests := []struct {+ name string+ statement string+ expectedCount int+ expectedStarts []*storepb.Position+ }{+ {+ name: "empty statement",+ statement: "",+ expectedCount: 0,+ },+ {+ name: "single GET request",+ statement: "GET _search",+ expectedCount: 1,+ expectedStarts: []*storepb.Position{+ {Line: 1, Column: 1},+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ statements, err := SplitMultiSQL(tc.statement)+ require.NoError(t, err)+ require.Len(t, statements, tc.expectedCount)++ if tc.expectedStarts != nil {+ for i, expectedStart := range tc.expectedStarts {+ require.NotNil(t, statements[i].Start, "statement %d should have Start position", i)+ require.Equal(t, expectedStart.Line, statements[i].Start.Line, "statement %d start line", i)+ require.Equal(t, expectedStart.Column, statements[i].Start.Column, "statement %d start column", i)+ }+ }+ })+ }+}diff --git a/backend/plugin/parser/elasticsearch/test-data/query_span.yaml b/backend/plugin/parser/elasticsearch/test-data/query_span.yamlnew file mode 100644index 0000000000..6f36c94e90--- /dev/null+++ b/backend/plugin/parser/elasticsearch/test-data/query_span.yaml@@ -0,0 +1,196 @@+# GET requests - Select+- description: GET search+ statement: "GET _search"+ queryType: 1 # Select++- description: GET document+ statement: "GET /my_index/_doc/1"+ queryType: 1 # Select++- description: GET mapping+ statement: "GET /my_index/_mapping"+ queryType: 1 # Select++# GET requests - SelectInfoSchema+- description: GET cat indices+ statement: "GET /_cat/indices"+ queryType: 3 # SelectInfoSchema++- description: GET cluster health+ statement: "GET /_cluster/health"+ queryType: 3 # SelectInfoSchema++- description: GET nodes info+ statement: "GET /_nodes/stats"+ queryType: 3 # SelectInfoSchema++# HEAD requests - Select+- description: HEAD index exists+ statement: "HEAD /my_index"+ queryType: 1 # Select++- description: HEAD document exists+ statement: "HEAD /my_index/_doc/1"+ queryType: 1 # Select++# POST requests - Select (read-only endpoints)+- description: POST search+ statement: "POST /my_index/_search"+ queryType: 1 # Select++- description: POST msearch+ statement: "POST /_msearch"+ queryType: 1 # Select++- description: POST count+ statement: "POST /my_index/_count"+ queryType: 1 # Select++- description: POST field_caps+ statement: "POST /_field_caps"+ queryType: 1 # Select++- description: POST validate query+ statement: "POST /my_index/_validate/query"+ queryType: 1 # Select++- description: POST SQL query+ statement: "POST /_sql"+ queryType: 1 # Select++- description: POST ESQL query+ statement: "POST /_esql/query"+ queryType: 1 # Select++- description: POST EQL search+ statement: "POST /my_index/_eql/search"+ queryType: 1 # Select++- description: POST graph explore+ statement: "POST /my_index/_graph/explore"+ queryType: 1 # Select++- description: POST knn search+ statement: "POST /my_index/_knn_search"+ queryType: 1 # Select++- description: POST async search submit+ statement: "POST /my_index/_async_search"+ queryType: 1 # Select++# POST requests - Explain+- description: POST explain+ statement: "POST /my_index/_explain/1"+ queryType: 2 # Explain++# POST requests - DML (document writes)+- description: POST document with auto ID+ statement: "POST /my_index/_doc"+ queryType: 5 # DML++- description: POST document with ID+ statement: "POST /my_index/_doc/1"+ queryType: 5 # DML++- description: POST create document+ statement: "POST /my_index/_create/1"+ queryType: 5 # DML++- description: POST update document+ statement: "POST /my_index/_update/1"+ queryType: 5 # DML++- description: POST bulk+ statement: "POST /_bulk"+ queryType: 5 # DML++- description: POST delete by query+ statement: "POST /my_index/_delete_by_query"+ queryType: 5 # DML++- description: POST update by query+ statement: "POST /my_index/_update_by_query"+ queryType: 5 # DML++- description: POST reindex+ statement: "POST /_reindex"+ queryType: 5 # DML++# POST requests - DDL (index/admin operations)+- description: POST open index+ statement: "POST /my_index/_open"+ queryType: 4 # DDL++- description: POST close index+ statement: "POST /my_index/_close"+ queryType: 4 # DDL++- description: POST aliases+ statement: "POST /_aliases"+ queryType: 4 # DDL++- description: POST rollover+ statement: "POST /my_index/_rollover"+ queryType: 4 # DDL++- description: POST mapping+ statement: "POST /my_index/_mapping"+ queryType: 4 # DDL++# PUT requests - DML (document writes)+- description: PUT document+ statement: "PUT /my_index/_doc/1"+ queryType: 5 # DML++- description: PUT create document+ statement: "PUT /my_index/_create/1"+ queryType: 5 # DML++- description: PUT bulk+ statement: "PUT /_bulk"+ queryType: 5 # DML++# PUT requests - DDL (index/schema operations)+- description: PUT create index+ statement: "PUT /my_index"+ queryType: 4 # DDL++- description: PUT mapping+ statement: "PUT /my_index/_mapping"+ queryType: 4 # DDL++- description: PUT settings+ statement: "PUT /my_index/_settings"+ queryType: 4 # DDL++- description: PUT index template+ statement: "PUT /_index_template/my_template"+ queryType: 4 # DDL++- description: PUT ingest pipeline+ statement: "PUT /_ingest/pipeline/my_pipeline"+ queryType: 4 # DDL++# DELETE requests - DML (document deletion)+- description: DELETE document+ statement: "DELETE /my_index/_doc/1"+ queryType: 5 # DML++# DELETE requests - DDL (index deletion)+- description: DELETE index+ statement: "DELETE /my_index"+ queryType: 4 # DDL++- description: DELETE index template+ statement: "DELETE /_index_template/my_template"+ queryType: 4 # DDL++# PATCH requests - DML+- description: PATCH document+ statement: "PATCH /my_index/_doc/1"+ queryType: 5 # DML++# Unknown method+- description: Unknown method+ statement: "UNKNOWN /my_index"+ queryType: 0 # QueryTypeUnknowndiff --git a/backend/plugin/parser/elasticsearch/test-data/splitter.yaml b/backend/plugin/parser/elasticsearch/test-data/splitter.yamlnew file mode 100644index 0000000000..80ea112aa1--- /dev/null+++ b/backend/plugin/parser/elasticsearch/test-data/splitter.yaml@@ -0,0 +1,80 @@+- description: empty statement+ statement: ""+ expectedCount: 0++- description: whitespace only+ statement: " \n\t "+ expectedCount: 0++- description: single GET request+ statement: "GET _search"+ expectedCount: 1+ statements:+ - text: "GET _search"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 12+ range:+ start: 0+ end: 11++- description: single POST request with body+ statement: "POST /my_index/_search\n{\n \"query\": {\n \"match_all\": {}\n }\n}"+ expectedCount: 1+ statements:+ - text: "POST /my_index/_search\n{\n \"query\": {\n \"match_all\": {}\n }\n}"+ start:+ line: 1+ column: 1++- description: multiple requests+ statement: "GET _search\nPOST _test_index"+ expectedCount: 2+ statements:+ - text: "GET _search"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 12+ range:+ start: 0+ end: 11+ - text: "POST _test_index"+ start:+ line: 2+ column: 1+ end:+ line: 2+ column: 17+ range:+ start: 12+ end: 28++- description: multiple requests with bodies+ statement: |+ GET _search+ {+ "query": {+ "match_all": {}+ }+ }++ GET _search+ {+ "query": {+ "match_all": {}+ }+ }+ expectedCount: 2+ statements:+ - start:+ line: 1+ column: 1+ - start:+ line: 8+ column: 1