Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(elasticsearch): rewrite parser as thin omni adapters (#19944)
h3n4l295d8274bb4a
diff --git a/backend/plugin/parser/elasticsearch/diagnose.go b/backend/plugin/parser/elasticsearch/diagnose.goindex 1510f8a279..dfafca7b50 100644--- a/backend/plugin/parser/elasticsearch/diagnose.go+++ b/backend/plugin/parser/elasticsearch/diagnose.go@@ -3,6 +3,10 @@ package elasticsearchimport ("context"+ lsp "github.com/bytebase/lsp-protocol"++ es "github.com/bytebase/omni/elasticsearch"+"github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -12,16 +16,31 @@ func init() {}func Diagnose(_ context.Context, _ base.DiagnoseContext, statement string) ([]base.Diagnostic, error) {- var diagnostics []base.Diagnostic- parseResult, _ := ParseElasticsearchREST(statement)- if parseResult == nil {- return nil, nil+ omniDiags, err := es.Diagnose(statement)+ if err != nil {+ return nil, err}- for _, err := range parseResult.Errors {- if err != nil {- diagnostics = append(diagnostics, base.ConvertSyntaxErrorToDiagnostic(err, statement))- }+ if omniDiags == nil {+ return nil, nil}+ var diagnostics []base.Diagnostic+ for _, d := range omniDiags {+ diagnostics = append(diagnostics, base.Diagnostic{+ Range: lsp.Range{+ Start: lsp.Position{+ Line: uint32(d.Range.Start.Line),+ Character: uint32(d.Range.Start.Column),+ },+ End: lsp.Position{+ Line: uint32(d.Range.End.Line),+ Character: uint32(d.Range.End.Column),+ },+ },+ Severity: lsp.DiagnosticSeverity(d.Severity),+ Source: "Syntax check",+ Message: d.Message,+ })+ }return diagnostics, nil}diff --git a/backend/plugin/parser/elasticsearch/masking.go b/backend/plugin/parser/elasticsearch/masking.goindex 4631bcefb1..39c3a1441b 100644--- a/backend/plugin/parser/elasticsearch/masking.go+++ b/backend/plugin/parser/elasticsearch/masking.go@@ -1,9 +1,7 @@package elasticsearchimport (- "encoding/json"- "slices"- "strings"+ "github.com/bytebase/omni/elasticsearch/analysis""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -38,425 +36,23 @@ const (// BlockedFeatureNames maps BlockedFeature to human-readable names for error messages.var BlockedFeatureNames = base.ElasticsearchBlockedFeatureNames-// blockedEndpoints lists URL patterns that must be rejected when masking is active.-// These overlap with read patterns but cannot be safely masked.-// Includes both Elasticsearch and OpenSearch equivalents.-var blockedEndpoints = []string{- // Elasticsearch endpoints.- "_async_search",- "_search/scroll",- "_search_template",- "_msearch/template",- "_sql",- "_eql/search",- "_esql/query",- "_terms_enum",- "_termvectors",- "_mtermvectors",- "_knn_search",- // OpenSearch equivalents (plugin-based paths).- "_plugins/_asynchronous_search",- "_plugins/_sql",- "_plugins/_ppl",-}--// classifyMaskableAPI determines the MaskableAPI type and target index for an ES request.-func classifyMaskableAPI(method, url string) (MaskableAPI, string) {- method = strings.ToUpper(method)-- // Strip query parameters.- path := url- if i := strings.IndexByte(path, '?'); i >= 0 {- path = path[:i]- }-- // Strip leading slash.- path = strings.TrimPrefix(path, "/")-- // Only GET and POST can return user data.- switch method {- case "GET":- if blocked, index := isBlockedEndpoint(path); blocked {- return APIBlocked, index- }- return classifyGetForMasking(path)- case "POST":- if blocked, index := isBlockedEndpoint(path); blocked {- return APIBlocked, index- }- return classifyPostForMasking(path)- default:- // HEAD, PUT, DELETE, PATCH do not return user data.- return APIUnsupported, ""- }-}--// isBlockedEndpoint checks whether the path matches a blocked endpoint pattern.-// Returns true and the extracted index if blocked.-func isBlockedEndpoint(path string) (bool, string) {- for _, pattern := range blockedEndpoints {- if strings.Contains(path, pattern) {- return true, extractIndex(path)- }- }- return false, ""-}--// classifyGetForMasking classifies a GET request path for masking.-func classifyGetForMasking(path string) (MaskableAPI, string) {- for part := range strings.SplitSeq(path, "/") {- switch part {- case "_search", "_msearch":- return APIMaskSearch, extractIndex(path)- case "_doc":- return APIMaskGetDoc, extractIndex(path)- case "_source":- return APIMaskGetSource, extractIndex(path)- case "_mget":- return APIMaskMGet, extractIndex(path)- }- }-- return APIUnsupported, ""-}--// classifyPostForMasking classifies a POST request path for masking.-func classifyPostForMasking(path string) (MaskableAPI, string) {- for part := range strings.SplitSeq(path, "/") {- switch part {- case "_search", "_msearch":- return APIMaskSearch, extractIndex(path)- case "_mget":- return APIMaskMGet, extractIndex(path)- case "_explain":- return APIMaskExplain, extractIndex(path)- }- }-- return APIUnsupported, ""-}--// extractIndex returns the first path segment if it does not start with "_".-// This extracts the index name from ES URL paths like "<index>/_search".-func extractIndex(path string) string {- if path == "" {- return ""- }- parts := strings.SplitN(path, "/", 2)- if len(parts) == 0 {- return ""- }- first := parts[0]- if strings.HasPrefix(first, "_") {- return ""- }- return first-}-// AnalyzeRequest combines URL classification with body analysis for an ES request.// Body analysis is only performed for search and explain APIs.func AnalyzeRequest(method, url, body string) *RequestAnalysis {- api, index := classifyMaskableAPI(method, url)- result := &RequestAnalysis{- API: api,- Index: index,- }-- // Only analyze the body for APIs that accept a query body.- switch api {- case APIMaskSearch, APIMaskExplain:- bodyResult := analyzeRequestBody(body)- result.BlockedFeatures = bodyResult.BlockedFeatures- result.SourceFields = bodyResult.SourceFields- result.SourceDisabled = bodyResult.SourceDisabled- result.RequestedFields = bodyResult.RequestedFields- result.HighlightFields = bodyResult.HighlightFields- result.SortFields = bodyResult.SortFields- result.HasInnerHits = bodyResult.HasInnerHits- result.PredicateFields = bodyResult.PredicateFields- default:- // No body analysis for other API types.- }+ omniResult := analysis.AnalyzeRequest(method, url, body)+ return convertAnalysis(omniResult)+}- return result+// classifyMaskableAPI determines the MaskableAPI type and target index for an ES request.+func classifyMaskableAPI(method, url string) (MaskableAPI, string) {+ omniAPI, index := analysis.ClassifyMaskableAPI(method, url)+ return convertMaskableAPI(omniAPI), index}// analyzeRequestBody parses the JSON body and extracts fields and blocked features.+// This is an internal adapter kept for test compatibility.func analyzeRequestBody(body string) *RequestAnalysis {- result := &RequestAnalysis{}-- parsed := make(map[string]any)- if json.Unmarshal([]byte(body), &parsed) != nil {- return result- }-- result.BlockedFeatures = detectBlockedFeatures(parsed)- extractSource(parsed, result)- result.RequestedFields = extractStringArray(parsed, "fields")- result.HighlightFields = extractHighlightFields(parsed)- result.SortFields = extractSortFields(parsed)- result.HasInnerHits = containsKey(parsed, "inner_hits")- result.PredicateFields = extractPredicateFields(parsed)-- return result-}--// detectBlockedFeatures checks for top-level keys that indicate blocked features.-func detectBlockedFeatures(parsed map[string]any) []BlockedFeature {- var blocked []BlockedFeature-- if _, ok := parsed["aggs"]; ok {- blocked = append(blocked, BlockedFeatureAggs)- } else if _, ok := parsed["aggregations"]; ok {- blocked = append(blocked, BlockedFeatureAggs)- }- if _, ok := parsed["suggest"]; ok {- blocked = append(blocked, BlockedFeatureSuggest)- }- if _, ok := parsed["script_fields"]; ok {- blocked = append(blocked, BlockedFeatureScriptFields)- }- if _, ok := parsed["runtime_mappings"]; ok {- blocked = append(blocked, BlockedFeatureRuntimeMappings)- }- if _, ok := parsed["stored_fields"]; ok {- blocked = append(blocked, BlockedFeatureStoredFields)- }- if _, ok := parsed["docvalue_fields"]; ok {- blocked = append(blocked, BlockedFeatureDocvalueFields)- }-- return blocked-}--// extractSource handles the three forms of _source in the request body.-func extractSource(parsed map[string]any, result *RequestAnalysis) {- src, ok := parsed["_source"]- if !ok {- return- }-- switch v := src.(type) {- case bool:- if !v {- result.SourceDisabled = true- }- case []any:- result.SourceFields = toStringSlice(v)- case map[string]any:- if includes, ok := v["includes"]; ok {- if arr, ok := includes.([]any); ok {- result.SourceFields = toStringSlice(arr)- }- }- default:- }-}--// extractStringArray extracts a string array value from a top-level key.-func extractStringArray(parsed map[string]any, key string) []string {- val, ok := parsed[key]- if !ok {- return nil- }- arr, ok := val.([]any)- if !ok {- return nil- }- return toStringSlice(arr)-}--// extractHighlightFields extracts field names from highlight.fields.-func extractHighlightFields(parsed map[string]any) []string {- highlight, ok := parsed["highlight"]- if !ok {- return nil- }- hlMap, ok := highlight.(map[string]any)- if !ok {- return nil- }- fields, ok := hlMap["fields"]- if !ok {- return nil- }- fieldsMap, ok := fields.(map[string]any)- if !ok {- return nil- }- result := make([]string, 0, len(fieldsMap))- for k := range fieldsMap {- result = append(result, k)- }- slices.Sort(result)- return result-}--// extractSortFields extracts field names from the sort array.-func extractSortFields(parsed map[string]any) []string {- sortVal, ok := parsed["sort"]- if !ok {- return nil- }- arr, ok := sortVal.([]any)- if !ok {- return nil- }- var fields []string- for _, item := range arr {- switch v := item.(type) {- case string:- if strings.HasPrefix(v, "_") {- continue- }- fields = append(fields, v)- case map[string]any:- for k := range v {- if !strings.HasPrefix(k, "_") {- fields = append(fields, k)- }- }- default:- }- }- return fields-}--// containsKey recursively searches for a key in a nested map structure.-func containsKey(data map[string]any, key string) bool {- for k, v := range data {- if k == key {- return true- }- if containsKeyInValue(v, key) {- return true- }- }- return false-}--// containsKeyInValue checks if any nested map within v contains the given key.-func containsKeyInValue(v any, key string) bool {- switch child := v.(type) {- case map[string]any:- return containsKey(child, key)- case []any:- for _, item := range child {- if m, ok := item.(map[string]any); ok && containsKey(m, key) {- return true- }- }- default:- }- return false-}--// toStringSlice converts a []any to a []string, skipping non-string elements.-func toStringSlice(arr []any) []string {- result := make([]string, 0, len(arr))- for _, item := range arr {- if s, ok := item.(string); ok {- result = append(result, s)- }- }- return result-}--// directFieldClauses are clause types where the keys inside the clause object are field names.-var directFieldClauses = map[string]bool{- "match": true, "match_phrase": true, "match_phrase_prefix": true, "match_bool_prefix": true,- "term": true, "terms": true, "terms_set": true,- "range": true,- "wildcard": true, "prefix": true, "fuzzy": true, "regexp": true,- "geo_distance": true, "geo_bounding_box": true, "geo_shape": true, "geo_polygon": true,- "span_term": true,-}--// compoundClauses maps clause types to the keys that contain sub-queries.-var compoundClauses = map[string][]string{- "bool": {"must", "must_not", "should", "filter"},- "nested": {"query"},- "has_child": {"query"},- "has_parent": {"query"},- "dis_max": {"queries"},- "constant_score": {"filter"},- "boosting": {"positive", "negative"},- "function_score": {"query"},-}--// extractPredicateFields extracts field names from ES query clauses.-func extractPredicateFields(parsed map[string]any) []string {- queryVal, ok := parsed["query"]- if !ok {- return nil- }- queryMap, ok := queryVal.(map[string]any)- if !ok {- return nil- }- var fields []string- extractFieldsFromQuery(queryMap, &fields)- return fields-}--// extractFieldsFromQuery recursively walks ES query clauses to collect field names.-func extractFieldsFromQuery(queryMap map[string]any, fields *[]string) {- for clauseType, clauseVal := range queryMap {- clauseObj, ok := clauseVal.(map[string]any)- if !ok {- extractFieldsFromQueryArray(clauseVal, fields)- continue- }- switch {- case directFieldClauses[clauseType]:- extractDirectFieldNames(clauseObj, fields)- case clauseType == "exists":- if fieldVal, ok := clauseObj["field"].(string); ok {- *fields = append(*fields, fieldVal)- }- case compoundClauses[clauseType] != nil:- extractCompoundClauseFields(clauseObj, compoundClauses[clauseType], fields)- default:- extractFieldsFromQuery(clauseObj, fields)- }- }-}--// extractFieldsFromQueryArray handles the case where a clause value is an array of query maps.-func extractFieldsFromQueryArray(val any, fields *[]string) {- arr, ok := val.([]any)- if !ok {- return- }- for _, item := range arr {- if m, ok := item.(map[string]any); ok {- extractFieldsFromQuery(m, fields)- }- }-}--// extractDirectFieldNames extracts field names from a direct field clause object.-func extractDirectFieldNames(clauseObj map[string]any, fields *[]string) {- for fieldName := range clauseObj {- if !strings.HasPrefix(fieldName, "_") && fieldName != "boost" {- *fields = append(*fields, fieldName)- }- }-}--// extractCompoundClauseFields recurses into sub-query keys of a compound clause.-func extractCompoundClauseFields(clauseObj map[string]any, subQueryKeys []string, fields *[]string) {- for _, sqKey := range subQueryKeys {- sqVal, ok := clauseObj[sqKey]- if !ok {- continue- }- switch sq := sqVal.(type) {- case map[string]any:- extractFieldsFromQuery(sq, fields)- case []any:- extractFieldsFromQueryArray(sq, fields)- default:- }- }+ // To match the omni behavior exactly, we use a search endpoint to trigger body analysis.+ omniResult := analysis.AnalyzeRequest("POST", "test/_search", body)+ return convertAnalysis(omniResult)}diff --git a/backend/plugin/parser/elasticsearch/parser.go b/backend/plugin/parser/elasticsearch/parser.goindex d01d757eef..9c1c17fed8 100644--- a/backend/plugin/parser/elasticsearch/parser.go+++ b/backend/plugin/parser/elasticsearch/parser.go@@ -1,25 +1,19 @@package elasticsearchimport (- "encoding/json"- "regexp"- "slices"- "strconv"- "strings"- "unicode/utf8"-- hjson "github.com/hjson/hjson-go/v4"- "github.com/pkg/errors"+ es "github.com/bytebase/omni/elasticsearch"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")+// ParseResult holds parsed requests and any syntax errors encountered.type ParseResult struct {Requests []*RequestErrors []*base.SyntaxError `yaml:"errors,omitempty"`}+// Request is a single parsed Kibana Dev Console REST request.type Request struct {Method stringURL string@@ -28,1235 +22,53 @@ type Request struct {EndOffset int}-type editorRequest struct {- method string- url string- data []string-}--type syntaxError struct {- // byteOffset is the byte offset of the first character of the error occurred, start by 0.- byteOffset int- message string-}--// parsedRequest is the range of a request, it is left inclusive and right exclusive.-type parsedRequest struct {- // startOffset is the byte offset of the first character of the request.- startOffset int- // endOffset is the byte offset of the end position.- endOffset int-}--type adjustedParsedRequest struct {- // startLineNumber is the line number of the first character of the request, starting from 0.- startLineNumber int- // endLineNumber is the line number of the end position, starting from 0.- endLineNumber int-}--// See https://sourcegraph.com/github.com/elastic/kibana/-/blob/src/platform/plugins/shared/console/public/application/containers/editor/utils/requests_utils.ts?L76.-// Combine getRequestStartLineNumber and getRequestEndLineNumber.-func getAdjustedParsedRequest(r parsedRequest, text string, nextRequest *parsedRequest) adjustedParsedRequest {- bs := []byte(text)- startLineNumber := 0- endLineNumber := 0- startOffset := r.startOffset- // The startOffset is out of range, returning the end of document like- // what the model.getPositionAt does.- if r.startOffset >= len(bs) {- startOffset = len(bs) - 1- }- for i := 0; i < r.startOffset; i++ {- if bs[i] == '\n' {- startLineNumber++- }- }-- if r.endOffset >= 0 {- // if the parser set an end offset for this request , then find the line number for it.- endLineNumber = startLineNumber- endOffset := r.endOffset- if endOffset >= len(bs) {- endOffset = len(bs) - 1- }- for i := startOffset; i < endOffset; i++ {- if bs[i] == '\n' {- endLineNumber++- }- }- } else {- // if no end offset, try to find the line before the next request starts.- if nextRequest != nil {- nextRequestStartLine := 0- nextRequestOffset := nextRequest.startOffset- if nextRequestOffset >= len(bs) {- nextRequestOffset = len(bs) - 1- }- for i := 0; i < nextRequestOffset; i++ {- if bs[i] == '\n' {- nextRequestStartLine++- }- }- if nextRequestStartLine > startLineNumber {- endLineNumber = nextRequestStartLine - 1- } else {- endLineNumber = startLineNumber- }- } else {- // If there is no next request, find the end of the text or the line that starts with a method.- lines := strings.Split(text, "\n")- nextLineNumber := 0- for i := 0; i < r.startOffset; i++ {- if bs[i] == '\n' {- nextLineNumber++- }- }- nextLineNumber++- for nextLineNumber < len(lines) {- content := strings.TrimSpace(lines[nextLineNumber])- startsWithMethodRegex := regexp.MustCompile(`(?i)^\s*(GET|POST|PUT|PATCH|DELETE)`)- if startsWithMethodRegex.MatchString(content) {- break- }- nextLineNumber++- }- // nextLineNumber is now either the line with a method or 1 line after the end of the text- // set the end line for this request to the line before nextLineNumber- if nextLineNumber > startLineNumber {- endLineNumber = nextLineNumber - 1- } else {- endLineNumber = startLineNumber- }- }- }- // if the end is empty, go up to find the first non-empty line.- lines := strings.Split(text, "\n")- for endLineNumber >= 0 && strings.TrimSpace(lines[endLineNumber]) == "" {- endLineNumber--- }- return adjustedParsedRequest{- startLineNumber: startLineNumber,- endLineNumber: endLineNumber,- }-}--type parser struct {- // at is the current rune's byte offset in the input string.- at int- ch rune- escapee map[rune]string- text string- errors []*syntaxError- requests []parsedRequest- // requestStart is the byte offset of the first character of the request.- requestStartOffset int- // requestEnd is the byte offset of the first character of the next request.- requestEndOffset int-}--// ParseElasticsearchREST parses the Elasticsearch REST API request.+// ParseElasticsearchREST parses the Elasticsearch REST API request text.+// It delegates to the omni elasticsearch package and converts types.func ParseElasticsearchREST(text string) (*ParseResult, error) {- p := newParser(text)- parsedResults, err := p.parse()+ result, err := es.ParseElasticsearchREST(text)if err != nil {return nil, err}+ return convertParseResult(result), nil+}++// convertParseResult converts an omni ParseResult to a bytebase ParseResult.+func convertParseResult(r *es.ParseResult) *ParseResult {+ if r == nil {+ return &ParseResult{}+ }+var requests []*Request- for i, parsedResult := range parsedResults {- // See https://sourcegraph.com/github.com/elastic/kibana/-/blob/src/platform/plugins/shared/console/public/application/containers/editor/monaco_editor_actions_provider.ts?L261.- var nextRequest *parsedRequest- if i < len(parsedResults)-1 {- nextRequest = &parsedResults[i+1]- }- adjustedOffset := getAdjustedParsedRequest(parsedResult, text, nextRequest)- if adjustedOffset.startLineNumber > adjustedOffset.endLineNumber {+ for _, req := range r.Requests {+ if req == nil {continue}- editorRequest := getEditorRequest(text, adjustedOffset)- if editorRequest == nil {- continue- }- if len(editorRequest.data) > 0 {- for i, v := range editorRequest.data {- if containsComments(v) {- // parse and stringify to remove comments.- editorRequest.data[i] = indentData(v)- }- editorRequest.data[i] = collapseLiteralString(v)- }- }requests = append(requests, &Request{- Method: editorRequest.method,- URL: editorRequest.url,- Data: editorRequest.data,- StartOffset: parsedResult.startOffset,- EndOffset: parsedResult.endOffset,+ Method: req.Method,+ URL: req.URL,+ Data: req.Data,+ StartOffset: req.StartOffset,+ EndOffset: req.EndOffset,})}- // Convert Error- var syntaxErrors []*base.SyntaxError- slices.SortFunc(p.errors, func(a, b *syntaxError) int {- if a.byteOffset < b.byteOffset {- return -1- }- if a.byteOffset > b.byteOffset {- return 1- }- return 0- })-- for i, err := range p.errors {- line := 0- column := 0- pos := 0- if i > 0 {- line = int(syntaxErrors[i-1].Position.Line)- column = int(syntaxErrors[i-1].Position.Column)- pos = p.errors[i-1].byteOffset- }- boundary := p.errors[i].byteOffset- if boundary >= len(text) {- boundary = len(text) - 1- }-- for j := pos; j <= boundary; j++ {- if text[j] == '\n' {- if j == boundary {- // Decorate the \n position instead of the next line.- column++- continue- }- line++- column = 0- } else {- column++- }+ var errors []*base.SyntaxError+ for _, e := range r.Errors {+ if e == nil {+ continue}- syntaxErrors = append(syntaxErrors, &base.SyntaxError{+ errors = append(errors, &base.SyntaxError{Position: &storepb.Position{- Line: int32(line),- Column: int32(column),+ Line: int32(e.Position.Line),+ Column: int32(e.Position.Column),},- Message: err.message,+ Message: e.Message,+ RawMessage: e.RawMessage,})}return &ParseResult{Requests: requests,- Errors: syntaxErrors,- }, nil-}--func collapseLiteralString(s string) string {- splitData := strings.Split(s, `"""`)- for idx := 1; idx < len(splitData)-1; idx += 2 {- v, err := json.Marshal(splitData[idx])- if err != nil {- continue- }- splitData[idx] = string(v)- }- return strings.Join(splitData, "")-}--func indentData(s string) string {- v := make(map[string]any)- if err := hjson.Unmarshal([]byte(s), &v); err != nil {- return s- }- m, err := json.MarshalIndent(v, "", " ")- if err != nil {- return s- }- return string(m)-}--func containsComments(s string) bool {- insideString := false- var prevR rune- rs := []rune(s)- for i, r := range rs {- nextR := rune(0)- if i+1 < len(rs) {- nextR = rs[i+1]- }-- if !insideString && r == '"' {- insideString = true- } else if insideString && r == '"' && prevR != '\\' {- insideString = false- } else if !insideString {- if r == '/' && (nextR == '/' || nextR == '*') {- return true- }- }- prevR = r- }- return false-}--// See https://sourcegraph.com/github.com/elastic/kibana/-/blob/src/platform/plugins/shared/console/public/application/containers/editor/utils/requests_utils.ts?L204.-func getEditorRequest(text string, a adjustedParsedRequest) *editorRequest {- e := &editorRequest{}- lines := strings.Split(text, "\n")- methodURLLine := strings.TrimSpace(lines[a.startLineNumber])- if methodURLLine == "" {- return nil- }- method, url := parseLine(methodURLLine)- if method == "" || url == "" {- return nil- }- e.method = method- e.url = url-- if a.endLineNumber <= a.startLineNumber {- return e- }-- dataString := ""- if a.startLineNumber < len(lines)-1 {- var validLines []string- for i := a.startLineNumber + 1; i <= a.endLineNumber; i++ {- validLines = append(validLines, lines[i])- }- dataString = strings.TrimSpace(strings.Join(validLines, "\n"))- }-- data := splitDataIntoJSONObjects(dataString)- return &editorRequest{- method: method,- url: url,- data: data,- }-}--// Splits a concatenated string of JSON objects into individual JSON objects.-// This function takes a string containing one or more JSON objects concatenated together,-// separated by optional whitespace, and splits them into an array of individual JSON strings.-// It ensures that nested objects and strings containing braces do not interfere with the splitting logic.-//-// Example inputs:-// - '{ "query": "test"} { "query": "test" }' -> ['{ "query": "test"}', '{ "query": "test" }']-// - '{ "query": "test"}' -> ['{ "query": "test"}']-// - '{ "query": "{a} {b}"}' -> ['{ "query": "{a} {b}"}'].-func splitDataIntoJSONObjects(s string) []string {- var jsonObjects []string- // Track the depth of nested braces- depth := 0- // Holds the current JSON object as we iterate- currentObject := ""- // Tracks whether the current position is inside a string- insideString := false- // Iterate through each character in the input string- rs := []rune(s)- for i, r := range rs {- // Append the character to the current JSON object string- currentObject += string(r)-- // If the character is a double quote and it is not escaped, toggle the `insideString` state- if r == '"' && (i == 0 || rs[i-1] != '\\') {- insideString = !insideString- } else if !insideString {- // Only modify depth if not inside a string- switch r {- case '{':- depth++- case '}':- depth--- default:- // Other characters don't affect depth- }-- if depth == 0 {- jsonObjects = append(jsonObjects, strings.TrimSpace(currentObject))- currentObject = ""- }- }- }-- // If there's remaining data in currentObject, add it as the last JSON object.- if strings.TrimSpace(currentObject) != "" {- jsonObjects = append(jsonObjects, strings.TrimSpace(currentObject))- }-- // Filter out any empty strings from the result- var result []string- for i := 0; i < len(jsonObjects); i++ {- if jsonObjects[i] != "" {- result = append(result, jsonObjects[i])- }- }- return result-}--func parseLine(line string) (method string, url string) {- line = strings.TrimSpace(line)- firstWhitespaceIndex := strings.Index(line, " ")- if firstWhitespaceIndex < 0 {- // There is no url, only method- return line, ""- }-- // 1st part is the method- method = strings.ToUpper(strings.TrimSpace(line[0:firstWhitespaceIndex]))- // 2nd part is the url- url = removeTrailingWhitespace(strings.TrimSpace(line[firstWhitespaceIndex:]))- return method, url-}--// This function removes any trailing comments, for example:-// "_search // comment" -> "_search"-// Ideally the parser would do that, but currently they are included in the url.-func removeTrailingWhitespace(s string) string {- index := 0- whitespaceIndex := -1- isQueryParam := false- for {- r, sz := utf8.DecodeRuneInString(s[index:])- if r == utf8.RuneError {- break- }- if r == '"' {- isQueryParam = !isQueryParam- } else if r == ' ' && !isQueryParam {- whitespaceIndex = index- break- }- index += sz- }- if whitespaceIndex > 0 {- return s[:whitespaceIndex]- }- return s-}--func newParser(text string) *parser {- return &parser{- at: 0,- ch: 0,- escapee: map[rune]string{- '"': `"`,- '\\': `\`,- '/': `/`,- 'b': "\b",- 'f': "\f",- 'n': "\n",- 'r': "\r",- 't': "\t",- },- text: text,- errors: []*syntaxError{},- }-}--// See https://sourcegraph.com/github.com/elastic/kibana/-/blob/src/platform/packages/shared/kbn-monaco/src/languages/console/parser.js.-func (p *parser) parse() ([]parsedRequest, error) {- if _, err := p.nextEmptyInput(); err != nil {- return nil, err- }- if err := p.multiRequest(); err != nil {- return nil, err- }-- if err := p.white(); err != nil {- return nil, err- }- if p.ch != 0 {- return nil, errors.Errorf("Syntax error")- }- return p.requests, nil-}--func (p *parser) multiRequest() error {- catch := func(e error) int {- p.errors = append(p.errors, &syntaxError{- byteOffset: p.at,- message: e.Error(),- })- if p.at >= len(p.text) {- return -1- }- remain := p.text[p.at:]- re := regexp.MustCompile(`(?m)^(POST|HEAD|GET|PUT|DELETE|PATCH)`)- match := re.FindStringIndex(remain)- if match != nil {- return match[0] + p.at- }- return -1- }- for p.ch != 0 {- if err := p.white(); err != nil {- if next := catch(err); next >= 0 {- if err := p.reset(next); err != nil {- return err- }- } else {- return nil- }- }- if p.ch == 0 {- continue- }- if err := p.comment(); err != nil {- if next := catch(err); next >= 0 {- if err := p.reset(next); err != nil {- return err- }- } else {- return nil- }- }- if err := p.white(); err != nil {- if next := catch(err); next >= 0 {- if err := p.reset(next); err != nil {- return err- }- } else {- return nil- }- }- if p.ch == 0 {- continue- }- if err := p.request(); err != nil {- if next := catch(err); next >= 0 {- if err := p.reset(next); err != nil {- return err- }- } else {- return nil- }- }- if err := p.white(); err != nil {- if next := catch(err); next >= 0 {- if err := p.reset(next); err != nil {- return err- }- } else {- return nil- }- }- }-- return nil-}--func (p *parser) request() error {- if err := p.white(); err != nil {- return err- }- if err := p.addRequestStart(); err != nil {- return err- }- if _, err := p.method(); err != nil {- return err- }- if err := p.updateRequestEnd(); err != nil {- return err- }- if err := p.strictWhite(); err != nil {- return err- }- if _, err := p.url(); err != nil {- return err- }- if err := p.updateRequestEnd(); err != nil {- return err- }- // advance to one new line- if err := p.strictWhite(); err != nil {- return err- }- if err := p.newLine(); err != nil {- return err- }- if err := p.strictWhite(); err != nil {- return err- }- if p.ch == '{' {- if _, err := p.object(); err != nil {- return err- }- if err := p.updateRequestEnd(); err != nil {- return err- }- }- // multi doc request- // advance to one new line- if err := p.strictWhite(); err != nil {- return err- }- if err := p.newLine(); err != nil {- return err- }- if err := p.strictWhite(); err != nil {- return err- }- for p.ch == '{' {- // another object- if _, err := p.object(); err != nil {- return err- }- if err := p.updateRequestEnd(); err != nil {- return err- }- if err := p.strictWhite(); err != nil {- return err- }- if err := p.newLine(); err != nil {- return err- }- if err := p.strictWhite(); err != nil {- return err- }- }- return p.addRequestEnd()-}--func (p *parser) url() (string, error) {- url := ""- for p.ch != 0 && p.ch != '\n' {- url += string(p.ch)- if _, err := p.nextEmptyInput(); err != nil {- return "", err- }- }- if url == "" {- return "", errors.Errorf("Missing url")- }- return url, nil-}--func (p *parser) object() (map[string]any, error) {- key := ""- object := make(map[string]any)-- if p.ch == '{' {- if err := p.next('{'); err != nil {- return nil, err- }- if err := p.white(); err != nil {- return nil, err- }- if p.ch == '}' {- if err := p.next('}'); err != nil {- return nil, err- }- // empty object- return object, nil- }- for p.ch != 0 {- var err error- key, err = p.string()- if err != nil {- return nil, err- }- if err := p.white(); err != nil {- return nil, err- }- if err := p.next(':'); err != nil {- return nil, err- }- if _, ok := object[key]; ok {- return nil, errors.Errorf("duplicate key '%s'", key)- }- v, err := p.value()- if err != nil {- return nil, err- }- object[key] = v- if err := p.white(); err != nil {- return nil, err- }- if p.ch == '}' {- if err := p.next('}'); err != nil {- return nil, err- }- return object, nil- }- if err := p.next(','); err != nil {- return nil, err- }- if err := p.white(); err != nil {- return nil, err- }- }- }- return nil, errors.Errorf("bad object")-}--func (p *parser) value() (any, error) {- if err := p.white(); err != nil {- return nil, err- }- switch p.ch {- case '{':- return p.object()- case '[':- return p.array()- case '"':- return p.string()- case '-':- return p.number()- default:- if p.ch >= '0' && p.ch <= '9' {- return p.number()- }- return p.word()- }-}--func (p *parser) word() (any, error) {- switch p.ch {- case 't':- if err := p.next('t'); err != nil {- return nil, err- }- if err := p.next('r'); err != nil {- return nil, err- }- if err := p.next('u'); err != nil {- return nil, err- }- if err := p.next('e'); err != nil {- return nil, err- }- return true, nil- case 'f':- if err := p.next('f'); err != nil {- return nil, err- }- if err := p.next('a'); err != nil {- return nil, err- }- if err := p.next('l'); err != nil {- return nil, err- }- if err := p.next('s'); err != nil {- return nil, err- }- if err := p.next('e'); err != nil {- return nil, err- }- return false, nil- case 'n':- if err := p.next('n'); err != nil {- return nil, err- }- if err := p.next('u'); err != nil {- return nil, err- }- if err := p.next('l'); err != nil {- return nil, err- }- if err := p.next('l'); err != nil {- return nil, err- }- return nil, nil- default:- return nil, errors.Errorf("unexpected '%c'", p.ch)- }-}--func (p *parser) number() (float64, error) {- s := ""- if p.ch == '-' {- s = "-"- if err := p.next('-'); err != nil {- return 0, err- }- }- for p.ch >= '0' && p.ch <= '9' {- s += string(p.ch)- if _, err := p.nextEmptyInput(); err != nil {- return 0, err- }- }- if p.ch == '.' {- s += "."- for {- if _, err := p.nextEmptyInput(); err != nil {- return 0, err- }- if p.ch >= '0' && p.ch <= '9' {- s += string(p.ch)- } else {- break- }- }- }- if p.ch == 'e' || p.ch == 'E' {- s += string(p.ch)- if _, err := p.nextEmptyInput(); err != nil {- return 0, err- }- if p.ch == '+' || p.ch == '-' {- s += string(p.ch)- if _, err := p.nextEmptyInput(); err != nil {- return 0, err- }- }- for p.ch >= '0' && p.ch <= '9' {- s += string(p.ch)- if _, err := p.nextEmptyInput(); err != nil {- return 0, err- }- }- }- num, err := strconv.ParseFloat(s, 64)- if err != nil {- return 0, errors.Errorf("bad number")- }- return num, nil-}--func (p *parser) array() ([]any, error) {- var array []any- if p.ch == '[' {- if err := p.next('['); err != nil {- return nil, err- }- if err := p.white(); err != nil {- return nil, err- }- if p.ch == ']' {- if err := p.next(']'); err != nil {- return nil, err- }- // empty array- return array, nil- }- for p.ch != 0 {- var err error- v, err := p.value()- if err != nil {- return nil, err- }- array = append(array, v)- if err := p.white(); err != nil {- return nil, err- }- if p.ch == ']' {- if err := p.next(']'); err != nil {- return nil, err- }- return array, nil- }- if err := p.next(','); err != nil {- return nil, err- }- if err := p.white(); err != nil {- return nil, err- }- }- }- return nil, errors.Errorf("bad array")-}--func (p *parser) string() (string, error) {- s := ""- var uffff int32- if p.ch == '"' {- if p.peek(0) == '"' && p.peek(1) == '"' {- // literal- if err := p.next('"'); err != nil {- return "", err- }- if err := p.next('"'); err != nil {- return "", err- }- return p.nextUpTo(`"""`, `failed to find closing '"""'`)- }- for {- r, err := p.nextEmptyInput()- if err != nil {- return "", err- }- if r == 0 {- break- }- if p.ch == '"' {- if _, err := p.nextEmptyInput(); err != nil {- return "", err- }- return s, nil- } else if p.ch == '\\' {- if _, err := p.nextEmptyInput(); err != nil {- return "", err- }- if p.ch == 'u' {- uffff = 0- for i := 0; i < 4; i++ {- nextRune, err := p.nextEmptyInput()- if err != nil {- return "", err- }- // Parse next rune into hex.- hex, err := strconv.ParseUint(string(nextRune), 16, 32)- if err != nil {- break- }- uffff = (uffff << 4) | int32(hex&0xF)- }- // Treat uffff as UTF-16 encoded rune.- s += string(rune(uffff))- } else if v, ok := p.escapee[p.ch]; ok {- s += v- } else {- break- }- } else {- s += string(p.ch)- }- }- }-- return "", errors.Errorf("bad string")-}--func (p *parser) nextUpTo(upTo string, errorMessage string) (string, error) {- currentAt := p.at- i := strings.Index(p.text[p.at:], upTo)- if i < 0 {- if errorMessage != "" {- return "", errors.New(errorMessage)- }- return "", errors.Errorf("expected '%s'", upTo)- }- i += currentAt- if err := p.reset(i + len(upTo)); err != nil {- return "", err- }- return p.text[currentAt:i], nil-}--func (p *parser) reset(newAt int) error {- ch, sz := utf8.DecodeRuneInString(p.text[newAt:])- if ch == utf8.RuneError {- if sz == 0 {- return errors.Errorf("unexpected empty input")- }- if sz == 1 {- return errors.Errorf("invalid UTF-8 character")- }- return errors.Errorf("unknown decoding rune error")- }- p.ch = ch- p.at = newAt + sz- return nil-}--func (p *parser) newLine() error {- if p.ch == '\n' {- if _, err := p.nextEmptyInput(); err != nil {- return err- }- }- return nil-}--func (p *parser) strictWhite() error {- for p.ch != 0 && (p.ch == ' ' || p.ch == '\t') {- if _, err := p.nextEmptyInput(); err != nil {- return err- }- }- return nil-}--func (p *parser) nextOneOf(rs []rune) error {- if !includes(rs, p.ch) {- return errors.Errorf("expected one of %+v instead of '%c'", rs, p.ch)- }- if p.at >= len(p.text) {- // EOF, just increase the at by 1 and set the ch to 0.- p.at++- p.ch = 0- return nil- }- ch, sz := utf8.DecodeRuneInString(p.text[p.at:])- if ch == utf8.RuneError {- if sz == 0 {- return errors.Errorf("unexpected empty input")- }- if sz == 1 {- return errors.Errorf("invalid UTF-8 character")- }- return errors.Errorf("unknown decoding rune error")- }- p.ch = ch- p.at += sz- return nil-}--func (p *parser) method() (string, error) {- uppercase := strings.ToUpper(string(p.ch))- switch uppercase {- case "G":- if err := p.nextOneOf([]rune{'G', 'g'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'E', 'e'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'T', 't'}); err != nil {- return "", err- }- return "GET", nil- case "H":- if err := p.nextOneOf([]rune{'H', 'h'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'E', 'e'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'A', 'a'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'D', 'd'}); err != nil {- return "", err- }- return "HEAD", nil- case "D":- if err := p.nextOneOf([]rune{'D', 'd'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'E', 'e'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'L', 'l'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'E', 'e'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'T', 't'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'E', 'e'}); err != nil {- return "", err- }- return "DELETE", nil- case "P":- if err := p.nextOneOf([]rune{'P', 'p'}); err != nil {- return "", err- }- nextUppercase := strings.ToUpper(string(p.ch))- switch nextUppercase {- case "A":- if err := p.nextOneOf([]rune{'A', 'a'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'T', 't'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'C', 'c'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'H', 'h'}); err != nil {- return "", err- }- return "PATCH", nil- case "U":- if err := p.nextOneOf([]rune{'U', 'u'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'T', 't'}); err != nil {- return "", err- }- return "PUT", nil- case "O":- if err := p.nextOneOf([]rune{'O', 'o'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'S', 's'}); err != nil {- return "", err- }- if err := p.nextOneOf([]rune{'T', 't'}); err != nil {- return "", err- }- return "POST", nil- default:- return "", errors.Errorf("unexpected '%c'", p.ch)- }- default:- return "", errors.Errorf("expected one of GET/POST/PUT/DELETE/HEAD/PATCH")- }-}--func (p *parser) addRequestStart() error {- previousRune, sz := utf8.DecodeLastRuneInString(p.text[:p.at])- if previousRune == utf8.RuneError {- if sz == 0 {- return errors.Errorf("unexpected empty input")- }- if sz == 1 {- return errors.Errorf("invalid UTF-8 character")- }- return errors.Errorf("unknown decoding rune error")- }- p.requestStartOffset = p.at - sz- p.requests = append(p.requests, parsedRequest{- startOffset: p.requestStartOffset,- endOffset: -1,- })- return nil-}--func (p *parser) addRequestEnd() error {- if len(p.requests) == 0 {- return errors.Errorf("unexpected empty requests")- }- p.requests[len(p.requests)-1].endOffset = p.requestEndOffset- return nil-}--func (p *parser) updateRequestEnd() error {- if p.at >= len(p.text) {- p.requestEndOffset = p.at - 1- return nil- }- previousRune, sz := utf8.DecodeLastRuneInString(p.text[:p.at])- if previousRune == utf8.RuneError {- if sz == 0 {- return errors.Errorf("unexpected empty input")- }- if sz == 1 {- return errors.Errorf("invalid UTF-8 character")- }- return errors.Errorf("unknown decoding rune error")- }- p.requestEndOffset = p.at - sz- return nil-}--func (p *parser) comment() error {- for p.ch == '#' {- for p.ch != 0 && p.ch != '\n' {- if _, err := p.nextEmptyInput(); err != nil {- return err- }- }- if err := p.white(); err != nil {- return err- }- }- return nil-}--func (p *parser) peek(offset uint) rune {- if p.at >= len(p.text) {- return 0- }- tempAt := p.at- var peekCh rune- var sz int- for i := uint(0); i <= offset; i++ {- if tempAt >= len(p.text) {- return 0- }- peekCh, sz = utf8.DecodeRuneInString(p.text[tempAt:])- if peekCh == utf8.RuneError {- return 0- }- tempAt += sz- }- return peekCh-}--func (p *parser) white() error {- for p.ch != 0 {- // Skip whitespace.- for p.ch != 0 && p.ch <= ' ' {- if _, err := p.nextEmptyInput(); err != nil {- return err- }- }-- // if the current rune in iteration is '#' or the rune and the next rune is equal to '//'- // we are on the single line comment.- if p.ch == '#' || (p.ch == '/' && p.peek(0) == '/') {- // Until we are on the new line, skip to the next char.- for p.ch != 0 && p.ch != '\n' {- if _, err := p.nextEmptyInput(); err != nil {- return err- }- }- } else if p.ch == '/' && p.peek(0) == '*' {- // If the chars starts with '/*', we are on the multiline comment.- if err := p.nNextEmptyInput(2); err != nil {- return err- }- for p.ch != 0 && (p.ch != '*' || p.peek(0) != '/') {- // Until we have closing tags '*', skip to the next char.- if _, err := p.nextEmptyInput(); err != nil {- return err- }- }- if p.ch != 0 {- if err := p.nNextEmptyInput(2); err != nil {- return err- }- }- } else {- break- }- }-- return nil-}--func (p *parser) next(c rune) error {- if c != p.ch {- return errors.Errorf("expected '%c' instead of '%c'", c, p.ch)- }-- _, err := p.nextEmptyInput()- return err-}--func (p *parser) nextEmptyInput() (rune, error) {- if p.at >= len(p.text) {- // EOF- p.at++- p.ch = 0- return 0, nil- }- nextCh, sz := utf8.DecodeRuneInString(p.text[p.at:])- if nextCh == utf8.RuneError {- if sz == 0 {- return 0, errors.Errorf("unexpected empty input")- }- if sz == 1 {- return 0, errors.Errorf("invalid UTF-8 character")- }- return 0, errors.Errorf("unknown decoding rune error")- }- p.ch = nextCh-- p.at += sz- return nextCh, nil-}--// call p.nextEmptyInput n times.-func (p *parser) nNextEmptyInput(n int) error {- var err error- for i := 0; i < n; i++ {- if _, err = p.nextEmptyInput(); err != nil {- return err- }- }- return nil-}--func includes[T rune](s []T, e T) bool {- for _, v := range s {- if v == e {- return true- }+ Errors: errors,}- return false}diff --git a/backend/plugin/parser/elasticsearch/parser_test.go b/backend/plugin/parser/elasticsearch/parser_test.goindex 74536f71be..e4aef0181c 100644--- a/backend/plugin/parser/elasticsearch/parser_test.go+++ b/backend/plugin/parser/elasticsearch/parser_test.go@@ -3,7 +3,6 @@ package elasticsearchimport ("io""os"- "strings""testing""github.com/stretchr/testify/require"@@ -49,380 +48,3 @@ func TestParseElasticsearchREST(t *testing.T) {a.NoError(err)}}--func TestParse(t *testing.T) {- // See https://sourcegraph.com/github.com/elastic/kibana/-/blob/src/platform/packages/shared/kbn-monaco/src/languages/console/parser.test.ts.- testCases := []struct {- description string- input string- got []parsedRequest- }{- {- description: "returns parsedRequests if the input is correct",- input: "GET _search",- got: []parsedRequest{- {- startOffset: 0,- endOffset: 11,- },- },- },- {- description: "parses several requests",- input: `GET _search-POST _test_index`,- got: []parsedRequest{- {- startOffset: 0,- endOffset: 11,- },- {- startOffset: 12,- endOffset: 28,- },- },- },- {- description: "parses a request with a request body",- input: `GET _search-{- "query": {- "match_all": {}- }-}`,- got: []parsedRequest{- {- startOffset: 0,- endOffset: 52,- },- },- },- {- description: "allows upper case methods",- input: "GET _search\nPOST _search\nPATCH _search\nPUT _search\nHEAD _search",- got: []parsedRequest{- {- startOffset: 0,- endOffset: 11,- },- {- startOffset: 12,- endOffset: 24,- },- {- startOffset: 25,- endOffset: 38,- },- {- startOffset: 39,- endOffset: 50,- },- {- startOffset: 51,- endOffset: 63,- },- },- },- {- description: "allows lower case methods",- input: "get _search\npost _search\npatch _search\nput _search\nhead _search",- got: []parsedRequest{- {- startOffset: 0,- endOffset: 11,- },- {- startOffset: 12,- endOffset: 24,- },- {- startOffset: 25,- endOffset: 38,- },- {- startOffset: 39,- endOffset: 50,- },- {- startOffset: 51,- endOffset: 63,- },- },- },- {- description: "allows mixed case methods",- input: "GeT _search\npOSt _search\nPaTch _search\nPut _search\nheAD _search",- got: []parsedRequest{- {- startOffset: 0,- endOffset: 11,- },- {- startOffset: 12,- endOffset: 24,- },- {- startOffset: 25,- endOffset: 38,- },- {- startOffset: 39,- endOffset: 50,- },- {- startOffset: 51,- endOffset: 63,- },- },- },- }-- a := require.New(t)- for _, tc := range testCases {- p := newParser(tc.input)- got, err := p.parse()- require.NoError(t, err)- a.Equal(tc.got, got, "description: %s", tc.description)- }-}--func TestGetEditorRequest(t *testing.T) {- testCases := []struct {- description string- content []string- adjustedParsedRequest adjustedParsedRequest- want *editorRequest- }{- {- description: "cleans up any text following the url",- content: []string{"GET _search // inline comment"},- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 0,- },- want: &editorRequest{- method: "GET",- url: "_search",- data: nil,- },- },- {- description: "doesn't incorrectly removes parts of url params that include whitespaces",- content: []string{`GET _search?query="test test"`},- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 0,- },- want: &editorRequest{- method: "GET",- url: `_search?query="test test"`,- data: nil,- },- },- {- description: "correctly includes the request body",- content: []string{- "GET _search",- "{",- " \"query\": {}",- "}",- },- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 3,- },- want: &editorRequest{- method: "GET",- url: `_search`,- data: []string{- "{\n \"query\": {}\n}",- },- },- },- {- description: "correctly handles nested braces",- content: []string{- "GET _search",- "{",- ` "query": "{a} {b}"`,- "}",- "{",- ` "query": {}`,- "}",- },- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 6,- },- want: &editorRequest{- method: "GET",- url: `_search`,- data: []string{- "{\n \"query\": \"{a} {b}\"\n}",- "{\n \"query\": {}\n}",- },- },- },- {- description: "works for several request bodies",- content: []string{- "GET _search",- "{",- ` "query": {}`,- "}",- "{",- ` "query": {}`,- "}",- },- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 6,- },- want: &editorRequest{- method: "GET",- url: `_search`,- data: []string{- "{\n \"query\": {}\n}",- "{\n \"query\": {}\n}",- },- },- },- {- description: "splits several json objects",- content: []string{- "GET _search",- `{"query":"test"}`,- `{`,- ` "query": "test"`,- `}`,- `{"query":"test"}`,- },- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 5,- },- want: &editorRequest{- method: "GET",- url: `_search`,- data: []string{- `{"query":"test"}`,- "{\n \"query\": \"test\"\n}",- `{"query":"test"}`,- },- },- },- {- description: "works for invalid json objects",- content: []string{- "GET _search",- `{"query":"test"}`,- `{`,- ` "query":`,- `{`,- },- adjustedParsedRequest: adjustedParsedRequest{- startLineNumber: 0,- endLineNumber: 4,- },- want: &editorRequest{- method: "GET",- url: `_search`,- data: []string{- `{"query":"test"}`,- "{\n \"query\":\n{",- },- },- },- }-- a := require.New(t)- for _, tc := range testCases {- got := getEditorRequest(strings.Join(tc.content, "\n"), tc.adjustedParsedRequest)- a.Equal(tc.want, got, "description: %s", tc.description)- }-}--func TestContainsComments(t *testing.T) {- testCases := []struct {- description string- input string- want bool- }{- {- description: "should return false for JSON with // and /* inside strings",- input: `{- "docs": [- {- "_source": {- "trace": {- "name": "GET /actuator/health/**"- },- "transaction": {- "outcome": "success"- }- }- },- {- "_source": {- "vulnerability": {- "reference": [- "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-15778"- ]- }- }- }- ]- }`,- want: false,- },- {- description: "should return true for text with actual line comment",- input: `{- // This is a comment- "query": { "match_all": {} }- }`,- want: true,- },- {- description: "should return true for text with actual block comment",- input: `{- /* Bulk insert */- "index": { "_index": "test" },- "field1": "value1"- }`,- want: true,- },- {- description: "should return false for text without any comments",- input: `{- "field": "value"- }`,- want: false,- },- {- description: "should return false for empty string",- input: ``,- want: false,- },- {- description: "should correctly handle escaped quotes within strings",- input: `{- "field": \"value with \\\"escaped quotes\\\"\"- }`,- want: false,- },- {- description: "should return true if comment is outside of strings",- input: `{- "field": "value" // comment here- }`,- want: true,- },- }-- a := require.New(t)- for _, tc := range testCases {- got := containsComments(tc.input)- a.Equal(tc.want, got, "description: %s", tc.description)- }-}diff --git a/backend/plugin/parser/elasticsearch/query_span.go b/backend/plugin/parser/elasticsearch/query_span.goindex 9086878212..ad05908d73 100644--- a/backend/plugin/parser/elasticsearch/query_span.go+++ b/backend/plugin/parser/elasticsearch/query_span.go@@ -2,9 +2,9 @@ package elasticsearchimport ("context"- "strings"- "github.com/pkg/errors"+ es "github.com/bytebase/omni/elasticsearch"+ "github.com/bytebase/omni/elasticsearch/analysis"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -22,57 +22,112 @@ func GetQuerySpan(_, _ string,_ bool,) (*base.QuerySpan, error) {- parseResult, err := ParseElasticsearchREST(stmt.Text)+ omniSpan, err := es.GetQuerySpan(stmt.Text)if err != nil {return nil, err}-- if parseResult == nil {+ if omniSpan == nil {return &base.QuerySpan{Type: base.QueryTypeUnknown}, nil}- if len(parseResult.Errors) > 0 {- firstErr := parseResult.Errors[0]- return nil, errors.Errorf("syntax error at line %d, column %d: %s", firstErr.Position.Line, firstErr.Position.Column, firstErr.Message)+ span := &base.QuerySpan{+ Type: convertQueryType(omniSpan.Type),}- if len(parseResult.Requests) == 0 {- return &base.QuerySpan{Type: base.QueryTypeUnknown}, nil+ if omniSpan.ElasticsearchAnalysis != nil {+ span.ElasticsearchAnalysis = convertAnalysis(omniSpan.ElasticsearchAnalysis)}- // After splitting, each statement should contain a single request.- // Use the first request for classification.- req := parseResult.Requests[0]- queryType := ClassifyRequest(req.Method, req.URL)-- span := &base.QuerySpan{Type: queryType}-- analysis := AnalyzeRequest(req.Method, req.URL, strings.Join(req.Data, "\n"))- span.ElasticsearchAnalysis = analysis-- if len(analysis.PredicateFields) > 0 {- span.PredicatePaths = make(map[string]*base.PathAST, len(analysis.PredicateFields))- for _, field := range analysis.PredicateFields {- span.PredicatePaths[field] = dotPathToPathAST(field)+ if omniSpan.PredicatePaths != nil {+ span.PredicatePaths = make(map[string]*base.PathAST, len(omniSpan.PredicatePaths))+ for key, p := range omniSpan.PredicatePaths {+ span.PredicatePaths[key] = convertPathAST(p)}}return span, nil}-// dotPathToPathAST converts a dot-delimited field path (e.g. "contact.phone")-// into a PathAST with ItemSelector nodes.-func dotPathToPathAST(dotPath string) *base.PathAST {- parts := strings.Split(dotPath, ".")- if len(parts) == 0 {+// convertPathAST converts an omni PathAST to a bytebase base.PathAST.+func convertPathAST(p *es.PathAST) *base.PathAST {+ if p == nil || p.Root == nil {return nil}- root := base.NewItemSelector(parts[0])+ root := base.NewItemSelector(p.Root.Name)current := base.SelectorNode(root)- for _, part := range parts[1:] {- next := base.NewItemSelector(part)+ omniCurrent := p.Root.Next+ for omniCurrent != nil {+ next := base.NewItemSelector(omniCurrent.Name)current.SetNext(next)current = next+ omniCurrent = omniCurrent.Next}return base.NewPathAST(root)}++// convertAnalysis converts an omni analysis.RequestAnalysis to a bytebase base.ElasticsearchAnalysis.+func convertAnalysis(a *analysis.RequestAnalysis) *base.ElasticsearchAnalysis {+ if a == nil {+ return nil+ }+ result := &base.ElasticsearchAnalysis{+ API: convertMaskableAPI(a.API),+ Index: a.Index,+ SourceFields: a.SourceFields,+ SourceDisabled: a.SourceDisabled,+ RequestedFields: a.RequestedFields,+ HighlightFields: a.HighlightFields,+ SortFields: a.SortFields,+ HasInnerHits: a.HasInnerHits,+ PredicateFields: a.PredicateFields,+ }++ if len(a.BlockedFeatures) > 0 {+ result.BlockedFeatures = make([]base.ElasticsearchBlockedFeature, len(a.BlockedFeatures))+ for i, bf := range a.BlockedFeatures {+ result.BlockedFeatures[i] = convertBlockedFeature(bf)+ }+ }++ return result+}++// convertMaskableAPI maps an omni analysis.MaskableAPI to a bytebase base.ElasticsearchMaskableAPI.+func convertMaskableAPI(api analysis.MaskableAPI) base.ElasticsearchMaskableAPI {+ switch api {+ case analysis.APIMaskSearch:+ return base.ElasticsearchAPIMaskSearch+ case analysis.APIMaskGetDoc:+ return base.ElasticsearchAPIMaskGetDoc+ case analysis.APIMaskGetSource:+ return base.ElasticsearchAPIMaskGetSource+ case analysis.APIMaskMGet:+ return base.ElasticsearchAPIMaskMGet+ case analysis.APIMaskExplain:+ return base.ElasticsearchAPIMaskExplain+ case analysis.APIBlocked:+ return base.ElasticsearchAPIBlocked+ default:+ return base.ElasticsearchAPIUnsupported+ }+}++// convertBlockedFeature maps an omni analysis.BlockedFeature to a bytebase base.ElasticsearchBlockedFeature.+func convertBlockedFeature(bf analysis.BlockedFeature) base.ElasticsearchBlockedFeature {+ switch bf {+ case analysis.BlockedFeatureAggs:+ return base.ElasticsearchBlockedFeatureAggs+ case analysis.BlockedFeatureSuggest:+ return base.ElasticsearchBlockedFeatureSuggest+ case analysis.BlockedFeatureScriptFields:+ return base.ElasticsearchBlockedFeatureScriptFields+ case analysis.BlockedFeatureRuntimeMappings:+ return base.ElasticsearchBlockedFeatureRuntimeMappings+ case analysis.BlockedFeatureStoredFields:+ return base.ElasticsearchBlockedFeatureStoredFields+ case analysis.BlockedFeatureDocvalueFields:+ return base.ElasticsearchBlockedFeatureDocvalueFields+ default:+ return base.ElasticsearchBlockedFeatureAggs // fallback+ }+}diff --git a/backend/plugin/parser/elasticsearch/query_type.go b/backend/plugin/parser/elasticsearch/query_type.goindex 289852fbb5..e51e64b8ce 100644--- a/backend/plugin/parser/elasticsearch/query_type.go+++ b/backend/plugin/parser/elasticsearch/query_type.go@@ -1,156 +1,30 @@package elasticsearchimport (- "strings"+ es "github.com/bytebase/omni/elasticsearch""github.com/bytebase/bytebase/backend/plugin/parser/base")-// readOnlyPostEndpoints lists URL patterns that are read-only despite using POST.-// These endpoints accept a request body for query parameters but do not modify data.-var readOnlyPostEndpoints = []string{- "_search",- "_msearch",- "_count",- "_field_caps",- "_validate/query",- "_search_shards",- "_terms_enum",- "_termvectors",- "_mtermvectors",- "_search_template",- "_msearch_template",- "_render/template",- "_rank_eval",- "_knn_search",- "_search_mvt",- "_sql",- "_esql/query",- "_eql/search",- "_fleet/_search",- "_fleet/_msearch",- "_graph/explore",- "rollup_search",- "async_search",-}--// infoSchemaEndpoints lists URL patterns for cluster/node metadata queries.-var infoSchemaEndpoints = []string{- "_cat/",- "_cluster/",- "_nodes/",-}--// dmlPostEndpoints lists URL patterns for document write operations via POST.-var dmlPostEndpoints = []string{- "_doc",- "_create/",- "_update/",- "_bulk",- "_delete_by_query",- "_update_by_query",- "_reindex",-}-// ClassifyRequest determines the QueryType for an ElasticSearch REST API request.func ClassifyRequest(method, url string) base.QueryType {- method = strings.ToUpper(method)- urlLower := strings.ToLower(url)-- switch method {- case "HEAD":- return base.Select+ return convertQueryType(es.ClassifyRequest(method, url))+}- case "GET":- if isInfoSchemaURL(urlLower) {- return base.SelectInfoSchema- }+// convertQueryType maps an omni QueryType to a bytebase base.QueryType.+func convertQueryType(qt es.QueryType) base.QueryType {+ switch qt {+ case es.QueryTypeSelect:return base.Select-- case "DELETE":- if isDocumentURL(urlLower) {- return base.DML- }- return base.DDL-- case "PUT":- if isDocumentWriteURL(urlLower) {- return base.DML- }+ case es.QueryTypeExplain:+ return base.Explain+ case es.QueryTypeSelectInfoSchema:+ return base.SelectInfoSchema+ case es.QueryTypeDDL:return base.DDL-- case "PATCH":+ case es.QueryTypeDML:return base.DML-- case "POST":- return classifyPostRequest(urlLower)-default:return base.QueryTypeUnknown}}--func classifyPostRequest(url string) base.QueryType {- // Explain- if strings.Contains(url, "_explain/") || strings.HasSuffix(url, "_explain") {- return base.Explain- }-- // Info schema (metadata reads)- if isInfoSchemaURL(url) {- return base.SelectInfoSchema- }-- // Read-only POST endpoints- if isReadOnlyPostURL(url) {- return base.Select- }-- // Document writes (DML)- if isDMLPostURL(url) {- return base.DML- }-- // Default: DDL for other POST operations (index management, admin, etc.)- return base.DDL-}--func isInfoSchemaURL(url string) bool {- for _, pattern := range infoSchemaEndpoints {- if strings.Contains(url, pattern) {- return true- }- }- return false-}--func isReadOnlyPostURL(url string) bool {- for _, pattern := range readOnlyPostEndpoints {- if strings.Contains(url, pattern) {- return true- }- }- return false-}--func isDMLPostURL(url string) bool {- for _, pattern := range dmlPostEndpoints {- if strings.Contains(url, pattern) {- return true- }- }- return false-}--func isDocumentURL(url string) bool {- // DELETE /{index}/_doc/{id} is DML (document deletion)- return strings.Contains(url, "_doc/")-}--func isDocumentWriteURL(url string) bool {- // PUT /{index}/_doc/{id}, PUT /{index}/_create/{id}, PUT /_bulk are DML- return strings.Contains(url, "_doc/") ||- strings.Contains(url, "_doc") ||- strings.Contains(url, "_create/") ||- strings.Contains(url, "_bulk")-}diff --git a/backend/plugin/parser/elasticsearch/splitter.go b/backend/plugin/parser/elasticsearch/splitter.goindex df52a469d2..16f0ca6ee0 100644--- a/backend/plugin/parser/elasticsearch/splitter.go+++ b/backend/plugin/parser/elasticsearch/splitter.go@@ -3,6 +3,8 @@ package elasticsearchimport ("strings"+ es "github.com/bytebase/omni/elasticsearch"+storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -13,12 +15,11 @@ func init() {// SplitMultiSQL splits the input into individual ElasticSearch REST API requests.func SplitMultiSQL(statement string) ([]base.Statement, error) {- parseResult, err := ParseElasticsearchREST(statement)+ omniStmts, err := es.SplitMultiSQL(statement)if err != nil {return nil, err}-- if parseResult == nil || len(parseResult.Requests) == 0 {+ if len(omniStmts) == 0 {if len(strings.TrimSpace(statement)) == 0 {return nil, nil}@@ -26,55 +27,23 @@ func SplitMultiSQL(statement string) ([]base.Statement, error) {}var statements []base.Statement- for _, req := range parseResult.Requests {- if req == nil {- continue- }- text := statement[req.StartOffset:req.EndOffset]- empty := len(strings.TrimSpace(text)) == 0-- // Calculate 1-based line and column positions- startLine, startColumn := byteOffsetToPosition(statement, req.StartOffset)- endLine, endColumn := byteOffsetToPosition(statement, req.EndOffset)-+ for _, s := range omniStmts {statements = append(statements, base.Statement{- Text: text,- Empty: empty,+ Text: s.Text,+ Empty: s.Empty,Start: &storepb.Position{- Line: int32(startLine),- Column: int32(startColumn),+ Line: int32(s.Start.Line),+ Column: int32(s.Start.Column),},End: &storepb.Position{- Line: int32(endLine),- Column: int32(endColumn),+ Line: int32(s.End.Line),+ Column: int32(s.End.Column),},Range: &storepb.Range{- Start: int32(req.StartOffset),- End: int32(req.EndOffset),+ Start: int32(s.Range.Start),+ End: int32(s.Range.End),},})}return statements, nil}--// byteOffsetToPosition converts a byte offset to 1-based line and column numbers.-// Column is measured in Unicode code points (runes), not bytes.-func byteOffsetToPosition(text string, byteOffset int) (line, column int) {- line = 1- column = 1- currentByte := 0-- for _, r := range text {- if currentByte >= byteOffset {- break- }- if r == '\n' {- line++- column = 1- } else {- column++- }- currentByte += len(string(r))- }- return line, column-}diff --git a/backend/plugin/parser/elasticsearch/statement_ranges.go b/backend/plugin/parser/elasticsearch/statement_ranges.goindex 48d7394708..4977860d60 100644--- a/backend/plugin/parser/elasticsearch/statement_ranges.go+++ b/backend/plugin/parser/elasticsearch/statement_ranges.go@@ -2,10 +2,11 @@ package elasticsearchimport ("context"- "unicode/utf8"lsp "github.com/bytebase/lsp-protocol"+ es "github.com/bytebase/omni/elasticsearch"+storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -15,57 +16,26 @@ func init() {}func GetStatementRanges(_ context.Context, _ base.StatementRangeContext, statement string) ([]base.Range, error) {- parseResult, _ := ParseElasticsearchREST(statement)- if parseResult == nil {+ omniRanges, err := es.GetStatementRanges(statement)+ if err != nil {+ return nil, err+ }+ if omniRanges == nil {return []base.Range{}, nil}- bs := []byte(statement)- var ranges []base.Range- for _, r := range parseResult.Requests {- if r == nil {- continue- }- if r.EndOffset <= r.StartOffset {- continue- }- // Get the start and end position of the request.- startPosition := getPositionByByteOffset(r.StartOffset, bs)- endPosition := getPositionByByteOffset(r.EndOffset, bs)- if startPosition == nil || endPosition == nil {- continue- }+ ranges := make([]base.Range, 0, len(omniRanges))+ for _, r := range omniRanges {ranges = append(ranges, base.Range{- Start: *startPosition,- End: *endPosition,+ Start: lsp.Position{+ Line: r.Start.Line,+ Character: r.Start.Character,+ },+ End: lsp.Position{+ Line: r.End.Line,+ Character: r.End.Character,+ },})}return ranges, nil}--func getPositionByByteOffset(byteOffset int, bs []byte) *lsp.Position {- var position lsp.Position- for i := 0; ; {- if i >= byteOffset || i > len(bs) {- break- }- if bs[i] == '\n' {- position.Line++- position.Character = 0- i++- continue- }- r, size := utf8.DecodeRune(bs[i:])- if r == utf8.RuneError {- return nil- }- position.Character++- if r > 0xFFFF {- // Out of BMP, need surrogate pair.- position.Character++- }- i += size- }-- return &position-}diff --git a/go.mod b/go.modindex b1f5a90c63..2a2683cbe5 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260410074538-2f7eebc4b32e+ github.com/bytebase/omni v0.0.0-20260410092340-771f9d9350aagithub.com/caarlos0/env/v11 v11.4.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2@@ -323,7 +323,7 @@ require (github.com/googleapis/gax-go/v2 v2.17.0 // indirectgithub.com/gosimple/unidecode v1.0.1 // indirectgithub.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect- github.com/hjson/hjson-go/v4 v4.6.0+ github.com/hjson/hjson-go/v4 v4.6.0 // indirectgithub.com/inconshreveable/mousetrap v1.1.0 // indirectgithub.com/jackc/pgio v1.0.0 // indirectgithub.com/jackc/pgpassfile v1.0.0 // indirectdiff --git a/go.sum b/go.sumindex f1875c6e29..74e4a4bf31 100644--- a/go.sum+++ b/go.sum@@ -253,8 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888 h1:E8tz8maxpih/vtgithub.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888/go.mod h1:IgLUOjyiHehdE0G3C92IjGYu9lJQgkPwf2mdNSvPWq8=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260410074538-2f7eebc4b32e h1:FyHUYAu9RU6kdOgqZdslMjCb+FAImKCrJDibrZ6ybhs=-github.com/bytebase/omni v0.0.0-20260410074538-2f7eebc4b32e/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=+github.com/bytebase/omni v0.0.0-20260410092340-771f9d9350aa h1:P8rnhEtTELj6k9dO3TidjAYhFfnfSXL5Jcv1qDY3SoE=+github.com/bytebase/omni v0.0.0-20260410092340-771f9d9350aa/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50 h1:Kq5wlhZ586Rch39y/IQV9MnrEzHob4iiXBG6zrbvSMU=github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=