RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(es): prevent Lucene injection in session_id and enforce dataset scope in DeleteChunks (#21981)
ZZZZYf1d387553b0e
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex d6257f0b17..4f7e383374 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -1019,6 +1019,9 @@ func formatOrderedTagFeas(v any) (json.RawMessage, bool) {// DeleteChunks deletes chunks from a dataset index by conditionfunc (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {+ if datasetID == "" {+ return 0, fmt.Errorf("dataset_id is required")+ }// For ES, index name is just indexName (e.g., "ragflow_{tenantID}"), not indexName_datasetIDfullIndexName := indexNamecommon.Info("Deleting chunks from Elasticsearch index", zap.String("index_name", fullIndexName), zap.Any("condition", condition))@@ -1033,6 +1036,17 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfacreturn 0, nil}+ // Scope the delete to the dataset the caller asked for. The tenant index is+ // shared by every dataset, so without this a condition that omits kb_id+ // deletes matching chunks in other datasets too.+ condition = copyFields(condition)+ if idVal, ok := condition["id"]; ok {+ items, isList := conditionTermsList(idVal)+ if idVal == nil || (isList && len(items) == 0) {+ return 0, fmt.Errorf("ES delete aborted: empty id list on index %s", fullIndexName)+ }+ }+ condition["kb_id"] = datasetIDcondition = types.CompilationFilter(condition)// Build bool query from conditionvar mustClauses []map[string]interface{}@@ -2350,22 +2364,13 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strinif v == nil || v == "" {continue}- if isMemoryIndex && k == "session_id_contains" {+ if isMemoryIndex && (k == "session_id" || k == "session_id_contains") {if text, ok := v.(string); ok {pattern := strings.NewReplacer(`\`, `\\`, "*", `\*`, "?", `\?`).Replace(text)filterClauses = append(filterClauses, map[string]interface{}{"wildcard": map[string]interface{}{"session_id": "*" + pattern + "*"}})+ continue}- continue- }- if isMemoryIndex && k == "session_id" {- if strVal, ok := v.(string); ok && strVal != "" {- filterClauses = append(filterClauses, map[string]interface{}{- "query_string": map[string]interface{}{- "query": fmt.Sprintf("*%s*", strVal),- "fields": []string{"session_id"},- "analyze_wildcard": true,- },- })+ if k == "session_id_contains" {continue}}diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex d097303cb8..448494503c 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -6,6 +6,7 @@ import ("net/http/httptest""reflect""strings"+ "sync/atomic""testing""github.com/elastic/go-elasticsearch/v8"@@ -322,6 +323,68 @@ func TestDeleteChunksIDStringSlice(t *testing.T) {assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})}+// TestDeleteChunksScopesByDataset pins the engine-side dataset scope. The ES+// index is shared by every dataset of a tenant, so DeleteChunks must turn the+// datasetID argument into a kb_id filter even when the caller's condition does+// not carry one; otherwise a doc_id-only delete reaches other datasets.+func TestDeleteChunksScopesByDataset(t *testing.T) {+ cases := []struct {+ name string+ condition map[string]interface{}+ }{+ {name: "condition without kb_id", condition: map[string]interface{}{"doc_id": "doc-1"}},+ {name: "datasetID wins over conflicting kb_id", condition: map[string]interface{}{"doc_id": "doc-1", "kb_id": "kb-2"}},+ {name: "typed kb_id slice cannot widen the scope", condition: map[string]interface{}{"id": []string{"chunk-1"}, "kb_id": []string{"kb-2"}}},+ {name: "empty condition stays dataset-scoped", condition: nil},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var deleteQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ switch r.Method {+ case http.MethodHead:+ w.WriteHeader(http.StatusOK)+ case http.MethodPost:+ if r.URL.Path != "/ragflow_tenant/_delete_by_query" {+ t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)+ http.Error(w, "unexpected request path", http.StatusNotFound)+ return+ }+ if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {+ t.Errorf("decode delete query: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"deleted":1}`))+ default:+ t.Errorf("method=%s", r.Method)+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)+ }+ }))+ defer server.Close()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ if _, err = engine.DeleteChunks(t.Context(), tc.condition, "ragflow_tenant", "kb-1"); err != nil {+ t.Fatalf("DeleteChunks: %v", err)+ }++ query, _ := json.Marshal(deleteQuery)+ if string(query) == `{"query":{"match_all":{}}}` {+ t.Fatalf("empty condition must stay dataset-scoped, got %s", query)+ }+ if !strings.Contains(string(query), `"kb_id":"kb-1"`) {+ t.Fatalf("delete query lost dataset scope kb-1: %s", query)+ }+ if strings.Contains(string(query), `"kb_id":"kb-2"`) {+ t.Fatalf("delete query kept conflicting kb_id kb-2: %s", query)+ }+ })+ }+}+// TestUpdateChunksPreservesStringSliceCondition guards the document// availability switch: the doc-service caller of UpdateChunks// (updateDocumentChunkAvailability) passes a typed []string id list, and a@@ -1132,3 +1195,81 @@ func TestCloneFilterSnapshotNonBoolQuery(t *testing.T) {t.Fatalf("snapshot = %#v, want the query_string clause preserved", snapshot)}}++func TestDeleteChunksRejectsEmptyDatasetID(t *testing.T) {+ for _, condition := range []map[string]interface{}{+ {"kb_id": "kb-2", "doc_id": "doc-1"},+ {"doc_id": "doc-1"},+ nil,+ } {+ var requests atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ requests.Add(1)+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ if r.Method != http.MethodHead {+ _, _ = w.Write([]byte(`{"deleted":1}`))+ }+ }))+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ server.Close()+ t.Fatal(err)+ }+ engine := &Engine{client: client}+ deleted, err := engine.DeleteChunks(t.Context(), condition, "ragflow_tenant", "")+ server.Close()+ if err == nil {+ t.Errorf("condition=%v: expected empty datasetID error, got deleted=%d", condition, deleted)+ }+ if deleted != 0 {+ t.Errorf("condition=%v: deleted=%d, want 0", condition, deleted)+ }+ if requests.Load() != 0 {+ t.Errorf("condition=%v: sent %d ES requests, want 0", condition, requests.Load())+ }+ }+}++func TestDeleteChunksRejectsEmptyIDs(t *testing.T) {+ cases := []struct {+ name string+ id interface{}+ }{+ {name: "nil", id: nil},+ {name: "empty string slice", id: []string{}},+ {name: "nil string slice", id: []string(nil)},+ {name: "empty interface slice", id: []interface{}{}},+ {name: "nil interface slice", id: []interface{}(nil)},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var deletes atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ if r.Method == http.MethodPost {+ deletes.Add(1)+ _, _ = w.Write([]byte(`{"deleted":1}`))+ }+ }))+ defer server.Close()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatal(err)+ }+ engine := &Engine{client: client}+ condition := map[string]interface{}{"id": tc.id}+ deleted, err := engine.DeleteChunks(t.Context(), condition, "ragflow_tenant", "kb-1")+ if err == nil {+ t.Error("expected an error for empty IDs")+ }+ if deleted != 0 {+ t.Errorf("deleted=%d, want 0", deleted)+ }+ if deletes.Load() != 0 {+ t.Errorf("sent %d delete requests, want 0", deletes.Load())+ }+ })+ }+}diff --git a/internal/engine/elasticsearch/message_contract_test.go b/internal/engine/elasticsearch/message_contract_test.goindex a45c0c9ca5..5c5353640e 100644--- a/internal/engine/elasticsearch/message_contract_test.go+++ b/internal/engine/elasticsearch/message_contract_test.go@@ -28,3 +28,53 @@ func TestMessageSubstringEscapesWildcardCharacters(t *testing.T) {t.Fatalf("query = %#v, want %#v", got, want)}}++func TestMemorySessionFilterMatchesLiteralSubstring(t *testing.T) {+ cases := []struct {+ input string+ want string+ }{+ {input: "alpha", want: "*alpha*"},+ {input: "lph", want: "*lph*"},+ {input: "alpha OR beta", want: "*alpha OR beta*"},+ {input: "alpha* OR session_id:*", want: `*alpha\* OR session_id:\**`},+ {input: `a*b?c\d`, want: `*a\*b\?c\\d*`},+ {input: "x:(y)", want: "*x:(y)*"},+ }+ for _, tc := range cases {+ t.Run(tc.input, func(t *testing.T) {+ got := buildBoolQueryFromCondition(map[string]interface{}{"session_id": tc.input}, nil, false, true)+ want := map[string]interface{}{"bool": map[string]interface{}{"filter": []interface{}{+ map[string]interface{}{"wildcard": map[string]interface{}{"session_id": tc.want}},+ }}}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("query = %#v, want %#v", got, want)+ }+ })+ }+}++func TestSessionFilterPreservesOtherQueryForms(t *testing.T) {+ cases := []struct {+ name string+ value interface{}+ memory bool+ clause map[string]interface{}+ }{+ {name: "non-memory string", value: "alpha", clause: map[string]interface{}{"term": map[string]interface{}{"session_id": "alpha"}}},+ {name: "memory list", value: []string{"alpha", "beta"}, memory: true, clause: map[string]interface{}{"terms": map[string]interface{}{"session_id": []string{"alpha", "beta"}}}},+ {name: "empty memory string", value: "", memory: true},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := buildBoolQueryFromCondition(map[string]interface{}{"session_id": tc.value}, nil, false, tc.memory)+ var want map[string]interface{}+ if tc.clause != nil {+ want = map[string]interface{}{"bool": map[string]interface{}{"filter": []interface{}{tc.clause}}}+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("query = %#v, want %#v", got, want)+ }+ })+ }+}