Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(es): prevent Lucene injection in session_id and enforce dataset scope in DeleteChunks (#21981)

ZZZZYf1d387553b0e
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index d6257f0b17..4f7e383374 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -1019,6 +1019,9 @@ func formatOrderedTagFeas(v any) (json.RawMessage, bool) {
// DeleteChunks deletes chunks from a dataset index by condition
func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
+ if datasetID == "" {
+ return 0, fmt.Errorf("dataset_id is required")
+ }
// For ES, index name is just indexName (e.g., "ragflow_{tenantID}"), not indexName_datasetID
fullIndexName := indexName
common.Info("Deleting chunks from Elasticsearch index", zap.String("index_name", fullIndexName), zap.Any("condition", condition))
@@ -1033,6 +1036,17 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
return 0, nil
}
+ // Scope the delete to the dataset the caller asked for. The tenant index is
+ // shared by every dataset, so without this a condition that omits kb_id
+ // deletes matching chunks in other datasets too.
+ condition = copyFields(condition)
+ if idVal, ok := condition["id"]; ok {
+ items, isList := conditionTermsList(idVal)
+ if idVal == nil || (isList && len(items) == 0) {
+ return 0, fmt.Errorf("ES delete aborted: empty id list on index %s", fullIndexName)
+ }
+ }
+ condition["kb_id"] = datasetID
condition = types.CompilationFilter(condition)
// Build bool query from condition
var mustClauses []map[string]interface{}
@@ -2350,22 +2364,13 @@ func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []strin
if v == nil || v == "" {
continue
}
- if isMemoryIndex && k == "session_id_contains" {
+ if isMemoryIndex && (k == "session_id" || k == "session_id_contains") {
if text, ok := v.(string); ok {
pattern := strings.NewReplacer(`\`, `\\`, "*", `\*`, "?", `\?`).Replace(text)
filterClauses = append(filterClauses, map[string]interface{}{"wildcard": map[string]interface{}{"session_id": "*" + pattern + "*"}})
+ continue
}
- continue
- }
- if isMemoryIndex && k == "session_id" {
- if strVal, ok := v.(string); ok && strVal != "" {
- filterClauses = append(filterClauses, map[string]interface{}{
- "query_string": map[string]interface{}{
- "query": fmt.Sprintf("*%s*", strVal),
- "fields": []string{"session_id"},
- "analyze_wildcard": true,
- },
- })
+ if k == "session_id_contains" {
continue
}
}
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index d097303cb8..448494503c 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -6,6 +6,7 @@ import (
"net/http/httptest"
"reflect"
"strings"
+ "sync/atomic"
"testing"
"github.com/elastic/go-elasticsearch/v8"
@@ -322,6 +323,68 @@ func TestDeleteChunksIDStringSlice(t *testing.T) {
assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})
}
+// TestDeleteChunksScopesByDataset pins the engine-side dataset scope. The ES
+// index is shared by every dataset of a tenant, so DeleteChunks must turn the
+// datasetID argument into a kb_id filter even when the caller's condition does
+// not carry one; otherwise a doc_id-only delete reaches other datasets.
+func TestDeleteChunksScopesByDataset(t *testing.T) {
+ cases := []struct {
+ name string
+ condition map[string]interface{}
+ }{
+ {name: "condition without kb_id", condition: map[string]interface{}{"doc_id": "doc-1"}},
+ {name: "datasetID wins over conflicting kb_id", condition: map[string]interface{}{"doc_id": "doc-1", "kb_id": "kb-2"}},
+ {name: "typed kb_id slice cannot widen the scope", condition: map[string]interface{}{"id": []string{"chunk-1"}, "kb_id": []string{"kb-2"}}},
+ {name: "empty condition stays dataset-scoped", condition: nil},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var deleteQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ switch r.Method {
+ case http.MethodHead:
+ w.WriteHeader(http.StatusOK)
+ case http.MethodPost:
+ if r.URL.Path != "/ragflow_tenant/_delete_by_query" {
+ t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)
+ http.Error(w, "unexpected request path", http.StatusNotFound)
+ return
+ }
+ if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {
+ t.Errorf("decode delete query: %v", err)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"deleted":1}`))
+ default:
+ t.Errorf("method=%s", r.Method)
+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
+ }
+ }))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ if _, err = engine.DeleteChunks(t.Context(), tc.condition, "ragflow_tenant", "kb-1"); err != nil {
+ t.Fatalf("DeleteChunks: %v", err)
+ }
+
+ query, _ := json.Marshal(deleteQuery)
+ if string(query) == `{"query":{"match_all":{}}}` {
+ t.Fatalf("empty condition must stay dataset-scoped, got %s", query)
+ }
+ if !strings.Contains(string(query), `"kb_id":"kb-1"`) {
+ t.Fatalf("delete query lost dataset scope kb-1: %s", query)
+ }
+ if strings.Contains(string(query), `"kb_id":"kb-2"`) {
+ t.Fatalf("delete query kept conflicting kb_id kb-2: %s", query)
+ }
+ })
+ }
+}
+
// TestUpdateChunksPreservesStringSliceCondition guards the document
// availability switch: the doc-service caller of UpdateChunks
// (updateDocumentChunkAvailability) passes a typed []string id list, and a
@@ -1132,3 +1195,81 @@ func TestCloneFilterSnapshotNonBoolQuery(t *testing.T) {
t.Fatalf("snapshot = %#v, want the query_string clause preserved", snapshot)
}
}
+
+func TestDeleteChunksRejectsEmptyDatasetID(t *testing.T) {
+ for _, condition := range []map[string]interface{}{
+ {"kb_id": "kb-2", "doc_id": "doc-1"},
+ {"doc_id": "doc-1"},
+ nil,
+ } {
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ if r.Method != http.MethodHead {
+ _, _ = w.Write([]byte(`{"deleted":1}`))
+ }
+ }))
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ server.Close()
+ t.Fatal(err)
+ }
+ engine := &Engine{client: client}
+ deleted, err := engine.DeleteChunks(t.Context(), condition, "ragflow_tenant", "")
+ server.Close()
+ if err == nil {
+ t.Errorf("condition=%v: expected empty datasetID error, got deleted=%d", condition, deleted)
+ }
+ if deleted != 0 {
+ t.Errorf("condition=%v: deleted=%d, want 0", condition, deleted)
+ }
+ if requests.Load() != 0 {
+ t.Errorf("condition=%v: sent %d ES requests, want 0", condition, requests.Load())
+ }
+ }
+}
+
+func TestDeleteChunksRejectsEmptyIDs(t *testing.T) {
+ cases := []struct {
+ name string
+ id interface{}
+ }{
+ {name: "nil", id: nil},
+ {name: "empty string slice", id: []string{}},
+ {name: "nil string slice", id: []string(nil)},
+ {name: "empty interface slice", id: []interface{}{}},
+ {name: "nil interface slice", id: []interface{}(nil)},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var deletes atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ if r.Method == http.MethodPost {
+ deletes.Add(1)
+ _, _ = w.Write([]byte(`{"deleted":1}`))
+ }
+ }))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ engine := &Engine{client: client}
+ condition := map[string]interface{}{"id": tc.id}
+ deleted, err := engine.DeleteChunks(t.Context(), condition, "ragflow_tenant", "kb-1")
+ if err == nil {
+ t.Error("expected an error for empty IDs")
+ }
+ if deleted != 0 {
+ t.Errorf("deleted=%d, want 0", deleted)
+ }
+ if deletes.Load() != 0 {
+ t.Errorf("sent %d delete requests, want 0", deletes.Load())
+ }
+ })
+ }
+}
diff --git a/internal/engine/elasticsearch/message_contract_test.go b/internal/engine/elasticsearch/message_contract_test.go
index a45c0c9ca5..5c5353640e 100644
--- a/internal/engine/elasticsearch/message_contract_test.go
+++ b/internal/engine/elasticsearch/message_contract_test.go
@@ -28,3 +28,53 @@ func TestMessageSubstringEscapesWildcardCharacters(t *testing.T) {
t.Fatalf("query = %#v, want %#v", got, want)
}
}
+
+func TestMemorySessionFilterMatchesLiteralSubstring(t *testing.T) {
+ cases := []struct {
+ input string
+ want string
+ }{
+ {input: "alpha", want: "*alpha*"},
+ {input: "lph", want: "*lph*"},
+ {input: "alpha OR beta", want: "*alpha OR beta*"},
+ {input: "alpha* OR session_id:*", want: `*alpha\* OR session_id:\**`},
+ {input: `a*b?c\d`, want: `*a\*b\?c\\d*`},
+ {input: "x:(y)", want: "*x:(y)*"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.input, func(t *testing.T) {
+ got := buildBoolQueryFromCondition(map[string]interface{}{"session_id": tc.input}, nil, false, true)
+ want := map[string]interface{}{"bool": map[string]interface{}{"filter": []interface{}{
+ map[string]interface{}{"wildcard": map[string]interface{}{"session_id": tc.want}},
+ }}}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("query = %#v, want %#v", got, want)
+ }
+ })
+ }
+}
+
+func TestSessionFilterPreservesOtherQueryForms(t *testing.T) {
+ cases := []struct {
+ name string
+ value interface{}
+ memory bool
+ clause map[string]interface{}
+ }{
+ {name: "non-memory string", value: "alpha", clause: map[string]interface{}{"term": map[string]interface{}{"session_id": "alpha"}}},
+ {name: "memory list", value: []string{"alpha", "beta"}, memory: true, clause: map[string]interface{}{"terms": map[string]interface{}{"session_id": []string{"alpha", "beta"}}}},
+ {name: "empty memory string", value: "", memory: true},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := buildBoolQueryFromCondition(map[string]interface{}{"session_id": tc.value}, nil, false, tc.memory)
+ var want map[string]interface{}
+ if tc.clause != nil {
+ want = map[string]interface{}{"bool": map[string]interface{}{"filter": []interface{}{tc.clause}}}
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("query = %#v, want %#v", got, want)
+ }
+ })
+ }
+}