Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(engine): filter chunks by requested IDs in update-by-query (#19665)

qinling0210b9f4d26aadf3
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index ecdad76dd8..bb50aeee86 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -638,12 +638,59 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin
return nil
}
+// conditionTermsList normalizes a condition or new-value field into the value
+// list a `terms` query / stored array field needs. Callers hand over either
+// []interface{} (decoded JSON payloads) or a typed slice such as []string
+// (internal callers, e.g. the document availability switch). A typed slice the
+// builder does not recognize silently drops the field, which turns a bounded
+// update/delete by query into an unfiltered one over the whole index.
+func conditionTermsList(value interface{}) ([]interface{}, bool) {
+ items, ok := value.([]interface{})
+ if ok {
+ return items, true
+ }
+ rv := reflect.ValueOf(value)
+ if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array {
+ return nil, false
+ }
+ if rv.Type().Elem().Kind() == reflect.Uint8 {
+ // []byte is a scalar blob, not a list of ids/keywords.
+ return nil, false
+ }
+ out := make([]interface{}, 0, rv.Len())
+ for i := 0; i < rv.Len(); i++ {
+ out = append(out, rv.Index(i).Interface())
+ }
+ return out, true
+}
+
+// mustNotExistsClauses renders the `must_not` exclusions a condition carries.
+// The only form RAGFlow produces is must_not={"exists": <field>} (e.g.
+// hybrid_search excluding compiled products); any other shape yields nothing.
+func mustNotExistsClauses(value interface{}) []map[string]interface{} {
+ exclusions, ok := value.(map[string]interface{})
+ if !ok {
+ return nil
+ }
+ clauses := make([]map[string]interface{}, 0, len(exclusions))
+ for field, value := range exclusions {
+ if field != "exists" {
+ continue
+ }
+ clauses = append(clauses, map[string]interface{}{
+ "exists": map[string]interface{}{"field": value},
+ })
+ }
+ return clauses
+}
+
// updateChunksByQuery handles multi-document update
func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error {
common.Debug("ElasticsearchConnection.updateChunksByQuery called", zap.String("indexName", indexName))
// Build bool query from condition
var mustClauses []map[string]interface{}
+ var mustNotClauses []map[string]interface{}
for k, v := range condition {
if k == "exists" {
mustClauses = append(mustClauses, map[string]interface{}{
@@ -651,29 +698,46 @@ func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, cond
})
continue
}
+ if k == "must_not" {
+ mustNotClauses = append(mustNotClauses, mustNotExistsClauses(v)...)
+ continue
+ }
if v == nil || v == "" {
continue
}
- if listVal, ok := v.([]interface{}); ok {
+ if items, ok := conditionTermsList(v); ok {
mustClauses = append(mustClauses, map[string]interface{}{
- "terms": map[string]interface{}{k: listVal},
+ "terms": map[string]interface{}{k: items},
})
- } else if _, ok := v.(string); ok {
+ continue
+ }
+ if _, ok := v.(string); ok {
mustClauses = append(mustClauses, map[string]interface{}{
"term": map[string]interface{}{k: v},
})
- } else if _, ok := v.(int); ok {
+ continue
+ }
+ if _, ok := v.(int); ok {
mustClauses = append(mustClauses, map[string]interface{}{
"term": map[string]interface{}{k: v},
})
}
}
+ if len(mustClauses) == 0 && len(mustNotClauses) == 0 && len(condition) > 0 {
+ // Every condition key was dropped (empty value or unsupported type).
+ // update-by-query without a filter rewrites the whole index, so refuse
+ // instead — mirrors DeleteChunks.
+ return fmt.Errorf("ES update aborted: non-empty condition yielded unfiltered update on index %s", indexName)
+ }
- boolQuery := map[string]interface{}{
- "bool": map[string]interface{}{
- "filter": mustClauses,
- },
+ boolClauses := map[string]interface{}{}
+ if len(mustClauses) > 0 {
+ boolClauses["filter"] = mustClauses
+ }
+ if len(mustNotClauses) > 0 {
+ boolClauses["must_not"] = mustNotClauses
}
+ boolQuery := map[string]interface{}{"bool": boolClauses}
// Build painless scripts from newValue
var scripts []string
@@ -712,6 +776,11 @@ func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, cond
continue
}
+ if items, ok := conditionTermsList(v); ok {
+ params[fmt.Sprintf("pp_%s", k)] = items
+ scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))
+ continue
+ }
switch val := v.(type) {
case string:
// Sanitize: replace ' \n \r with space
@@ -720,9 +789,6 @@ func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, cond
scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))
case int, int8, int16, int32, int64, float32, float64:
scripts = append(scripts, fmt.Sprintf("ctx._source.%s=%v;", k, val))
- case []interface{}:
- params[fmt.Sprintf("pp_%s", k)] = val
- scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))
}
}
@@ -895,12 +961,15 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
// Handle chunk IDs - use terms query on "id" field instead of ids query on _id
if idVal, ok := condition["id"]; ok && idVal != nil {
- switch v := idVal.(type) {
- case []interface{}:
- ids := make([]string, 0, len(v))
- for _, id := range v {
- if s, ok := id.(string); ok {
- ids = append(ids, s)
+ if id, isString := idVal.(string); isString {
+ mustClauses = append(mustClauses, map[string]interface{}{
+ "term": map[string]interface{}{"id": id},
+ })
+ } else if items, isList := conditionTermsList(idVal); isList {
+ ids := make([]interface{}, 0, len(items))
+ for _, item := range items {
+ if id, isString := item.(string); isString {
+ ids = append(ids, id)
}
}
if len(ids) > 0 {
@@ -908,20 +977,6 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
"terms": map[string]interface{}{"id": ids},
})
}
- case []string:
- // A typed []string must be handled here; the generic loop below
- // skips the "id" key, so without this branch a caller passing
- // map[string]interface{}{"id": []string{...}} would build a query
- // with no id filter and DeleteChunks could match every document.
- if len(v) > 0 {
- mustClauses = append(mustClauses, map[string]interface{}{
- "terms": map[string]interface{}{"id": v},
- })
- }
- case string:
- mustClauses = append(mustClauses, map[string]interface{}{
- "term": map[string]interface{}{"id": v},
- })
}
}
@@ -942,23 +997,11 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
"exists": map[string]interface{}{"field": v},
})
} else if k == "must_not" {
- if m, ok := v.(map[string]interface{}); ok {
- for kk, vv := range m {
- if kk == "exists" {
- mustNotClauses = append(mustNotClauses, map[string]interface{}{
- "exists": map[string]interface{}{"field": vv},
- })
- }
- }
- }
+ mustNotClauses = append(mustNotClauses, mustNotExistsClauses(v)...)
} else if v != nil {
- if listVal, ok := v.([]interface{}); ok {
- mustClauses = append(mustClauses, map[string]interface{}{
- "terms": map[string]interface{}{k: listVal},
- })
- } else if listVal, ok := v.([]string); ok {
+ if items, ok := conditionTermsList(v); ok {
mustClauses = append(mustClauses, map[string]interface{}{
- "terms": map[string]interface{}{k: listVal},
+ "terms": map[string]interface{}{k: items},
})
} else if _, ok := v.(string); ok {
mustClauses = append(mustClauses, map[string]interface{}{
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index 98f1dd3242..807d7fb3a4 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -249,6 +249,200 @@ func TestDeleteChunksIDStringSlice(t *testing.T) {
assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})
}
+// TestUpdateChunksPreservesStringSliceCondition guards the document
+// availability switch: updateSourceChunkAvailability passes a typed []string id
+// list, and a builder that only understands []interface{} silently drops the id
+// clause, widening the update-by-query to every chunk of the dataset (kb_id is
+// then the only remaining filter).
+func TestUpdateChunksPreservesStringSliceCondition(t *testing.T) {
+ var updateQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ switch r.Method {
+ case http.MethodHead:
+ w.WriteHeader(http.StatusOK)
+ case http.MethodPost:
+ if r.URL.Path != "/ragflow_tenant/_update_by_query" {
+ t.Errorf("path=%s, want /ragflow_tenant/_update_by_query", r.URL.Path)
+ http.Error(w, "unexpected request path", http.StatusNotFound)
+ return
+ }
+ if err := json.NewDecoder(r.Body).Decode(&updateQuery); err != nil {
+ t.Errorf("decode update query: %v", err)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"updated":2}`))
+ default:
+ t.Errorf("method=%s", r.Method)
+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
+ }
+ }))
+ defer server.Close()
+
+ ctx := t.Context()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ if err = engine.UpdateChunks(ctx,
+ map[string]interface{}{"id": []string{"chunk-a", "chunk-b"}},
+ map[string]interface{}{"available_int": 1, "source_doc_ids": []string{"doc-a"}},
+ "ragflow_tenant", "kb-1"); err != nil {
+ t.Fatalf("UpdateChunks: %v", err)
+ }
+
+ query, ok := updateQuery["query"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("update query missing query: %#v", updateQuery)
+ }
+ boolQuery, ok := query["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("update query missing bool: %#v", query)
+ }
+ filter, ok := boolQuery["filter"].([]interface{})
+ if !ok {
+ t.Fatalf("update query missing filter: %#v", boolQuery)
+ }
+ var gotIDs, gotKB interface{}
+ for _, raw := range filter {
+ clause, ok := raw.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ if terms, ok := clause["terms"].(map[string]interface{}); ok {
+ if ids, ok := terms["id"]; ok {
+ gotIDs = ids
+ }
+ }
+ if term, ok := clause["term"].(map[string]interface{}); ok {
+ if kb, ok := term["kb_id"]; ok {
+ gotKB = kb
+ }
+ }
+ }
+ if gotIDs == nil {
+ t.Fatalf("update filter dropped the id terms clause (would hit the whole dataset): %#v", filter)
+ }
+ assertEqual(t, gotIDs, []interface{}{"chunk-a", "chunk-b"})
+ assertEqual(t, gotKB, "kb-1")
+
+ script, ok := updateQuery["script"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("update query missing script: %#v", updateQuery)
+ }
+ source, _ := script["source"].(string)
+ if !strings.Contains(source, "ctx._source.available_int=1;") {
+ t.Fatalf("script source=%q, want available_int assignment", source)
+ }
+ if !strings.Contains(source, "ctx._source.source_doc_ids=params.pp_source_doc_ids;") {
+ t.Fatalf("script source=%q, want typed []string value assignment", source)
+ }
+ scriptParams, ok := script["params"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("update script missing params: %#v", script)
+ }
+ assertEqual(t, scriptParams["pp_source_doc_ids"], []interface{}{"doc-a"})
+}
+
+// TestUpdateChunksPreservesMustNotCondition locks the exclusion half of the
+// condition contract. A dropped must_not widens the update to exactly the rows
+// the caller asked to exclude (e.g. hybrid_search's
+// must_not={"exists":"compile_kwd"} excluding compiled products).
+func TestUpdateChunksPreservesMustNotCondition(t *testing.T) {
+ var updateQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ switch r.Method {
+ case http.MethodHead:
+ w.WriteHeader(http.StatusOK)
+ case http.MethodPost:
+ if r.URL.Path != "/ragflow_tenant/_update_by_query" {
+ t.Errorf("path=%s, want /ragflow_tenant/_update_by_query", r.URL.Path)
+ http.Error(w, "unexpected request path", http.StatusNotFound)
+ return
+ }
+ if err := json.NewDecoder(r.Body).Decode(&updateQuery); err != nil {
+ t.Errorf("decode update query: %v", err)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"updated":1}`))
+ default:
+ t.Errorf("method=%s", r.Method)
+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
+ }
+ }))
+ defer server.Close()
+
+ ctx := t.Context()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ exclusion := []interface{}{
+ map[string]interface{}{"exists": map[string]interface{}{"field": "compile_kwd"}},
+ }
+ if err = engine.UpdateChunks(ctx,
+ map[string]interface{}{
+ "doc_id": "doc-1",
+ "must_not": map[string]interface{}{"exists": "compile_kwd"},
+ },
+ map[string]interface{}{"available_int": 0},
+ "ragflow_tenant", "kb-1"); err != nil {
+ t.Fatalf("UpdateChunks: %v", err)
+ }
+ boolQuery := mustBoolQuery(t, updateQuery)
+ assertEqual(t, boolQuery["must_not"], exclusion)
+ if _, ok := boolQuery["filter"]; !ok {
+ t.Fatalf("update query lost the positive filter: %#v", boolQuery)
+ }
+
+ // A must_not-only condition is an explicit "everything except" scope rather
+ // than a dropped filter, so it is accepted and forwarded. DeleteChunks
+ // behaves the same way.
+ if err = engine.updateChunksByQuery(ctx, "ragflow_tenant",
+ map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},
+ map[string]interface{}{"available_int": 0}); err != nil {
+ t.Fatalf("updateChunksByQuery must_not only: %v", err)
+ }
+ boolQuery = mustBoolQuery(t, updateQuery)
+ assertEqual(t, boolQuery["must_not"], exclusion)
+ if _, ok := boolQuery["filter"]; ok {
+ t.Fatalf("must_not-only update should carry no positive filter: %#v", boolQuery)
+ }
+}
+
+// mustBoolQuery extracts query.bool from a captured update-by-query body.
+func mustBoolQuery(t *testing.T, body map[string]interface{}) map[string]interface{} {
+ t.Helper()
+ query, ok := body["query"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("update query missing query: %#v", body)
+ }
+ boolQuery, ok := query["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("update query missing bool: %#v", query)
+ }
+ return boolQuery
+}
+
+// TestUpdateChunksByQueryRejectsUnfilteredUpdate locks the guard: a non-empty
+// condition whose keys all get dropped must fail instead of rewriting every row
+// of the index.
+func TestUpdateChunksByQueryRejectsUnfilteredUpdate(t *testing.T) {
+ engine := &Engine{}
+ err := engine.updateChunksByQuery(t.Context(), "ragflow_tenant",
+ map[string]interface{}{"unsupported_field": struct{}{}},
+ map[string]interface{}{"available_int": 1})
+ if err == nil {
+ t.Fatal("updateChunksByQuery accepted a condition that yields no filter clause")
+ }
+ if !strings.Contains(err.Error(), "unfiltered update") {
+ t.Fatalf("err=%v, want unfiltered update refusal", err)
+ }
+}
+
func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {
engine := &Engine{}
chunks := []map[string]interface{}{
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 5299fbb9a9..5c4dabc928 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -446,6 +446,12 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interfac
// Build filter string from condition
filter := buildFilterFromCondition(condition, clmns)
+ if len(condition) > 0 && (filter == "" || filter == "1=1") {
+ // Every condition key was dropped (blank value, empty list, unknown
+ // column). table.Update("1=1", ...) would rewrite every row of the
+ // dataset, so refuse instead — mirrors DeleteChunks/DeleteMetadata.
+ return fmt.Errorf("INFINITY update aborted: non-empty condition yielded unconstrained filter on table %s", tableName)
+ }
// Process remove operation first
removeValue := make(map[string]interface{})
diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.go
index 98993a7599..232e77e464 100644
--- a/internal/engine/infinity/sql_test.go
+++ b/internal/engine/infinity/sql_test.go
@@ -400,3 +400,26 @@ func TestBuildFilterFromCondition_UnconstrainedFilter(t *testing.T) {
t.Errorf("non-empty condition with blank values: got %q, want '1=1'", got)
}
}
+
+// TestBuildFilterFromCondition_StringSliceIDPreservesScope pins the shape the
+// document availability switch relies on: updateSourceChunkAvailability passes a
+// typed []string id list, and Infinity must render it as an IN clause. Dropping
+// it leaves whatever other clauses the caller passed (none, for that path), i.e.
+// an update scoped to the whole dataset table.
+func TestBuildFilterFromCondition_StringSliceIDPreservesScope(t *testing.T) {
+ clmns := map[string]struct {
+ Type string
+ Default interface{}
+ }{
+ "id": {"Varchar", ""},
+ }
+ got := buildFilterFromCondition(map[string]interface{}{"id": []string{"chunk-a", "chunk-b"}}, clmns)
+ if got != "id IN ('chunk-a', 'chunk-b')" {
+ t.Errorf("condition []string id: got %q, want %q", got, "id IN ('chunk-a', 'chunk-b')")
+ }
+ // An empty list contributes no clause, so the caller is left with an
+ // unconstrained filter — the case UpdateChunks/DeleteChunks refuse.
+ if got := buildFilterFromCondition(map[string]interface{}{"id": []string{}}, clmns); got != "1=1" {
+ t.Errorf("empty []string id: got %q, want '1=1'", got)
+ }
+}