RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(engine): filter chunks by requested IDs in update-by-query (#19665)
qinling0210b9f4d26aadf3
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex ecdad76dd8..bb50aeee86 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -638,12 +638,59 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strinreturn nil}+// conditionTermsList normalizes a condition or new-value field into the value+// list a `terms` query / stored array field needs. Callers hand over either+// []interface{} (decoded JSON payloads) or a typed slice such as []string+// (internal callers, e.g. the document availability switch). A typed slice the+// builder does not recognize silently drops the field, which turns a bounded+// update/delete by query into an unfiltered one over the whole index.+func conditionTermsList(value interface{}) ([]interface{}, bool) {+ items, ok := value.([]interface{})+ if ok {+ return items, true+ }+ rv := reflect.ValueOf(value)+ if rv.Kind() != reflect.Slice && rv.Kind() != reflect.Array {+ return nil, false+ }+ if rv.Type().Elem().Kind() == reflect.Uint8 {+ // []byte is a scalar blob, not a list of ids/keywords.+ return nil, false+ }+ out := make([]interface{}, 0, rv.Len())+ for i := 0; i < rv.Len(); i++ {+ out = append(out, rv.Index(i).Interface())+ }+ return out, true+}++// mustNotExistsClauses renders the `must_not` exclusions a condition carries.+// The only form RAGFlow produces is must_not={"exists": <field>} (e.g.+// hybrid_search excluding compiled products); any other shape yields nothing.+func mustNotExistsClauses(value interface{}) []map[string]interface{} {+ exclusions, ok := value.(map[string]interface{})+ if !ok {+ return nil+ }+ clauses := make([]map[string]interface{}, 0, len(exclusions))+ for field, value := range exclusions {+ if field != "exists" {+ continue+ }+ clauses = append(clauses, map[string]interface{}{+ "exists": map[string]interface{}{"field": value},+ })+ }+ return clauses+}+// updateChunksByQuery handles multi-document updatefunc (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error {common.Debug("ElasticsearchConnection.updateChunksByQuery called", zap.String("indexName", indexName))// Build bool query from conditionvar mustClauses []map[string]interface{}+ var mustNotClauses []map[string]interface{}for k, v := range condition {if k == "exists" {mustClauses = append(mustClauses, map[string]interface{}{@@ -651,29 +698,46 @@ func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, cond})continue}+ if k == "must_not" {+ mustNotClauses = append(mustNotClauses, mustNotExistsClauses(v)...)+ continue+ }if v == nil || v == "" {continue}- if listVal, ok := v.([]interface{}); ok {+ if items, ok := conditionTermsList(v); ok {mustClauses = append(mustClauses, map[string]interface{}{- "terms": map[string]interface{}{k: listVal},+ "terms": map[string]interface{}{k: items},})- } else if _, ok := v.(string); ok {+ continue+ }+ if _, ok := v.(string); ok {mustClauses = append(mustClauses, map[string]interface{}{"term": map[string]interface{}{k: v},})- } else if _, ok := v.(int); ok {+ continue+ }+ if _, ok := v.(int); ok {mustClauses = append(mustClauses, map[string]interface{}{"term": map[string]interface{}{k: v},})}}+ if len(mustClauses) == 0 && len(mustNotClauses) == 0 && len(condition) > 0 {+ // Every condition key was dropped (empty value or unsupported type).+ // update-by-query without a filter rewrites the whole index, so refuse+ // instead — mirrors DeleteChunks.+ return fmt.Errorf("ES update aborted: non-empty condition yielded unfiltered update on index %s", indexName)+ }- boolQuery := map[string]interface{}{- "bool": map[string]interface{}{- "filter": mustClauses,- },+ boolClauses := map[string]interface{}{}+ if len(mustClauses) > 0 {+ boolClauses["filter"] = mustClauses+ }+ if len(mustNotClauses) > 0 {+ boolClauses["must_not"] = mustNotClauses}+ boolQuery := map[string]interface{}{"bool": boolClauses}// Build painless scripts from newValuevar scripts []string@@ -712,6 +776,11 @@ func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condcontinue}+ if items, ok := conditionTermsList(v); ok {+ params[fmt.Sprintf("pp_%s", k)] = items+ scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))+ continue+ }switch val := v.(type) {case string:// Sanitize: replace ' \n \r with space@@ -720,9 +789,6 @@ func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condscripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))case int, int8, int16, int32, int64, float32, float64:scripts = append(scripts, fmt.Sprintf("ctx._source.%s=%v;", k, val))- case []interface{}:- params[fmt.Sprintf("pp_%s", k)] = val- scripts = append(scripts, fmt.Sprintf("ctx._source.%s=params.pp_%s;", k, k))}}@@ -895,12 +961,15 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac// Handle chunk IDs - use terms query on "id" field instead of ids query on _idif idVal, ok := condition["id"]; ok && idVal != nil {- switch v := idVal.(type) {- case []interface{}:- ids := make([]string, 0, len(v))- for _, id := range v {- if s, ok := id.(string); ok {- ids = append(ids, s)+ if id, isString := idVal.(string); isString {+ mustClauses = append(mustClauses, map[string]interface{}{+ "term": map[string]interface{}{"id": id},+ })+ } else if items, isList := conditionTermsList(idVal); isList {+ ids := make([]interface{}, 0, len(items))+ for _, item := range items {+ if id, isString := item.(string); isString {+ ids = append(ids, id)}}if len(ids) > 0 {@@ -908,20 +977,6 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac"terms": map[string]interface{}{"id": ids},})}- case []string:- // A typed []string must be handled here; the generic loop below- // skips the "id" key, so without this branch a caller passing- // map[string]interface{}{"id": []string{...}} would build a query- // with no id filter and DeleteChunks could match every document.- if len(v) > 0 {- mustClauses = append(mustClauses, map[string]interface{}{- "terms": map[string]interface{}{"id": v},- })- }- case string:- mustClauses = append(mustClauses, map[string]interface{}{- "term": map[string]interface{}{"id": v},- })}}@@ -942,23 +997,11 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac"exists": map[string]interface{}{"field": v},})} else if k == "must_not" {- if m, ok := v.(map[string]interface{}); ok {- for kk, vv := range m {- if kk == "exists" {- mustNotClauses = append(mustNotClauses, map[string]interface{}{- "exists": map[string]interface{}{"field": vv},- })- }- }- }+ mustNotClauses = append(mustNotClauses, mustNotExistsClauses(v)...)} else if v != nil {- if listVal, ok := v.([]interface{}); ok {- mustClauses = append(mustClauses, map[string]interface{}{- "terms": map[string]interface{}{k: listVal},- })- } else if listVal, ok := v.([]string); ok {+ if items, ok := conditionTermsList(v); ok {mustClauses = append(mustClauses, map[string]interface{}{- "terms": map[string]interface{}{k: listVal},+ "terms": map[string]interface{}{k: items},})} else if _, ok := v.(string); ok {mustClauses = append(mustClauses, map[string]interface{}{diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex 98f1dd3242..807d7fb3a4 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -249,6 +249,200 @@ func TestDeleteChunksIDStringSlice(t *testing.T) {assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})}+// TestUpdateChunksPreservesStringSliceCondition guards the document+// availability switch: updateSourceChunkAvailability passes a typed []string id+// list, and a builder that only understands []interface{} silently drops the id+// clause, widening the update-by-query to every chunk of the dataset (kb_id is+// then the only remaining filter).+func TestUpdateChunksPreservesStringSliceCondition(t *testing.T) {+ var updateQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ switch r.Method {+ case http.MethodHead:+ w.WriteHeader(http.StatusOK)+ case http.MethodPost:+ if r.URL.Path != "/ragflow_tenant/_update_by_query" {+ t.Errorf("path=%s, want /ragflow_tenant/_update_by_query", r.URL.Path)+ http.Error(w, "unexpected request path", http.StatusNotFound)+ return+ }+ if err := json.NewDecoder(r.Body).Decode(&updateQuery); err != nil {+ t.Errorf("decode update query: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"updated":2}`))+ default:+ t.Errorf("method=%s", r.Method)+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)+ }+ }))+ defer server.Close()++ ctx := t.Context()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ if err = engine.UpdateChunks(ctx,+ map[string]interface{}{"id": []string{"chunk-a", "chunk-b"}},+ map[string]interface{}{"available_int": 1, "source_doc_ids": []string{"doc-a"}},+ "ragflow_tenant", "kb-1"); err != nil {+ t.Fatalf("UpdateChunks: %v", err)+ }++ query, ok := updateQuery["query"].(map[string]interface{})+ if !ok {+ t.Fatalf("update query missing query: %#v", updateQuery)+ }+ boolQuery, ok := query["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("update query missing bool: %#v", query)+ }+ filter, ok := boolQuery["filter"].([]interface{})+ if !ok {+ t.Fatalf("update query missing filter: %#v", boolQuery)+ }+ var gotIDs, gotKB interface{}+ for _, raw := range filter {+ clause, ok := raw.(map[string]interface{})+ if !ok {+ continue+ }+ if terms, ok := clause["terms"].(map[string]interface{}); ok {+ if ids, ok := terms["id"]; ok {+ gotIDs = ids+ }+ }+ if term, ok := clause["term"].(map[string]interface{}); ok {+ if kb, ok := term["kb_id"]; ok {+ gotKB = kb+ }+ }+ }+ if gotIDs == nil {+ t.Fatalf("update filter dropped the id terms clause (would hit the whole dataset): %#v", filter)+ }+ assertEqual(t, gotIDs, []interface{}{"chunk-a", "chunk-b"})+ assertEqual(t, gotKB, "kb-1")++ script, ok := updateQuery["script"].(map[string]interface{})+ if !ok {+ t.Fatalf("update query missing script: %#v", updateQuery)+ }+ source, _ := script["source"].(string)+ if !strings.Contains(source, "ctx._source.available_int=1;") {+ t.Fatalf("script source=%q, want available_int assignment", source)+ }+ if !strings.Contains(source, "ctx._source.source_doc_ids=params.pp_source_doc_ids;") {+ t.Fatalf("script source=%q, want typed []string value assignment", source)+ }+ scriptParams, ok := script["params"].(map[string]interface{})+ if !ok {+ t.Fatalf("update script missing params: %#v", script)+ }+ assertEqual(t, scriptParams["pp_source_doc_ids"], []interface{}{"doc-a"})+}++// TestUpdateChunksPreservesMustNotCondition locks the exclusion half of the+// condition contract. A dropped must_not widens the update to exactly the rows+// the caller asked to exclude (e.g. hybrid_search's+// must_not={"exists":"compile_kwd"} excluding compiled products).+func TestUpdateChunksPreservesMustNotCondition(t *testing.T) {+ var updateQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ switch r.Method {+ case http.MethodHead:+ w.WriteHeader(http.StatusOK)+ case http.MethodPost:+ if r.URL.Path != "/ragflow_tenant/_update_by_query" {+ t.Errorf("path=%s, want /ragflow_tenant/_update_by_query", r.URL.Path)+ http.Error(w, "unexpected request path", http.StatusNotFound)+ return+ }+ if err := json.NewDecoder(r.Body).Decode(&updateQuery); err != nil {+ t.Errorf("decode update query: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"updated":1}`))+ default:+ t.Errorf("method=%s", r.Method)+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)+ }+ }))+ defer server.Close()++ ctx := t.Context()+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ exclusion := []interface{}{+ map[string]interface{}{"exists": map[string]interface{}{"field": "compile_kwd"}},+ }+ if err = engine.UpdateChunks(ctx,+ map[string]interface{}{+ "doc_id": "doc-1",+ "must_not": map[string]interface{}{"exists": "compile_kwd"},+ },+ map[string]interface{}{"available_int": 0},+ "ragflow_tenant", "kb-1"); err != nil {+ t.Fatalf("UpdateChunks: %v", err)+ }+ boolQuery := mustBoolQuery(t, updateQuery)+ assertEqual(t, boolQuery["must_not"], exclusion)+ if _, ok := boolQuery["filter"]; !ok {+ t.Fatalf("update query lost the positive filter: %#v", boolQuery)+ }++ // A must_not-only condition is an explicit "everything except" scope rather+ // than a dropped filter, so it is accepted and forwarded. DeleteChunks+ // behaves the same way.+ if err = engine.updateChunksByQuery(ctx, "ragflow_tenant",+ map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},+ map[string]interface{}{"available_int": 0}); err != nil {+ t.Fatalf("updateChunksByQuery must_not only: %v", err)+ }+ boolQuery = mustBoolQuery(t, updateQuery)+ assertEqual(t, boolQuery["must_not"], exclusion)+ if _, ok := boolQuery["filter"]; ok {+ t.Fatalf("must_not-only update should carry no positive filter: %#v", boolQuery)+ }+}++// mustBoolQuery extracts query.bool from a captured update-by-query body.+func mustBoolQuery(t *testing.T, body map[string]interface{}) map[string]interface{} {+ t.Helper()+ query, ok := body["query"].(map[string]interface{})+ if !ok {+ t.Fatalf("update query missing query: %#v", body)+ }+ boolQuery, ok := query["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("update query missing bool: %#v", query)+ }+ return boolQuery+}++// TestUpdateChunksByQueryRejectsUnfilteredUpdate locks the guard: a non-empty+// condition whose keys all get dropped must fail instead of rewriting every row+// of the index.+func TestUpdateChunksByQueryRejectsUnfilteredUpdate(t *testing.T) {+ engine := &Engine{}+ err := engine.updateChunksByQuery(t.Context(), "ragflow_tenant",+ map[string]interface{}{"unsupported_field": struct{}{}},+ map[string]interface{}{"available_int": 1})+ if err == nil {+ t.Fatal("updateChunksByQuery accepted a condition that yields no filter clause")+ }+ if !strings.Contains(err.Error(), "unfiltered update") {+ t.Fatalf("err=%v, want unfiltered update refusal", err)+ }+}+func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {engine := &Engine{}chunks := []map[string]interface{}{diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 5299fbb9a9..5c4dabc928 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -446,6 +446,12 @@ func (e *Engine) UpdateChunks(ctx context.Context, condition map[string]interfac// Build filter string from conditionfilter := buildFilterFromCondition(condition, clmns)+ if len(condition) > 0 && (filter == "" || filter == "1=1") {+ // Every condition key was dropped (blank value, empty list, unknown+ // column). table.Update("1=1", ...) would rewrite every row of the+ // dataset, so refuse instead — mirrors DeleteChunks/DeleteMetadata.+ return fmt.Errorf("INFINITY update aborted: non-empty condition yielded unconstrained filter on table %s", tableName)+ }// Process remove operation firstremoveValue := make(map[string]interface{})diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.goindex 98993a7599..232e77e464 100644--- a/internal/engine/infinity/sql_test.go+++ b/internal/engine/infinity/sql_test.go@@ -400,3 +400,26 @@ func TestBuildFilterFromCondition_UnconstrainedFilter(t *testing.T) {t.Errorf("non-empty condition with blank values: got %q, want '1=1'", got)}}++// TestBuildFilterFromCondition_StringSliceIDPreservesScope pins the shape the+// document availability switch relies on: updateSourceChunkAvailability passes a+// typed []string id list, and Infinity must render it as an IN clause. Dropping+// it leaves whatever other clauses the caller passed (none, for that path), i.e.+// an update scoped to the whole dataset table.+func TestBuildFilterFromCondition_StringSliceIDPreservesScope(t *testing.T) {+ clmns := map[string]struct {+ Type string+ Default interface{}+ }{+ "id": {"Varchar", ""},+ }+ got := buildFilterFromCondition(map[string]interface{}{"id": []string{"chunk-a", "chunk-b"}}, clmns)+ if got != "id IN ('chunk-a', 'chunk-b')" {+ t.Errorf("condition []string id: got %q, want %q", got, "id IN ('chunk-a', 'chunk-b')")+ }+ // An empty list contributes no clause, so the caller is left with an+ // unconstrained filter — the case UpdateChunks/DeleteChunks refuse.+ if got := buildFilterFromCondition(map[string]interface{}{"id": []string{}}, clmns); got != "1=1" {+ t.Errorf("empty []string id: got %q, want '1=1'", got)+ }+}