RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(es): return error on item-level failures in metadata bulk write (#21956)
ZZZZY5482c2e299cb
diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.goindex 8c9aba6ccc..f64d468a62 100644--- a/internal/engine/elasticsearch/metadata.go+++ b/internal/engine/elasticsearch/metadata.go@@ -159,9 +159,50 @@ func (e *Engine) InsertMetadata(ctx context.Context, metadata []map[string]interreturn nil, fmt.Errorf("failed to parse bulk response: %w", err)}- // Check for errors in bulk response- if errors, ok := bulkResponse["errors"].(bool); ok && errors {- common.Warn("Bulk request had some errors")+ // Check for errors in bulk response. ES reports a 200 at the top level+ // even when individual bulk items fail, so inspect the items and surface+ // the reasons instead of returning success while metadata was silently+ // dropped.+ if hasErrs, ok := bulkResponse["errors"].(bool); ok && hasErrs {+ var failures []string+ failed := 0+ if items, ok := bulkResponse["items"].([]interface{}); ok {+ for _, it := range items {+ im, ok := it.(map[string]interface{})+ if !ok {+ continue+ }+ for _, op := range im {+ om, ok := op.(map[string]interface{})+ if !ok {+ continue+ }+ errObj, ok := om["error"].(map[string]interface{})+ if !ok {+ continue+ }+ failed+++ reason, _ := errObj["reason"].(string)+ docID, _ := om["_id"].(string)+ switch {+ case docID != "" && reason != "":+ failures = append(failures, docID+": "+reason)+ case docID != "":+ failures = append(failures, docID)+ case reason != "":+ failures = append(failures, reason)+ }+ }+ }+ }+ shown := failures+ if len(shown) > 5 {+ shown = shown[:5]+ }+ common.Error("Elasticsearch bulk request had item errors",+ fmt.Errorf("index %s: %d failed items; first failures: %v", indexName, failed, shown))+ return nil, fmt.Errorf("elasticsearch bulk request had %d item errors (index %s); first failures: %v",+ failed, indexName, shown)}common.Info("ElasticsearchConnection.InsertMetadata result", zap.String("index_name", indexName), zap.Int("count", len(metadata)))diff --git a/internal/engine/elasticsearch/metadata_bulk_test.go b/internal/engine/elasticsearch/metadata_bulk_test.gonew file mode 100644index 0000000000..2b377a6cb6--- /dev/null+++ b/internal/engine/elasticsearch/metadata_bulk_test.go@@ -0,0 +1,126 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package elasticsearch++import (+ "fmt"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+)++// newBulkReplyEngine returns an Engine whose bulk requests are answered with+// body, and whose update-by-query requests are answered with ubqBody.+func newBulkReplyEngine(t *testing.T, ubqBody, bulkBody string) *Engine {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ w.Header().Set("Content-Type", "application/json")+ if strings.HasSuffix(r.URL.Path, "/_update_by_query") {+ _, _ = w.Write([]byte(ubqBody))+ return+ }+ _, _ = w.Write([]byte(bulkBody))+ }))+ t.Cleanup(server.Close)+ return newTestEngine(t, server.URL)+}++func oneMetadata() []map[string]interface{} {+ return []map[string]interface{}{{+ "id": "doc-1",+ "kb_id": "kb-1",+ "meta_fields": map[string]interface{}{"author": "alice"},+ }}+}++func TestInsertMetadataReturnsErrorOnBulkItemError(t *testing.T) {+ engine := newBulkReplyEngine(t, `{}`,+ `{"took":1,"errors":true,"items":[{"index":{"_index":"ragflow_doc_meta_t1","_id":"doc-1","status":400,"error":{"type":"document_parsing_exception","reason":"failed to parse field [meta_fields.author]"}}}]}`)++ _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1")+ if err == nil {+ t.Fatal("InsertMetadata returned nil for a bulk response with a failed item")+ }+ for _, want := range []string{"ragflow_doc_meta_t1", "1 item errors", "doc-1: failed to parse field"} {+ if !strings.Contains(err.Error(), want) {+ t.Fatalf("error %q does not mention %q", err.Error(), want)+ }+ }+}++func TestInsertMetadataReportsFailedItemCountBeforeTruncatingReasons(t *testing.T) {+ var items []string+ for i := 0; i < 7; i++ {+ items = append(items, fmt.Sprintf(+ `{"index":{"_id":"doc-%d","status":400,"error":{"reason":"reason-%d"}}}`, i, i))+ }+ body := `{"errors":true,"items":[` + strings.Join(items, ",") + `]}`+ engine := newBulkReplyEngine(t, `{}`, body)++ _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1")+ if err == nil {+ t.Fatal("InsertMetadata returned nil for 7 failed items")+ }+ if !strings.Contains(err.Error(), "7 item errors") {+ t.Fatalf("error %q does not report all 7 failed items", err.Error())+ }+ if strings.Contains(err.Error(), "reason-5") || strings.Contains(err.Error(), "reason-6") {+ t.Fatalf("error %q should only show the first 5 reasons", err.Error())+ }+}++func TestInsertMetadataIdentifiesEachFailedDocument(t *testing.T) {+ engine := newBulkReplyEngine(t, `{}`,+ `{"errors":true,"items":[`++ `{"index":{"_id":"doc-a","status":400,"error":{"reason":"failed to parse field [meta_fields.author]"}}},`++ `{"index":{"_id":"doc-b","status":400,"error":{"reason":"failed to parse field [meta_fields.author]"}}}]}`)++ _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1")+ if err == nil {+ t.Fatal("InsertMetadata returned nil for failed bulk items")+ }+ for _, want := range []string{"doc-a:", "doc-b:", "2 item errors"} {+ if !strings.Contains(err.Error(), want) {+ t.Fatalf("error %q does not mention %q", err.Error(), want)+ }+ }+}++func TestInsertMetadataSucceedsOnCleanBulkResponse(t *testing.T) {+ engine := newBulkReplyEngine(t, `{}`,+ `{"took":1,"errors":false,"items":[{"index":{"_id":"doc-1","status":201}}]}`)++ if _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1"); err != nil {+ t.Fatalf("InsertMetadata returned %v for a clean bulk response", err)+ }+}++func TestUpdateMetadataPropagatesFallbackBulkItemError(t *testing.T) {+ engine := newBulkReplyEngine(t, `{"took":1,"total":0,"updated":0,"failures":[]}`,+ `{"errors":true,"items":[{"index":{"_id":"doc-1","status":400,"error":{"reason":"failed to parse field [meta_fields.author]"}}}]}`)++ err := engine.UpdateMetadata(t.Context(), "doc-1", "kb-1",+ map[string]interface{}{"author": "alice"}, "t1")+ if err == nil {+ t.Fatal("UpdateMetadata returned nil when its insert fallback had a failed item")+ }+ if !strings.Contains(err.Error(), "failed to insert metadata") {+ t.Fatalf("error %q does not wrap the fallback failure", err.Error())+ }+}