Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(es): return error on item-level failures in metadata bulk write (#21956)

ZZZZY5482c2e299cb
diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.go
index 8c9aba6ccc..f64d468a62 100644
--- a/internal/engine/elasticsearch/metadata.go
+++ b/internal/engine/elasticsearch/metadata.go
@@ -159,9 +159,50 @@ func (e *Engine) InsertMetadata(ctx context.Context, metadata []map[string]inter
return nil, fmt.Errorf("failed to parse bulk response: %w", err)
}
- // Check for errors in bulk response
- if errors, ok := bulkResponse["errors"].(bool); ok && errors {
- common.Warn("Bulk request had some errors")
+ // Check for errors in bulk response. ES reports a 200 at the top level
+ // even when individual bulk items fail, so inspect the items and surface
+ // the reasons instead of returning success while metadata was silently
+ // dropped.
+ if hasErrs, ok := bulkResponse["errors"].(bool); ok && hasErrs {
+ var failures []string
+ failed := 0
+ if items, ok := bulkResponse["items"].([]interface{}); ok {
+ for _, it := range items {
+ im, ok := it.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ for _, op := range im {
+ om, ok := op.(map[string]interface{})
+ if !ok {
+ continue
+ }
+ errObj, ok := om["error"].(map[string]interface{})
+ if !ok {
+ continue
+ }
+ failed++
+ reason, _ := errObj["reason"].(string)
+ docID, _ := om["_id"].(string)
+ switch {
+ case docID != "" && reason != "":
+ failures = append(failures, docID+": "+reason)
+ case docID != "":
+ failures = append(failures, docID)
+ case reason != "":
+ failures = append(failures, reason)
+ }
+ }
+ }
+ }
+ shown := failures
+ if len(shown) > 5 {
+ shown = shown[:5]
+ }
+ common.Error("Elasticsearch bulk request had item errors",
+ fmt.Errorf("index %s: %d failed items; first failures: %v", indexName, failed, shown))
+ return nil, fmt.Errorf("elasticsearch bulk request had %d item errors (index %s); first failures: %v",
+ failed, indexName, shown)
}
common.Info("ElasticsearchConnection.InsertMetadata result", zap.String("index_name", indexName), zap.Int("count", len(metadata)))
diff --git a/internal/engine/elasticsearch/metadata_bulk_test.go b/internal/engine/elasticsearch/metadata_bulk_test.go
new file mode 100644
index 0000000000..2b377a6cb6
--- /dev/null
+++ b/internal/engine/elasticsearch/metadata_bulk_test.go
@@ -0,0 +1,126 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package elasticsearch
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+// newBulkReplyEngine returns an Engine whose bulk requests are answered with
+// body, and whose update-by-query requests are answered with ubqBody.
+func newBulkReplyEngine(t *testing.T, ubqBody, bulkBody string) *Engine {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ if strings.HasSuffix(r.URL.Path, "/_update_by_query") {
+ _, _ = w.Write([]byte(ubqBody))
+ return
+ }
+ _, _ = w.Write([]byte(bulkBody))
+ }))
+ t.Cleanup(server.Close)
+ return newTestEngine(t, server.URL)
+}
+
+func oneMetadata() []map[string]interface{} {
+ return []map[string]interface{}{{
+ "id": "doc-1",
+ "kb_id": "kb-1",
+ "meta_fields": map[string]interface{}{"author": "alice"},
+ }}
+}
+
+func TestInsertMetadataReturnsErrorOnBulkItemError(t *testing.T) {
+ engine := newBulkReplyEngine(t, `{}`,
+ `{"took":1,"errors":true,"items":[{"index":{"_index":"ragflow_doc_meta_t1","_id":"doc-1","status":400,"error":{"type":"document_parsing_exception","reason":"failed to parse field [meta_fields.author]"}}}]}`)
+
+ _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1")
+ if err == nil {
+ t.Fatal("InsertMetadata returned nil for a bulk response with a failed item")
+ }
+ for _, want := range []string{"ragflow_doc_meta_t1", "1 item errors", "doc-1: failed to parse field"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Fatalf("error %q does not mention %q", err.Error(), want)
+ }
+ }
+}
+
+func TestInsertMetadataReportsFailedItemCountBeforeTruncatingReasons(t *testing.T) {
+ var items []string
+ for i := 0; i < 7; i++ {
+ items = append(items, fmt.Sprintf(
+ `{"index":{"_id":"doc-%d","status":400,"error":{"reason":"reason-%d"}}}`, i, i))
+ }
+ body := `{"errors":true,"items":[` + strings.Join(items, ",") + `]}`
+ engine := newBulkReplyEngine(t, `{}`, body)
+
+ _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1")
+ if err == nil {
+ t.Fatal("InsertMetadata returned nil for 7 failed items")
+ }
+ if !strings.Contains(err.Error(), "7 item errors") {
+ t.Fatalf("error %q does not report all 7 failed items", err.Error())
+ }
+ if strings.Contains(err.Error(), "reason-5") || strings.Contains(err.Error(), "reason-6") {
+ t.Fatalf("error %q should only show the first 5 reasons", err.Error())
+ }
+}
+
+func TestInsertMetadataIdentifiesEachFailedDocument(t *testing.T) {
+ engine := newBulkReplyEngine(t, `{}`,
+ `{"errors":true,"items":[`+
+ `{"index":{"_id":"doc-a","status":400,"error":{"reason":"failed to parse field [meta_fields.author]"}}},`+
+ `{"index":{"_id":"doc-b","status":400,"error":{"reason":"failed to parse field [meta_fields.author]"}}}]}`)
+
+ _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1")
+ if err == nil {
+ t.Fatal("InsertMetadata returned nil for failed bulk items")
+ }
+ for _, want := range []string{"doc-a:", "doc-b:", "2 item errors"} {
+ if !strings.Contains(err.Error(), want) {
+ t.Fatalf("error %q does not mention %q", err.Error(), want)
+ }
+ }
+}
+
+func TestInsertMetadataSucceedsOnCleanBulkResponse(t *testing.T) {
+ engine := newBulkReplyEngine(t, `{}`,
+ `{"took":1,"errors":false,"items":[{"index":{"_id":"doc-1","status":201}}]}`)
+
+ if _, err := engine.InsertMetadata(t.Context(), oneMetadata(), "t1"); err != nil {
+ t.Fatalf("InsertMetadata returned %v for a clean bulk response", err)
+ }
+}
+
+func TestUpdateMetadataPropagatesFallbackBulkItemError(t *testing.T) {
+ engine := newBulkReplyEngine(t, `{"took":1,"total":0,"updated":0,"failures":[]}`,
+ `{"errors":true,"items":[{"index":{"_id":"doc-1","status":400,"error":{"reason":"failed to parse field [meta_fields.author]"}}}]}`)
+
+ err := engine.UpdateMetadata(t.Context(), "doc-1", "kb-1",
+ map[string]interface{}{"author": "alice"}, "t1")
+ if err == nil {
+ t.Fatal("UpdateMetadata returned nil when its insert fallback had a failed item")
+ }
+ if !strings.Contains(err.Error(), "failed to insert metadata") {
+ t.Fatalf("error %q does not wrap the fallback failure", err.Error())
+ }
+}