RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(elasticsearch): propagate search failures with index context (#20546)
quifox7a5a337756c6
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex 88f81c0c5f..d987389572 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -1169,7 +1169,8 @@ type SearchResponse struct {Aggregations map[string]interface{} `json:"aggregations"`}-// Search executes search with unified types.SearchRequest+// Search executes a unified search across the requested indices. Ordinary+// searches fail when any index request fails, without returning partial results.func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SearchResult, error) {types.LogSearchRequest("Elasticsearch", req)@@ -1454,8 +1455,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.Sfor _, indexName := range req.IndexNames {searchChunks, indexTotal, esErr := e.searchOneIndex(ctx, indexName, payload)if esErr != nil {- common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(esErr))- continue+ return nil, fmt.Errorf("elasticsearch: search index %q: %w", indexName, esErr)}if hybrid {// A window the backend could not fill is a candidate shortfall@@ -1712,6 +1712,8 @@ func sortByFields(chunks []map[string]interface{}, expr *types.OrderByExpr) []mareturn chunks}+// searchOneIndex sends an ordinary search with a fresh request body. A valid+// missing-index response is empty; other request and response failures propagate.func (e *Engine) searchOneIndex(ctx context.Context, indexName string, payload []byte) ([]map[string]interface{}, int64, error) {res, err := e.client.Search(e.client.Search.WithContext(ctx),@@ -1726,7 +1728,22 @@ func (e *Engine) searchOneIndex(ctx context.Context, indexName string, payload [defer res.Body.Close()if res.IsError() {- bodyBytes, _ := io.ReadAll(res.Body)+ bodyBytes, readErr := io.ReadAll(res.Body)+ if readErr != nil {+ return nil, 0, fmt.Errorf("read elasticsearch error response: %w", readErr)+ }+ if res.StatusCode == http.StatusNotFound {+ var failure struct {+ Error struct {+ Type string `json:"type"`+ } `json:"error"`+ }+ // A tenant without indexed documents has no physical index yet.+ // Preserve that empty result without hiding unrelated 404 errors.+ if json.Unmarshal(bodyBytes, &failure) == nil && failure.Error.Type == "index_not_found_exception" {+ return nil, 0, nil+ }+ }common.Warn("Elasticsearch error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))return nil, 0, fmt.Errorf("elasticsearch error response: %s", string(bodyBytes))}diff --git a/internal/engine/elasticsearch/search_errors_test.go b/internal/engine/elasticsearch/search_errors_test.gonew file mode 100644index 0000000000..43588444ac--- /dev/null+++ b/internal/engine/elasticsearch/search_errors_test.go@@ -0,0 +1,154 @@+package elasticsearch++import (+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "strings"+ "testing"++ elastic "github.com/elastic/go-elasticsearch/v8"+ "ragflow/internal/common"+ "ragflow/internal/engine/types"+)++type searchErrorTransport func(*http.Request) (*http.Response, error)++// RoundTrip supplies controlled responses and failures through the injected transport.+func (f searchErrorTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }++// newSearchErrorEngine uses the injected transport and disables retries so+// each index query makes a single controlled request.+func newSearchErrorEngine(t *testing.T, transport searchErrorTransport) *Engine {+ t.Helper()+ if err := common.InitLogger("error", common.FileOutput{}, "elasticsearch_search_test"); err != nil {+ t.Fatal(err)+ }+ client, err := elastic.NewClient(elastic.Config{Addresses: []string{"http://search.test"}, Transport: transport, DisableRetry: true})+ if err != nil {+ t.Fatal(err)+ }+ return &Engine{client: client}+}++// searchErrorResponse includes the product header required by the Elasticsearch+// client when accepting a synthetic backend response.+func searchErrorResponse(status int, body string) *http.Response {+ return &http.Response{StatusCode: status, Header: http.Header{"X-Elastic-Product": []string{"Elasticsearch"}, "Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(body))}+}++// TestSearchPropagatesIndexErrors forbids returning apparently complete results after any requested index fails.+func TestSearchPropagatesIndexErrors(t *testing.T) {+ for _, test := range []struct {+ name string+ indexes []string+ status int+ body string+ }{+ {"backend", []string{"failed"}, 400, `{"error":{"reason":"controlled failure"},"status":400}`},+ {"decode", []string{"failed"}, 200, `not-json`},+ {"unrelated_404", []string{"failed"}, 404, `{"error":{"type":"route_not_found"},"status":404}`},+ {"partial", []string{"ok", "failed"}, 400, `{"error":{"reason":"controlled failure"},"status":400}`},+ } {+ t.Run(test.name, func(t *testing.T) {+ engine := newSearchErrorEngine(t, func(r *http.Request) (*http.Response, error) {+ if strings.HasPrefix(r.URL.Path, "/ok/") {+ return searchErrorResponse(200, `{"hits":{"total":{"value":1},"hits":[{"_id":"one","_source":{"id":"one"}}]}}`), nil+ }+ return searchErrorResponse(test.status, test.body), nil+ })+ result, err := engine.Search(t.Context(), &types.SearchRequest{IndexNames: test.indexes, Limit: 10})+ if err == nil || result != nil || !strings.Contains(err.Error(), "failed") {+ t.Fatalf("result=%+v err=%v, want scoped error and no partial result", result, err)+ }+ })+ }+}++// TestSearchPreservesTransportError lets callers distinguish transport failures through errors.Is.+func TestSearchPreservesTransportError(t *testing.T) {+ controlled := errors.New("controlled transport failure")+ engine := newSearchErrorEngine(t, func(*http.Request) (*http.Response, error) { return nil, controlled })+ result, err := engine.Search(t.Context(), &types.SearchRequest{IndexNames: []string{"failed"}, Limit: 10})+ if result != nil || !errors.Is(err, controlled) {+ t.Fatalf("result=%+v err=%v", result, err)+ }+}++// TestSearchSuccessfulIndicesRemainAggregated checks fresh request bodies and a genuine empty response.+func TestSearchSuccessfulIndicesRemainAggregated(t *testing.T) {+ for _, empty := range []bool{false, true} {+ t.Run(map[bool]string{false: "hits", true: "empty"}[empty], func(t *testing.T) {+ requests := 0+ engine := newSearchErrorEngine(t, func(r *http.Request) (*http.Response, error) {+ var body map[string]any+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Fatal(err)+ }+ requests+++ if empty {+ return searchErrorResponse(200, `{"hits":{"total":{"value":0},"hits":[]}}`), nil+ }+ return searchErrorResponse(200, `{"hits":{"total":{"value":1},"hits":[{"_id":"one","_source":{"id":"one"}}]}}`), nil+ })+ result, err := engine.Search(t.Context(), &types.SearchRequest{IndexNames: []string{"first", "second"}, Limit: 10})+ if err != nil {+ t.Fatal(err)+ }+ want := 2+ if empty {+ want = 0+ }+ if requests != 2 || len(result.Chunks) != want || result.Total != int64(want) {+ t.Fatalf("requests=%d result=%+v", requests, result)+ }+ })+ }+}++// TestSearchMissingIndexRemainsEmpty preserves the unindexed-tenant case alongside populated indices.+func TestSearchMissingIndexRemainsEmpty(t *testing.T) {+ for _, mixed := range []bool{false, true} {+ t.Run(map[bool]string{false: "empty", true: "mixed"}[mixed], func(t *testing.T) {+ engine := newSearchErrorEngine(t, func(r *http.Request) (*http.Response, error) {+ if strings.HasPrefix(r.URL.Path, "/missing/") {+ return searchErrorResponse(404, `{"error":{"type":"index_not_found_exception","reason":"no such index"},"status":404}`), nil+ }+ return searchErrorResponse(200, `{"hits":{"total":{"value":1},"hits":[{"_id":"one","_source":{"id":"one"}}]}}`), nil+ })+ names := []string{"missing"}+ want := 0+ if mixed {+ names = append(names, "populated")+ want = 1+ }+ result, err := engine.Search(t.Context(), &types.SearchRequest{IndexNames: names, Limit: 10})+ if err != nil {+ t.Fatal(err)+ }+ if len(result.Chunks) != want || result.Total != int64(want) {+ t.Fatalf("result=%+v", result)+ }+ })+ }+}++// TestSearchPreservesResponseReadError rejects a truncated error response even when its prefix names a missing index.+func TestSearchPreservesResponseReadError(t *testing.T) {+ controlled := errors.New("controlled response read failure")+ engine := newSearchErrorEngine(t, func(*http.Request) (*http.Response, error) {+ response := searchErrorResponse(404, "")+ response.Body = io.NopCloser(io.MultiReader(strings.NewReader(`{"error":{"type":"index_not_found_exception"}}`), failedSearchReader{controlled}))+ return response, nil+ })+ result, err := engine.Search(t.Context(), &types.SearchRequest{IndexNames: []string{"failed"}, Limit: 10})+ if result != nil || !errors.Is(err, controlled) {+ t.Fatalf("result=%+v err=%v", result, err)+ }+}++type failedSearchReader struct{ err error }++// Read surfaces the injected error after any preceding response prefix has been read.+func (r failedSearchReader) Read([]byte) (int, error) { return 0, r.err }