RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Go: fix resource leak in ES connector (#19912)
Jin Hai1e5fc587f087
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex c99fa737c3..cff83c9f2a 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -589,18 +589,17 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin"source": fmt.Sprintf("ctx._source.remove(\"%s\");", k),},}- body, _ := json.Marshal(scriptBody)+ body, _ = json.Marshal(scriptBody)req := esapi.UpdateRequest{Index: indexName,DocumentID: actualID,Body: bytes.NewReader(body),}- res, err := req.Do(ctx, e.client)+ res, err = req.Do(ctx, e.client)if err != nil {common.Warn("Failed to remove feas field", zap.String("field", k), zap.Error(err))- } else {- res.Body.Close()}+ closeESBody(res)}// Remove specific field if removeField is set@@ -610,18 +609,17 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin"source": fmt.Sprintf("ctx._source.remove('%s');", removeField),},}- body, _ := json.Marshal(scriptBody)+ body, _ = json.Marshal(scriptBody)req := esapi.UpdateRequest{Index: indexName,DocumentID: actualID,Body: bytes.NewReader(body),}- res, err := req.Do(ctx, e.client)+ res, err = req.Do(ctx, e.client)if err != nil {common.Warn("Failed to remove field", zap.String("field", removeField), zap.Error(err))- } else {- res.Body.Close()}+ closeESBody(res)}// Remove specific values from array fields (removeDict)@@ -641,18 +639,17 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin"params": params,},}- body, _ := json.Marshal(scriptBody)+ body, _ = json.Marshal(scriptBody)req := esapi.UpdateRequest{Index: indexName,DocumentID: actualID,Body: bytes.NewReader(body),}- res, err := req.Do(ctx, e.client)+ res, err = req.Do(ctx, e.client)if err != nil {common.Warn("Failed to remove dict fields", zap.Error(err))- } else {- res.Body.Close()}+ closeESBody(res)}}@@ -892,6 +889,17 @@ func sanitizeString(s string) string {return strings.TrimSpace(s)}+// closeESBody releases an ES response body, tolerating the nil response the+// client returns alongside a transport error. Callers that check `err` first+// and close only on the success branch leak the body whenever the client ever+// hands back a non-nil response with an error, so every Do/Search call site+// funnels its close through here.+func closeESBody(res *esapi.Response) {+ if res != nil && res.Body != nil {+ _ = res.Body.Close()+ }+}+// copyFields creates a shallow copy of a mapfunc copyFields(m map[string]interface{}) map[string]interface{} {result := make(map[string]interface{})@@ -1434,34 +1442,12 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S// each iteration a fresh bytes.NewReader.payload := append([]byte(nil), buf.Bytes()...)for _, indexName := range req.IndexNames {- res, err := e.client.Search(- e.client.Search.WithContext(ctx),- e.client.Search.WithIndex(indexName),- e.client.Search.WithBody(bytes.NewReader(payload)),- e.client.Search.WithTrackTotalHits(true),- )- if err != nil {- common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(err))- continue- }- defer res.Body.Close()-- if res.IsError() {- bodyBytes, _ := io.ReadAll(res.Body)- common.Warn("Elasticsearch error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))- continue- }-- // Parse response and return results- var esResp SearchResponse- if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {- common.Warn("Elasticsearch failed to parse response", zap.String("index", indexName), zap.Error(err))+ searchChunks, indexTotal, esErr := e.searchOneIndex(ctx, indexName, payload)+ if esErr != nil {+ common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(esErr))continue}-- searchChunks := convertESResponse(&esResp, "")- totalHits += esResp.Hits.Total.Value-+ totalHits += indexTotalallResults = append(allResults, searchChunks...)}}@@ -1495,6 +1481,35 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S}, nil}+func (e *Engine) searchOneIndex(ctx context.Context, indexName string, payload []byte) ([]map[string]interface{}, int64, error) {+ res, err := e.client.Search(+ e.client.Search.WithContext(ctx),+ e.client.Search.WithIndex(indexName),+ e.client.Search.WithBody(bytes.NewReader(payload)),+ e.client.Search.WithTrackTotalHits(true),+ )+ if err != nil {+ common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(err))+ return nil, 0, err+ }+ defer res.Body.Close()++ if res.IsError() {+ bodyBytes, _ := io.ReadAll(res.Body)+ common.Warn("Elasticsearch error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))+ return nil, 0, fmt.Errorf("elasticsearch error response: %s", string(bodyBytes))+ }++ // Parse response and return results+ var esResp SearchResponse+ if err = json.NewDecoder(res.Body).Decode(&esResp); err != nil {+ common.Warn("Elasticsearch failed to parse response", zap.String("index", indexName), zap.Error(err))+ return nil, 0, err+ }++ return convertESResponse(&esResp, ""), esResp.Hits.Total.Value, nil+}+// searchAfterFetcher issues one ES search request with the given batch// size and search_after cursor, returning the decoded response. Defined// as a function type so the pagination logic below can be unit-tested@@ -2175,70 +2190,80 @@ func (e *Engine) GetChunk(ctx context.Context, baseName, chunkID string, dataset// Try search by doc_id field (which is stored in the document)for _, datasetID := range datasetIDs {- searchReq := map[string]interface{}{- "query": map[string]interface{}{- "bool": map[string]interface{}{- "must": []map[string]interface{}{- {"term": map[string]interface{}{"id": chunkID}},- {"term": map[string]interface{}{"kb_id": datasetID}},- },+ source, found, err := e.searchChunkInDataset(ctx, baseName, chunkID, datasetID)+ if err != nil {+ return nil, err+ }+ if !found {+ continue+ }+ common.Info("GetChunk found hit", zap.String("baseName", baseName), zap.String("chunkID", chunkID))+ source["id"] = chunkID+ return source, nil+ }++ common.Info("GetChunk no hits found", zap.String("baseName", baseName), zap.String("chunkID", chunkID))+ return nil, nil+}++func (e *Engine) searchChunkInDataset(ctx context.Context, baseName, chunkID, datasetID string) (map[string]interface{}, bool, error) {+ searchReq := map[string]interface{}{+ "query": map[string]interface{}{+ "bool": map[string]interface{}{+ "must": []map[string]interface{}{+ {"term": map[string]interface{}{"id": chunkID}},+ {"term": map[string]interface{}{"kb_id": datasetID}},},},- }+ },+ }- body, err := json.Marshal(searchReq)- if err != nil {- return nil, fmt.Errorf("failed to marshal search request: %w", err)- }+ body, err := json.Marshal(searchReq)+ if err != nil {+ return nil, false, fmt.Errorf("failed to marshal search request: %w", err)+ }- res, err := e.client.Search(- e.client.Search.WithContext(ctx),- e.client.Search.WithIndex(baseName),- e.client.Search.WithBody(bytes.NewReader(body)),- )- if err != nil {- return nil, fmt.Errorf("failed to search for chunk: %w", err)- }+ res, err := e.client.Search(+ e.client.Search.WithContext(ctx),+ e.client.Search.WithIndex(baseName),+ e.client.Search.WithBody(bytes.NewReader(body)),+ )- if res.IsError() {- res.Body.Close()- return nil, fmt.Errorf("failed to search for chunk: %s", res.Status())- }+ defer closeESBody(res)- var searchResult map[string]interface{}- if err := json.NewDecoder(res.Body).Decode(&searchResult); err != nil {- res.Body.Close()- return nil, fmt.Errorf("failed to parse search response: %w", err)- }- res.Body.Close()+ if err != nil {+ return nil, false, fmt.Errorf("failed to search for chunk: %w", err)+ }- hits, ok := searchResult["hits"].(map[string]interface{})- if !ok {- continue- }+ if res.IsError() {+ return nil, false, fmt.Errorf("failed to search for chunk: %s", res.Status())+ }- hitList, ok := hits["hits"].([]interface{})- if !ok || len(hitList) == 0 {- continue- }+ var searchResult map[string]interface{}+ if err = json.NewDecoder(res.Body).Decode(&searchResult); err != nil {+ return nil, false, fmt.Errorf("failed to parse search response: %w", err)+ }- firstHit, ok := hitList[0].(map[string]interface{})- if !ok {- continue- }+ hits, ok := searchResult["hits"].(map[string]interface{})+ if !ok {+ return nil, false, nil+ }- source, ok := firstHit["_source"].(map[string]interface{})- if !ok {- continue- }+ hitList, ok := hits["hits"].([]interface{})+ if !ok || len(hitList) == 0 {+ return nil, false, nil+ }- common.Info("GetChunk found hit", zap.String("baseName", baseName), zap.String("chunkID", chunkID))- source["id"] = chunkID- return source, nil+ firstHit, ok := hitList[0].(map[string]interface{})+ if !ok {+ return nil, false, nil}- common.Info("GetChunk no hits found", zap.String("baseName", baseName), zap.String("chunkID", chunkID))- return nil, nil+ source, ok := firstHit["_source"].(map[string]interface{})+ if !ok {+ return nil, false, nil+ }+ return source, true, nil}func (e *Engine) getMemoryMessage(ctx context.Context, indexName, docID string) (interface{}, error) {@@ -2263,7 +2288,7 @@ func (e *Engine) getMemoryMessage(ctx context.Context, indexName, docID string)Found bool `json:"found"`Source map[string]interface{} `json:"_source"`}- if err := json.NewDecoder(res.Body).Decode(&getResult); err != nil {+ if err = json.NewDecoder(res.Body).Decode(&getResult); err != nil {return nil, fmt.Errorf("failed to parse memory message get response: %w", err)}if !getResult.Found || getResult.Source == nil {@@ -2321,23 +2346,23 @@ func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map}}- if _, ok := val.([]interface{}); ok {+ if _, ok = val.([]interface{}); ok {m[field] = valcontinue}if field == "available_int" {- if _, ok := val.(int); ok {+ if _, ok = val.(int); ok {m[field] = valcontinue}- if _, ok := val.(float64); ok {+ if _, ok = val.(float64); ok {m[field] = valcontinue}}- if _, ok := val.(string); !ok {+ if _, ok = val.(string); !ok {val = fmt.Sprintf("%v", val)}m[field] = val@@ -2730,10 +2755,10 @@ func (e *Engine) memoryMessageVectorMappingExists(ctx context.Context, indexNameIndex: []string{indexName},}res, err := req.Do(ctx, e.client)+ defer closeESBody(res)if err != nil {return false, fmt.Errorf("failed to get memory vector mapping: %w", err)}- defer res.Body.Close()if res.StatusCode == http.StatusNotFound {return false, nil@@ -2797,10 +2822,10 @@ func (e *Engine) ensureMemoryMessageVectorMapping(ctx context.Context, indexNameBody: bytes.NewReader(data),}res, err := req.Do(ctx, e.client)+ defer closeESBody(res)if err != nil {return fmt.Errorf("failed to update memory vector mapping: %w", err)}- defer res.Body.Close()if res.IsError() {bodyBytes, _ := io.ReadAll(res.Body)diff --git a/internal/service/nlp/wordnet.go b/internal/service/nlp/wordnet.goindex 297c4998c5..922a571193 100644--- a/internal/service/nlp/wordnet.go+++ b/internal/service/nlp/wordnet.go@@ -14,8 +14,6 @@// limitations under the License.//-// Package wordnet provides a Go implementation of NLTK's WordNet synsets functionality.-// This implementation reads WordNet 3.0 database files and provides synonym set lookup.package nlpimport (@@ -234,6 +232,7 @@ func (wn *WordNet) loadLemmaPosOffsetMap() error {for i := 0; i < nSynsets && fieldIdx < len(fields); i++ {offset, err := strconv.Atoi(fields[fieldIdx])if err != nil {+ fieldIdx++continue}offsets = append(offsets, offset)@@ -246,7 +245,7 @@ func (wn *WordNet) loadLemmaPosOffsetMap() error {}wn.lemmaPosOffsetMap[lemma][filePos] = offsets}- if err := scanner.Err(); err != nil {+ if err = scanner.Err(); err != nil {return fmt.Errorf("error reading %s: %w", filename, err)}}