Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Go: fix resource leak in ES connector (#19912)

Jin Hai1e5fc587f087
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index c99fa737c3..cff83c9f2a 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -589,18 +589,17 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin
"source": fmt.Sprintf("ctx._source.remove(\"%s\");", k),
},
}
- body, _ := json.Marshal(scriptBody)
+ body, _ = json.Marshal(scriptBody)
req := esapi.UpdateRequest{
Index: indexName,
DocumentID: actualID,
Body: bytes.NewReader(body),
}
- res, err := req.Do(ctx, e.client)
+ res, err = req.Do(ctx, e.client)
if err != nil {
common.Warn("Failed to remove feas field", zap.String("field", k), zap.Error(err))
- } else {
- res.Body.Close()
}
+ closeESBody(res)
}
// Remove specific field if removeField is set
@@ -610,18 +609,17 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin
"source": fmt.Sprintf("ctx._source.remove('%s');", removeField),
},
}
- body, _ := json.Marshal(scriptBody)
+ body, _ = json.Marshal(scriptBody)
req := esapi.UpdateRequest{
Index: indexName,
DocumentID: actualID,
Body: bytes.NewReader(body),
}
- res, err := req.Do(ctx, e.client)
+ res, err = req.Do(ctx, e.client)
if err != nil {
common.Warn("Failed to remove field", zap.String("field", removeField), zap.Error(err))
- } else {
- res.Body.Close()
}
+ closeESBody(res)
}
// Remove specific values from array fields (removeDict)
@@ -641,18 +639,17 @@ func (e *Engine) updateSingleChunk(ctx context.Context, indexName, chunkID strin
"params": params,
},
}
- body, _ := json.Marshal(scriptBody)
+ body, _ = json.Marshal(scriptBody)
req := esapi.UpdateRequest{
Index: indexName,
DocumentID: actualID,
Body: bytes.NewReader(body),
}
- res, err := req.Do(ctx, e.client)
+ res, err = req.Do(ctx, e.client)
if err != nil {
common.Warn("Failed to remove dict fields", zap.Error(err))
- } else {
- res.Body.Close()
}
+ closeESBody(res)
}
}
@@ -892,6 +889,17 @@ func sanitizeString(s string) string {
return strings.TrimSpace(s)
}
+// closeESBody releases an ES response body, tolerating the nil response the
+// client returns alongside a transport error. Callers that check `err` first
+// and close only on the success branch leak the body whenever the client ever
+// hands back a non-nil response with an error, so every Do/Search call site
+// funnels its close through here.
+func closeESBody(res *esapi.Response) {
+ if res != nil && res.Body != nil {
+ _ = res.Body.Close()
+ }
+}
+
// copyFields creates a shallow copy of a map
func copyFields(m map[string]interface{}) map[string]interface{} {
result := make(map[string]interface{})
@@ -1434,34 +1442,12 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
// each iteration a fresh bytes.NewReader.
payload := append([]byte(nil), buf.Bytes()...)
for _, indexName := range req.IndexNames {
- res, err := e.client.Search(
- e.client.Search.WithContext(ctx),
- e.client.Search.WithIndex(indexName),
- e.client.Search.WithBody(bytes.NewReader(payload)),
- e.client.Search.WithTrackTotalHits(true),
- )
- if err != nil {
- common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(err))
- continue
- }
- defer res.Body.Close()
-
- if res.IsError() {
- bodyBytes, _ := io.ReadAll(res.Body)
- common.Warn("Elasticsearch error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))
- continue
- }
-
- // Parse response and return results
- var esResp SearchResponse
- if err := json.NewDecoder(res.Body).Decode(&esResp); err != nil {
- common.Warn("Elasticsearch failed to parse response", zap.String("index", indexName), zap.Error(err))
+ searchChunks, indexTotal, esErr := e.searchOneIndex(ctx, indexName, payload)
+ if esErr != nil {
+ common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(esErr))
continue
}
-
- searchChunks := convertESResponse(&esResp, "")
- totalHits += esResp.Hits.Total.Value
-
+ totalHits += indexTotal
allResults = append(allResults, searchChunks...)
}
}
@@ -1495,6 +1481,35 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
}, nil
}
+func (e *Engine) searchOneIndex(ctx context.Context, indexName string, payload []byte) ([]map[string]interface{}, int64, error) {
+ res, err := e.client.Search(
+ e.client.Search.WithContext(ctx),
+ e.client.Search.WithIndex(indexName),
+ e.client.Search.WithBody(bytes.NewReader(payload)),
+ e.client.Search.WithTrackTotalHits(true),
+ )
+ if err != nil {
+ common.Warn("Elasticsearch query failed", zap.String("index", indexName), zap.Error(err))
+ return nil, 0, err
+ }
+ defer res.Body.Close()
+
+ if res.IsError() {
+ bodyBytes, _ := io.ReadAll(res.Body)
+ common.Warn("Elasticsearch error response", zap.String("index", indexName), zap.String("body", string(bodyBytes)))
+ return nil, 0, fmt.Errorf("elasticsearch error response: %s", string(bodyBytes))
+ }
+
+ // Parse response and return results
+ var esResp SearchResponse
+ if err = json.NewDecoder(res.Body).Decode(&esResp); err != nil {
+ common.Warn("Elasticsearch failed to parse response", zap.String("index", indexName), zap.Error(err))
+ return nil, 0, err
+ }
+
+ return convertESResponse(&esResp, ""), esResp.Hits.Total.Value, nil
+}
+
// searchAfterFetcher issues one ES search request with the given batch
// size and search_after cursor, returning the decoded response. Defined
// as a function type so the pagination logic below can be unit-tested
@@ -2175,70 +2190,80 @@ func (e *Engine) GetChunk(ctx context.Context, baseName, chunkID string, dataset
// Try search by doc_id field (which is stored in the document)
for _, datasetID := range datasetIDs {
- searchReq := map[string]interface{}{
- "query": map[string]interface{}{
- "bool": map[string]interface{}{
- "must": []map[string]interface{}{
- {"term": map[string]interface{}{"id": chunkID}},
- {"term": map[string]interface{}{"kb_id": datasetID}},
- },
+ source, found, err := e.searchChunkInDataset(ctx, baseName, chunkID, datasetID)
+ if err != nil {
+ return nil, err
+ }
+ if !found {
+ continue
+ }
+ common.Info("GetChunk found hit", zap.String("baseName", baseName), zap.String("chunkID", chunkID))
+ source["id"] = chunkID
+ return source, nil
+ }
+
+ common.Info("GetChunk no hits found", zap.String("baseName", baseName), zap.String("chunkID", chunkID))
+ return nil, nil
+}
+
+func (e *Engine) searchChunkInDataset(ctx context.Context, baseName, chunkID, datasetID string) (map[string]interface{}, bool, error) {
+ searchReq := map[string]interface{}{
+ "query": map[string]interface{}{
+ "bool": map[string]interface{}{
+ "must": []map[string]interface{}{
+ {"term": map[string]interface{}{"id": chunkID}},
+ {"term": map[string]interface{}{"kb_id": datasetID}},
},
},
- }
+ },
+ }
- body, err := json.Marshal(searchReq)
- if err != nil {
- return nil, fmt.Errorf("failed to marshal search request: %w", err)
- }
+ body, err := json.Marshal(searchReq)
+ if err != nil {
+ return nil, false, fmt.Errorf("failed to marshal search request: %w", err)
+ }
- res, err := e.client.Search(
- e.client.Search.WithContext(ctx),
- e.client.Search.WithIndex(baseName),
- e.client.Search.WithBody(bytes.NewReader(body)),
- )
- if err != nil {
- return nil, fmt.Errorf("failed to search for chunk: %w", err)
- }
+ res, err := e.client.Search(
+ e.client.Search.WithContext(ctx),
+ e.client.Search.WithIndex(baseName),
+ e.client.Search.WithBody(bytes.NewReader(body)),
+ )
- if res.IsError() {
- res.Body.Close()
- return nil, fmt.Errorf("failed to search for chunk: %s", res.Status())
- }
+ defer closeESBody(res)
- var searchResult map[string]interface{}
- if err := json.NewDecoder(res.Body).Decode(&searchResult); err != nil {
- res.Body.Close()
- return nil, fmt.Errorf("failed to parse search response: %w", err)
- }
- res.Body.Close()
+ if err != nil {
+ return nil, false, fmt.Errorf("failed to search for chunk: %w", err)
+ }
- hits, ok := searchResult["hits"].(map[string]interface{})
- if !ok {
- continue
- }
+ if res.IsError() {
+ return nil, false, fmt.Errorf("failed to search for chunk: %s", res.Status())
+ }
- hitList, ok := hits["hits"].([]interface{})
- if !ok || len(hitList) == 0 {
- continue
- }
+ var searchResult map[string]interface{}
+ if err = json.NewDecoder(res.Body).Decode(&searchResult); err != nil {
+ return nil, false, fmt.Errorf("failed to parse search response: %w", err)
+ }
- firstHit, ok := hitList[0].(map[string]interface{})
- if !ok {
- continue
- }
+ hits, ok := searchResult["hits"].(map[string]interface{})
+ if !ok {
+ return nil, false, nil
+ }
- source, ok := firstHit["_source"].(map[string]interface{})
- if !ok {
- continue
- }
+ hitList, ok := hits["hits"].([]interface{})
+ if !ok || len(hitList) == 0 {
+ return nil, false, nil
+ }
- common.Info("GetChunk found hit", zap.String("baseName", baseName), zap.String("chunkID", chunkID))
- source["id"] = chunkID
- return source, nil
+ firstHit, ok := hitList[0].(map[string]interface{})
+ if !ok {
+ return nil, false, nil
}
- common.Info("GetChunk no hits found", zap.String("baseName", baseName), zap.String("chunkID", chunkID))
- return nil, nil
+ source, ok := firstHit["_source"].(map[string]interface{})
+ if !ok {
+ return nil, false, nil
+ }
+ return source, true, nil
}
func (e *Engine) getMemoryMessage(ctx context.Context, indexName, docID string) (interface{}, error) {
@@ -2263,7 +2288,7 @@ func (e *Engine) getMemoryMessage(ctx context.Context, indexName, docID string)
Found bool `json:"found"`
Source map[string]interface{} `json:"_source"`
}
- if err := json.NewDecoder(res.Body).Decode(&getResult); err != nil {
+ if err = json.NewDecoder(res.Body).Decode(&getResult); err != nil {
return nil, fmt.Errorf("failed to parse memory message get response: %w", err)
}
if !getResult.Found || getResult.Source == nil {
@@ -2321,23 +2346,23 @@ func (e *Engine) GetFields(chunks []map[string]interface{}, fields []string) map
}
}
- if _, ok := val.([]interface{}); ok {
+ if _, ok = val.([]interface{}); ok {
m[field] = val
continue
}
if field == "available_int" {
- if _, ok := val.(int); ok {
+ if _, ok = val.(int); ok {
m[field] = val
continue
}
- if _, ok := val.(float64); ok {
+ if _, ok = val.(float64); ok {
m[field] = val
continue
}
}
- if _, ok := val.(string); !ok {
+ if _, ok = val.(string); !ok {
val = fmt.Sprintf("%v", val)
}
m[field] = val
@@ -2730,10 +2755,10 @@ func (e *Engine) memoryMessageVectorMappingExists(ctx context.Context, indexName
Index: []string{indexName},
}
res, err := req.Do(ctx, e.client)
+ defer closeESBody(res)
if err != nil {
return false, fmt.Errorf("failed to get memory vector mapping: %w", err)
}
- defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
return false, nil
@@ -2797,10 +2822,10 @@ func (e *Engine) ensureMemoryMessageVectorMapping(ctx context.Context, indexName
Body: bytes.NewReader(data),
}
res, err := req.Do(ctx, e.client)
+ defer closeESBody(res)
if err != nil {
return fmt.Errorf("failed to update memory vector mapping: %w", err)
}
- defer res.Body.Close()
if res.IsError() {
bodyBytes, _ := io.ReadAll(res.Body)
diff --git a/internal/service/nlp/wordnet.go b/internal/service/nlp/wordnet.go
index 297c4998c5..922a571193 100644
--- a/internal/service/nlp/wordnet.go
+++ b/internal/service/nlp/wordnet.go
@@ -14,8 +14,6 @@
// limitations under the License.
//
-// Package wordnet provides a Go implementation of NLTK's WordNet synsets functionality.
-// This implementation reads WordNet 3.0 database files and provides synonym set lookup.
package nlp
import (
@@ -234,6 +232,7 @@ func (wn *WordNet) loadLemmaPosOffsetMap() error {
for i := 0; i < nSynsets && fieldIdx < len(fields); i++ {
offset, err := strconv.Atoi(fields[fieldIdx])
if err != nil {
+ fieldIdx++
continue
}
offsets = append(offsets, offset)
@@ -246,7 +245,7 @@ func (wn *WordNet) loadLemmaPosOffsetMap() error {
}
wn.lemmaPosOffsetMap[lemma][filePos] = offsets
}
- if err := scanner.Err(); err != nil {
+ if err = scanner.Err(); err != nil {
return fmt.Errorf("error reading %s: %w", filename, err)
}
}