Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: filter chunks by requested IDs (#19377)

buua436afb01aa887ff
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 72e571d772..ecdad76dd8 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -1865,6 +1865,11 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
map[string]interface{}{"terms": map[string]interface{}{"id": listVal}},
map[string]interface{}{"terms": map[string]interface{}{"_id": listVal}},
)
+ } else if strListVal, ok := v.([]string); ok && len(strListVal) > 0 {
+ shouldClauses = append(shouldClauses,
+ map[string]interface{}{"terms": map[string]interface{}{"id": strListVal}},
+ map[string]interface{}{"terms": map[string]interface{}{"_id": strListVal}},
+ )
} else if strVal, ok := v.(string); ok && strVal != "" {
shouldClauses = append(shouldClauses,
map[string]interface{}{"term": map[string]interface{}{"id": strVal}},
diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.go
index 119da8daab..61952e3fbc 100644
--- a/internal/engine/elasticsearch/chunk_test.go
+++ b/internal/engine/elasticsearch/chunk_test.go
@@ -424,6 +424,10 @@ func TestBuildBoolQueryFromConditionIDFilter(t *testing.T) {
"id": []interface{}{"a", "b", "c"},
}, []string{"id", "_id"})
+ check("string_list_value", map[string]interface{}{
+ "id": []string{"a", "b", "c"},
+ }, []string{"id", "_id"})
+
check("string_value", map[string]interface{}{
"id": "doc-42",
}, []string{"id", "_id"})
diff --git a/internal/handler/chunk.go b/internal/handler/chunk.go
index 5cc4aa170c..7013b1ea9d 100644
--- a/internal/handler/chunk.go
+++ b/internal/handler/chunk.go
@@ -235,6 +235,7 @@ func (h *ChunkHandler) ListChunks(c *gin.Context) {
req := service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
+ ChunkIDs: queryStringList(c, "chunk_ids"),
Page: &page,
Size: &size,
Keywords: c.Query("keywords"),
@@ -258,6 +259,26 @@ func (h *ChunkHandler) ListChunks(c *gin.Context) {
common.SuccessWithData(c, resp, "success")
}
+func queryStringList(c *gin.Context, name string) []string {
+ values := c.QueryArray(name)
+ seen := make(map[string]struct{}, len(values))
+ ids := make([]string, 0, len(values))
+ for _, value := range values {
+ for _, item := range strings.Split(value, ",") {
+ item = strings.TrimSpace(item)
+ if item == "" {
+ continue
+ }
+ if _, ok := seen[item]; ok {
+ continue
+ }
+ seen[item] = struct{}{}
+ ids = append(ids, item)
+ }
+ }
+ return ids
+}
+
func parsePositiveQueryInt(c *gin.Context, name string, defaultValue int) (int, error) {
raw := strings.TrimSpace(c.Query(name))
if raw == "" {
diff --git a/internal/handler/chunk_test.go b/internal/handler/chunk_test.go
index 15f578879f..03679f8d44 100644
--- a/internal/handler/chunk_test.go
+++ b/internal/handler/chunk_test.go
@@ -155,6 +155,9 @@ func TestChunkHandlerListChunksMapsPathAndQuery(t *testing.T) {
if req.Keywords != "AI" {
t.Fatalf("keywords = %q, want AI", req.Keywords)
}
+ if !reflect.DeepEqual(req.ChunkIDs, []string{"chunk-1", "chunk-2", "chunk-3"}) {
+ t.Fatalf("chunk_ids = %#v, want %#v", req.ChunkIDs, []string{"chunk-1", "chunk-2", "chunk-3"})
+ }
if req.AvailableInt == nil || *req.AvailableInt != 1 {
t.Fatalf("available_int = %v, want 1", req.AvailableInt)
}
@@ -167,7 +170,7 @@ func TestChunkHandlerListChunksMapsPathAndQuery(t *testing.T) {
}, nil
}
- req := httptest.NewRequest(http.MethodGet, "/api/v1/datasets/kb-1/documents/doc-1/chunks?page=2&page_size=5&keywords=AI&available=true", nil)
+ req := httptest.NewRequest(http.MethodGet, "/api/v1/datasets/kb-1/documents/doc-1/chunks?page=2&page_size=5&keywords=AI&available=true&chunk_ids=chunk-1&chunk_ids=chunk-2%2Cchunk-3&chunk_ids=chunk-1", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index d0ec0564b4..898fbf024e 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -901,6 +901,9 @@ func (s *ChunkService) List(ctx context.Context, req *service.ListChunksRequest,
"must_not": map[string]interface{}{"exists": "compile_kwd"},
},
}
+ if len(req.ChunkIDs) > 0 {
+ searchReq.Filter["id"] = req.ChunkIDs
+ }
// Add available_int filter if specified
if req.AvailableInt != nil {
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index 8dad50c3cc..c96956ed39 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -383,6 +383,7 @@ func TestListBuildsMatchTextExprForKeywords(t *testing.T) {
resp, err := svc.List(ctx, &service.ListChunksRequest{
DatasetID: datasetID,
DocID: documentID,
+ ChunkIDs: []string{"chunk-1", "chunk-2"},
Page: &page,
Size: &size,
Keywords: " invoice terms ",
@@ -405,6 +406,9 @@ func TestListBuildsMatchTextExprForKeywords(t *testing.T) {
if got := engine.searchReq.Filter["doc_id"]; got != documentID {
t.Fatalf("doc_id filter = %#v, want %q", got, documentID)
}
+ if got := engine.searchReq.Filter["id"]; !reflect.DeepEqual(got, []string{"chunk-1", "chunk-2"}) {
+ t.Fatalf("id filter = %#v, want %#v", got, []string{"chunk-1", "chunk-2"})
+ }
mustNot, ok := engine.searchReq.Filter["must_not"].(map[string]interface{})
if !ok || mustNot["exists"] != "compile_kwd" {
t.Fatalf("must_not filter = %#v, want compile_kwd existence exclusion", engine.searchReq.Filter["must_not"])
diff --git a/internal/service/chunk_types.go b/internal/service/chunk_types.go
index d8d86afeee..3dd01f2433 100644
--- a/internal/service/chunk_types.go
+++ b/internal/service/chunk_types.go
@@ -329,12 +329,13 @@ func (s *ChunkService) StopParsing(ctx context.Context, userID, datasetID string
// ListChunksRequest request for listing chunks
type ListChunksRequest struct {
- DatasetID string `json:"dataset_id,omitempty"`
- DocID string `json:"doc_id" binding:"required"`
- Page *int `json:"page,omitempty"`
- Size *int `json:"size,omitempty"`
- Keywords string `json:"keywords,omitempty"`
- AvailableInt *int `json:"available_int,omitempty"`
+ DatasetID string `json:"dataset_id,omitempty"`
+ DocID string `json:"doc_id" binding:"required"`
+ ChunkIDs []string `json:"chunk_ids,omitempty"`
+ Page *int `json:"page,omitempty"`
+ Size *int `json:"size,omitempty"`
+ Keywords string `json:"keywords,omitempty"`
+ AvailableInt *int `json:"available_int,omitempty"`
}
// ListChunksResponse response for listing chunks