RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor: do not include compliation chunks in retrieval testing (#18364)
Wang Qi57b907cd1ef1
diff --git a/api/apps/restful_apis/dataset_api.py b/api/apps/restful_apis/dataset_api.pyindex 4cc0f969c8..341f01b683 100644--- a/api/apps/restful_apis/dataset_api.py+++ b/api/apps/restful_apis/dataset_api.py@@ -515,7 +515,7 @@ async def search_datasets(tenant_id):POST /api/v1/datasets/searchJSON body: {"dataset_ids": list[str] (required), "question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int,"similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,- "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict}+ "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors."""@@ -538,7 +538,7 @@ async def search(tenant_id, dataset_id):POST /api/v1/datasets/<dataset_id>/searchJSON body: {"question": str (required), "doc_ids": list[str], "top_k": int, "page": int, "size": int,"similarity_threshold": float, "vector_similarity_weight": float, "use_kg": bool,- "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict}+ "cross_languages": list[str], "keyword": bool, "meta_data_filter": dict, "include_knowledge_compilation": bool (default true)}Success: {"code": 0, "data": {"chunks": [...], "total": int, "labels": [...]}}Errors: ARGUMENT_ERROR (101) for invalid payload; DATA_ERROR (102) for access denied or internal errors."""diff --git a/api/apps/services/dataset_api_service.py b/api/apps/services/dataset_api_service.pyindex a5cb969f41..f71b973bc3 100644--- a/api/apps/services/dataset_api_service.py+++ b/api/apps/services/dataset_api_service.py@@ -1530,6 +1530,7 @@ async def search_datasets(tenant_id: str, req: dict):rerank_mdl=rerank_mdl,rank_feature=labels,trace_id=search_id,+ must_not=None if req.get("include_knowledge_compilation", True) else {"exists": "compile_kwd"},)if use_kg:diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.pyindex ddb455762c..054c6f19c5 100644--- a/api/utils/validation_utils.py+++ b/api/utils/validation_utils.py@@ -992,6 +992,7 @@ class SearchDatasetReq(BaseModel):rerank_id: Annotated[str | None, Field(default=None)]tenant_rerank_id: Annotated[str | None, Field(default=None)]meta_data_filter: Annotated[dict | None, Field(default=None)]+ include_knowledge_compilation: Annotated[bool, Field(default=True)]class SearchDatasetsReq(BaseModel):@@ -1014,6 +1015,7 @@ class SearchDatasetsReq(BaseModel):rerank_id: Annotated[str | None, Field(default=None)]tenant_rerank_id: Annotated[str | None, Field(default=None)]meta_data_filter: Annotated[dict | None, Field(default=None)]+ include_knowledge_compilation: Annotated[bool, Field(default=True)]class BaseListReq(BaseModel):diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex ae98fa1371..9255d1f29c 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -1679,6 +1679,7 @@ func memoryMessageStatusBool(value interface{}) bool {// message indexes use memory_id plus message-specific storage fields.func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {var mustClauses []interface{}+ var mustNotClauses []interface{}var filterClauses []interface{}var shouldClauses []interface{}@@ -1753,6 +1754,14 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,}continue}+ if k == "must_not" {+ if condition, ok := v.(map[string]interface{}); ok {+ if field, ok := condition["exists"].(string); ok && field != "" {+ mustNotClauses = append(mustNotClauses, map[string]interface{}{"exists": map[string]interface{}{"field": field}})+ }+ }+ continue+ }if k == "id" {if v == nil || v == "" {continue@@ -1818,6 +1827,9 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,if len(mustClauses) > 0 {boolQuery["must"] = mustClauses}+ if len(mustNotClauses) > 0 {+ boolQuery["must_not"] = mustNotClauses+ }if len(filterClauses) > 0 {boolQuery["filter"] = filterClauses}diff --git a/internal/handler/dataset_search_test.go b/internal/handler/dataset_search_test.goindex 9eaf94773c..2163da217e 100644--- a/internal/handler/dataset_search_test.go+++ b/internal/handler/dataset_search_test.go@@ -157,7 +157,7 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {h := &DatasetsHandler{searchDatasetsService: fake}rec := httptest.NewRecorder()- req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7}`))+ req := httptest.NewRequest(http.MethodPost, "/api/v1/datasets/search", strings.NewReader(`{"question":" hello ","dataset_ids":["ds-1"],"top_k":7,"include_knowledge_compilation":false}`))req.Header.Set("Content-Type", "application/json")c, _ := gin.CreateTestContext(rec)c.Request = req@@ -171,6 +171,9 @@ func TestDatasetsHandlerSearchDatasetsSuccess(t *testing.T) {if fake.userID != "user-1" || fake.req == nil || fake.req.Question != "hello" || len(fake.req.DatasetIDs) != 1 || fake.req.DatasetIDs[0] != "ds-1" {t.Fatalf("call args userID=%q req=%#v", fake.userID, fake.req)}+ if fake.req.IncludeCompiledChunks == nil || *fake.req.IncludeCompiledChunks {+ t.Fatalf("include_knowledge_compilation=%v want false", fake.req.IncludeCompiledChunks)+ }body := decodeSearchResponse(t, rec)if body["code"] != float64(common.CodeSuccess) {t.Fatalf("code=%v want=%d body=%s", body["code"], common.CodeSuccess, rec.Body.String())diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 4394af649e..5549365d5a 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -270,6 +270,9 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.SearchRankFeature: &labels,EmbeddingModel: embeddingModel,}+ if req.IncludeCompiledChunks != nil && !*req.IncludeCompiledChunks {+ retrievalReq.Filter = map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}}+ }retrievalResult, err := nlp.NewRetrievalService(d.docEngine, d.documentDAO).Retrieval(ctx, retrievalReq)if err != nil {diff --git a/internal/service/dataset/search_test.go b/internal/service/dataset/search_test.goindex 1b90d535d6..b898964d6a 100644--- a/internal/service/dataset/search_test.go+++ b/internal/service/dataset/search_test.go@@ -16,6 +16,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {vectorSimilarityWeight := 0.8searchID := "search-1"rerankID := "rerank-1"+ includeKnowledgeCompilation := falsereq := &service.SearchDatasetRequest{Question: "hello world",Page: &page,@@ -30,6 +31,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {Keyword: &keyword,SimilarityThreshold: &similarityThreshold,VectorSimilarityWeight: &vectorSimilarityWeight,+ IncludeCompiledChunks: &includeKnowledgeCompilation,}converted := req.ToSearchDatasetsRequest("dataset-1")@@ -51,4 +53,7 @@ func TestSearchDatasetRequestToSearchDatasetsRequest(t *testing.T) {if converted.SimilarityThreshold != req.SimilarityThreshold || converted.VectorSimilarityWeight != req.VectorSimilarityWeight {t.Fatalf("converted request did not preserve thresholds: %#v", converted)}+ if converted.IncludeCompiledChunks != req.IncludeCompiledChunks {+ t.Fatalf("converted request did not preserve include_knowledge_compilation: %#v", converted)+ }}diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.goindex 0864d66aa1..6d61253eba 100644--- a/internal/service/dataset_types.go+++ b/internal/service/dataset_types.go@@ -51,6 +51,7 @@ type SearchDatasetsRequest struct {Keyword *bool `json:"keyword,omitempty"`SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"`VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`+ IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`ForceRefresh bool `json:"force_refresh"`}@@ -77,6 +78,7 @@ type SearchDatasetRequest struct {Keyword *bool `json:"keyword,omitempty"`SimilarityThreshold *float64 `json:"similarity_threshold,omitempty"`VectorSimilarityWeight *float64 `json:"vector_similarity_weight,omitempty"`+ IncludeCompiledChunks *bool `json:"include_knowledge_compilation,omitempty"`}// ToSearchDatasetsRequest converts a single-dataset search request into the multi-dataset form.@@ -99,6 +101,7 @@ func (req *SearchDatasetRequest) ToSearchDatasetsRequest(datasetID string) *SearKeyword: req.Keyword,SimilarityThreshold: req.SimilarityThreshold,VectorSimilarityWeight: req.VectorSimilarityWeight,+ IncludeCompiledChunks: req.IncludeCompiledChunks,}}diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 50ae32467f..63b5a21653 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -61,6 +61,7 @@ type RetrievalRequest struct {EmbeddingModel *models.EmbeddingModelAggs *boolHighlight *bool+ Filter map[string]interface{}}// RetrievalResult result from retrieval search@@ -145,6 +146,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)Top: *req.Top,RankFeature: *req.RankFeature,EmbeddingModel: req.EmbeddingModel,+ Filter: req.Filter,}searchResult, err := s.Search(ctx, searchReq)if err != nil {@@ -434,6 +436,7 @@ func (s *RetrievalService) countThresholdValidMatches(ctx context.Context, req *Top: *req.Top,RankFeature: *req.RankFeature,EmbeddingModel: req.EmbeddingModel,+ Filter: req.Filter,}searchResult, err := s.Search(ctx, searchReq)if err != nil {diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 531616ecb0..1cb7f782d1 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -41,6 +41,7 @@ func TestRetrievalTotalCountsThresholdValidMatchesBeyondRerankWindow(t *testing.SimilarityThreshold: &threshold,VectorSimilarityWeight: &vectorWeight,Aggs: &aggs,+ Filter: map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},})if err != nil {t.Fatalf("Retrieval failed: %v", err)@@ -54,15 +55,23 @@ func TestRetrievalTotalCountsThresholdValidMatchesBeyondRerankWindow(t *testing.if len(engine.searchLimits) != 2 || engine.searchLimits[0] != 70 || engine.searchLimits[1] != 100 {t.Fatalf("search limits = %v, want [70 100]", engine.searchLimits)}+ for _, filters := range engine.searchFilters {+ mustNot, ok := filters["must_not"].(map[string]interface{})+ if !ok || mustNot["exists"] != "compile_kwd" {+ t.Fatalf("must_not filter = %#v", filters["must_not"])+ }+ }}type retrievalCountEngine struct {- rows []map[string]interface{}- searchLimits []int+ rows []map[string]interface{}+ searchLimits []int+ searchFilters []map[string]interface{}}func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {e.searchLimits = append(e.searchLimits, req.Limit)+ e.searchFilters = append(e.searchFilters, req.Filter)offset := req.Offsetif offset > len(e.rows) {offset = len(e.rows)diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.tsindex e35c38236a..1302179ea2 100644--- a/web/src/hooks/use-knowledge-request.ts+++ b/web/src/hooks/use-knowledge-request.ts@@ -136,6 +136,7 @@ export const useTestRetrieval = () => {page: 1,doc_ids: filterValue.doc_ids,highlight: true,+ include_knowledge_compilation: false,};}, [filterValue, knowledgeBaseId, values]);