RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
test(retrieval) : add unit tests for retrieval and reranker (#21611)
ZZZZY7df338422fa1
diff --git a/internal/handler/mcp_retrieval.go b/internal/handler/mcp_retrieval.goindex d885f7d7d8..25a820c808 100644--- a/internal/handler/mcp_retrieval.go+++ b/internal/handler/mcp_retrieval.go@@ -9,7 +9,6 @@ import ("ragflow/internal/dao""ragflow/internal/mcp""ragflow/internal/service"- dataset "ragflow/internal/service/dataset""ragflow/internal/service/document")@@ -36,10 +35,19 @@ func validateRetrievalWindow(page, pageSize int) error {return nil}+// mcpDatasetService is the dataset surface mcpRetrieval depends on. It exists so+// the MCP retrieval entry point can be tested against a recording fake without+// standing up the full DatasetService.+type mcpDatasetService interface {+ ListDatasets(ctx context.Context, id, name string, page, pageSize int, terms []dao.OrderTerm, keywords string, ownerIDs []string, parserID, userID string, ids []string, canvasID string) ([]map[string]interface{}, int64, common.ErrorCode, error)+ SearchDatasets(ctx context.Context, req *service.SearchDatasetsRequest, userID string) (*service.SearchDatasetsResponse, error)+ GetDataset(ctx context.Context, datasetID, userID string) (map[string]interface{}, common.ErrorCode, error)+}+// mcpRetrieval executes a retrieval request on behalf of the MCP tool handler.// It translates the mcp.RetrievalRequest into a service.SearchDatasetsRequest// and calls DatasetService.SearchDatasets. The result is serialized as JSON.-func mcpRetrieval(ctx context.Context, ds *dataset.DatasetService, userID string, req mcp.RetrievalRequest) (string, error) {+func mcpRetrieval(ctx context.Context, ds mcpDatasetService, userID string, req mcp.RetrievalRequest) (string, error) {if err := validateRetrievalWindow(req.Page, req.PageSize); err != nil {return "", err}diff --git a/internal/handler/mcp_retrieval_test.go b/internal/handler/mcp_retrieval_test.gonew file mode 100644index 0000000000..69e02ff884--- /dev/null+++ b/internal/handler/mcp_retrieval_test.go@@ -0,0 +1,230 @@+package handler++import (+ "context"+ "encoding/json"+ "errors"+ "math"+ "slices"+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/ingestion/knowledge_compile"+ "ragflow/internal/mcp"+ "ragflow/internal/service"+)++// recordingDatasetService records every call mcpRetrieval makes downstream. It+// implements each method explicitly (rather than embedding a nil service) so+// calls are counted even on paths that never reach them.+type recordingDatasetService struct {+ listCalls int+ searchCalls int+ getCalls int++ listPages []int+ listSizes []int+ request *service.SearchDatasetsRequest+ searchUser string+ getIDs []string++ listRows []map[string]interface{}+ response *service.SearchDatasetsResponse+ searchErr error+}++func (f *recordingDatasetService) ListDatasets(_ context.Context, _, _ string, page, pageSize int, _ []dao.OrderTerm, _ string, _ []string, _, _ string, _ []string, _ string) ([]map[string]interface{}, int64, common.ErrorCode, error) {+ f.listCalls+++ f.listPages = append(f.listPages, page)+ f.listSizes = append(f.listSizes, pageSize)+ return f.listRows, int64(len(f.listRows)), common.CodeSuccess, nil+}++func (f *recordingDatasetService) SearchDatasets(_ context.Context, req *service.SearchDatasetsRequest, userID string) (*service.SearchDatasetsResponse, error) {+ f.searchCalls+++ f.request = req+ f.searchUser = userID+ if f.searchErr != nil {+ return nil, f.searchErr+ }+ if f.response == nil {+ return &service.SearchDatasetsResponse{}, nil+ }+ return f.response, nil+}++func (f *recordingDatasetService) GetDataset(_ context.Context, datasetID, _ string) (map[string]interface{}, common.ErrorCode, error) {+ f.getCalls+++ f.getIDs = append(f.getIDs, datasetID)+ return map[string]interface{}{"name": "dataset-" + datasetID}, common.CodeSuccess, nil+}++// isolateKnowledgeCompilePublisher restores the process-wide publisher that the+// success path installs through document.NewDocumentService.+func isolateKnowledgeCompilePublisher(t *testing.T) {+ t.Helper()+ prev := knowledge_compile.DefaultPublisher()+ t.Cleanup(func() { knowledge_compile.SetScheduler(prev) })+}++func TestMCPRetrievalUsesFixedWindowAndForwardsPagination(t *testing.T) {+ cases := []struct {+ name string+ page int+ pageSize int+ wantPage *int+ wantSize *int+ wantJSONPg int+ wantJSONPS int+ }{+ {name: "first page", page: 1, pageSize: 10, wantPage: new(1), wantSize: new(10), wantJSONPg: 1, wantJSONPS: 10},+ {name: "half-full page", page: 1, pageSize: 50, wantPage: new(1), wantSize: new(50), wantJSONPg: 1, wantJSONPS: 50},+ {name: "mid page", page: 2, pageSize: 50, wantPage: new(2), wantSize: new(50), wantJSONPg: 2, wantJSONPS: 50},+ {name: "last full page inside window", page: 7, pageSize: 10, wantPage: new(7), wantSize: new(10), wantJSONPg: 7, wantJSONPS: 10},+ {name: "schema max page size", page: 1, pageSize: 100, wantPage: new(1), wantSize: new(100), wantJSONPg: 1, wantJSONPS: 100},+ {name: "exact window boundary is allowed", page: 8, pageSize: 64, wantPage: new(8), wantSize: new(64), wantJSONPg: 8, wantJSONPS: 64},+ {name: "unset pagination is forwarded as nil", page: 0, pageSize: 0, wantJSONPg: 1, wantJSONPS: 30},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ isolateKnowledgeCompilePublisher(t)+ probe := &recordingDatasetService{}++ result, err := mcpRetrieval(t.Context(), probe, "user-1", mcp.RetrievalRequest{+ DatasetIDs: []string{"ds-1"},+ DocumentIDs: []string{"doc-1"},+ Question: "alpha",+ Page: tc.page,+ PageSize: tc.pageSize,+ })+ if err != nil {+ t.Fatalf("mcpRetrieval failed: %v", err)+ }+ if probe.searchCalls != 1 {+ t.Fatalf("SearchDatasets calls = %d, want 1", probe.searchCalls)+ }+ if probe.listCalls != 0 {+ t.Fatalf("ListDatasets calls = %d, want 0 when dataset IDs are given", probe.listCalls)+ }+ got := probe.request+ if got == nil {+ t.Fatal("SearchDatasets did not receive a request")+ }+ if got.RerankCandidatesCount == nil || *got.RerankCandidatesCount != 512 {+ t.Fatalf("unexpected rerank candidate count: %#v", got.RerankCandidatesCount)+ }+ if !sameIntPtr(got.Page, tc.wantPage) {+ t.Fatalf("page = %#v, want %#v", got.Page, tc.wantPage)+ }+ if !sameIntPtr(got.PageSize, tc.wantSize) {+ t.Fatalf("page size = %#v, want %#v", got.PageSize, tc.wantSize)+ }+ if !slices.Equal(got.DatasetIDs, []string{"ds-1"}) {+ t.Fatalf("dataset IDs = %v, want [ds-1]", got.DatasetIDs)+ }+ if !slices.Equal(got.DocumentIDs, []string{"doc-1"}) {+ t.Fatalf("document IDs = %v, want [doc-1]", got.DocumentIDs)+ }+ if got.Question != "alpha" {+ t.Fatalf("question = %q, want alpha", got.Question)+ }+ if probe.searchUser != "user-1" {+ t.Fatalf("search userID = %q, want user-1", probe.searchUser)+ }+ if probe.getCalls != 1 || !slices.Equal(probe.getIDs, []string{"ds-1"}) {+ t.Fatalf("GetDataset calls = %d (%v), want 1 ([ds-1])", probe.getCalls, probe.getIDs)+ }++ var payload map[string]interface{}+ if err := json.Unmarshal([]byte(result), &payload); err != nil {+ t.Fatalf("result is not valid JSON: %v (%q)", err, result)+ }+ pagination, ok := payload["pagination"].(map[string]interface{})+ if !ok {+ t.Fatalf("pagination missing from result: %#v", payload)+ }+ if pagination["page"] != float64(tc.wantJSONPg) || pagination["page_size"] != float64(tc.wantJSONPS) {+ t.Fatalf("pagination = %#v, want page=%d page_size=%d", pagination, tc.wantJSONPg, tc.wantJSONPS)+ }+ if pagination["total_chunks"] != float64(0) {+ t.Fatalf("total_chunks = %#v, want 0", pagination["total_chunks"])+ }+ })+ }+}++func TestMCPRetrievalRejectsOutOfWindowBeforeAnyDownstreamCall(t *testing.T) {+ cases := []struct {+ name string+ page int+ pageSize int+ datasetIDs []string+ }{+ {name: "one past the boundary", page: 513, pageSize: 1, datasetIDs: []string{"ds-1"}},+ {name: "common overflow", page: 6, pageSize: 100, datasetIDs: []string{"ds-1"}},+ {name: "multiplication overflow", page: math.MaxInt, pageSize: math.MaxInt, datasetIDs: []string{"ds-1"}},+ {name: "no dataset ids", page: 513, pageSize: 1},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ probe := &recordingDatasetService{}++ result, err := mcpRetrieval(t.Context(), probe, "user-1", mcp.RetrievalRequest{+ DatasetIDs: tc.datasetIDs,+ Question: "alpha",+ Page: tc.page,+ PageSize: tc.pageSize,+ })+ if err == nil || !strings.Contains(err.Error(), "exceeds the fixed rerank candidate window") {+ t.Fatalf("unexpected error: %v", err)+ }+ if result != "" {+ t.Fatalf("unexpected result on rejection: %q", result)+ }+ if probe.listCalls != 0 || probe.searchCalls != 0 || probe.getCalls != 0 {+ t.Fatalf("out-of-window request invoked downstream services: list=%d search=%d get=%d",+ probe.listCalls, probe.searchCalls, probe.getCalls)+ }+ })+ }+}++func TestMCPRetrievalPropagatesSearchError(t *testing.T) {+ isolateKnowledgeCompilePublisher(t)+ sentinel := errors.New("search exploded")+ probe := &recordingDatasetService{searchErr: sentinel}++ result, err := mcpRetrieval(t.Context(), probe, "user-1", mcp.RetrievalRequest{+ DatasetIDs: []string{"ds-1"},+ Question: "alpha",+ Page: 1,+ PageSize: 10,+ })+ if !errors.Is(err, sentinel) {+ t.Fatalf("error = %v, want the downstream sentinel", err)+ }+ if result != "" {+ t.Fatalf("unexpected result on failure: %q", result)+ }+ if probe.searchCalls != 1 {+ t.Fatalf("SearchDatasets calls = %d, want 1", probe.searchCalls)+ }+ if probe.getCalls != 0 {+ t.Fatalf("GetDataset calls = %d, want 0 after a search error", probe.getCalls)+ }+ if got := probe.request; got == nil ||+ got.RerankCandidatesCount == nil || *got.RerankCandidatesCount != 512 {+ t.Fatalf("search request not populated before the error: %#v", got)+ }+}++func sameIntPtr(got, want *int) bool {+ if got == nil || want == nil {+ return got == nil && want == nil+ }+ return *got == *want+}diff --git a/internal/service/nlp/reranker_question_tks_test.go b/internal/service/nlp/reranker_question_tks_test.goindex 0a1edfead6..6a867e6f21 100644--- a/internal/service/nlp/reranker_question_tks_test.go+++ b/internal/service/nlp/reranker_question_tks_test.go@@ -75,6 +75,56 @@ func fullChunk() map[string]interface{} {}}+func TestRerankStandardScoresQuestionOnlyMatch(t *testing.T) {+ chunks := []map[string]interface{}{+ {"content_ltks": "alpha", "question_tks": "quux"},+ {"content_ltks": "alpha"},+ }++ _, tsim, _ := RerankStandard(+ chunks, []string{"quux"}, nil, "quux", 1, 0,+ "content_ltks", NewQueryBuilder(), nil, "",+ )++ if len(tsim) != 2 {+ t.Fatalf("term similarities = %v, want two scores", tsim)+ }+ if !(tsim[0] > 0.5 && tsim[0] <= 1) {+ t.Errorf("question-only match term similarity = %v, want a positive match above 0.5", tsim[0])+ }+ if !(tsim[1] >= 0 && tsim[1] < 1e-6) {+ t.Errorf("term similarity without question_tks = %v, want near zero", tsim[1])+ }+}++func TestRerankWithKNNIncludesQuestionTks(t *testing.T) {+ chunks := []map[string]interface{}{+ {"content_ltks": "alpha", "question_tks": "quux"},+ {"content_ltks": "alpha"},+ }+ ids := []string{"with-question", "without-question"}+ field := map[string]map[string]interface{}{+ ids[0]: chunks[0],+ ids[1]: chunks[1],+ }++ _, tsim, _ := RerankWithKNN(+ t.Context(), chunks, ids, field,+ map[string]float64{ids[0]: 0.5, ids[1]: 0.5},+ "quux", 1, 0, "content_ltks", NewQueryBuilder(), nil, "",+ )++ if len(tsim) != 2 {+ t.Fatalf("term similarities = %v, want two scores", tsim)+ }+ if !(tsim[0] > 0.5 && tsim[0] <= 1) {+ t.Errorf("question-only match term similarity = %v, want a positive match above 0.5", tsim[0])+ }+ if !(tsim[1] >= 0 && tsim[1] < 1e-6) {+ t.Errorf("term similarity without question_tks = %v, want near zero", tsim[1])+ }+}+// TestRerankByModelIncludesQuestionTks pins the fix: RerankStandard and// RerankWithKNN both fold question_tks into their token lists, so a chunk// whose generated questions carry the match was scored on those tokens on@@ -161,3 +211,76 @@ func TestRerankByModelFallsBackToTokensWithoutNaturalText(t *testing.T) {}}}++// TestRerankByModelUsesTokenizedFieldsForTermSimilarity pins the split between+// the two consumers in RerankByModel: the cross-encoder scores the natural text,+// while term similarity must keep reading the tokenized content_ltks. Both+// chunks carry the same natural text containing the query term, so only the+// tokenized field can separate them. Weights are pinned to term similarity+// alone (tkWeight=1, vtWeight=0, no rank features) so neither the model score+// nor pagerank can mask a wrong field choice.+func TestRerankByModelUsesTokenizedFieldsForTermSimilarity(t *testing.T) {+ const naturalText = "quux appears in the natural text"+ chunks := []map[string]interface{}{+ {"content_with_weight": naturalText, "content_ltks": "quux"},+ {"content_with_weight": naturalText, "content_ltks": "alpha"},+ }+ ids := []string{"with-quux", "without-quux"}+ field := map[string]map[string]interface{}{+ ids[0]: chunks[0],+ ids[1]: chunks[1],+ }+ driver := &captureDriver{}++ sim, tsim, _, err := RerankByModel(+ t.Context(),+ models.NewRerankModel(driver, nil, nil, 0),+ chunks,+ ids,+ field,+ "quux",+ 1,+ 0,+ "content_ltks",+ NewQueryBuilder(),+ nil,+ "",+ )+ if err != nil {+ t.Fatalf("RerankByModel failed: %v", err)+ }+ if len(tsim) != 2 {+ t.Fatalf("term similarities = %v, want two scores", tsim)+ }+ if len(sim) != 2 {+ t.Fatalf("similarities = %v, want two scores", sim)+ }+ if len(driver.documents) != 2 {+ t.Fatalf("model documents = %v, want two documents", driver.documents)+ }+ for i, doc := range driver.documents {+ if doc != naturalText {+ t.Errorf("model document[%d] = %q, want unchanged natural text %q", i, doc, naturalText)+ }+ }+ if !(tsim[0] > 0.5 && tsim[0] <= 1) {+ t.Errorf("term similarity for the matching content_ltks = %v, want a positive match above 0.5", tsim[0])+ }+ if !(tsim[1] >= 0 && tsim[1] < 1e-6) {+ t.Errorf("term similarity for the non-matching content_ltks = %v, want near zero", tsim[1])+ }+}++// TestRerankByModelCleansTokenizedFallback pins the fallback formatting: when no+// natural text exists, the joined tokens are passed through+// common.RemoveRedundantSpaces so punctuation does not keep the spaces its+// tokenization introduced.+func TestRerankByModelCleansTokenizedFallback(t *testing.T) {+ doc := rerankOneChunk(t, map[string]interface{}{+ "content_ltks": "( alpha ) beta !",+ })++ if want := "(alpha) beta!"; doc != want {+ t.Errorf("got %q, want %q", doc, want)+ }+}diff --git a/internal/service/nlp/reranker_rank_feature_test.go b/internal/service/nlp/reranker_rank_feature_test.goindex 37ccd3dd66..3e4f06faf9 100644--- a/internal/service/nlp/reranker_rank_feature_test.go+++ b/internal/service/nlp/reranker_rank_feature_test.go@@ -21,6 +21,141 @@ import ("ragflow/internal/common")+// rankFeatureCase drives the same document shape through both rank-feature+// entry points. The document PageRank is fixed at 0.25 by chunk(); the query+// weight for common.PAGERANK_FLD is deliberately large (7.0) so using it as a+// multiplier on the document PageRank would produce a visibly wrong score.+type rankFeatureCase struct {+ name string+ tags interface{}+ rankFeature map[string]float64+ base float64+ want float64+}++func (tc rankFeatureCase) chunk() map[string]interface{} {+ return map[string]interface{}{+ common.TAG_FLD: tc.tags,+ common.PAGERANK_FLD: 0.25,+ }+}++func rankFeatureCases() []rankFeatureCase {+ weightedTags := map[string]float64{"tag_a": 2.0, "tag_b": 1.0, common.PAGERANK_FLD: 7.0}+ // tag_score = (2*0.6 + 1*0.8) / |(0.6, 0.8)| / |(2, 1)|+ // = 2.0 / 1.0 / sqrt(5)+ // final = base + tag_score*10 + docPagerank+ tagged := 0.3 + (2.0/math.Sqrt(5.0))*10 + 0.25+ baseOnly := 0.3 + 0.25++ return []rankFeatureCase{+ {+ name: "json_string_tag_feas",+ tags: `{"tag_a": 0.6, "tag_b": 0.8}`,+ rankFeature: weightedTags,+ base: 0.3,+ want: tagged,+ },+ {+ name: "go_map_tag_feas",+ tags: map[string]interface{}{"tag_a": 0.6, "tag_b": 0.8},+ rankFeature: weightedTags,+ base: 0.3,+ want: tagged,+ },+ {+ name: "invalid_json_tag_feas",+ tags: `{"tag_a": 0.6,`,+ rankFeature: weightedTags,+ base: 0.3,+ want: baseOnly,+ },+ {+ name: "query_pagerank_only",+ tags: map[string]interface{}{"tag_a": 0.6, "tag_b": 0.8},+ rankFeature: map[string]float64{common.PAGERANK_FLD: 7.0},+ base: 0.3,+ want: baseOnly,+ },+ {+ name: "document_tags_all_zero",+ tags: map[string]interface{}{"tag_a": 0.0, "tag_b": 0.0},+ rankFeature: weightedTags,+ base: 0.3,+ want: baseOnly,+ },+ }+}++// TestApplyRankFeatureScores_SameDocumentBothEntryPoints covers the chunk-based+// and ID-based scoring paths with identical inputs, each invocation receiving+// a fresh similarity slice because both functions mutate it in place.+func TestApplyRankFeatureScores_SameDocumentBothEntryPoints(t *testing.T) {+ for _, tc := range rankFeatureCases() {+ tc := tc+ t.Run(tc.name+"/chunks", func(t *testing.T) {+ sim := []float64{tc.base}+ got := applyRankFeatureScores([]map[string]interface{}{tc.chunk()}, sim, tc.rankFeature)+ assertRankFeatureScores(t, got, []float64{tc.want})+ })+ t.Run(tc.name+"/ids", func(t *testing.T) {+ const id = "c1"+ sim := []float64{tc.base}+ field := map[string]map[string]interface{}{id: tc.chunk()}+ got := applyRankFeatureScoresForIDs([]string{id}, field, sim, tc.rankFeature)+ assertRankFeatureScores(t, got, []float64{tc.want})+ })+ }+}++// TestApplyRankFeatureScoresForIDs_AlignsToRequestedIDs pins ID-to-score+// alignment: the field map is keyed independently of ids, so a positional read+// from field instead of a lookup by chunk ID would attach the wrong tag and+// PageRank to each result.+func TestApplyRankFeatureScoresForIDs_AlignsToRequestedIDs(t *testing.T) {+ ids := []string{"c3", "c1", "c2"}+ field := map[string]map[string]interface{}{+ "c1": {+ common.TAG_FLD: map[string]interface{}{"tag_a": 0.6, "tag_b": 0.8},+ common.PAGERANK_FLD: 0.25,+ },+ "c2": {+ common.TAG_FLD: map[string]interface{}{"tag_a": 0.3, "tag_b": 0.4},+ common.PAGERANK_FLD: 0.5,+ },+ "c3": {+ common.TAG_FLD: map[string]interface{}{"tag_a": 0.9, "tag_b": 0.1},+ common.PAGERANK_FLD: 0.1,+ },+ }+ rankFeature := map[string]float64{"tag_a": 2.0, "tag_b": 1.0, common.PAGERANK_FLD: 7.0}+ sim := []float64{0.11, 0.22, 0.33}++ got := applyRankFeatureScoresForIDs(ids, field, sim, rankFeature)++ want := []float64{+ 0.11 + ((2*0.9+1*0.1)/math.Hypot(0.9, 0.1)/math.Sqrt(5.0))*10 + 0.1,+ 0.22 + ((2*0.6+1*0.8)/math.Hypot(0.6, 0.8)/math.Sqrt(5.0))*10 + 0.25,+ 0.33 + ((2*0.3+1*0.4)/math.Hypot(0.3, 0.4)/math.Sqrt(5.0))*10 + 0.5,+ }+ assertRankFeatureScores(t, got, want)+}++func assertRankFeatureScores(t *testing.T, got, want []float64) {+ t.Helper()+ if len(got) != len(want) {+ t.Fatalf("length = %d, want %d (got %v)", len(got), len(want), got)+ }+ for i, v := range got {+ if math.IsNaN(v) || math.IsInf(v, 0) {+ t.Fatalf("score[%d] = %v, want finite value", i, v)+ }+ if math.Abs(v-want[i]) > 1e-9 {+ t.Errorf("score[%d] = %v, want %v (all got %v)", i, v, want[i], got)+ }+ }+}+// wantRankFeature computes the expected final score for one chunk independently// of applyRankFeatureScores. It mirrors the documented formula://diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex b8e04a27d1..07b06b8900 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -21,6 +21,117 @@ import ()func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {+ for _, tc := range []struct {+ name string+ page int+ candidates int+ wantChunkIDs []string+ wantTotal int64+ pageSize int+ }{+ {+ name: "first page of 64 candidates",+ page: 1,+ pageSize: 10,+ candidates: 64,+ wantChunkIDs: chunkIDs(0, 10),+ wantTotal: 64,+ },+ {+ name: "sixth page of 64 candidates",+ page: 6,+ pageSize: 10,+ candidates: 64,+ wantChunkIDs: chunkIDs(50, 60),+ wantTotal: 64,+ },+ {+ name: "seventh page of 70 candidates",+ page: 7,+ pageSize: 10,+ candidates: 70,+ wantChunkIDs: chunkIDs(60, 70),+ wantTotal: 70,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ rows := make([]map[string]interface{}, 75)+ for i := range rows {+ rows[i] = map[string]interface{}{+ "id": fmt.Sprintf("chunk-%02d", i),+ "content_ltks": "alpha",+ "content_with_weight": "alpha",+ "_score": 0.9,+ }+ }+ engine := &retrievalCountEngine{rows: rows}+ service := NewRetrievalService(engine, &dao.DocumentDAO{})+ top := 100+ threshold := 0.5+ vectorWeight := 1.0+ aggs := false+ rerankCandidatesCount := tc.candidates++ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: tc.page,+ PageSize: tc.pageSize,+ KNNTopK: &top,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ Filter: map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},+ RerankCandidatesCount: &rerankCandidatesCount,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ var gotChunkIDs []string+ for _, chunk := range result.Chunks {+ id, ok := chunk["chunk_id"].(string)+ if !ok {+ t.Fatalf("chunk without chunk_id: %#v", chunk)+ }+ gotChunkIDs = append(gotChunkIDs, id)+ }+ if !slices.Equal(gotChunkIDs, tc.wantChunkIDs) {+ t.Fatalf("page chunk ids = %v, want %v", gotChunkIDs, tc.wantChunkIDs)+ }+ if result.Total != tc.wantTotal {+ t.Fatalf("total = %d, want %d", result.Total, tc.wantTotal)+ }+ if len(engine.searchLimits) != 1 || engine.searchLimits[0] != rerankCandidatesCount {+ t.Fatalf("search limits = %v, want [%d]", engine.searchLimits, rerankCandidatesCount)+ }+ for _, filters := range engine.searchFilters {+ mustNot, ok := filters["must_not"].(map[string]interface{})+ if !ok || mustNot["exists"] != "compile_kwd" {+ t.Fatalf("must_not filter = %#v", filters["must_not"])+ }+ }+ if engine.highlightCalls != 0 {+ t.Fatalf("GetHighlight calls = %d with highlighting disabled, want 0", engine.highlightCalls)+ }+ })+ }+}++// chunkIDs returns the sequential chunk identifiers prepared by the pagination+// tests for the half-open range [from, to).+func chunkIDs(from, to int) []string {+ ids := make([]string, 0, to-from)+ for i := from; i < to; i++ {+ ids = append(ids, fmt.Sprintf("chunk-%02d", i))+ }+ return ids+}++func TestRetrievalRejectsPageBeyondCandidateWindow(t *testing.T) {oldQueryBuilder := globalQueryBuilderglobalQueryBuilder = NewQueryBuilder()defer func() { globalQueryBuilder = oldQueryBuilder }()@@ -40,40 +151,78 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {threshold := 0.5vectorWeight := 1.0aggs := false- rerankCandidatesCount := 70+ rerankCandidatesCount := 64result, err := service.Retrieval(t.Context(), &RetrievalRequest{Question: "alpha",TenantIDs: []string{"tenant-1"},- Page: 1,+ Page: 7,PageSize: 10,KNNTopK: &top,SimilarityThreshold: &threshold,VectorSimilarityWeight: &vectorWeight,Aggs: &aggs,- Filter: map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},RerankCandidatesCount: &rerankCandidatesCount,})- if err != nil {- t.Fatalf("Retrieval failed: %v", err)+ if err == nil {+ t.Fatalf("Retrieval succeeded with page beyond the candidate window: %#v", result)}- if len(result.Chunks) != 10 {- t.Fatalf("page chunk count = %d, want 10", len(result.Chunks))+ if result != nil {+ t.Fatalf("result = %#v, want nil on rejection", result)}- if result.Total != 70 {- t.Fatalf("total = %d, want 70", result.Total)+ if !strings.Contains(err.Error(), "rerank_candidates_count") || !strings.Contains(err.Error(), "page") {+ t.Fatalf("error = %q, want candidate-window rejection mentioning rerank_candidates_count and page", err)}- if len(engine.searchLimits) != 1 || engine.searchLimits[0] != rerankCandidatesCount {- t.Fatalf("search limits = %v, want [%d]", engine.searchLimits, rerankCandidatesCount)+ if len(engine.searchLimits) != 0 {+ t.Fatalf("engine Search calls = %v, want none before candidate-window validation", engine.searchLimits)}- for _, filters := range engine.searchFilters {- mustNot, ok := filters["must_not"].(map[string]interface{})- if !ok || mustNot["exists"] != "compile_kwd" {- t.Fatalf("must_not filter = %#v", filters["must_not"])+}++func TestRetrievalRejectsPaginationWithRerankModelBeforeSearch(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ rows := make([]map[string]interface{}, 75)+ for i := range rows {+ rows[i] = map[string]interface{}{+ "id": fmt.Sprintf("chunk-%02d", i),+ "content_ltks": "alpha",+ "content_with_weight": "alpha",+ "_score": 0.9,}}- if engine.highlightCalls != 0 {- t.Fatalf("GetHighlight calls = %d with highlighting disabled, want 0", engine.highlightCalls)+ engine := &retrievalCountEngine{rows: rows}+ service := NewRetrievalService(engine, &dao.DocumentDAO{})+ top := 100+ threshold := 0.5+ vectorWeight := 1.0+ aggs := false+ rerankCandidatesCount := 64++ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 2,+ PageSize: 10,+ KNNTopK: &top,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Aggs: &aggs,+ RerankCandidatesCount: &rerankCandidatesCount,+ RerankModel: &modelModule.RerankModel{},+ })+ if err == nil {+ t.Fatalf("Retrieval succeeded with pagination and a rerank model: %#v", result)+ }+ if result != nil {+ t.Fatalf("result = %#v, want nil on rejection", result)+ }+ if !strings.Contains(strings.ToLower(err.Error()), "pagination") || !strings.Contains(err.Error(), "rerank_mdl") {+ t.Fatalf("error = %q, want reranker pagination rejection mentioning pagination and rerank_mdl", err)+ }+ if len(engine.searchLimits) != 0 {+ t.Fatalf("engine Search calls = %v, want none before reranker pagination validation", engine.searchLimits)}}