Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

test(retrieval): add integration tests for retrieval and search (#21690)

ZZZZY7d6191c03b6e
diff --git a/internal/service/dataset/search_integration_test.go b/internal/service/dataset/search_integration_test.go
new file mode 100644
index 0000000000..7c44b1733f
--- /dev/null
+++ b/internal/service/dataset/search_integration_test.go
@@ -0,0 +1,317 @@
+//go:build integration
+
+package dataset
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "math"
+ "net/http"
+ "os"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine/elasticsearch"
+ "ragflow/internal/engine/types"
+ "ragflow/internal/entity"
+ "ragflow/internal/entity/models"
+ "ragflow/internal/permission"
+ "ragflow/internal/server/config"
+ "ragflow/internal/service"
+ "ragflow/internal/service/nlp"
+ "ragflow/internal/tokenizer"
+)
+
+// Requires ES_HOST (optional ES_USERNAME/ES_PASSWORD), TEI_BASE_URL serving
+// BAAI/bge-small-en-v1.5, and the production tokenizer resources. No parallel
+// tests: SQLite temporarily replaces the process-wide dao.DB.
+// Missing ES_HOST or TEI_BASE_URL skips these tests; configured service failures
+// and retrieval assertion failures remain fatal.
+const searchIntegrationQuestion = "rocket propulsion engines exhaust thrust"
+
+type searchIntegrationFixture struct {
+ service *DatasetService
+ ctx context.Context
+ owner string
+}
+
+// Catches a saved config that is loaded but ignored when preparing retrieval.
+func TestSearchDatasetsSearchIDOverridesIntegration(t *testing.T) {
+ f := newSearchIntegrationFixture(t)
+ for _, tt := range []struct {
+ name string
+ weight, threshold float64
+ wrongWeight, wrongThreshold float64
+ }{
+ {"weight", 0.7, 0, 0.2, 0},
+ {"threshold", 0.7, 0.85, 0.7, 0},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ searches := service.NewSearchService()
+ created, err := searches.CreateSearch(f.ctx, f.owner, "retrieval_"+tt.name, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = searches.UpdateSearch(f.ctx, f.owner, created.SearchID, &service.UpdateSearchRequest{
+ Name: "retrieval_" + tt.name,
+ SearchConfig: map[string]interface{}{
+ "keywords_similarity_weight": 1 - tt.weight,
+ "similarity_threshold": tt.threshold,
+ "top_k": 16, "rerank_candidates_count": 16,
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ expected := f.search(t, f.request(tt.weight, tt.threshold))
+ conflicting := f.request(tt.wrongWeight, tt.wrongThreshold)
+ control := f.search(t, conflicting)
+ conflicting.SearchID = &created.SearchID
+ actual := f.search(t, conflicting)
+ if expected.Total == 0 || actual.Total != expected.Total || !reflect.DeepEqual(actual.Chunks, expected.Chunks) {
+ t.Fatalf("saved parameters must determine results: expected=%+v actual=%+v", expected, actual)
+ }
+ if tt.name == "weight" {
+ mustHaveSearchIntegrationChunks(t, expected, "high-a", "low")
+ mustHaveSearchIntegrationChunks(t, control, "high-a", "low")
+ if math.Abs(searchIntegrationScore(t, expected, "low")-searchIntegrationScore(t, control, "low")) < 1e-5 {
+ t.Fatal("weight fixture must distinguish saved and conflicting request weights")
+ }
+ } else {
+ mustHaveSearchIntegrationChunks(t, expected, "high-a")
+ mustHaveSearchIntegrationChunks(t, control, "high-a", "low")
+ }
+ })
+ }
+}
+
+// Catches comparing model row IDs instead of base names, or bypassing the
+// consistency guard before loading a model and executing retrieval.
+func TestSearchDatasetsEmbeddingConsistencyIntegration(t *testing.T) {
+ f := newSearchIntegrationFixture(t)
+ t.Run("same_base_different_records", func(t *testing.T) {
+ req := f.request(0.7, 0)
+ req.DatasetIDs = []string{"kb-a", "kb-b"}
+ result := f.search(t, req)
+ mustHaveSearchIntegrationChunks(t, result, "high-a", "high-b", "low")
+ for _, chunk := range result.Chunks {
+ wantKB := "kb-a"
+ if chunk["id"] == "high-b" {
+ wantKB = "kb-b"
+ }
+ if chunk["dataset_id"] != wantKB {
+ t.Fatalf("incorrect dataset ownership: %v", chunk)
+ }
+ }
+ })
+ t.Run("different_bases", func(t *testing.T) {
+ if err := dao.DB.Model(&entity.TenantModel{}).Where("id = ?", "22222222222222222222222222222222").Update("model_name", "BAAI/bge-large-en-v1.5").Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := dao.DB.Model(&entity.Knowledgebase{}).Where("id = ?", "kb-b").Update("embd_id", "BAAI/bge-large-en-v1.5@Builtin").Error; err != nil {
+ t.Fatal(err)
+ }
+ req := f.request(0.7, 0)
+ req.DatasetIDs = []string{"kb-a", "kb-b"}
+ result, err := f.service.SearchDatasets(f.ctx, req, f.owner)
+ if err == nil || !strings.Contains(err.Error(), "datasets use different embedding models") || result != nil {
+ t.Fatalf("different base models must be rejected before retrieval: result=%+v err=%v", result, err)
+ }
+ })
+}
+
+func newSearchIntegrationFixture(t *testing.T) *searchIntegrationFixture {
+ t.Helper()
+ if os.Getenv("ES_HOST") == "" || os.Getenv("TEI_BASE_URL") == "" {
+ t.Skip("set ES_HOST and TEI_BASE_URL to run real SearchDatasets integration tests")
+ }
+ if err := common.InitLogger("error", common.FileOutput{}, ""); err != nil {
+ t.Fatal(err)
+ }
+ if !tokenizer.IsInitialized() {
+ if err := tokenizer.Init(&tokenizer.PoolConfig{MinSize: 1, MaxSize: 2}); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(tokenizer.Close)
+ }
+ if err := tokenizer.InitCL100KEncoder(); err != nil {
+ t.Fatal(err)
+ }
+ if err := nlp.InitQueryBuilder(""); err != nil {
+ t.Fatal(err)
+ }
+ // Verify the deployed model rather than accepting any model with 384 dimensions.
+ client := &http.Client{Timeout: 30 * time.Second}
+ response, err := client.Get(strings.TrimRight(os.Getenv("TEI_BASE_URL"), "/") + "/info")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var info struct {
+ ModelID string `json:"model_id"`
+ }
+ decodeErr := json.NewDecoder(response.Body).Decode(&info)
+ closeErr := response.Body.Close()
+ if response.StatusCode != http.StatusOK || decodeErr != nil || closeErr != nil || info.ModelID != "BAAI/bge-small-en-v1.5" {
+ t.Fatalf("TEI must serve BAAI/bge-small-en-v1.5: status=%d model=%q decode=%v close=%v", response.StatusCode, info.ModelID, decodeErr, closeErr)
+ }
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Search{}); err != nil {
+ t.Fatal(err)
+ }
+ owner := fmt.Sprintf("search_%d", time.Now().UnixNano())
+ if err := db.Create(&entity.Tenant{ID: owner, ParserIDs: "general"}).Error; err != nil {
+ t.Fatal(err)
+ }
+ ensureDatasetTestMembership(t, owner, owner, string(permission.RoleOwner))
+ ctx, cancel := context.WithTimeout(withDatasetTestContext(t.Context(), owner, owner), 3*time.Minute)
+ t.Cleanup(cancel)
+ de, err := elasticsearch.NewEngine(ctx, config.ElasticsearchConfig{Hosts: os.Getenv("ES_HOST"), Username: os.Getenv("ES_USERNAME"), Password: os.Getenv("ES_PASSWORD")})
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := de.Close(); err != nil {
+ t.Errorf("close Elasticsearch: %v", err)
+ }
+ })
+ f := &searchIntegrationFixture{ctx: ctx, owner: owner, service: &DatasetService{kbDAO: dao.NewKnowledgebaseDAO(), documentDAO: dao.NewDocumentDAO(), searchService: service.NewSearchService(), docEngine: de}}
+ // Different model records resolve to the same base. The ordinary Builtin
+ // reference still goes through ModelFactory to the real TEI driver.
+ if err := db.Create(&entity.TenantModelProvider{ID: "provider", ProviderName: "Builtin", UserID: owner, TenantID: owner}).Error; err != nil {
+ t.Fatal(err)
+ }
+ for i, id := range []string{"11111111111111111111111111111111", "22222222222222222222222222222222"} {
+ instanceID := fmt.Sprintf("instance-%d", i)
+ if err := db.Create(&entity.TenantModelInstance{ID: instanceID, InstanceName: instanceID, ProviderID: "provider", Status: "active", Extra: "{}"}).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Create(&entity.TenantModel{ID: id, ModelName: info.ModelID, ProviderID: "provider", InstanceID: instanceID, ModelType: int(entity.ModelTypeEmbedding), Status: "active"}).Error; err != nil {
+ t.Fatal(err)
+ }
+ kb := fmt.Sprintf("kb-%c", 'a'+i)
+ if err := db.Create(&entity.Knowledgebase{ID: kb, Name: kb, TenantID: owner, CreatedBy: owner, Permission: "me", EmbdID: info.ModelID + "@Builtin", TenantEmbdID: &id, ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatal(err)
+ }
+ grantDatasetTestResource(t, owner, "kb", kb, owner, "owner")
+ }
+ model, err := service.NewModelFactory().NewEmbeddingModel(ctx, service.ModelAccess{TenantID: owner}, info.ModelID+"@Builtin")
+ if err != nil {
+ t.Fatal(err)
+ }
+ rows := []struct{ id, kb, content string }{
+ {"high-a", "kb-a", searchIntegrationQuestion},
+ {"low", "kb-a", "Rocket Engines is a restaurant serving peppery salad leaves, fresh tomatoes, garden vegetables and homemade bread."},
+ {"high-b", "kb-b", searchIntegrationQuestion + " spacecraft"},
+ }
+ texts := make([]string, len(rows))
+ for i, row := range rows {
+ texts[i] = row.content
+ }
+ embeddings, err := model.Embed(ctx, models.EmbedRequest{Texts: texts}, &models.EmbeddingConfig{}, nil)
+ if err != nil || len(embeddings) != len(rows) {
+ t.Fatalf("embed fixture: count=%d err=%v", len(embeddings), err)
+ }
+ // Both datasets share an isolated tenant index, just as production does.
+ index := "ragflow_" + owner
+ t.Cleanup(func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cleanupCancel()
+ if err := de.DropChunkStore(cleanupCtx, index, "kb-a"); err != nil {
+ t.Errorf("drop fixture index: %v", err)
+ }
+ })
+ for i, row := range rows {
+ if len(embeddings[i].Embedding) != 384 {
+ t.Fatal("expected 384-dimensional BGE embeddings")
+ }
+ if err := db.Create(&entity.Document{ID: row.id, KbID: row.kb, ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := de.CreateChunkStore(ctx, index, row.kb, 384, "naive", "English"); err != nil {
+ t.Fatal(err)
+ }
+ tokens, err := tokenizer.Tokenize(row.content)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fine, err := tokenizer.FineGrainedTokenize(tokens)
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunk := map[string]interface{}{"id": row.id, "doc_id": row.id, "kb_id": row.kb, "docnm_kwd": row.id + ".txt", "content_with_weight": row.content, "content_ltks": tokens, "content_sm_ltks": fine, "available_int": 1, "q_384_vec": embeddings[i].Embedding}
+ if row.id == "low" {
+ chunk["important_kwd"] = strings.Fields(searchIntegrationQuestion)
+ }
+ if _, err := de.InsertChunks(ctx, []map[string]interface{}{chunk}, index, row.kb, "English"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ visibilityCtx, visibilityCancel := context.WithTimeout(ctx, 30*time.Second)
+ defer visibilityCancel()
+ ticker := time.NewTicker(200 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ result, err := de.Search(visibilityCtx, &types.SearchRequest{IndexNames: []string{index}, KbIDs: []string{"kb-a", "kb-b"}, Limit: 16, SelectFields: []string{"id"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(result.Chunks) == len(rows) {
+ break
+ }
+ select {
+ case <-visibilityCtx.Done():
+ t.Fatal("fixture index visibility timeout")
+ case <-ticker.C:
+ }
+ }
+ return f
+}
+
+func (f *searchIntegrationFixture) request(weight, threshold float64) *service.SearchDatasetsRequest {
+ return &service.SearchDatasetsRequest{Question: searchIntegrationQuestion, DatasetIDs: []string{"kb-a"}, Page: new(1), PageSize: new(16), RerankCandidatesCount: new(16), KNNTopK: new(16), KNNNumCandidates: new(32), VectorSimilarityWeight: &weight, SimilarityThreshold: &threshold, IncludeCompiledChunks: new(false)}
+}
+
+func (f *searchIntegrationFixture) search(t *testing.T, req *service.SearchDatasetsRequest) *service.SearchDatasetsResponse {
+ t.Helper()
+ result, err := f.service.SearchDatasets(f.ctx, req, f.owner)
+ if err != nil {
+ t.Fatalf("SearchDatasets: %v", err)
+ }
+ return result
+}
+
+func mustHaveSearchIntegrationChunks(t *testing.T, result *service.SearchDatasetsResponse, want ...string) {
+ t.Helper()
+ got := make([]string, 0, len(result.Chunks))
+ for _, chunk := range result.Chunks {
+ id, _ := chunk["id"].(string)
+ got = append(got, id)
+ }
+ slices.Sort(got)
+ slices.Sort(want)
+ if !slices.Equal(got, want) || result.Total != int64(len(want)) {
+ t.Fatalf("chunks=%v total=%d, want %v", got, result.Total, want)
+ }
+}
+
+func searchIntegrationScore(t *testing.T, result *service.SearchDatasetsResponse, id string) float64 {
+ t.Helper()
+ for _, chunk := range result.Chunks {
+ if chunk["id"] == id {
+ score, ok := chunk["similarity"].(common.PyFloat64)
+ if !ok || math.IsNaN(float64(score)) || math.IsInf(float64(score), 0) {
+ t.Fatalf("invalid similarity: %v", chunk)
+ }
+ return float64(score)
+ }
+ }
+ t.Fatalf("missing chunk %s", id)
+ return 0
+}
diff --git a/internal/service/nlp/retrieval_integration_test.go b/internal/service/nlp/retrieval_integration_test.go
new file mode 100644
index 0000000000..5646abce4e
--- /dev/null
+++ b/internal/service/nlp/retrieval_integration_test.go
@@ -0,0 +1,415 @@
+//go:build integration
+
+package nlp
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "math"
+ "net/http"
+ "os"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
+ "ragflow/internal/engine/elasticsearch"
+ "ragflow/internal/engine/infinity"
+ "ragflow/internal/engine/types"
+ "ragflow/internal/entity"
+ "ragflow/internal/entity/models"
+ "ragflow/internal/server/config"
+ "ragflow/internal/tokenizer"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+// Requires ES_HOST (and optional ES_USERNAME/ES_PASSWORD), RAGFLOW_INFINITY_URI,
+// TEI_BASE_URL serving BAAI/bge-small-en-v1.5, and the usual tokenizer resources.
+// Missing service configuration skips the affected backend; configured service
+// failures and retrieval assertion failures remain fatal.
+// All three tests cover both backends when their configuration is supplied.
+const retrievalIntegrationQuestion = "rocket propulsion engines exhaust thrust"
+
+type retrievalIntegrationFixture struct {
+ service *RetrievalService
+ model *models.EmbeddingModel
+ tenant string
+ kb string
+ engine string
+}
+
+// Catches a disconnected dense leg, ignored weights, or an ES second-pass score
+// that no longer uses the caller's weight. No cross-engine score parity is assumed.
+func TestRetrievalModesAndWeightsIntegration(t *testing.T) {
+ for _, backend := range []string{"elasticsearch", "infinity"} {
+ t.Run(backend, func(t *testing.T) {
+ f := newRetrievalIntegrationFixture(t, backend)
+ t.Run("text", func(t *testing.T) {
+ req := f.request(0, 0)
+ req.EmbeddingModel = nil
+ mustContainRetrievalChunks(t, f.retrieve(t, req), "lexical")
+ })
+ t.Run("dense", func(t *testing.T) {
+ req := f.request(1, 0)
+ req.VectorOnly = true
+ mustContainRetrievalChunks(t, f.retrieve(t, req), "semantic")
+ })
+ t.Run("hybrid_weights", func(t *testing.T) {
+ low := f.retrieve(t, f.request(0.2, 0))
+ high := f.retrieve(t, f.request(0.7, 0))
+ mustContainRetrievalChunks(t, low, "lexical", "semantic")
+ mustContainRetrievalChunks(t, high, "lexical", "semantic")
+ gap := func(result *RetrievalResult) float64 {
+ return retrievalIntegrationScore(t, result, "semantic", "similarity") - retrievalIntegrationScore(t, result, "lexical", "similarity")
+ }
+ if gap(high) <= gap(low)+1e-5 {
+ t.Fatalf("increasing vector weight must improve semantic relative to lexical candidate: gaps %g -> %g", gap(low), gap(high))
+ }
+ if backend == "elasticsearch" {
+ for _, tt := range []struct {
+ result *RetrievalResult
+ weight float64
+ }{{low, 0.2}, {high, 0.7}} {
+ for _, id := range []string{"lexical", "semantic"} {
+ term := retrievalIntegrationScore(t, tt.result, id, "term_similarity")
+ vector := retrievalIntegrationScore(t, tt.result, id, "vector_similarity")
+ want := (1-tt.weight)*term + tt.weight*vector
+ if got := retrievalIntegrationScore(t, tt.result, id, "similarity"); math.Abs(got-want) > 1e-6 {
+ t.Fatalf("%s: similarity=%g, want blended score %g", id, got, want)
+ }
+ }
+ }
+ }
+ })
+ })
+ }
+}
+
+// Catches a missing final threshold or a total copied from unfiltered hits.
+func TestRetrievalThresholdIntegration(t *testing.T) {
+ for _, backend := range []string{"elasticsearch", "infinity"} {
+ t.Run(backend, func(t *testing.T) {
+ f := newRetrievalIntegrationFixture(t, backend, backend != "infinity")
+ low := f.scopedRequest(0)
+ result := f.retrieve(t, low)
+ for _, chunk := range result.Chunks {
+ t.Logf("threshold fixture: id=%v similarity=%v cutoff=%g", chunk["chunk_id"], chunk["similarity"], f.threshold())
+ }
+ mustHaveRetrievalChunks(t, result, []string{"a1", "a2", "b1", "lexical"})
+ result = f.retrieve(t, f.scopedRequest(f.threshold()))
+ mustHaveRetrievalChunks(t, result, []string{"a1", "a2", "b1"})
+ })
+ }
+}
+
+// Catches pagination before filtering, page-local totals/aggregations, and a
+// lost document scope. Literal IDs/counts keep the baseline from being an oracle.
+func TestRetrievalPaginationAndAggregationsIntegration(t *testing.T) {
+ for _, backend := range []string{"elasticsearch", "infinity"} {
+ t.Run(backend, func(t *testing.T) {
+ f := newRetrievalIntegrationFixture(t, backend, backend != "infinity")
+ full := f.retrieve(t, f.scopedRequest(f.threshold()))
+ mustHaveRetrievalChunks(t, full, []string{"a1", "a2", "b1"})
+ for i := 1; i < len(full.Chunks); i++ {
+ previous := full.Chunks[i-1]["similarity"].(float64)
+ current := full.Chunks[i]["similarity"].(float64)
+ if previous-current < 1e-6 {
+ t.Fatal("pagination fixture requires distinct scores; tied backend order cannot define page boundaries")
+ }
+ }
+ wantAggs := []map[string]interface{}{
+ {"doc_name": "alpha.txt", "doc_id": "doc-a", "count": 2},
+ {"doc_name": "beta.txt", "doc_id": "doc-b", "count": 1},
+ }
+ if !reflect.DeepEqual(full.DocAggs, wantAggs) {
+ t.Fatalf("doc_aggs=%v, want %v", full.DocAggs, wantAggs)
+ }
+ var all []string
+ for page := 1; page <= 2; page++ {
+ req := f.scopedRequest(f.threshold())
+ req.Page, req.PageSize = page, 2
+ result := f.retrieve(t, req)
+ begin, end := (page-1)*2, min(page*2, len(full.Chunks))
+ if !reflect.DeepEqual(result.Chunks, full.Chunks[begin:end]) {
+ t.Fatalf("page %d differs from sorted result slice: %v", page, result.Chunks)
+ }
+ if result.Total != 3 || !reflect.DeepEqual(result.DocAggs, wantAggs) {
+ t.Fatalf("page %d: total=%d doc_aggs=%v", page, result.Total, result.DocAggs)
+ }
+ all = append(all, retrievalIntegrationIDs(result)...)
+ }
+ slices.Sort(all)
+ if !slices.Equal(all, []string{"a1", "a2", "b1"}) {
+ t.Fatalf("pagination duplicated, lost, or leaked chunks: %v", all)
+ }
+ })
+ }
+}
+
+func newRetrievalIntegrationFixture(t *testing.T, backend string, lexicalBoost ...bool) *retrievalIntegrationFixture {
+ t.Helper()
+ if os.Getenv("TEI_BASE_URL") == "" {
+ t.Skip("set TEI_BASE_URL to run real retrieval integration tests with BAAI/bge-small-en-v1.5")
+ }
+ switch backend {
+ case "elasticsearch":
+ if os.Getenv("ES_HOST") == "" {
+ t.Skip("set ES_HOST to run Elasticsearch retrieval integration tests")
+ }
+ case "infinity":
+ if os.Getenv("RAGFLOW_INFINITY_URI") == "" {
+ t.Skip("set RAGFLOW_INFINITY_URI to run Infinity retrieval integration tests")
+ }
+ default:
+ t.Fatalf("unknown backend %q", backend)
+ }
+ // Weight competition needs a strong lexical candidate. Infinity threshold
+ // and pagination fixtures opt out so unrelated content is not boosted by
+ // important_kwd^30/important_tks^20. Existing ES and reranker fixtures retain it.
+ boostLexical := len(lexicalBoost) == 0 || lexicalBoost[0]
+ if err := common.InitLogger("error", common.FileOutput{}, ""); err != nil {
+ t.Fatal(err)
+ }
+ if !tokenizer.IsInitialized() {
+ if err := tokenizer.Init(&tokenizer.PoolConfig{MinSize: 1, MaxSize: 2}); err != nil {
+ t.Fatalf("initialize production tokenizer: %v", err)
+ }
+ t.Cleanup(tokenizer.Close)
+ }
+ if err := tokenizer.InitCL100KEncoder(); err != nil {
+ t.Fatal(err)
+ }
+ previousQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ t.Cleanup(func() { globalQueryBuilder = previousQueryBuilder })
+
+ baseURL := strings.TrimRight(os.Getenv("TEI_BASE_URL"), "/")
+ if baseURL == "" {
+ t.Fatal("TEI_BASE_URL must point to BAAI/bge-small-en-v1.5")
+ }
+ client := &http.Client{Timeout: 30 * time.Second}
+ response, err := client.Get(baseURL + "/info")
+ if err != nil {
+ t.Fatalf("TEI /info: %v", err)
+ }
+ var info struct {
+ ModelID string `json:"model_id"`
+ }
+ decodeErr := json.NewDecoder(response.Body).Decode(&info)
+ closeErr := response.Body.Close()
+ if response.StatusCode != http.StatusOK || decodeErr != nil || closeErr != nil || info.ModelID != "BAAI/bge-small-en-v1.5" {
+ t.Fatalf("TEI must serve BAAI/bge-small-en-v1.5: status=%d model=%q decode=%v close=%v", response.StatusCode, info.ModelID, decodeErr, closeErr)
+ }
+ modelName := info.ModelID
+ model := models.NewEmbeddingModel(models.NewBuiltinModel(baseURL, modelName), &modelName, &models.APIConfig{}, 512)
+ db, err := gorm.Open(sqlite.Open(filepath.Join(t.TempDir(), "documents.db")), &gorm.Config{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := sqlDB.Close(); err != nil {
+ t.Errorf("close SQLite: %v", err)
+ }
+ })
+ if err := db.AutoMigrate(&entity.Document{}); err != nil {
+ t.Fatal(err)
+ }
+ previousDB := dao.DB
+ dao.DB = db
+ t.Cleanup(func() { dao.DB = previousDB })
+
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Minute)
+ defer cancel()
+ var de engine.DocEngine
+ switch backend {
+ case "elasticsearch":
+ de, err = elasticsearch.NewEngine(ctx, config.ElasticsearchConfig{Hosts: os.Getenv("ES_HOST"), Username: os.Getenv("ES_USERNAME"), Password: os.Getenv("ES_PASSWORD")})
+ case "infinity":
+ de, err = infinity.NewEngine(ctx, config.InfinityConfig{URI: os.Getenv("RAGFLOW_INFINITY_URI"), DBName: "ragflow_retrieval_test", PostgresPort: 5432})
+ }
+ if err != nil {
+ t.Fatalf("connect to %s: %v", backend, err)
+ }
+ t.Cleanup(func() {
+ if err := de.Close(); err != nil {
+ t.Errorf("close %s: %v", backend, err)
+ }
+ })
+ f := &retrievalIntegrationFixture{service: NewRetrievalService(de, dao.NewDocumentDAO()), model: model, tenant: fmt.Sprintf("retrieval_%d", time.Now().UnixNano()), kb: "retrieval", engine: backend}
+ index := "ragflow_" + f.tenant
+ rows := []struct{ id, doc, name, content string }{
+ {"a1", "doc-a", "alpha.txt", retrievalIntegrationQuestion},
+ {"a2", "doc-a", "alpha.txt", retrievalIntegrationQuestion + " vacuum"},
+ {"b1", "doc-b", "beta.txt", retrievalIntegrationQuestion + " spacecraft"},
+ {"lexical", "doc-low", "garden.txt", "Rocket Engines is a restaurant serving peppery salad leaves, fresh tomatoes, garden vegetables and homemade bread."},
+ {"semantic", "doc-semantic", "science.txt", "A rocket engine produces thrust by ejecting hot gas into space."},
+ {"excluded", "doc-excluded", "excluded.txt", retrievalIntegrationQuestion},
+ }
+ texts := make([]string, len(rows))
+ for i, row := range rows {
+ texts[i] = row.content
+ }
+ embeddings, err := model.Embed(ctx, models.EmbedRequest{Texts: texts}, &models.EmbeddingConfig{}, nil)
+ if err != nil {
+ t.Fatalf("embed fixture with real model: %v", err)
+ }
+ if len(embeddings) != len(rows) || len(embeddings[0].Embedding) != 384 {
+ t.Fatal("fixture requires all six 384-dimensional bge-small-en-v1.5 embeddings")
+ }
+ if err := de.CreateChunkStore(ctx, index, f.kb, 384, "naive", "English"); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cleanupCancel()
+ if err := de.DropChunkStore(cleanupCtx, index, f.kb); err != nil {
+ t.Errorf("drop fixture store: %v", err)
+ }
+ })
+ chunks := make([]map[string]interface{}, 0, len(rows))
+ documents := map[string]bool{}
+ for i, row := range rows {
+ if !documents[row.doc] {
+ if err := db.Create(&entity.Document{ID: row.doc, KbID: f.kb, Name: &row.name, ParserConfig: entity.JSONMap{}}).Error; err != nil {
+ t.Fatal(err)
+ }
+ documents[row.doc] = true
+ }
+ tokens, err := tokenizer.Tokenize(row.content)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fine, err := tokenizer.FineGrainedTokenize(tokens)
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunk := map[string]interface{}{"id": row.id, "doc_id": row.doc, "kb_id": f.kb, "docnm_kwd": row.name, "content_with_weight": row.content, "content_ltks": tokens, "content_sm_ltks": fine, "available_int": 1, "q_384_vec": embeddings[i].Embedding}
+ if row.id == "lexical" && boostLexical {
+ chunk["important_kwd"] = strings.Fields(retrievalIntegrationQuestion)
+ chunk["important_tks"], err = tokenizer.Tokenize(retrievalIntegrationQuestion)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ chunks = append(chunks, chunk)
+ }
+ if _, err := de.InsertChunks(ctx, chunks, index, f.kb, "English"); err != nil {
+ t.Fatal(err)
+ }
+ // Retry visibility only. Authentication, transport and query errors fail now.
+ visibilityCtx, visibilityCancel := context.WithTimeout(ctx, 30*time.Second)
+ defer visibilityCancel()
+ ticker := time.NewTicker(200 * time.Millisecond)
+ defer ticker.Stop()
+ for {
+ visible, err := de.Search(visibilityCtx, &types.SearchRequest{IndexNames: []string{index}, KbIDs: []string{f.kb}, Limit: 16, SelectFields: []string{"id"}})
+ if err != nil {
+ t.Fatalf("fixture visibility query: %v", err)
+ }
+ if len(visible.Chunks) == len(rows) {
+ break
+ }
+ select {
+ case <-visibilityCtx.Done():
+ t.Fatalf("index visibility timeout: got %d chunks, want %d", len(visible.Chunks), len(rows))
+ case <-ticker.C:
+ }
+ }
+ return f
+}
+
+func (f *retrievalIntegrationFixture) request(weight, threshold float64) *RetrievalRequest {
+ return &RetrievalRequest{Question: retrievalIntegrationQuestion, TenantIDs: []string{f.tenant}, KbIDs: []string{f.kb}, Page: 1, PageSize: 16, RerankCandidatesCount: new(16), KNNTopK: new(16), KNNNumCandidates: new(32), VectorSimilarityWeight: &weight, SimilarityThreshold: &threshold, RankFeature: &map[string]float64{}, EmbeddingModel: f.model, AllowDenseFallback: new(false), Language: "English"}
+}
+
+func (f *retrievalIntegrationFixture) scopedRequest(threshold float64) *RetrievalRequest {
+ req := f.request(0.7, threshold)
+ req.DocIDs = []string{"doc-a", "doc-b", "doc-low"}
+ return req
+}
+
+func (f *retrievalIntegrationFixture) threshold() float64 {
+ // ES returns its clean kNN score on a different scale from Infinity's atan
+ // fusion. These are fixed cutoffs, not thresholds fitted to retrieved scores.
+ if f.engine == "elasticsearch" {
+ return 0.85
+ }
+ return 0.70
+}
+
+func (f *retrievalIntegrationFixture) retrieve(t *testing.T, req *RetrievalRequest) *RetrievalResult {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ result, err := f.service.Retrieval(ctx, req)
+ if err != nil {
+ t.Fatalf("Retrieval: %v", err)
+ }
+ previousScore := math.Inf(1)
+ for _, chunk := range result.Chunks {
+ score, ok := chunk["similarity"].(float64)
+ if !ok || math.IsNaN(score) || math.IsInf(score, 0) || score > previousScore {
+ t.Fatalf("invalid or unsorted similarity: %v", chunk)
+ }
+ if chunk["kb_id"] != f.kb {
+ t.Fatalf("chunk escaped fixture dataset: %v", chunk)
+ }
+ previousScore = score
+ }
+ return result
+}
+
+func retrievalIntegrationIDs(result *RetrievalResult) []string {
+ ids := make([]string, 0, len(result.Chunks))
+ for _, chunk := range result.Chunks {
+ id, _ := chunk["chunk_id"].(string)
+ ids = append(ids, id)
+ }
+ return ids
+}
+
+func mustContainRetrievalChunks(t *testing.T, result *RetrievalResult, ids ...string) {
+ t.Helper()
+ got := retrievalIntegrationIDs(result)
+ for _, id := range ids {
+ if !slices.Contains(got, id) {
+ t.Fatalf("missing target chunk %q: got %v", id, got)
+ }
+ }
+}
+
+func mustHaveRetrievalChunks(t *testing.T, result *RetrievalResult, want []string) {
+ t.Helper()
+ got := retrievalIntegrationIDs(result)
+ slices.Sort(got)
+ if !slices.Equal(got, want) || result.Total != int64(len(want)) {
+ t.Fatalf("chunks=%v total=%d, want chunks=%v total=%d", got, result.Total, want, len(want))
+ }
+}
+
+func retrievalIntegrationScore(t *testing.T, result *RetrievalResult, id, field string) float64 {
+ t.Helper()
+ for _, chunk := range result.Chunks {
+ if chunk["chunk_id"] == id {
+ score, ok := chunk[field].(float64)
+ if !ok || math.IsNaN(score) || math.IsInf(score, 0) {
+ t.Fatalf("%s has invalid %s: %v", id, field, chunk[field])
+ }
+ return score
+ }
+ }
+ t.Fatalf("chunk %s missing", id)
+ return 0
+}
diff --git a/internal/service/nlp/retrieval_rerank_integration_test.go b/internal/service/nlp/retrieval_rerank_integration_test.go
new file mode 100644
index 0000000000..f5bebab249
--- /dev/null
+++ b/internal/service/nlp/retrieval_rerank_integration_test.go
@@ -0,0 +1,183 @@
+//go:build integration
+
+package nlp
+
+import (
+ "context"
+ "math"
+ "os"
+ "slices"
+ "testing"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity/models"
+)
+
+// Uses the Elasticsearch/TEI fixture in retrieval_integration_test.go plus
+// SILICONFLOW_API_KEY for the fixed BAAI/bge-reranker-v2-m3 model.
+// SILICONFLOW_BASE_URL may select another deployment of the same provider.
+// Missing required configuration skips these tests; configured service failures,
+// including authentication failures, remain fatal.
+// No tests run in parallel: the fixture temporarily installs dao.DB.
+const retrievalIntegrationReranker = "BAAI/bge-reranker-v2-m3"
+
+type rerankIntegrationCall struct {
+ query string
+ documents []string
+ results []models.RerankResult
+}
+
+// Records actual driver traffic without changing the request, response or error.
+type rerankIntegrationRecorder struct {
+ models.ModelDriver
+ calls []rerankIntegrationCall
+}
+
+func (r *rerankIntegrationRecorder) Rerank(ctx context.Context, name *string, request models.RerankRequest, api *models.APIConfig, config *models.RerankConfig, usage *common.ModelUsage) (*models.RerankResponse, error) {
+ response, err := r.ModelDriver.Rerank(ctx, name, request, api, config, usage)
+ call := rerankIntegrationCall{query: request.Query, documents: slices.Clone(request.Documents)}
+ if response != nil {
+ call.results = slices.Clone(response.Data)
+ }
+ r.calls = append(r.calls, call)
+ return response, err
+}
+
+// Catches truncation to page_size before the real model call, missing natural
+// text, and returning engine order instead of the real model's highest score.
+func TestRetrievalRerankIntegration(t *testing.T) {
+ f, model, recorder := newRerankIntegrationFixture(t)
+ req := f.request(1, 0)
+ req.RerankModel = model
+ req.RerankCandidatesCount = new(3)
+ req.PageSize = 1
+ result := f.retrieve(t, req)
+ call := mustHaveRerankIntegrationCall(t, recorder, 1)
+ if len(call.documents) != 3 {
+ t.Fatalf("reranker received %d candidates, want 3 before page_size=1 truncation", len(call.documents))
+ }
+ if len(result.Chunks) != 1 || result.Total != 3 {
+ t.Fatalf("chunks=%d total=%d, want one returned chunk and three scored candidates", len(result.Chunks), result.Total)
+ }
+ scores := rerankIntegrationScores(t, call)
+ best := math.Inf(-1)
+ for _, score := range scores {
+ best = math.Max(best, score)
+ }
+ text, ok := result.Chunks[0]["content_with_weight"].(string)
+ score, exists := scores[text]
+ if !ok || !exists || math.Abs(score-best) > 1e-6 {
+ t.Fatalf("returned chunk is not a highest-scoring real rerank candidate: %v", result.Chunks[0])
+ }
+ if got := result.Chunks[0]["similarity"].(float64); math.Abs(got-score) > 1e-6 {
+ t.Fatalf("final similarity=%g, want real reranker score %g", got, score)
+ }
+}
+
+// Catches filtering on engine scores instead of rerank scores. Both calls must
+// reach the model with the same two candidates before the fixed 0.5 cutoff.
+func TestRetrievalRerankThresholdIntegration(t *testing.T) {
+ f, model, recorder := newRerankIntegrationFixture(t)
+ request := func(threshold float64) *RetrievalRequest {
+ req := f.request(1, threshold)
+ req.RerankModel = model
+ req.DocIDs = []string{"doc-b", "doc-low"}
+ return req
+ }
+ low := f.retrieve(t, request(0))
+ mustHaveRetrievalChunks(t, low, []string{"b1", "lexical"})
+ first := mustHaveRerankIntegrationCall(t, recorder, 1)
+ firstScores := rerankIntegrationScores(t, first)
+ high := f.retrieve(t, request(0.5))
+ second := mustHaveRerankIntegrationCall(t, recorder, 2)
+ secondScores := rerankIntegrationScores(t, second)
+ wantTexts := []string{
+ retrievalIntegrationQuestion + " spacecraft",
+ "Rocket Engines is a restaurant serving peppery salad leaves, fresh tomatoes, garden vegetables and homemade bread.",
+ }
+ wantSet := slices.Clone(wantTexts)
+ slices.Sort(wantSet)
+ for _, call := range []rerankIntegrationCall{first, second} {
+ got := slices.Clone(call.documents)
+ slices.Sort(got)
+ if !slices.Equal(got, wantSet) {
+ t.Fatalf("model input changed before rerank threshold filtering: got %q, want %q", got, wantSet)
+ }
+ }
+ for _, scores := range []map[string]float64{firstScores, secondScores} {
+ if scores[wantTexts[0]] < 0.5 || scores[wantTexts[1]] >= 0.5 {
+ t.Fatalf("fixed model/fixture must distinguish relevant and irrelevant passages at 0.5: %v", scores)
+ }
+ }
+ mustHaveRetrievalChunks(t, high, []string{"b1"})
+ for _, result := range []*RetrievalResult{low, high} {
+ scores := firstScores
+ if result == high {
+ scores = secondScores
+ }
+ for _, chunk := range result.Chunks {
+ text, _ := chunk["content_with_weight"].(string)
+ if score, ok := scores[text]; !ok || math.Abs(chunk["similarity"].(float64)-score) > 1e-6 {
+ t.Fatalf("chunk does not carry its real rerank score: %v", chunk)
+ }
+ }
+ }
+}
+
+func newRerankIntegrationFixture(t *testing.T) (*retrievalIntegrationFixture, *models.RerankModel, *rerankIntegrationRecorder) {
+ t.Helper()
+ key := os.Getenv("SILICONFLOW_API_KEY")
+ if key == "" {
+ t.Skip("set SILICONFLOW_API_KEY to run real reranker integration tests")
+ }
+ baseURL := os.Getenv("SILICONFLOW_BASE_URL")
+ if baseURL == "" {
+ baseURL = "https://api.siliconflow.cn/v1"
+ }
+ f := newRetrievalIntegrationFixture(t, "elasticsearch")
+ name := retrievalIntegrationReranker
+ driver := models.NewSiliconflowModel(map[string]string{"default": baseURL}, models.URLSuffix{Rerank: "rerank"})
+ recorder := &rerankIntegrationRecorder{ModelDriver: driver}
+ model := models.NewRerankModel(recorder, &name, &models.APIConfig{ApiKey: &key, BaseURL: &baseURL}, 8192)
+ return f, model, recorder
+}
+
+func mustHaveRerankIntegrationCall(t *testing.T, recorder *rerankIntegrationRecorder, count int) rerankIntegrationCall {
+ t.Helper()
+ if len(recorder.calls) != count {
+ t.Fatalf("real rerank calls=%d, want %d", len(recorder.calls), count)
+ }
+ call := recorder.calls[count-1]
+ if call.query != retrievalIntegrationQuestion {
+ t.Fatalf("rerank query=%q, want %q", call.query, retrievalIntegrationQuestion)
+ }
+ return call
+}
+
+func rerankIntegrationScores(t *testing.T, call rerankIntegrationCall) map[string]float64 {
+ t.Helper()
+ if len(call.results) != len(call.documents) || len(call.documents) == 0 {
+ t.Fatalf("real model returned %d scores for %d candidates", len(call.results), len(call.documents))
+ }
+ seen := make(map[int]bool, len(call.documents))
+ scores := make(map[string]float64, len(call.documents))
+ for _, result := range call.results {
+ if result.Index < 0 || result.Index >= len(call.documents) || seen[result.Index] || math.IsNaN(result.RelevanceScore) || math.IsInf(result.RelevanceScore, 0) || result.RelevanceScore < 0 || result.RelevanceScore > 1 {
+ t.Fatalf("invalid calibrated score/index from fixed reranker: %+v", result)
+ }
+ seen[result.Index] = true
+ text := call.documents[result.Index]
+ // Match against the natural indexed content, never tokenized text.
+ switch text {
+ case retrievalIntegrationQuestion,
+ retrievalIntegrationQuestion + " vacuum",
+ retrievalIntegrationQuestion + " spacecraft",
+ "Rocket Engines is a restaurant serving peppery salad leaves, fresh tomatoes, garden vegetables and homemade bread.",
+ "A rocket engine produces thrust by ejecting hot gas into space.":
+ default:
+ t.Fatalf("reranker received altered or tokenized content: %q", text)
+ }
+ scores[text] = result.RelevanceScore
+ }
+ return scores
+}