Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: delete not used code (#19164)

Wang Qicf0f56a9038b
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 4831a3da8b..78aea00840 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -37,7 +37,7 @@ import (
"ragflow/internal/tokenizer"
"github.com/elastic/go-elasticsearch/v8/esapi"
- "github.com/json-iterator/go"
+ jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
)
@@ -2988,51 +2988,6 @@ func getDefaultSkillMapping() map[string]interface{} {
}
}
-// rerankWindow returns the candidate-window size shared by retrieval's
-// block fetch and slice. Mirrors Dealer._rerank_window in rag/nlp/search.py.
-//
-// `size` is the per-page size; the window MUST be an exact multiple of it,
-// otherwise the block fetched (offset // window) and the in-block page slice
-// (offset % window) drift apart and deep pagination silently drops results.
-//
-// The window targets a provider-friendly pool of ~64 candidates, bounded by
-// `topK` when given (i.e. when an external reranker is active), and is always
-// rounded UP to a whole number of pages to preserve the alignment invariant.
-func rerankWindow(size, topK int) int {
- if size <= 1 {
- if topK > 0 {
- return min(30, topK)
- }
- return 30
- }
- window := ((64 + size - 1) / size) * size // ceil(64/size) * size
- if topK > 0 {
- if aligned := ((topK + size - 1) / size) * size; window > aligned {
- window = aligned
- }
- }
- return window
-}
-
-// calculatePagination calculates offset and limit based on page, size and topK
-func calculatePagination(page, size, topK int) (int, int) {
- if page < 1 {
- page = 1
- }
- if size <= 0 {
- size = 30
- }
- if topK <= 0 {
- topK = 1024
- }
-
- window := rerankWindow(size, topK)
-
- offset := max((page-1)*window, 0)
-
- return offset, window
-}
-
// convertESResponse converts ES SearchResponse to unified chunks format
func convertESResponse(esResp *SearchResponse, vectorFieldName string) []map[string]interface{} {
if esResp == nil || esResp.Hits.Hits == nil {
diff --git a/internal/engine/elasticsearch/chunk_test.go b/internal/engine/elasticsearch/chunk_test.go
index 6222a10014..119da8daab 100644
--- a/internal/engine/elasticsearch/chunk_test.go
+++ b/internal/engine/elasticsearch/chunk_test.go
@@ -449,90 +449,6 @@ var paginationGRID = func() []struct{ size, topK int } {
return out
}()
-// paginate replays the (block-fetch + in-block slice) math that
-// calculatePagination's window is consumed by: for every page whose start is
-// inside the candidate pool, return the in-block page slice. The block is
-// window-aligned, so on the aligned invariant every page is full and the
-// concatenation reconstructs [0, cap).
-func paginate(total, size, topK int) (window, capN int, surfaced []int) {
- window = rerankWindow(size, topK)
- capN = total
- if topK > 0 && capN > topK {
- capN = topK
- }
- for page := 1; (page-1)*size < capN; page++ {
- globalOffset := (page - 1) * size
- blockIndex := globalOffset / window
- blockStart := blockIndex * window
- block := make([]int, 0, window)
- for i := blockStart; i < blockStart+window && i < capN; i++ {
- block = append(block, i)
- }
- begin := globalOffset % window
- end := min(begin+size, len(block))
- surfaced = append(surfaced, block[begin:end]...)
- }
- return window, capN, surfaced
-}
-
-func TestRerankWindowIsPageAligned(t *testing.T) {
- for _, g := range paginationGRID {
- window := rerankWindow(g.size, g.topK)
- if window < 1 {
- t.Errorf("rerankWindow(%d, %d) = %d, want >= 1", g.size, g.topK, window)
- }
- if g.size > 1 && window%g.size != 0 {
- t.Errorf("rerankWindow(%d, %d) = %d, want multiple of %d", g.size, g.topK, window, g.size)
- }
- }
-}
-
-func TestRerankWindowPaginationReconstructsPool(t *testing.T) {
- // Walking every page reconstructs the candidate pool exactly: in order,
- // no gaps, no duplicates, and no short interior pages.
- const total = 250
- for _, g := range paginationGRID {
- window, capN, surfaced := paginate(total, g.size, g.topK)
- if len(surfaced) != capN {
- t.Errorf("size=%d topK=%d: surfaced %d, want %d (window=%d)",
- g.size, g.topK, len(surfaced), capN, window)
- continue
- }
- for i, v := range surfaced {
- if v != i {
- t.Errorf("size=%d topK=%d: surfaced[%d] = %d, want %d (window=%d)",
- g.size, g.topK, i, v, i, window)
- break
- }
- }
- }
-}
-
-func TestCalculatePaginationReportedRegression(t *testing.T) {
- // The reported case: size=10, topK=1024. Legacy min(..., 64) clamped the
- // window to 64 (not a multiple of 10), so page 7 (global offset 60) used
- // to return only 4 of 10 results. With the fix, the window is 70 and
- // page 7 is full and contiguous.
- _, limit := calculatePagination(7, 10, 1024)
- if limit != 70 {
- t.Fatalf("calculatePagination(7, 10, 1024) limit = %d, want 70", limit)
- }
- if limit%10 != 0 {
- t.Fatalf("calculatePagination(7, 10, 1024) limit = %d, want multiple of 10", limit)
- }
-
- // And the simulated end-to-end page walk covers positions 60..69 fully.
- _, capN, surfaced := paginate(250, 10, 1024)
- if capN < 70 || len(surfaced) < 70 {
- t.Fatalf("paginate(250, 10, 1024) returned cap=%d surfaced=%d, want >= 70", capN, len(surfaced))
- }
- for i := 60; i < 70; i++ {
- if surfaced[i] != i {
- t.Errorf("page 7: surfaced[%d] = %d, want %d", i, surfaced[i], i)
- }
- }
-}
-
func TestFormatOrderedTagFeas(t *testing.T) {
tagFeas := map[string]any{
"价格咨询": 4,