RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(search): push pagination down to ListByOwnerIDs (#20386)
cielc05662fce91d
diff --git a/internal/dao/search.go b/internal/dao/search.goindex cd9d99eb8c..4bd7033ab7 100644--- a/internal/dao/search.go+++ b/internal/dao/search.go@@ -18,9 +18,10 @@ package daoimport ("context"- "ragflow/internal/entity""strings"+ "ragflow/internal/entity"+"gorm.io/gorm")@@ -99,11 +100,12 @@ func (dao *SearchDAO) ListByTenantIDs(ctx context.Context, db *gorm.DB, tenantIDreturn searches, total, nil}-// ListByOwnerIDs list searches by owner IDs with filtering (manual pagination)-func (dao *SearchDAO) ListByOwnerIDs(ctx context.Context, db *gorm.DB, ownerIDs []string, userID string, terms []OrderTerm, keywords string) ([]*entity.SearchListItem, int64, error) {+// ListByOwnerIDs list searches by owner IDs with pagination and filtering+func (dao *SearchDAO) ListByOwnerIDs(ctx context.Context, db *gorm.DB, ownerIDs []string, userID string, page, pageSize int, terms []OrderTerm, keywords string) ([]*entity.SearchListItem, int64, error) {var searches []*entity.SearchListItem+ var total int64- // Build query with join to user table+ // Build query with join to user table for nickname and avatarquery := db.WithContext(ctx).Model(&entity.Search{}).Select(`search.*,@@ -128,12 +130,22 @@ func (dao *SearchDAO) ListByOwnerIDs(ctx context.Context, db *gorm.DB, ownerIDs// codeql[go/sql-injection] False positive: searchOrderClausequery = query.Order(searchOrderClause(terms))- // Get all matching records- if err := query.Scan(&searches).Error; err != nil {+ // Count total+ if err := query.Count(&total).Error; err != nil {return nil, 0, err}- total := int64(len(searches))+ // Apply pagination+ if page > 0 && pageSize > 0 {+ offset := (page - 1) * pageSize+ if err := query.Offset(offset).Limit(pageSize).Scan(&searches).Error; err != nil {+ return nil, 0, err+ }+ } else {+ if err := query.Scan(&searches).Error; err != nil {+ return nil, 0, err+ }+ }return searches, total, nil}diff --git a/internal/dao/search_test.go b/internal/dao/search_test.goindex c3ca9c99ab..b6c2b1531d 100644--- a/internal/dao/search_test.go+++ b/internal/dao/search_test.go@@ -17,6 +17,7 @@package daoimport (+ "fmt""testing""gorm.io/gorm"@@ -100,7 +101,7 @@ func TestSearchDAOListByOwnerIDsOrderByExpressionFallsBack(t *testing.T) {for _, orderby := range searchOrderExpressions {t.Run(orderby, func(t *testing.T) {- rows, _, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", []OrderTerm{{Column: orderby}}, "")+ rows, _, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", 1, 10, []OrderTerm{{Column: orderby}}, "")if err != nil {t.Fatalf("ListByOwnerIDs with orderby %q: %v", orderby, err)}@@ -130,3 +131,151 @@ func TestSearchDAOListOrderByAllowedColumn(t *testing.T) {}assertSearchOrder(t, rows, []string{"s-1", "s-2", "s-3"}, "name")}++// assertSearchIDs compares the returned row IDs against wantIDs, naming the+// call under test in the failure message. assertSearchOrder is for the+// ORDER BY allowlist tests, whose failure message talks about expressions+// reaching the ORDER BY clause.+func assertSearchIDs(t *testing.T, rows []*entity.SearchListItem, wantIDs []string, call string) {+ t.Helper()+ gotIDs := make([]string, 0, len(rows))+ for _, row := range rows {+ gotIDs = append(gotIDs, row.ID)+ }+ if len(gotIDs) != len(wantIDs) {+ t.Fatalf("%s returned %v, want %v", call, gotIDs, wantIDs)+ }+ for i := range gotIDs {+ if gotIDs[i] != wantIDs[i] {+ t.Fatalf("%s returned %v, want %v", call, gotIDs, wantIDs)+ }+ }+}++// seedSearchRowsWithCreateTime creates count searches owned by tenantID whose+// create_time grows with their index, so page boundaries are deterministic.+func seedSearchRowsWithCreateTime(t *testing.T, db *gorm.DB, tenantID string, count int) {+ t.Helper()+ for i := 1; i <= count; i++ {+ createTime := int64(i * 100)+ search := entity.Search{+ ID: fmt.Sprintf("s-%d", i),+ TenantID: tenantID,+ Name: fmt.Sprintf("search-%d", i),+ CreatedBy: tenantID,+ SearchConfig: entity.JSONMap{},+ Status: stringPtr("1"),+ BaseModel: entity.BaseModel{CreateTime: &createTime},+ }+ if err := db.Create(&search).Error; err != nil {+ t.Fatalf("failed to create search %s: %v", search.ID, err)+ }+ }+}++// TestSearchDAOListByOwnerIDsPaginatesInSQL checks that page/pageSize reach the+// query as LIMIT/OFFSET while total keeps reporting the full COUNT(*).+// SearchService.ListSearches returns that total as the response's total field+// and no longer slices the returned rows itself.+func TestSearchDAOListByOwnerIDsPaginatesInSQL(t *testing.T) {+ db := setupSearchDetailDAOTestDB(t)+ pushDB(t, db)+ seedSearchRowsWithCreateTime(t, db, "t1", 5)+ ctx := t.Context()+ d := NewSearchDAO()+ terms := []OrderTerm{{Column: "create_time", Desc: true}}++ tests := []struct {+ name string+ page int+ pageSize int+ wantIDs []string+ }{+ {name: "first page", page: 1, pageSize: 2, wantIDs: []string{"s-5", "s-4"}},+ {name: "middle page", page: 2, pageSize: 2, wantIDs: []string{"s-3", "s-2"}},+ {name: "partial last page", page: 3, pageSize: 2, wantIDs: []string{"s-1"}},+ {name: "page past the end", page: 4, pageSize: 2, wantIDs: []string{}},+ {name: "unpaginated returns every row", page: 0, pageSize: 0, wantIDs: []string{"s-5", "s-4", "s-3", "s-2", "s-1"}},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ rows, total, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", tt.page, tt.pageSize, terms, "")+ if err != nil {+ t.Fatalf("ListByOwnerIDs(page=%d, pageSize=%d): %v", tt.page, tt.pageSize, err)+ }+ if total != 5 {+ t.Fatalf("total = %d, want 5: total must count every matching row, not just the page", total)+ }+ assertSearchIDs(t, rows, tt.wantIDs, fmt.Sprintf("page=%d pageSize=%d", tt.page, tt.pageSize))+ })+ }+}++// TestSearchDAOListByOwnerIDsKeepsJoinedOwnerColumnsWhenPaginated guards the+// COUNT followed by the paged Scan: the second statement must still select the+// joined user columns instead of inheriting count(*) from the first one.+func TestSearchDAOListByOwnerIDsKeepsJoinedOwnerColumnsWhenPaginated(t *testing.T) {+ db := setupSearchDetailDAOTestDB(t)+ pushDB(t, db)+ seedSearchRowsWithCreateTime(t, db, "t1", 3)+ ctx := t.Context()++ if err := db.Create(&entity.User{ID: "t1", Nickname: "owner of t1", Status: stringPtr("1")}).Error; err != nil {+ t.Fatalf("failed to create user: %v", err)+ }++ rows, total, err := NewSearchDAO().ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", 1, 2, []OrderTerm{{Column: "create_time", Desc: true}}, "")+ if err != nil {+ t.Fatalf("ListByOwnerIDs: %v", err)+ }+ if total != 3 {+ t.Fatalf("total = %d, want 3", total)+ }+ if len(rows) != 2 {+ t.Fatalf("got %d rows, want 2", len(rows))+ }+ for i, row := range rows {+ if row.Name == "" || row.TenantID != "t1" {+ t.Fatalf("row %d = %+v, want the search columns of tenant t1", i, row.Search)+ }+ if row.Nickname == nil || *row.Nickname != "owner of t1" {+ t.Fatalf("row %d nickname = %v, want owner of t1: the paged scan dropped the joined owner column", i, row.Nickname)+ }+ }+}++// TestSearchDAOListByOwnerIDsKeepsOwnerFilter pins the owner_ids filter: a+// search owned by the requesting user is still excluded when its tenant_id is+// not among ownerIDs, which the endpoint relies on after+// SearchService.filterAccessibleSearchOwnerIDs has narrowed the list.+func TestSearchDAOListByOwnerIDsKeepsOwnerFilter(t *testing.T) {+ db := setupSearchDetailDAOTestDB(t)+ pushDB(t, db)+ ctx := t.Context()+ d := NewSearchDAO()++ for _, tenantID := range []string{"t1", "u-self"} {+ createTime := int64(100)+ search := entity.Search{+ ID: "search-" + tenantID,+ TenantID: tenantID,+ Name: "search " + tenantID,+ CreatedBy: tenantID,+ SearchConfig: entity.JSONMap{},+ Status: stringPtr("1"),+ BaseModel: entity.BaseModel{CreateTime: &createTime},+ }+ if err := db.Create(&search).Error; err != nil {+ t.Fatalf("failed to create search %s: %v", search.ID, err)+ }+ }++ rows, total, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "u-self", 0, 0, nil, "")+ if err != nil {+ t.Fatalf("ListByOwnerIDs: %v", err)+ }+ if total != 1 {+ t.Fatalf("total = %d, want 1: only the t1 search has a tenant_id in ownerIDs", total)+ }+ assertSearchIDs(t, rows, []string{"search-t1"}, "owner filter")+}diff --git a/internal/service/search.go b/internal/service/search.goindex 000c954f58..930f086c0c 100644--- a/internal/service/search.go+++ b/internal/service/search.go@@ -19,11 +19,12 @@ package serviceimport ("context""fmt"+ "strings"+"ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity""ragflow/internal/utility"- "strings")// SearchService search service@@ -102,23 +103,10 @@ func (s *SearchService) ListSearches(ctx context.Context, userID string, keyword}, nil}- searches, total, err = s.searchDAO.ListByOwnerIDs(ctx, dao.DB, ownerIDs, userID, terms, keywords)+ searches, total, err = s.searchDAO.ListByOwnerIDs(ctx, dao.DB, ownerIDs, userID, page, pageSize, terms, keywords)if err != nil {return nil, err}-- if page > 0 && pageSize > 0 {- start := (page - 1) * pageSize- end := start + pageSize- if start < int(total) {- if end > int(total) {- end = int(total)- }- searches = searches[start:end]- } else {- searches = []*entity.SearchListItem{}- }- }}// Convert to response format@@ -220,7 +208,6 @@ func (s *SearchService) CreateSearch(ctx context.Context, userID string, name stexisting, _ := s.searchDAO.GetByNameAndTenant(ctx, dao.DB, name, tid)return len(existing) > 0}, name, userID)-if err != nil {return nil, err}@@ -684,7 +671,6 @@ func (s *SearchService) UpdateSearch(ctx context.Context, userID string, searchI// GetDetail gets search details by ID including search_configfunc (s *SearchService) GetDetail(ctx context.Context, searchID string) (map[string]interface{}, error) {search, err := s.searchDAO.GetByID(ctx, dao.DB, searchID)-if err != nil {return nil, err}diff --git a/internal/service/search_test.go b/internal/service/search_test.goindex d167174875..296c1c9fbc 100644--- a/internal/service/search_test.go+++ b/internal/service/search_test.go@@ -17,6 +17,7 @@package serviceimport (+ "fmt""math""strings""testing"@@ -191,3 +192,56 @@ func TestSearchServiceListSearchesNicknameFallsBackToTenantID(t *testing.T) {t.Fatalf("nickname = %v, want %v", got, want)}}++// TestSearchServiceListSearchesPaginatesOwnerBranch covers the owner_ids branch+// end to end: pagination happens in SQL now, so each page must carry only its+// own rows while total still reports every matching search.+func TestSearchServiceListSearchesPaginatesOwnerBranch(t *testing.T) {+ setupSearchServiceTestDB(t)+ ctx := t.Context()++ for i := 1; i <= 5; i++ {+ createTime := int64(i * 100)+ status := string(entity.StatusValid)+ if err := dao.DB.Create(&entity.Search{+ ID: fmt.Sprintf("search-%d", i),+ TenantID: "tenant-1",+ Name: fmt.Sprintf("Search %d", i),+ CreatedBy: "tenant-1",+ SearchConfig: entity.JSONMap{},+ Status: &status,+ BaseModel: entity.BaseModel{CreateTime: &createTime},+ }).Error; err != nil {+ t.Fatalf("failed to create search %d: %v", i, err)+ }+ }++ tests := []struct {+ name string+ page int+ pageSize int+ wantIDs []string+ }{+ {name: "middle page", page: 2, pageSize: 2, wantIDs: []string{"search-3", "search-2"}},+ {name: "unpaginated returns every row", page: 0, pageSize: 0, wantIDs: []string{"search-5", "search-4", "search-3", "search-2", "search-1"}},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ result, err := NewSearchService().ListSearches(ctx, "tenant-1", "", tt.page, tt.pageSize, []dao.OrderTerm{{Column: "create_time", Desc: true}}, []string{"tenant-1"})+ if err != nil {+ t.Fatalf("ListSearches failed: %v", err)+ }+ if result.Total != 5 {+ t.Fatalf("total = %d, want 5", result.Total)+ }+ if len(result.SearchApps) != len(tt.wantIDs) {+ t.Fatalf("got %d search apps, want %d", len(result.SearchApps), len(tt.wantIDs))+ }+ for i, app := range result.SearchApps {+ if got := app["id"]; got != tt.wantIDs[i] {+ t.Fatalf("search app %d id = %v, want %s", i, got, tt.wantIDs[i])+ }+ }+ })+ }+}