Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(search): push pagination down to ListByOwnerIDs (#20386)

cielc05662fce91d
diff --git a/internal/dao/search.go b/internal/dao/search.go
index cd9d99eb8c..4bd7033ab7 100644
--- a/internal/dao/search.go
+++ b/internal/dao/search.go
@@ -18,9 +18,10 @@ package dao
import (
"context"
- "ragflow/internal/entity"
"strings"
+ "ragflow/internal/entity"
+
"gorm.io/gorm"
)
@@ -99,11 +100,12 @@ func (dao *SearchDAO) ListByTenantIDs(ctx context.Context, db *gorm.DB, tenantID
return searches, total, nil
}
-// ListByOwnerIDs list searches by owner IDs with filtering (manual pagination)
-func (dao *SearchDAO) ListByOwnerIDs(ctx context.Context, db *gorm.DB, ownerIDs []string, userID string, terms []OrderTerm, keywords string) ([]*entity.SearchListItem, int64, error) {
+// ListByOwnerIDs list searches by owner IDs with pagination and filtering
+func (dao *SearchDAO) ListByOwnerIDs(ctx context.Context, db *gorm.DB, ownerIDs []string, userID string, page, pageSize int, terms []OrderTerm, keywords string) ([]*entity.SearchListItem, int64, error) {
var searches []*entity.SearchListItem
+ var total int64
- // Build query with join to user table
+ // Build query with join to user table for nickname and avatar
query := db.WithContext(ctx).Model(&entity.Search{}).
Select(`
search.*,
@@ -128,12 +130,22 @@ func (dao *SearchDAO) ListByOwnerIDs(ctx context.Context, db *gorm.DB, ownerIDs
// codeql[go/sql-injection] False positive: searchOrderClause
query = query.Order(searchOrderClause(terms))
- // Get all matching records
- if err := query.Scan(&searches).Error; err != nil {
+ // Count total
+ if err := query.Count(&total).Error; err != nil {
return nil, 0, err
}
- total := int64(len(searches))
+ // Apply pagination
+ if page > 0 && pageSize > 0 {
+ offset := (page - 1) * pageSize
+ if err := query.Offset(offset).Limit(pageSize).Scan(&searches).Error; err != nil {
+ return nil, 0, err
+ }
+ } else {
+ if err := query.Scan(&searches).Error; err != nil {
+ return nil, 0, err
+ }
+ }
return searches, total, nil
}
diff --git a/internal/dao/search_test.go b/internal/dao/search_test.go
index c3ca9c99ab..b6c2b1531d 100644
--- a/internal/dao/search_test.go
+++ b/internal/dao/search_test.go
@@ -17,6 +17,7 @@
package dao
import (
+ "fmt"
"testing"
"gorm.io/gorm"
@@ -100,7 +101,7 @@ func TestSearchDAOListByOwnerIDsOrderByExpressionFallsBack(t *testing.T) {
for _, orderby := range searchOrderExpressions {
t.Run(orderby, func(t *testing.T) {
- rows, _, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", []OrderTerm{{Column: orderby}}, "")
+ rows, _, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", 1, 10, []OrderTerm{{Column: orderby}}, "")
if err != nil {
t.Fatalf("ListByOwnerIDs with orderby %q: %v", orderby, err)
}
@@ -130,3 +131,151 @@ func TestSearchDAOListOrderByAllowedColumn(t *testing.T) {
}
assertSearchOrder(t, rows, []string{"s-1", "s-2", "s-3"}, "name")
}
+
+// assertSearchIDs compares the returned row IDs against wantIDs, naming the
+// call under test in the failure message. assertSearchOrder is for the
+// ORDER BY allowlist tests, whose failure message talks about expressions
+// reaching the ORDER BY clause.
+func assertSearchIDs(t *testing.T, rows []*entity.SearchListItem, wantIDs []string, call string) {
+ t.Helper()
+ gotIDs := make([]string, 0, len(rows))
+ for _, row := range rows {
+ gotIDs = append(gotIDs, row.ID)
+ }
+ if len(gotIDs) != len(wantIDs) {
+ t.Fatalf("%s returned %v, want %v", call, gotIDs, wantIDs)
+ }
+ for i := range gotIDs {
+ if gotIDs[i] != wantIDs[i] {
+ t.Fatalf("%s returned %v, want %v", call, gotIDs, wantIDs)
+ }
+ }
+}
+
+// seedSearchRowsWithCreateTime creates count searches owned by tenantID whose
+// create_time grows with their index, so page boundaries are deterministic.
+func seedSearchRowsWithCreateTime(t *testing.T, db *gorm.DB, tenantID string, count int) {
+ t.Helper()
+ for i := 1; i <= count; i++ {
+ createTime := int64(i * 100)
+ search := entity.Search{
+ ID: fmt.Sprintf("s-%d", i),
+ TenantID: tenantID,
+ Name: fmt.Sprintf("search-%d", i),
+ CreatedBy: tenantID,
+ SearchConfig: entity.JSONMap{},
+ Status: stringPtr("1"),
+ BaseModel: entity.BaseModel{CreateTime: &createTime},
+ }
+ if err := db.Create(&search).Error; err != nil {
+ t.Fatalf("failed to create search %s: %v", search.ID, err)
+ }
+ }
+}
+
+// TestSearchDAOListByOwnerIDsPaginatesInSQL checks that page/pageSize reach the
+// query as LIMIT/OFFSET while total keeps reporting the full COUNT(*).
+// SearchService.ListSearches returns that total as the response's total field
+// and no longer slices the returned rows itself.
+func TestSearchDAOListByOwnerIDsPaginatesInSQL(t *testing.T) {
+ db := setupSearchDetailDAOTestDB(t)
+ pushDB(t, db)
+ seedSearchRowsWithCreateTime(t, db, "t1", 5)
+ ctx := t.Context()
+ d := NewSearchDAO()
+ terms := []OrderTerm{{Column: "create_time", Desc: true}}
+
+ tests := []struct {
+ name string
+ page int
+ pageSize int
+ wantIDs []string
+ }{
+ {name: "first page", page: 1, pageSize: 2, wantIDs: []string{"s-5", "s-4"}},
+ {name: "middle page", page: 2, pageSize: 2, wantIDs: []string{"s-3", "s-2"}},
+ {name: "partial last page", page: 3, pageSize: 2, wantIDs: []string{"s-1"}},
+ {name: "page past the end", page: 4, pageSize: 2, wantIDs: []string{}},
+ {name: "unpaginated returns every row", page: 0, pageSize: 0, wantIDs: []string{"s-5", "s-4", "s-3", "s-2", "s-1"}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ rows, total, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", tt.page, tt.pageSize, terms, "")
+ if err != nil {
+ t.Fatalf("ListByOwnerIDs(page=%d, pageSize=%d): %v", tt.page, tt.pageSize, err)
+ }
+ if total != 5 {
+ t.Fatalf("total = %d, want 5: total must count every matching row, not just the page", total)
+ }
+ assertSearchIDs(t, rows, tt.wantIDs, fmt.Sprintf("page=%d pageSize=%d", tt.page, tt.pageSize))
+ })
+ }
+}
+
+// TestSearchDAOListByOwnerIDsKeepsJoinedOwnerColumnsWhenPaginated guards the
+// COUNT followed by the paged Scan: the second statement must still select the
+// joined user columns instead of inheriting count(*) from the first one.
+func TestSearchDAOListByOwnerIDsKeepsJoinedOwnerColumnsWhenPaginated(t *testing.T) {
+ db := setupSearchDetailDAOTestDB(t)
+ pushDB(t, db)
+ seedSearchRowsWithCreateTime(t, db, "t1", 3)
+ ctx := t.Context()
+
+ if err := db.Create(&entity.User{ID: "t1", Nickname: "owner of t1", Status: stringPtr("1")}).Error; err != nil {
+ t.Fatalf("failed to create user: %v", err)
+ }
+
+ rows, total, err := NewSearchDAO().ListByOwnerIDs(ctx, db, []string{"t1"}, "t1", 1, 2, []OrderTerm{{Column: "create_time", Desc: true}}, "")
+ if err != nil {
+ t.Fatalf("ListByOwnerIDs: %v", err)
+ }
+ if total != 3 {
+ t.Fatalf("total = %d, want 3", total)
+ }
+ if len(rows) != 2 {
+ t.Fatalf("got %d rows, want 2", len(rows))
+ }
+ for i, row := range rows {
+ if row.Name == "" || row.TenantID != "t1" {
+ t.Fatalf("row %d = %+v, want the search columns of tenant t1", i, row.Search)
+ }
+ if row.Nickname == nil || *row.Nickname != "owner of t1" {
+ t.Fatalf("row %d nickname = %v, want owner of t1: the paged scan dropped the joined owner column", i, row.Nickname)
+ }
+ }
+}
+
+// TestSearchDAOListByOwnerIDsKeepsOwnerFilter pins the owner_ids filter: a
+// search owned by the requesting user is still excluded when its tenant_id is
+// not among ownerIDs, which the endpoint relies on after
+// SearchService.filterAccessibleSearchOwnerIDs has narrowed the list.
+func TestSearchDAOListByOwnerIDsKeepsOwnerFilter(t *testing.T) {
+ db := setupSearchDetailDAOTestDB(t)
+ pushDB(t, db)
+ ctx := t.Context()
+ d := NewSearchDAO()
+
+ for _, tenantID := range []string{"t1", "u-self"} {
+ createTime := int64(100)
+ search := entity.Search{
+ ID: "search-" + tenantID,
+ TenantID: tenantID,
+ Name: "search " + tenantID,
+ CreatedBy: tenantID,
+ SearchConfig: entity.JSONMap{},
+ Status: stringPtr("1"),
+ BaseModel: entity.BaseModel{CreateTime: &createTime},
+ }
+ if err := db.Create(&search).Error; err != nil {
+ t.Fatalf("failed to create search %s: %v", search.ID, err)
+ }
+ }
+
+ rows, total, err := d.ListByOwnerIDs(ctx, db, []string{"t1"}, "u-self", 0, 0, nil, "")
+ if err != nil {
+ t.Fatalf("ListByOwnerIDs: %v", err)
+ }
+ if total != 1 {
+ t.Fatalf("total = %d, want 1: only the t1 search has a tenant_id in ownerIDs", total)
+ }
+ assertSearchIDs(t, rows, []string{"search-t1"}, "owner filter")
+}
diff --git a/internal/service/search.go b/internal/service/search.go
index 000c954f58..930f086c0c 100644
--- a/internal/service/search.go
+++ b/internal/service/search.go
@@ -19,11 +19,12 @@ package service
import (
"context"
"fmt"
+ "strings"
+
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/utility"
- "strings"
)
// SearchService search service
@@ -102,23 +103,10 @@ func (s *SearchService) ListSearches(ctx context.Context, userID string, keyword
}, nil
}
- searches, total, err = s.searchDAO.ListByOwnerIDs(ctx, dao.DB, ownerIDs, userID, terms, keywords)
+ searches, total, err = s.searchDAO.ListByOwnerIDs(ctx, dao.DB, ownerIDs, userID, page, pageSize, terms, keywords)
if err != nil {
return nil, err
}
-
- if page > 0 && pageSize > 0 {
- start := (page - 1) * pageSize
- end := start + pageSize
- if start < int(total) {
- if end > int(total) {
- end = int(total)
- }
- searches = searches[start:end]
- } else {
- searches = []*entity.SearchListItem{}
- }
- }
}
// Convert to response format
@@ -220,7 +208,6 @@ func (s *SearchService) CreateSearch(ctx context.Context, userID string, name st
existing, _ := s.searchDAO.GetByNameAndTenant(ctx, dao.DB, name, tid)
return len(existing) > 0
}, name, userID)
-
if err != nil {
return nil, err
}
@@ -684,7 +671,6 @@ func (s *SearchService) UpdateSearch(ctx context.Context, userID string, searchI
// GetDetail gets search details by ID including search_config
func (s *SearchService) GetDetail(ctx context.Context, searchID string) (map[string]interface{}, error) {
search, err := s.searchDAO.GetByID(ctx, dao.DB, searchID)
-
if err != nil {
return nil, err
}
diff --git a/internal/service/search_test.go b/internal/service/search_test.go
index d167174875..296c1c9fbc 100644
--- a/internal/service/search_test.go
+++ b/internal/service/search_test.go
@@ -17,6 +17,7 @@
package service
import (
+ "fmt"
"math"
"strings"
"testing"
@@ -191,3 +192,56 @@ func TestSearchServiceListSearchesNicknameFallsBackToTenantID(t *testing.T) {
t.Fatalf("nickname = %v, want %v", got, want)
}
}
+
+// TestSearchServiceListSearchesPaginatesOwnerBranch covers the owner_ids branch
+// end to end: pagination happens in SQL now, so each page must carry only its
+// own rows while total still reports every matching search.
+func TestSearchServiceListSearchesPaginatesOwnerBranch(t *testing.T) {
+ setupSearchServiceTestDB(t)
+ ctx := t.Context()
+
+ for i := 1; i <= 5; i++ {
+ createTime := int64(i * 100)
+ status := string(entity.StatusValid)
+ if err := dao.DB.Create(&entity.Search{
+ ID: fmt.Sprintf("search-%d", i),
+ TenantID: "tenant-1",
+ Name: fmt.Sprintf("Search %d", i),
+ CreatedBy: "tenant-1",
+ SearchConfig: entity.JSONMap{},
+ Status: &status,
+ BaseModel: entity.BaseModel{CreateTime: &createTime},
+ }).Error; err != nil {
+ t.Fatalf("failed to create search %d: %v", i, err)
+ }
+ }
+
+ tests := []struct {
+ name string
+ page int
+ pageSize int
+ wantIDs []string
+ }{
+ {name: "middle page", page: 2, pageSize: 2, wantIDs: []string{"search-3", "search-2"}},
+ {name: "unpaginated returns every row", page: 0, pageSize: 0, wantIDs: []string{"search-5", "search-4", "search-3", "search-2", "search-1"}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result, err := NewSearchService().ListSearches(ctx, "tenant-1", "", tt.page, tt.pageSize, []dao.OrderTerm{{Column: "create_time", Desc: true}}, []string{"tenant-1"})
+ if err != nil {
+ t.Fatalf("ListSearches failed: %v", err)
+ }
+ if result.Total != 5 {
+ t.Fatalf("total = %d, want 5", result.Total)
+ }
+ if len(result.SearchApps) != len(tt.wantIDs) {
+ t.Fatalf("got %d search apps, want %d", len(result.SearchApps), len(tt.wantIDs))
+ }
+ for i, app := range result.SearchApps {
+ if got := app["id"]; got != tt.wantIDs[i] {
+ t.Fatalf("search app %d id = %v, want %s", i, got, tt.wantIDs[i])
+ }
+ }
+ })
+ }
+}