RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Go: support ids filter in list datasets API (#17705)
euvre04918f88bfa7
diff --git a/internal/dao/kb.go b/internal/dao/kb.goindex 1eda211460..e10ac12002 100644--- a/internal/dao/kb.go+++ b/internal/dao/kb.go@@ -173,7 +173,7 @@ func (dao *KnowledgebaseDAO) Count(ctx context.Context, db *gorm.DB, filters map// GetByTenantIDs retrieves knowledge bases by tenant IDs with pagination// This matches the Python get_by_tenant_ids method-func (dao *KnowledgebaseDAO) GetByTenantIDs(ctx context.Context, db *gorm.DB, tenantIDs []string, userID string, pageNumber, itemsPerPage int, orderby string, desc bool, keywords, parserID, id, name string) ([]*entity.KnowledgebaseListItem, int64, error) {+func (dao *KnowledgebaseDAO) GetByTenantIDs(ctx context.Context, db *gorm.DB, tenantIDs []string, userID string, pageNumber, itemsPerPage int, orderby string, desc bool, keywords, parserID, id, name string, ids []string) ([]*entity.KnowledgebaseListItem, int64, error) {var kbs []*entity.KnowledgebaseListItemvar total int64@@ -193,6 +193,10 @@ func (dao *KnowledgebaseDAO) GetByTenantIDs(ctx context.Context, db *gorm.DB, tequery = query.Where("knowledgebase.id = ?", id)}+ if len(ids) > 0 {+ query = query.Where("knowledgebase.id IN ?", ids)+ }+if name != "" {query = query.Where("knowledgebase.name = ?", name)}@@ -318,6 +322,21 @@ func (dao *KnowledgebaseDAO) Accessible(ctx context.Context, db *gorm.DB, datasereturn count > 0}+// GetAccessibleIDs returns the subset of ids that are visible to the user:+// team-permission KBs owned by any joined tenant, plus the user's own KBs.+// This matches the Python get_accessible_ids method.+func (dao *KnowledgebaseDAO) GetAccessibleIDs(ctx context.Context, db *gorm.DB, joinedTenantIDs []string, userID string, ids []string) ([]string, error) {+ accessibleIDs := make([]string, 0, len(ids))+ err := db.WithContext(ctx).Model(&entity.Knowledgebase{}).+ Where("id IN ? AND ((tenant_id IN ? AND permission = ?) OR tenant_id = ?) AND status = ?",+ ids, joinedTenantIDs, string(entity.TenantPermissionTeam), userID, string(entity.StatusValid)).+ Pluck("id", &accessibleIDs).Error+ if err != nil {+ return nil, err+ }+ return accessibleIDs, nil+}+// Accessible4Deletion checks if a knowledge base can be deleted by a user// This matches the Python accessible4deletion methodfunc (dao *KnowledgebaseDAO) Accessible4Deletion(ctx context.Context, db *gorm.DB, kbID, userID string) bool {diff --git a/internal/handler/dataset.go b/internal/handler/dataset.goindex cb2748c86b..37feaec5e4 100644--- a/internal/handler/dataset.go+++ b/internal/handler/dataset.go@@ -168,6 +168,39 @@ func (h *DatasetsHandler) ListDatasets(c *gin.Context) {}}+ // Mirror pydantic's ListDatasetReq.ids: each occurrence is comma-split,+ // every value must be a valid UUID, and duplicates are rejected.+ var ids []string+ if rawIDs, exists := c.Request.URL.Query()["ids"]; exists {+ seen := make(map[string]int)+ for _, item := range rawIDs {+ for _, value := range strings.Split(item, ",") {+ if value == "" {+ continue+ }+ normalizedID, err := dataset.NormalizeDatasetID(strings.TrimSpace(value))+ if err != nil {+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, err.Error())+ return+ }+ seen[normalizedID]+++ ids = append(ids, normalizedID)+ }+ }+ duplicates := make([]string, 0, len(ids))+ reported := make(map[string]bool)+ for _, normalizedID := range ids {+ if seen[normalizedID] > 1 && !reported[normalizedID] {+ reported[normalizedID] = true+ duplicates = append(duplicates, normalizedID)+ }+ }+ if len(duplicates) > 0 {+ common.ResponseWithCodeData(c, common.CodeArgumentError, nil, fmt.Sprintf("Duplicate ids: '%s'", strings.Join(duplicates, ", ")))+ return+ }+ }+ctx := c.Request.Context()data, total, code, err := h.datasetsService.ListDatasets(ctx,@@ -181,6 +214,7 @@ func (h *DatasetsHandler) ListDatasets(c *gin.Context) {ownerIDs,parserID,user.ID,+ ids,)if err != nil {common.ErrorWithCode(c, code, err.Error())@@ -326,7 +360,7 @@ func pythonJSONTypeName(v interface{}) string {// ListDatasetReq (BaseListReq + include_parsing_status/ext; `type` is handled// before validation in the Python endpoint).var listDatasetsAllowedParams = map[string]bool{- "id": true, "name": true, "page": true, "page_size": true,+ "id": true, "ids": true, "name": true, "page": true, "page_size": true,"orderby": true, "desc": true, "include_parsing_status": true,"ext": true, "type": true,}diff --git a/internal/handler/dataset_list_test.go b/internal/handler/dataset_list_test.gonew file mode 100644index 0000000000..6792300946--- /dev/null+++ b/internal/handler/dataset_list_test.go@@ -0,0 +1,147 @@+package handler++import (+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "net/url"+ "testing"++ "github.com/gin-gonic/gin"+ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ dataset "ragflow/internal/service/dataset"+)++const (+ listDatasetsTestKBID = "123e4567e89b12d3a456426614174000"+ listDatasetsTestKBIDDashed = "123e4567-e89b-12d3-a456-426614174000"+)++func setupListDatasetsTestDB(t *testing.T) *gorm.DB {+ t.Helper()++ db, err := gorm.Open(sqlite.Open("file:"+url.QueryEscape(t.Name())+"?mode=memory&cache=shared"), &gorm.Config{+ TranslateError: true,+ })+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }++ if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.User{}, &entity.UserTenant{}, &entity.Tenant{}); err != nil {+ t.Fatalf("failed to migrate test schema: %v", err)+ }++ origDB := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = origDB })++ return db+}++func insertListDatasetsTestKB(t *testing.T, id, tenantID, name string) {+ t.Helper()++ status := string(entity.StatusValid)+ kb := &entity.Knowledgebase{+ ID: id,+ TenantID: tenantID,+ Name: name,+ EmbdID: "BAAI/bge-large-zh-v1.5@Builtin",+ CreatedBy: tenantID,+ Permission: string(entity.TenantPermissionMe),+ ParserID: string(entity.ParserTypeNaive),+ ParserConfig: entity.JSONMap{"chunk_token_num": float64(128)},+ Status: &status,+ }+ if err := dao.DB.Create(kb).Error; err != nil {+ t.Fatalf("insert test kb: %v", err)+ }+}++func newListDatasetsTestRouter() *gin.Engine {+ gin.SetMode(gin.TestMode)+ h := NewDatasetsHandler(dataset.NewDatasetService(), nil)+ r := gin.New()+ r.GET("/api/v1/datasets", func(c *gin.Context) {+ c.Set("user", &entity.User{ID: "user-1"})+ h.ListDatasets(c)+ })+ return r+}++type listDatasetsTestResponse struct {+ Code int `json:"code"`+ Data []map[string]interface{} `json:"data"`+ Message string `json:"message"`+ TotalDatasets int64 `json:"total_datasets"`+}++func getListDatasets(t *testing.T, r *gin.Engine, rawQuery string) listDatasetsTestResponse {+ t.Helper()++ resp := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodGet, "/api/v1/datasets?"+rawQuery, nil)+ r.ServeHTTP(resp, req)++ if resp.Code != http.StatusOK {+ t.Fatalf("status=%d body=%s", resp.Code, resp.Body.String())+ }++ var body listDatasetsTestResponse+ if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {+ t.Fatalf("unmarshal response: %v body=%s", err, resp.Body.String())+ }+ return body+}++func TestDatasetsHandlerListDatasetsFiltersByIDs(t *testing.T) {+ setupListDatasetsTestDB(t)+ insertListDatasetsTestKB(t, listDatasetsTestKBID, "user-1", "Alpha")++ body := getListDatasets(t, newListDatasetsTestRouter(),+ fmt.Sprintf("ids=%s&page_size=1", listDatasetsTestKBIDDashed))++ if body.Code != int(common.CodeSuccess) {+ t.Fatalf("code=%d message=%q", body.Code, body.Message)+ }+ if body.TotalDatasets != 1 || len(body.Data) != 1 {+ t.Fatalf("expected exactly one dataset, got total=%d len=%d", body.TotalDatasets, len(body.Data))+ }+ if body.Data[0]["id"] != listDatasetsTestKBID {+ t.Fatalf("expected dataset id %q, got %#v", listDatasetsTestKBID, body.Data[0]["id"])+ }+}++func TestDatasetsHandlerListDatasetsRejectsInvalidIDInIDs(t *testing.T) {+ setupListDatasetsTestDB(t)++ body := getListDatasets(t, newListDatasetsTestRouter(), "ids=not-a-uuid")++ if body.Code != int(common.CodeArgumentError) {+ t.Fatalf("code=%d want=%d", body.Code, common.CodeArgumentError)+ }+ if body.Message != "Invalid UUID format" {+ t.Fatalf("message=%q want=%q", body.Message, "Invalid UUID format")+ }+}++func TestDatasetsHandlerListDatasetsRejectsDuplicateIDs(t *testing.T) {+ setupListDatasetsTestDB(t)++ rawQuery := fmt.Sprintf("ids=%s,%s", listDatasetsTestKBIDDashed, listDatasetsTestKBIDDashed)+ body := getListDatasets(t, newListDatasetsTestRouter(), rawQuery)++ if body.Code != int(common.CodeArgumentError) {+ t.Fatalf("code=%d want=%d", body.Code, common.CodeArgumentError)+ }+ expected := fmt.Sprintf("Duplicate ids: '%s'", listDatasetsTestKBID)+ if body.Message != expected {+ t.Fatalf("message=%q want=%q", body.Message, expected)+ }+}diff --git a/internal/handler/mcp_server.go b/internal/handler/mcp_server.goindex fe0334ce80..d7a8c509d7 100644--- a/internal/handler/mcp_server.go+++ b/internal/handler/mcp_server.go@@ -35,7 +35,7 @@ import (// by the MCP server handler.type MCPRetrievalService interface {SearchDatasets(req *service.SearchDatasetsRequest, userID string) (*service.SearchDatasetsResponse, error)- ListDatasets(id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string) ([]map[string]interface{}, int64, common.ErrorCode, error)+ ListDatasets(id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string, ids []string) ([]map[string]interface{}, int64, common.ErrorCode, error)}// MCPServerHandler handles MCP protocol requests (JSON-RPC over HTTP).@@ -118,7 +118,7 @@ func (h *MCPServerHandler) HandleMCP(c *gin.Context) {func MCPListDatasets(ctx context.Context, ds *dataset.DatasetService, userID string, page, pageSize int, orderby string, desc bool) ([]map[string]interface{}, int64, error) {data, total, _, err := ds.ListDatasets(ctx,"", "", page, pageSize, orderby, desc,- "", nil, "", userID,+ "", nil, "", userID, nil,)return data, total, err}@@ -154,7 +154,7 @@ func MCPRetrieval(ctx context.Context, ds *dataset.DatasetService, userID stringfor {data, _, _, err := ds.ListDatasets(ctx,"", "", page, maxPageSize, "create_time", true,- "", nil, "", userID,+ "", nil, "", userID, nil,)if err != nil {return "", fmt.Errorf("cannot resolve accessible datasets: %w", err)diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex 7f88d34ed7..959b2356b2 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -328,8 +328,11 @@ func (d *DatasetService) deleteDataset(tenantID string, kb *entity.Knowledgebase})}-func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string) ([]map[string]interface{}, int64, common.ErrorCode, error) {+func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page, pageSize int, orderby string, desc bool, keywords string, ownerIDs []string, parserID, userID string, ids []string) ([]map[string]interface{}, int64, common.ErrorCode, error) {id = strings.TrimSpace(id)+ if id != "" && len(ids) > 0 {+ return nil, 0, common.CodeDataError, fmt.Errorf("Should not provide both 'id':%s and 'ids'%s", id, pythonStringListRepr(ids))+ }if id != "" {normalizedID, err := normalizeDatasetID(id)if err != nil {@@ -380,6 +383,7 @@ func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, page}}queryUserID := userID+ var joinedTenantIDs []stringif len(tenantIDs) > 0 {joinedTenants, err := d.tenantDAO.GetJoinedTenantsByUserID(ctx, dao.DB, userID)if err != nil {@@ -391,6 +395,7 @@ func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, pagecontinue}allowedTenantIDs[joinedTenant.TenantID] = struct{}{}+ joinedTenantIDs = append(joinedTenantIDs, joinedTenant.TenantID)}filteredTenantIDs := tenantIDs[:0]queryUserID = ""@@ -414,10 +419,33 @@ func (d *DatasetService) ListDatasets(ctx context.Context, id, name string, pagecontinue}tenantIDs = append(tenantIDs, joinedTenant.TenantID)+ joinedTenantIDs = append(joinedTenantIDs, joinedTenant.TenantID)+ }+ }++ // Mirror Python: ids are checked for accessibility against the joined+ // tenants (not the owner-filtered tenant list) before filtering.+ if len(ids) > 0 {+ accessibleIDs, err := d.kbDAO.GetAccessibleIDs(ctx, dao.DB, joinedTenantIDs, userID, ids)+ if err != nil {+ return nil, 0, common.CodeServerError, errors.New("database operation failed")+ }+ accessible := make(map[string]struct{}, len(accessibleIDs))+ for _, accessibleID := range accessibleIDs {+ accessible[accessibleID] = struct{}{}+ }+ deniedIDs := make([]string, 0, len(ids))+ for _, datasetID := range ids {+ if _, ok := accessible[datasetID]; !ok {+ deniedIDs = append(deniedIDs, datasetID)+ }+ }+ if len(deniedIDs) > 0 {+ return nil, 0, common.CodeDataError, fmt.Errorf("User '%s' lacks permission for datasets: '%s'", userID, strings.Join(deniedIDs, ", "))}}- kbs, total, err := d.kbDAO.GetByTenantIDs(ctx, dao.DB, tenantIDs, queryUserID, page, pageSize, orderby, desc, keywords, parserID, id, name)+ kbs, total, err := d.kbDAO.GetByTenantIDs(ctx, dao.DB, tenantIDs, queryUserID, page, pageSize, orderby, desc, keywords, parserID, id, name, ids)if err != nil {return nil, 0, common.CodeServerError, errors.New("database operation failed")}diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex 36966489c2..d1559b594d 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -180,6 +180,16 @@ func normalizeDatasetID(id string) (string, error) {return strings.ReplaceAll(parsedUUID.String(), "-", ""), nil}+// pythonStringListRepr renders a string slice the way Python prints a list of+// strings, e.g. ['a', 'b'], for error messages that mirror the Python API.+func pythonStringListRepr(items []string) string {+ quoted := make([]string, 0, len(items))+ for _, item := range items {+ quoted = append(quoted, "'"+item+"'")+ }+ return "[" + strings.Join(quoted, ", ") + "]"+}+func canvasAccessibleForUser(ctx context.Context, userID, canvasID string) (bool, error) {tenantIDs, _ := dao.NewUserTenantDAO().GetTenantIDsByUserID(ctx, dao.DB, userID)return dao.NewUserCanvasDAO().Accessible(ctx, dao.DB, canvasID, userID, tenantIDs), nildiff --git a/internal/service/dataset/list_test.go b/internal/service/dataset/list_test.gonew file mode 100644index 0000000000..7c2e072c9c--- /dev/null+++ b/internal/service/dataset/list_test.go@@ -0,0 +1,119 @@+package dataset++import (+ "strings"+ "testing"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+)++func testDatasetListService(t *testing.T) *DatasetService {+ t.Helper()++ return &DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ documentDAO: dao.NewDocumentDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }+}++func TestDatasetServiceListDatasetsFiltersByIDs(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Alpha")+ insertDatasetUpdateKB(t, "kb-2", "tenant-1", "Beta")++ ctx := t.Context()+ data, total, code, err := testDatasetListService(t).ListDatasets(ctx,+ "", "", 1, 30, "create_time", true,+ "", nil, "", "tenant-1", []string{"kb-1"},+ )+ if err != nil {+ t.Fatalf("ListDatasets failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }+ if total != 1 || len(data) != 1 {+ t.Fatalf("expected exactly one dataset, got total=%d len=%d", total, len(data))+ }+ if data[0]["id"] != "kb-1" {+ t.Fatalf("expected kb-1, got %#v", data[0]["id"])+ }+}++func TestDatasetServiceListDatasetsIDsAccessibleViaTeamTenant(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-team", "owner-1", "Shared")+ if err := dao.DB.Create(&entity.Tenant{ID: "owner-1", Name: sptr("owner"), Status: sptr("1")}).Error; err != nil {+ t.Fatalf("insert owner tenant: %v", err)+ }+ insertDatasetUpdateTeamMember(t, "user-1", "owner-1")+ if err := dao.DB.Model(&entity.Knowledgebase{}).+ Where("id = ?", "kb-team").+ Update("permission", string(entity.TenantPermissionTeam)).Error; err != nil {+ t.Fatalf("update kb permission: %v", err)+ }++ ctx := t.Context()+ data, total, code, err := testDatasetListService(t).ListDatasets(ctx,+ "", "", 1, 30, "create_time", true,+ "", nil, "", "user-1", []string{"kb-team"},+ )+ if err != nil {+ t.Fatalf("ListDatasets failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }+ if total != 1 || len(data) != 1 || data[0]["id"] != "kb-team" {+ t.Fatalf("expected the shared dataset, got total=%d data=%#v", total, data)+ }+}++func TestDatasetServiceListDatasetsRejectsIDAndIDsTogether(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-1", "tenant-1", "Alpha")++ ctx := t.Context()+ _, _, code, err := testDatasetListService(t).ListDatasets(ctx,+ "kb-1", "", 1, 30, "create_time", true,+ "", nil, "", "tenant-1", []string{"kb-1"},+ )+ if err == nil {+ t.Fatal("expected id/ids conflict error")+ }+ if code != common.CodeDataError {+ t.Fatalf("expected data error code, got %d", code)+ }+ expected := "Should not provide both 'id':kb-1 and 'ids'['kb-1']"+ if err.Error() != expected {+ t.Fatalf("unexpected error: %v", err)+ }+}++func TestDatasetServiceListDatasetsRejectsDeniedIDs(t *testing.T) {+ db := setupDatasetUpdateTestDB(t)+ pushServiceDB(t, db)+ insertDatasetUpdateKB(t, "kb-private", "owner-1", "Private")++ ctx := t.Context()+ _, _, code, err := testDatasetListService(t).ListDatasets(ctx,+ "", "", 1, 30, "create_time", true,+ "", nil, "", "user-1", []string{"kb-private"},+ )+ if err == nil {+ t.Fatal("expected permission error")+ }+ if code != common.CodeDataError {+ t.Fatalf("expected data error code, got %d", code)+ }+ expected := "User 'user-1' lacks permission for datasets: 'kb-private'"+ if !strings.Contains(err.Error(), expected) {+ t.Fatalf("unexpected error: %v", err)+ }+}