RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(memory): expose server-side filter discovery (#19840)
Sbaaoui Idriss67aa682027a2
diff --git a/api/apps/restful_apis/memory_api.py b/api/apps/restful_apis/memory_api.pyindex 2152957ac7..56e761b304 100644--- a/api/apps/restful_apis/memory_api.py+++ b/api/apps/restful_apis/memory_api.py@@ -139,6 +139,13 @@ async def delete_memory(memory_id):@manager.route("/memories", methods=["GET"]) # noqa: F821@login_requiredasync def list_memory():+ if request.args.get("type") == "filter":+ try:+ return get_json_result(message=True, data=await memory_api_service.list_memory_filters())+ except Exception as e:+ logging.error(e)+ return get_json_result(code=RetCode.SERVER_ERROR, message="Internal server error")+# Every list filter accepts repeated query keys and comma-joined values. The getlist idiom comes from# api/utils/validation_utils.py, which does not strip; the strip matches _split_filter_values in api/apps/services/memory_api_service.py.filter_params = {diff --git a/api/apps/services/memory_api_service.py b/api/apps/services/memory_api_service.pyindex 78e8f116e9..9cb23a5781 100644--- a/api/apps/services/memory_api_service.py+++ b/api/apps/services/memory_api_service.py@@ -291,6 +291,24 @@ async def list_memory(filter_params: dict, keywords: str, page: int = 1, page_sireturn {"memory_list": memory_list, "total_count": count}+async def list_memory_filters():+ """Return filter aggregations for all memories visible to the caller."""+ filter_dict = {"tenant_id": list(_joined_tenant_ids(current_user.id)), "accessible_user_id": current_user.id}+ _, total = MemoryService.get_by_filter(filter_dict, None, 1, 1)+ memories, _ = MemoryService.get_by_filter(filter_dict, None, 1, max(total, 1))+ owner = {}+ memory_type = {}+ storage_type = {}+ for memory in memories:+ owner_item = owner.setdefault(memory["tenant_id"], {"id": memory["tenant_id"], "label": memory.get("owner_name") or memory["tenant_id"], "count": 0})+ owner_item["count"] += 1+ for item in get_memory_type_human(memory["memory_type"]):+ memory_type.setdefault(item, {"id": item, "label": item, "count": 0})["count"] += 1+ item = memory["storage_type"]+ storage_type.setdefault(item, {"id": item, "label": item, "count": 0})["count"] += 1+ return {"filter": {"owner": list(owner.values()), "memory_type": list(memory_type.values()), "storage_type": list(storage_type.values())}, "total": total}++async def get_memory_config(memory_id):memory = MemoryService.get_with_owner_name_by_id(memory_id)if not memory or not _memory_accessible(memory):diff --git a/internal/dao/memory.go b/internal/dao/memory.goindex c93bf33cb6..1e64fbced1 100644--- a/internal/dao/memory.go+++ b/internal/dao/memory.go@@ -348,7 +348,6 @@ func (dao *MemoryDAO) GetByFilter(ctx context.Context, db *gorm.DB, userID strinreturn nil, 0, err}- offset := (page - 1) * pageSizequerySQL := fmt.Sprintf(`SELECT m.id, m.name, m.avatar, m.tenant_id, m.memory_type,m.storage_type, m.embd_id, m.tenant_embd_id, m.llm_id, m.tenant_llm_id,@@ -360,10 +359,12 @@ func (dao *MemoryDAO) GetByFilter(ctx context.Context, db *gorm.DB, userID strinLEFT JOIN user u ON m.tenant_id = u.id%sORDER BY m.create_time DESC- LIMIT ? OFFSET ?`, whereClause)-- queryArgs := append(args, pageSize, offset)+ queryArgs := args+ if pageSize > 0 {+ querySQL += " LIMIT ? OFFSET ?"+ queryArgs = append(queryArgs, pageSize, (page-1)*pageSize)+ }var rawResults []struct {entity.Memorydiff --git a/internal/handler/memory.go b/internal/handler/memory.goindex 66bcb3605e..6f7f665de5 100644--- a/internal/handler/memory.go+++ b/internal/handler/memory.go@@ -335,6 +335,15 @@ func (h *MemoryHandler) ListMemories(c *gin.Context) {common.ErrorWithCode(c, errorCode, errorMessage)return}+ if c.Query("type") == "filter" {+ result, err := h.memoryService.ListMemoryFilters(c.Request.Context(), user.ID)+ if err != nil {+ common.ResponseWithCodeData(c, common.CodeServerError, nil, err.Error())+ return+ }+ common.SuccessWithData(c, result, "success")+ return+ }// Parse query parametersstorageType := c.Query("storage_type")diff --git a/internal/service/memory.go b/internal/service/memory.goindex f3203f4ccb..0a6178c0d3 100644--- a/internal/service/memory.go+++ b/internal/service/memory.go@@ -360,6 +360,66 @@ type ListMemoryResponse struct {TotalCount int64 `json:"total_count"`}+type MemoryFilterOption struct {+ ID string `json:"id"`+ Label string `json:"label"`+ Count int64 `json:"count"`+}++type MemoryFiltersResponse struct {+ Filter struct {+ Owner []MemoryFilterOption `json:"owner"`+ MemoryType []MemoryFilterOption `json:"memory_type"`+ StorageType []MemoryFilterOption `json:"storage_type"`+ } `json:"filter"`+ Total int64 `json:"total"`+}++func (s *MemoryService) ListMemoryFilters(ctx context.Context, userID string) (*MemoryFiltersResponse, error) {+ userTenants, err := NewUserTenantService().GetUserTenantRelationByUserIDWithContext(ctx, userID)+ if err != nil {+ return nil, fmt.Errorf("failed to get user tenants: %w", err)+ }+ tenantIDs := make([]string, 0, len(userTenants)+1)+ tenantIDs = append(tenantIDs, userID)+ for _, tenant := range userTenants {+ tenantIDs = append(tenantIDs, tenant.TenantID)+ }+ memories, _, err := s.memoryDAO.GetByFilter(ctx, dao.DB, userID, tenantIDs, nil, "", "", 1, 0)+ if err != nil {+ return nil, err+ }+ ownerCounts := map[string]int64{}+ ownerLabels := map[string]string{}+ typeCounts := map[string]int64{}+ storageCounts := map[string]int64{}+ for _, memory := range memories {+ owner := memory.TenantID+ label := owner+ if memory.OwnerName != nil && *memory.OwnerName != "" {+ label = *memory.OwnerName+ }+ ownerCounts[owner]+++ ownerLabels[owner] = label+ for _, memoryType := range dao.GetMemoryTypeHuman(memory.MemoryType) {+ typeCounts[memoryType]+++ }+ storageCounts[memory.StorageType]+++ }+ resp := &MemoryFiltersResponse{}+ for id, count := range ownerCounts {+ resp.Filter.Owner = append(resp.Filter.Owner, MemoryFilterOption{ID: id, Label: ownerLabels[id], Count: count})+ }+ for id, count := range typeCounts {+ resp.Filter.MemoryType = append(resp.Filter.MemoryType, MemoryFilterOption{ID: id, Label: id, Count: count})+ }+ for id, count := range storageCounts {+ resp.Filter.StorageType = append(resp.Filter.StorageType, MemoryFilterOption{ID: id, Label: id, Count: count})+ }+ resp.Total = int64(len(memories))+ return resp, nil+}+// CreateMemory creates a new memory with the given parameters// It validates the request, generates a unique name if needed, and creates the memory record//diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.goindex 7a7fe44739..cb59378fec 100644--- a/internal/service/memory_message_test.go+++ b/internal/service/memory_message_test.go@@ -189,6 +189,35 @@ func setupMemoryMessageTestDB(t *testing.T) {})}+func TestListMemoryFiltersUsesAccessibleMemoryAggregates(t *testing.T) {+ setupMemoryMessageTestDB(t)+ if err := dao.DB.Create(&entity.User{ID: "user-1", Nickname: "Owner"}).Error; err != nil {+ t.Fatalf("seed user: %v", err)+ }+ for _, memory := range []*entity.Memory{+ {ID: "mem-1", Name: "one", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw | dao.MemoryTypeSemantic, StorageType: "table", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO)},+ {ID: "mem-2", Name: "two", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw, StorageType: "graph", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO)},+ } {+ if err := dao.DB.Create(memory).Error; err != nil {+ t.Fatalf("seed memory: %v", err)+ }+ }++ filters, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")+ if err != nil {+ t.Fatalf("ListMemoryFilters: %v", err)+ }+ if filters.Total != 2 || len(filters.Filter.Owner) != 1 {+ t.Fatalf("unexpected owner aggregate: %+v", filters)+ }+ if filters.Filter.Owner[0].Label != "Owner" || filters.Filter.Owner[0].Count != 2 {+ t.Fatalf("owner filter = %+v", filters.Filter.Owner)+ }+ if len(filters.Filter.MemoryType) != 2 || len(filters.Filter.StorageType) != 2 {+ t.Fatalf("unexpected type/storage filters: %+v", filters)+ }+}+func TestForgetMessageKeepsCompanionFieldForNonOceanBaseEngines(t *testing.T) {setupMemoryMessageTestDB(t)diff --git a/web/src/pages/memories/hooks.ts b/web/src/pages/memories/hooks.tsindex ca9ad84ecf..4a9c8e1d80 100644--- a/web/src/pages/memories/hooks.ts+++ b/web/src/pages/memories/hooks.ts@@ -9,11 +9,6 @@ import { useHandleSearchChange } from '@/hooks/logic-hooks';import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';import memoryService, { updateMemoryById } from '@/services/memory-service';import { markListItemsDeleted } from '@/utils/list-deletion-util';-import {- buildOwnersFilter,- groupListByArray,- groupListByType,-} from '@/utils/list-filter-util';import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';import { useDebounce } from 'ahooks';import { omit } from 'lodash';@@ -30,10 +25,17 @@ import {IMemoryAppDetailProps,MemoryDetailResponse,MemoryListResponse,+ MemoryFiltersResponse,} from './interface';+export const MemoryKeys = {+ list: () => ['memoryList'] as const,+ filters: () => ['memoryFilters'] as const,+};+export const useCreateMemory = () => {const { t } = useTranslation();+ const queryClient = useQueryClient();const createMemory = useCallback(async (props: ICreateMemoryProps): Promise<CreateMemoryResponse> => {@@ -43,10 +45,11 @@ export const useCreateMemory = () => {}if (response.code === 0) {message.success(t('message.created'));+ queryClient.invalidateQueries({ queryKey: MemoryKeys.filters() });}return response.data;},- [t],+ [queryClient, t],);return { createMemory };@@ -87,7 +90,7 @@ export const useFetchMemoryList = () => {Error>({queryKey: [- 'memoryList',+ ...MemoryKeys.list(),{debouncedSearchString,...pagination,@@ -105,7 +108,6 @@ export const useFetchMemoryList = () => {if (response.code !== 0) {throw new Error(response.message || 'Failed to fetch memory list');}- console.log(response);return response;},});@@ -133,6 +135,32 @@ export const useFetchMemoryList = () => {};};+export const useFetchMemoryFilters = () => {+ const {+ data,+ isFetching: isLoading,+ isError,+ } = useQuery<MemoryFiltersResponse, Error>({+ queryKey: MemoryKeys.filters(),+ initialData: {+ filter: { owner: [], memory_type: [], storage_type: [] },+ total: 0,+ },+ queryFn: async () => {+ const { data: response } = await memoryService.getMemoryFilters(+ { params: { type: 'filter' } },+ true,+ );+ if (response.code !== 0) {+ throw new Error(response.message || 'Failed to fetch memory filters');+ }+ return response.data;+ },+ });++ return { data, isLoading, isError };+};+export const useFetchMemoryDetail = (tenantId?: string) => {const { id } = useParams();@@ -184,7 +212,8 @@ export const useDeleteMemory = () => {throw new Error(response.message || 'Failed to delete memory');}- queryClient.invalidateQueries({ queryKey: ['memoryList'] });+ queryClient.invalidateQueries({ queryKey: MemoryKeys.list() });+ queryClient.invalidateQueries({ queryKey: MemoryKeys.filters() });markListItemsDeleted(ListDeletionKey.MemoryList);return response;},@@ -232,6 +261,7 @@ export const useUpdateMemory = () => {queryClient.invalidateQueries({queryKey: [MemoryApiAction.FetchMemoryDetail],});+ queryClient.invalidateQueries({ queryKey: MemoryKeys.filters() });},});@@ -315,49 +345,37 @@ export const useRenameMemory = () => {};/**- * Build the filter facet collections for the memory list page from the- * memory items that are already loaded by the page's list query.- *- * The filters must be derived from the same query data that renders the- * memory cards so that creating, updating or deleting a memory and- * refetching the list refreshes the cards and the filter options in- * lockstep. Firing a second list query here creates an independent react-- * query observer whose key diverges from the page query whenever a search- * keyword or a filter is active, leaving the filter options stale after a- * memory is created.+ * Build the filter facet collections from the server-side aggregation query.*- * @param memoryList - The memory items of the currently loaded list page.+ * @param filterData - Server-side filter aggregations for visible memories.* @returns The filter collections consumed by ListFilterBar.** @example* const { data: list } = useFetchMemoryList();* const { filters } = useSelectFilters(list?.data?.memory_list ?? []);*/-export function useSelectFilters(memoryList: IMemory[]) {+export function useSelectFilters(filterData: MemoryFiltersResponse) {const { t } = useTranslation();const filters: FilterCollection[] = useMemo(() => {- const memoryType = groupListByArray(memoryList, 'memory_type');- const storageType = groupListByType(- memoryList,- 'storage_type',- 'storage_type',- );-return [- buildOwnersFilter(memoryList, 'owner_name', t('common.owner')),+ {+ field: 'owner',+ list: filterData.filter.owner,+ label: t('common.owner'),+ },{field: 'memoryType',- list: memoryType,+ list: filterData.filter.memory_type,label: t('memories.memoryType'),},{field: 'storageType',- list: storageType,+ list: filterData.filter.storage_type,label: t('memory.config.storageType'),},];- }, [memoryList, t]);+ }, [filterData, t]);return { filters };}diff --git a/web/src/pages/memories/index.tsx b/web/src/pages/memories/index.tsxindex 3148aa3da7..f7cf2220a0 100644--- a/web/src/pages/memories/index.tsx+++ b/web/src/pages/memories/index.tsx@@ -13,7 +13,12 @@ import { useCallback, useEffect, useState } from 'react';import { useSearchParams } from 'react-router';import { AddOrEditModal } from './add-or-edit-modal';import { defaultMemoryFields } from './constants';-import { useFetchMemoryList, useRenameMemory, useSelectFilters } from './hooks';+import {+ useFetchMemoryFilters,+ useFetchMemoryList,+ useRenameMemory,+ useSelectFilters,+} from './hooks';import { ICreateMemoryProps, IMemory } from './interface';import { MemoryCard } from './memory-card';@@ -35,6 +40,7 @@ export default function MemoryList() {setFilterValue,handleFilterSubmit,} = useFetchMemoryList();+ const { data: filterData } = useFetchMemoryFilters();const {openCreateModal,@@ -70,7 +76,7 @@ export default function MemoryList() {});const [searchUrl, setMemoryUrl] = useSearchParams();- const { filters } = useSelectFilters(list?.data?.memory_list ?? []);+ const { filters } = useSelectFilters(filterData);const isCreate = searchUrl.get('isCreate') === 'true';useEffect(() => {if (isCreate) {diff --git a/web/src/pages/memories/interface.ts b/web/src/pages/memories/interface.tsindex 4b33a62536..33e8bcd292 100644--- a/web/src/pages/memories/interface.ts+++ b/web/src/pages/memories/interface.ts@@ -48,6 +48,15 @@ export interface MemoryListResponse {message: string;}+export interface MemoryFiltersResponse {+ filter: {+ owner: Array<{ id: string; label: string; count: number }>;+ memory_type: Array<{ id: string; label: string; count: number }>;+ storage_type: Array<{ id: string; label: string; count: number }>;+ };+ total: number;+}+export interface DeleteMemoryProps {memory_id: string;}diff --git a/web/src/services/memory-service.ts b/web/src/services/memory-service.tsindex bbde3cc498..5dc325052a 100644--- a/web/src/services/memory-service.ts+++ b/web/src/services/memory-service.ts@@ -39,6 +39,10 @@ const methods = {url: getMemoryList,method: 'get',},+ getMemoryFilters: {+ url: getMemoryList,+ method: 'get',+ },deleteMemory: { url: deleteMemory, method: 'delete' },getMemoryConfig: {url: getMemoryConfig,