Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(memory): expose server-side filter discovery (#19840)

Sbaaoui Idriss67aa682027a2
diff --git a/api/apps/restful_apis/memory_api.py b/api/apps/restful_apis/memory_api.py
index 2152957ac7..56e761b304 100644
--- a/api/apps/restful_apis/memory_api.py
+++ b/api/apps/restful_apis/memory_api.py
@@ -139,6 +139,13 @@ async def delete_memory(memory_id):
@manager.route("/memories", methods=["GET"]) # noqa: F821
@login_required
async def list_memory():
+ if request.args.get("type") == "filter":
+ try:
+ return get_json_result(message=True, data=await memory_api_service.list_memory_filters())
+ except Exception as e:
+ logging.error(e)
+ return get_json_result(code=RetCode.SERVER_ERROR, message="Internal server error")
+
# Every list filter accepts repeated query keys and comma-joined values. The getlist idiom comes from
# api/utils/validation_utils.py, which does not strip; the strip matches _split_filter_values in api/apps/services/memory_api_service.py.
filter_params = {
diff --git a/api/apps/services/memory_api_service.py b/api/apps/services/memory_api_service.py
index 78e8f116e9..9cb23a5781 100644
--- a/api/apps/services/memory_api_service.py
+++ b/api/apps/services/memory_api_service.py
@@ -291,6 +291,24 @@ async def list_memory(filter_params: dict, keywords: str, page: int = 1, page_si
return {"memory_list": memory_list, "total_count": count}
+async def list_memory_filters():
+ """Return filter aggregations for all memories visible to the caller."""
+ filter_dict = {"tenant_id": list(_joined_tenant_ids(current_user.id)), "accessible_user_id": current_user.id}
+ _, total = MemoryService.get_by_filter(filter_dict, None, 1, 1)
+ memories, _ = MemoryService.get_by_filter(filter_dict, None, 1, max(total, 1))
+ owner = {}
+ memory_type = {}
+ storage_type = {}
+ for memory in memories:
+ owner_item = owner.setdefault(memory["tenant_id"], {"id": memory["tenant_id"], "label": memory.get("owner_name") or memory["tenant_id"], "count": 0})
+ owner_item["count"] += 1
+ for item in get_memory_type_human(memory["memory_type"]):
+ memory_type.setdefault(item, {"id": item, "label": item, "count": 0})["count"] += 1
+ item = memory["storage_type"]
+ storage_type.setdefault(item, {"id": item, "label": item, "count": 0})["count"] += 1
+ return {"filter": {"owner": list(owner.values()), "memory_type": list(memory_type.values()), "storage_type": list(storage_type.values())}, "total": total}
+
+
async def get_memory_config(memory_id):
memory = MemoryService.get_with_owner_name_by_id(memory_id)
if not memory or not _memory_accessible(memory):
diff --git a/internal/dao/memory.go b/internal/dao/memory.go
index c93bf33cb6..1e64fbced1 100644
--- a/internal/dao/memory.go
+++ b/internal/dao/memory.go
@@ -348,7 +348,6 @@ func (dao *MemoryDAO) GetByFilter(ctx context.Context, db *gorm.DB, userID strin
return nil, 0, err
}
- offset := (page - 1) * pageSize
querySQL := fmt.Sprintf(`
SELECT m.id, m.name, m.avatar, m.tenant_id, m.memory_type,
m.storage_type, m.embd_id, m.tenant_embd_id, m.llm_id, m.tenant_llm_id,
@@ -360,10 +359,12 @@ func (dao *MemoryDAO) GetByFilter(ctx context.Context, db *gorm.DB, userID strin
LEFT JOIN user u ON m.tenant_id = u.id
%s
ORDER BY m.create_time DESC
- LIMIT ? OFFSET ?
`, whereClause)
-
- queryArgs := append(args, pageSize, offset)
+ queryArgs := args
+ if pageSize > 0 {
+ querySQL += " LIMIT ? OFFSET ?"
+ queryArgs = append(queryArgs, pageSize, (page-1)*pageSize)
+ }
var rawResults []struct {
entity.Memory
diff --git a/internal/handler/memory.go b/internal/handler/memory.go
index 66bcb3605e..6f7f665de5 100644
--- a/internal/handler/memory.go
+++ b/internal/handler/memory.go
@@ -335,6 +335,15 @@ func (h *MemoryHandler) ListMemories(c *gin.Context) {
common.ErrorWithCode(c, errorCode, errorMessage)
return
}
+ if c.Query("type") == "filter" {
+ result, err := h.memoryService.ListMemoryFilters(c.Request.Context(), user.ID)
+ if err != nil {
+ common.ResponseWithCodeData(c, common.CodeServerError, nil, err.Error())
+ return
+ }
+ common.SuccessWithData(c, result, "success")
+ return
+ }
// Parse query parameters
storageType := c.Query("storage_type")
diff --git a/internal/service/memory.go b/internal/service/memory.go
index f3203f4ccb..0a6178c0d3 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -360,6 +360,66 @@ type ListMemoryResponse struct {
TotalCount int64 `json:"total_count"`
}
+type MemoryFilterOption struct {
+ ID string `json:"id"`
+ Label string `json:"label"`
+ Count int64 `json:"count"`
+}
+
+type MemoryFiltersResponse struct {
+ Filter struct {
+ Owner []MemoryFilterOption `json:"owner"`
+ MemoryType []MemoryFilterOption `json:"memory_type"`
+ StorageType []MemoryFilterOption `json:"storage_type"`
+ } `json:"filter"`
+ Total int64 `json:"total"`
+}
+
+func (s *MemoryService) ListMemoryFilters(ctx context.Context, userID string) (*MemoryFiltersResponse, error) {
+ userTenants, err := NewUserTenantService().GetUserTenantRelationByUserIDWithContext(ctx, userID)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get user tenants: %w", err)
+ }
+ tenantIDs := make([]string, 0, len(userTenants)+1)
+ tenantIDs = append(tenantIDs, userID)
+ for _, tenant := range userTenants {
+ tenantIDs = append(tenantIDs, tenant.TenantID)
+ }
+ memories, _, err := s.memoryDAO.GetByFilter(ctx, dao.DB, userID, tenantIDs, nil, "", "", 1, 0)
+ if err != nil {
+ return nil, err
+ }
+ ownerCounts := map[string]int64{}
+ ownerLabels := map[string]string{}
+ typeCounts := map[string]int64{}
+ storageCounts := map[string]int64{}
+ for _, memory := range memories {
+ owner := memory.TenantID
+ label := owner
+ if memory.OwnerName != nil && *memory.OwnerName != "" {
+ label = *memory.OwnerName
+ }
+ ownerCounts[owner]++
+ ownerLabels[owner] = label
+ for _, memoryType := range dao.GetMemoryTypeHuman(memory.MemoryType) {
+ typeCounts[memoryType]++
+ }
+ storageCounts[memory.StorageType]++
+ }
+ resp := &MemoryFiltersResponse{}
+ for id, count := range ownerCounts {
+ resp.Filter.Owner = append(resp.Filter.Owner, MemoryFilterOption{ID: id, Label: ownerLabels[id], Count: count})
+ }
+ for id, count := range typeCounts {
+ resp.Filter.MemoryType = append(resp.Filter.MemoryType, MemoryFilterOption{ID: id, Label: id, Count: count})
+ }
+ for id, count := range storageCounts {
+ resp.Filter.StorageType = append(resp.Filter.StorageType, MemoryFilterOption{ID: id, Label: id, Count: count})
+ }
+ resp.Total = int64(len(memories))
+ return resp, nil
+}
+
// CreateMemory creates a new memory with the given parameters
// It validates the request, generates a unique name if needed, and creates the memory record
//
diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.go
index 7a7fe44739..cb59378fec 100644
--- a/internal/service/memory_message_test.go
+++ b/internal/service/memory_message_test.go
@@ -189,6 +189,35 @@ func setupMemoryMessageTestDB(t *testing.T) {
})
}
+func TestListMemoryFiltersUsesAccessibleMemoryAggregates(t *testing.T) {
+ setupMemoryMessageTestDB(t)
+ if err := dao.DB.Create(&entity.User{ID: "user-1", Nickname: "Owner"}).Error; err != nil {
+ t.Fatalf("seed user: %v", err)
+ }
+ for _, memory := range []*entity.Memory{
+ {ID: "mem-1", Name: "one", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw | dao.MemoryTypeSemantic, StorageType: "table", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO)},
+ {ID: "mem-2", Name: "two", TenantID: "user-1", MemoryType: dao.MemoryTypeRaw, StorageType: "graph", EmbdID: "embd", LLMID: "llm", Permissions: string(entity.TenantPermissionMe), ForgettingPolicy: string(ForgettingPolicyFIFO)},
+ } {
+ if err := dao.DB.Create(memory).Error; err != nil {
+ t.Fatalf("seed memory: %v", err)
+ }
+ }
+
+ filters, err := NewMemoryService().ListMemoryFilters(t.Context(), "user-1")
+ if err != nil {
+ t.Fatalf("ListMemoryFilters: %v", err)
+ }
+ if filters.Total != 2 || len(filters.Filter.Owner) != 1 {
+ t.Fatalf("unexpected owner aggregate: %+v", filters)
+ }
+ if filters.Filter.Owner[0].Label != "Owner" || filters.Filter.Owner[0].Count != 2 {
+ t.Fatalf("owner filter = %+v", filters.Filter.Owner)
+ }
+ if len(filters.Filter.MemoryType) != 2 || len(filters.Filter.StorageType) != 2 {
+ t.Fatalf("unexpected type/storage filters: %+v", filters)
+ }
+}
+
func TestForgetMessageKeepsCompanionFieldForNonOceanBaseEngines(t *testing.T) {
setupMemoryMessageTestDB(t)
diff --git a/web/src/pages/memories/hooks.ts b/web/src/pages/memories/hooks.ts
index ca9ad84ecf..4a9c8e1d80 100644
--- a/web/src/pages/memories/hooks.ts
+++ b/web/src/pages/memories/hooks.ts
@@ -9,11 +9,6 @@ import { useHandleSearchChange } from '@/hooks/logic-hooks';
import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';
import memoryService, { updateMemoryById } from '@/services/memory-service';
import { markListItemsDeleted } from '@/utils/list-deletion-util';
-import {
- buildOwnersFilter,
- groupListByArray,
- groupListByType,
-} from '@/utils/list-filter-util';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { useDebounce } from 'ahooks';
import { omit } from 'lodash';
@@ -30,10 +25,17 @@ import {
IMemoryAppDetailProps,
MemoryDetailResponse,
MemoryListResponse,
+ MemoryFiltersResponse,
} from './interface';
+export const MemoryKeys = {
+ list: () => ['memoryList'] as const,
+ filters: () => ['memoryFilters'] as const,
+};
+
export const useCreateMemory = () => {
const { t } = useTranslation();
+ const queryClient = useQueryClient();
const createMemory = useCallback(
async (props: ICreateMemoryProps): Promise<CreateMemoryResponse> => {
@@ -43,10 +45,11 @@ export const useCreateMemory = () => {
}
if (response.code === 0) {
message.success(t('message.created'));
+ queryClient.invalidateQueries({ queryKey: MemoryKeys.filters() });
}
return response.data;
},
- [t],
+ [queryClient, t],
);
return { createMemory };
@@ -87,7 +90,7 @@ export const useFetchMemoryList = () => {
Error
>({
queryKey: [
- 'memoryList',
+ ...MemoryKeys.list(),
{
debouncedSearchString,
...pagination,
@@ -105,7 +108,6 @@ export const useFetchMemoryList = () => {
if (response.code !== 0) {
throw new Error(response.message || 'Failed to fetch memory list');
}
- console.log(response);
return response;
},
});
@@ -133,6 +135,32 @@ export const useFetchMemoryList = () => {
};
};
+export const useFetchMemoryFilters = () => {
+ const {
+ data,
+ isFetching: isLoading,
+ isError,
+ } = useQuery<MemoryFiltersResponse, Error>({
+ queryKey: MemoryKeys.filters(),
+ initialData: {
+ filter: { owner: [], memory_type: [], storage_type: [] },
+ total: 0,
+ },
+ queryFn: async () => {
+ const { data: response } = await memoryService.getMemoryFilters(
+ { params: { type: 'filter' } },
+ true,
+ );
+ if (response.code !== 0) {
+ throw new Error(response.message || 'Failed to fetch memory filters');
+ }
+ return response.data;
+ },
+ });
+
+ return { data, isLoading, isError };
+};
+
export const useFetchMemoryDetail = (tenantId?: string) => {
const { id } = useParams();
@@ -184,7 +212,8 @@ export const useDeleteMemory = () => {
throw new Error(response.message || 'Failed to delete memory');
}
- queryClient.invalidateQueries({ queryKey: ['memoryList'] });
+ queryClient.invalidateQueries({ queryKey: MemoryKeys.list() });
+ queryClient.invalidateQueries({ queryKey: MemoryKeys.filters() });
markListItemsDeleted(ListDeletionKey.MemoryList);
return response;
},
@@ -232,6 +261,7 @@ export const useUpdateMemory = () => {
queryClient.invalidateQueries({
queryKey: [MemoryApiAction.FetchMemoryDetail],
});
+ queryClient.invalidateQueries({ queryKey: MemoryKeys.filters() });
},
});
@@ -315,49 +345,37 @@ export const useRenameMemory = () => {
};
/**
- * Build the filter facet collections for the memory list page from the
- * memory items that are already loaded by the page's list query.
- *
- * The filters must be derived from the same query data that renders the
- * memory cards so that creating, updating or deleting a memory and
- * refetching the list refreshes the cards and the filter options in
- * lockstep. Firing a second list query here creates an independent react-
- * query observer whose key diverges from the page query whenever a search
- * keyword or a filter is active, leaving the filter options stale after a
- * memory is created.
+ * Build the filter facet collections from the server-side aggregation query.
*
- * @param memoryList - The memory items of the currently loaded list page.
+ * @param filterData - Server-side filter aggregations for visible memories.
* @returns The filter collections consumed by ListFilterBar.
*
* @example
* const { data: list } = useFetchMemoryList();
* const { filters } = useSelectFilters(list?.data?.memory_list ?? []);
*/
-export function useSelectFilters(memoryList: IMemory[]) {
+export function useSelectFilters(filterData: MemoryFiltersResponse) {
const { t } = useTranslation();
const filters: FilterCollection[] = useMemo(() => {
- const memoryType = groupListByArray(memoryList, 'memory_type');
- const storageType = groupListByType(
- memoryList,
- 'storage_type',
- 'storage_type',
- );
-
return [
- buildOwnersFilter(memoryList, 'owner_name', t('common.owner')),
+ {
+ field: 'owner',
+ list: filterData.filter.owner,
+ label: t('common.owner'),
+ },
{
field: 'memoryType',
- list: memoryType,
+ list: filterData.filter.memory_type,
label: t('memories.memoryType'),
},
{
field: 'storageType',
- list: storageType,
+ list: filterData.filter.storage_type,
label: t('memory.config.storageType'),
},
];
- }, [memoryList, t]);
+ }, [filterData, t]);
return { filters };
}
diff --git a/web/src/pages/memories/index.tsx b/web/src/pages/memories/index.tsx
index 3148aa3da7..f7cf2220a0 100644
--- a/web/src/pages/memories/index.tsx
+++ b/web/src/pages/memories/index.tsx
@@ -13,7 +13,12 @@ import { useCallback, useEffect, useState } from 'react';
import { useSearchParams } from 'react-router';
import { AddOrEditModal } from './add-or-edit-modal';
import { defaultMemoryFields } from './constants';
-import { useFetchMemoryList, useRenameMemory, useSelectFilters } from './hooks';
+import {
+ useFetchMemoryFilters,
+ useFetchMemoryList,
+ useRenameMemory,
+ useSelectFilters,
+} from './hooks';
import { ICreateMemoryProps, IMemory } from './interface';
import { MemoryCard } from './memory-card';
@@ -35,6 +40,7 @@ export default function MemoryList() {
setFilterValue,
handleFilterSubmit,
} = useFetchMemoryList();
+ const { data: filterData } = useFetchMemoryFilters();
const {
openCreateModal,
@@ -70,7 +76,7 @@ export default function MemoryList() {
});
const [searchUrl, setMemoryUrl] = useSearchParams();
- const { filters } = useSelectFilters(list?.data?.memory_list ?? []);
+ const { filters } = useSelectFilters(filterData);
const isCreate = searchUrl.get('isCreate') === 'true';
useEffect(() => {
if (isCreate) {
diff --git a/web/src/pages/memories/interface.ts b/web/src/pages/memories/interface.ts
index 4b33a62536..33e8bcd292 100644
--- a/web/src/pages/memories/interface.ts
+++ b/web/src/pages/memories/interface.ts
@@ -48,6 +48,15 @@ export interface MemoryListResponse {
message: string;
}
+export interface MemoryFiltersResponse {
+ filter: {
+ owner: Array<{ id: string; label: string; count: number }>;
+ memory_type: Array<{ id: string; label: string; count: number }>;
+ storage_type: Array<{ id: string; label: string; count: number }>;
+ };
+ total: number;
+}
+
export interface DeleteMemoryProps {
memory_id: string;
}
diff --git a/web/src/services/memory-service.ts b/web/src/services/memory-service.ts
index bbde3cc498..5dc325052a 100644
--- a/web/src/services/memory-service.ts
+++ b/web/src/services/memory-service.ts
@@ -39,6 +39,10 @@ const methods = {
url: getMemoryList,
method: 'get',
},
+ getMemoryFilters: {
+ url: getMemoryList,
+ method: 'get',
+ },
deleteMemory: { url: deleteMemory, method: 'delete' },
getMemoryConfig: {
url: getMemoryConfig,