Skip to content

Build and run agents you can see, understand and trust.

feat(rag): add chunk browsing, raw document preview and enriched knowledge base listing (#2372)

DavdGao284a7bffeb81
diff --git a/examples/web_ui/frontend/src/api/knowledgeBase.ts b/examples/web_ui/frontend/src/api/knowledgeBase.ts
index 202e5b1ce..3541a08e2 100644
--- a/examples/web_ui/frontend/src/api/knowledgeBase.ts
+++ b/examples/web_ui/frontend/src/api/knowledgeBase.ts
@@ -2,11 +2,15 @@ import { ApiError, client, getBaseUrl, getUserId } from './client';
import type {
CreateKnowledgeBaseRequest,
CreateKnowledgeBaseResponse,
+ DocumentDownloadTokenResponse,
KbMiddlewareParametersSchemaResponse,
KnowledgeBaseView,
ListChunkersResponse,
+ ListDocumentChunksResponse,
ListKbEmbeddingModelsResponse,
+ ListKnowledgeBasesParams,
ListKnowledgeBasesResponse,
+ ListKnowledgeDocumentsParams,
ListKnowledgeDocumentsResponse,
ListKnowledgeDocumentStatusResponse,
ListSupportedContentTypesResponse,
@@ -16,6 +20,40 @@ import type {
UploadKnowledgeDocumentResponse,
} from './types';
+/** Drop undefined values and stringify the rest for `client` params. */
+function toQuery(params: Record<string, unknown>): Record<string, string> {
+ const query: Record<string, string> = {};
+ for (const [key, value] of Object.entries(params)) {
+ if (value !== undefined) query[key] = String(value);
+ }
+ return query;
+}
+
+/** The backend caps `page_size` at 128. */
+const MAX_PAGE_SIZE = 128;
+
+/**
+ * Drain a paginated endpoint into one flat array — genuinely all of
+ * it, however many pages that takes.
+ *
+ * Keeps requesting pages while the accumulated count is below the
+ * server-reported `total` and the last page still made progress. An
+ * empty page means the server has nothing more to give, so a lying
+ * `total` degrades into a clean stop instead of an infinite loop —
+ * no arbitrary page cap that would silently truncate large lists.
+ */
+async function fetchAllPages<T>(
+ fetchPage: (page: number, pageSize: number) => Promise<{ items: T[]; total: number }>,
+): Promise<T[]> {
+ const all: T[] = [];
+ for (let page = 1; ; page++) {
+ const { items, total } = await fetchPage(page, MAX_PAGE_SIZE);
+ all.push(...items);
+ if (items.length === 0 || all.length >= total) break;
+ }
+ return all;
+}
+
/**
* Callback invoked while bytes are pushed across the wire.
*
@@ -118,7 +156,22 @@ function uploadDocumentXhr(
* Client for the `/knowledge_bases` router.
*/
export const knowledgeBaseApi = {
- list: () => client.get<ListKnowledgeBasesResponse>('/knowledge_bases/'),
+ list: (params: ListKnowledgeBasesParams = {}) =>
+ client.get<ListKnowledgeBasesResponse>('/knowledge_bases/', toQuery({ ...params })),
+
+ /**
+ * Fetch every visible knowledge base across all pages — for views
+ * that render one flat list (e.g. the sidebar).
+ */
+ listAll: (params: Omit<ListKnowledgeBasesParams, 'page' | 'page_size'> = {}) =>
+ fetchAllPages(async (page, pageSize) => {
+ const res = await knowledgeBaseApi.list({
+ ...params,
+ page,
+ page_size: pageSize,
+ });
+ return { items: res.knowledge_bases, total: res.total };
+ }),
listEmbeddingModels: () =>
client.get<ListKbEmbeddingModelsResponse>('/knowledge_bases/embedding_models'),
@@ -143,9 +196,80 @@ export const knowledgeBaseApi = {
delete: (knowledgeBaseId: string) => client.delete(`/knowledge_bases/${knowledgeBaseId}`),
- /** List every document registered against a knowledge base. */
- listDocuments: (knowledgeBaseId: string) =>
- client.get<ListKnowledgeDocumentsResponse>(`/knowledge_bases/${knowledgeBaseId}/documents`),
+ /** List one page of the documents registered against a knowledge base. */
+ listDocuments: (knowledgeBaseId: string, params: ListKnowledgeDocumentsParams = {}) =>
+ client.get<ListKnowledgeDocumentsResponse>(
+ `/knowledge_bases/${knowledgeBaseId}/documents`,
+ toQuery({ ...params }),
+ ),
+
+ /**
+ * Fetch every document of a knowledge base across all pages — for
+ * views that render one flat list (e.g. the documents panel).
+ */
+ listAllDocuments: (
+ knowledgeBaseId: string,
+ params: Omit<ListKnowledgeDocumentsParams, 'page' | 'page_size'> = {},
+ ) =>
+ fetchAllPages(async (page, pageSize) => {
+ const res = await knowledgeBaseApi.listDocuments(knowledgeBaseId, {
+ ...params,
+ page,
+ page_size: pageSize,
+ });
+ return { items: res.documents, total: res.total };
+ }),
+
+ /** Browse one page of a document's chunks in `chunk_index` order. */
+ listDocumentChunks: (knowledgeBaseId: string, documentId: string, page = 1, pageSize = 30) =>
+ client.get<ListDocumentChunksResponse>(
+ `/knowledge_bases/${knowledgeBaseId}/documents/${documentId}/chunks`,
+ toQuery({ page, page_size: pageSize }),
+ // 501 means the configured vector store cannot list chunks —
+ // the caller hides the chunk view instead of toasting.
+ { silent: true },
+ ),
+
+ /**
+ * Mint a short-lived token so a browser-native fetch (`<iframe>`,
+ * `<img>`, a download navigation) can retrieve the raw file
+ * without the `X-User-ID` header.
+ */
+ createDocumentDownloadToken: (knowledgeBaseId: string, documentId: string) =>
+ client.post<DocumentDownloadTokenResponse>(
+ `/knowledge_bases/${knowledgeBaseId}/documents/${documentId}/download_token`,
+ ),
+
+ /**
+ * Absolute URL of a document's raw bytes, using a token minted via
+ * `createDocumentDownloadToken`. Pass `download: true` to force a
+ * `Content-Disposition: attachment`.
+ */
+ documentContentUrl: (
+ knowledgeBaseId: string,
+ documentId: string,
+ token: string,
+ download = false,
+ ) => {
+ const url = new URL(
+ `/knowledge_bases/${knowledgeBaseId}/documents/${documentId}`,
+ getBaseUrl(),
+ );
+ url.searchParams.set('token', token);
+ if (download) url.searchParams.set('download', 'true');
+ return url.toString();
+ },
+
+ /**
+ * Fetch the raw file as text through the authenticated client —
+ * for markdown / plain-text previews that render in-app.
+ */
+ fetchDocumentText: async (knowledgeBaseId: string, documentId: string) => {
+ const res = await client.stream(
+ `/knowledge_bases/${knowledgeBaseId}/documents/${documentId}`,
+ );
+ return res.text();
+ },
/**
* Batch-query lifecycle status for a list of documents.
diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.ts
index 06f423da8..4c22daf9c 100644
--- a/examples/web_ui/frontend/src/api/types.ts
+++ b/examples/web_ui/frontend/src/api/types.ts
@@ -874,11 +874,50 @@ export interface KnowledgeBaseView {
* shared with read-only permission.
*/
editable: boolean;
+ /** Number of documents registered in the knowledge base. */
+ document_count: number;
+ /** Total indexed chunks across all documents. */
+ chunk_count: number;
+ /**
+ * Display name of the credential behind
+ * `embedding_model_config.credential_id`, resolved server-side so
+ * shared viewers see it too. `null` when the credential was deleted.
+ */
+ credential_name: string | null;
+ /** Per-indexing-status document counts; always served. */
+ status_counts: KnowledgeBaseStatusCounts;
+}
+
+/** Documents of one knowledge base, counted by indexing status. */
+export interface KnowledgeBaseStatusCounts {
+ pending: number;
+ parsing: number;
+ chunking: number;
+ indexing: number;
+ ready: number;
+ error: number;
+}
+
+/** Query parameters accepted by `GET /knowledge_bases/`. */
+export interface ListKnowledgeBasesParams {
+ /** Filter down to one knowledge base — list doubles as get-single. */
+ id?: string;
+ /** Case-insensitive substring filter on the name. */
+ name?: string;
+ /** 1-based page number (default 1). */
+ page?: number;
+ /** Page size (default 30, max 128). */
+ page_size?: number;
+ orderby?: 'create_time' | 'update_time';
+ desc?: boolean;
}
export interface ListKnowledgeBasesResponse {
knowledge_bases: KnowledgeBaseView[];
+ /** Total across all pages (after filters), for page counts. */
total: number;
+ page: number;
+ page_size: number;
}
export interface CreateKnowledgeBaseRequest {
@@ -936,9 +975,28 @@ export interface KnowledgeDocumentView {
updated_at: string;
}
+/** Query parameters accepted by `GET /knowledge_bases/{id}/documents`. */
+export interface ListKnowledgeDocumentsParams {
+ /** Filter down to one document by id. */
+ id?: string;
+ /** Case-insensitive substring filter on the filename. */
+ keywords?: string;
+ /** Filter by indexing status. */
+ status?: KnowledgeDocumentStatus;
+ /** 1-based page number (default 1). */
+ page?: number;
+ /** Page size (default 30, max 128). */
+ page_size?: number;
+ orderby?: 'create_time' | 'update_time';
+ desc?: boolean;
+}
+
export interface ListKnowledgeDocumentsResponse {
documents: KnowledgeDocumentView[];
+ /** Total across all pages (after filters), for page counts. */
total: number;
+ page: number;
+ page_size: number;
}
export interface ListKnowledgeDocumentStatusResponse {
@@ -969,6 +1027,29 @@ export interface KnowledgeChunk {
metadata: Record<string, unknown>;
}
+/**
+ * Response of `GET /knowledge_bases/{id}/documents/{doc}/chunks` —
+ * one page of a document's chunks in `chunk_index` order.
+ */
+export interface ListDocumentChunksResponse {
+ chunks: KnowledgeChunk[];
+ /** Total chunks in the document; `0` while it is still indexing. */
+ total: number;
+ page: number;
+ page_size: number;
+}
+
+/**
+ * Response of `POST /knowledge_bases/{id}/documents/{doc}/download_token`
+ * — a short-lived capability for browser-native fetches (`<iframe>`,
+ * `<img>`, download links) that cannot carry the `X-User-ID` header.
+ */
+export interface DocumentDownloadTokenResponse {
+ token: string;
+ /** Unix timestamp after which the token is refused. */
+ expires_at: number;
+}
+
/**
* One vector search hit returned by the knowledge base search endpoint.
* Mirrors :class:`agentscope.rag.VectorSearchResult` on the backend.
diff --git a/examples/web_ui/frontend/src/components/knowledge/DocumentDetailDrawer.tsx b/examples/web_ui/frontend/src/components/knowledge/DocumentDetailDrawer.tsx
new file mode 100644
index 000000000..827ae0361
--- /dev/null
+++ b/examples/web_ui/frontend/src/components/knowledge/DocumentDetailDrawer.tsx
@@ -0,0 +1,362 @@
+import { ChevronLeft, ChevronRight, Download, FileText, Loader2 } from 'lucide-react';
+import { useCallback, useEffect, useState } from 'react';
+
+import { knowledgeBaseApi } from '@/api';
+import type { KnowledgeChunk, KnowledgeDocumentView } from '@/api';
+import { ApiError } from '@/api/client';
+import { Markdown } from '@/components/markdown';
+import { Badge } from '@/components/ui/badge.tsx';
+import { Button } from '@/components/ui/button.tsx';
+import {
+ Empty,
+ EmptyDescription,
+ EmptyHeader,
+ EmptyMedia,
+ EmptyTitle,
+} from '@/components/ui/empty.tsx';
+import {
+ Sheet,
+ SheetContent,
+ SheetDescription,
+ SheetFooter,
+ SheetHeader,
+ SheetTitle,
+} from '@/components/ui/sheet.tsx';
+import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs.tsx';
+import { useTranslation } from '@/i18n/useI18n.ts';
+
+const CHUNK_PAGE_SIZE = 20;
+
+/**
+ * Image types the backend serves with `Content-Disposition: inline`.
+ * Anything else — notably `image/svg+xml`, which can carry script —
+ * comes back as an attachment, so an `<img>` preview would break;
+ * those types fall through to the download-button branch instead.
+ */
+const INLINE_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp', 'image/bmp'];
+
+/**
+ * Largest text file rendered inline. Text previews pull the whole body
+ * into the tab, so a multi-hundred-megabyte upload would freeze it —
+ * past this the drawer offers a download instead.
+ */
+const MAX_INLINE_TEXT_BYTES = 2 * 1024 * 1024;
+
+interface Props {
+ open: boolean;
+ onOpenChange: (open: boolean) => void;
+ knowledgeBaseId: string;
+ document: KnowledgeDocumentView;
+}
+
+/** Media type with parameters stripped, lowercased — `''` when unknown. */
+function mediaType(doc: KnowledgeDocumentView): string {
+ return (doc.content_type ?? '').split(';')[0].trim().toLowerCase();
+}
+
+/**
+ * Right-side drawer showing a ready document's indexed chunks (paged,
+ * in `chunk_index` order) and a preview of the original uploaded file.
+ */
+export function DocumentDetailDrawer({ open, onOpenChange, knowledgeBaseId, document }: Props) {
+ const { t } = useTranslation();
+
+ return (
+ <Sheet open={open} onOpenChange={onOpenChange}>
+ <SheetContent className="flex w-full sm:!max-w-[560px] flex-col gap-y-4 p-4">
+ <SheetHeader className="px-0">
+ <SheetTitle className="flex items-center gap-x-2">
+ <FileText className="size-4 shrink-0" />
+ <span className="truncate">{document.filename}</span>
+ </SheetTitle>
+ <SheetDescription>
+ {t('knowledge.documentDetail.description', {
+ count: document.chunk_count,
+ })}
+ </SheetDescription>
+ </SheetHeader>
+
+ <Tabs defaultValue="chunks" className="flex min-h-0 flex-1 flex-col gap-y-3">
+ <TabsList className="w-fit">
+ <TabsTrigger value="chunks">
+ {t('knowledge.documentDetail.chunksTab')}
+ </TabsTrigger>
+ <TabsTrigger value="preview">
+ {t('knowledge.documentDetail.previewTab')}
+ </TabsTrigger>
+ </TabsList>
+ <TabsContent value="chunks" className="min-h-0 flex-1 overflow-y-auto">
+ <ChunksTab
+ open={open}
+ knowledgeBaseId={knowledgeBaseId}
+ document={document}
+ />
+ </TabsContent>
+ <TabsContent value="preview" className="min-h-0 flex-1 overflow-y-auto">
+ <PreviewTab
+ open={open}
+ knowledgeBaseId={knowledgeBaseId}
+ document={document}
+ />
+ </TabsContent>
+ </Tabs>
+
+ <SheetFooter className="px-0">
+ <Button variant="ghost" onClick={() => onOpenChange(false)}>
+ {t('common.close')}
+ </Button>
+ </SheetFooter>
+ </SheetContent>
+ </Sheet>
+ );
+}
+
+interface TabProps {
+ open: boolean;
+ knowledgeBaseId: string;
+ document: KnowledgeDocumentView;
+}
+
+function ChunksTab({ open, knowledgeBaseId, document }: TabProps) {
+ const { t } = useTranslation();
+ const [page, setPage] = useState(1);
+ const [chunks, setChunks] = useState<KnowledgeChunk[]>([]);
+ const [total, setTotal] = useState(document.chunk_count);
+ const [loading, setLoading] = useState(false);
+ const [unsupported, setUnsupported] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+
+ useEffect(() => {
+ if (!open) return;
+ let cancelled = false;
+ setLoading(true);
+ setError(null);
+ knowledgeBaseApi
+ .listDocumentChunks(knowledgeBaseId, document.id, page, CHUNK_PAGE_SIZE)
+ .then((res) => {
+ if (cancelled) return;
+ setChunks(res.chunks);
+ setTotal(res.total);
+ })
+ .catch((e) => {
+ if (cancelled) return;
+ // 501 — the configured vector store cannot list chunks.
+ if (e instanceof ApiError && e.status === 501) setUnsupported(true);
+ else setError((e as Error).message || String(e));
+ })
+ .finally(() => {
+ if (!cancelled) setLoading(false);
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [open, knowledgeBaseId, document.id, page]);
+
+ if (unsupported) {
+ return (
+ <Empty className="border-none py-6">
+ <EmptyHeader>
+ <EmptyMedia variant="icon">
+ <FileText />
+ </EmptyMedia>
+ <EmptyTitle>{t('knowledge.documentDetail.chunksUnsupportedTitle')}</EmptyTitle>
+ <EmptyDescription>
+ {t('knowledge.documentDetail.chunksUnsupportedDescription')}
+ </EmptyDescription>
+ </EmptyHeader>
+ </Empty>
+ );
+ }
+
+ const pageCount = Math.max(1, Math.ceil(total / CHUNK_PAGE_SIZE));
+
+ return (
+ <div className="flex flex-col gap-y-3">
+ {error && <p className="text-destructive text-sm">{error}</p>}
+ {loading ? (
+ <div className="flex justify-center py-8">
+ <Loader2 className="text-muted-foreground size-4 animate-spin" />
+ </div>
+ ) : chunks.length === 0 ? (
+ <Empty className="border-none py-6">
+ <EmptyHeader>
+ <EmptyMedia variant="icon">
+ <FileText />
+ </EmptyMedia>
+ <EmptyTitle>{t('knowledge.documentDetail.chunksEmptyTitle')}</EmptyTitle>
+ <EmptyDescription>
+ {t('knowledge.documentDetail.chunksEmptyDescription')}
+ </EmptyDescription>
+ </EmptyHeader>
+ </Empty>
+ ) : (
+ chunks.map((chunk) => (
+ <div
+ key={chunk.chunk_index}
+ className="bg-card flex flex-col gap-y-2 rounded-md border p-3"
+ >
+ <div className="text-muted-foreground flex items-center gap-x-2 text-xs">
+ <Badge variant="secondary" className="font-mono">
+ {t('knowledge.test.chunkPosition', {
+ index: chunk.chunk_index + 1,
+ total: chunk.total_chunks,
+ })}
+ </Badge>
+ </div>
+ <p className="text-sm break-words whitespace-pre-wrap">
+ {chunk.content &&
+ typeof chunk.content === 'object' &&
+ 'text' in chunk.content
+ ? String(chunk.content.text ?? '')
+ : JSON.stringify(chunk.content)}
+ </p>
+ </div>
+ ))
+ )}
+ {total > CHUNK_PAGE_SIZE && (
+ <div className="flex items-center justify-between pt-1">
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={loading || page <= 1}
+ onClick={() => setPage((p) => Math.max(1, p - 1))}
+ >
+ <ChevronLeft className="size-3.5" />
+ {t('knowledge.documentDetail.previousPage')}
+ </Button>
+ <span className="text-muted-foreground text-xs">
+ {t('knowledge.documentDetail.pageIndicator', {
+ page,
+ pageCount,
+ })}
+ </span>
+ <Button
+ variant="outline"
+ size="sm"
+ disabled={loading || page >= pageCount}
+ onClick={() => setPage((p) => Math.min(pageCount, p + 1))}
+ >
+ {t('knowledge.documentDetail.nextPage')}
+ <ChevronRight className="size-3.5" />
+ </Button>
+ </div>
+ )}
+ </div>
+ );
+}
+
+function PreviewTab({ open, knowledgeBaseId, document }: TabProps) {
+ const { t } = useTranslation();
+ const media = mediaType(document);
+ // Oversized text falls through to the download branch rather than
+ // being streamed into the tab.
+ const isTextType = media === 'text/markdown' || media === 'text/plain';
+ const tooLargeToInline = isTextType && document.size > MAX_INLINE_TEXT_BYTES;
+ const isText = isTextType && !tooLargeToInline;
+ const isPdf = media === 'application/pdf';
+ const isImage = INLINE_IMAGE_TYPES.includes(media);
+
+ const [text, setText] = useState<string | null>(null);
+ const [tokenUrl, setTokenUrl] = useState<string | null>(null);
+ const [loading, setLoading] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+
+ useEffect(() => {
+ if (!open) return;
+ let cancelled = false;
+ setLoading(true);
+ setError(null);
+ const load = async () => {
+ if (isText) {
+ const body = await knowledgeBaseApi.fetchDocumentText(knowledgeBaseId, document.id);
+ if (!cancelled) setText(body);
+ } else if (isPdf || isImage) {
+ const { token } = await knowledgeBaseApi.createDocumentDownloadToken(
+ knowledgeBaseId,
+ document.id,
+ );
+ if (!cancelled) {
+ setTokenUrl(
+ knowledgeBaseApi.documentContentUrl(knowledgeBaseId, document.id, token),
+ );
+ }
+ }
+ };
+ load()
+ .catch((e) => {
+ if (!cancelled) setError((e as Error).message || String(e));
+ })
+ .finally(() => {
+ if (!cancelled) setLoading(false);
+ });
+ return () => {
+ cancelled = true;
+ };
+ }, [open, knowledgeBaseId, document.id, isText, isPdf, isImage]);
+
+ const handleDownload = useCallback(async () => {
+ try {
+ const { token } = await knowledgeBaseApi.createDocumentDownloadToken(
+ knowledgeBaseId,
+ document.id,
+ );
+ window.open(
+ knowledgeBaseApi.documentContentUrl(knowledgeBaseId, document.id, token, true),
+ '_blank',
+ );
+ } catch {
+ // The client already toasts the failure.
+ }
+ }, [knowledgeBaseId, document.id]);
+
+ if (loading) {
+ return (
+ <div className="flex justify-center py-8">
+ <Loader2 className="text-muted-foreground size-4 animate-spin" />
+ </div>
+ );
+ }
+ if (error) {
+ return <p className="text-destructive text-sm">{error}</p>;
+ }
+ if (isText && text !== null) {
+ return media === 'text/markdown' ? (
+ <Markdown>{text}</Markdown>
+ ) : (
+ <pre className="text-sm break-words whitespace-pre-wrap">{text}</pre>
+ );
+ }
+ if (isPdf && tokenUrl) {
+ return <iframe src={tokenUrl} title={document.filename} className="h-full w-full" />;
+ }
+ if (isImage && tokenUrl) {
+ return <img src={tokenUrl} alt={document.filename} className="max-w-full" />;
+ }
+ return (
+ <Empty className="border-none py-6">
+ <EmptyHeader>
+ <EmptyMedia variant="icon">
+ <Download />
+ </EmptyMedia>
+ <EmptyTitle>
+ {t(
+ tooLargeToInline
+ ? 'knowledge.documentDetail.previewTooLargeTitle'
+ : 'knowledge.documentDetail.previewUnavailableTitle',
+ )}
+ </EmptyTitle>
+ <EmptyDescription>
+ {t(
+ tooLargeToInline
+ ? 'knowledge.documentDetail.previewTooLargeDescription'
+ : 'knowledge.documentDetail.previewUnavailableDescription',
+ )}
+ </EmptyDescription>
+ </EmptyHeader>
+ <Button size="sm" variant="outline" onClick={handleDownload}>
+ <Download className="size-3.5" />
+ {t('knowledge.documentDetail.downloadButton')}
+ </Button>
+ </Empty>
+ );
+}
diff --git a/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx b/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx
index 022c80b81..c06c46531 100644
--- a/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx
+++ b/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx
@@ -6,6 +6,7 @@ import { toast } from 'sonner';
import { knowledgeBaseApi } from '@/api';
import type { KnowledgeDocumentStatus, KnowledgeDocumentView } from '@/api';
import { DeleteDialog } from '@/components/dialog/DeleteDialog.tsx';
+import { DocumentDetailDrawer } from '@/components/knowledge/DocumentDetailDrawer.tsx';
import { Button } from '@/components/ui/button.tsx';
import {
Empty,
@@ -96,9 +97,10 @@ interface RowViewProps {
onCancel: (taskId: string) => void;
onDismiss: (taskId: string) => void;
onDelete: (doc: KnowledgeDocumentView) => void;
+ onOpen: (doc: KnowledgeDocumentView) => void;
}
-function RowView({ row, onCancel, onDismiss, onDelete }: RowViewProps) {
+function RowView({ row, onCancel, onDismiss, onDelete, onOpen }: RowViewProps) {
const { t } = useTranslation();
const filename = row.kind === 'server' ? row.doc.filename : row.task.filename;
@@ -155,8 +157,19 @@ function RowView({ row, onCancel, onDismiss, onDelete }: RowViewProps) {
const showProgress = phase !== 'ready' && phase !== 'cancelled';
+ // Ready server rows open the detail drawer (chunks + preview).
+ const openable = row.kind === 'server' && row.doc.status === 'ready';
+
return (
- <div className="border-border bg-card flex flex-col gap-y-2 rounded-lg border p-3">
+ <div
+ className={cn(
+ 'border-border bg-card flex flex-col gap-y-2 rounded-lg border p-3',
+ openable && 'hover:bg-accent/50 cursor-pointer transition-colors',
+ )}
+ onClick={
+ openable ? () => onOpen((row as Extract<Row, { kind: 'server' }>).doc) : undefined
+ }
+ >
<div className="flex items-start gap-x-3">
<FileText className="text-muted-foreground mt-0.5 size-4 shrink-0" />
<div className="flex min-w-0 flex-1 flex-col gap-y-0.5">
@@ -178,7 +191,10 @@ function RowView({ row, onCancel, onDismiss, onDelete }: RowViewProps) {
)}
</div>
</div>
- <div className="flex shrink-0 items-center gap-x-1">
+ <div
+ className="flex shrink-0 items-center gap-x-1"
+ onClick={(e) => e.stopPropagation()}
+ >
{phase === 'queued' || phase === 'uploading' ? (
<Button
variant="ghost"
@@ -242,6 +258,7 @@ export function KnowledgeDocumentsPanel({ knowledgeBaseId }: KnowledgeDocumentsP
const fileInputRef = useRef<HTMLInputElement>(null);
const [dragOver, setDragOver] = useState(false);
const [deleteTarget, setDeleteTarget] = useState<KnowledgeDocumentView | null>(null);
+ const [detailTarget, setDetailTarget] = useState<KnowledgeDocumentView | null>(null);
const { enqueue, cancel, dismiss, tasksForKb, clearFinishedForKb } = useUploadContext();
const { documents, refetch } = useKnowledgeDocuments(knowledgeBaseId);
@@ -460,12 +477,27 @@ export function KnowledgeDocumentsPanel({ knowledgeBaseId }: KnowledgeDocumentsP
onCancel={cancel}
onDismiss={dismiss}
onDelete={setDeleteTarget}
+ onOpen={setDetailTarget}
/>
))}
</div>
)}
</div>
+ {detailTarget && (
+ <DocumentDetailDrawer
+ // Keyed so switching documents remounts the drawer —
+ // otherwise chunk page / preview state leaks across.
+ key={detailTarget.id}
+ open={detailTarget !== null}
+ onOpenChange={(open) => {
+ if (!open) setDetailTarget(null);
+ }}
+ knowledgeBaseId={knowledgeBaseId}
+ document={detailTarget}
+ />
+ )}
+
<DeleteDialog
open={deleteTarget !== null}
onOpenChange={(open) => {
diff --git a/examples/web_ui/frontend/src/hooks/useKnowledgeBases.ts b/examples/web_ui/frontend/src/hooks/useKnowledgeBases.ts
index b66631e9e..fd593ff99 100644
--- a/examples/web_ui/frontend/src/hooks/useKnowledgeBases.ts
+++ b/examples/web_ui/frontend/src/hooks/useKnowledgeBases.ts
@@ -25,8 +25,9 @@ export function useKnowledgeBases() {
setLoading(true);
setError(null);
try {
- const { knowledge_bases } = await knowledgeBaseApi.list();
- setKnowledgeBases(knowledge_bases);
+ // The sidebar shows every knowledge base as one flat list,
+ // so drain all pages of the paginated endpoint.
+ setKnowledgeBases(await knowledgeBaseApi.listAll());
} catch (e) {
setError(e as Error);
} finally {
diff --git a/examples/web_ui/frontend/src/hooks/useKnowledgeDocuments.ts b/examples/web_ui/frontend/src/hooks/useKnowledgeDocuments.ts
index 115f5cd53..4f8ddf73d 100644
--- a/examples/web_ui/frontend/src/hooks/useKnowledgeDocuments.ts
+++ b/examples/web_ui/frontend/src/hooks/useKnowledgeDocuments.ts
@@ -28,7 +28,10 @@ export function useKnowledgeDocuments(knowledgeBaseId: string | null) {
setLoading(true);
setError(null);
try {
- const { documents: list } = await knowledgeBaseApi.listDocuments(knowledgeBaseId);
+ // The panel merges rows with in-flight upload tasks and the
+ // status poller into one flat list, so drain all pages of
+ // the paginated endpoint.
+ const list = await knowledgeBaseApi.listAllDocuments(knowledgeBaseId);
if (seq !== requestSeq.current) return;
setDocuments(list);
} catch (e) {
diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.json
index 939641c7b..d3fcbff43 100644
--- a/examples/web_ui/frontend/src/i18n/locales/en.json
+++ b/examples/web_ui/frontend/src/i18n/locales/en.json
@@ -385,6 +385,35 @@
"chunkPosition": "Chunk {{index}} of {{total}}",
"emptyTitle": "No matches",
"emptyDescription": "No chunks scored above the threshold for this query."
+ },
+ "config": {
+ "title": "Configuration",
+ "embeddingModel": "Embedding model",
+ "dimensions": "Dimensions",
+ "credential": "Credential",
+ "chunker": "Chunker",
+ "counts": "Contents",
+ "countsValue": "{{documents}} documents · {{chunks}} chunks",
+ "status": "Indexing",
+ "statusValue": "{{ready}} ready · {{indexing}} in progress · {{failed}} failed",
+ "createdAt": "Created"
+ },
+ "documentDetail": {
+ "description": "{{count}} indexed chunks",
+ "chunksTab": "Chunks",
+ "previewTab": "Preview",
+ "chunksEmptyTitle": "No chunks",
+ "chunksEmptyDescription": "This document has no indexed chunks yet.",
+ "chunksUnsupportedTitle": "Chunk browsing unavailable",
+ "chunksUnsupportedDescription": "The configured vector store does not support listing chunks.",
+ "previousPage": "Previous",
+ "nextPage": "Next",
+ "pageIndicator": "Page {{page}} of {{pageCount}}",
+ "previewUnavailableTitle": "No inline preview",
+ "previewUnavailableDescription": "This file type cannot be previewed in the browser.",
+ "previewTooLargeTitle": "File too large to preview",
+ "previewTooLargeDescription": "This file exceeds the inline preview limit. Download it to view the original.",
+ "downloadButton": "Download original file"
}
},
"llm-select": {
diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.json
index 2c9ceb301..c13ec5196 100644
--- a/examples/web_ui/frontend/src/i18n/locales/zh.json
+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json
@@ -385,6 +385,35 @@
"chunkPosition": "第 {{index}} / {{total}} 段",
"emptyTitle": "无匹配",
"emptyDescription": "本次查询没有返回任何分块。"
+ },
+ "config": {
+ "title": "配置信息",
+ "embeddingModel": "嵌入模型",
+ "dimensions": "向量维度",
+ "credential": "凭证",
+ "chunker": "分块器",
+ "counts": "内容",
+ "countsValue": "{{documents}} 个文档 · {{chunks}} 个分块",
+ "status": "索引状态",
+ "statusValue": "{{ready}} 就绪 · {{indexing}} 处理中 · {{failed}} 失败",
+ "createdAt": "创建时间"
+ },
+ "documentDetail": {
+ "description": "已索引 {{count}} 个分块",
+ "chunksTab": "分块",
+ "previewTab": "预览",
+ "chunksEmptyTitle": "暂无分块",
+ "chunksEmptyDescription": "该文档尚未产生已索引的分块。",
+ "chunksUnsupportedTitle": "无法浏览分块",
+ "chunksUnsupportedDescription": "当前配置的向量数据库不支持分块列表。",
+ "previousPage": "上一页",
+ "nextPage": "下一页",
+ "pageIndicator": "第 {{page}} / {{pageCount}} 页",
+ "previewUnavailableTitle": "无法在线预览",
+ "previewUnavailableDescription": "该文件类型无法在浏览器中预览。",
+ "previewTooLargeTitle": "文件过大,无法预览",
+ "previewTooLargeDescription": "该文件超出在线预览大小限制,请下载后查看源文件。",
+ "downloadButton": "下载源文件"
}
},
"llm-select": {
diff --git a/examples/web_ui/frontend/src/pages/knowledge/index.tsx b/examples/web_ui/frontend/src/pages/knowledge/index.tsx
index 6191ecc08..64b0f0f93 100644
--- a/examples/web_ui/frontend/src/pages/knowledge/index.tsx
+++ b/examples/web_ui/frontend/src/pages/knowledge/index.tsx
@@ -1,3 +1,4 @@
+import { format } from 'date-fns';
import { Ellipsis, Files, FlaskConical, Pencil, Plus, Trash2 } from 'lucide-react';
import { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
@@ -92,9 +93,89 @@ function DetailPanel({ knowledgeBase, onTest }: DetailPanelProps) {
<Separator className="shrink-0" />
- {/* Documents */}
+ {/* Configuration + documents */}
<div className="min-h-0 flex-1 overflow-y-auto p-[20px_18px_24px]">
- <KnowledgeDocumentsPanel knowledgeBaseId={knowledgeBase.id} />
+ <div className="flex flex-col gap-y-6">
+ <ConfigCard knowledgeBase={knowledgeBase} />
+ <KnowledgeDocumentsPanel knowledgeBaseId={knowledgeBase.id} />
+ </div>
+ </div>
+ </div>
+ );
+}
+
+function ConfigItem({ label, value }: { label: string; value: string }) {
+ return (
+ <div className="flex min-w-0 flex-col gap-y-0.5">
+ <span className="text-muted-foreground text-xs">{label}</span>
+ <span className="truncate text-sm text-foreground" title={value}>
+ {value}
+ </span>
+ </div>
+ );
+}
+
+/**
+ * Read-only card surfacing the full configuration pinned at creation
+ * time — embedding model, chunker, credential — plus the aggregated
+ * counts the list endpoint now serves (#2360).
+ */
+function ConfigCard({ knowledgeBase }: { knowledgeBase: KnowledgeBaseView }) {
+ const { t } = useTranslation();
+ const embedding = knowledgeBase.embedding_model_config;
+ const chunker = knowledgeBase.chunker_config;
+
+ // Only worth showing when something is not ready — a KB where every
+ // document indexed cleanly says nothing extra beyond the totals.
+ const counts = knowledgeBase.status_counts;
+ const unfinished = counts.pending + counts.parsing + counts.chunking + counts.indexing;
+ const statusValue =
+ counts.error > 0 || unfinished > 0
+ ? t('knowledge.config.statusValue', {
+ ready: counts.ready,
+ indexing: unfinished,
+ failed: counts.error,
+ })
+ : null;
+
+ const chunkerValue = chunker
+ ? Object.keys(chunker.parameters).length > 0
+ ? `${chunker.type} · ${Object.entries(chunker.parameters)
+ .map(([k, v]) => `${k}=${String(v)}`)
+ .join(', ')}`
+ : chunker.type
+ : '—';
+
+ return (
+ <div className="flex flex-col gap-y-3">
+ <h3 className="text-[13.5px] font-medium text-foreground">
+ {t('knowledge.config.title')}
+ </h3>
+ <div className="border-border bg-card grid grid-cols-2 gap-x-4 gap-y-3 rounded-lg border p-3 sm:grid-cols-3">
+ <ConfigItem label={t('knowledge.config.embeddingModel')} value={embedding.model} />
+ <ConfigItem
+ label={t('knowledge.config.dimensions')}
+ value={String(embedding.dimensions)}
+ />
+ <ConfigItem
+ label={t('knowledge.config.credential')}
+ value={knowledgeBase.credential_name ?? embedding.credential_id}
+ />
+ <ConfigItem label={t('knowledge.config.chunker')} value={chunkerValue} />
+ <ConfigItem
+ label={t('knowledge.config.counts')}
+ value={t('knowledge.config.countsValue', {
+ documents: knowledgeBase.document_count,
+ chunks: knowledgeBase.chunk_count,
+ })}
+ />
+ {statusValue && (
+ <ConfigItem label={t('knowledge.config.status')} value={statusValue} />
+ )}
+ <ConfigItem
+ label={t('knowledge.config.createdAt')}
+ value={format(new Date(knowledgeBase.created_at), 'yyyy-MM-dd HH:mm')}
+ />
</div>
</div>
);
diff --git a/src/agentscope/app/_router/_knowledge_base.py b/src/agentscope/app/_router/_knowledge_base.py
index 9db702200..f851ce691 100644
--- a/src/agentscope/app/_router/_knowledge_base.py
+++ b/src/agentscope/app/_router/_knowledge_base.py
@@ -7,20 +7,26 @@ The HTTP layer is intentionally thin — every endpoint translates the
request into a single :class:`~agentscope.app._service.
KnowledgeBaseService` call and returns the result.
"""
+from urllib.parse import quote
+
from fastapi import (
APIRouter,
Depends,
File,
Form,
+ Header,
+ HTTPException,
Path,
Query,
UploadFile,
status,
)
+from fastapi.responses import StreamingResponse
from ..access import ResourceKind
from ..deps import (
get_current_user_id,
+ get_download_secret,
get_knowledge_base_manager,
get_knowledge_base_service,
get_knowledge_chunkers,
@@ -29,6 +35,8 @@ from ..deps import (
)
from ._schema import (
ChunkerInfo,
+ DocumentDownloadTokenResponse,
+ ListDocumentChunksResponse,
CreateKnowledgeBaseRequest,
CreateKnowledgeBaseResponse,
KbEmbeddingProvider,
@@ -49,6 +57,8 @@ from ...credential import CredentialFactory
from ..rag.knowledge_base_manager import KnowledgeBaseManagerBase
from .._service import (
KnowledgeBaseService,
+ sign_download_token,
+ verify_download_token,
KnowledgeBaseView,
ResourceAccessService,
)
@@ -300,31 +310,75 @@ async def create_knowledge_base(
summary="List the caller's knowledge bases",
)
async def list_knowledge_bases(
+ id: str # pylint: disable=redefined-builtin # noqa: A002
+ | None = Query(
+ default=None,
+ description=(
+ "Filter down to one knowledge base — the list endpoint "
+ "doubles as get-single, RAGFlow style."
+ ),
+ ),
+ name: str
+ | None = Query(
+ default=None,
+ description="Case-insensitive substring filter on the name.",
+ ),
+ page: int = Query(default=1, ge=1, description="1-based page number."),
+ page_size: int = Query(default=30, ge=1, le=128),
+ orderby: str = Query(
+ default="create_time",
+ pattern="^(create_time|update_time)$",
+ ),
+ desc: bool = Query(default=True, description="Sort newest first."),
user_id: str = Depends(get_current_user_id),
- access: "ResourceAccessService" = Depends(get_resource_access_service),
+ service: "KnowledgeBaseService" = Depends(get_knowledge_base_service),
) -> ListKnowledgeBasesResponse:
- """Return all knowledge bases visible to the authenticated user.
+ """Return the caller's knowledge bases, filtered and paginated.
Includes the caller's own knowledge bases plus any shared to them
through :class:`ResourceAccessPolicyBase`. Each entry carries an
- ``editable`` flag: ``read`` grants search + attach; ``edit`` also
- grants document add/delete and metadata update.
+ ``editable`` flag (``read`` grants search + attach; ``edit`` also
+ grants document add/delete and metadata update) plus the
+ aggregated document / chunk / per-status counts and the resolved
+ ``credential_name`` a detail page needs — the list is the single
+ source of truth; there is no separate get-single endpoint.
Args:
+ id (`str | None`, optional):
+ Filter down to one knowledge base by id.
+ name (`str | None`, optional):
+ Case-insensitive substring filter on the display name.
+ page (`int`, defaults to ``1``):
+ 1-based page number.
+ page_size (`int`, defaults to ``30``):
+ Knowledge bases per page (max 128).
+ orderby (`str`, defaults to ``"create_time"``):
+ Sort key — ``"create_time"`` or ``"update_time"``.
+ desc (`bool`, defaults to ``True``):
+ Sort newest first.
user_id (`str`):
Injected authenticated user ID.
- access (`ResourceAccessService`):
- Injected resource access service.
+ service (`KnowledgeBaseService`):
+ Injected knowledge base service.
Returns:
`ListKnowledgeBasesResponse`:
- All visible knowledge bases paired with per-viewer
- editability.
+ The requested page of views plus the filtered total.
"""
- entries = await access.list_resource(user_id, ResourceKind.KNOWLEDGE_BASE)
+ views, total = await service.list_knowledge_base_views(
+ user_id,
+ knowledge_base_id=id,
+ name=name,
+ page=page,
+ page_size=page_size,
+ orderby=orderby,
+ desc=desc,
+ )
return ListKnowledgeBasesResponse(
- knowledge_bases=entries,
- total=len(entries),
+ knowledge_bases=views,
+ total=total,
+ page=page,
+ page_size=page_size,
)
@@ -410,10 +464,34 @@ async def delete_knowledge_base(
)
async def list_knowledge_documents(
knowledge_base_id: str = Path(description="The knowledge base id."),
+ id: str # pylint: disable=redefined-builtin # noqa: A002
+ | None = Query(
+ default=None,
+ description="Filter down to one document by id.",
+ ),
+ keywords: str
+ | None = Query(
+ default=None,
+ description="Case-insensitive substring filter on the filename.",
+ ),
+ status_filter: str
+ | None = Query(
+ default=None,
+ alias="status",
+ pattern="^(pending|parsing|chunking|indexing|ready|error)$",
+ description="Filter by indexing status.",
+ ),
+ page: int = Query(default=1, ge=1, description="1-based page number."),
+ page_size: int = Query(default=30, ge=1, le=128),
+ orderby: str = Query(
+ default="create_time",
+ pattern="^(create_time|update_time)$",
+ ),
+ desc: bool = Query(default=True, description="Sort newest first."),
user_id: str = Depends(get_current_user_id),
service: "KnowledgeBaseService" = Depends(get_knowledge_base_service),
) -> ListKnowledgeDocumentsResponse:
- """List every document registered against a knowledge base.
+ """List a knowledge base's documents, filtered and paginated.
Reads from the storage backend (service-mode source of truth), so
documents in any lifecycle state — including ``pending`` /
@@ -422,6 +500,20 @@ async def list_knowledge_documents(
Args:
knowledge_base_id (`str`):
The target knowledge base id.
+ id (`str | None`, optional):
+ Filter down to one document by id.
+ keywords (`str | None`, optional):
+ Case-insensitive substring filter on the filename.
+ status_filter (`str | None`, optional):
+ Filter by indexing status.
+ page (`int`, defaults to ``1``):
+ 1-based page number.
+ page_size (`int`, defaults to ``30``):
+ Documents per page (max 128).
+ orderby (`str`, defaults to ``"create_time"``):
+ Sort key — ``"create_time"`` or ``"update_time"``.
+ desc (`bool`, defaults to ``True``):
+ Sort newest first.
user_id (`str`):
Injected authenticated user ID.
service (`KnowledgeBaseService`):
@@ -429,13 +521,25 @@ async def list_knowledge_documents(
Returns:
`ListKnowledgeDocumentsResponse`:
- One view per registered document.
+ The requested page of views plus the filtered total.
"""
- records = await service.list_documents(user_id, knowledge_base_id)
+ records, total = await service.list_documents(
+ user_id,
+ knowledge_base_id,
+ document_id=id,
+ keywords=keywords,
+ doc_status=status_filter,
+ page=page,
+ page_size=page_size,
+ orderby=orderby,
+ desc=desc,
+ )
views = [KnowledgeDocumentView.from_record(r) for r in records]
return ListKnowledgeDocumentsResponse(
documents=views,
- total=len(views),
+ total=total,
+ page=page,
+ page_size=page_size,
)
@@ -617,3 +721,240 @@ async def search_knowledge_base(
top_k=body.top_k,
)
return SearchKnowledgeBaseResponse(results=results, total=len(results))
+
+
+# The media types a browser may render inline. Anything else (notably
+# text/html and image/svg+xml, which can run script) is forced to an
+# attachment so a crafted upload cannot XSS the app origin. Raster
+# image formats are enumerated rather than matched by ``image/``
+# prefix precisely so that SVG never slips through.
+_INLINE_MEDIA_TYPES = frozenset(
+ {
+ "text/plain",
+ "text/markdown",
+ "application/pdf",
+ "image/png",
+ "image/jpeg",
+ "image/gif",
+ "image/webp",
+ "image/bmp",
+ },
+)
+
+
+# Preview tokens outlive a download capability on purpose: a PDF
+# viewer embedded in an <iframe> re-requests the file as the reader
+# pages through it, and a 60-second window would 401 mid-read.
+_PREVIEW_TOKEN_TTL = 600
+
+
+def _document_token_path(knowledge_base_id: str, document_id: str) -> str:
+ """The resource string a document download token is bound to."""
+ return f"kb/{knowledge_base_id}/{document_id}"
+
+
+@knowledge_base_router.get(
+ "/{knowledge_base_id}/documents/{document_id}/chunks",
+ response_model=ListDocumentChunksResponse,
+ summary="Browse one document's chunks in order",
+)
+async def list_document_chunks(
+ knowledge_base_id: str = Path(description="The knowledge base id."),
+ document_id: str = Path(description="The document id."),
+ page: int = Query(default=1, ge=1, description="1-based page number."),
+ page_size: int = Query(default=30, ge=1, le=128),
+ user_id: str = Depends(get_current_user_id),
+ service: "KnowledgeBaseService" = Depends(get_knowledge_base_service),
+) -> ListDocumentChunksResponse:
+ """Return one page of a document's chunks, ``chunk_index`` ascending.
+
+ Pagination is stable: ``chunk_index`` is a dense, immutable
+ ``0..N-1`` sequence within a document, so page ``N`` always maps to
+ the same chunk range. A document that is still indexing serves
+ the chunks persisted so far.
+
+ Args:
+ knowledge_base_id (`str`):
+ The parent knowledge base.
+ document_id (`str`):
+ The document whose chunks should be listed.
+ page (`int`, defaults to ``1``):
+ 1-based page number.
+ page_size (`int`, defaults to ``30``):
+ Chunks per page (max 128).
+ user_id (`str`):
+ Injected authenticated user ID.
+ service (`KnowledgeBaseService`):
+ Injected knowledge base service.
+
+ Returns:
+ `ListDocumentChunksResponse`:
+ The page of chunks plus the document's total chunk count.
+ """
+ chunks, total = await service.list_document_chunks(
+ user_id=user_id,
+ knowledge_base_id=knowledge_base_id,
+ document_id=document_id,
+ page=page,
+ page_size=page_size,
+ )
+ return ListDocumentChunksResponse(
+ chunks=chunks,
+ total=total,
+ page=page,
+ page_size=page_size,
+ )
+
+
+@knowledge_base_router.post(
+ "/{knowledge_base_id}/documents/{document_id}/download_token",
+ response_model=DocumentDownloadTokenResponse,
+ summary="Mint a short-lived token for a browser-native fetch",
+)
+async def create_document_download_token(
+ knowledge_base_id: str = Path(description="The knowledge base id."),
+ document_id: str = Path(description="The document id."),
+ user_id: str = Depends(get_current_user_id),
+ service: "KnowledgeBaseService" = Depends(get_knowledge_base_service),
+ download_secret: str = Depends(get_download_secret),
+) -> DocumentDownloadTokenResponse:
+ """Mint a token so a browser can fetch the raw file directly.
+
+ ``<iframe>`` PDF previews, ``<img>`` tags and click-to-download
+ navigations carry no custom headers, so they cannot present
+ ``X-User-ID``; the token rides in the URL instead and is bound to
+ exactly this document.
+
+ Args:
+ knowledge_base_id (`str`):
+ The parent knowledge base.
+ document_id (`str`):
+ The document the token authorizes.
+ user_id (`str`):
+ Injected authenticated user ID.
+ service (`KnowledgeBaseService`):
+ Injected knowledge base service — resolved here only to
+ fail early with a proper 404 instead of a raw error page
+ on the browser navigation.
+ download_secret (`str`):
+ Injected app-wide signing secret.
+
+ Returns:
+ `DocumentDownloadTokenResponse`:
+ The token and its expiry.
+ """
+ await service.get_document(user_id, knowledge_base_id, document_id)
+ token, expires_at = sign_download_token(
+ download_secret,
+ user_id,
+ _document_token_path(knowledge_base_id, document_id),
+ ttl=_PREVIEW_TOKEN_TTL,
+ )
+ return DocumentDownloadTokenResponse(token=token, expires_at=expires_at)
+
+
+@knowledge_base_router.get(
+ "/{knowledge_base_id}/documents/{document_id}",
+ summary="Fetch the original uploaded file of a document",
+)
+async def read_knowledge_document(
+ knowledge_base_id: str = Path(description="The knowledge base id."),
+ document_id: str = Path(description="The document id."),
+ download: bool = Query(
+ default=False,
+ description="Force a Content-Disposition attachment.",
+ ),
+ token: str
+ | None = Query(
+ default=None,
+ description=(
+ "A token from ``POST .../documents/{document_id}"
+ "/download_token``, accepted in place of the "
+ "``X-User-ID`` header so a browser navigation can fetch "
+ "the file directly."
+ ),
+ ),
+ x_user_id: str | None = Header(default=None),
+ service: "KnowledgeBaseService" = Depends(get_knowledge_base_service),
+ download_secret: str = Depends(get_download_secret),
+) -> StreamingResponse:
+ """Stream the raw uploaded file back for preview or download.
+
+ Mirrors ``GET /workspace/files``: the body is piped chunk by chunk
+ from the blob store rather than read whole, so one large file
+ cannot exhaust the shared API process. Only media types that
+ cannot carry script are served inline; everything else is forced
+ to an attachment.
+
+ Args:
+ knowledge_base_id (`str`):
+ The parent knowledge base.
+ document_id (`str`):
+ The document whose original file should be fetched.
+ download (`bool`, defaults to ``False``):
+ Force an attachment disposition.
+ token (`str | None`, optional):
+ Signed download token, accepted instead of ``X-User-ID``.
+ x_user_id (`str | None`, optional):
+ The normal identity header.
+ service (`KnowledgeBaseService`):
+ Injected knowledge base service.
+ download_secret (`str`):
+ Injected app-wide signing secret.
+
+ Returns:
+ `StreamingResponse`:
+ The file bytes with content-type / length / disposition
+ headers derived from the upload-time record.
+ """
+ if token is not None:
+ try:
+ user_id = verify_download_token(
+ download_secret,
+ token,
+ _document_token_path(knowledge_base_id, document_id),
+ )
+ except ValueError as e:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail=str(e),
+ ) from e
+ elif x_user_id:
+ user_id = x_user_id
+ else:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="X-User-ID header or download token is required.",
+ )
+
+ record, size, content = await service.stream_document_content(
+ user_id=user_id,
+ knowledge_base_id=knowledge_base_id,
+ document_id=document_id,
+ )
+ data = record.data
+ media_type = (data.content_type or "").split(";")[
+ 0
+ ].strip().lower() or "application/octet-stream"
+ inline = not download and media_type in _INLINE_MEDIA_TYPES
+ disposition = "inline" if inline else "attachment"
+ filename = quote(data.filename or "download")
+ headers = {
+ "Content-Disposition": (f"{disposition}; filename*=UTF-8''{filename}"),
+ "Cache-Control": "private, max-age=60",
+ # The declared type is authoritative — never let the browser
+ # sniff a scriptable type out of the bytes.
+ "X-Content-Type-Options": "nosniff",
+ }
+ # Measured on the blob, never read off the record: a wrong length
+ # truncates the body. Present, it turns the browser's download
+ # progress from a byte counter into a percentage; absent (backend
+ # cannot measure), the response is chunked — same rule as
+ # ``GET /workspace/files``.
+ if size is not None:
+ headers["Content-Length"] = str(size)
+ return StreamingResponse(
+ content,
+ media_type=media_type,
+ headers=headers,
+ )
diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.py
index cb7cf0ceb..07de4b6a5 100644
--- a/src/agentscope/app/_router/_schema/__init__.py
+++ b/src/agentscope/app/_router/_schema/__init__.py
@@ -64,6 +64,8 @@ from ._knowledge_base import (
ListChunkersResponse,
ListKbEmbeddingModelsResponse,
ListKnowledgeBasesResponse,
+ ListDocumentChunksResponse,
+ DocumentDownloadTokenResponse,
ListKnowledgeDocumentsResponse,
ListKnowledgeDocumentStatusResponse,
ListSupportedContentTypesResponse,
@@ -141,6 +143,8 @@ __all__ = [
"ListChunkersResponse",
"ListKbEmbeddingModelsResponse",
"ListKnowledgeBasesResponse",
+ "ListDocumentChunksResponse",
+ "DocumentDownloadTokenResponse",
"ListKnowledgeDocumentsResponse",
"ListKnowledgeDocumentStatusResponse",
"ListSupportedContentTypesResponse",
diff --git a/src/agentscope/app/_router/_schema/_knowledge_base.py b/src/agentscope/app/_router/_schema/_knowledge_base.py
index 7a92ffc1c..2bef7a2aa 100644
--- a/src/agentscope/app/_router/_schema/_knowledge_base.py
+++ b/src/agentscope/app/_router/_schema/_knowledge_base.py
@@ -12,7 +12,7 @@ from ...storage import (
)
from ..._service import CredentialView, KnowledgeBaseView
from ....embedding import EmbeddingModelCard
-from ....rag import VectorSearchResult
+from ....rag import Chunk, VectorSearchResult
from ...rag.knowledge_base_manager._dimension_policy import DimensionPolicy
@@ -72,9 +72,16 @@ class ListKnowledgeBasesResponse(BaseModel):
"""Response body for listing the caller's knowledge bases."""
knowledge_bases: list[KnowledgeBaseView] = Field(
- description="All knowledge bases owned by the caller.",
+ description="The requested page of visible knowledge bases.",
)
- total: int = Field(description="Total number of returned items.")
+ total: int = Field(
+ description=(
+ "Total number of knowledge bases matching the filters — "
+ "across all pages, for computing page counts."
+ ),
+ )
+ page: int = Field(default=1, description="The 1-based page served.")
+ page_size: int = Field(default=30, description="The page size used.")
class KnowledgeDocumentView(BaseModel):
@@ -140,9 +147,16 @@ class ListKnowledgeDocumentsResponse(BaseModel):
"""Response body for listing documents inside a knowledge base."""
documents: list[KnowledgeDocumentView] = Field(
- description="One view per registered document.",
+ description="The requested page of document views.",
+ )
+ total: int = Field(
+ description=(
+ "Total number of documents matching the filters — across "
+ "all pages, for computing page counts."
+ ),
)
- total: int = Field(description="Total number of returned items.")
+ page: int = Field(default=1, description="The 1-based page served.")
+ page_size: int = Field(default=30, description="The page size used.")
class ListKnowledgeDocumentStatusResponse(BaseModel):
@@ -302,3 +316,39 @@ class ListChunkersResponse(BaseModel):
chunkers: list[ChunkerInfo] = Field(
description="The available chunker types, in configured order.",
)
+
+
+class ListDocumentChunksResponse(BaseModel):
+ """Response body for browsing one document's chunks page by page."""
+
+ chunks: list[Chunk] = Field(
+ description=(
+ "The requested page of chunks, ordered by ``chunk_index`` "
+ "ascending."
+ ),
+ )
+ total: int = Field(
+ description=(
+ "Total number of chunks indexed for this document, taken "
+ "from the document record. ``0`` while the document is "
+ "still being indexed."
+ ),
+ )
+ page: int = Field(description="The 1-based page number served.")
+ page_size: int = Field(description="The page size used.")
+
+
+class DocumentDownloadTokenResponse(BaseModel):
+ """A capability authorizing one fetch of one document's raw file."""
+
+ token: str = Field(
+ description=(
+ "Pass as the ``token`` query parameter of "
+ "``GET /knowledge_bases/{kb_id}/documents/{document_id}``, "
+ "in place of the ``X-User-ID`` header. Valid for this "
+ "document only."
+ ),
+ )
+ expires_at: float = Field(
+ description="Unix timestamp after which the token is refused.",
+ )
diff --git a/src/agentscope/app/_service/__init__.py b/src/agentscope/app/_service/__init__.py
index 1cfb55a88..a1ce68f15 100644
--- a/src/agentscope/app/_service/__init__.py
+++ b/src/agentscope/app/_service/__init__.py
@@ -3,6 +3,7 @@
from ._access import (
AgentView,
CredentialView,
+ KnowledgeBaseStatusCounts,
KnowledgeBaseView,
ResourceAccessService,
)
@@ -20,6 +21,7 @@ from ._session import SessionService, SessionStatus
from ._session_projection import SessionProjection
from ._projectors import SubagentHitlProjector
from ._toolkit import get_toolkit
+from ._download_token import sign_download_token, verify_download_token
from ._workspace import GitStatus, WorkspaceService, WorkspaceStatus
__all__ = [
@@ -32,6 +34,7 @@ __all__ = [
"IndexTaskConsumer",
"IndexWorker",
"KnowledgeBaseService",
+ "KnowledgeBaseStatusCounts",
"KnowledgeBaseView",
"MCPRenderError",
"ResourceAccessService",
@@ -40,6 +43,8 @@ __all__ = [
"SessionProjection",
"SubagentHitlProjector",
"WorkspaceService",
+ "sign_download_token",
+ "verify_download_token",
"WorkspaceStatus",
"get_embedding_model",
"get_model",
diff --git a/src/agentscope/app/_service/_access.py b/src/agentscope/app/_service/_access.py
index 9773efac6..8e4101d33 100644
--- a/src/agentscope/app/_service/_access.py
+++ b/src/agentscope/app/_service/_access.py
@@ -6,7 +6,7 @@ from datetime import datetime
from typing import Any, Literal, TypeVar, overload
from fastapi import HTTPException, status
-from pydantic import BaseModel, Field, model_validator
+from pydantic import AliasChoices, BaseModel, Field, model_validator
from ..access import (
ResourceAccessPolicyBase,
@@ -61,6 +61,23 @@ class CredentialView(CredentialRecord):
)
+class KnowledgeBaseStatusCounts(BaseModel):
+ """Documents of one knowledge base, counted by indexing status.
+
+ Built from a :class:`collections.Counter` keyed by
+ ``KnowledgeDocumentStatus``; a status without a field here is
+ ignored rather than raising, so adding a lifecycle state cannot
+ break the list endpoint.
+ """
+
+ pending: int = 0
+ parsing: int = 0
+ chunking: int = 0
+ indexing: int = 0
+ ready: int = 0
+ error: int = 0
+
+
class KnowledgeBaseView(BaseModel):
"""Flat, viewer-facing projection of a knowledge base.
@@ -72,8 +89,10 @@ class KnowledgeBaseView(BaseModel):
nesting of those fields under :attr:`KnowledgeBaseRecord.data`
(introduced so the SQL backend can serialise a single JSON column)
must not leak into the HTTP response, so :meth:`_lift_data_payload`
- lifts them back up. ``user_id`` and the opaque ``collection_name``
- are dropped (pydantic ignores the surplus keys).
+ lifts them back up. The record's ``user_id`` is kept as
+ :attr:`owner_id` for server-side reads but excluded from the wire
+ shape; the opaque ``collection_name`` is dropped outright (pydantic
+ ignores the surplus key).
"""
id: str = Field(description="The knowledge base id.")
@@ -99,6 +118,39 @@ class KnowledgeBaseView(BaseModel):
"Whether the current viewer may modify this knowledge base."
),
)
+ owner_id: str = Field(
+ validation_alias=AliasChoices("owner_id", "user_id"),
+ exclude=True,
+ description=(
+ "The owning user id, carried for server-side reads (shared "
+ "viewers must query the owner's storage) and excluded from "
+ "the wire shape."
+ ),
+ )
+ document_count: int = Field(
+ default=0,
+ description="Number of documents registered in the knowledge base.",
+ )
+ chunk_count: int = Field(
+ default=0,
+ description=(
+ "Total indexed chunks across all documents, summed from the "
+ "per-document records."
+ ),
+ )
+ credential_name: str | None = Field(
+ default=None,
+ description=(
+ "Display name of the credential behind "
+ "``embedding_model_config.credential_id``, resolved against "
+ "the owner so shared viewers see it too. ``None`` when the "
+ "credential has been deleted."
+ ),
+ )
+ status_counts: KnowledgeBaseStatusCounts = Field(
+ default_factory=KnowledgeBaseStatusCounts,
+ description="Documents broken down by indexing status.",
+ )
@model_validator(mode="before")
@classmethod
diff --git a/src/agentscope/app/_service/_download_token.py b/src/agentscope/app/_service/_download_token.py
new file mode 100644
index 000000000..9905be8e9
--- /dev/null
+++ b/src/agentscope/app/_service/_download_token.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+"""Short-lived signed download tokens.
+
+A browser-native fetch (an ``<iframe>`` PDF preview, an ``<img>`` tag,
+or a click-to-download navigation) carries no custom headers, so it
+cannot present ``X-User-ID``. These helpers mint and verify a
+capability that rides in the URL instead: an HMAC over
+``(expires_at, user_id, path)`` keyed by the app-wide download secret.
+
+The scheme is identical to the one
+:class:`~agentscope.app._service._workspace.WorkspaceService` uses for
+workspace files; it lives here as free functions so other routers
+(knowledge base document preview in v1) can sign against the same
+``app.state.download_secret`` without depending on the workspace
+service.
+"""
+import base64
+import hashlib
+import hmac
+import time
+
+# Long enough to survive a slow round trip and the user's click, short
+# enough that a token leaked through a log or history is already dead.
+DEFAULT_DOWNLOAD_TOKEN_TTL = 60
+
+
+def sign_download_token(
+ secret: str,
+ user_id: str,
+ path: str,
+ ttl: int = DEFAULT_DOWNLOAD_TOKEN_TTL,
+) -> tuple[str, int]:
+ """Mint a token authorizing one download.
+
+ Args:
+ secret (`str`):
+ The app-wide signing secret
+ (``app.state.download_secret``).
+ user_id (`str`):
+ The already-authenticated caller.
+ path (`str`):
+ The resource the token authorizes, verbatim as the
+ download request will re-derive it.
+ ttl (`int`, defaults to ``60``):
+ Seconds the token stays valid.
+
+ Returns:
+ `tuple[str, int]`:
+ The token and its expiry as a Unix timestamp.
+ """
+ expires_at = int(time.time()) + ttl
+ signature = _signature(secret, expires_at, user_id, path)
+ token = (
+ f"{expires_at}"
+ f".{_b64(user_id.encode('utf-8'))}"
+ f".{_b64(signature)}"
+ )
+ return token, expires_at
+
+
+def verify_download_token(secret: str, token: str, path: str) -> str:
+ """Return the user a token was granted to, for this path.
+
+ The path is not read out of the token but re-derived from the
+ request, so a token for one resource cannot be replayed against
+ another.
+
+ Args:
+ secret (`str`):
+ The app-wide signing secret.
+ token (`str`):
+ The token from the request.
+ path (`str`):
+ The resource the request is asking for.
+
+ Returns:
+ `str`:
+ The user ID the token was minted for.
+
+ Raises:
+ `ValueError`:
+ The token is malformed, expired, or does not match.
+ """
+ try:
+ raw_expiry, raw_user, raw_signature = token.split(".")
+ expires_at = int(raw_expiry)
+ user_id = _unb64(raw_user).decode("utf-8")
+ signature = _unb64(raw_signature)
+ except (ValueError, UnicodeDecodeError) as e:
+ raise ValueError("Malformed download token.") from e
+
+ expected = _signature(secret, expires_at, user_id, path)
+ if not hmac.compare_digest(signature, expected):
+ raise ValueError("Invalid download token.")
+ if expires_at < time.time():
+ raise ValueError("Expired download token.")
+ return user_id
+
+
+def _signature(
+ secret: str,
+ expires_at: int,
+ user_id: str,
+ path: str,
+) -> bytes:
+ """Compute the MAC binding an expiry, a user and a path.
+
+ ``\\0`` separates the fields because it cannot occur in any of
+ them, so no combination can be re-cut into a different triple.
+
+ Args:
+ secret (`str`): The signing secret.
+ expires_at (`int`): Unix timestamp after which to refuse.
+ user_id (`str`): The user the capability was granted to.
+ path (`str`): The one resource the capability covers.
+
+ Returns:
+ `bytes`: The raw HMAC-SHA256 digest.
+ """
+ message = f"{expires_at}\0{user_id}\0{path}".encode("utf-8")
+ return hmac.new(
+ secret.encode("utf-8"),
+ message,
+ hashlib.sha256,
+ ).digest()
+
+
+def _b64(raw: bytes) -> str:
+ """Encode without padding, which is not URL-safe to round-trip."""
+ return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
+
+
+def _unb64(text: str) -> bytes:
+ """Reverse :func:`_b64`, restoring the stripped padding."""
+ return base64.urlsafe_b64decode(text + "=" * (-len(text) % 4))
diff --git a/src/agentscope/app/_service/_index_worker.py b/src/agentscope/app/_service/_index_worker.py
index 0625e7553..f79511bc1 100644
--- a/src/agentscope/app/_service/_index_worker.py
+++ b/src/agentscope/app/_service/_index_worker.py
@@ -419,6 +419,19 @@ class IndexWorker:
user_id,
knowledge_base_id,
)
+ # A retry re-runs the whole pipeline. Records are keyed by
+ # (document_id, chunk_index), so re-inserting overwrites in
+ # place — but a re-parse that yields fewer chunks would leave
+ # the old tail behind, so drop the previous vectors first.
+ #
+ # The trade-off: between this delete and the insert below the
+ # document is unsearchable, where before it kept a complete but
+ # stale set; and a first-time index pays a no-op delete. The
+ # narrow alternative — insert, then drop only
+ # ``chunk_index >= len(chunks)`` — needs a vector-store API that
+ # deletes by more than ``document_id``, which is not worth a new
+ # abstract method for a retry-only window.
+ await knowledge.delete_document(document_id)
await knowledge.insert_document(
chunks=chunks,
document_id=document_id,
diff --git a/src/agentscope/app/_service/_knowledge_base.py b/src/agentscope/app/_service/_knowledge_base.py
index f0a921bd0..a2d12b62d 100644
--- a/src/agentscope/app/_service/_knowledge_base.py
+++ b/src/agentscope/app/_service/_knowledge_base.py
@@ -18,7 +18,8 @@ project's stance is that a knowledge base is managed end-to-end in one
mode.
"""
import uuid
-from typing import IO, TYPE_CHECKING
+from collections import Counter
+from typing import IO, TYPE_CHECKING, AsyncIterator
from fastapi import HTTPException, status
from pydantic import ValidationError
@@ -34,9 +35,13 @@ from ..storage import (
KnowledgeDocumentRecord,
)
from ..._logging import logger
-from ...rag import ApproxTokenChunker
+from ...rag import ApproxTokenChunker, Chunk
from .._bus_ops import enqueue_index_task
-from ._access import ResourceAccessService
+from ._access import (
+ KnowledgeBaseStatusCounts,
+ KnowledgeBaseView,
+ ResourceAccessService,
+)
if TYPE_CHECKING:
from ..rag.blob_store import BlobStoreBase
@@ -198,6 +203,101 @@ class KnowledgeBaseService:
"""
return await self._manager.list_knowledge_bases(user_id)
+ async def list_knowledge_base_views(
+ self,
+ user_id: str,
+ *,
+ knowledge_base_id: str | None = None,
+ name: str | None = None,
+ page: int = 1,
+ page_size: int = 30,
+ orderby: str = "create_time",
+ desc: bool = True,
+ ) -> tuple[list[KnowledgeBaseView], int]:
+ """List visible knowledge bases as enriched, paginated views.
+
+ Serves the single list endpoint that doubles as "get one"
+ (filter by ``knowledge_base_id``), mirroring RAGFlow's dataset
+ API. Filtering and ordering happen before the page is cut;
+ the returned total counts the filtered set so clients can
+ compute page numbers.
+
+ Only the served page is enriched: per view, the documents are
+ read once from storage to derive the counts, and the embedding
+ credential is resolved against the owner so shared viewers see
+ its display name too.
+
+ Args:
+ user_id (`str`):
+ The viewer user id — own + shared knowledge bases.
+ knowledge_base_id (`str | None`, optional):
+ Filter down to one knowledge base by id.
+ name (`str | None`, optional):
+ Case-insensitive substring filter on the display name.
+ page (`int`, defaults to ``1``):
+ 1-based page number.
+ page_size (`int`, defaults to ``30``):
+ Knowledge bases per page.
+ orderby (`str`, defaults to ``"create_time"``):
+ Sort key — ``"create_time"`` or ``"update_time"``.
+ desc (`bool`, defaults to ``True``):
+ Sort newest first.
+
+ Returns:
+ `tuple[list[KnowledgeBaseView], int]`:
+ The requested page of views and the filtered total.
+ """
+ views = await self._access.list_resource(
+ user_id,
+ ResourceKind.KNOWLEDGE_BASE,
+ )
+ if knowledge_base_id is not None:
+ views = [view for view in views if view.id == knowledge_base_id]
+ if name is not None:
+ needle = name.lower()
+ views = [view for view in views if needle in view.name.lower()]
+ total = len(views)
+
+ # The id breaks ties: storage returns rows in an undefined
+ # order and MySQL DATETIME has no sub-second precision, so
+ # records created together would otherwise shuffle between
+ # requests and make pages drop or repeat rows.
+ sort_key = "updated_at" if orderby == "update_time" else "created_at"
+ views.sort(
+ key=lambda view: (getattr(view, sort_key), view.id),
+ reverse=desc,
+ )
+ page_views = views[(page - 1) * page_size : page * page_size]
+
+ credential_names: dict[tuple[str, str], str | None] = {}
+ for view in page_views:
+ documents = await self._storage.list_knowledge_documents(
+ view.owner_id,
+ view.id,
+ )
+ view.document_count = len(documents)
+ view.chunk_count = sum(
+ document.data.chunk_count for document in documents
+ )
+ view.status_counts = KnowledgeBaseStatusCounts.model_validate(
+ Counter(document.status for document in documents),
+ )
+
+ cache_key = (
+ view.owner_id,
+ view.embedding_model_config.credential_id,
+ )
+ if cache_key not in credential_names:
+ credential = await self._storage.get_credential(*cache_key)
+ credential_names[cache_key] = (
+ credential.data.get("name")
+ if credential is not None
+ else None
+ )
+ view.credential_name = credential_names[cache_key]
+
+ return page_views, total
+
async def update_knowledge_base(
self,
user_id: str,
@@ -347,8 +447,16 @@ class KnowledgeBaseService:
self,
user_id: str,
knowledge_base_id: str,
- ) -> list[KnowledgeDocumentRecord]:
- """List every document registered against a knowledge base.
+ *,
+ document_id: str | None = None,
+ keywords: str | None = None,
+ doc_status: str | None = None,
+ page: int = 1,
+ page_size: int = 30,
+ orderby: str = "create_time",
+ desc: bool = True,
+ ) -> tuple[list[KnowledgeDocumentRecord], int]:
+ """List documents of a knowledge base, filtered and paginated.
Service-mode source of truth: reads from storage, NOT the
vector store. Documents in ``pending`` / ``parsing`` /
@@ -361,11 +469,25 @@ class KnowledgeBaseService:
granted access through the resource access policy.
knowledge_base_id (`str`):
The target knowledge base id.
+ document_id (`str | None`, optional):
+ Filter down to one document by id.
+ keywords (`str | None`, optional):
+ Case-insensitive substring filter on the filename.
+ doc_status (`str | None`, optional):
+ Filter by indexing status (``pending`` / ``parsing`` /
+ ``chunking`` / ``indexing`` / ``ready`` / ``error``).
+ page (`int`, defaults to ``1``):
+ 1-based page number.
+ page_size (`int`, defaults to ``30``):
+ Documents per page.
+ orderby (`str`, defaults to ``"create_time"``):
+ Sort key — ``"create_time"`` or ``"update_time"``.
+ desc (`bool`, defaults to ``True``):
+ Sort newest first.
Returns:
- `list[KnowledgeDocumentRecord]`:
- Every document registered against the knowledge base,
- in unspecified order.
+ `tuple[list[KnowledgeDocumentRecord], int]`:
+ The requested page of records and the filtered total.
Raises:
`HTTPException`:
@@ -376,10 +498,28 @@ class KnowledgeBaseService:
user_id,
knowledge_base_id,
)
- return await self._storage.list_knowledge_documents(
+ records = await self._storage.list_knowledge_documents(
record.user_id,
knowledge_base_id,
)
+ if document_id is not None:
+ records = [r for r in records if r.id == document_id]
+ if keywords is not None:
+ needle = keywords.lower()
+ records = [r for r in records if needle in r.data.filename.lower()]
+ if doc_status is not None:
+ records = [r for r in records if r.status == doc_status]
+ total = len(records)
+
+ # See ``list_knowledge_base_views`` — the id breaks timestamp
+ # ties so pages stay stable; a batch upload lands within one
+ # MySQL DATETIME tick.
+ sort_key = "updated_at" if orderby == "update_time" else "created_at"
+ records.sort(
+ key=lambda record: (getattr(record, sort_key), record.id),
+ reverse=desc,
+ )
+ return records[(page - 1) * page_size : page * page_size], total
async def get_document_status(
self,
@@ -428,6 +568,183 @@ class KnowledgeBaseService:
records.append(record_doc)
return records
+ async def get_document(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ ) -> KnowledgeDocumentRecord:
+ """Fetch one document record, or 404.
+
+ Args:
+ user_id (`str`):
+ The viewer user id — owner or shared reader.
+ knowledge_base_id (`str`):
+ The parent knowledge base id.
+ document_id (`str`):
+ The document to fetch.
+
+ Returns:
+ `KnowledgeDocumentRecord`:
+ The matching record.
+
+ Raises:
+ `HTTPException`:
+ ``404`` if the knowledge base is not visible to the
+ caller or the document does not exist in it.
+ """
+ record = await self._access.resolve_knowledge_base(
+ user_id,
+ knowledge_base_id,
+ )
+ document = await self._storage.get_knowledge_document(
+ record.user_id,
+ knowledge_base_id,
+ document_id,
+ )
+ if document is None:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail=f"Document {document_id} not found.",
+ )
+ return document
+
+ async def list_document_chunks(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ *,
+ page: int = 1,
+ page_size: int = 30,
+ ) -> tuple[list[Chunk], int]:
+ """List one page of a document's chunks in ``chunk_index`` order.
+
+ ``page`` / ``page_size`` translate to the dense-``chunk_index``
+ slice ``[(page - 1) * page_size, page * page_size)`` — stable
+ under concurrent writes because chunk indices never move. The
+ total comes from the document record's ``chunk_count`` rather
+ than a vector-store count, so a document that is still
+ indexing reports the chunks persisted so far with a total
+ of ``0`` until it turns ``ready``.
+
+ Args:
+ user_id (`str`):
+ The viewer user id — owner or shared reader.
+ knowledge_base_id (`str`):
+ The parent knowledge base id.
+ document_id (`str`):
+ The document whose chunks should be listed.
+ page (`int`, defaults to ``1``):
+ 1-based page number.
+ page_size (`int`, defaults to ``30``):
+ Chunks per page.
+
+ Returns:
+ `tuple[list[Chunk], int]`:
+ The page of chunks (``chunk_index`` ascending) and the
+ document's total chunk count.
+
+ Raises:
+ `HTTPException`:
+ ``404`` if the knowledge base or document is not
+ visible to the caller; ``501`` if the configured
+ vector store does not implement chunk listing.
+ """
+ document = await self.get_document(
+ user_id,
+ knowledge_base_id,
+ document_id,
+ )
+ knowledge = await self._resolve_knowledge(user_id, knowledge_base_id)
+ try:
+ chunks = await knowledge.list_chunks(
+ document_id,
+ offset=(page - 1) * page_size,
+ limit=page_size,
+ )
+ except NotImplementedError as exc:
+ raise HTTPException(
+ status_code=status.HTTP_501_NOT_IMPLEMENTED,
+ detail=(
+ "The configured vector store does not support "
+ "chunk listing."
+ ),
+ ) from exc
+ return chunks, document.data.chunk_count
+
+ async def stream_document_content(
+ self,
+ user_id: str,
+ knowledge_base_id: str,
+ document_id: str,
+ ) -> tuple[KnowledgeDocumentRecord, int | None, AsyncIterator[bytes]]:
+ """Open the original uploaded file of a document for streaming.
+
+ The bytes come straight from the blob store — the same blob
+ the index worker parsed — in bounded 1 MiB chunks so a large
+ download never holds the whole file in the API process.
+
+ The size is measured on the blob rather than read off the
+ record: ``data.size`` is what the upload request happened to
+ carry, and a ``Content-Length`` that disagrees with the body
+ breaks the response.
+
+ Args:
+ user_id (`str`):
+ The viewer user id — owner or shared reader.
+ knowledge_base_id (`str`):
+ The parent knowledge base id.
+ document_id (`str`):
+ The document whose original file should be streamed.
+
+ Returns:
+ `tuple[KnowledgeDocumentRecord, int | None, AsyncIterator[bytes]]`:
+ The document record (for the filename / content-type
+ headers), the blob's measured byte length — ``None``
+ when the backend cannot measure it — and a lazy byte
+ iterator that opens the blob on first pull.
+
+ Raises:
+ `HTTPException`:
+ ``404`` if the knowledge base or document is not
+ visible to the caller, or the underlying blob is gone
+ (legacy data, switched blob backend).
+ """
+ document = await self.get_document(
+ user_id,
+ knowledge_base_id,
+ document_id,
+ )
+ blob_uri = document.data.blob_uri
+ try:
+ # A measured size doubles as proof the blob is there, so the
+ # extra existence check only runs for backends that cannot
+ # measure.
+ size = await self._blob_store.size(blob_uri)
+ available = size is not None or await self._blob_store.exists(
+ blob_uri,
+ )
+ except ValueError:
+ # URI scheme unknown to this backend (e.g. records written
+ # by a local:// deployment now running on s3://).
+ size, available = None, False
+ if not available:
+ raise HTTPException(
+ status_code=status.HTTP_404_NOT_FOUND,
+ detail="The original file is no longer available.",
+ )
+
+ async def _iter_content() -> AsyncIterator[bytes]:
+ async with self._blob_store.open(blob_uri) as fp:
+ while True:
+ data = await fp.read(1 << 20)
+ if not data:
+ break
+ yield data
+
+ return document, size, _iter_content()
+
async def delete_document(
self,
user_id: str,
diff --git a/src/agentscope/app/_service/_workspace.py b/src/agentscope/app/_service/_workspace.py
index 32f03b6a7..c60401a1f 100644
--- a/src/agentscope/app/_service/_workspace.py
+++ b/src/agentscope/app/_service/_workspace.py
@@ -1,27 +1,24 @@
# -*- coding: utf-8 -*-
"""Workspace service — resolution, download tokens, uploads, git."""
import asyncio
-import base64
-import hashlib
-import hmac
import re
import tarfile
-import time
from typing import AsyncIterator, Literal
from fastapi import HTTPException, UploadFile, status
from pydantic import BaseModel, Field
+from ._download_token import (
+ DEFAULT_DOWNLOAD_TOKEN_TTL,
+ sign_download_token,
+ verify_download_token,
+)
from ..storage import SessionRecord, StorageBase
from ..workspace_manager import WorkspaceManagerBase
from ..._logging import logger
from ...tool import BackendBase
from ...workspace import WorkspaceBase
-# Long enough to survive a slow round trip and the user's click, short
-# enough that a token leaked through a log or history is already dead.
-DEFAULT_DOWNLOAD_TOKEN_TTL = 60
-
#: Ceilings on one upload. Peak memory is a chunk per concurrent
#: install; these bound the sandbox's disk and how long a slot is held.
MAX_FILE_BYTES = 50 * 1024 * 1024
@@ -317,14 +314,12 @@ class WorkspaceService:
`tuple[str, int]`: The token and its expiry as a Unix
timestamp.
"""
- expires_at = int(time.time()) + ttl
- signature = self._signature(expires_at, user_id, path)
- token = (
- f"{expires_at}"
- f".{self._b64(user_id.encode('utf-8'))}"
- f".{self._b64(signature)}"
+ return sign_download_token(
+ self._download_secret,
+ user_id,
+ path,
+ ttl=ttl,
)
- return token, expires_at
def verify_download_token(self, token: str, path: str) -> str:
"""Return the user a token was granted to, for this path.
@@ -344,51 +339,7 @@ class WorkspaceService:
`ValueError`: The token is malformed, expired, or does not
match.
"""
- try:
- raw_expiry, raw_user, raw_signature = token.split(".")
- expires_at = int(raw_expiry)
- user_id = self._unb64(raw_user).decode("utf-8")
- signature = self._unb64(raw_signature)
- except (ValueError, UnicodeDecodeError) as e:
- raise ValueError("Malformed download token.") from e
-
- expected = self._signature(expires_at, user_id, path)
- if not hmac.compare_digest(signature, expected):
- raise ValueError("Invalid download token.")
- if expires_at < time.time():
- raise ValueError("Expired download token.")
- return user_id
-
- def _signature(self, expires_at: int, user_id: str, path: str) -> bytes:
- """Compute the MAC binding an expiry, a user and a path.
-
- ``\\0`` separates the fields because it cannot occur in any of
- them, so no combination can be re-cut into a different triple.
-
- Args:
- expires_at (`int`): Unix timestamp after which to refuse.
- user_id (`str`): The user the capability was granted to.
- path (`str`): The one path the capability covers.
-
- Returns:
- `bytes`: The raw HMAC-SHA256 digest.
- """
- message = f"{expires_at}\0{user_id}\0{path}".encode("utf-8")
- return hmac.new(
- self._download_secret.encode("utf-8"),
- message,
- hashlib.sha256,
- ).digest()
-
- @staticmethod
- def _b64(raw: bytes) -> str:
- """Encode without padding, which is not URL-safe to round-trip."""
- return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
-
- @staticmethod
- def _unb64(text: str) -> bytes:
- """Reverse :meth:`_b64`, restoring the stripped padding."""
- return base64.urlsafe_b64decode(text + "=" * (-len(text) % 4))
+ return verify_download_token(self._download_secret, token, path)
# ── Skill upload ───────────────────────────────────────────────────
#
diff --git a/src/agentscope/app/rag/blob_store/_base.py b/src/agentscope/app/rag/blob_store/_base.py
index a7c4bed68..ebabca641 100644
--- a/src/agentscope/app/rag/blob_store/_base.py
+++ b/src/agentscope/app/rag/blob_store/_base.py
@@ -144,6 +144,34 @@ class BlobStoreBase(ABC):
A URI produced by :meth:`write_stream`.
"""
+ async def size(self, uri: str) -> int | None:
+ """Return the stored byte length of the blob, if measurable.
+
+ Callers use this to set ``Content-Length`` on a download, which
+ is what turns a browser's progress bar from "bytes so far" into
+ a percentage. The number therefore has to be the length of the
+ bytes :meth:`open` will actually yield — a stale copy recorded
+ elsewhere would truncate the response — so backends must
+ measure the object itself (``stat`` / ``HEAD``), never echo a
+ size supplied at upload time.
+
+ Not abstract: it was added after third-party subclasses already
+ existed, and the default keeps them working. ``None`` means "I
+ cannot cheaply measure this", and callers then omit the header
+ rather than guessing.
+
+ Args:
+ uri (`str`):
+ A URI produced by :meth:`write_stream`.
+
+ Returns:
+ `int | None`:
+ The byte length, or ``None`` when the blob is missing
+ or the backend cannot measure it.
+ """
+ del uri # the default cannot measure anything
+ return None
+
@abstractmethod
async def exists(self, uri: str) -> bool:
"""Return whether the blob at ``uri`` is present.
diff --git a/src/agentscope/app/rag/blob_store/_local.py b/src/agentscope/app/rag/blob_store/_local.py
index 10ddd2826..62241dfee 100644
--- a/src/agentscope/app/rag/blob_store/_local.py
+++ b/src/agentscope/app/rag/blob_store/_local.py
@@ -116,6 +116,14 @@ class LocalBlobStore(BlobStoreBase):
async with aiofiles.open(path, "rb") as fp:
yield fp
+ async def size(self, uri: str) -> int | None:
+ """Return the file's size on disk, or ``None`` if it is gone."""
+ path = self._path_for(self._key_from_uri(uri))
+ try:
+ return (await aiofiles.os.stat(path)).st_size
+ except FileNotFoundError:
+ return None
+
async def delete(self, uri: str) -> None:
"""Remove the blob at ``uri`` if present (idempotent)."""
path = self._path_for(self._key_from_uri(uri))
diff --git a/src/agentscope/app/rag/blob_store/_s3.py b/src/agentscope/app/rag/blob_store/_s3.py
index 0c21a7141..dbef355e9 100644
--- a/src/agentscope/app/rag/blob_store/_s3.py
+++ b/src/agentscope/app/rag/blob_store/_s3.py
@@ -253,6 +253,21 @@ class S3BlobStore(BlobStoreBase):
# no special handling needed for the "already gone" case.
await s3.delete_object(Bucket=self._bucket, Key=key)
+ async def size(self, uri: str) -> int | None:
+ """Return the object's ``ContentLength``, or ``None`` if gone.
+
+ One ``HEAD`` — the same call :meth:`exists` makes, so a caller
+ that needs both can ask for the size and treat a number as
+ proof of existence.
+ """
+ key = self._key_from_uri(uri, self._bucket)
+ async with self._client() as s3:
+ try:
+ head = await s3.head_object(Bucket=self._bucket, Key=key)
+ except Exception: # noqa: BLE001 — see ``exists``
+ return None
+ return head.get("ContentLength")
+
async def exists(self, uri: str) -> bool:
"""Return whether the object at ``uri`` is present."""
key = self._key_from_uri(uri, self._bucket)
diff --git a/src/agentscope/rag/_knowledge.py b/src/agentscope/rag/_knowledge.py
index cc87c0877..14c862c39 100644
--- a/src/agentscope/rag/_knowledge.py
+++ b/src/agentscope/rag/_knowledge.py
@@ -382,3 +382,43 @@ class KnowledgeBase:
self._collection,
metadata_filter=self._metadata_filter,
)
+
+ async def list_chunks(
+ self,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ ) -> list[Chunk]:
+ """List one document's chunks ordered by ``chunk_index``.
+
+ Filtered by :attr:`metadata_filter` when set, so callers only
+ ever see chunks within their own scope.
+
+ Args:
+ document_id (`str`):
+ The source document whose chunks should be listed.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip; equals the first
+ returned chunk's ``chunk_index``.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+
+ Returns:
+ `list[Chunk]`:
+ At most ``limit`` chunks ordered by ``chunk_index``
+ ascending.
+
+ Raises:
+ `NotImplementedError`:
+ If the configured vector store does not support chunk
+ listing.
+ """
+ await self.ensure_collection()
+ return await self._vector_store.list_chunks(
+ self._collection,
+ document_id,
+ offset=offset,
+ limit=limit,
+ metadata_filter=self._metadata_filter,
+ )
diff --git a/src/agentscope/rag/_vdb/_elasticsearch.py b/src/agentscope/rag/_vdb/_elasticsearch.py
index 79e149bc0..ff2593668 100644
--- a/src/agentscope/rag/_vdb/_elasticsearch.py
+++ b/src/agentscope/rag/_vdb/_elasticsearch.py
@@ -269,6 +269,105 @@ class ElasticsearchStore(VectorStoreBase):
return summaries
+ # ------------------------------------------------------------------
+ # Chunk listing
+ # ------------------------------------------------------------------
+
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list[Chunk]:
+ """List one document's chunks ordered by ``chunk_index``.
+
+ The index mapping stores ``chunk`` with ``enabled: false`` (it
+ is a verbatim payload, neither indexed nor sortable), so unlike
+ the other backends this implementation cannot push the
+ ``chunk_index`` range down to the server. Instead it retrieves
+ **all** chunks of the one document (a ``term`` query on the
+ indexed ``document_id`` field, paged through a point-in-time
+ with ``_shard_doc`` / ``search_after`` — which needs
+ Elasticsearch >= 7.12 and is unsupported on OpenSearch — so
+ documents beyond the 10000-hit window still work), sorts them
+ in Python, and slices the requested page — O(chunks of one
+ document) per page, which is bounded and acceptable for an
+ interactive detail view.
+
+ Args:
+ collection (`str`):
+ The target collection name.
+ document_id (`str`):
+ The source document whose chunks should be listed.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+ metadata_filter (`dict[str, Any] | None`, optional):
+ Extra ``chunk.metadata`` equality constraints.
+
+ Returns:
+ `list[Chunk]`:
+ At most ``limit`` chunks, ``chunk_index`` ascending.
+ """
+ if limit <= 0:
+ return []
+ filters: list[dict[str, Any]] = [
+ {"term": {"document_id": document_id}},
+ ]
+ filters.extend(self._metadata_filters(metadata_filter))
+
+ client = self.get_client()
+ pit = await client.open_point_in_time(
+ index=collection,
+ keep_alive="1m",
+ )
+ pit_id = pit["id"]
+ # Only the requested window is materialised: the scan still has
+ # to visit every hit of the document (``chunk`` is not
+ # filterable server-side), but chunks outside
+ # ``[offset, offset + limit)`` are discarded on sight, keeping
+ # memory O(page_size) regardless of the document's total size.
+ by_index: dict[int, Chunk] = {}
+ try:
+ search_after: list[Any] | None = None
+ while True:
+ body: dict[str, Any] = {
+ "query": {"bool": {"filter": filters}},
+ "size": 1000,
+ "pit": {"id": pit_id, "keep_alive": "1m"},
+ "sort": [{"_shard_doc": "asc"}],
+ "_source": ["chunk"],
+ }
+ if search_after is not None:
+ body["search_after"] = search_after
+ response = await client.search(**body)
+ hits = response["hits"]["hits"]
+ if not hits:
+ break
+ for hit in hits:
+ payload = hit["_source"]["chunk"]
+ index = payload.get("chunk_index")
+ if (
+ not isinstance(index, int)
+ or index < offset
+ or index >= offset + limit
+ ):
+ continue
+ by_index.setdefault(
+ index,
+ Chunk.model_validate(payload),
+ )
+ pit_id = response.get("pit_id", pit_id)
+ search_after = hits[-1]["sort"]
+ finally:
+ await client.close_point_in_time(id=pit_id)
+
+ return [by_index[index] for index in sorted(by_index)]
+
@staticmethod
def _record_id(record: VectorRecord) -> str:
"""Build a stable ID so re-indexing replaces the same chunk."""
diff --git a/src/agentscope/rag/_vdb/_milvus_lite.py b/src/agentscope/rag/_vdb/_milvus_lite.py
index ebd963f99..62aa8d1bc 100644
--- a/src/agentscope/rag/_vdb/_milvus_lite.py
+++ b/src/agentscope/rag/_vdb/_milvus_lite.py
@@ -6,9 +6,9 @@ started automatically when the URI points to a local ``.db`` file, so it
is convenient for local development, tests, and small RAG workloads.
"""
import asyncio
+import hashlib
import json
import os
-import uuid
from typing import TYPE_CHECKING, Any, Literal
from ._vector_store import (
@@ -204,6 +204,11 @@ class MilvusLiteStore(VectorStoreBase):
JSON field used by :meth:`search` and :meth:`list_documents` for
flat equality filtering.
+ Entity IDs are derived deterministically from
+ ``(document_id, chunk_index)`` and written via ``upsert`` so
+ that re-indexing the same document after a mid-pipeline crash
+ replaces the previous records instead of duplicating them.
+
Args:
collection (`str`):
The target collection name.
@@ -217,11 +222,11 @@ class MilvusLiteStore(VectorStoreBase):
for start in range(0, len(records), self._batch_size):
batch = records[start : start + self._batch_size]
await asyncio.to_thread(
- self.get_client().insert,
+ self.get_client().upsert,
collection_name=collection,
data=[
{
- "id": str(uuid.uuid4()),
+ "id": self._record_id(record),
"vector": record.vector,
"document_id": record.document_id,
"chunk": record.chunk.model_dump(mode="json"),
@@ -231,6 +236,26 @@ class MilvusLiteStore(VectorStoreBase):
],
)
+ @staticmethod
+ def _record_id(record: VectorRecord) -> str:
+ """Deterministic entity ID for ``(document_id, chunk_index)``.
+
+ Mirrors the Elasticsearch backend's recipe: a SHA-256 over the
+ ``\0``-joined pair, whose 64 hex chars exactly fit the
+ ``VARCHAR(64)`` primary key. Stable IDs make indexing retries
+ idempotent (via ``upsert``).
+
+ Args:
+ record (`VectorRecord`):
+ The record to derive an ID for.
+
+ Returns:
+ `str`:
+ The 64-character hex digest.
+ """
+ raw = f"{record.document_id}\0{record.chunk.chunk_index}"
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
async def delete(
self,
collection: str,
@@ -403,6 +428,73 @@ class MilvusLiteStore(VectorStoreBase):
offset += self._batch_size
return rows
+ # ------------------------------------------------------------------
+ # Chunk listing
+ # ------------------------------------------------------------------
+
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list[Chunk]:
+ """List one document's chunks ordered by ``chunk_index``.
+
+ Selects the page with a JSON-path range expression
+ ``offset <= chunk["chunk_index"] < offset + limit`` instead of
+ sort-and-skip — Milvus ``query`` has no ORDER BY, but because
+ ``chunk_index`` is dense the range filter yields exactly the
+ requested page, which is then sorted in Python (at most
+ ``limit`` items).
+
+ Args:
+ collection (`str`):
+ The target collection name.
+ document_id (`str`):
+ The source document whose chunks should be listed.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+ metadata_filter (`dict[str, Any] | None`, optional):
+ Extra ``chunk.metadata`` equality constraints.
+
+ Returns:
+ `list[Chunk]`:
+ At most ``limit`` chunks, ``chunk_index`` ascending.
+ """
+ if limit <= 0:
+ return []
+ clauses = [
+ self._build_document_filter(document_id),
+ f'chunk["chunk_index"] >= {offset}',
+ f'chunk["chunk_index"] < {offset + limit}',
+ ]
+ meta_expr = self._build_metadata_filter(metadata_filter)
+ if meta_expr:
+ clauses.append(meta_expr)
+
+ # Query the WHOLE index range (bounded: its width is `limit`)
+ # and deduplicate by chunk_index — collections written before
+ # deterministic entity IDs may hold duplicates from indexing
+ # retries, and truncating at `limit` raw rows could then drop a
+ # low index while returning a duplicate of a higher one.
+ rows = await asyncio.to_thread(
+ self.get_client().query,
+ collection_name=collection,
+ filter=" and ".join(clauses),
+ output_fields=["chunk"],
+ limit=16_384,
+ )
+ by_index: dict[int, Chunk] = {}
+ for row in rows:
+ chunk = Chunk.model_validate(row["chunk"])
+ by_index.setdefault(chunk.chunk_index, chunk)
+ return [by_index[index] for index in sorted(by_index)][:limit]
+
@staticmethod
def _build_document_filter(document_id: str) -> str:
"""Build a Milvus scalar filter for a document id."""
diff --git a/src/agentscope/rag/_vdb/_mongodb.py b/src/agentscope/rag/_vdb/_mongodb.py
index a0e9529a5..8ae414026 100644
--- a/src/agentscope/rag/_vdb/_mongodb.py
+++ b/src/agentscope/rag/_vdb/_mongodb.py
@@ -462,3 +462,53 @@ class MongoDBStore(VectorStoreBase):
),
)
return summaries
+
+ # ------------------------------------------------------------------
+ # Chunk listing
+ # ------------------------------------------------------------------
+
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list[Chunk]:
+ """List one document's chunks ordered by ``chunk_index``.
+
+ Uses a native ``find + sort + skip + limit`` query on the
+ ``chunk.chunk_index`` field — MongoDB is the only backend that
+ supports server-side ordering directly.
+
+ Args:
+ collection (`str`):
+ The target collection name.
+ document_id (`str`):
+ The source document whose chunks should be listed.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+ metadata_filter (`dict[str, Any] | None`, optional):
+ Extra ``chunk.metadata`` equality constraints.
+
+ Returns:
+ `list[Chunk]`:
+ At most ``limit`` chunks, ``chunk_index`` ascending.
+ """
+ if limit <= 0:
+ return []
+ query: dict[str, Any] = {"document_id": document_id}
+ for key, value in (metadata_filter or {}).items():
+ query[f"chunk.metadata.{key}"] = value
+
+ cursor = (
+ self._col(collection)
+ .find(query, projection={"chunk": True})
+ .sort("chunk.chunk_index", 1)
+ .skip(offset)
+ .limit(limit)
+ )
+ return [Chunk.model_validate(row["chunk"]) async for row in cursor]
diff --git a/src/agentscope/rag/_vdb/_qdrant.py b/src/agentscope/rag/_vdb/_qdrant.py
index d03426305..a5f8b28cd 100644
--- a/src/agentscope/rag/_vdb/_qdrant.py
+++ b/src/agentscope/rag/_vdb/_qdrant.py
@@ -191,6 +191,11 @@ class QdrantStore(VectorStoreBase):
:class:`Chunk` under the ``chunk`` key, so that :meth:`delete`
can remove all records of one document.
+ Point IDs are derived deterministically from
+ ``(document_id, chunk_index)`` so that re-indexing the same
+ document after a mid-pipeline crash upserts over the previous
+ records instead of duplicating them.
+
Args:
collection (`str`):
The target collection name.
@@ -207,7 +212,7 @@ class QdrantStore(VectorStoreBase):
collection_name=collection,
points=[
models.PointStruct(
- id=str(uuid.uuid4()),
+ id=self._point_id(record),
vector=record.vector,
payload={
"document_id": record.document_id,
@@ -218,6 +223,26 @@ class QdrantStore(VectorStoreBase):
],
)
+ @staticmethod
+ def _point_id(record: VectorRecord) -> str:
+ """Deterministic point ID for ``(document_id, chunk_index)``.
+
+ ``\0`` separates the fields because it cannot occur in either,
+ so no pair can collide with another. Re-inserting the same
+ chunk therefore upserts in place — the idempotency the indexing
+ pipeline relies on when it retries after a crash.
+
+ Args:
+ record (`VectorRecord`):
+ The record to derive an ID for.
+
+ Returns:
+ `str`:
+ A UUIDv5 string (Qdrant requires UUID-shaped IDs).
+ """
+ raw = f"{record.document_id}\0{record.chunk.chunk_index}"
+ return str(uuid.uuid5(uuid.NAMESPACE_URL, raw))
+
async def delete(
self,
collection: str,
@@ -357,6 +382,88 @@ class QdrantStore(VectorStoreBase):
return list(summaries.values())
+ # ------------------------------------------------------------------
+ # Chunk listing
+ # ------------------------------------------------------------------
+
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list[Chunk]:
+ """List one document's chunks ordered by ``chunk_index``.
+
+ Selects the page with a payload range condition
+ ``offset <= chunk.chunk_index < offset + limit`` instead of
+ sort-and-skip — Qdrant's ``scroll`` has no payload ordering,
+ but because ``chunk_index`` is dense the range filter yields
+ exactly the requested page, which is then sorted in Python
+ (at most ``limit`` items).
+
+ Args:
+ collection (`str`):
+ The target collection name.
+ document_id (`str`):
+ The source document whose chunks should be listed.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+ metadata_filter (`dict[str, Any] | None`, optional):
+ Extra ``chunk.metadata`` equality constraints.
+
+ Returns:
+ `list[Chunk]`:
+ At most ``limit`` chunks, ``chunk_index`` ascending.
+ """
+ from qdrant_client import models
+
+ if limit <= 0:
+ return []
+ conditions = [
+ models.FieldCondition(
+ key="document_id",
+ match=models.MatchValue(value=document_id),
+ ),
+ models.FieldCondition(
+ key="chunk.chunk_index",
+ range=models.Range(gte=offset, lt=offset + limit),
+ ),
+ ]
+ base_filter = self._build_metadata_filter(metadata_filter)
+ if base_filter is not None:
+ conditions.extend(base_filter.must)
+
+ client = self.get_client()
+ # Scroll the WHOLE index range (bounded: its width is `limit`)
+ # and deduplicate by chunk_index — collections written before
+ # deterministic point IDs may hold duplicates from indexing
+ # retries, and an early stop at `limit` raw points could then
+ # drop a low index while returning a duplicate of a higher one.
+ by_index: dict[int, Chunk] = {}
+ scroll_offset: Any = None
+ while True:
+ points, next_offset = await client.scroll(
+ collection_name=collection,
+ scroll_filter=models.Filter(must=conditions),
+ limit=256,
+ offset=scroll_offset,
+ with_payload=True,
+ with_vectors=False,
+ )
+ for point in points:
+ chunk = Chunk.model_validate(point.payload["chunk"])
+ by_index.setdefault(chunk.chunk_index, chunk)
+ if next_offset is None:
+ break
+ scroll_offset = next_offset
+
+ return [by_index[index] for index in sorted(by_index)][:limit]
+
@staticmethod
def _build_metadata_filter(
metadata_filter: dict[str, Any] | None,
diff --git a/src/agentscope/rag/_vdb/_vector_store.py b/src/agentscope/rag/_vdb/_vector_store.py
index e491c701a..8e2684b0e 100644
--- a/src/agentscope/rag/_vdb/_vector_store.py
+++ b/src/agentscope/rag/_vdb/_vector_store.py
@@ -289,3 +289,66 @@ class VectorStoreBase(ABC):
One summary per distinct ``document_id``, in
unspecified order.
"""
+
+ # ------------------------------------------------------------------
+ # Chunk listing
+ # ------------------------------------------------------------------
+
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list[Chunk]:
+ """List the chunks of one source document in ``chunk_index`` order.
+
+ Returns the slice ``[offset, offset + limit)`` of the document's
+ chunks, ordered by :attr:`Chunk.chunk_index` ascending. Because
+ ``chunk_index`` is a dense, immutable ``0..N-1`` sequence within
+ a document, the slice is equivalent to the payload filter
+ ``offset <= chunk_index < offset + limit`` — backends should
+ prefer that filter over sort-and-skip so a page never requires
+ scanning the whole document.
+
+ Deliberately **not** an ``@abstractmethod``: it was added after
+ third-party subclasses of :class:`VectorStoreBase` already
+ existed, and making it abstract would break them on upgrade.
+ Backends that do not override it raise
+ :class:`NotImplementedError`, which the application layer maps
+ to HTTP 501 so clients can hide the chunk-browsing UI.
+
+ Args:
+ collection (`str`):
+ The target collection name.
+ document_id (`str`):
+ The source document whose chunks should be listed —
+ the same value carried by
+ :attr:`VectorRecord.document_id` at insertion time.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip; equals the first
+ returned chunk's ``chunk_index``.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+ metadata_filter (`dict[str, Any] | None`, optional):
+ If provided, restrict listing to records whose
+ ``chunk.metadata`` matches every ``key == value`` pair
+ in this dict. Used for defense-in-depth cross-tenant
+ scoping when an isolation strategy co-locates multiple
+ knowledge bases inside the same collection.
+
+ Returns:
+ `list[Chunk]`:
+ At most ``limit`` chunks ordered by ``chunk_index``
+ ascending; empty when the document has no records or
+ ``offset`` is past the end.
+
+ Raises:
+ `NotImplementedError`:
+ If the backend does not support chunk listing.
+ """
+ raise NotImplementedError(
+ f"{type(self).__name__} does not implement list_chunks().",
+ )
diff --git a/tests/blob_store_s3_test.py b/tests/blob_store_s3_test.py
index 67a865f63..30b3ff1cf 100644
--- a/tests/blob_store_s3_test.py
+++ b/tests/blob_store_s3_test.py
@@ -142,6 +142,17 @@ class S3BlobStoreTest(IsolatedAsyncioTestCase):
# Second delete is a no-op.
await store.delete(uri)
+ async def test_size_measures_the_stored_object(self) -> None:
+ """``size`` returns the object's byte length, ``None`` if gone."""
+ async with self._store() as store:
+ uri = await store.write_stream(
+ "kb/abc/doc-4",
+ io.BytesIO(b"twelve bytes"),
+ )
+ self.assertEqual(await store.size(uri), 12)
+ await store.delete(uri)
+ self.assertIsNone(await store.size(uri))
+
async def test_bad_uri_scheme_rejected(self) -> None:
"""Non-``s3://`` URIs are refused before any network call."""
async with self._store() as store:
diff --git a/tests/rag_vdb_elasticsearch_test.py b/tests/rag_vdb_elasticsearch_test.py
index 1f70f7f2c..dee6f5100 100644
--- a/tests/rag_vdb_elasticsearch_test.py
+++ b/tests/rag_vdb_elasticsearch_test.py
@@ -49,6 +49,8 @@ class _FakeClient:
def __init__(self) -> None:
self.indices = _FakeIndices()
self.bulk = AsyncMock(return_value={"errors": False, "items": []})
+ self.open_point_in_time = AsyncMock()
+ self.close_point_in_time = AsyncMock()
self.delete_by_query = AsyncMock()
self.search = AsyncMock()
self.close = AsyncMock()
@@ -247,3 +249,86 @@ class ElasticsearchStoreTest(IsolatedAsyncioTestCase):
second_query["aggs"]["documents"]["composite"]["after"],
{"document_id": "doc-1"},
)
+
+ async def test_list_chunks_pages_through_pit(self) -> None:
+ chunks = [_record("doc-1", i).chunk for i in (2, 0, 3, 1)]
+ self.client.open_point_in_time.return_value = {"id": "pit-1"}
+ self.client.search.side_effect = [
+ {
+ "hits": {
+ "hits": [
+ {
+ "_source": {
+ "chunk": chunk.model_dump(mode="json"),
+ },
+ "sort": [index],
+ }
+ for index, chunk in enumerate(chunks[:2])
+ ],
+ },
+ "pit_id": "pit-1",
+ },
+ {
+ "hits": {
+ "hits": [
+ {
+ "_source": {
+ "chunk": chunk.model_dump(mode="json"),
+ },
+ "sort": [2 + index],
+ }
+ for index, chunk in enumerate(chunks[2:])
+ ],
+ },
+ "pit_id": "pit-1",
+ },
+ {"hits": {"hits": []}, "pit_id": "pit-1"},
+ ]
+
+ page = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=1,
+ limit=2,
+ metadata_filter={"tenant": "bank-a"},
+ )
+
+ self.client.open_point_in_time.assert_awaited_once_with(
+ index="kb-1",
+ keep_alive="1m",
+ )
+ first_call = self.client.search.await_args_list[0].kwargs
+ self.assertEqual(
+ first_call["query"],
+ {
+ "bool": {
+ "filter": [
+ {"term": {"document_id": "doc-1"}},
+ {"term": {"metadata.tenant": "bank-a"}},
+ ],
+ },
+ },
+ )
+ self.assertEqual(
+ first_call["pit"],
+ {"id": "pit-1", "keep_alive": "1m"},
+ )
+ self.assertEqual(first_call["sort"], [{"_shard_doc": "asc"}])
+ second_call = self.client.search.await_args_list[1].kwargs
+ self.assertEqual(second_call["search_after"], [1])
+ self.client.close_point_in_time.assert_awaited_once_with(id="pit-1")
+ self.assertEqual([c.chunk_index for c in page], [1, 2])
+
+ async def test_list_chunks_closes_pit_on_error(self) -> None:
+ self.client.open_point_in_time.return_value = {"id": "pit-1"}
+ self.client.search.side_effect = RuntimeError("boom")
+ with self.assertRaises(RuntimeError):
+ await self.store.list_chunks("kb-1", "doc-1")
+ self.client.close_point_in_time.assert_awaited_once_with(id="pit-1")
+
+ async def test_list_chunks_zero_limit_short_circuits(self) -> None:
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", limit=0),
+ [],
+ )
+ self.client.search.assert_not_awaited()
diff --git a/tests/rag_vdb_milvus_lite_test.py b/tests/rag_vdb_milvus_lite_test.py
index fd7878864..af1a524c6 100644
--- a/tests/rag_vdb_milvus_lite_test.py
+++ b/tests/rag_vdb_milvus_lite_test.py
@@ -360,3 +360,161 @@ class MilvusLiteStoreTest(IsolatedAsyncioTestCase):
)
self.assertEqual([r.document_id for r in results], ["doc-1"])
+
+ async def test_list_chunks_pagination_and_isolation(self) -> None:
+ """list_chunks pages by chunk_index and isolates documents."""
+ await self.store.create_collection("kb-1", dimensions=3)
+ records = [
+ _make_record(
+ f"doc1-chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=5,
+ )
+ # Insert out of order to prove ordering is restored.
+ for i in (3, 0, 4, 1, 2)
+ ] + [
+ _make_record(
+ f"doc2-chunk{i}",
+ [0.0, 1.0, 0.0],
+ document_id="doc-2",
+ chunk_index=i,
+ total_chunks=2,
+ )
+ for i in (1, 0)
+ ]
+ await self.store.insert("kb-1", records)
+
+ page = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=0,
+ limit=3,
+ )
+ self.assertEqual([c.chunk_index for c in page], [0, 1, 2])
+ self.assertEqual(
+ [c.content.text for c in page],
+ ["doc1-chunk0", "doc1-chunk1", "doc1-chunk2"],
+ )
+
+ tail = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=3,
+ limit=30,
+ )
+ self.assertEqual([c.chunk_index for c in tail], [3, 4])
+
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", offset=5, limit=3),
+ [],
+ )
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", limit=0),
+ [],
+ )
+
+ other = await self.store.list_chunks("kb-1", "doc-2", limit=30)
+ self.assertEqual([c.chunk_index for c in other], [0, 1])
+ self.assertTrue(
+ all(c.content.text.startswith("doc2") for c in other),
+ )
+
+ async def test_list_chunks_metadata_filter(self) -> None:
+ """list_chunks applies the metadata_filter tenant scoping."""
+ await self.store.create_collection("kb-1", dimensions=3)
+ record = _make_record(
+ "scoped",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ )
+ record.chunk.metadata["kb_scope"] = "kb-a"
+ await self.store.insert("kb-1", [record])
+
+ hit = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ metadata_filter={"kb_scope": "kb-a"},
+ )
+ self.assertEqual(len(hit), 1)
+
+ miss = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ metadata_filter={"kb_scope": "kb-b"},
+ )
+ self.assertEqual(miss, [])
+
+ async def test_list_chunks_deduplicates_reindex_duplicates(self) -> None:
+ """Re-indexing the same document never duplicates or drops chunks.
+
+ Stable record IDs make the second insert an in-place upsert; and
+ even against legacy duplicate rows the listing deduplicates by
+ ``chunk_index``, so a page is never ``[1, 2, 3, 3, 4]``.
+ """
+ await self.store.create_collection("kb-1", dimensions=3)
+ records = [
+ _make_record(
+ f"chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=5,
+ )
+ for i in range(5)
+ ]
+ # Simulate the index worker crashing after the vector insert
+ # and retrying the whole pipeline.
+ await self.store.insert("kb-1", records)
+ await self.store.insert("kb-1", records)
+
+ page = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=0,
+ limit=5,
+ )
+ self.assertEqual([c.chunk_index for c in page], [0, 1, 2, 3, 4])
+
+ window = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=1,
+ limit=3,
+ )
+ self.assertEqual([c.chunk_index for c in window], [1, 2, 3])
+
+ async def test_list_chunks_after_reindexing_into_fewer_chunks(
+ self,
+ ) -> None:
+ """A shorter re-index leaves no tail from the previous run.
+
+ The worker deletes a document's vectors before re-inserting, so
+ even though upserts alone would keep the old high indices, the
+ listing reflects only the latest run.
+ """
+ await self.store.create_collection("kb-1", dimensions=3)
+
+ def _records(count: int) -> list:
+ return [
+ _make_record(
+ f"v{count}-chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=count,
+ )
+ for i in range(count)
+ ]
+
+ await self.store.insert("kb-1", _records(5))
+ await self.store.delete("kb-1", "doc-1")
+ await self.store.insert("kb-1", _records(2))
+
+ chunks = await self.store.list_chunks("kb-1", "doc-1", limit=30)
+ self.assertEqual([c.chunk_index for c in chunks], [0, 1])
+ self.assertEqual(
+ [c.content.text for c in chunks],
+ ["v2-chunk0", "v2-chunk1"],
+ )
diff --git a/tests/rag_vdb_mongodb_test.py b/tests/rag_vdb_mongodb_test.py
index 19c0b9211..0ac038d8c 100644
--- a/tests/rag_vdb_mongodb_test.py
+++ b/tests/rag_vdb_mongodb_test.py
@@ -100,6 +100,45 @@ class _FakeAsyncIterator:
return item
+def _read_dotted(doc: dict[str, Any], path: str) -> Any:
+ """Read a dotted path (e.g. ``chunk.chunk_index``) from a dict."""
+ value: Any = doc
+ for part in path.split("."):
+ if not isinstance(value, dict):
+ return None
+ value = value.get(part)
+ return value
+
+
+class _FakeFindCursor:
+ """Chainable find cursor supporting sort / skip / limit."""
+
+ def __init__(self, documents: list[dict[str, Any]]) -> None:
+ self._documents = documents
+
+ def sort(self, key: str, direction: int) -> "_FakeFindCursor":
+ """Sort documents by a dotted key path."""
+ self._documents = sorted(
+ self._documents,
+ key=lambda doc: _read_dotted(doc, key),
+ reverse=direction < 0,
+ )
+ return self
+
+ def skip(self, count: int) -> "_FakeFindCursor":
+ """Skip the first ``count`` documents."""
+ self._documents = self._documents[count:]
+ return self
+
+ def limit(self, count: int) -> "_FakeFindCursor":
+ """Keep at most ``count`` documents."""
+ self._documents = self._documents[:count]
+ return self
+
+ def __aiter__(self) -> _FakeAsyncIterator:
+ return _FakeAsyncIterator(list(self._documents))
+
+
class _FakeMongoCollection:
"""In-memory collection that implements the async API MongoDBStore uses."""
@@ -109,6 +148,23 @@ class _FakeMongoCollection:
self._docs: dict[str, dict[str, Any]] = {}
self._index_queryable = False
+ def find(
+ self,
+ filter_doc: dict[str, Any],
+ projection: dict[str, Any] | None = None,
+ ) -> "_FakeFindCursor":
+ """Return a chainable cursor over matching documents."""
+ del projection # the fake returns whole documents regardless
+ matched = [
+ doc
+ for doc in self._docs.values()
+ if all(
+ _read_dotted(doc, key) == value
+ for key, value in filter_doc.items()
+ )
+ ]
+ return _FakeFindCursor(matched)
+
async def bulk_write(
self,
operations: list[Any],
@@ -657,3 +713,88 @@ class MongoDBStoreTest(IsolatedAsyncioTestCase):
},
],
)
+
+ async def test_list_chunks_pagination_and_isolation(self) -> None:
+ """list_chunks pages by chunk_index and isolates documents."""
+ await self.store.create_collection("kb-1", dimensions=3)
+ records = [
+ _make_record(
+ f"doc1-chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=5,
+ )
+ # Insert out of order to prove ordering is restored.
+ for i in (3, 0, 4, 1, 2)
+ ] + [
+ _make_record(
+ f"doc2-chunk{i}",
+ [0.0, 1.0, 0.0],
+ document_id="doc-2",
+ chunk_index=i,
+ total_chunks=2,
+ )
+ for i in (1, 0)
+ ]
+ await self.store.insert("kb-1", records)
+
+ page = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=0,
+ limit=3,
+ )
+ self.assertEqual([c.chunk_index for c in page], [0, 1, 2])
+ self.assertEqual(
+ [c.content.text for c in page],
+ ["doc1-chunk0", "doc1-chunk1", "doc1-chunk2"],
+ )
+
+ tail = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=3,
+ limit=30,
+ )
+ self.assertEqual([c.chunk_index for c in tail], [3, 4])
+
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", offset=5, limit=3),
+ [],
+ )
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", limit=0),
+ [],
+ )
+
+ other = await self.store.list_chunks("kb-1", "doc-2", limit=30)
+ self.assertEqual([c.chunk_index for c in other], [0, 1])
+ self.assertTrue(
+ all(c.content.text.startswith("doc2") for c in other),
+ )
+
+ async def test_list_chunks_metadata_filter(self) -> None:
+ """list_chunks applies the metadata_filter tenant scoping."""
+ await self.store.create_collection("kb-1", dimensions=3)
+ record = _make_record(
+ "scoped",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ )
+ record.chunk.metadata["kb_scope"] = "kb-a"
+ await self.store.insert("kb-1", [record])
+
+ hit = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ metadata_filter={"kb_scope": "kb-a"},
+ )
+ self.assertEqual(len(hit), 1)
+
+ miss = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ metadata_filter={"kb_scope": "kb-b"},
+ )
+ self.assertEqual(miss, [])
diff --git a/tests/rag_vdb_qdrant_test.py b/tests/rag_vdb_qdrant_test.py
index 6f4848c06..fbbe68893 100644
--- a/tests/rag_vdb_qdrant_test.py
+++ b/tests/rag_vdb_qdrant_test.py
@@ -382,3 +382,161 @@ class QdrantStoreTest(IsolatedAsyncioTestCase):
metadata_filter={"kb_scope": "kb-b"},
)
self.assertEqual([r.document_id for r in results], ["doc-2"])
+
+ async def test_list_chunks_pagination_and_isolation(self) -> None:
+ """list_chunks pages by chunk_index and isolates documents."""
+ await self.store.create_collection("kb-1", dimensions=3)
+ records = [
+ _make_record(
+ f"doc1-chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=5,
+ )
+ # Insert out of order to prove ordering is restored.
+ for i in (3, 0, 4, 1, 2)
+ ] + [
+ _make_record(
+ f"doc2-chunk{i}",
+ [0.0, 1.0, 0.0],
+ document_id="doc-2",
+ chunk_index=i,
+ total_chunks=2,
+ )
+ for i in (1, 0)
+ ]
+ await self.store.insert("kb-1", records)
+
+ page = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=0,
+ limit=3,
+ )
+ self.assertEqual([c.chunk_index for c in page], [0, 1, 2])
+ self.assertEqual(
+ [c.content.text for c in page],
+ ["doc1-chunk0", "doc1-chunk1", "doc1-chunk2"],
+ )
+
+ tail = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=3,
+ limit=30,
+ )
+ self.assertEqual([c.chunk_index for c in tail], [3, 4])
+
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", offset=5, limit=3),
+ [],
+ )
+ self.assertEqual(
+ await self.store.list_chunks("kb-1", "doc-1", limit=0),
+ [],
+ )
+
+ other = await self.store.list_chunks("kb-1", "doc-2", limit=30)
+ self.assertEqual([c.chunk_index for c in other], [0, 1])
+ self.assertTrue(
+ all(c.content.text.startswith("doc2") for c in other),
+ )
+
+ async def test_list_chunks_metadata_filter(self) -> None:
+ """list_chunks applies the metadata_filter tenant scoping."""
+ await self.store.create_collection("kb-1", dimensions=3)
+ record = _make_record(
+ "scoped",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ )
+ record.chunk.metadata["kb_scope"] = "kb-a"
+ await self.store.insert("kb-1", [record])
+
+ hit = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ metadata_filter={"kb_scope": "kb-a"},
+ )
+ self.assertEqual(len(hit), 1)
+
+ miss = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ metadata_filter={"kb_scope": "kb-b"},
+ )
+ self.assertEqual(miss, [])
+
+ async def test_list_chunks_deduplicates_reindex_duplicates(self) -> None:
+ """Re-indexing the same document never duplicates or drops chunks.
+
+ Stable record IDs make the second insert an in-place upsert; and
+ even against legacy duplicate rows the listing deduplicates by
+ ``chunk_index``, so a page is never ``[1, 2, 3, 3, 4]``.
+ """
+ await self.store.create_collection("kb-1", dimensions=3)
+ records = [
+ _make_record(
+ f"chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=5,
+ )
+ for i in range(5)
+ ]
+ # Simulate the index worker crashing after the vector insert
+ # and retrying the whole pipeline.
+ await self.store.insert("kb-1", records)
+ await self.store.insert("kb-1", records)
+
+ page = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=0,
+ limit=5,
+ )
+ self.assertEqual([c.chunk_index for c in page], [0, 1, 2, 3, 4])
+
+ window = await self.store.list_chunks(
+ "kb-1",
+ "doc-1",
+ offset=1,
+ limit=3,
+ )
+ self.assertEqual([c.chunk_index for c in window], [1, 2, 3])
+
+ async def test_list_chunks_after_reindexing_into_fewer_chunks(
+ self,
+ ) -> None:
+ """A shorter re-index leaves no tail from the previous run.
+
+ The worker deletes a document's vectors before re-inserting, so
+ even though upserts alone would keep the old high indices, the
+ listing reflects only the latest run.
+ """
+ await self.store.create_collection("kb-1", dimensions=3)
+
+ def _records(count: int) -> list:
+ return [
+ _make_record(
+ f"v{count}-chunk{i}",
+ [1.0, 0.0, 0.0],
+ document_id="doc-1",
+ chunk_index=i,
+ total_chunks=count,
+ )
+ for i in range(count)
+ ]
+
+ await self.store.insert("kb-1", _records(5))
+ await self.store.delete("kb-1", "doc-1")
+ await self.store.insert("kb-1", _records(2))
+
+ chunks = await self.store.list_chunks("kb-1", "doc-1", limit=30)
+ self.assertEqual([c.chunk_index for c in chunks], [0, 1])
+ self.assertEqual(
+ [c.content.text for c in chunks],
+ ["v2-chunk0", "v2-chunk1"],
+ )
diff --git a/tests/resource_access_policy_test.py b/tests/resource_access_policy_test.py
index 8d754b3c0..a6b833bff 100644
--- a/tests/resource_access_policy_test.py
+++ b/tests/resource_access_policy_test.py
@@ -422,6 +422,17 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):
"created_at": AnyValue(),
"updated_at": AnyValue(),
"editable": True,
+ "document_count": 0,
+ "chunk_count": 0,
+ "credential_name": None,
+ "status_counts": {
+ "pending": 0,
+ "parsing": 0,
+ "chunking": 0,
+ "indexing": 0,
+ "ready": 0,
+ "error": 0,
+ },
},
)
@@ -450,6 +461,17 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):
"created_at": AnyValue(),
"updated_at": AnyValue(),
"editable": True,
+ "document_count": 0,
+ "chunk_count": 0,
+ "credential_name": None,
+ "status_counts": {
+ "pending": 0,
+ "parsing": 0,
+ "chunking": 0,
+ "indexing": 0,
+ "ready": 0,
+ "error": 0,
+ },
},
],
)
diff --git a/tests/service_knowledge_base_detail_test.py b/tests/service_knowledge_base_detail_test.py
new file mode 100644
index 000000000..78815239f
--- /dev/null
+++ b/tests/service_knowledge_base_detail_test.py
@@ -0,0 +1,774 @@
+# -*- coding: utf-8 -*-
+# pylint: disable=protected-access
+"""Router tests for document chunk browsing and raw-file preview.
+
+Boots the full FastAPI app via :func:`create_app` against fakeredis +
+the in-memory KB fakes from the upload-flow test, seeds a knowledge
+base, a ready document and its chunks directly through storage / the
+fake vector store, then exercises the three endpoints added for
+issue #2360:
+
+* ``GET /knowledge_bases/{kb}/documents/{doc}/chunks`` — ordered,
+ stable ``page`` / ``page_size`` pagination, 404s, and the 501
+ contract for vector stores without ``list_chunks``;
+* ``POST /knowledge_bases/{kb}/documents/{doc}/download_token`` — the
+ browser-native capability mint;
+* ``GET /knowledge_bases/{kb}/documents/{doc}`` — streamed raw bytes
+ with inline-vs-attachment disposition and token / header auth.
+"""
+import io
+import tempfile
+from typing import Any
+from unittest.async_case import IsolatedAsyncioTestCase
+
+import fakeredis.aioredis
+from fastapi.testclient import TestClient
+
+from service_knowledge_base_upload_test import (
+ _FakeKbManager,
+ _FakeVectorStore,
+ _NoopWorkspaceManager,
+ _make_bus,
+ _make_storage,
+)
+
+from agentscope.app import create_app
+from agentscope.app.access import (
+ ResourceAccessPolicyBase,
+ ResourceKind,
+ ResourcePermission,
+ ResourceRef,
+)
+from agentscope.app.rag.blob_store import LocalBlobStore
+from agentscope.app.storage import (
+ EmbeddingModelConfig,
+ KnowledgeBaseData,
+ KnowledgeBaseRecord,
+ KnowledgeDocumentData,
+ KnowledgeDocumentRecord,
+)
+from agentscope.credential import OpenAICredential
+from agentscope.message import TextBlock
+from agentscope.rag import Chunk
+from agentscope.rag._vdb._vector_store import VectorRecord
+
+
+class _NoChunkListingVectorStore(_FakeVectorStore):
+ """Fake store that pretends chunk listing is unsupported."""
+
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list:
+ """Refuse — models a backend predating ``list_chunks``."""
+ raise NotImplementedError("no chunk listing here")
+
+
+class _ShareToViewerPolicy(ResourceAccessPolicyBase):
+ """Grant ``user-shared`` read access to one knowledge base.
+
+ ``knowledge_base_id`` is filled in by the test setup once the
+ seeded record's id is known; every other viewer gets nothing, so
+ the foreign-viewer 404 tests keep their meaning.
+ """
+
+ def __init__(self) -> None:
+ self.knowledge_base_id: str | None = None
+
+ async def list_accessible(
+ self,
+ viewer_id: str,
+ kind: ResourceKind,
+ storage: object,
+ ) -> list[ResourceRef]:
+ """Return the single read grant for ``user-shared``."""
+ del storage # static grant — nothing to look up
+ if (
+ viewer_id == "user-shared"
+ and kind == ResourceKind.KNOWLEDGE_BASE
+ and self.knowledge_base_id is not None
+ ):
+ return [
+ ResourceRef(
+ kind=ResourceKind.KNOWLEDGE_BASE,
+ owner_id="user-1",
+ resource_id=self.knowledge_base_id,
+ permission=ResourcePermission.READ,
+ ),
+ ]
+ return []
+
+
+class _KnowledgeBaseDetailTestBase(IsolatedAsyncioTestCase):
+ """Shared app bootstrap + seed data for the detail endpoints."""
+
+ vector_store_cls: type[_FakeVectorStore] = _FakeVectorStore
+
+ async def asyncSetUp(self) -> None:
+ """Boot the app and seed a KB, a ready document and chunks."""
+ # pylint: disable=consider-using-with
+ self._tmp = tempfile.TemporaryDirectory()
+ self._fr = fakeredis.aioredis.FakeRedis(decode_responses=True)
+ self._vector_store = self.vector_store_cls()
+ storage = _make_storage(self._fr)
+ message_bus = _make_bus(self._fr)
+ self._blob_store = LocalBlobStore(root_dir=self._tmp.name)
+
+ self._share_policy = _ShareToViewerPolicy()
+ self._app = create_app(
+ storage=storage,
+ message_bus=message_bus,
+ workspace_manager=_NoopWorkspaceManager(),
+ knowledge_base_manager=_FakeKbManager(
+ storage=storage,
+ vector_store=self._vector_store,
+ ),
+ blob_store=self._blob_store,
+ resource_access_policy=self._share_policy,
+ )
+
+ # Seed a knowledge base, a ready document (blob included) and
+ # its chunks directly, bypassing the async indexing pipeline.
+ kb_record = KnowledgeBaseRecord(
+ user_id="user-1",
+ data=KnowledgeBaseData(
+ name="kb",
+ description="",
+ embedding_model_config=EmbeddingModelConfig(
+ type="openai_credential",
+ credential_id="cred-1",
+ model="text-embedding-3-small",
+ dimensions=1,
+ ),
+ collection_name="",
+ ),
+ )
+ kb_record.data.collection_name = f"kb_{kb_record.id}"
+ collection = kb_record.data.collection_name
+ await self._vector_store.create_collection(collection, 1)
+ self._kb_id = kb_record.id
+ self._share_policy.knowledge_base_id = kb_record.id
+
+ self._file_bytes = b"# Hello\n\nchunked markdown body\n"
+ async with self._blob_store as blob_store:
+ blob_uri = await blob_store.write_stream(
+ key=f"kb/{self._kb_id}/doc-1",
+ stream=io.BytesIO(self._file_bytes),
+ )
+ document = KnowledgeDocumentRecord(
+ id="doc-1",
+ user_id="user-1",
+ knowledge_base_id=self._kb_id,
+ status="ready",
+ data=KnowledgeDocumentData(
+ filename="hello.md",
+ size=len(self._file_bytes),
+ content_type="text/markdown",
+ blob_uri=blob_uri,
+ chunk_count=5,
+ ),
+ )
+ await self._vector_store.insert(
+ collection,
+ [
+ VectorRecord(
+ vector=[0.0],
+ document_id="doc-1",
+ chunk=Chunk(
+ content=TextBlock(text=f"chunk-{index}"),
+ source="hello.md",
+ chunk_index=index,
+ total_chunks=5,
+ ),
+ )
+ # Insert out of order to prove ordering is restored.
+ for index in (3, 0, 4, 1, 2)
+ ],
+ )
+
+ storage._client = self._fr
+ await storage.upsert_knowledge_base("user-1", kb_record)
+ await storage.upsert_knowledge_document("user-1", document)
+ await storage.upsert_credential(
+ "user-1",
+ OpenAICredential(
+ id="cred-1",
+ name="My OpenAI Key",
+ api_key="sk-secret",
+ ),
+ )
+ storage._client = None
+
+ async def asyncTearDown(self) -> None:
+ """Release fakeredis and the temporary blob directory."""
+ await self._fr.aclose()
+ self._tmp.cleanup()
+
+
+class DocumentChunkBrowsingTest(_KnowledgeBaseDetailTestBase):
+ """``GET .../documents/{doc}/chunks`` behaviour."""
+
+ def test_pages_are_ordered_and_stable(self) -> None:
+ """Pages come back chunk_index-ascending with a stable window."""
+ headers = {"X-User-ID": "user-1"}
+ with TestClient(self._app) as client:
+ first = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1/chunks",
+ params={"page": 1, "page_size": 2},
+ headers=headers,
+ )
+ self.assertEqual(first.status_code, 200)
+ body = first.json()
+ self.assertEqual(body["total"], 5)
+ self.assertEqual(body["page"], 1)
+ self.assertEqual(body["page_size"], 2)
+ self.assertEqual(
+ [c["chunk_index"] for c in body["chunks"]],
+ [0, 1],
+ )
+ self.assertEqual(body["chunks"][0]["content"]["text"], "chunk-0")
+
+ last = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1/chunks",
+ params={"page": 3, "page_size": 2},
+ headers=headers,
+ )
+ self.assertEqual(
+ [c["chunk_index"] for c in last.json()["chunks"]],
+ [4],
+ )
+
+ past_end = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1/chunks",
+ params={"page": 4, "page_size": 2},
+ headers=headers,
+ )
+ self.assertEqual(past_end.json()["chunks"], [])
+ self.assertEqual(past_end.json()["total"], 5)
+
+ def test_missing_document_and_foreign_viewer_are_404(self) -> None:
+ """Unknown documents and invisible KBs both surface 404."""
+ with TestClient(self._app) as client:
+ missing = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/nope/chunks",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(missing.status_code, 404)
+
+ foreign = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1/chunks",
+ headers={"X-User-ID": "user-2"},
+ )
+ self.assertEqual(foreign.status_code, 404)
+
+
+class DocumentChunkBrowsingUnsupportedTest(_KnowledgeBaseDetailTestBase):
+ """Vector stores without ``list_chunks`` surface HTTP 501."""
+
+ vector_store_cls = _NoChunkListingVectorStore
+
+ def test_not_implemented_maps_to_501(self) -> None:
+ """A store without list_chunks maps to HTTP 501."""
+ with TestClient(self._app) as client:
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1/chunks",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 501)
+
+
+class DocumentContentTest(_KnowledgeBaseDetailTestBase):
+ """``GET .../documents/{doc}`` raw-file streaming behaviour."""
+
+ def test_header_auth_streams_inline(self) -> None:
+ """Header-authenticated fetch streams the bytes inline."""
+ with TestClient(self._app) as client:
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(response.content, self._file_bytes)
+ self.assertTrue(
+ response.headers["content-type"].startswith(
+ "text/markdown",
+ ),
+ )
+ self.assertTrue(
+ response.headers["content-disposition"].startswith(
+ "inline;",
+ ),
+ )
+ self.assertIn(
+ "hello.md",
+ response.headers["content-disposition"],
+ )
+ self.assertEqual(
+ response.headers["content-length"],
+ str(len(self._file_bytes)),
+ )
+
+ def test_content_length_is_measured_not_declared(self) -> None:
+ """The header comes from the blob, so a wrong record cannot lie.
+
+ A determinate download progress bar needs Content-Length, and a
+ value that disagrees with the body truncates the response — so
+ it is measured on the stored bytes, even when the record's
+ declared size is stale or zero.
+ """
+
+ async def _corrupt_declared_size() -> None:
+ storage = self._app.state.storage
+ record = await storage.get_knowledge_document(
+ "user-1",
+ self._kb_id,
+ "doc-1",
+ )
+ record.data.size = 0
+ await storage.upsert_knowledge_document("user-1", record)
+
+ with TestClient(self._app) as client:
+ client.portal.call(_corrupt_declared_size)
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertEqual(
+ response.headers["content-length"],
+ str(len(self._file_bytes)),
+ )
+ self.assertEqual(response.content, self._file_bytes)
+
+ def test_download_flag_forces_attachment(self) -> None:
+ """``download=true`` switches the disposition to attachment."""
+ with TestClient(self._app) as client:
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ params={"download": "true"},
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertTrue(
+ response.headers["content-disposition"].startswith(
+ "attachment;",
+ ),
+ )
+
+ def test_scriptable_media_type_is_never_inline(self) -> None:
+ """Scriptable media types are forced to attachment."""
+
+ async def _seed_html_document() -> None:
+ async with self._blob_store as blob_store:
+ blob_uri = await blob_store.write_stream(
+ key=f"kb/{self._kb_id}/doc-html",
+ stream=io.BytesIO(b"<script>alert(1)</script>"),
+ )
+ # Lifespan has already bound the storage client.
+ await self._app.state.storage.upsert_knowledge_document(
+ "user-1",
+ KnowledgeDocumentRecord(
+ id="doc-html",
+ user_id="user-1",
+ knowledge_base_id=self._kb_id,
+ status="ready",
+ data=KnowledgeDocumentData(
+ filename="page.html",
+ size=25,
+ content_type="text/html",
+ blob_uri=blob_uri,
+ chunk_count=0,
+ ),
+ ),
+ )
+
+ with TestClient(self._app) as client:
+ client.portal.call(_seed_html_document)
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-html",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertTrue(
+ response.headers["content-disposition"].startswith(
+ "attachment;",
+ ),
+ )
+
+ def test_raster_image_previews_inline(self) -> None:
+ """Safe raster formats (PNG here) are served inline for <img>."""
+
+ async def _seed_png_document() -> None:
+ async with self._blob_store as blob_store:
+ blob_uri = await blob_store.write_stream(
+ key=f"kb/{self._kb_id}/doc-png",
+ stream=io.BytesIO(b"\x89PNG fake bytes"),
+ )
+ await self._app.state.storage.upsert_knowledge_document(
+ "user-1",
+ KnowledgeDocumentRecord(
+ id="doc-png",
+ user_id="user-1",
+ knowledge_base_id=self._kb_id,
+ status="ready",
+ data=KnowledgeDocumentData(
+ filename="picture.png",
+ size=15,
+ content_type="image/png",
+ blob_uri=blob_uri,
+ chunk_count=0,
+ ),
+ ),
+ )
+
+ with TestClient(self._app) as client:
+ client.portal.call(_seed_png_document)
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-png",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertTrue(
+ response.headers["content-disposition"].startswith("inline;"),
+ )
+ self.assertEqual(
+ response.headers["x-content-type-options"],
+ "nosniff",
+ )
+
+ def test_svg_is_never_inline(self) -> None:
+ """SVG can carry script, so it must always be an attachment."""
+
+ async def _seed_svg_document() -> None:
+ async with self._blob_store as blob_store:
+ blob_uri = await blob_store.write_stream(
+ key=f"kb/{self._kb_id}/doc-svg",
+ stream=io.BytesIO(b"<svg><script>alert(1)</script></svg>"),
+ )
+ await self._app.state.storage.upsert_knowledge_document(
+ "user-1",
+ KnowledgeDocumentRecord(
+ id="doc-svg",
+ user_id="user-1",
+ knowledge_base_id=self._kb_id,
+ status="ready",
+ data=KnowledgeDocumentData(
+ filename="image.svg",
+ size=37,
+ content_type="image/svg+xml",
+ blob_uri=blob_uri,
+ chunk_count=0,
+ ),
+ ),
+ )
+
+ with TestClient(self._app) as client:
+ client.portal.call(_seed_svg_document)
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-svg",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 200)
+ self.assertTrue(
+ response.headers["content-disposition"].startswith(
+ "attachment;",
+ ),
+ )
+ self.assertEqual(
+ response.headers["x-content-type-options"],
+ "nosniff",
+ )
+
+ def test_missing_auth_is_401_and_foreign_viewer_404(self) -> None:
+ """No credentials is 401; an invisible KB is 404."""
+ with TestClient(self._app) as client:
+ anonymous = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ )
+ self.assertEqual(anonymous.status_code, 401)
+
+ foreign = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ headers={"X-User-ID": "user-2"},
+ )
+ self.assertEqual(foreign.status_code, 404)
+
+ def test_missing_blob_is_404(self) -> None:
+ """A record whose blob is gone yields 404, not a 500."""
+
+ async def _seed_blobless_document() -> None:
+ # Lifespan has already bound the storage client.
+ await self._app.state.storage.upsert_knowledge_document(
+ "user-1",
+ KnowledgeDocumentRecord(
+ id="doc-gone",
+ user_id="user-1",
+ knowledge_base_id=self._kb_id,
+ status="ready",
+ data=KnowledgeDocumentData(
+ filename="gone.md",
+ size=1,
+ content_type="text/markdown",
+ blob_uri="local://kb/nowhere/doc-gone",
+ chunk_count=0,
+ ),
+ ),
+ )
+
+ with TestClient(self._app) as client:
+ client.portal.call(_seed_blobless_document)
+ response = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-gone",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 404)
+
+
+class DocumentDownloadTokenTest(_KnowledgeBaseDetailTestBase):
+ """``POST .../download_token`` mint + token-authenticated fetch."""
+
+ def test_token_round_trip(self) -> None:
+ """A minted token fetches the file with no header at all."""
+ with TestClient(self._app) as client:
+ minted = client.post(
+ f"/knowledge_bases/{self._kb_id}"
+ "/documents/doc-1/download_token",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(minted.status_code, 200)
+ token = minted.json()["token"]
+
+ # The token alone fetches the file — no header needed.
+ fetched = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ params={"token": token},
+ )
+ self.assertEqual(fetched.status_code, 200)
+ self.assertEqual(fetched.content, self._file_bytes)
+
+ def test_token_is_bound_to_one_document(self) -> None:
+ """Tokens replayed against another document or garbage are 401."""
+ with TestClient(self._app) as client:
+ minted = client.post(
+ f"/knowledge_bases/{self._kb_id}"
+ "/documents/doc-1/download_token",
+ headers={"X-User-ID": "user-1"},
+ )
+ token = minted.json()["token"]
+
+ replayed = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-other",
+ params={"token": token},
+ )
+ self.assertEqual(replayed.status_code, 401)
+
+ garbage = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ params={"token": "not-a-token"},
+ )
+ self.assertEqual(garbage.status_code, 401)
+
+ def test_mint_requires_visibility(self) -> None:
+ """Minting is gated by the same 404 visibility rule."""
+ with TestClient(self._app) as client:
+ foreign = client.post(
+ f"/knowledge_bases/{self._kb_id}"
+ "/documents/doc-1/download_token",
+ headers={"X-User-ID": "user-2"},
+ )
+ self.assertEqual(foreign.status_code, 404)
+
+
+class KnowledgeBaseListEnrichmentTest(_KnowledgeBaseDetailTestBase):
+ """``GET /knowledge_bases/`` filters, pagination and enrichment."""
+
+ def test_list_serves_counts_and_credential_name(self) -> None:
+ """The list carries counts and the resolved credential name."""
+ with TestClient(self._app) as client:
+ response = client.get(
+ "/knowledge_bases/",
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(response.status_code, 200)
+ body = response.json()
+ self.assertEqual(body["total"], 1)
+ self.assertEqual(body["page"], 1)
+ self.assertEqual(body["page_size"], 30)
+ view = body["knowledge_bases"][0]
+ self.assertEqual(view["document_count"], 1)
+ self.assertEqual(view["chunk_count"], 5)
+ self.assertEqual(view["credential_name"], "My OpenAI Key")
+ self.assertEqual(view["status_counts"]["ready"], 1)
+ # The masked credential secret must never ride along.
+ self.assertNotIn("api_key", str(body))
+
+ def test_id_filter_doubles_as_get_single(self) -> None:
+ """``?id=`` narrows the list to one knowledge base."""
+ with TestClient(self._app) as client:
+ hit = client.get(
+ "/knowledge_bases/",
+ params={"id": self._kb_id},
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(hit.json()["total"], 1)
+
+ miss = client.get(
+ "/knowledge_bases/",
+ params={"id": "kb-nope"},
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(miss.json()["total"], 0)
+ self.assertEqual(miss.json()["knowledge_bases"], [])
+
+ def test_name_filter_and_pagination_window(self) -> None:
+ """Name filtering and page windows behave as documented."""
+ with TestClient(self._app) as client:
+ named = client.get(
+ "/knowledge_bases/",
+ params={"name": "KB"}, # case-insensitive substring
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(named.json()["total"], 1)
+
+ beyond = client.get(
+ "/knowledge_bases/",
+ params={"page": 2, "page_size": 30},
+ headers={"X-User-ID": "user-1"},
+ )
+ self.assertEqual(beyond.json()["knowledge_bases"], [])
+ self.assertEqual(beyond.json()["total"], 1)
+
+
+class KnowledgeDocumentListFilterTest(_KnowledgeBaseDetailTestBase):
+ """``GET .../documents`` filters and pagination."""
+
+ def test_filters_and_pagination(self) -> None:
+ """Document filters, status validation and paging all work."""
+ headers = {"X-User-ID": "user-1"}
+ base = f"/knowledge_bases/{self._kb_id}/documents"
+ with TestClient(self._app) as client:
+ plain = client.get(base, headers=headers)
+ body = plain.json()
+ self.assertEqual(body["total"], 1)
+ self.assertEqual(body["page"], 1)
+ self.assertEqual(body["page_size"], 30)
+ self.assertEqual(body["documents"][0]["id"], "doc-1")
+
+ by_id = client.get(base, params={"id": "doc-1"}, headers=headers)
+ self.assertEqual(by_id.json()["total"], 1)
+
+ by_kw = client.get(
+ base,
+ params={"keywords": "HELLO"},
+ headers=headers,
+ )
+ self.assertEqual(by_kw.json()["total"], 1)
+
+ kw_miss = client.get(
+ base,
+ params={"keywords": "nothing"},
+ headers=headers,
+ )
+ self.assertEqual(kw_miss.json()["total"], 0)
+
+ by_status = client.get(
+ base,
+ params={"status": "ready"},
+ headers=headers,
+ )
+ self.assertEqual(by_status.json()["total"], 1)
+
+ status_miss = client.get(
+ base,
+ params={"status": "error"},
+ headers=headers,
+ )
+ self.assertEqual(status_miss.json()["total"], 0)
+
+ bad_status = client.get(
+ base,
+ params={"status": "bogus"},
+ headers=headers,
+ )
+ self.assertEqual(bad_status.status_code, 422)
+
+ beyond = client.get(
+ base,
+ params={"page": 2},
+ headers=headers,
+ )
+ self.assertEqual(beyond.json()["documents"], [])
+ self.assertEqual(beyond.json()["total"], 1)
+
+
+class SharedViewerAccessTest(_KnowledgeBaseDetailTestBase):
+ """A read-only shared viewer can use every new read endpoint."""
+
+ def test_shared_viewer_sees_enriched_list(self) -> None:
+ """The shared KB lists with counts + the owner's credential name."""
+ with TestClient(self._app) as client:
+ response = client.get(
+ "/knowledge_bases/",
+ headers={"X-User-ID": "user-shared"},
+ )
+ self.assertEqual(response.status_code, 200)
+ body = response.json()
+ self.assertEqual(body["total"], 1)
+ view = body["knowledge_bases"][0]
+ self.assertEqual(view["id"], self._kb_id)
+ self.assertFalse(view["editable"])
+ self.assertEqual(view["document_count"], 1)
+ self.assertEqual(view["chunk_count"], 5)
+ self.assertEqual(view["credential_name"], "My OpenAI Key")
+ self.assertNotIn("api_key", str(body))
+
+ def test_shared_viewer_lists_documents_and_chunks(self) -> None:
+ """Documents and chunk pages resolve through the owner's data."""
+ headers = {"X-User-ID": "user-shared"}
+ with TestClient(self._app) as client:
+ documents = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents",
+ headers=headers,
+ )
+ self.assertEqual(documents.status_code, 200)
+ self.assertEqual(documents.json()["total"], 1)
+
+ chunks = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1/chunks",
+ params={"page": 1, "page_size": 2},
+ headers=headers,
+ )
+ self.assertEqual(chunks.status_code, 200)
+ self.assertEqual(chunks.json()["total"], 5)
+ self.assertEqual(
+ [c["chunk_index"] for c in chunks.json()["chunks"]],
+ [0, 1],
+ )
+
+ def test_shared_viewer_previews_and_mints_tokens(self) -> None:
+ """Raw-file fetch works via header auth and via a minted token."""
+ headers = {"X-User-ID": "user-shared"}
+ with TestClient(self._app) as client:
+ direct = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ headers=headers,
+ )
+ self.assertEqual(direct.status_code, 200)
+ self.assertEqual(direct.content, self._file_bytes)
+
+ minted = client.post(
+ f"/knowledge_bases/{self._kb_id}"
+ "/documents/doc-1/download_token",
+ headers=headers,
+ )
+ self.assertEqual(minted.status_code, 200)
+ fetched = client.get(
+ f"/knowledge_bases/{self._kb_id}/documents/doc-1",
+ params={"token": minted.json()["token"]},
+ )
+ self.assertEqual(fetched.status_code, 200)
+ self.assertEqual(fetched.content, self._file_bytes)
diff --git a/tests/service_knowledge_base_upload_test.py b/tests/service_knowledge_base_upload_test.py
index 3e4c58d10..25457a47e 100644
--- a/tests/service_knowledge_base_upload_test.py
+++ b/tests/service_knowledge_base_upload_test.py
@@ -106,6 +106,26 @@ class _FakeVectorStore(VectorStoreBase):
) -> list[DocumentSummary]:
return []
+ async def list_chunks(
+ self,
+ collection: str,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ metadata_filter: dict[str, Any] | None = None,
+ ) -> list:
+ del metadata_filter # single-tenant fake — nothing to scope
+ matched = sorted(
+ (
+ record.chunk
+ for record in self._collections.get(collection, [])
+ if record.document_id == document_id
+ ),
+ key=lambda chunk: chunk.chunk_index,
+ )
+ return matched[offset : offset + limit]
+
class _FakeKnowledge:
"""Minimal stand-in for :class:`KnowledgeBase` used by the worker.
@@ -171,6 +191,34 @@ class _FakeKnowledge:
"""
await self._vector_store.delete(self._collection_name, document_id)
+ async def list_chunks(
+ self,
+ document_id: str,
+ *,
+ offset: int = 0,
+ limit: int = 30,
+ ) -> list:
+ """Delegate chunk listing to the bound fake vector store.
+
+ Args:
+ document_id (`str`):
+ The document whose chunks should be listed.
+ offset (`int`, defaults to ``0``):
+ Number of leading chunks to skip.
+ limit (`int`, defaults to ``30``):
+ Maximum number of chunks to return.
+
+ Returns:
+ `list`:
+ The requested page of chunks.
+ """
+ return await self._vector_store.list_chunks(
+ self._collection_name,
+ document_id,
+ offset=offset,
+ limit=limit,
+ )
+
async def search(self, queries: list, top_k: int = 5) -> list:
"""Return an empty result list — search is out of scope here.