Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(rag): make chunker config explicit and schema-driven (#2083)
qbc7cee10c3bf5f
diff --git a/examples/agent_service/main.py b/examples/agent_service/main.pyindex a41baba1d..9a6a0bbf3 100644--- a/examples/agent_service/main.py+++ b/examples/agent_service/main.py@@ -16,7 +16,7 @@ from agentscope.app.workspace_manager import LocalWorkspaceManagerfrom agentscope.mcp import MCPClient, StdioMCPConfig, HttpMCPConfigfrom agentscope.middleware import AgenticMemoryMiddleware, MiddlewareBasefrom agentscope.permission import PermissionContext, PermissionMode-from agentscope.rag import QdrantStore+from agentscope.rag import ApproxTokenChunker, QdrantStorefrom agentscope.workspace import WorkspaceBasedefault_mcps = [@@ -94,6 +94,9 @@ app = create_app(storage=storage,vector_store=vector_store,),+ # Chunker classes users can pick from when creating a knowledge base;+ # the chosen type and parameters are pinned on the knowledge base.+ knowledge_chunkers=[ApproxTokenChunker],# Resource hubs the UI browses under /hub. Neither needs credentials# of its own — an individual MCP card declares whatever key it wants# from the user in its ``inputs_schema``. Passing a ClawHub tokendiff --git a/examples/web_ui/frontend/src/api/knowledgeBase.ts b/examples/web_ui/frontend/src/api/knowledgeBase.tsindex b6f1f1f63..202e5b1ce 100644--- a/examples/web_ui/frontend/src/api/knowledgeBase.ts+++ b/examples/web_ui/frontend/src/api/knowledgeBase.ts@@ -4,6 +4,7 @@ import type {CreateKnowledgeBaseResponse,KbMiddlewareParametersSchemaResponse,KnowledgeBaseView,+ ListChunkersResponse,ListKbEmbeddingModelsResponse,ListKnowledgeBasesResponse,ListKnowledgeDocumentsResponse,@@ -122,6 +123,8 @@ export const knowledgeBaseApi = {listEmbeddingModels: () =>client.get<ListKbEmbeddingModelsResponse>('/knowledge_bases/embedding_models'),+ listChunkers: () => client.get<ListChunkersResponse>('/knowledge_bases/chunkers'),+/** Fetch the JSON Schema describing the KB middleware's tunable params. */middlewareParametersSchema: () =>client.get<KbMiddlewareParametersSchemaResponse>(diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.tsindex ef4997d67..06f423da8 100644--- a/examples/web_ui/frontend/src/api/types.ts+++ b/examples/web_ui/frontend/src/api/types.ts@@ -838,6 +838,22 @@ export interface ListEmbeddingModelResponse {total: number;}+// ─── Chunker ──────────────────────────────────────────────────────────────────++export interface ChunkerConfig {+ type: string;+ parameters: Record<string, unknown>;+}++export interface ChunkerInfo {+ type: string;+ parameter_schema: JSONSchema;+}++export interface ListChunkersResponse {+ chunkers: ChunkerInfo[];+}+// ─── Knowledge Base ───────────────────────────────────────────────────────────/**@@ -849,6 +865,7 @@ export interface KnowledgeBaseView {name: string;description: string;embedding_model_config: EmbeddingModelConfig;+ chunker_config?: ChunkerConfig;created_at: string;updated_at: string;/**@@ -868,6 +885,7 @@ export interface CreateKnowledgeBaseRequest {name: string;description?: string;embedding_model_config: EmbeddingModelConfig;+ chunker_config: ChunkerConfig;}export interface CreateKnowledgeBaseResponse {diff --git a/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx b/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsxindex bf6a4fc1e..ce89ae167 100644--- a/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx+++ b/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx@@ -1,7 +1,19 @@import { CircleAlert, Info, Loader2, PlusCircle } from 'lucide-react';-import { useEffect, useMemo, useState } from 'react';+import { useCallback, useEffect, useMemo, useState } from 'react';-import type { EmbeddingModelCard, EmbeddingModelConfig } from '@/api';+import type {+ ChunkerConfig,+ ChunkerInfo,+ EmbeddingModelCard,+ EmbeddingModelConfig,+ JSONSchema,+} from '@/api';+import {+ type SchemaFormValue,+ SchemaForm,+ defaultValuesFromSchema,+} from '@/components/form/SchemaForm';+import { ChunkerSelect } from '@/components/select/ChunkerSelect';import { DimensionSelect } from '@/components/select/DimensionSelect';import { EmbeddingSelect } from '@/components/select/EmbeddingSelect';import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert.tsx';@@ -16,7 +28,9 @@ import {} from '@/components/ui/dialog.tsx';import { Field, FieldGroup, FieldLabel } from '@/components/ui/field.tsx';import { Input } from '@/components/ui/input.tsx';+import { Separator } from '@/components/ui/separator.tsx';import { Textarea } from '@/components/ui/textarea.tsx';+import { useChunkers } from '@/hooks/useChunkers';import { useKbEmbeddingModels } from '@/hooks/useKbEmbeddingModels';import { useKnowledgeBases } from '@/hooks/useKnowledgeBases';import { useTranslation } from '@/i18n/useI18n.ts';@@ -33,6 +47,10 @@ interface Props {credentialRefetchTrigger?: number;}+const NO_SKIP = new Set<string>();+/** Same control width for every label-left field, as in the schedule dialog. */+const HORIZONTAL_FIELD_WIDTH = '[&>[data-orientation=horizontal]>:last-child]:w-48';+interface SelectedEmbedding {type: string;credentialId: string;@@ -50,20 +68,58 @@ export function CreateKnowledgeBaseDialog({const { t } = useTranslation();const { create } = useKnowledgeBases();const { providers, policy, loading } = useKbEmbeddingModels(credentialRefetchTrigger);+ const { chunkers } = useChunkers();const [name, setName] = useState('');const [description, setDescription] = useState('');const [selected, setSelected] = useState<SelectedEmbedding | null>(null);const [dimension, setDimension] = useState<number | null>(null);+ const [selectedChunkerType, setSelectedChunkerType] = useState<string>('');+ const [chunkerParams, setChunkerParams] = useState<Record<string, SchemaFormValue>>({});const [submitting, setSubmitting] = useState(false);const [errorKey, setErrorKey] = useState<string | null>(null);+ const selectedChunker = useMemo<ChunkerInfo | null>(+ () => chunkers.find((c) => c.type === selectedChunkerType) ?? null,+ [chunkers, selectedChunkerType],+ );++ const chunkerParamSchema = useMemo<JSONSchema | null>(+ () => selectedChunker?.parameter_schema ?? null,+ [selectedChunker],+ );++ const handleSelectChunker = useCallback(+ (type: string) => {+ setSelectedChunkerType(type);+ const info = chunkers.find((c) => c.type === type);+ const schema = info?.parameter_schema;+ setChunkerParams(schema ? defaultValuesFromSchema(schema, NO_SKIP) : {});+ },+ [chunkers],+ );++ const handleChunkerParamChange = useCallback((key: string, value: SchemaFormValue) => {+ setChunkerParams((prev) => ({ ...prev, [key]: value }));+ }, []);++ // Default to the first available chunker so the form is submittable+ // without an explicit pick; re-pick when the list changes underneath.+ useEffect(() => {+ if (!open || chunkers.length === 0) return;+ if (!chunkers.some((c) => c.type === selectedChunkerType)) {+ handleSelectChunker(chunkers[0].type);+ }+ }, [open, chunkers, selectedChunkerType, handleSelectChunker]);+useEffect(() => {if (!open) {setName('');setDescription('');setSelected(null);setDimension(null);+ setSelectedChunkerType('');+ setChunkerParams({});setErrorKey(null);setSubmitting(false);}@@ -100,6 +156,10 @@ export function CreateKnowledgeBaseDialog({setErrorKey('dialog-knowledge-base-create.errors.dimensionRequired');return;}+ if (!selectedChunkerType) {+ setErrorKey('dialog-knowledge-base-create.errors.chunkerRequired');+ return;+ }setErrorKey(null);setSubmitting(true);try {@@ -110,10 +170,19 @@ export function CreateKnowledgeBaseDialog({dimensions: dimension,parameters: {},};+ const params: Record<string, unknown> = {};+ for (const [k, v] of Object.entries(chunkerParams)) {+ if (v !== undefined && v !== null && v !== '') params[k] = v;+ }+ const chunkerConfig: ChunkerConfig = {+ type: selectedChunkerType,+ parameters: params,+ };const knowledgeBaseId = await create({name: name.trim(),description: description.trim(),embedding_model_config: config,+ chunker_config: chunkerConfig,});onCreated?.(knowledgeBaseId);onOpenChange(false);@@ -167,7 +236,7 @@ export function CreateKnowledgeBaseDialog({</Alert>)}- <FieldGroup>+ <FieldGroup className={HORIZONTAL_FIELD_WIDTH}><Field><FieldLabel>{t('dialog-knowledge-base-create.name.label')}</FieldLabel><Input@@ -191,6 +260,9 @@ export function CreateKnowledgeBaseDialog({rows={3}/></Field>++ <Separator />+<Field orientation="horizontal"><FieldLabel>{t('dialog-knowledge-base-create.embeddingModel.label')}@@ -220,6 +292,53 @@ export function CreateKnowledgeBaseDialog({disabled={submitting}/></Field>++ {chunkers.length > 0 && (+ <>+ <Separator />++ <Field orientation="horizontal">+ <FieldLabel>+ {t('dialog-knowledge-base-create.chunker.label')}+ </FieldLabel>+ <ChunkerSelect+ value={selectedChunkerType}+ chunkers={chunkers}+ onChange={handleSelectChunker}+ disabled={submitting}+ />+ </Field>++ {chunkerParamSchema && (+ <SchemaForm+ schema={chunkerParamSchema}+ values={chunkerParams}+ onChange={handleChunkerParamChange}+ skipFields={NO_SKIP}+ idPrefix="chunker-param"+ orientation="horizontal"+ className={HORIZONTAL_FIELD_WIDTH}+ labelFor={(key, prop) =>+ t(+ `chunker-types.${selectedChunkerType}.params.${key}.label`,+ { defaultValue: '' },+ ) ||+ prop.title ||+ undefined+ }+ descriptionFor={(key, prop) =>+ t(+ `chunker-types.${selectedChunkerType}.params.${key}.description`,+ { defaultValue: '' },+ ) ||+ prop.description ||+ undefined+ }+ />+ )}+ </>+ )}+{errorKey && <p className="text-destructive text-sm">{t(errorKey)}</p>}</FieldGroup><DialogFooter>diff --git a/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx b/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsxindex 7ffd58e5a..0fb9c268d 100644--- a/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx+++ b/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx@@ -72,6 +72,21 @@ export function EditKnowledgeBaseDialog({ open, onOpenChange, knowledgeBase, onU? `${knowledgeBase.embedding_model_config.model} · ${knowledgeBase.embedding_model_config.dimensions}d`: '';+ const chunkerLabel = (() => {+ if (!knowledgeBase?.chunker_config) return null;+ const cfg = knowledgeBase.chunker_config;+ const typeLabel = t(`chunker-types.${cfg.type}.label`, { defaultValue: cfg.type });+ const paramStr = Object.entries(cfg.parameters)+ .map(([k, v]) => {+ const paramLabel = t(`chunker-types.${cfg.type}.params.${k}.label`, {+ defaultValue: k,+ });+ return `${paramLabel}=${v}`;+ })+ .join(', ');+ return paramStr ? `${typeLabel} (${paramStr})` : typeLabel;+ })();+return (<Dialog open={open} onOpenChange={onOpenChange}><DialogContent className="!w-[500px] !max-w-[500px]">@@ -113,6 +128,14 @@ export function EditKnowledgeBaseDialog({ open, onOpenChange, knowledgeBase, onU{embeddingModelLabel}</Badge></Field>+ {chunkerLabel && (+ <Field orientation="horizontal">+ <FieldLabel>{t('dialog-knowledge-base-edit.chunker.label')}</FieldLabel>+ <Badge variant="secondary" className="font-mono">+ {chunkerLabel}+ </Badge>+ </Field>+ )}{errorKey && <p className="text-destructive text-sm">{t(errorKey)}</p>}</FieldGroup><DialogFooter>diff --git a/examples/web_ui/frontend/src/components/form/SchemaForm.tsx b/examples/web_ui/frontend/src/components/form/SchemaForm.tsxindex d55ee8857..b64d1ffef 100644--- a/examples/web_ui/frontend/src/components/form/SchemaForm.tsx+++ b/examples/web_ui/frontend/src/components/form/SchemaForm.tsx@@ -1,6 +1,14 @@+import type { ReactNode } from 'react';+import type { JSONSchema, JSONSchemaProperty } from '@/api';import { Checkbox } from '@/components/ui/checkbox';-import { Field, FieldDescription, FieldGroup, FieldLabel } from '@/components/ui/field.tsx';+import {+ Field,+ FieldContent,+ FieldDescription,+ FieldGroup,+ FieldLabel,+} from '@/components/ui/field.tsx';import { Input } from '@/components/ui/input';import {Select,@@ -29,6 +37,10 @@ interface Props {descriptionFor?: (key: string, prop: JSONSchemaProperty) => string | undefined;/** Prefix for generated DOM IDs (so multiple SchemaForms on one page don't collide). */idPrefix?: string;+ /** Layout of non-boolean fields. `horizontal` puts the label left of the control. */+ orientation?: 'vertical' | 'horizontal';+ /** Extra classes for the wrapping FieldGroup. */+ className?: string;}function effectiveType(prop: JSONSchemaProperty): string {@@ -77,13 +89,15 @@ export function SchemaForm({placeholderFor,descriptionFor,idPrefix = 'schema-form',+ orientation = 'vertical',+ className,}: Props) {const entries = Object.entries(schema.properties ?? {}).filter(([key, prop]) => !skipFields.has(key) && prop.const === undefined,);return (- <FieldGroup>+ <FieldGroup className={className}>{entries.map(([key, prop]) => {const fieldId = `${idPrefix}-${key}`;const isRequired = schema.required?.includes(key) ?? false;@@ -99,6 +113,33 @@ export function SchemaForm({const description = descriptionFor?.(key, prop);const current = values[key];+ const labelNode = (+ <FieldLabel htmlFor={fieldId}>+ {label}+ {isRequired && <span className="text-destructive ml-0.5">*</span>}+ </FieldLabel>+ );+ const descriptionNode = description ? (+ <FieldDescription>{description}</FieldDescription>+ ) : null;+ /** Label (+ helper text) left of the control when horizontal. */+ const wrap = (control: ReactNode) =>+ orientation === 'horizontal' ? (+ <Field key={key} orientation="horizontal">+ <FieldContent>+ {labelNode}+ {descriptionNode}+ </FieldContent>+ {control}+ </Field>+ ) : (+ <Field key={key}>+ {labelNode}+ {control}+ {descriptionNode}+ </Field>+ );+if (isBoolean) {return (<Field key={key} orientation="horizontal">@@ -118,101 +159,71 @@ export function SchemaForm({if (enumOpts) {const currentStr =current === undefined || current === null ? '' : String(current);- return (- <Field key={key}>- <FieldLabel htmlFor={fieldId}>- {label}- {isRequired && <span className="text-destructive ml-0.5">*</span>}- </FieldLabel>- <Select value={currentStr} onValueChange={(v) => onChange(key, v)}>- <SelectTrigger id={fieldId} className="w-full">- <SelectValue placeholder={placeholder} />- </SelectTrigger>- <SelectContent>- {enumOpts.map((opt) => (- <SelectItem key={String(opt)} value={String(opt)}>- {String(opt)}- </SelectItem>- ))}- </SelectContent>- </Select>- {description && <FieldDescription>{description}</FieldDescription>}- </Field>+ return wrap(+ <Select value={currentStr} onValueChange={(v) => onChange(key, v)}>+ <SelectTrigger id={fieldId} className="w-full">+ <SelectValue placeholder={placeholder} />+ </SelectTrigger>+ <SelectContent>+ {enumOpts.map((opt) => (+ <SelectItem key={String(opt)} value={String(opt)}>+ {String(opt)}+ </SelectItem>+ ))}+ </SelectContent>+ </Select>,);}if (isTextarea) {- return (- <Field key={key}>- <FieldLabel htmlFor={fieldId}>- {label}- {isRequired && <span className="text-destructive ml-0.5">*</span>}- </FieldLabel>- <Textarea- id={fieldId}- rows={3}- value={(current as string | undefined) ?? ''}- onChange={(e) => onChange(key, e.target.value)}- placeholder={placeholder}- />- {description && <FieldDescription>{description}</FieldDescription>}- </Field>+ return wrap(+ <Textarea+ id={fieldId}+ rows={3}+ value={(current as string | undefined) ?? ''}+ onChange={(e) => onChange(key, e.target.value)}+ placeholder={placeholder}+ />,);}if (isNumber) {const min = prop.minimum ?? prop.exclusiveMinimum;const max = prop.maximum ?? prop.exclusiveMaximum;- return (- <Field key={key}>- <FieldLabel htmlFor={fieldId}>- {label}- {isRequired && <span className="text-destructive ml-0.5">*</span>}- </FieldLabel>- <Input- id={fieldId}- type="number"- min={min}- max={max}- step={inferStep(type)}- value={- current === undefined || current === null ? '' : String(current)+ return wrap(+ <Input+ id={fieldId}+ type="number"+ min={min}+ max={max}+ step={inferStep(type)}+ value={current === undefined || current === null ? '' : String(current)}+ onChange={(e) => {+ const raw = e.target.value;+ // Empty input → undefined so JSON.stringify drops the key+ // and the backend applies its own default. Sending "" would+ // fail Pydantic float/int coercion.+ if (raw === '') {+ onChange(key, undefined);+ return;}- onChange={(e) => {- const raw = e.target.value;- // Empty input → undefined so JSON.stringify drops the key- // and the backend applies its own default. Sending "" would- // fail Pydantic float/int coercion.- if (raw === '') {- onChange(key, undefined);- return;- }- const parsed =- type === 'integer' ? parseInt(raw, 10) : parseFloat(raw);- onChange(key, Number.isNaN(parsed) ? raw : parsed);- }}- placeholder={placeholder}- />- {description && <FieldDescription>{description}</FieldDescription>}- </Field>+ const parsed =+ type === 'integer' ? parseInt(raw, 10) : parseFloat(raw);+ onChange(key, Number.isNaN(parsed) ? raw : parsed);+ }}+ placeholder={placeholder}+ />,);}- return (- <Field key={key}>- <FieldLabel htmlFor={fieldId}>- {label}- {isRequired && <span className="text-destructive ml-0.5">*</span>}- </FieldLabel>- <Input- id={fieldId}- type={isPassword ? 'password' : 'text'}- value={(current as string | undefined) ?? ''}- onChange={(e) => onChange(key, e.target.value)}- placeholder={placeholder}- />- {description && <FieldDescription>{description}</FieldDescription>}- </Field>+ return wrap(+ <Input+ id={fieldId}+ type={isPassword ? 'password' : 'text'}+ value={(current as string | undefined) ?? ''}+ onChange={(e) => onChange(key, e.target.value)}+ placeholder={placeholder}+ />,);})}</FieldGroup>diff --git a/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx b/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsxindex c8fedce5e..022c80b81 100644--- a/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx+++ b/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx@@ -75,7 +75,7 @@ function StatusBadge({ phase }: { phase: UploadPhase }) {return (<spanclassName={cn(- 'inline-flex items-center gap-x-1 rounded-md px-1.5 py-0.5 text-[10px] font-medium',+ 'inline-flex shrink-0 items-center gap-x-1 rounded-md px-1.5 py-0.5 text-[10px] font-medium whitespace-nowrap',tone,)}>diff --git a/examples/web_ui/frontend/src/components/select/ChunkerSelect.tsx b/examples/web_ui/frontend/src/components/select/ChunkerSelect.tsxnew file mode 100644index 000000000..b08fc0f76--- /dev/null+++ b/examples/web_ui/frontend/src/components/select/ChunkerSelect.tsx@@ -0,0 +1,74 @@+import { ChevronDown } from 'lucide-react';++import type { ChunkerInfo } from '@/api';+import { Button } from '@/components/ui/button';+import {+ DropdownMenu,+ DropdownMenuContent,+ DropdownMenuItem,+ DropdownMenuTrigger,+} from '@/components/ui/dropdown-menu';+import { useTranslation } from '@/i18n/useI18n.ts';++interface Props {+ value: string;+ chunkers: ChunkerInfo[];+ loading?: boolean;+ onChange?: (type: string) => void;+ disabled?: boolean;+ placeholder?: string;+}++/**+ * Chunker type picker. Mirrors the same visual style used by+ * :component:`EmbeddingSelect` so the two selectors look consistent+ * when placed next to each other in the knowledge-base creation dialog.+ */+export function ChunkerSelect({+ value,+ chunkers,+ loading,+ onChange,+ disabled,+ placeholder,+}: Props) {+ const { t } = useTranslation();++ const localizedLabel = (type: string) =>+ t(`chunker-types.${type}.label`, { defaultValue: type });++ const displayLabel = value+ ? localizedLabel(value)+ : loading+ ? t('chunker-select.loading')+ : (placeholder ?? t('chunker-select.placeholder'));++ return (+ <DropdownMenu>+ <DropdownMenuTrigger asChild>+ <Button+ variant="outline"+ size="sm"+ className="justify-between gap-1 font-normal"+ disabled={disabled}+ >+ <span className="truncate">{displayLabel}</span>+ <ChevronDown className="size-3.5 text-muted-foreground" />+ </Button>+ </DropdownMenuTrigger>+ <DropdownMenuContent align="start" className="min-w-48 max-h-72 overflow-y-auto">+ {!loading && chunkers.length === 0 ? (+ <div className="px-2 py-3 text-center text-sm text-muted-foreground">+ {t('chunker-select.empty')}+ </div>+ ) : (+ chunkers.map((c) => (+ <DropdownMenuItem key={c.type} onSelect={() => onChange?.(c.type)}>+ {localizedLabel(c.type)}+ </DropdownMenuItem>+ ))+ )}+ </DropdownMenuContent>+ </DropdownMenu>+ );+}diff --git a/examples/web_ui/frontend/src/hooks/useChunkers.ts b/examples/web_ui/frontend/src/hooks/useChunkers.tsnew file mode 100644index 000000000..db42204e9--- /dev/null+++ b/examples/web_ui/frontend/src/hooks/useChunkers.ts@@ -0,0 +1,34 @@+import { useCallback, useEffect, useState } from 'react';++import { knowledgeBaseApi } from '@/api';+import type { ChunkerInfo } from '@/api';++/**+ * Fetches the list of registered chunker types and their parameter+ * schemas. Used by the create-KB dialog to render the chunker+ * selector and dynamic parameter form.+ */+export function useChunkers() {+ const [chunkers, setChunkers] = useState<ChunkerInfo[]>([]);+ const [loading, setLoading] = useState(false);++ const refetch = useCallback(async () => {+ setLoading(true);+ try {+ const res = await knowledgeBaseApi.listChunkers();+ setChunkers(res.chunkers);+ } catch {+ // Graceful fallback — if the endpoint is unavailable+ // (older backend), the dialog works without chunker+ // selection.+ } finally {+ setLoading(false);+ }+ }, []);++ useEffect(() => {+ refetch();+ }, [refetch]);++ return { chunkers, loading, refetch };+}diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.jsonindex 429ad1836..dc2d9423e 100644--- a/examples/web_ui/frontend/src/i18n/locales/en.json+++ b/examples/web_ui/frontend/src/i18n/locales/en.json@@ -262,6 +262,9 @@"dimension": {"label": "Dimension"},+ "chunker": {+ "label": "Chunker"+ },"policy": {"lockedTitle": "Dimension is locked to {{dimension}}d","lockedDescription": "This server already pins all knowledge bases to {{dimension}}-dim embeddings. Only compatible models are listed below.",@@ -272,7 +275,8 @@"errors": {"nameRequired": "Please give the knowledge base a name.","embeddingRequired": "Please pick an embedding model.",- "dimensionRequired": "Please pick a dimension."+ "dimensionRequired": "Please pick a dimension.",+ "chunkerRequired": "Please pick a chunking strategy."}},"dialog-knowledge-base-edit": {@@ -289,6 +293,9 @@"embeddingModel": {"label": "Embedding model"},+ "chunker": {+ "label": "Chunker"+ },"errors": {"nameRequired": "Please give the knowledge base a name."}@@ -395,6 +402,26 @@},"addCredential": "Add Credential"},+ "chunker-select": {+ "placeholder": "Select a chunker",+ "loading": "Loading…",+ "empty": "No chunkers available"+ },+ "chunker-types": {+ "approx_token": {+ "label": "Approx Token",+ "params": {+ "chunk_size": {+ "label": "Chunk Size",+ "description": "Maximum number of approximate tokens per chunk."+ },+ "overlap": {+ "label": "Overlap",+ "description": "Number of approximate tokens shared between consecutive chunks."+ }+ }+ }+ },"dimension-select": {"placeholder": "Select a dimension","pickModelFirst": "Pick a model first"diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.jsonindex 260e8fde0..62926125d 100644--- a/examples/web_ui/frontend/src/i18n/locales/zh.json+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json@@ -262,6 +262,9 @@"dimension": {"label": "维度"},+ "chunker": {+ "label": "分块策略"+ },"policy": {"lockedTitle": "维度已锁定为 {{dimension}}d","lockedDescription": "当前实例已要求所有知识库使用 {{dimension}} 维 embedding,下面仅展示兼容的模型。",@@ -272,7 +275,8 @@"errors": {"nameRequired": "请填写知识库名称。","embeddingRequired": "请选择 Embedding 模型。",- "dimensionRequired": "请选择 Embedding 维度。"+ "dimensionRequired": "请选择 Embedding 维度。",+ "chunkerRequired": "请选择分块策略。"}},"dialog-knowledge-base-edit": {@@ -289,6 +293,9 @@"embeddingModel": {"label": "Embedding 模型"},+ "chunker": {+ "label": "分块策略"+ },"errors": {"nameRequired": "请填写知识库名称。"}@@ -395,6 +402,26 @@},"addCredential": "添加凭证"},+ "chunker-select": {+ "placeholder": "选择分块策略",+ "loading": "加载中…",+ "empty": "暂无可用分块策略"+ },+ "chunker-types": {+ "approx_token": {+ "label": "近似 Token 数",+ "params": {+ "chunk_size": {+ "label": "块大小",+ "description": "每个分块的最大近似 Token 数量。"+ },+ "overlap": {+ "label": "重叠大小",+ "description": "相邻分块之间共享的近似 Token 数量。"+ }+ }+ }+ },"dimension-select": {"placeholder": "选择维度","pickModelFirst": "请先选择 Embedding 模型"diff --git a/examples/web_ui/frontend/src/pages/knowledge/index.tsx b/examples/web_ui/frontend/src/pages/knowledge/index.tsxindex 7acace7f6..6191ecc08 100644--- a/examples/web_ui/frontend/src/pages/knowledge/index.tsx+++ b/examples/web_ui/frontend/src/pages/knowledge/index.tsx@@ -1,5 +1,5 @@import { Ellipsis, Files, FlaskConical, Pencil, Plus, Trash2 } from 'lucide-react';-import { useState } from 'react';+import { useEffect, useState } from 'react';import { useTranslation } from 'react-i18next';import { useNavigate, useParams } from 'react-router-dom';@@ -117,6 +117,16 @@ export const KnowledgePage = () => {const selectedKb = knowledgeBases.find((kb) => kb.id === selectedKbId);+ // Default to the first knowledge base when none is selected (or the+ // selected one no longer exists), mirroring the URL so refresh keeps it.+ useEffect(() => {+ if (knowledgeBases.length === 0) return;+ if (knowledgeBases.some((kb) => kb.id === selectedKbId)) return;+ const first = knowledgeBases[0].id;+ setSelectedKbId(first);+ navigate(`/knowledge/${first}`, { replace: true });+ }, [knowledgeBases, selectedKbId, navigate]);+const handleCreated = async (knowledgeBaseId: string) => {await refetch();setSelectedKbId(knowledgeBaseId);diff --git a/src/agentscope/app/_app.py b/src/agentscope/app/_app.pyindex e43afc74c..091a17d13 100644--- a/src/agentscope/app/_app.py+++ b/src/agentscope/app/_app.py@@ -32,12 +32,9 @@ from .message_bus import MessageBusfrom .storage import StorageBasefrom ..agent import Agentfrom ..credential import CredentialFactory, CredentialBase-from ..rag import (- ApproxTokenChunker,- ChunkerBase,- ParserBase,- TextParser,-)+from ..rag import ApproxTokenChunker, ChunkerBase, ParserBase, TextParser++from .._logging import loggerfrom .._version import __version__@@ -84,7 +81,7 @@ def create_app(workspace_manager: WorkspaceManagerBase,knowledge_base_manager: KnowledgeBaseManagerBase | None = None,knowledge_parsers: list[ParserBase] | dict[str, ParserBase] | None = None,- knowledge_chunker: ChunkerBase | None = None,+ knowledge_chunkers: list[Type[ChunkerBase]] | None = None,blob_store: BlobStoreBase | None = None,enable_index_worker: bool = True,mcp_hubs: list[MCPHubBase] | None = None,@@ -101,6 +98,7 @@ def create_app(download_secret: str | None = None,title: str = "AgentScope",version: str = __version__,+ **kwargs: Any,) -> FastAPI:"""Create and configure a FastAPI application.@@ -164,9 +162,11 @@ def create_app((one parser bound to multiple types, type aliases, ...).Defaults to ``[TextParser()]`` when``knowledge_base_manager`` is set.- knowledge_chunker (`ChunkerBase | None`, optional):- The chunker shared across every knowledge base. Defaults- to :class:`~agentscope.rag.ApproxTokenChunker()` when+ knowledge_chunkers (`list[Type[ChunkerBase]] | None`, optional):+ The chunker classes users can choose from when creating a+ knowledge base. The chunker type and parameters are pinned+ on the knowledge base record and reconstructed by the index+ worker. Defaults to ``[ApproxTokenChunker]`` when``knowledge_base_manager`` is set.blob_store (`BlobStoreBase | None`, optional):Backend storing uploaded document bytes between the@@ -297,13 +297,44 @@ def create_app(# KB feature is actually enabled. When ``knowledge_base_manager`` is# ``None`` every KB endpoint is disabled, so leaving these as ``None``# avoids unused imports being eagerly constructed at app startup.+ unknown_kwargs = set(kwargs) - {"knowledge_chunker"}+ if unknown_kwargs:+ logger.warning(+ "Ignoring unknown create_app() arguments: %s",+ sorted(unknown_kwargs),+ )+if knowledge_base_manager is not None:app.state.knowledge_parsers = (knowledge_parsersif knowledge_parsers is not Noneelse [TextParser()])- app.state.knowledge_chunker = knowledge_chunker or ApproxTokenChunker()+ chunker_classes = list(+ knowledge_chunkers+ if knowledge_chunkers is not None+ else [ApproxTokenChunker],+ )+ # Backward compatibility: the deprecated ``knowledge_chunker``+ # instance is only used for its class.+ if "knowledge_chunker" in kwargs:+ logger.warning(+ "The `knowledge_chunker` argument of create_app() is "+ "deprecated, use `knowledge_chunkers` instead.",+ )+ legacy_cls = type(kwargs["knowledge_chunker"])+ if legacy_cls not in chunker_classes:+ chunker_classes.append(legacy_cls)+ seen_chunker_types: dict[str, Type[ChunkerBase]] = {}+ for cls in chunker_classes:+ if cls.chunker_type in seen_chunker_types:+ raise ValueError(+ f"Duplicate chunker_type {cls.chunker_type!r}: "+ f"{seen_chunker_types[cls.chunker_type].__name__} and "+ f"{cls.__name__}.",+ )+ seen_chunker_types[cls.chunker_type] = cls+ app.state.knowledge_chunkers = chunker_classesapp.state.blob_store = (blob_storeif blob_store is not None@@ -311,7 +342,7 @@ def create_app()else:app.state.knowledge_parsers = knowledge_parsers- app.state.knowledge_chunker = knowledge_chunker+ app.state.knowledge_chunkers = knowledge_chunkersapp.state.blob_store = blob_storeapp.state.enable_index_worker = (enable_index_worker and knowledge_base_manager is not Nonediff --git a/src/agentscope/app/_lifespan.py b/src/agentscope/app/_lifespan.pyindex e004a16e1..bd46ff9bd 100644--- a/src/agentscope/app/_lifespan.py+++ b/src/agentscope/app/_lifespan.py@@ -197,7 +197,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:blob_store=blob_store,knowledge_base_manager=knowledge_base_manager,parsers=app.state.knowledge_parsers,- chunker=app.state.knowledge_chunker,+ chunkers=app.state.knowledge_chunkers,node_id=node_id,)await stack.enter_async_context(@@ -220,6 +220,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:blob_store=blob_store,message_bus=message_bus,resource_access_service=resource_access_service,+ chunkers=app.state.knowledge_chunkers,)app.state.knowledge_base_service = knowledge_base_servicediff --git a/src/agentscope/app/_router/_knowledge_base.py b/src/agentscope/app/_router/_knowledge_base.pyindex f3951c430..9db702200 100644--- a/src/agentscope/app/_router/_knowledge_base.py+++ b/src/agentscope/app/_router/_knowledge_base.py@@ -23,15 +23,18 @@ from ..deps import (get_current_user_id,get_knowledge_base_manager,get_knowledge_base_service,+ get_knowledge_chunkers,get_knowledge_parsers,get_resource_access_service,)from ._schema import (+ ChunkerInfo,CreateKnowledgeBaseRequest,CreateKnowledgeBaseResponse,KbEmbeddingProvider,KbMiddlewareParametersSchemaResponse,KnowledgeDocumentView,+ ListChunkersResponse,ListKbEmbeddingModelsResponse,ListKnowledgeBasesResponse,ListKnowledgeDocumentsResponse,@@ -50,7 +53,7 @@ from .._service import (ResourceAccessService,)from ...middleware import RAGMiddleware-from ...rag import ParserBase+from ...rag import ChunkerBase, ParserBaseknowledge_base_router = APIRouter(@@ -129,6 +132,46 @@ async def list_kb_embedding_models(return ListKbEmbeddingModelsResponse(providers=providers, policy=policy)+@knowledge_base_router.get(+ "/chunkers",+ response_model=ListChunkersResponse,+ summary="List available chunker types and their parameter schemas",+)+async def list_chunkers(+ _: str = Depends(get_current_user_id),+ chunker_classes: list[type[ChunkerBase]] = Depends(+ get_knowledge_chunkers,+ ),+) -> ListChunkersResponse:+ """List every available chunker type with its parameter schema.++ The front-end uses this to populate the chunker selector and+ to render a dynamic parameter form when creating a knowledge+ base.++ Args:+ _ (`str`):+ Injected authenticated user ID; only used to gate the+ endpoint behind authentication.+ chunker_classes (`list[type[ChunkerBase]]`):+ The chunker classes configured via+ ``create_app(knowledge_chunkers=...)``.++ Returns:+ `ListChunkersResponse`:+ All available chunker types with their JSON Schemas.+ """+ return ListChunkersResponse(+ chunkers=[+ ChunkerInfo(+ type=cls.chunker_type,+ parameter_schema=cls.Parameters.model_json_schema(),+ )+ for cls in chunker_classes+ ],+ )++@knowledge_base_router.get("/middleware/parameters_schema",response_model=KbMiddlewareParametersSchemaResponse,@@ -246,6 +289,7 @@ async def create_knowledge_base(name=body.name,description=body.description,embedding_model_config=body.embedding_model_config,+ chunker_config=body.chunker_config,)return CreateKnowledgeBaseResponse(knowledge_base_id=record.id)diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.pyindex a8c8e2b6b..cb7cf0ceb 100644--- a/src/agentscope/app/_router/_schema/__init__.py+++ b/src/agentscope/app/_router/_schema/__init__.py@@ -55,11 +55,13 @@ from ._credential import (ListCredentialSchemasResponse,)from ._knowledge_base import (+ ChunkerInfo,CreateKnowledgeBaseRequest,CreateKnowledgeBaseResponse,KbEmbeddingProvider,KbMiddlewareParametersSchemaResponse,KnowledgeDocumentView,+ ListChunkersResponse,ListKbEmbeddingModelsResponse,ListKnowledgeBasesResponse,ListKnowledgeDocumentsResponse,@@ -130,11 +132,13 @@ __all__ = ["ListCredentialsResponse","ListCredentialSchemasResponse",# Knowledge base+ "ChunkerInfo","CreateKnowledgeBaseRequest","CreateKnowledgeBaseResponse","KbEmbeddingProvider","KbMiddlewareParametersSchemaResponse","KnowledgeDocumentView",+ "ListChunkersResponse","ListKbEmbeddingModelsResponse","ListKnowledgeBasesResponse","ListKnowledgeDocumentsResponse",diff --git a/src/agentscope/app/_router/_schema/_knowledge_base.py b/src/agentscope/app/_router/_schema/_knowledge_base.pyindex 99c8effea..7a92ffc1c 100644--- a/src/agentscope/app/_router/_schema/_knowledge_base.py+++ b/src/agentscope/app/_router/_schema/_knowledge_base.py@@ -5,6 +5,7 @@ from datetime import datetimefrom pydantic import BaseModel, Fieldfrom ...storage import (+ ChunkerConfig,EmbeddingModelConfig,KnowledgeDocumentRecord,KnowledgeDocumentStatus,@@ -30,6 +31,15 @@ class CreateKnowledgeBaseRequest(BaseModel):"invalidate every previously inserted vector."),)+ chunker_config: ChunkerConfig | None = Field(+ default=None,+ description=(+ "Chunker configuration determining how uploaded documents "+ "are split into chunks. Pinned at creation time and "+ "cannot be changed afterwards. Defaults to the first "+ "configured chunker with its default parameters."+ ),+ )class CreateKnowledgeBaseResponse(BaseModel):@@ -272,3 +282,23 @@ class ListSupportedContentTypesResponse(BaseModel):"subclasses may override the default."),)+++class ChunkerInfo(BaseModel):+ """One available chunker type and its parameter schema."""++ type: str = Field(description="The chunker type identifier.")+ parameter_schema: dict = Field(+ description=(+ "JSON Schema produced by `ChunkerBase.Parameters."+ "model_json_schema()`, used to render the parameter form."+ ),+ )+++class ListChunkersResponse(BaseModel):+ """Response body listing the available chunker types."""++ chunkers: list[ChunkerInfo] = Field(+ description="The available chunker types, in configured order.",+ )diff --git a/src/agentscope/app/_service/_access.py b/src/agentscope/app/_service/_access.pyindex 2c583f7ee..9773efac6 100644--- a/src/agentscope/app/_service/_access.py+++ b/src/agentscope/app/_service/_access.py@@ -16,6 +16,7 @@ from ..access import ()from ..storage import (AgentRecord,+ ChunkerConfig,CredentialRecord,EmbeddingModelConfig,KnowledgeBaseRecord,@@ -84,6 +85,13 @@ class KnowledgeBaseView(BaseModel):embedding_model_config: EmbeddingModelConfig = Field(description="Embedding model configuration pinned at creation.",)+ chunker_config: ChunkerConfig | None = Field(+ default=None,+ description=(+ "Chunker configuration pinned at creation. ``None`` for "+ "legacy records created before per-KB chunker support."+ ),+ )created_at: datetime = Field(description="Creation timestamp.")updated_at: datetime = Field(description="Last-update timestamp.")editable: bool = Field(diff --git a/src/agentscope/app/_service/_index_worker.py b/src/agentscope/app/_service/_index_worker.pyindex 3c2eae7d7..14bcb8518 100644--- a/src/agentscope/app/_service/_index_worker.py+++ b/src/agentscope/app/_service/_index_worker.py@@ -25,14 +25,17 @@ import contextlibimport mimetypesfrom concurrent.futures import ProcessPoolExecutorfrom datetime import timedelta-from typing import TYPE_CHECKING+from typing import Any, TYPE_CHECKING++from pydantic import ValidationErrorfrom ..._logging import logger+from ...rag import ApproxTokenChunkerif TYPE_CHECKING:from ..rag.blob_store import BlobStoreBasefrom ..rag.knowledge_base_manager import KnowledgeBaseManagerBase- from ..storage import StorageBase+ from ..storage import KnowledgeBaseRecord, StorageBasefrom ...rag import ChunkerBase, ParserBase, Section# Read blob bytes in chunks bounded so the worker never holds the whole@@ -115,11 +118,12 @@ class IndexWorker:blob_store: "BlobStoreBase",knowledge_base_manager: "KnowledgeBaseManagerBase",parsers: "list[ParserBase] | dict[str, ParserBase]",- chunker: "ChunkerBase",node_id: str,+ chunkers: "list[type[ChunkerBase]] | None" = None,max_concurrency: int = 4,lease_ttl: timedelta = timedelta(seconds=90),parser_executor: ProcessPoolExecutor | None = None,+ **kwargs: Any,) -> None:"""Initialize the worker.@@ -146,13 +150,15 @@ class IndexWorker:declare.Same registry the upload service uses, passed in by DI.- chunker (`ChunkerBase`):- The shared chunker.node_id (`str`):Stable identifier for this worker process. Used as``processing_node`` on the lease so the sweeper cantell whose work expired. Typically``f"{hostname}:{pid}:{uuid}"``.+ chunkers (`list[type[ChunkerBase]] | None`, optional):+ The chunker classes that can be rebuilt from a knowledge+ base's ``chunker_config``. Defaults to+ ``[ApproxTokenChunker]``.max_concurrency (`int`, defaults to ``4``):Maximum number of documents processed concurrently bythis worker. Higher values trade memory for@@ -169,12 +175,33 @@ class IndexWorker:third-party byte-oriented parsers. Injected so asingle pool can be shared across the app (built inlifespan).+ **kwargs (`Any`):+ Deprecated. ``chunker`` (a shared chunker instance) is+ still accepted for backward compatibility; only its+ class is used."""+ chunker_classes = list(chunkers or [ApproxTokenChunker])+ if "chunker" in kwargs:+ logger.warning(+ "The `chunker` argument of IndexWorker is deprecated, "+ "use `chunkers` instead.",+ )+ legacy_cls = type(kwargs.pop("chunker"))+ if legacy_cls not in chunker_classes:+ chunker_classes.append(legacy_cls)+ if kwargs:+ logger.warning(+ "Ignoring unknown IndexWorker arguments: %s",+ sorted(kwargs),+ )+self._storage = storageself._blob_store = blob_storeself._manager = knowledge_base_managerself._parsers_by_media_type = _build_parser_registry(parsers)- self._chunker = chunker+ self._chunkers_by_type = {+ cls.chunker_type: cls for cls in chunker_classes+ }self._node_id = node_idself._lease_ttl = lease_ttlself._sem = asyncio.Semaphore(max_concurrency)@@ -326,6 +353,19 @@ class IndexWorker:)return+ kb_record = await self._manager.get_knowledge_base(+ user_id,+ knowledge_base_id,+ )+ if kb_record is None:+ logger.warning(+ "Knowledge base %s vanished before processing document %s.",+ knowledge_base_id,+ document_id,+ )+ return+ chunker = self._resolve_chunker_from_record(kb_record)+data = record.datamedia_type = (data.content_type or mimetypes.guess_type(data.filename)[0]@@ -357,7 +397,7 @@ class IndexWorker:document_id,"chunking",)- chunks = await self._chunker.chunk(sections)+ chunks = await chunker.chunk(sections)# ---- indexing ----await self._storage.update_knowledge_document_status(@@ -407,6 +447,42 @@ class IndexWorker:filename,)+ def _resolve_chunker_from_record(+ self,+ kb_record: "KnowledgeBaseRecord",+ ) -> "ChunkerBase":+ """Resolve the chunker from a pre-fetched KB record.++ Instantiates the matching chunker class using the record's+ ``chunker_config``. Falls back to+ :class:`~agentscope.rag.ApproxTokenChunker` for legacy+ records that predate per-KB chunker support+ (``chunker_config`` is ``None``).+ """+ cfg = kb_record.data.chunker_config+ if cfg is not None:+ chunker_cls = self._chunkers_by_type.get(cfg.type)+ if chunker_cls is None:+ logger.warning(+ "Unknown chunker type %r on KB %s — "+ "falling back to ApproxTokenChunker.",+ cfg.type,+ kb_record.id,+ )+ return ApproxTokenChunker()+ try:+ return chunker_cls(+ parameters=chunker_cls.Parameters(**cfg.parameters),+ )+ except (ValidationError, TypeError, ValueError) as exc:+ logger.warning(+ "Invalid chunker parameters on KB %s: %s — "+ "falling back to ApproxTokenChunker.",+ kb_record.id,+ exc,+ )+ return ApproxTokenChunker()+async def _read_blob(self, blob_uri: str) -> bytes:"""Stream the blob into memory in bounded chunks.diff --git a/src/agentscope/app/_service/_knowledge_base.py b/src/agentscope/app/_service/_knowledge_base.pyindex 180e0bcda..f0a921bd0 100644--- a/src/agentscope/app/_service/_knowledge_base.py+++ b/src/agentscope/app/_service/_knowledge_base.py@@ -21,6 +21,7 @@ import uuidfrom typing import IO, TYPE_CHECKINGfrom fastapi import HTTPException, status+from pydantic import ValidationErrorfrom ..access import ResourceKindfrom ..rag.knowledge_base_manager import (@@ -28,10 +29,12 @@ from ..rag.knowledge_base_manager import (KnowledgeBaseNotFoundError,)from ..storage import (+ ChunkerConfig,KnowledgeDocumentData,KnowledgeDocumentRecord,)from ..._logging import logger+from ...rag import ApproxTokenChunkerfrom .._bus_ops import enqueue_index_taskfrom ._access import ResourceAccessService@@ -44,7 +47,7 @@ if TYPE_CHECKING:KnowledgeBaseRecord,StorageBase,)- from ...rag import VectorSearchResult+ from ...rag import ChunkerBase, VectorSearchResultclass KnowledgeBaseService:@@ -65,6 +68,7 @@ class KnowledgeBaseService:blob_store: "BlobStoreBase",message_bus: "MessageBus",resource_access_service: "ResourceAccessService",+ chunkers: "list[type[ChunkerBase]] | None" = None,) -> None:"""Initialize the service.@@ -90,12 +94,19 @@ class KnowledgeBaseService:*and* mutation) goes through it so shared knowledgebases work end-to-end: readers see documents +search hits, editors can also upload / delete.+ chunkers (`list[type[ChunkerBase]] | None`, optional):+ The chunker classes users can choose from when creating+ a knowledge base; used to validate ``chunker_config``.+ Defaults to ``[ApproxTokenChunker]``."""self._storage = storageself._manager = knowledge_base_managerself._blob_store = blob_storeself._bus = message_busself._access = resource_access_service+ self._chunkers_by_type = {+ cls.chunker_type: cls for cls in (chunkers or [ApproxTokenChunker])+ }# ------------------------------------------------------------------# Knowledge base CRUD@@ -107,6 +118,7 @@ class KnowledgeBaseService:name: str,description: str,embedding_model_config: "EmbeddingModelConfig",+ chunker_config: "ChunkerConfig | None" = None,) -> "KnowledgeBaseRecord":"""Delegate creation to the manager, mapping policy errors.@@ -119,6 +131,9 @@ class KnowledgeBaseService:Free-form description.embedding_model_config (`EmbeddingModelConfig`):Embedding model configuration; pinned to the record.+ chunker_config (`ChunkerConfig | None`, optional):+ Chunker configuration; pinned to the record. Defaults+ to the first configured chunker with default parameters.Returns:`KnowledgeBaseRecord`:@@ -128,13 +143,38 @@ class KnowledgeBaseService:`HTTPException`:``409`` when the requested embedding dimensionviolates the manager's dimension policy.+ ``422`` when the chunker type or parameters are+ invalid."""+ if chunker_config is None:+ chunker_config = ChunkerConfig(+ type=next(iter(self._chunkers_by_type)),+ parameters={},+ )+ chunker_cls = self._chunkers_by_type.get(chunker_config.type)+ if chunker_cls is None:+ raise HTTPException(+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,+ detail=(+ f"Unknown chunker type: {chunker_config.type!r}, "+ f"available: {sorted(self._chunkers_by_type)}"+ ),+ )+ try:+ chunker_cls.Parameters(**chunker_config.parameters)+ except (TypeError, ValueError, ValidationError) as exc:+ raise HTTPException(+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,+ detail=f"Invalid chunker parameters: {exc}",+ ) from exc+try:return await self._manager.create_knowledge_base(user_id=user_id,name=name,description=description,embedding_model_config=embedding_model_config,+ chunker_config=chunker_config,)except DimensionPolicyError as exc:raise HTTPException(diff --git a/src/agentscope/app/deps.py b/src/agentscope/app/deps.pyindex 3be59c79e..0eabb8fe3 100644--- a/src/agentscope/app/deps.py+++ b/src/agentscope/app/deps.py@@ -26,7 +26,7 @@ from .message_bus import MessageBusfrom .rag.blob_store import BlobStoreBasefrom .rag.knowledge_base_manager import KnowledgeBaseManagerBasefrom .storage import StorageBase-from ..rag import ParserBase+from ..rag import ChunkerBase, ParserBaseasync def get_current_user_id(@@ -346,6 +346,37 @@ async def get_knowledge_parsers(return parsers+async def get_knowledge_chunkers(+ request: Request,+) -> list[type[ChunkerBase]]:+ """Return the chunker classes configured on the app.++ Args:+ request (`Request`):+ The incoming FastAPI request.++ Returns:+ `list[type[ChunkerBase]]`:+ The chunker classes stored in ``app.state.knowledge_chunkers``+ — the same value the index worker uses to rebuild chunkers.++ Raises:+ `HTTPException`:+ ``503`` when the KB feature is disabled (no chunkers+ configured).+ """+ chunkers = getattr(request.app.state, "knowledge_chunkers", None)+ if not chunkers:+ raise HTTPException(+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,+ detail=(+ "Knowledge base feature is disabled — pass a "+ "knowledge_base_manager to create_app() to enable it."+ ),+ )+ return chunkers++async def get_mcp_hubs(request: Request) -> dict[str, MCPHubBase]:"""Return the registered MCP hubs, keyed by hub id.diff --git a/src/agentscope/app/rag/index_worker/__init__.py b/src/agentscope/app/rag/index_worker/__init__.pyindex bb4612b35..03066610f 100644--- a/src/agentscope/app/rag/index_worker/__init__.py+++ b/src/agentscope/app/rag/index_worker/__init__.py@@ -36,7 +36,7 @@ Example::from agentscope.app.message_bus import RedisMessageBusfrom agentscope.app.storage import RedisStoragefrom agentscope.app.rag.index_worker import run_worker- from agentscope.rag import ApproxTokenChunker, TextParser, ...+ from agentscope.rag import TextParser, ...async def main() -> None:storage = RedisStorage(url=os.environ["REDIS_URL"])@@ -47,14 +47,12 @@ Example::)kb_manager = DefaultKnowledgeBaseManager(...)parsers = [TextParser()]- chunker = ApproxTokenChunker()await run_worker(storage=storage,message_bus=message_bus,blob_store=blob_store,knowledge_base_manager=kb_manager,parsers=parsers,- chunker=chunker,)if __name__ == "__main__":@@ -66,7 +64,7 @@ import socketimport uuidfrom concurrent.futures import ProcessPoolExecutorfrom contextlib import AsyncExitStack-from typing import TYPE_CHECKING+from typing import Any, TYPE_CHECKINGfrom ..._service import IndexTaskConsumer, IndexWorkerfrom ...._logging import logger@@ -86,11 +84,12 @@ async def run_worker(blob_store: "BlobStoreBase",knowledge_base_manager: "KnowledgeBaseManagerBase",parsers: "list[ParserBase] | dict[str, ParserBase]",- chunker: "ChunkerBase",+ chunkers: "list[type[ChunkerBase]] | None" = None,node_id: str | None = None,worker_max_concurrency: int = 4,consumer_max_batch: int = 32,parser_executor: ProcessPoolExecutor | None = None,+ **kwargs: Any,) -> None:"""Run the out-of-process index worker until cancelled.@@ -117,8 +116,10 @@ async def run_worker(``supported_media_types`` (later entries override earlierones, with a warning); dict mode is used verbatim forexplicit routing.- chunker (`ChunkerBase`):- Shared chunker.+ chunkers (`list[type[ChunkerBase]] | None`, optional):+ The chunker classes that can be rebuilt from a knowledge+ base's ``chunker_config``. Defaults to+ ``[ApproxTokenChunker]``.node_id (`str | None`, optional):Stable identifier for this worker process used on thestorage lease. Defaults to@@ -136,6 +137,9 @@ async def run_worker(parser_executor (`ProcessPoolExecutor | None`, optional):Process pool for CPU-bound parses. ``None`` runs parsesinline (fine for text-only deployments).+ **kwargs (`Any`):+ Deprecated arguments forwarded to :class:`IndexWorker`+ (e.g. the old ``chunker`` instance)."""resolved_node_id = (node_id or f"{socket.gethostname()}:{uuid.uuid4().hex[:8]}"@@ -152,10 +156,11 @@ async def run_worker(blob_store=blob_store,knowledge_base_manager=knowledge_base_manager,parsers=parsers,- chunker=chunker,+ chunkers=chunkers,node_id=resolved_node_id,max_concurrency=worker_max_concurrency,parser_executor=parser_executor,+ **kwargs,)await stack.enter_async_context(IndexTaskConsumer(diff --git a/src/agentscope/app/rag/index_worker/__main__.py b/src/agentscope/app/rag/index_worker/__main__.pyindex 805391222..8addeddfc 100644--- a/src/agentscope/app/rag/index_worker/__main__.py+++ b/src/agentscope/app/rag/index_worker/__main__.py@@ -21,7 +21,7 @@ Example bootstrap (``mydeploy/worker_bootstrap.py``)::)from agentscope.app.message_bus import RedisMessageBusfrom agentscope.app.storage import RedisStorage- from agentscope.rag import ApproxTokenChunker, TextParser+ from agentscope.rag import TextParserdef bootstrap() -> dict:return {@@ -33,7 +33,6 @@ Example bootstrap (``mydeploy/worker_bootstrap.py``)::),"knowledge_base_manager": DefaultKnowledgeBaseManager(...),"parsers": [TextParser()],- "chunker": ApproxTokenChunker(),}And launch::diff --git a/src/agentscope/app/rag/knowledge_base_manager/_base.py b/src/agentscope/app/rag/knowledge_base_manager/_base.pyindex d50efa42f..492a21727 100644--- a/src/agentscope/app/rag/knowledge_base_manager/_base.py+++ b/src/agentscope/app/rag/knowledge_base_manager/_base.py@@ -28,6 +28,7 @@ if TYPE_CHECKING:from types import TracebackTypefrom ...storage import (+ ChunkerConfig,EmbeddingModelConfig,KnowledgeBaseRecord,StorageBase,@@ -127,6 +128,7 @@ class KnowledgeBaseManagerBase(ABC):name: str,description: str,embedding_model_config: "EmbeddingModelConfig",+ chunker_config: "ChunkerConfig | None" = None,) -> "KnowledgeBaseRecord":"""Create a new knowledge base for the given user.@@ -147,6 +149,9 @@ class KnowledgeBaseManagerBase(ABC):Free-form description.embedding_model_config (`EmbeddingModelConfig`):Embedding model configuration; pinned to the record.+ chunker_config (`ChunkerConfig | None`, optional):+ Chunker configuration; pinned to the record. ``None``+ leaves the record without a pinned chunker (legacy).Returns:`KnowledgeBaseRecord`:diff --git a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.pyindex aa0ca3280..8ad9b560b 100644--- a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py+++ b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py@@ -25,7 +25,7 @@ from ..._service._embedding import build_embedding_modelfrom ...storage import KnowledgeBaseRecord, KnowledgeBaseDataif TYPE_CHECKING:- from ...storage import EmbeddingModelConfig+ from ...storage import ChunkerConfig, EmbeddingModelConfigclass CollectionPerKbManager(KnowledgeBaseManagerBase):@@ -55,6 +55,7 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):name: str,description: str,embedding_model_config: "EmbeddingModelConfig",+ chunker_config: "ChunkerConfig | None" = None,) -> KnowledgeBaseRecord:"""Allocate a new collection and persist the knowledge base record.@@ -67,6 +68,9 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):Free-form description.embedding_model_config (`EmbeddingModelConfig`):Embedding model configuration; pinned to the record.+ chunker_config (`ChunkerConfig | None`, optional):+ Chunker configuration; pinned to the record. ``None``+ leaves the record without a pinned chunker (legacy).Returns:`KnowledgeBaseRecord`:@@ -78,6 +82,7 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):name=name,description=description,embedding_model_config=embedding_model_config,+ chunker_config=chunker_config,collection_name="",),)diff --git a/src/agentscope/app/storage/__init__.py b/src/agentscope/app/storage/__init__.pyindex 3fee8470c..8db3e1b4b 100644--- a/src/agentscope/app/storage/__init__.py+++ b/src/agentscope/app/storage/__init__.py@@ -9,6 +9,7 @@ from ._model import (AgentRecord,ChannelBinding,ChannelRecord,+ ChunkerConfig,RoutingConfig,SessionScope,SessionSettings,@@ -74,6 +75,7 @@ __all__ = ["AgentRecord","ChannelBinding","ChannelRecord",+ "ChunkerConfig","RoutingConfig","SessionScope","SessionSettings",diff --git a/src/agentscope/app/storage/_model/__init__.py b/src/agentscope/app/storage/_model/__init__.pyindex 614dc0e9c..f31a3ffa1 100644--- a/src/agentscope/app/storage/_model/__init__.py+++ b/src/agentscope/app/storage/_model/__init__.py@@ -10,7 +10,11 @@ from ._channel import (SessionSettings,)from ._credential import CredentialRecord-from ._knowledge_base import KnowledgeBaseData, KnowledgeBaseRecord+from ._knowledge_base import (+ ChunkerConfig,+ KnowledgeBaseData,+ KnowledgeBaseRecord,+)from ._knowledge_document import (KnowledgeDocumentData,KnowledgeDocumentRecord,@@ -36,6 +40,7 @@ __all__ = ["AgentRecord","ChannelBinding","ChannelRecord",+ "ChunkerConfig","RoutingConfig","SessionScope","SessionSettings",diff --git a/src/agentscope/app/storage/_model/_knowledge_base.py b/src/agentscope/app/storage/_model/_knowledge_base.pyindex 2e9be619a..1f9c7ede1 100644--- a/src/agentscope/app/storage/_model/_knowledge_base.py+++ b/src/agentscope/app/storage/_model/_knowledge_base.py@@ -8,6 +8,28 @@ from ._base import _RecordBasefrom ._session import EmbeddingModelConfig+class ChunkerConfig(BaseModel):+ """Chunker configuration persisted alongside a knowledge base.++ Stores the chunker ``type`` identifier (as declared by+ ``ChunkerBase.chunker_type``) and the keyword-argument+ ``parameters`` used to reconstruct it at indexing time.+ """++ type: str = Field(+ description=(+ "Chunker type identifier, as declared by "+ "``ChunkerBase.chunker_type``."+ ),+ )+ parameters: dict = Field(+ default_factory=dict,+ description=(+ "Keyword arguments used to build ``ChunkerBase.Parameters``."+ ),+ )++class KnowledgeBaseData(BaseModel):"""The mutable payload of a knowledge base record.@@ -38,6 +60,21 @@ class KnowledgeBaseData(BaseModel):)"""Embedding model configuration pinned at creation time."""+ chunker_config: ChunkerConfig | None = Field(+ default=None,+ description=(+ "Chunker configuration pinned at creation time. "+ "``None`` only for legacy records created before "+ "per-KB chunker support was introduced; the index "+ "worker falls back to ``ApproxTokenChunker()`` in "+ "that case. New records always have an explicit value."+ ),+ )+ """Chunker configuration pinned at creation time.++ ``None`` only for legacy records; new creations always set this.+ """+collection_name: str = Field(description=("The vector store collection that physically backs this "@@ -71,9 +108,9 @@ class KnowledgeBaseRecord(_RecordBase):Pre-refactor records stored the payload fields (``name`` /``description`` / ``embedding_model_config`` /- ``collection_name``) at the record top level rather than- under ``data``. This validator runs in ``mode='before'`` so- it hooks every pydantic entry point —+ ``chunker_config`` / ``collection_name``) at the record top+ level rather than under ``data``. This validator runs in+ ``mode='before'`` so it hooks every pydantic entry point —:meth:`~pydantic.BaseModel.model_validate`,:meth:`~pydantic.BaseModel.model_validate_json` (the paththe storage backend takes when reading from Redis / SQL),@@ -97,6 +134,7 @@ class KnowledgeBaseRecord(_RecordBase):"name","description","embedding_model_config",+ "chunker_config","collection_name",)if not any(key in obj for key in legacy_keys):diff --git a/src/agentscope/rag/_chunker/_approx_token_chunker.py b/src/agentscope/rag/_chunker/_approx_token_chunker.pyindex fe26674d4..3ecf52261 100644--- a/src/agentscope/rag/_chunker/_approx_token_chunker.py+++ b/src/agentscope/rag/_chunker/_approx_token_chunker.py@@ -7,9 +7,13 @@ within the right order of magnitude for most LLM tokenizers."""from bisect import bisect_rightfrom itertools import accumulate+from typing import Any++from pydantic import Field, model_validatorfrom ._base import ChunkerBasefrom .._document import Chunk, Section+from ..._logging import loggerfrom ...message import TextBlock, DataBlock@@ -30,35 +34,97 @@ class ApproxTokenChunker(ChunkerBase):required by :class:`ChunkerBase`."""- def __init__(self, chunk_size: int = 512, overlap: int = 50) -> None:+ chunker_type = "approx_token"++ class Parameters(ChunkerBase.Parameters):+ """The tunable parameters of the approximate-token chunker."""++ chunk_size: int = Field(+ default=512,+ ge=1,+ title="Chunk Size",+ description="Maximum number of approximate tokens per chunk.",+ )+ overlap: int = Field(+ default=50,+ ge=0,+ title="Overlap",+ description=(+ "Number of approximate tokens shared between "+ "consecutive chunks."+ ),+ )++ @model_validator(mode="after")+ def _overlap_less_than_chunk_size(+ self,+ ) -> "ApproxTokenChunker.Parameters":+ if self.overlap >= self.chunk_size:+ raise ValueError(+ "overlap must be less than chunk_size, got "+ f"overlap={self.overlap}, chunk_size={self.chunk_size}.",+ )+ return self++ def __init__(+ self,+ parameters: "ApproxTokenChunker.Parameters | None" = None,+ **kwargs: Any,+ ) -> None:"""Initialize the approx token chunker.Args:- chunk_size (`int`, defaults to `512`):- The maximum number of approximate tokens per chunk.- Must be a positive integer.- overlap (`int`, defaults to `50`):- The number of approximate tokens shared between two- consecutive chunks. Must be non-negative and smaller- than ``chunk_size``.+ parameters (`ApproxTokenChunker.Parameters | None`, optional):+ The chunker parameters (``chunk_size`` and ``overlap``).+ Defaults to ``Parameters()`` when not provided.+ **kwargs (`Any`):+ Deprecated. ``chunk_size`` and ``overlap`` are still+ accepted for backward compatibility and override the+ corresponding fields in ``parameters``; other keys are+ ignored.Raises:+ `TypeError`:+ If ``parameters`` is not a ``Parameters`` instance.`ValueError`:If ``chunk_size`` is not positive, or ``overlap`` isnegative or not smaller than ``chunk_size``."""- if chunk_size <= 0:- raise ValueError(- f"chunk_size must be positive, got {chunk_size}.",+ if parameters is not None and not isinstance(+ parameters,+ self.Parameters,+ ):+ raise TypeError(+ "The first argument of ApproxTokenChunker must be an "+ "ApproxTokenChunker.Parameters instance, got "+ f"{type(parameters).__name__}. Use keyword arguments "+ "chunk_size=/overlap= or parameters=Parameters(...).",)- if overlap < 0 or overlap >= chunk_size:- raise ValueError(- "overlap must satisfy 0 <= overlap < chunk_size, "- f"got overlap={overlap}, chunk_size={chunk_size}.",+ legacy = {+ key: kwargs[key]+ for key in ("chunk_size", "overlap")+ if key in kwargs+ }+ if legacy:+ logger.warning(+ "Passing %s to ApproxTokenChunker directly is deprecated, "+ "use ApproxTokenChunker.Parameters instead.",+ ", ".join(f"``{k}``" for k in legacy),)+ base = parameters.model_dump() if parameters is not None else {}+ parameters = self.Parameters(**{**base, **legacy})++ super().__init__(parameters)++ @property+ def chunk_size(self) -> int:+ """The maximum number of approximate tokens per chunk."""+ return self.parameters.chunk_size- self.chunk_size = chunk_size- self.overlap = overlap+ @property+ def overlap(self) -> int:+ """The number of approximate tokens shared between chunks."""+ return self.parameters.overlapasync def chunk(self, sections: list[Section]) -> list[Chunk]:"""Chunk the input sections into smaller chunks based on an approxdiff --git a/src/agentscope/rag/_chunker/_base.py b/src/agentscope/rag/_chunker/_base.pyindex a766d20ac..c80c16075 100644--- a/src/agentscope/rag/_chunker/_base.py+++ b/src/agentscope/rag/_chunker/_base.py@@ -18,6 +18,8 @@ guarantee preserves the structural metadata attached by the Parser"""from abc import ABC, abstractmethod+from pydantic import BaseModel+from .._document import Chunk, Section@@ -29,6 +31,9 @@ class ChunkerBase(ABC):chunker is configured once at construction time and reusedacross many ``chunk()`` calls within the same knowledge base.+ Subclasses must set :attr:`chunker_type` and may override the+ :class:`Parameters` inner class to declare their tunable parameters.+Subclasses must guarantee:- **No cross-Section merging**: every output :class:`Chunk` is@@ -45,6 +50,27 @@ class ChunkerBase(ABC):``metadata`` are copied from its parent Section."""+ chunker_type: str+ """The unique identifier of the chunking strategy."""++ class Parameters(BaseModel):+ """The tunable parameters of the chunker."""++ def __init__(+ self,+ parameters: "ChunkerBase.Parameters | None" = None,+ ) -> None:+ """Initialize the chunker.++ Args:+ parameters (`ChunkerBase.Parameters | None`, optional):+ The chunker parameters. Defaults to+ ``self.Parameters()`` when not provided.+ """+ self.parameters = (+ parameters if parameters is not None else self.Parameters()+ )+@abstractmethodasync def chunk(self, sections: list[Section]) -> list[Chunk]:"""Split a list of Sections into Chunks.diff --git a/tests/app_lifespan_dedicated_test.py b/tests/app_lifespan_dedicated_test.pyindex 2aa658da8..4addbee66 100644--- a/tests/app_lifespan_dedicated_test.py+++ b/tests/app_lifespan_dedicated_test.py@@ -46,6 +46,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import ()from agentscope.app.message_bus import RedisMessageBusfrom agentscope.app.storage import (+ ChunkerConfig,EmbeddingModelConfig,KnowledgeBaseData,KnowledgeBaseRecord,@@ -129,6 +130,7 @@ class _FakeKbManager(KnowledgeBaseManagerBase):name: str,description: str,embedding_model_config: EmbeddingModelConfig,+ chunker_config: ChunkerConfig | None = None,) -> KnowledgeBaseRecord:raise NotImplementedErrordiff --git a/tests/resource_access_policy_test.py b/tests/resource_access_policy_test.pyindex 46bcdf77f..8d754b3c0 100644--- a/tests/resource_access_policy_test.py+++ b/tests/resource_access_policy_test.py@@ -418,6 +418,7 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):"dimensions": 1536,"parameters": {},},+ "chunker_config": None,"created_at": AnyValue(),"updated_at": AnyValue(),"editable": True,@@ -445,6 +446,7 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):"dimensions": 1536,"parameters": {},},+ "chunker_config": None,"created_at": AnyValue(),"updated_at": AnyValue(),"editable": True,diff --git a/tests/service_knowledge_base_upload_test.py b/tests/service_knowledge_base_upload_test.pyindex a01845ef2..3e4c58d10 100644--- a/tests/service_knowledge_base_upload_test.py+++ b/tests/service_knowledge_base_upload_test.py@@ -35,6 +35,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import ()from agentscope.app.message_bus import RedisMessageBusfrom agentscope.app.storage import (+ ChunkerConfig,EmbeddingModelConfig,KnowledgeBaseData,KnowledgeBaseRecord,@@ -206,6 +207,7 @@ class _FakeKbManager(KnowledgeBaseManagerBase):name: str,description: str,embedding_model_config: EmbeddingModelConfig,+ chunker_config: ChunkerConfig | None = None,) -> KnowledgeBaseRecord:record = KnowledgeBaseRecord(user_id=user_id,