Skip to content

Build and run agents you can see, understand and trust.

feat(rag): make chunker config explicit and schema-driven (#2083)

qbc7cee10c3bf5f
diff --git a/examples/agent_service/main.py b/examples/agent_service/main.py
index a41baba1d..9a6a0bbf3 100644
--- a/examples/agent_service/main.py
+++ b/examples/agent_service/main.py
@@ -16,7 +16,7 @@ from agentscope.app.workspace_manager import LocalWorkspaceManager
from agentscope.mcp import MCPClient, StdioMCPConfig, HttpMCPConfig
from agentscope.middleware import AgenticMemoryMiddleware, MiddlewareBase
from agentscope.permission import PermissionContext, PermissionMode
-from agentscope.rag import QdrantStore
+from agentscope.rag import ApproxTokenChunker, QdrantStore
from agentscope.workspace import WorkspaceBase
default_mcps = [
@@ -94,6 +94,9 @@ app = create_app(
storage=storage,
vector_store=vector_store,
),
+ # Chunker classes users can pick from when creating a knowledge base;
+ # the chosen type and parameters are pinned on the knowledge base.
+ knowledge_chunkers=[ApproxTokenChunker],
# Resource hubs the UI browses under /hub. Neither needs credentials
# of its own — an individual MCP card declares whatever key it wants
# from the user in its ``inputs_schema``. Passing a ClawHub token
diff --git a/examples/web_ui/frontend/src/api/knowledgeBase.ts b/examples/web_ui/frontend/src/api/knowledgeBase.ts
index b6f1f1f63..202e5b1ce 100644
--- a/examples/web_ui/frontend/src/api/knowledgeBase.ts
+++ b/examples/web_ui/frontend/src/api/knowledgeBase.ts
@@ -4,6 +4,7 @@ import type {
CreateKnowledgeBaseResponse,
KbMiddlewareParametersSchemaResponse,
KnowledgeBaseView,
+ ListChunkersResponse,
ListKbEmbeddingModelsResponse,
ListKnowledgeBasesResponse,
ListKnowledgeDocumentsResponse,
@@ -122,6 +123,8 @@ export const knowledgeBaseApi = {
listEmbeddingModels: () =>
client.get<ListKbEmbeddingModelsResponse>('/knowledge_bases/embedding_models'),
+ listChunkers: () => client.get<ListChunkersResponse>('/knowledge_bases/chunkers'),
+
/** Fetch the JSON Schema describing the KB middleware's tunable params. */
middlewareParametersSchema: () =>
client.get<KbMiddlewareParametersSchemaResponse>(
diff --git a/examples/web_ui/frontend/src/api/types.ts b/examples/web_ui/frontend/src/api/types.ts
index ef4997d67..06f423da8 100644
--- a/examples/web_ui/frontend/src/api/types.ts
+++ b/examples/web_ui/frontend/src/api/types.ts
@@ -838,6 +838,22 @@ export interface ListEmbeddingModelResponse {
total: number;
}
+// ─── Chunker ──────────────────────────────────────────────────────────────────
+
+export interface ChunkerConfig {
+ type: string;
+ parameters: Record<string, unknown>;
+}
+
+export interface ChunkerInfo {
+ type: string;
+ parameter_schema: JSONSchema;
+}
+
+export interface ListChunkersResponse {
+ chunkers: ChunkerInfo[];
+}
+
// ─── Knowledge Base ───────────────────────────────────────────────────────────
/**
@@ -849,6 +865,7 @@ export interface KnowledgeBaseView {
name: string;
description: string;
embedding_model_config: EmbeddingModelConfig;
+ chunker_config?: ChunkerConfig;
created_at: string;
updated_at: string;
/**
@@ -868,6 +885,7 @@ export interface CreateKnowledgeBaseRequest {
name: string;
description?: string;
embedding_model_config: EmbeddingModelConfig;
+ chunker_config: ChunkerConfig;
}
export interface CreateKnowledgeBaseResponse {
diff --git a/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx b/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx
index bf6a4fc1e..ce89ae167 100644
--- a/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx
+++ b/examples/web_ui/frontend/src/components/dialog/CreateKnowledgeBaseDialog.tsx
@@ -1,7 +1,19 @@
import { CircleAlert, Info, Loader2, PlusCircle } from 'lucide-react';
-import { useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
-import type { EmbeddingModelCard, EmbeddingModelConfig } from '@/api';
+import type {
+ ChunkerConfig,
+ ChunkerInfo,
+ EmbeddingModelCard,
+ EmbeddingModelConfig,
+ JSONSchema,
+} from '@/api';
+import {
+ type SchemaFormValue,
+ SchemaForm,
+ defaultValuesFromSchema,
+} from '@/components/form/SchemaForm';
+import { ChunkerSelect } from '@/components/select/ChunkerSelect';
import { DimensionSelect } from '@/components/select/DimensionSelect';
import { EmbeddingSelect } from '@/components/select/EmbeddingSelect';
import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert.tsx';
@@ -16,7 +28,9 @@ import {
} from '@/components/ui/dialog.tsx';
import { Field, FieldGroup, FieldLabel } from '@/components/ui/field.tsx';
import { Input } from '@/components/ui/input.tsx';
+import { Separator } from '@/components/ui/separator.tsx';
import { Textarea } from '@/components/ui/textarea.tsx';
+import { useChunkers } from '@/hooks/useChunkers';
import { useKbEmbeddingModels } from '@/hooks/useKbEmbeddingModels';
import { useKnowledgeBases } from '@/hooks/useKnowledgeBases';
import { useTranslation } from '@/i18n/useI18n.ts';
@@ -33,6 +47,10 @@ interface Props {
credentialRefetchTrigger?: number;
}
+const NO_SKIP = new Set<string>();
+/** Same control width for every label-left field, as in the schedule dialog. */
+const HORIZONTAL_FIELD_WIDTH = '[&>[data-orientation=horizontal]>:last-child]:w-48';
+
interface SelectedEmbedding {
type: string;
credentialId: string;
@@ -50,20 +68,58 @@ export function CreateKnowledgeBaseDialog({
const { t } = useTranslation();
const { create } = useKnowledgeBases();
const { providers, policy, loading } = useKbEmbeddingModels(credentialRefetchTrigger);
+ const { chunkers } = useChunkers();
const [name, setName] = useState('');
const [description, setDescription] = useState('');
const [selected, setSelected] = useState<SelectedEmbedding | null>(null);
const [dimension, setDimension] = useState<number | null>(null);
+ const [selectedChunkerType, setSelectedChunkerType] = useState<string>('');
+ const [chunkerParams, setChunkerParams] = useState<Record<string, SchemaFormValue>>({});
const [submitting, setSubmitting] = useState(false);
const [errorKey, setErrorKey] = useState<string | null>(null);
+ const selectedChunker = useMemo<ChunkerInfo | null>(
+ () => chunkers.find((c) => c.type === selectedChunkerType) ?? null,
+ [chunkers, selectedChunkerType],
+ );
+
+ const chunkerParamSchema = useMemo<JSONSchema | null>(
+ () => selectedChunker?.parameter_schema ?? null,
+ [selectedChunker],
+ );
+
+ const handleSelectChunker = useCallback(
+ (type: string) => {
+ setSelectedChunkerType(type);
+ const info = chunkers.find((c) => c.type === type);
+ const schema = info?.parameter_schema;
+ setChunkerParams(schema ? defaultValuesFromSchema(schema, NO_SKIP) : {});
+ },
+ [chunkers],
+ );
+
+ const handleChunkerParamChange = useCallback((key: string, value: SchemaFormValue) => {
+ setChunkerParams((prev) => ({ ...prev, [key]: value }));
+ }, []);
+
+ // Default to the first available chunker so the form is submittable
+ // without an explicit pick; re-pick when the list changes underneath.
+ useEffect(() => {
+ if (!open || chunkers.length === 0) return;
+ if (!chunkers.some((c) => c.type === selectedChunkerType)) {
+ handleSelectChunker(chunkers[0].type);
+ }
+ }, [open, chunkers, selectedChunkerType, handleSelectChunker]);
+
useEffect(() => {
if (!open) {
setName('');
setDescription('');
setSelected(null);
setDimension(null);
+ setSelectedChunkerType('');
+ setChunkerParams({});
setErrorKey(null);
setSubmitting(false);
}
@@ -100,6 +156,10 @@ export function CreateKnowledgeBaseDialog({
setErrorKey('dialog-knowledge-base-create.errors.dimensionRequired');
return;
}
+ if (!selectedChunkerType) {
+ setErrorKey('dialog-knowledge-base-create.errors.chunkerRequired');
+ return;
+ }
setErrorKey(null);
setSubmitting(true);
try {
@@ -110,10 +170,19 @@ export function CreateKnowledgeBaseDialog({
dimensions: dimension,
parameters: {},
};
+ const params: Record<string, unknown> = {};
+ for (const [k, v] of Object.entries(chunkerParams)) {
+ if (v !== undefined && v !== null && v !== '') params[k] = v;
+ }
+ const chunkerConfig: ChunkerConfig = {
+ type: selectedChunkerType,
+ parameters: params,
+ };
const knowledgeBaseId = await create({
name: name.trim(),
description: description.trim(),
embedding_model_config: config,
+ chunker_config: chunkerConfig,
});
onCreated?.(knowledgeBaseId);
onOpenChange(false);
@@ -167,7 +236,7 @@ export function CreateKnowledgeBaseDialog({
</Alert>
)}
- <FieldGroup>
+ <FieldGroup className={HORIZONTAL_FIELD_WIDTH}>
<Field>
<FieldLabel>{t('dialog-knowledge-base-create.name.label')}</FieldLabel>
<Input
@@ -191,6 +260,9 @@ export function CreateKnowledgeBaseDialog({
rows={3}
/>
</Field>
+
+ <Separator />
+
<Field orientation="horizontal">
<FieldLabel>
{t('dialog-knowledge-base-create.embeddingModel.label')}
@@ -220,6 +292,53 @@ export function CreateKnowledgeBaseDialog({
disabled={submitting}
/>
</Field>
+
+ {chunkers.length > 0 && (
+ <>
+ <Separator />
+
+ <Field orientation="horizontal">
+ <FieldLabel>
+ {t('dialog-knowledge-base-create.chunker.label')}
+ </FieldLabel>
+ <ChunkerSelect
+ value={selectedChunkerType}
+ chunkers={chunkers}
+ onChange={handleSelectChunker}
+ disabled={submitting}
+ />
+ </Field>
+
+ {chunkerParamSchema && (
+ <SchemaForm
+ schema={chunkerParamSchema}
+ values={chunkerParams}
+ onChange={handleChunkerParamChange}
+ skipFields={NO_SKIP}
+ idPrefix="chunker-param"
+ orientation="horizontal"
+ className={HORIZONTAL_FIELD_WIDTH}
+ labelFor={(key, prop) =>
+ t(
+ `chunker-types.${selectedChunkerType}.params.${key}.label`,
+ { defaultValue: '' },
+ ) ||
+ prop.title ||
+ undefined
+ }
+ descriptionFor={(key, prop) =>
+ t(
+ `chunker-types.${selectedChunkerType}.params.${key}.description`,
+ { defaultValue: '' },
+ ) ||
+ prop.description ||
+ undefined
+ }
+ />
+ )}
+ </>
+ )}
+
{errorKey && <p className="text-destructive text-sm">{t(errorKey)}</p>}
</FieldGroup>
<DialogFooter>
diff --git a/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx b/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx
index 7ffd58e5a..0fb9c268d 100644
--- a/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx
+++ b/examples/web_ui/frontend/src/components/dialog/EditKnowledgeBaseDialog.tsx
@@ -72,6 +72,21 @@ export function EditKnowledgeBaseDialog({ open, onOpenChange, knowledgeBase, onU
? `${knowledgeBase.embedding_model_config.model} · ${knowledgeBase.embedding_model_config.dimensions}d`
: '';
+ const chunkerLabel = (() => {
+ if (!knowledgeBase?.chunker_config) return null;
+ const cfg = knowledgeBase.chunker_config;
+ const typeLabel = t(`chunker-types.${cfg.type}.label`, { defaultValue: cfg.type });
+ const paramStr = Object.entries(cfg.parameters)
+ .map(([k, v]) => {
+ const paramLabel = t(`chunker-types.${cfg.type}.params.${k}.label`, {
+ defaultValue: k,
+ });
+ return `${paramLabel}=${v}`;
+ })
+ .join(', ');
+ return paramStr ? `${typeLabel} (${paramStr})` : typeLabel;
+ })();
+
return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="!w-[500px] !max-w-[500px]">
@@ -113,6 +128,14 @@ export function EditKnowledgeBaseDialog({ open, onOpenChange, knowledgeBase, onU
{embeddingModelLabel}
</Badge>
</Field>
+ {chunkerLabel && (
+ <Field orientation="horizontal">
+ <FieldLabel>{t('dialog-knowledge-base-edit.chunker.label')}</FieldLabel>
+ <Badge variant="secondary" className="font-mono">
+ {chunkerLabel}
+ </Badge>
+ </Field>
+ )}
{errorKey && <p className="text-destructive text-sm">{t(errorKey)}</p>}
</FieldGroup>
<DialogFooter>
diff --git a/examples/web_ui/frontend/src/components/form/SchemaForm.tsx b/examples/web_ui/frontend/src/components/form/SchemaForm.tsx
index d55ee8857..b64d1ffef 100644
--- a/examples/web_ui/frontend/src/components/form/SchemaForm.tsx
+++ b/examples/web_ui/frontend/src/components/form/SchemaForm.tsx
@@ -1,6 +1,14 @@
+import type { ReactNode } from 'react';
+
import type { JSONSchema, JSONSchemaProperty } from '@/api';
import { Checkbox } from '@/components/ui/checkbox';
-import { Field, FieldDescription, FieldGroup, FieldLabel } from '@/components/ui/field.tsx';
+import {
+ Field,
+ FieldContent,
+ FieldDescription,
+ FieldGroup,
+ FieldLabel,
+} from '@/components/ui/field.tsx';
import { Input } from '@/components/ui/input';
import {
Select,
@@ -29,6 +37,10 @@ interface Props {
descriptionFor?: (key: string, prop: JSONSchemaProperty) => string | undefined;
/** Prefix for generated DOM IDs (so multiple SchemaForms on one page don't collide). */
idPrefix?: string;
+ /** Layout of non-boolean fields. `horizontal` puts the label left of the control. */
+ orientation?: 'vertical' | 'horizontal';
+ /** Extra classes for the wrapping FieldGroup. */
+ className?: string;
}
function effectiveType(prop: JSONSchemaProperty): string {
@@ -77,13 +89,15 @@ export function SchemaForm({
placeholderFor,
descriptionFor,
idPrefix = 'schema-form',
+ orientation = 'vertical',
+ className,
}: Props) {
const entries = Object.entries(schema.properties ?? {}).filter(
([key, prop]) => !skipFields.has(key) && prop.const === undefined,
);
return (
- <FieldGroup>
+ <FieldGroup className={className}>
{entries.map(([key, prop]) => {
const fieldId = `${idPrefix}-${key}`;
const isRequired = schema.required?.includes(key) ?? false;
@@ -99,6 +113,33 @@ export function SchemaForm({
const description = descriptionFor?.(key, prop);
const current = values[key];
+ const labelNode = (
+ <FieldLabel htmlFor={fieldId}>
+ {label}
+ {isRequired && <span className="text-destructive ml-0.5">*</span>}
+ </FieldLabel>
+ );
+ const descriptionNode = description ? (
+ <FieldDescription>{description}</FieldDescription>
+ ) : null;
+ /** Label (+ helper text) left of the control when horizontal. */
+ const wrap = (control: ReactNode) =>
+ orientation === 'horizontal' ? (
+ <Field key={key} orientation="horizontal">
+ <FieldContent>
+ {labelNode}
+ {descriptionNode}
+ </FieldContent>
+ {control}
+ </Field>
+ ) : (
+ <Field key={key}>
+ {labelNode}
+ {control}
+ {descriptionNode}
+ </Field>
+ );
+
if (isBoolean) {
return (
<Field key={key} orientation="horizontal">
@@ -118,101 +159,71 @@ export function SchemaForm({
if (enumOpts) {
const currentStr =
current === undefined || current === null ? '' : String(current);
- return (
- <Field key={key}>
- <FieldLabel htmlFor={fieldId}>
- {label}
- {isRequired && <span className="text-destructive ml-0.5">*</span>}
- </FieldLabel>
- <Select value={currentStr} onValueChange={(v) => onChange(key, v)}>
- <SelectTrigger id={fieldId} className="w-full">
- <SelectValue placeholder={placeholder} />
- </SelectTrigger>
- <SelectContent>
- {enumOpts.map((opt) => (
- <SelectItem key={String(opt)} value={String(opt)}>
- {String(opt)}
- </SelectItem>
- ))}
- </SelectContent>
- </Select>
- {description && <FieldDescription>{description}</FieldDescription>}
- </Field>
+ return wrap(
+ <Select value={currentStr} onValueChange={(v) => onChange(key, v)}>
+ <SelectTrigger id={fieldId} className="w-full">
+ <SelectValue placeholder={placeholder} />
+ </SelectTrigger>
+ <SelectContent>
+ {enumOpts.map((opt) => (
+ <SelectItem key={String(opt)} value={String(opt)}>
+ {String(opt)}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>,
);
}
if (isTextarea) {
- return (
- <Field key={key}>
- <FieldLabel htmlFor={fieldId}>
- {label}
- {isRequired && <span className="text-destructive ml-0.5">*</span>}
- </FieldLabel>
- <Textarea
- id={fieldId}
- rows={3}
- value={(current as string | undefined) ?? ''}
- onChange={(e) => onChange(key, e.target.value)}
- placeholder={placeholder}
- />
- {description && <FieldDescription>{description}</FieldDescription>}
- </Field>
+ return wrap(
+ <Textarea
+ id={fieldId}
+ rows={3}
+ value={(current as string | undefined) ?? ''}
+ onChange={(e) => onChange(key, e.target.value)}
+ placeholder={placeholder}
+ />,
);
}
if (isNumber) {
const min = prop.minimum ?? prop.exclusiveMinimum;
const max = prop.maximum ?? prop.exclusiveMaximum;
- return (
- <Field key={key}>
- <FieldLabel htmlFor={fieldId}>
- {label}
- {isRequired && <span className="text-destructive ml-0.5">*</span>}
- </FieldLabel>
- <Input
- id={fieldId}
- type="number"
- min={min}
- max={max}
- step={inferStep(type)}
- value={
- current === undefined || current === null ? '' : String(current)
+ return wrap(
+ <Input
+ id={fieldId}
+ type="number"
+ min={min}
+ max={max}
+ step={inferStep(type)}
+ value={current === undefined || current === null ? '' : String(current)}
+ onChange={(e) => {
+ const raw = e.target.value;
+ // Empty input → undefined so JSON.stringify drops the key
+ // and the backend applies its own default. Sending "" would
+ // fail Pydantic float/int coercion.
+ if (raw === '') {
+ onChange(key, undefined);
+ return;
}
- onChange={(e) => {
- const raw = e.target.value;
- // Empty input → undefined so JSON.stringify drops the key
- // and the backend applies its own default. Sending "" would
- // fail Pydantic float/int coercion.
- if (raw === '') {
- onChange(key, undefined);
- return;
- }
- const parsed =
- type === 'integer' ? parseInt(raw, 10) : parseFloat(raw);
- onChange(key, Number.isNaN(parsed) ? raw : parsed);
- }}
- placeholder={placeholder}
- />
- {description && <FieldDescription>{description}</FieldDescription>}
- </Field>
+ const parsed =
+ type === 'integer' ? parseInt(raw, 10) : parseFloat(raw);
+ onChange(key, Number.isNaN(parsed) ? raw : parsed);
+ }}
+ placeholder={placeholder}
+ />,
);
}
- return (
- <Field key={key}>
- <FieldLabel htmlFor={fieldId}>
- {label}
- {isRequired && <span className="text-destructive ml-0.5">*</span>}
- </FieldLabel>
- <Input
- id={fieldId}
- type={isPassword ? 'password' : 'text'}
- value={(current as string | undefined) ?? ''}
- onChange={(e) => onChange(key, e.target.value)}
- placeholder={placeholder}
- />
- {description && <FieldDescription>{description}</FieldDescription>}
- </Field>
+ return wrap(
+ <Input
+ id={fieldId}
+ type={isPassword ? 'password' : 'text'}
+ value={(current as string | undefined) ?? ''}
+ onChange={(e) => onChange(key, e.target.value)}
+ placeholder={placeholder}
+ />,
);
})}
</FieldGroup>
diff --git a/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx b/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx
index c8fedce5e..022c80b81 100644
--- a/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx
+++ b/examples/web_ui/frontend/src/components/knowledge/KnowledgeDocumentsPanel.tsx
@@ -75,7 +75,7 @@ function StatusBadge({ phase }: { phase: UploadPhase }) {
return (
<span
className={cn(
- 'inline-flex items-center gap-x-1 rounded-md px-1.5 py-0.5 text-[10px] font-medium',
+ 'inline-flex shrink-0 items-center gap-x-1 rounded-md px-1.5 py-0.5 text-[10px] font-medium whitespace-nowrap',
tone,
)}
>
diff --git a/examples/web_ui/frontend/src/components/select/ChunkerSelect.tsx b/examples/web_ui/frontend/src/components/select/ChunkerSelect.tsx
new file mode 100644
index 000000000..b08fc0f76
--- /dev/null
+++ b/examples/web_ui/frontend/src/components/select/ChunkerSelect.tsx
@@ -0,0 +1,74 @@
+import { ChevronDown } from 'lucide-react';
+
+import type { ChunkerInfo } from '@/api';
+import { Button } from '@/components/ui/button';
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuTrigger,
+} from '@/components/ui/dropdown-menu';
+import { useTranslation } from '@/i18n/useI18n.ts';
+
+interface Props {
+ value: string;
+ chunkers: ChunkerInfo[];
+ loading?: boolean;
+ onChange?: (type: string) => void;
+ disabled?: boolean;
+ placeholder?: string;
+}
+
+/**
+ * Chunker type picker. Mirrors the same visual style used by
+ * :component:`EmbeddingSelect` so the two selectors look consistent
+ * when placed next to each other in the knowledge-base creation dialog.
+ */
+export function ChunkerSelect({
+ value,
+ chunkers,
+ loading,
+ onChange,
+ disabled,
+ placeholder,
+}: Props) {
+ const { t } = useTranslation();
+
+ const localizedLabel = (type: string) =>
+ t(`chunker-types.${type}.label`, { defaultValue: type });
+
+ const displayLabel = value
+ ? localizedLabel(value)
+ : loading
+ ? t('chunker-select.loading')
+ : (placeholder ?? t('chunker-select.placeholder'));
+
+ return (
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <Button
+ variant="outline"
+ size="sm"
+ className="justify-between gap-1 font-normal"
+ disabled={disabled}
+ >
+ <span className="truncate">{displayLabel}</span>
+ <ChevronDown className="size-3.5 text-muted-foreground" />
+ </Button>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent align="start" className="min-w-48 max-h-72 overflow-y-auto">
+ {!loading && chunkers.length === 0 ? (
+ <div className="px-2 py-3 text-center text-sm text-muted-foreground">
+ {t('chunker-select.empty')}
+ </div>
+ ) : (
+ chunkers.map((c) => (
+ <DropdownMenuItem key={c.type} onSelect={() => onChange?.(c.type)}>
+ {localizedLabel(c.type)}
+ </DropdownMenuItem>
+ ))
+ )}
+ </DropdownMenuContent>
+ </DropdownMenu>
+ );
+}
diff --git a/examples/web_ui/frontend/src/hooks/useChunkers.ts b/examples/web_ui/frontend/src/hooks/useChunkers.ts
new file mode 100644
index 000000000..db42204e9
--- /dev/null
+++ b/examples/web_ui/frontend/src/hooks/useChunkers.ts
@@ -0,0 +1,34 @@
+import { useCallback, useEffect, useState } from 'react';
+
+import { knowledgeBaseApi } from '@/api';
+import type { ChunkerInfo } from '@/api';
+
+/**
+ * Fetches the list of registered chunker types and their parameter
+ * schemas. Used by the create-KB dialog to render the chunker
+ * selector and dynamic parameter form.
+ */
+export function useChunkers() {
+ const [chunkers, setChunkers] = useState<ChunkerInfo[]>([]);
+ const [loading, setLoading] = useState(false);
+
+ const refetch = useCallback(async () => {
+ setLoading(true);
+ try {
+ const res = await knowledgeBaseApi.listChunkers();
+ setChunkers(res.chunkers);
+ } catch {
+ // Graceful fallback — if the endpoint is unavailable
+ // (older backend), the dialog works without chunker
+ // selection.
+ } finally {
+ setLoading(false);
+ }
+ }, []);
+
+ useEffect(() => {
+ refetch();
+ }, [refetch]);
+
+ return { chunkers, loading, refetch };
+}
diff --git a/examples/web_ui/frontend/src/i18n/locales/en.json b/examples/web_ui/frontend/src/i18n/locales/en.json
index 429ad1836..dc2d9423e 100644
--- a/examples/web_ui/frontend/src/i18n/locales/en.json
+++ b/examples/web_ui/frontend/src/i18n/locales/en.json
@@ -262,6 +262,9 @@
"dimension": {
"label": "Dimension"
},
+ "chunker": {
+ "label": "Chunker"
+ },
"policy": {
"lockedTitle": "Dimension is locked to {{dimension}}d",
"lockedDescription": "This server already pins all knowledge bases to {{dimension}}-dim embeddings. Only compatible models are listed below.",
@@ -272,7 +275,8 @@
"errors": {
"nameRequired": "Please give the knowledge base a name.",
"embeddingRequired": "Please pick an embedding model.",
- "dimensionRequired": "Please pick a dimension."
+ "dimensionRequired": "Please pick a dimension.",
+ "chunkerRequired": "Please pick a chunking strategy."
}
},
"dialog-knowledge-base-edit": {
@@ -289,6 +293,9 @@
"embeddingModel": {
"label": "Embedding model"
},
+ "chunker": {
+ "label": "Chunker"
+ },
"errors": {
"nameRequired": "Please give the knowledge base a name."
}
@@ -395,6 +402,26 @@
},
"addCredential": "Add Credential"
},
+ "chunker-select": {
+ "placeholder": "Select a chunker",
+ "loading": "Loading…",
+ "empty": "No chunkers available"
+ },
+ "chunker-types": {
+ "approx_token": {
+ "label": "Approx Token",
+ "params": {
+ "chunk_size": {
+ "label": "Chunk Size",
+ "description": "Maximum number of approximate tokens per chunk."
+ },
+ "overlap": {
+ "label": "Overlap",
+ "description": "Number of approximate tokens shared between consecutive chunks."
+ }
+ }
+ }
+ },
"dimension-select": {
"placeholder": "Select a dimension",
"pickModelFirst": "Pick a model first"
diff --git a/examples/web_ui/frontend/src/i18n/locales/zh.json b/examples/web_ui/frontend/src/i18n/locales/zh.json
index 260e8fde0..62926125d 100644
--- a/examples/web_ui/frontend/src/i18n/locales/zh.json
+++ b/examples/web_ui/frontend/src/i18n/locales/zh.json
@@ -262,6 +262,9 @@
"dimension": {
"label": "维度"
},
+ "chunker": {
+ "label": "分块策略"
+ },
"policy": {
"lockedTitle": "维度已锁定为 {{dimension}}d",
"lockedDescription": "当前实例已要求所有知识库使用 {{dimension}} 维 embedding,下面仅展示兼容的模型。",
@@ -272,7 +275,8 @@
"errors": {
"nameRequired": "请填写知识库名称。",
"embeddingRequired": "请选择 Embedding 模型。",
- "dimensionRequired": "请选择 Embedding 维度。"
+ "dimensionRequired": "请选择 Embedding 维度。",
+ "chunkerRequired": "请选择分块策略。"
}
},
"dialog-knowledge-base-edit": {
@@ -289,6 +293,9 @@
"embeddingModel": {
"label": "Embedding 模型"
},
+ "chunker": {
+ "label": "分块策略"
+ },
"errors": {
"nameRequired": "请填写知识库名称。"
}
@@ -395,6 +402,26 @@
},
"addCredential": "添加凭证"
},
+ "chunker-select": {
+ "placeholder": "选择分块策略",
+ "loading": "加载中…",
+ "empty": "暂无可用分块策略"
+ },
+ "chunker-types": {
+ "approx_token": {
+ "label": "近似 Token 数",
+ "params": {
+ "chunk_size": {
+ "label": "块大小",
+ "description": "每个分块的最大近似 Token 数量。"
+ },
+ "overlap": {
+ "label": "重叠大小",
+ "description": "相邻分块之间共享的近似 Token 数量。"
+ }
+ }
+ }
+ },
"dimension-select": {
"placeholder": "选择维度",
"pickModelFirst": "请先选择 Embedding 模型"
diff --git a/examples/web_ui/frontend/src/pages/knowledge/index.tsx b/examples/web_ui/frontend/src/pages/knowledge/index.tsx
index 7acace7f6..6191ecc08 100644
--- a/examples/web_ui/frontend/src/pages/knowledge/index.tsx
+++ b/examples/web_ui/frontend/src/pages/knowledge/index.tsx
@@ -1,5 +1,5 @@
import { Ellipsis, Files, FlaskConical, Pencil, Plus, Trash2 } from 'lucide-react';
-import { useState } from 'react';
+import { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useNavigate, useParams } from 'react-router-dom';
@@ -117,6 +117,16 @@ export const KnowledgePage = () => {
const selectedKb = knowledgeBases.find((kb) => kb.id === selectedKbId);
+ // Default to the first knowledge base when none is selected (or the
+ // selected one no longer exists), mirroring the URL so refresh keeps it.
+ useEffect(() => {
+ if (knowledgeBases.length === 0) return;
+ if (knowledgeBases.some((kb) => kb.id === selectedKbId)) return;
+ const first = knowledgeBases[0].id;
+ setSelectedKbId(first);
+ navigate(`/knowledge/${first}`, { replace: true });
+ }, [knowledgeBases, selectedKbId, navigate]);
+
const handleCreated = async (knowledgeBaseId: string) => {
await refetch();
setSelectedKbId(knowledgeBaseId);
diff --git a/src/agentscope/app/_app.py b/src/agentscope/app/_app.py
index e43afc74c..091a17d13 100644
--- a/src/agentscope/app/_app.py
+++ b/src/agentscope/app/_app.py
@@ -32,12 +32,9 @@ from .message_bus import MessageBus
from .storage import StorageBase
from ..agent import Agent
from ..credential import CredentialFactory, CredentialBase
-from ..rag import (
- ApproxTokenChunker,
- ChunkerBase,
- ParserBase,
- TextParser,
-)
+from ..rag import ApproxTokenChunker, ChunkerBase, ParserBase, TextParser
+
+from .._logging import logger
from .._version import __version__
@@ -84,7 +81,7 @@ def create_app(
workspace_manager: WorkspaceManagerBase,
knowledge_base_manager: KnowledgeBaseManagerBase | None = None,
knowledge_parsers: list[ParserBase] | dict[str, ParserBase] | None = None,
- knowledge_chunker: ChunkerBase | None = None,
+ knowledge_chunkers: list[Type[ChunkerBase]] | None = None,
blob_store: BlobStoreBase | None = None,
enable_index_worker: bool = True,
mcp_hubs: list[MCPHubBase] | None = None,
@@ -101,6 +98,7 @@ def create_app(
download_secret: str | None = None,
title: str = "AgentScope",
version: str = __version__,
+ **kwargs: Any,
) -> FastAPI:
"""Create and configure a FastAPI application.
@@ -164,9 +162,11 @@ def create_app(
(one parser bound to multiple types, type aliases, ...).
Defaults to ``[TextParser()]`` when
``knowledge_base_manager`` is set.
- knowledge_chunker (`ChunkerBase | None`, optional):
- The chunker shared across every knowledge base. Defaults
- to :class:`~agentscope.rag.ApproxTokenChunker()` when
+ knowledge_chunkers (`list[Type[ChunkerBase]] | None`, optional):
+ The chunker classes users can choose from when creating a
+ knowledge base. The chunker type and parameters are pinned
+ on the knowledge base record and reconstructed by the index
+ worker. Defaults to ``[ApproxTokenChunker]`` when
``knowledge_base_manager`` is set.
blob_store (`BlobStoreBase | None`, optional):
Backend storing uploaded document bytes between the
@@ -297,13 +297,44 @@ def create_app(
# KB feature is actually enabled. When ``knowledge_base_manager`` is
# ``None`` every KB endpoint is disabled, so leaving these as ``None``
# avoids unused imports being eagerly constructed at app startup.
+ unknown_kwargs = set(kwargs) - {"knowledge_chunker"}
+ if unknown_kwargs:
+ logger.warning(
+ "Ignoring unknown create_app() arguments: %s",
+ sorted(unknown_kwargs),
+ )
+
if knowledge_base_manager is not None:
app.state.knowledge_parsers = (
knowledge_parsers
if knowledge_parsers is not None
else [TextParser()]
)
- app.state.knowledge_chunker = knowledge_chunker or ApproxTokenChunker()
+ chunker_classes = list(
+ knowledge_chunkers
+ if knowledge_chunkers is not None
+ else [ApproxTokenChunker],
+ )
+ # Backward compatibility: the deprecated ``knowledge_chunker``
+ # instance is only used for its class.
+ if "knowledge_chunker" in kwargs:
+ logger.warning(
+ "The `knowledge_chunker` argument of create_app() is "
+ "deprecated, use `knowledge_chunkers` instead.",
+ )
+ legacy_cls = type(kwargs["knowledge_chunker"])
+ if legacy_cls not in chunker_classes:
+ chunker_classes.append(legacy_cls)
+ seen_chunker_types: dict[str, Type[ChunkerBase]] = {}
+ for cls in chunker_classes:
+ if cls.chunker_type in seen_chunker_types:
+ raise ValueError(
+ f"Duplicate chunker_type {cls.chunker_type!r}: "
+ f"{seen_chunker_types[cls.chunker_type].__name__} and "
+ f"{cls.__name__}.",
+ )
+ seen_chunker_types[cls.chunker_type] = cls
+ app.state.knowledge_chunkers = chunker_classes
app.state.blob_store = (
blob_store
if blob_store is not None
@@ -311,7 +342,7 @@ def create_app(
)
else:
app.state.knowledge_parsers = knowledge_parsers
- app.state.knowledge_chunker = knowledge_chunker
+ app.state.knowledge_chunkers = knowledge_chunkers
app.state.blob_store = blob_store
app.state.enable_index_worker = (
enable_index_worker and knowledge_base_manager is not None
diff --git a/src/agentscope/app/_lifespan.py b/src/agentscope/app/_lifespan.py
index e004a16e1..bd46ff9bd 100644
--- a/src/agentscope/app/_lifespan.py
+++ b/src/agentscope/app/_lifespan.py
@@ -197,7 +197,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
blob_store=blob_store,
knowledge_base_manager=knowledge_base_manager,
parsers=app.state.knowledge_parsers,
- chunker=app.state.knowledge_chunker,
+ chunkers=app.state.knowledge_chunkers,
node_id=node_id,
)
await stack.enter_async_context(
@@ -220,6 +220,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
blob_store=blob_store,
message_bus=message_bus,
resource_access_service=resource_access_service,
+ chunkers=app.state.knowledge_chunkers,
)
app.state.knowledge_base_service = knowledge_base_service
diff --git a/src/agentscope/app/_router/_knowledge_base.py b/src/agentscope/app/_router/_knowledge_base.py
index f3951c430..9db702200 100644
--- a/src/agentscope/app/_router/_knowledge_base.py
+++ b/src/agentscope/app/_router/_knowledge_base.py
@@ -23,15 +23,18 @@ from ..deps import (
get_current_user_id,
get_knowledge_base_manager,
get_knowledge_base_service,
+ get_knowledge_chunkers,
get_knowledge_parsers,
get_resource_access_service,
)
from ._schema import (
+ ChunkerInfo,
CreateKnowledgeBaseRequest,
CreateKnowledgeBaseResponse,
KbEmbeddingProvider,
KbMiddlewareParametersSchemaResponse,
KnowledgeDocumentView,
+ ListChunkersResponse,
ListKbEmbeddingModelsResponse,
ListKnowledgeBasesResponse,
ListKnowledgeDocumentsResponse,
@@ -50,7 +53,7 @@ from .._service import (
ResourceAccessService,
)
from ...middleware import RAGMiddleware
-from ...rag import ParserBase
+from ...rag import ChunkerBase, ParserBase
knowledge_base_router = APIRouter(
@@ -129,6 +132,46 @@ async def list_kb_embedding_models(
return ListKbEmbeddingModelsResponse(providers=providers, policy=policy)
+@knowledge_base_router.get(
+ "/chunkers",
+ response_model=ListChunkersResponse,
+ summary="List available chunker types and their parameter schemas",
+)
+async def list_chunkers(
+ _: str = Depends(get_current_user_id),
+ chunker_classes: list[type[ChunkerBase]] = Depends(
+ get_knowledge_chunkers,
+ ),
+) -> ListChunkersResponse:
+ """List every available chunker type with its parameter schema.
+
+ The front-end uses this to populate the chunker selector and
+ to render a dynamic parameter form when creating a knowledge
+ base.
+
+ Args:
+ _ (`str`):
+ Injected authenticated user ID; only used to gate the
+ endpoint behind authentication.
+ chunker_classes (`list[type[ChunkerBase]]`):
+ The chunker classes configured via
+ ``create_app(knowledge_chunkers=...)``.
+
+ Returns:
+ `ListChunkersResponse`:
+ All available chunker types with their JSON Schemas.
+ """
+ return ListChunkersResponse(
+ chunkers=[
+ ChunkerInfo(
+ type=cls.chunker_type,
+ parameter_schema=cls.Parameters.model_json_schema(),
+ )
+ for cls in chunker_classes
+ ],
+ )
+
+
@knowledge_base_router.get(
"/middleware/parameters_schema",
response_model=KbMiddlewareParametersSchemaResponse,
@@ -246,6 +289,7 @@ async def create_knowledge_base(
name=body.name,
description=body.description,
embedding_model_config=body.embedding_model_config,
+ chunker_config=body.chunker_config,
)
return CreateKnowledgeBaseResponse(knowledge_base_id=record.id)
diff --git a/src/agentscope/app/_router/_schema/__init__.py b/src/agentscope/app/_router/_schema/__init__.py
index a8c8e2b6b..cb7cf0ceb 100644
--- a/src/agentscope/app/_router/_schema/__init__.py
+++ b/src/agentscope/app/_router/_schema/__init__.py
@@ -55,11 +55,13 @@ from ._credential import (
ListCredentialSchemasResponse,
)
from ._knowledge_base import (
+ ChunkerInfo,
CreateKnowledgeBaseRequest,
CreateKnowledgeBaseResponse,
KbEmbeddingProvider,
KbMiddlewareParametersSchemaResponse,
KnowledgeDocumentView,
+ ListChunkersResponse,
ListKbEmbeddingModelsResponse,
ListKnowledgeBasesResponse,
ListKnowledgeDocumentsResponse,
@@ -130,11 +132,13 @@ __all__ = [
"ListCredentialsResponse",
"ListCredentialSchemasResponse",
# Knowledge base
+ "ChunkerInfo",
"CreateKnowledgeBaseRequest",
"CreateKnowledgeBaseResponse",
"KbEmbeddingProvider",
"KbMiddlewareParametersSchemaResponse",
"KnowledgeDocumentView",
+ "ListChunkersResponse",
"ListKbEmbeddingModelsResponse",
"ListKnowledgeBasesResponse",
"ListKnowledgeDocumentsResponse",
diff --git a/src/agentscope/app/_router/_schema/_knowledge_base.py b/src/agentscope/app/_router/_schema/_knowledge_base.py
index 99c8effea..7a92ffc1c 100644
--- a/src/agentscope/app/_router/_schema/_knowledge_base.py
+++ b/src/agentscope/app/_router/_schema/_knowledge_base.py
@@ -5,6 +5,7 @@ from datetime import datetime
from pydantic import BaseModel, Field
from ...storage import (
+ ChunkerConfig,
EmbeddingModelConfig,
KnowledgeDocumentRecord,
KnowledgeDocumentStatus,
@@ -30,6 +31,15 @@ class CreateKnowledgeBaseRequest(BaseModel):
"invalidate every previously inserted vector."
),
)
+ chunker_config: ChunkerConfig | None = Field(
+ default=None,
+ description=(
+ "Chunker configuration determining how uploaded documents "
+ "are split into chunks. Pinned at creation time and "
+ "cannot be changed afterwards. Defaults to the first "
+ "configured chunker with its default parameters."
+ ),
+ )
class CreateKnowledgeBaseResponse(BaseModel):
@@ -272,3 +282,23 @@ class ListSupportedContentTypesResponse(BaseModel):
"subclasses may override the default."
),
)
+
+
+class ChunkerInfo(BaseModel):
+ """One available chunker type and its parameter schema."""
+
+ type: str = Field(description="The chunker type identifier.")
+ parameter_schema: dict = Field(
+ description=(
+ "JSON Schema produced by `ChunkerBase.Parameters."
+ "model_json_schema()`, used to render the parameter form."
+ ),
+ )
+
+
+class ListChunkersResponse(BaseModel):
+ """Response body listing the available chunker types."""
+
+ chunkers: list[ChunkerInfo] = Field(
+ description="The available chunker types, in configured order.",
+ )
diff --git a/src/agentscope/app/_service/_access.py b/src/agentscope/app/_service/_access.py
index 2c583f7ee..9773efac6 100644
--- a/src/agentscope/app/_service/_access.py
+++ b/src/agentscope/app/_service/_access.py
@@ -16,6 +16,7 @@ from ..access import (
)
from ..storage import (
AgentRecord,
+ ChunkerConfig,
CredentialRecord,
EmbeddingModelConfig,
KnowledgeBaseRecord,
@@ -84,6 +85,13 @@ class KnowledgeBaseView(BaseModel):
embedding_model_config: EmbeddingModelConfig = Field(
description="Embedding model configuration pinned at creation.",
)
+ chunker_config: ChunkerConfig | None = Field(
+ default=None,
+ description=(
+ "Chunker configuration pinned at creation. ``None`` for "
+ "legacy records created before per-KB chunker support."
+ ),
+ )
created_at: datetime = Field(description="Creation timestamp.")
updated_at: datetime = Field(description="Last-update timestamp.")
editable: bool = Field(
diff --git a/src/agentscope/app/_service/_index_worker.py b/src/agentscope/app/_service/_index_worker.py
index 3c2eae7d7..14bcb8518 100644
--- a/src/agentscope/app/_service/_index_worker.py
+++ b/src/agentscope/app/_service/_index_worker.py
@@ -25,14 +25,17 @@ import contextlib
import mimetypes
from concurrent.futures import ProcessPoolExecutor
from datetime import timedelta
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
+
+from pydantic import ValidationError
from ..._logging import logger
+from ...rag import ApproxTokenChunker
if TYPE_CHECKING:
from ..rag.blob_store import BlobStoreBase
from ..rag.knowledge_base_manager import KnowledgeBaseManagerBase
- from ..storage import StorageBase
+ from ..storage import KnowledgeBaseRecord, StorageBase
from ...rag import ChunkerBase, ParserBase, Section
# Read blob bytes in chunks bounded so the worker never holds the whole
@@ -115,11 +118,12 @@ class IndexWorker:
blob_store: "BlobStoreBase",
knowledge_base_manager: "KnowledgeBaseManagerBase",
parsers: "list[ParserBase] | dict[str, ParserBase]",
- chunker: "ChunkerBase",
node_id: str,
+ chunkers: "list[type[ChunkerBase]] | None" = None,
max_concurrency: int = 4,
lease_ttl: timedelta = timedelta(seconds=90),
parser_executor: ProcessPoolExecutor | None = None,
+ **kwargs: Any,
) -> None:
"""Initialize the worker.
@@ -146,13 +150,15 @@ class IndexWorker:
declare.
Same registry the upload service uses, passed in by DI.
- chunker (`ChunkerBase`):
- The shared chunker.
node_id (`str`):
Stable identifier for this worker process. Used as
``processing_node`` on the lease so the sweeper can
tell whose work expired. Typically
``f"{hostname}:{pid}:{uuid}"``.
+ chunkers (`list[type[ChunkerBase]] | None`, optional):
+ The chunker classes that can be rebuilt from a knowledge
+ base's ``chunker_config``. Defaults to
+ ``[ApproxTokenChunker]``.
max_concurrency (`int`, defaults to ``4``):
Maximum number of documents processed concurrently by
this worker. Higher values trade memory for
@@ -169,12 +175,33 @@ class IndexWorker:
third-party byte-oriented parsers. Injected so a
single pool can be shared across the app (built in
lifespan).
+ **kwargs (`Any`):
+ Deprecated. ``chunker`` (a shared chunker instance) is
+ still accepted for backward compatibility; only its
+ class is used.
"""
+ chunker_classes = list(chunkers or [ApproxTokenChunker])
+ if "chunker" in kwargs:
+ logger.warning(
+ "The `chunker` argument of IndexWorker is deprecated, "
+ "use `chunkers` instead.",
+ )
+ legacy_cls = type(kwargs.pop("chunker"))
+ if legacy_cls not in chunker_classes:
+ chunker_classes.append(legacy_cls)
+ if kwargs:
+ logger.warning(
+ "Ignoring unknown IndexWorker arguments: %s",
+ sorted(kwargs),
+ )
+
self._storage = storage
self._blob_store = blob_store
self._manager = knowledge_base_manager
self._parsers_by_media_type = _build_parser_registry(parsers)
- self._chunker = chunker
+ self._chunkers_by_type = {
+ cls.chunker_type: cls for cls in chunker_classes
+ }
self._node_id = node_id
self._lease_ttl = lease_ttl
self._sem = asyncio.Semaphore(max_concurrency)
@@ -326,6 +353,19 @@ class IndexWorker:
)
return
+ kb_record = await self._manager.get_knowledge_base(
+ user_id,
+ knowledge_base_id,
+ )
+ if kb_record is None:
+ logger.warning(
+ "Knowledge base %s vanished before processing document %s.",
+ knowledge_base_id,
+ document_id,
+ )
+ return
+ chunker = self._resolve_chunker_from_record(kb_record)
+
data = record.data
media_type = (
data.content_type or mimetypes.guess_type(data.filename)[0]
@@ -357,7 +397,7 @@ class IndexWorker:
document_id,
"chunking",
)
- chunks = await self._chunker.chunk(sections)
+ chunks = await chunker.chunk(sections)
# ---- indexing ----
await self._storage.update_knowledge_document_status(
@@ -407,6 +447,42 @@ class IndexWorker:
filename,
)
+ def _resolve_chunker_from_record(
+ self,
+ kb_record: "KnowledgeBaseRecord",
+ ) -> "ChunkerBase":
+ """Resolve the chunker from a pre-fetched KB record.
+
+ Instantiates the matching chunker class using the record's
+ ``chunker_config``. Falls back to
+ :class:`~agentscope.rag.ApproxTokenChunker` for legacy
+ records that predate per-KB chunker support
+ (``chunker_config`` is ``None``).
+ """
+ cfg = kb_record.data.chunker_config
+ if cfg is not None:
+ chunker_cls = self._chunkers_by_type.get(cfg.type)
+ if chunker_cls is None:
+ logger.warning(
+ "Unknown chunker type %r on KB %s — "
+ "falling back to ApproxTokenChunker.",
+ cfg.type,
+ kb_record.id,
+ )
+ return ApproxTokenChunker()
+ try:
+ return chunker_cls(
+ parameters=chunker_cls.Parameters(**cfg.parameters),
+ )
+ except (ValidationError, TypeError, ValueError) as exc:
+ logger.warning(
+ "Invalid chunker parameters on KB %s: %s — "
+ "falling back to ApproxTokenChunker.",
+ kb_record.id,
+ exc,
+ )
+ return ApproxTokenChunker()
+
async def _read_blob(self, blob_uri: str) -> bytes:
"""Stream the blob into memory in bounded chunks.
diff --git a/src/agentscope/app/_service/_knowledge_base.py b/src/agentscope/app/_service/_knowledge_base.py
index 180e0bcda..f0a921bd0 100644
--- a/src/agentscope/app/_service/_knowledge_base.py
+++ b/src/agentscope/app/_service/_knowledge_base.py
@@ -21,6 +21,7 @@ import uuid
from typing import IO, TYPE_CHECKING
from fastapi import HTTPException, status
+from pydantic import ValidationError
from ..access import ResourceKind
from ..rag.knowledge_base_manager import (
@@ -28,10 +29,12 @@ from ..rag.knowledge_base_manager import (
KnowledgeBaseNotFoundError,
)
from ..storage import (
+ ChunkerConfig,
KnowledgeDocumentData,
KnowledgeDocumentRecord,
)
from ..._logging import logger
+from ...rag import ApproxTokenChunker
from .._bus_ops import enqueue_index_task
from ._access import ResourceAccessService
@@ -44,7 +47,7 @@ if TYPE_CHECKING:
KnowledgeBaseRecord,
StorageBase,
)
- from ...rag import VectorSearchResult
+ from ...rag import ChunkerBase, VectorSearchResult
class KnowledgeBaseService:
@@ -65,6 +68,7 @@ class KnowledgeBaseService:
blob_store: "BlobStoreBase",
message_bus: "MessageBus",
resource_access_service: "ResourceAccessService",
+ chunkers: "list[type[ChunkerBase]] | None" = None,
) -> None:
"""Initialize the service.
@@ -90,12 +94,19 @@ class KnowledgeBaseService:
*and* mutation) goes through it so shared knowledge
bases work end-to-end: readers see documents +
search hits, editors can also upload / delete.
+ chunkers (`list[type[ChunkerBase]] | None`, optional):
+ The chunker classes users can choose from when creating
+ a knowledge base; used to validate ``chunker_config``.
+ Defaults to ``[ApproxTokenChunker]``.
"""
self._storage = storage
self._manager = knowledge_base_manager
self._blob_store = blob_store
self._bus = message_bus
self._access = resource_access_service
+ self._chunkers_by_type = {
+ cls.chunker_type: cls for cls in (chunkers or [ApproxTokenChunker])
+ }
# ------------------------------------------------------------------
# Knowledge base CRUD
@@ -107,6 +118,7 @@ class KnowledgeBaseService:
name: str,
description: str,
embedding_model_config: "EmbeddingModelConfig",
+ chunker_config: "ChunkerConfig | None" = None,
) -> "KnowledgeBaseRecord":
"""Delegate creation to the manager, mapping policy errors.
@@ -119,6 +131,9 @@ class KnowledgeBaseService:
Free-form description.
embedding_model_config (`EmbeddingModelConfig`):
Embedding model configuration; pinned to the record.
+ chunker_config (`ChunkerConfig | None`, optional):
+ Chunker configuration; pinned to the record. Defaults
+ to the first configured chunker with default parameters.
Returns:
`KnowledgeBaseRecord`:
@@ -128,13 +143,38 @@ class KnowledgeBaseService:
`HTTPException`:
``409`` when the requested embedding dimension
violates the manager's dimension policy.
+ ``422`` when the chunker type or parameters are
+ invalid.
"""
+ if chunker_config is None:
+ chunker_config = ChunkerConfig(
+ type=next(iter(self._chunkers_by_type)),
+ parameters={},
+ )
+ chunker_cls = self._chunkers_by_type.get(chunker_config.type)
+ if chunker_cls is None:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=(
+ f"Unknown chunker type: {chunker_config.type!r}, "
+ f"available: {sorted(self._chunkers_by_type)}"
+ ),
+ )
+ try:
+ chunker_cls.Parameters(**chunker_config.parameters)
+ except (TypeError, ValueError, ValidationError) as exc:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=f"Invalid chunker parameters: {exc}",
+ ) from exc
+
try:
return await self._manager.create_knowledge_base(
user_id=user_id,
name=name,
description=description,
embedding_model_config=embedding_model_config,
+ chunker_config=chunker_config,
)
except DimensionPolicyError as exc:
raise HTTPException(
diff --git a/src/agentscope/app/deps.py b/src/agentscope/app/deps.py
index 3be59c79e..0eabb8fe3 100644
--- a/src/agentscope/app/deps.py
+++ b/src/agentscope/app/deps.py
@@ -26,7 +26,7 @@ from .message_bus import MessageBus
from .rag.blob_store import BlobStoreBase
from .rag.knowledge_base_manager import KnowledgeBaseManagerBase
from .storage import StorageBase
-from ..rag import ParserBase
+from ..rag import ChunkerBase, ParserBase
async def get_current_user_id(
@@ -346,6 +346,37 @@ async def get_knowledge_parsers(
return parsers
+async def get_knowledge_chunkers(
+ request: Request,
+) -> list[type[ChunkerBase]]:
+ """Return the chunker classes configured on the app.
+
+ Args:
+ request (`Request`):
+ The incoming FastAPI request.
+
+ Returns:
+ `list[type[ChunkerBase]]`:
+ The chunker classes stored in ``app.state.knowledge_chunkers``
+ — the same value the index worker uses to rebuild chunkers.
+
+ Raises:
+ `HTTPException`:
+ ``503`` when the KB feature is disabled (no chunkers
+ configured).
+ """
+ chunkers = getattr(request.app.state, "knowledge_chunkers", None)
+ if not chunkers:
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail=(
+ "Knowledge base feature is disabled — pass a "
+ "knowledge_base_manager to create_app() to enable it."
+ ),
+ )
+ return chunkers
+
+
async def get_mcp_hubs(request: Request) -> dict[str, MCPHubBase]:
"""Return the registered MCP hubs, keyed by hub id.
diff --git a/src/agentscope/app/rag/index_worker/__init__.py b/src/agentscope/app/rag/index_worker/__init__.py
index bb4612b35..03066610f 100644
--- a/src/agentscope/app/rag/index_worker/__init__.py
+++ b/src/agentscope/app/rag/index_worker/__init__.py
@@ -36,7 +36,7 @@ Example::
from agentscope.app.message_bus import RedisMessageBus
from agentscope.app.storage import RedisStorage
from agentscope.app.rag.index_worker import run_worker
- from agentscope.rag import ApproxTokenChunker, TextParser, ...
+ from agentscope.rag import TextParser, ...
async def main() -> None:
storage = RedisStorage(url=os.environ["REDIS_URL"])
@@ -47,14 +47,12 @@ Example::
)
kb_manager = DefaultKnowledgeBaseManager(...)
parsers = [TextParser()]
- chunker = ApproxTokenChunker()
await run_worker(
storage=storage,
message_bus=message_bus,
blob_store=blob_store,
knowledge_base_manager=kb_manager,
parsers=parsers,
- chunker=chunker,
)
if __name__ == "__main__":
@@ -66,7 +64,7 @@ import socket
import uuid
from concurrent.futures import ProcessPoolExecutor
from contextlib import AsyncExitStack
-from typing import TYPE_CHECKING
+from typing import Any, TYPE_CHECKING
from ..._service import IndexTaskConsumer, IndexWorker
from ...._logging import logger
@@ -86,11 +84,12 @@ async def run_worker(
blob_store: "BlobStoreBase",
knowledge_base_manager: "KnowledgeBaseManagerBase",
parsers: "list[ParserBase] | dict[str, ParserBase]",
- chunker: "ChunkerBase",
+ chunkers: "list[type[ChunkerBase]] | None" = None,
node_id: str | None = None,
worker_max_concurrency: int = 4,
consumer_max_batch: int = 32,
parser_executor: ProcessPoolExecutor | None = None,
+ **kwargs: Any,
) -> None:
"""Run the out-of-process index worker until cancelled.
@@ -117,8 +116,10 @@ async def run_worker(
``supported_media_types`` (later entries override earlier
ones, with a warning); dict mode is used verbatim for
explicit routing.
- chunker (`ChunkerBase`):
- Shared chunker.
+ chunkers (`list[type[ChunkerBase]] | None`, optional):
+ The chunker classes that can be rebuilt from a knowledge
+ base's ``chunker_config``. Defaults to
+ ``[ApproxTokenChunker]``.
node_id (`str | None`, optional):
Stable identifier for this worker process used on the
storage lease. Defaults to
@@ -136,6 +137,9 @@ async def run_worker(
parser_executor (`ProcessPoolExecutor | None`, optional):
Process pool for CPU-bound parses. ``None`` runs parses
inline (fine for text-only deployments).
+ **kwargs (`Any`):
+ Deprecated arguments forwarded to :class:`IndexWorker`
+ (e.g. the old ``chunker`` instance).
"""
resolved_node_id = (
node_id or f"{socket.gethostname()}:{uuid.uuid4().hex[:8]}"
@@ -152,10 +156,11 @@ async def run_worker(
blob_store=blob_store,
knowledge_base_manager=knowledge_base_manager,
parsers=parsers,
- chunker=chunker,
+ chunkers=chunkers,
node_id=resolved_node_id,
max_concurrency=worker_max_concurrency,
parser_executor=parser_executor,
+ **kwargs,
)
await stack.enter_async_context(
IndexTaskConsumer(
diff --git a/src/agentscope/app/rag/index_worker/__main__.py b/src/agentscope/app/rag/index_worker/__main__.py
index 805391222..8addeddfc 100644
--- a/src/agentscope/app/rag/index_worker/__main__.py
+++ b/src/agentscope/app/rag/index_worker/__main__.py
@@ -21,7 +21,7 @@ Example bootstrap (``mydeploy/worker_bootstrap.py``)::
)
from agentscope.app.message_bus import RedisMessageBus
from agentscope.app.storage import RedisStorage
- from agentscope.rag import ApproxTokenChunker, TextParser
+ from agentscope.rag import TextParser
def bootstrap() -> dict:
return {
@@ -33,7 +33,6 @@ Example bootstrap (``mydeploy/worker_bootstrap.py``)::
),
"knowledge_base_manager": DefaultKnowledgeBaseManager(...),
"parsers": [TextParser()],
- "chunker": ApproxTokenChunker(),
}
And launch::
diff --git a/src/agentscope/app/rag/knowledge_base_manager/_base.py b/src/agentscope/app/rag/knowledge_base_manager/_base.py
index d50efa42f..492a21727 100644
--- a/src/agentscope/app/rag/knowledge_base_manager/_base.py
+++ b/src/agentscope/app/rag/knowledge_base_manager/_base.py
@@ -28,6 +28,7 @@ if TYPE_CHECKING:
from types import TracebackType
from ...storage import (
+ ChunkerConfig,
EmbeddingModelConfig,
KnowledgeBaseRecord,
StorageBase,
@@ -127,6 +128,7 @@ class KnowledgeBaseManagerBase(ABC):
name: str,
description: str,
embedding_model_config: "EmbeddingModelConfig",
+ chunker_config: "ChunkerConfig | None" = None,
) -> "KnowledgeBaseRecord":
"""Create a new knowledge base for the given user.
@@ -147,6 +149,9 @@ class KnowledgeBaseManagerBase(ABC):
Free-form description.
embedding_model_config (`EmbeddingModelConfig`):
Embedding model configuration; pinned to the record.
+ chunker_config (`ChunkerConfig | None`, optional):
+ Chunker configuration; pinned to the record. ``None``
+ leaves the record without a pinned chunker (legacy).
Returns:
`KnowledgeBaseRecord`:
diff --git a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py
index aa0ca3280..8ad9b560b 100644
--- a/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py
+++ b/src/agentscope/app/rag/knowledge_base_manager/_collection_per_kb.py
@@ -25,7 +25,7 @@ from ..._service._embedding import build_embedding_model
from ...storage import KnowledgeBaseRecord, KnowledgeBaseData
if TYPE_CHECKING:
- from ...storage import EmbeddingModelConfig
+ from ...storage import ChunkerConfig, EmbeddingModelConfig
class CollectionPerKbManager(KnowledgeBaseManagerBase):
@@ -55,6 +55,7 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
name: str,
description: str,
embedding_model_config: "EmbeddingModelConfig",
+ chunker_config: "ChunkerConfig | None" = None,
) -> KnowledgeBaseRecord:
"""Allocate a new collection and persist the knowledge base record.
@@ -67,6 +68,9 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
Free-form description.
embedding_model_config (`EmbeddingModelConfig`):
Embedding model configuration; pinned to the record.
+ chunker_config (`ChunkerConfig | None`, optional):
+ Chunker configuration; pinned to the record. ``None``
+ leaves the record without a pinned chunker (legacy).
Returns:
`KnowledgeBaseRecord`:
@@ -78,6 +82,7 @@ class CollectionPerKbManager(KnowledgeBaseManagerBase):
name=name,
description=description,
embedding_model_config=embedding_model_config,
+ chunker_config=chunker_config,
collection_name="",
),
)
diff --git a/src/agentscope/app/storage/__init__.py b/src/agentscope/app/storage/__init__.py
index 3fee8470c..8db3e1b4b 100644
--- a/src/agentscope/app/storage/__init__.py
+++ b/src/agentscope/app/storage/__init__.py
@@ -9,6 +9,7 @@ from ._model import (
AgentRecord,
ChannelBinding,
ChannelRecord,
+ ChunkerConfig,
RoutingConfig,
SessionScope,
SessionSettings,
@@ -74,6 +75,7 @@ __all__ = [
"AgentRecord",
"ChannelBinding",
"ChannelRecord",
+ "ChunkerConfig",
"RoutingConfig",
"SessionScope",
"SessionSettings",
diff --git a/src/agentscope/app/storage/_model/__init__.py b/src/agentscope/app/storage/_model/__init__.py
index 614dc0e9c..f31a3ffa1 100644
--- a/src/agentscope/app/storage/_model/__init__.py
+++ b/src/agentscope/app/storage/_model/__init__.py
@@ -10,7 +10,11 @@ from ._channel import (
SessionSettings,
)
from ._credential import CredentialRecord
-from ._knowledge_base import KnowledgeBaseData, KnowledgeBaseRecord
+from ._knowledge_base import (
+ ChunkerConfig,
+ KnowledgeBaseData,
+ KnowledgeBaseRecord,
+)
from ._knowledge_document import (
KnowledgeDocumentData,
KnowledgeDocumentRecord,
@@ -36,6 +40,7 @@ __all__ = [
"AgentRecord",
"ChannelBinding",
"ChannelRecord",
+ "ChunkerConfig",
"RoutingConfig",
"SessionScope",
"SessionSettings",
diff --git a/src/agentscope/app/storage/_model/_knowledge_base.py b/src/agentscope/app/storage/_model/_knowledge_base.py
index 2e9be619a..1f9c7ede1 100644
--- a/src/agentscope/app/storage/_model/_knowledge_base.py
+++ b/src/agentscope/app/storage/_model/_knowledge_base.py
@@ -8,6 +8,28 @@ from ._base import _RecordBase
from ._session import EmbeddingModelConfig
+class ChunkerConfig(BaseModel):
+ """Chunker configuration persisted alongside a knowledge base.
+
+ Stores the chunker ``type`` identifier (as declared by
+ ``ChunkerBase.chunker_type``) and the keyword-argument
+ ``parameters`` used to reconstruct it at indexing time.
+ """
+
+ type: str = Field(
+ description=(
+ "Chunker type identifier, as declared by "
+ "``ChunkerBase.chunker_type``."
+ ),
+ )
+ parameters: dict = Field(
+ default_factory=dict,
+ description=(
+ "Keyword arguments used to build ``ChunkerBase.Parameters``."
+ ),
+ )
+
+
class KnowledgeBaseData(BaseModel):
"""The mutable payload of a knowledge base record.
@@ -38,6 +60,21 @@ class KnowledgeBaseData(BaseModel):
)
"""Embedding model configuration pinned at creation time."""
+ chunker_config: ChunkerConfig | None = Field(
+ default=None,
+ description=(
+ "Chunker configuration pinned at creation time. "
+ "``None`` only for legacy records created before "
+ "per-KB chunker support was introduced; the index "
+ "worker falls back to ``ApproxTokenChunker()`` in "
+ "that case. New records always have an explicit value."
+ ),
+ )
+ """Chunker configuration pinned at creation time.
+
+ ``None`` only for legacy records; new creations always set this.
+ """
+
collection_name: str = Field(
description=(
"The vector store collection that physically backs this "
@@ -71,9 +108,9 @@ class KnowledgeBaseRecord(_RecordBase):
Pre-refactor records stored the payload fields (``name`` /
``description`` / ``embedding_model_config`` /
- ``collection_name``) at the record top level rather than
- under ``data``. This validator runs in ``mode='before'`` so
- it hooks every pydantic entry point —
+ ``chunker_config`` / ``collection_name``) at the record top
+ level rather than under ``data``. This validator runs in
+ ``mode='before'`` so it hooks every pydantic entry point —
:meth:`~pydantic.BaseModel.model_validate`,
:meth:`~pydantic.BaseModel.model_validate_json` (the path
the storage backend takes when reading from Redis / SQL),
@@ -97,6 +134,7 @@ class KnowledgeBaseRecord(_RecordBase):
"name",
"description",
"embedding_model_config",
+ "chunker_config",
"collection_name",
)
if not any(key in obj for key in legacy_keys):
diff --git a/src/agentscope/rag/_chunker/_approx_token_chunker.py b/src/agentscope/rag/_chunker/_approx_token_chunker.py
index fe26674d4..3ecf52261 100644
--- a/src/agentscope/rag/_chunker/_approx_token_chunker.py
+++ b/src/agentscope/rag/_chunker/_approx_token_chunker.py
@@ -7,9 +7,13 @@ within the right order of magnitude for most LLM tokenizers.
"""
from bisect import bisect_right
from itertools import accumulate
+from typing import Any
+
+from pydantic import Field, model_validator
from ._base import ChunkerBase
from .._document import Chunk, Section
+from ..._logging import logger
from ...message import TextBlock, DataBlock
@@ -30,35 +34,97 @@ class ApproxTokenChunker(ChunkerBase):
required by :class:`ChunkerBase`.
"""
- def __init__(self, chunk_size: int = 512, overlap: int = 50) -> None:
+ chunker_type = "approx_token"
+
+ class Parameters(ChunkerBase.Parameters):
+ """The tunable parameters of the approximate-token chunker."""
+
+ chunk_size: int = Field(
+ default=512,
+ ge=1,
+ title="Chunk Size",
+ description="Maximum number of approximate tokens per chunk.",
+ )
+ overlap: int = Field(
+ default=50,
+ ge=0,
+ title="Overlap",
+ description=(
+ "Number of approximate tokens shared between "
+ "consecutive chunks."
+ ),
+ )
+
+ @model_validator(mode="after")
+ def _overlap_less_than_chunk_size(
+ self,
+ ) -> "ApproxTokenChunker.Parameters":
+ if self.overlap >= self.chunk_size:
+ raise ValueError(
+ "overlap must be less than chunk_size, got "
+ f"overlap={self.overlap}, chunk_size={self.chunk_size}.",
+ )
+ return self
+
+ def __init__(
+ self,
+ parameters: "ApproxTokenChunker.Parameters | None" = None,
+ **kwargs: Any,
+ ) -> None:
"""Initialize the approx token chunker.
Args:
- chunk_size (`int`, defaults to `512`):
- The maximum number of approximate tokens per chunk.
- Must be a positive integer.
- overlap (`int`, defaults to `50`):
- The number of approximate tokens shared between two
- consecutive chunks. Must be non-negative and smaller
- than ``chunk_size``.
+ parameters (`ApproxTokenChunker.Parameters | None`, optional):
+ The chunker parameters (``chunk_size`` and ``overlap``).
+ Defaults to ``Parameters()`` when not provided.
+ **kwargs (`Any`):
+ Deprecated. ``chunk_size`` and ``overlap`` are still
+ accepted for backward compatibility and override the
+ corresponding fields in ``parameters``; other keys are
+ ignored.
Raises:
+ `TypeError`:
+ If ``parameters`` is not a ``Parameters`` instance.
`ValueError`:
If ``chunk_size`` is not positive, or ``overlap`` is
negative or not smaller than ``chunk_size``.
"""
- if chunk_size <= 0:
- raise ValueError(
- f"chunk_size must be positive, got {chunk_size}.",
+ if parameters is not None and not isinstance(
+ parameters,
+ self.Parameters,
+ ):
+ raise TypeError(
+ "The first argument of ApproxTokenChunker must be an "
+ "ApproxTokenChunker.Parameters instance, got "
+ f"{type(parameters).__name__}. Use keyword arguments "
+ "chunk_size=/overlap= or parameters=Parameters(...).",
)
- if overlap < 0 or overlap >= chunk_size:
- raise ValueError(
- "overlap must satisfy 0 <= overlap < chunk_size, "
- f"got overlap={overlap}, chunk_size={chunk_size}.",
+ legacy = {
+ key: kwargs[key]
+ for key in ("chunk_size", "overlap")
+ if key in kwargs
+ }
+ if legacy:
+ logger.warning(
+ "Passing %s to ApproxTokenChunker directly is deprecated, "
+ "use ApproxTokenChunker.Parameters instead.",
+ ", ".join(f"``{k}``" for k in legacy),
)
+ base = parameters.model_dump() if parameters is not None else {}
+ parameters = self.Parameters(**{**base, **legacy})
+
+ super().__init__(parameters)
+
+ @property
+ def chunk_size(self) -> int:
+ """The maximum number of approximate tokens per chunk."""
+ return self.parameters.chunk_size
- self.chunk_size = chunk_size
- self.overlap = overlap
+ @property
+ def overlap(self) -> int:
+ """The number of approximate tokens shared between chunks."""
+ return self.parameters.overlap
async def chunk(self, sections: list[Section]) -> list[Chunk]:
"""Chunk the input sections into smaller chunks based on an approx
diff --git a/src/agentscope/rag/_chunker/_base.py b/src/agentscope/rag/_chunker/_base.py
index a766d20ac..c80c16075 100644
--- a/src/agentscope/rag/_chunker/_base.py
+++ b/src/agentscope/rag/_chunker/_base.py
@@ -18,6 +18,8 @@ guarantee preserves the structural metadata attached by the Parser
"""
from abc import ABC, abstractmethod
+from pydantic import BaseModel
+
from .._document import Chunk, Section
@@ -29,6 +31,9 @@ class ChunkerBase(ABC):
chunker is configured once at construction time and reused
across many ``chunk()`` calls within the same knowledge base.
+ Subclasses must set :attr:`chunker_type` and may override the
+ :class:`Parameters` inner class to declare their tunable parameters.
+
Subclasses must guarantee:
- **No cross-Section merging**: every output :class:`Chunk` is
@@ -45,6 +50,27 @@ class ChunkerBase(ABC):
``metadata`` are copied from its parent Section.
"""
+ chunker_type: str
+ """The unique identifier of the chunking strategy."""
+
+ class Parameters(BaseModel):
+ """The tunable parameters of the chunker."""
+
+ def __init__(
+ self,
+ parameters: "ChunkerBase.Parameters | None" = None,
+ ) -> None:
+ """Initialize the chunker.
+
+ Args:
+ parameters (`ChunkerBase.Parameters | None`, optional):
+ The chunker parameters. Defaults to
+ ``self.Parameters()`` when not provided.
+ """
+ self.parameters = (
+ parameters if parameters is not None else self.Parameters()
+ )
+
@abstractmethod
async def chunk(self, sections: list[Section]) -> list[Chunk]:
"""Split a list of Sections into Chunks.
diff --git a/tests/app_lifespan_dedicated_test.py b/tests/app_lifespan_dedicated_test.py
index 2aa658da8..4addbee66 100644
--- a/tests/app_lifespan_dedicated_test.py
+++ b/tests/app_lifespan_dedicated_test.py
@@ -46,6 +46,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import (
)
from agentscope.app.message_bus import RedisMessageBus
from agentscope.app.storage import (
+ ChunkerConfig,
EmbeddingModelConfig,
KnowledgeBaseData,
KnowledgeBaseRecord,
@@ -129,6 +130,7 @@ class _FakeKbManager(KnowledgeBaseManagerBase):
name: str,
description: str,
embedding_model_config: EmbeddingModelConfig,
+ chunker_config: ChunkerConfig | None = None,
) -> KnowledgeBaseRecord:
raise NotImplementedError
diff --git a/tests/resource_access_policy_test.py b/tests/resource_access_policy_test.py
index 46bcdf77f..8d754b3c0 100644
--- a/tests/resource_access_policy_test.py
+++ b/tests/resource_access_policy_test.py
@@ -418,6 +418,7 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):
"dimensions": 1536,
"parameters": {},
},
+ "chunker_config": None,
"created_at": AnyValue(),
"updated_at": AnyValue(),
"editable": True,
@@ -445,6 +446,7 @@ class ResourceAccessServiceTest(IsolatedAsyncioTestCase):
"dimensions": 1536,
"parameters": {},
},
+ "chunker_config": None,
"created_at": AnyValue(),
"updated_at": AnyValue(),
"editable": True,
diff --git a/tests/service_knowledge_base_upload_test.py b/tests/service_knowledge_base_upload_test.py
index a01845ef2..3e4c58d10 100644
--- a/tests/service_knowledge_base_upload_test.py
+++ b/tests/service_knowledge_base_upload_test.py
@@ -35,6 +35,7 @@ from agentscope.app.rag.knowledge_base_manager._dimension_policy import (
)
from agentscope.app.message_bus import RedisMessageBus
from agentscope.app.storage import (
+ ChunkerConfig,
EmbeddingModelConfig,
KnowledgeBaseData,
KnowledgeBaseRecord,
@@ -206,6 +207,7 @@ class _FakeKbManager(KnowledgeBaseManagerBase):
name: str,
description: str,
embedding_model_config: EmbeddingModelConfig,
+ chunker_config: ChunkerConfig | None = None,
) -> KnowledgeBaseRecord:
record = KnowledgeBaseRecord(
user_id=user_id,