DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""
CLI Knowledge Base Command
===========================
Manage knowledge bases from the command line.
"""
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from typing import TYPE_CHECKING, Optional
from rich.console import Console
from rich.table import Table
import typer
from deeptutor.knowledge.manager import KnowledgeBaseManager
from deeptutor.knowledge.naming import validate_knowledge_base_name
from deeptutor.services.path_service import get_path_service
from deeptutor.services.rag.factory import DEFAULT_PROVIDER
from deeptutor.services.rag.file_routing import FileTypeRouter
if TYPE_CHECKING: # pragma: no cover - imported for type annotations only
from deeptutor.services.rag.eval import EvalDataset, EvalReport, ProgressFn, QueryEvaluation
console = Console()
def _get_kb_manager() -> KnowledgeBaseManager:
"""Return a KnowledgeBaseManager rooted at the canonical project-level KB directory."""
base_dir = get_path_service().project_root / "data" / "knowledge_bases"
return KnowledgeBaseManager(base_dir=str(base_dir))
def _collect_documents(docs: list[str], docs_dir: Optional[str]) -> list[str]:
"""Collect and de-duplicate document files from explicit paths and a directory."""
candidates: list[Path] = []
for doc in docs:
path = Path(doc).expanduser().resolve()
if path.exists() and path.is_file():
candidates.append(path)
if docs_dir:
base = Path(docs_dir).expanduser().resolve()
if not base.exists() or not base.is_dir():
raise typer.BadParameter(f"docs directory does not exist: {base}")
candidates.extend(FileTypeRouter.collect_supported_files(base, recursive=True))
unique: list[str] = []
seen: set[str] = set()
for path in candidates:
key = str(path)
if key in seen:
continue
seen.add(key)
unique.append(key)
return unique
def _load_eval_dataset(dataset: str) -> EvalDataset:
"""Load the QA set for ``kb eval``, exiting with a readable message when it is invalid."""
from deeptutor.services.rag.eval import EvalDatasetError, load_dataset
try:
return load_dataset(dataset)
except EvalDatasetError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
def _resolve_eval_provider(mgr: KnowledgeBaseManager, name: str) -> str:
"""Return the provider bound to ``name``, rejecting engines that return no ranked chunks."""
from deeptutor.services.rag.factory import PAGEINDEX_OSS_PROVIDER, PAGEINDEX_PROVIDER
from deeptutor.services.rag.provider_binding import resolve_bound_provider
provider = resolve_bound_provider(str(mgr.base_dir), name)
if provider in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}:
console.print(
"[red]PageIndex answers by reasoning over pages inside an agent loop, so it "
"returns no ranked chunks to score. Evaluate a KB bound to llamaindex, "
"graphrag, lightrag, a LightRAG server, WeKnora or IMA instead.[/]"
)
raise typer.Exit(code=1)
return provider
def _build_eval_progress(top_k: int, fmt: str) -> ProgressFn:
"""Return a per-case progress printer that stays silent while emitting JSON."""
from deeptutor.services.rag.eval import format_metric
recall_key = f"recall@{top_k}"
def _report(index: int, total: int, item: QueryEvaluation) -> None:
if fmt == "json":
return
if item.failed:
console.print(f" {index}/{total} [red]failed[/] {item.case_id}: {item.error}")
return
value = format_metric(item.metrics.values()[recall_key])
console.print(f" {index}/{total} [green]ok[/] {item.case_id} {recall_key}={value}")
return _report
def _save_eval_report(save: str, payload: str) -> None:
"""Write a serialized evaluation report to ``save``, creating parent directories."""
target = Path(save).expanduser()
try:
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(payload, encoding="utf-8")
except OSError as exc:
typer.echo(f"Could not write report: {exc}", err=True)
raise typer.Exit(code=1) from exc
typer.echo(f"Report written to {target}", err=True)
def _render_eval_report(report: EvalReport, name: str, provider: str) -> None:
"""Print the per-case metric table followed by the aggregate summary lines."""
columns = report.metric_columns()
table = Table(title=f"Retrieval evaluation - {name} ({provider})")
table.add_column("Case", style="bold")
table.add_column("Status")
table.add_column("Query", overflow="fold", max_width=48)
for column in columns:
table.add_column(column, justify="right")
for item in report.evaluations:
cells = report.format_metrics(item)
table.add_row(
item.case_id,
"[red]failed[/]" if item.failed else "[green]ok[/]",
item.query,
*(cells[column] for column in columns),
)
console.print(table)
for line in report.summary_lines():
console.print(line)
def _parse_rewrite_flag(value: Optional[str]) -> Optional[bool]:
"""Map a ``--rewrite`` value to the three-state rewrite flag, or None to defer to settings.
Args:
value: ``None`` (flag omitted, defer to settings), ``"off"`` or ``"standalone"``.
Returns:
``False`` for ``"off"``, ``True`` for ``"standalone"``, ``None`` when omitted.
Raises:
typer.BadParameter: If ``value`` is not a recognized strategy.
"""
if value is None:
return None
normalized = value.strip().lower()
if normalized == "off":
return False
if normalized == "standalone":
return True
raise typer.BadParameter(
f"unknown rewrite strategy '{value}'; expected 'off' or 'standalone'",
param_hint="--rewrite",
)
def _load_eval_report(path: str) -> dict:
"""Load a saved ``kb eval --save`` report for ``kb eval-diff``, exiting readably on error.
Args:
path: Path to a report JSON file written by ``kb eval --save``.
Returns:
The parsed report as a mapping, ready for
:func:`deeptutor.services.rag.eval.diff_reports`.
Raises:
typer.Exit: If the file is missing, is not valid JSON, or is not a JSON object.
"""
target = Path(path).expanduser()
if not target.exists():
console.print(f"[red]Report not found: {target}[/]")
raise typer.Exit(code=1)
try:
loaded = json.loads(target.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
console.print(f"[red]Could not read report '{target}': {exc}[/]")
raise typer.Exit(code=1) from exc
if not isinstance(loaded, dict):
console.print(f"[red]Report '{target}' is not a JSON object.[/]")
raise typer.Exit(code=1)
return loaded
def register(app: typer.Typer) -> None:
@app.command("list")
def kb_list(
fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
) -> None:
"""List all knowledge bases."""
mgr = _get_kb_manager()
kb_names = mgr.list_knowledge_bases()
if not kb_names:
if fmt == "json":
console.print_json("[]")
else:
console.print("[dim]No knowledge bases found.[/]")
return
if fmt == "json":
items = []
for name in kb_names:
info = mgr.get_info(name)
stats = info.get("statistics", {})
metadata = info.get("metadata", {})
items.append(
{
"name": name,
"status": info.get("status", "unknown"),
"documents": stats.get("raw_documents", 0),
"rag_provider": metadata.get(
"rag_provider", stats.get("rag_provider", DEFAULT_PROVIDER)
),
"is_default": bool(info.get("is_default")),
}
)
console.print_json(json.dumps(items, ensure_ascii=False, default=str))
return
table = Table(title="Knowledge Bases")
table.add_column("Name", style="bold")
table.add_column("Status")
table.add_column("Documents", justify="right")
table.add_column("RAG Provider")
table.add_column("Default")
for name in kb_names:
info = mgr.get_info(name)
stats = info.get("statistics", {})
metadata = info.get("metadata", {})
table.add_row(
name,
str(info.get("status", "unknown")),
str(stats.get("raw_documents", 0)),
str(metadata.get("rag_provider", stats.get("rag_provider", DEFAULT_PROVIDER))),
"yes" if info.get("is_default") else "",
)
console.print(table)
@app.command("info")
def kb_info(name: str = typer.Argument(..., help="Knowledge base name.")) -> None:
"""Show details of a knowledge base."""
mgr = _get_kb_manager()
try:
info = mgr.get_info(name)
except Exception as exc:
console.print(f"[red]Knowledge base '{name}' not found: {exc}[/]")
raise typer.Exit(code=1) from exc
console.print_json(json.dumps(info, indent=2, ensure_ascii=False, default=str))
@app.command("set-default")
def kb_set_default(name: str = typer.Argument(..., help="Knowledge base name.")) -> None:
"""Set the default knowledge base."""
mgr = _get_kb_manager()
try:
mgr.set_default(name)
except Exception as exc:
console.print(f"[red]Failed to set default KB '{name}': {exc}[/]")
raise typer.Exit(code=1) from exc
console.print(f"[green]Set '{name}' as default knowledge base.[/]")
@app.command("create")
def kb_create(
name: str = typer.Argument(..., help="New KB name."),
docs: list[str] = typer.Option([], "--doc", "-d", help="Document paths."),
docs_dir: Optional[str] = typer.Option(None, "--docs-dir", help="Directory of documents."),
) -> None:
"""Initialize a new knowledge base from documents."""
mgr = _get_kb_manager()
try:
name = validate_knowledge_base_name(name)
except ValueError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
if name in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' already exists.[/]")
raise typer.Exit(code=1)
try:
doc_paths = _collect_documents(docs, docs_dir)
except typer.BadParameter as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
if not doc_paths:
console.print("[red]Provide at least one supported document (--doc or --docs-dir).[/]")
raise typer.Exit(code=1)
console.print(
f"Creating KB [bold]{name}[/] with {len(doc_paths)} document(s) via [bold]LlamaIndex[/]..."
)
from deeptutor.knowledge.initializer import initialize_knowledge_base
try:
asyncio.run(
initialize_knowledge_base(
kb_name=name,
source_files=doc_paths,
base_dir=str(mgr.base_dir),
)
)
except Exception as exc:
console.print(f"[red]KB creation failed: {exc}[/]")
raise typer.Exit(code=1) from exc
console.print("[green]Knowledge base created successfully.[/]")
@app.command("connect-kiwix")
def kb_connect_kiwix(
name: str = typer.Argument(..., help="New KB name."),
server_url: str = typer.Option(..., "--server-url", help="kiwix-serve base URL."),
zim_name: str = typer.Option(..., "--zim-name", help="One ZIM name served by Kiwix."),
) -> None:
"""Connect an existing ZIM archive without copying or indexing it."""
from deeptutor.services.rag.pipelines.kiwix.client import KiwixClient
try:
client = KiwixClient(server_url, zim_name)
title = asyncio.run(client.probe())
_get_kb_manager().register_kiwix_kb(
name, client.base_url, client.zim_name, zim_title=title
)
except Exception as exc:
console.print(f"[red]Could not connect Kiwix archive: {exc}[/]")
raise typer.Exit(code=1) from exc
console.print(f"[green]Connected '{name}' to Kiwix archive '{title}'.[/]")
@app.command("add")
def kb_add(
name: str = typer.Argument(..., help="KB name."),
docs: list[str] = typer.Option([], "--doc", "-d", help="Document paths to add."),
docs_dir: Optional[str] = typer.Option(None, "--docs-dir", help="Directory of documents."),
) -> None:
"""Add documents to an existing knowledge base."""
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
try:
doc_paths = _collect_documents(docs, docs_dir)
except typer.BadParameter as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
if not doc_paths:
console.print("[red]Provide at least one supported document.[/]")
raise typer.Exit(code=1)
console.print(f"Adding {len(doc_paths)} document(s) to [bold]{name}[/]...")
from deeptutor.knowledge.add_documents import add_documents
try:
processed_count = asyncio.run(
add_documents(
kb_name=name,
source_files=doc_paths,
base_dir=str(mgr.base_dir),
allow_duplicates=False,
)
)
except Exception as exc:
console.print(f"[red]Document upload failed: {exc}[/]")
raise typer.Exit(code=1) from exc
if processed_count:
console.print(f"[green]Done. Indexed {processed_count} document(s).[/]")
else:
console.print("[yellow]No new unique documents were indexed.[/]")
@app.command("delete")
def kb_delete(
name: str = typer.Argument(..., help="KB name."),
force: bool = typer.Option(False, "--force", "-f", help="Skip confirmation."),
) -> None:
"""Delete a knowledge base."""
if not force:
confirm = typer.confirm(f"Delete knowledge base '{name}'?")
if not confirm:
raise typer.Abort()
mgr = _get_kb_manager()
try:
deleted = mgr.delete_knowledge_base(name, confirm=True)
except Exception as exc:
console.print(f"[red]Failed to delete '{name}': {exc}[/]")
raise typer.Exit(code=1) from exc
if deleted:
console.print(f"[green]Deleted '{name}'.[/]")
else:
console.print(f"[yellow]Knowledge base '{name}' was not deleted.[/]")
@app.command("search")
def kb_search(
name: str = typer.Argument(..., help="KB name."),
query: str = typer.Argument(..., help="Search query."),
mode: str = typer.Option("hybrid", help="Search mode."),
fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
) -> None:
"""Search a knowledge base."""
from deeptutor.tools.rag_tool import rag_search
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
try:
from deeptutor.services.rag.factory import PAGEINDEX_OSS_PROVIDER, PAGEINDEX_PROVIDER
from deeptutor.services.rag.provider_binding import resolve_bound_provider
provider = resolve_bound_provider(str(mgr.base_dir), name)
if provider in {PAGEINDEX_PROVIDER, PAGEINDEX_OSS_PROVIDER}:
from deeptutor.services.rag.pipelines.pageindex.reasoning import (
read_pageindex_with_agent,
)
reading = asyncio.run(
read_pageindex_with_agent(
kb_name=name,
system_prompt=(
"Answer the CLI user's question from the selected PageIndex "
"knowledge base. Include page references from tool results."
),
user_prompt=query,
source="cli_kb_search",
stage="search",
)
)
result = {
"query": query,
"answer": reading.text,
"content": reading.text,
"sources": reading.sources,
"provider": reading.tool_context.provider,
}
else:
result = asyncio.run(
rag_search(
query=query,
kb_name=name,
mode=mode,
kb_base_dir=str(mgr.base_dir),
)
)
except Exception as exc:
console.print(f"[red]Search failed: {exc}[/]")
raise typer.Exit(code=1) from exc
if fmt == "json":
console.print_json(json.dumps(result, indent=2, ensure_ascii=False, default=str))
return
answer = result.get("answer") or result.get("content", "")
provider = str(result.get("provider", DEFAULT_PROVIDER))
console.print(f"[bold]Provider:[/] {provider}")
console.print(f"[bold]Answer:[/]\n{answer}")
@app.command("eval")
def kb_eval(
name: str = typer.Argument(..., help="KB name."),
dataset: str = typer.Option(..., "--dataset", "-d", help="QA set file (.jsonl or .json)."),
top_k: int = typer.Option(5, "--top-k", "-k", help="Rank cutoff for the metrics."),
mode: Optional[str] = typer.Option(
None, "--mode", help="Search mode for engines that support one (e.g. hybrid)."
),
limit: Optional[int] = typer.Option(
None, "--limit", help="Evaluate only the first N cases of the set."
),
min_ratio: float = typer.Option(
0.5, "--min-ratio", help="Token coverage a retrieved chunk needs to count as a match."
),
save: Optional[str] = typer.Option(None, "--save", help="Write the JSON report to a file."),
fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
rewrite: Optional[str] = typer.Option(
None,
"--rewrite",
help="Force the query-rewrite layer: off | standalone. Omit to defer to settings.",
),
) -> None:
"""Score a knowledge base's retrieval quality against a QA set."""
from deeptutor.services.rag.eval import MatchPolicy, RetrievalEvaluator
rewrite_flag = _parse_rewrite_flag(rewrite)
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
eval_set = _load_eval_dataset(dataset)
provider = _resolve_eval_provider(mgr, name)
try:
evaluator = RetrievalEvaluator(
kb_name=name,
kb_base_dir=str(mgr.base_dir),
k=top_k,
mode=mode,
query_limit=limit,
policy=MatchPolicy(min_ratio=min_ratio),
provider=provider,
rewrite=rewrite_flag,
)
except ValueError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
cases = eval_set.limited(limit)
if fmt != "json":
suffix = ""
if rewrite_flag is True:
suffix = ", rewrite=standalone"
elif rewrite_flag is False:
suffix = ", rewrite=off"
console.print(
f"[bold]Evaluating[/] '{name}' ({provider}) on {len(cases)} case(s), "
f"k={top_k}{suffix} ..."
)
progress = _build_eval_progress(top_k, fmt)
try:
report = asyncio.run(evaluator.evaluate(eval_set, progress=progress))
except Exception as exc:
console.print(f"[red]Evaluation failed: {exc}[/]")
raise typer.Exit(code=1) from exc
payload = report.to_json()
if save:
_save_eval_report(save, payload)
if fmt == "json":
console.print_json(payload)
return
_render_eval_report(report, name, provider)
@app.command("eval-diff")
def kb_eval_diff(
baseline: str = typer.Argument(..., help="Baseline report JSON (from `kb eval --save`)."),
candidate: str = typer.Argument(..., help="Candidate report JSON (from `kb eval --save`)."),
fmt: str = typer.Option("rich", "--format", "-f", help="Output format: rich | json."),
) -> None:
"""Diff two `kb eval` reports to quantify a retrieval change.
Records a baseline before a settings change (e.g. turning the query-rewrite
layer on), a candidate after, and shows the per-metric and per-case movement.
"""
from deeptutor.services.rag.eval.diff import diff_reports
base_report = _load_eval_report(baseline)
cand_report = _load_eval_report(candidate)
try:
diff = diff_reports(
base_report,
cand_report,
baseline_label=Path(baseline).stem,
candidate_label=Path(candidate).stem,
)
except ValueError as exc:
console.print(f"[red]{exc}[/]")
raise typer.Exit(code=1) from exc
if fmt == "json":
console.print_json(data=diff.to_dict())
return
for line in diff.summary_lines():
console.print(line)
if diff.has_regression:
raise typer.Exit(code=2)
# ── GitHub source commands ──────────────────────────────────────
@app.command("add-github-source")
def kb_add_github_source(
name: str = typer.Argument(..., help="KB name."),
repo: str = typer.Option(..., "--repo", "-r", help="GitHub repo as owner/name or URL."),
branch: str = typer.Option("main", "--branch", "-b", help="Branch to track."),
path: str = typer.Option("", "--path", "-p", help="Subdirectory prefix (e.g. docs/)."),
glob: str = typer.Option("*.md", "--glob", "-g", help="File glob pattern."),
) -> None:
"""Add a GitHub repo as a Markdown document source for a KB."""
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
try:
info = mgr.add_github_source(name, repo, branch, path, glob)
except Exception as exc:
console.print(f"[red]Failed: {exc}[/]")
raise typer.Exit(code=1) from exc
console.print("[green]GitHub source added:[/]")
console.print_json(json.dumps(info, ensure_ascii=False))
@app.command("remove-github-source")
def kb_remove_github_source(
name: str = typer.Argument(..., help="KB name."),
source_id: str = typer.Option(..., "--source-id", "-s", help="Source ID to remove."),
) -> None:
"""Remove a GitHub source from a KB."""
mgr = _get_kb_manager()
ok = mgr.remove_github_source(name, source_id)
if ok:
console.print(f"[green]Removed '{source_id}'.[/]")
else:
console.print(f"[yellow]Source '{source_id}' not found.[/]")
@app.command("add-web-source")
def kb_add_web_source(
name: str = typer.Argument(..., help="KB name."),
url: str = typer.Option(..., "--url", "-u", help="Documentation site base URL."),
max_depth: int = typer.Option(3, "--max-depth", help="Crawl depth."),
max_pages: int = typer.Option(200, "--max-pages", help="Max pages to crawl."),
) -> None:
"""Add a documentation site URL as a source for a KB."""
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
try:
info = mgr.add_web_source(name, url, max_depth, max_pages)
except Exception as exc:
console.print(f"[red]Failed: {exc}[/]")
raise typer.Exit(code=1) from exc
console.print("[green]Web source added:[/]")
console.print_json(json.dumps(info, ensure_ascii=False))
@app.command("remove-web-source")
def kb_remove_web_source(
name: str = typer.Argument(..., help="KB name."),
source_id: str = typer.Option(..., "--source-id", "-s", help="Source ID to remove."),
) -> None:
"""Remove a web source from a KB."""
mgr = _get_kb_manager()
removed = mgr.remove_web_source(name, source_id)
if removed:
console.print(f"[green]Removed '{source_id}'.[/]")
else:
console.print(f"[yellow]Source '{source_id}' not found.[/]")
@app.command("list-sources")
def kb_list_sources(
name: str = typer.Argument(..., help="KB name."),
) -> None:
"""List GitHub and web sources for a KB."""
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
gh_sources = mgr.get_github_sources(name)
web_sources = mgr.get_web_sources(name)
if not gh_sources and not web_sources:
console.print("[dim]No sources for this KB.[/]")
return
if gh_sources:
table = Table(title=f"GitHub Sources - {name}")
table.add_column("ID", style="dim")
table.add_column("Repo")
table.add_column("Branch")
table.add_column("Status")
table.add_column("Files", justify="right")
for source in gh_sources:
table.add_row(
source.get("id", ""),
source.get("repo", ""),
source.get("branch", ""),
source.get("last_sync_status", "pending"),
str(source.get("files_synced", 0)),
)
console.print(table)
if web_sources:
table = Table(title=f"Web Sources - {name}")
table.add_column("ID", style="dim")
table.add_column("URL")
table.add_column("Status")
table.add_column("Pages", justify="right")
for source in web_sources:
table.add_row(
source.get("id", ""),
source.get("url", ""),
source.get("last_sync_status", "pending"),
str(source.get("page_count", 0)),
)
console.print(table)
@app.command("sync")
def kb_sync(name: str = typer.Argument(..., help="KB name.")) -> None:
"""Immediately sync all GitHub and web sources for a KB."""
mgr = _get_kb_manager()
if name not in mgr.list_knowledge_bases():
console.print(f"[red]Knowledge base '{name}' not found.[/]")
raise typer.Exit(code=1)
gh_sources = mgr.get_github_sources(name)
web_sources = mgr.get_web_sources(name)
if not gh_sources and not web_sources:
console.print("[yellow]No sources to sync.[/]")
raise typer.Exit(code=0)
from deeptutor.services.github_source.sync import sync_source as gh_sync
from deeptutor.services.web_source.sync import sync_source as web_sync
for src in gh_sources:
if not src.get("enabled", True):
continue
console.print(f"Syncing GitHub [bold]{src.get('repo')}[/] ...")
try:
r = asyncio.run(gh_sync(kb_name=name, source=src, base_dir=str(mgr.base_dir)))
except Exception as exc:
console.print(f" [red]Error: {exc}[/]")
continue
if r.skipped:
console.print(" [dim]Up to date.[/]")
elif r.ok:
console.print(f" [green]+{r.files_added} ~{r.files_updated} -{r.files_removed}[/]")
else:
console.print(f" [red]Failed: {r.error}[/]")
for source in web_sources:
if not source.get("enabled", True):
continue
console.print(f"Crawling [bold]{source.get('url')}[/] ...")
try:
result = asyncio.run(
web_sync(kb_name=name, source=source, base_dir=str(mgr.base_dir))
)
except Exception as exc:
console.print(f" [red]Error: {exc}[/]")
continue
if result.ok:
console.print(
f" [green]+{result.pages_added} ~{result.pages_updated} "
f"-{result.pages_removed} ({result.pages_unchanged} unchanged)[/]"
)
else:
console.print(f" [red]Failed: {result.error}[/]")