DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""Update mode — chunk-based incremental fact extraction.
Algorithm
---------
1. Compute "new since last update" by id-set diff against ``*.meta.json``.
2. Concatenate the new inputs by time (oldest first).
3. ``chunk_with_boundary`` cuts the concat into ≤ budget pieces, never
truncating mid-paragraph (or mid-sentence, per settings).
4. For each chunk: LLM call → parse facts → filter by ref pool → append
to in-memory ``Document``.
5. Atomic flush to disk + update ``*.meta.json``.
6. If ``dedup.auto_after_update`` is set, kick off the dedup pass.
The append step uses the existing :class:`ops.AddOp` apply path so the
document's invariants (id allocation, validation, footnote rebuild on
serialize) stay centralized.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import json
import logging
from deeptutor.services.memory import paths
from deeptutor.services.memory import snapshot as snap
from deeptutor.services.memory.consolidator.chunker import (
chunk_with_boundary,
)
from deeptutor.services.memory.consolidator.meta import (
load_l2_meta,
load_l3_meta,
save_l2_meta,
save_l3_meta,
)
from deeptutor.services.memory.consolidator.modes._runtime import (
OnEvent,
call_llm,
emit,
load_doc,
load_prompt,
slot_focus,
surface_focus,
today_iso,
write_doc_checkpoint,
)
from deeptutor.services.memory.consolidator.references import (
ExtractedFact,
refs_in_span_l2,
refs_in_span_l3,
render_l2_entries_for_concat,
render_traces_for_concat,
validate_fact_refs,
)
from deeptutor.services.memory.document import Document, Entry, serialize
from deeptutor.services.memory.ops import AddOp
from deeptutor.services.memory.ops import apply as apply_ops
from deeptutor.services.memory.paths import L3Slot, Surface
from deeptutor.services.memory.settings import load_memory_settings
logger = logging.getLogger(__name__)
Layer = str # "L2" | "L3"
@dataclass
class UpdateResult:
layer: Layer
key: str
chunks_processed: int
facts_added: int
refs_dropped: int
new_entry_ids: list[str] = field(default_factory=list)
no_new_input: bool = False
# ── Public entry ────────────────────────────────────────────────────────
async def run_update(
layer: Layer,
key: str,
*,
language: str = "en",
user_label: str = "anonymous",
budget: int | None = None,
llm_selection: dict | None = None,
on_event: OnEvent | None = None,
) -> UpdateResult:
"""Dispatch to the layer-specific update implementation.
The chosen ``llm_selection`` (``{profile_id, model_id}``) is
installed as a scoped LLM config for the duration of the run so
every internal :func:`call_llm` resolves to the right provider.
"""
from deeptutor.services.model_selection.runtime import (
activate_llm_selection,
reset_llm_selection,
)
settings = load_memory_settings()
token = None
if llm_selection:
try:
_config, token = activate_llm_selection(llm_selection)
except Exception as exc: # noqa: BLE001
logger.warning(
"memory update: ignoring unresolvable llm_selection %s: %s", llm_selection, exc
)
token = None
try:
if layer == "L2":
return await _run_update_l2(
key, # type: ignore[arg-type]
language=language,
user_label=user_label,
budget=budget if budget is not None else settings.update.l2_budget,
llm_selection=llm_selection,
on_event=on_event,
settings=settings,
)
if layer == "L3":
return await _run_update_l3(
key, # type: ignore[arg-type]
language=language,
user_label=user_label,
budget=budget if budget is not None else settings.update.l3_budget,
llm_selection=llm_selection,
on_event=on_event,
settings=settings,
)
raise ValueError(f"unknown layer {layer!r}")
finally:
reset_llm_selection(token)
# ── L2 ──────────────────────────────────────────────────────────────────
async def _run_update_l2(
surface: Surface,
*,
language: str,
user_label: str,
budget: int,
llm_selection: dict | None,
on_event: OnEvent | None,
settings,
) -> UpdateResult:
meta = load_l2_meta(surface)
all_entities = sorted(
snap.read_snapshot(surface),
key=lambda e: (e.ts or "", e.id),
)
seen = meta.seen_entity_refs
new_entities = [e for e in all_entities if f"{surface}:{e.id}" not in seen]
seen_now = {f"{surface}:{e.id}" for e in all_entities}
await emit(
on_event,
{
"stage": "trace_loaded",
"surface": surface,
"total": len(all_entities),
"new": len(new_entities),
},
)
if not new_entities:
# Still persist a fresh meta so the "last_update_at" timestamp
# moves; this signals "we checked, nothing new".
save_l2_meta(surface, seen_entity_refs=seen_now)
# Even when no facts were added we still run merge — the doc may
# be in the legacy entry-keyed footnote layout and the user
# expects "click update" to clean it up.
if settings.merge.auto_after_update:
from deeptutor.services.memory.consolidator.modes.merge import run_merge
await run_merge(
"L2",
surface,
language=language,
user_label=user_label,
on_event=on_event,
)
await emit(on_event, {"stage": "done", "no_new_input": True, "facts_added": 0})
return UpdateResult(
layer="L2",
key=surface,
chunks_processed=0,
facts_added=0,
refs_dropped=0,
no_new_input=True,
)
text = render_traces_for_concat(new_entities, surface=surface)
chunks = chunk_with_boundary(
text,
budget=budget,
overlap_ratio=settings.chunking.overlap_ratio,
min_chunk_chars=settings.chunking.min_chunk_chars,
max_chunk_chars=settings.chunking.max_chunk_chars,
boundary=settings.chunking.boundary,
)
await emit(
on_event,
{"stage": "chunked", "chunks": len(chunks), "budget": budget, "chars": len(text)},
)
prompt = load_prompt("update_l2", language)
focus, sections = surface_focus(language, surface)
l2_path = paths.l2_file(surface)
doc = load_doc(l2_path, default_title=f"{surface} memory")
facts_added = 0
refs_dropped = 0
new_entry_ids: list[str] = []
for chunk in chunks:
await emit(
on_event,
{
"stage": "progress",
"mode": "update",
"turn": chunk.index + 1,
"total": len(chunks),
"chunk_start": chunk.start,
"chunk_end": chunk.end,
},
)
system = prompt["system"].format(
user_label=user_label,
surface=surface,
sections=", ".join(sections) if sections else "(any)",
focus=focus,
today=today_iso(),
)
allowed = refs_in_span_l2(
new_entities,
surface=surface,
full_text=text,
start=chunk.start,
end=chunk.end,
)
user = prompt["user"].format(
surface=surface,
existing=_render_existing_l2(doc),
chunk=_chunk_with_ref_header(chunk.text, allowed),
chunk_index=chunk.index + 1,
chunk_total=len(chunks),
chunk_start=chunk.start,
chunk_end=chunk.end,
)
raw = await call_llm(
system_prompt=system,
user_prompt=user,
on_event=on_event,
turn=chunk.index + 1,
chunk_index=chunk.index,
label="update",
)
facts = _parse_facts(raw)
kept_in_chunk: list[ExtractedFact] = []
for fact in facts:
kept_refs, reject_reason = validate_fact_refs(
fact,
allowed=allowed,
enforce_required=settings.reference.enforce_required,
drop_invalid=settings.reference.drop_invalid_refs,
)
if reject_reason is not None:
refs_dropped += 1
await emit(
on_event,
{
"stage": "refs_dropped",
"turn": chunk.index + 1,
"reason": reject_reason,
"text": fact.text[:120],
},
)
continue
kept_in_chunk.append(
ExtractedFact(text=fact.text, refs=kept_refs, section=fact.section)
)
added_now = _append_facts_to_doc(doc, kept_in_chunk, sections)
facts_added += len(added_now)
new_entry_ids.extend(added_now)
if added_now:
await write_doc_checkpoint(
l2_path,
doc,
layer="L2",
key=surface,
on_event=on_event,
turn=chunk.index + 1,
label="update",
action="append_facts",
)
await emit(
on_event,
{
"stage": "facts_extracted",
"turn": chunk.index + 1,
"kept": len(kept_in_chunk),
"added": len(added_now),
},
)
save_l2_meta(surface, seen_entity_refs=seen_now)
await emit(
on_event,
{
"stage": "done",
"facts_added": facts_added,
"refs_dropped": refs_dropped,
"chunks_processed": len(chunks),
"auto_dedup": settings.dedup.auto_after_update,
},
)
if settings.dedup.auto_after_update and facts_added > 0:
# Avoid a circular import: dedup imports settings, refs, line_doc.
from deeptutor.services.memory.consolidator.modes.dedup import run_dedup
await run_dedup(
"L2",
surface,
language=language,
user_label=user_label,
iterations=settings.dedup.iterations,
llm_selection=llm_selection,
on_event=on_event,
)
if settings.merge.auto_after_update:
from deeptutor.services.memory.consolidator.modes.merge import run_merge
await run_merge(
"L2",
surface,
language=language,
user_label=user_label,
on_event=on_event,
)
return UpdateResult(
layer="L2",
key=surface,
chunks_processed=len(chunks),
facts_added=facts_added,
refs_dropped=refs_dropped,
new_entry_ids=new_entry_ids,
)
# ── L3 ──────────────────────────────────────────────────────────────────
async def _run_update_l3(
slot: L3Slot,
*,
language: str,
user_label: str,
budget: int,
llm_selection: dict | None,
on_event: OnEvent | None,
settings,
) -> UpdateResult:
if slot == "preferences":
raise ValueError("preferences.md is not auto-consolidated")
meta = load_l3_meta(slot)
l2_docs = _load_all_l2_docs()
entries_by_surface: dict[str, list[Entry]] = {}
seen_now: dict[str, set[str]] = {}
for surface, doc in l2_docs.items():
all_entries = doc.all_entries()
seen_now[surface] = {e.id for e in all_entries}
# Sort by id (ULID) ascending → roughly time-ascending.
new_entries = sorted(
(e for e in all_entries if e.id not in meta.seen_l2_entry_ids.get(surface, set())),
key=lambda e: e.id,
)
entries_by_surface[surface] = new_entries
new_count = sum(len(v) for v in entries_by_surface.values())
total_count = sum(len(d.all_entries()) for d in l2_docs.values())
await emit(
on_event,
{
"stage": "trace_loaded",
"slot": slot,
"total_l2_entries": total_count,
"new_l2_entries": new_count,
},
)
if new_count == 0:
save_l3_meta(slot, seen_l2_entry_ids=seen_now)
if settings.merge.auto_after_update:
from deeptutor.services.memory.consolidator.modes.merge import run_merge
await run_merge(
"L3",
slot,
language=language,
user_label=user_label,
on_event=on_event,
)
await emit(on_event, {"stage": "done", "no_new_input": True, "facts_added": 0})
return UpdateResult(
layer="L3",
key=slot,
chunks_processed=0,
facts_added=0,
refs_dropped=0,
no_new_input=True,
)
text = render_l2_entries_for_concat(entries_by_surface)
chunks = chunk_with_boundary(
text,
budget=budget,
overlap_ratio=settings.chunking.overlap_ratio,
min_chunk_chars=settings.chunking.min_chunk_chars,
max_chunk_chars=settings.chunking.max_chunk_chars,
boundary=settings.chunking.boundary,
)
await emit(
on_event,
{"stage": "chunked", "chunks": len(chunks), "budget": budget, "chars": len(text)},
)
prompt = load_prompt("update_l3", language)
focus, sections = slot_focus(language, slot)
l3_path = paths.l3_file(slot)
doc = load_doc(l3_path, default_title=_default_l3_title(slot))
facts_added = 0
refs_dropped = 0
new_entry_ids: list[str] = []
for chunk in chunks:
await emit(
on_event,
{
"stage": "progress",
"mode": "update",
"turn": chunk.index + 1,
"total": len(chunks),
"chunk_start": chunk.start,
"chunk_end": chunk.end,
},
)
system = prompt["system"].format(
user_label=user_label,
slot=slot,
sections=", ".join(sections) if sections else "(any)",
focus=focus,
today=today_iso(),
)
# L3 refs are *surface names* (chat / notebook / ...). The pool
# is whichever surface blocks intersect this chunk; the LLM is
# told to cite from that list. Per-entry-id provenance was
# explicitly dropped — L3 points at L2 *files*, not L2 entries,
# which gives the user a clean 7-footnote chain
# (L3 → L2 md → L1 raw traces).
allowed = refs_in_span_l3(
entries_by_surface=entries_by_surface,
full_text=text,
start=chunk.start,
end=chunk.end,
)
user = prompt["user"].format(
slot=slot,
existing=_render_existing_l3(doc),
chunk=_chunk_with_ref_header(chunk.text, allowed),
chunk_index=chunk.index + 1,
chunk_total=len(chunks),
)
raw = await call_llm(
system_prompt=system,
user_prompt=user,
on_event=on_event,
turn=chunk.index + 1,
chunk_index=chunk.index,
label="update",
)
facts = _parse_facts(raw)
kept_in_chunk: list[ExtractedFact] = []
for fact in facts:
kept_refs, reject_reason = validate_fact_refs(
fact,
allowed=allowed,
enforce_required=settings.reference.enforce_required,
drop_invalid=settings.reference.drop_invalid_refs,
)
if reject_reason is not None:
refs_dropped += 1
await emit(
on_event,
{
"stage": "refs_dropped",
"turn": chunk.index + 1,
"reason": reject_reason,
"text": fact.text[:120],
},
)
continue
kept_in_chunk.append(
ExtractedFact(text=fact.text, refs=kept_refs, section=fact.section)
)
added_now = _append_facts_to_doc(doc, kept_in_chunk, sections)
facts_added += len(added_now)
new_entry_ids.extend(added_now)
if added_now:
await write_doc_checkpoint(
l3_path,
doc,
layer="L3",
key=slot,
on_event=on_event,
turn=chunk.index + 1,
label="update",
action="append_facts",
)
await emit(
on_event,
{
"stage": "facts_extracted",
"turn": chunk.index + 1,
"kept": len(kept_in_chunk),
"added": len(added_now),
},
)
save_l3_meta(slot, seen_l2_entry_ids=seen_now)
await emit(
on_event,
{
"stage": "done",
"facts_added": facts_added,
"refs_dropped": refs_dropped,
"chunks_processed": len(chunks),
"auto_dedup": settings.dedup.auto_after_update,
},
)
if settings.dedup.auto_after_update and facts_added > 0:
from deeptutor.services.memory.consolidator.modes.dedup import run_dedup
await run_dedup(
"L3",
slot,
language=language,
user_label=user_label,
iterations=settings.dedup.iterations,
llm_selection=llm_selection,
on_event=on_event,
)
if settings.merge.auto_after_update:
from deeptutor.services.memory.consolidator.modes.merge import run_merge
await run_merge(
"L3",
slot,
language=language,
user_label=user_label,
on_event=on_event,
)
return UpdateResult(
layer="L3",
key=slot,
chunks_processed=len(chunks),
facts_added=facts_added,
refs_dropped=refs_dropped,
new_entry_ids=new_entry_ids,
)
# ── Helpers ─────────────────────────────────────────────────────────────
def _parse_facts(raw: str) -> list[ExtractedFact]:
"""Tolerant JSON parse → list[ExtractedFact]. Empty on any failure."""
if not raw:
return []
snippet = _extract_json_object(raw)
if snippet is None:
return []
try:
data = json.loads(snippet)
except json.JSONDecodeError:
return []
items = data.get("facts") if isinstance(data, dict) else None
if not isinstance(items, list):
return []
out: list[ExtractedFact] = []
for item in items:
if not isinstance(item, dict):
continue
text = str(item.get("text", "")).strip()
section = str(item.get("section", "")).strip()
refs_raw = item.get("refs", [])
refs = [str(r).strip() for r in (refs_raw if isinstance(refs_raw, list) else []) if r]
if not text:
continue
out.append(ExtractedFact(text=text, refs=refs, section=section))
return out
def _extract_json_object(raw: str) -> str | None:
text = raw.strip()
if text.startswith("```"):
text = text.split("\n", 1)[1] if "\n" in text else text
if text.rstrip().endswith("```"):
text = text.rstrip()[:-3]
start = text.find("{")
if start == -1:
return None
try:
_parsed, end = json.JSONDecoder().raw_decode(text[start:])
except json.JSONDecodeError:
return None
return text[start : start + end]
def _append_facts_to_doc(
doc: Document, facts: list[ExtractedFact], allowed_sections: list[str]
) -> list[str]:
"""Append each fact as one AddOp; return the new entry ids."""
new_ids: list[str] = []
fallback_section = allowed_sections[0] if allowed_sections else "Notes"
for fact in facts:
section = fact.section if fact.section else fallback_section
if allowed_sections and section not in allowed_sections:
# Map an off-list section into the first allowed one — keeps
# the section catalog stable across runs.
section = fallback_section
op = AddOp(section=section, text=fact.text, refs=fact.refs)
report = apply_ops(doc, [op])
if report.accepted and report.results:
new_id = report.results[0].entry_id
if new_id:
new_ids.append(new_id)
else:
logger.warning(
"update: skipped fact (%s): %s",
report.reason,
fact.text[:80],
)
return new_ids
def _render_existing_l2(doc: Document) -> str:
if not doc.all_entries():
return "(empty — first run)"
return serialize(doc).strip()
def _render_existing_l3(doc: Document) -> str:
if not doc.all_entries():
return "(empty — first run)"
return serialize(doc).strip()
def _chunk_with_ref_header(chunk_text: str, allowed: set[str]) -> str:
if not allowed:
return chunk_text
refs = "\n".join(f"- {ref}" for ref in sorted(allowed))
return f"# Chunk-local citeable refs\n{refs}\n\n{chunk_text}"
def _load_all_l2_docs() -> dict[str, Document]:
from deeptutor.services.memory.document import parse
docs: dict[str, Document] = {}
for surface in paths.SURFACES:
path = paths.l2_file(surface)
if not path.exists():
continue
try:
docs[surface] = parse(path.read_text(encoding="utf-8"))
except Exception:
continue
return docs
def _default_l3_title(slot: L3Slot) -> str:
return {
"recent": "Recent summary",
"profile": "User profile",
"scope": "Knowledge scope",
"preferences": "Preferences",
}[slot]
__all__ = ["UpdateResult", "run_update"]