DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(memory): aggregate account-level L1 across workspaces (#1799) (#1800)
ZQR11015b2b99fab99f
diff --git a/deeptutor/services/memory/snapshot/adapters.py b/deeptutor/services/memory/snapshot/adapters.pyindex 46727ab2a..298cb8a49 100644--- a/deeptutor/services/memory/snapshot/adapters.py+++ b/deeptutor/services/memory/snapshot/adapters.py@@ -17,11 +17,13 @@ from datetime import datetime, timezoneimport hashlibimport jsonimport logging+from pathlib import Pathimport sqlite3+from typing import Anyfrom deeptutor.services.memory.paths import Surfacefrom deeptutor.services.memory.snapshot.entity import Entity, EntityStamp-from deeptutor.services.path_service import get_path_service+from deeptutor.services.path_service import PathService, get_path_servicelogger = logging.getLogger(__name__)@@ -58,6 +60,50 @@ def _iso(ts: float | int | str | None) -> str:return ""+def _iter_chat_dbs() -> list[tuple[str, Path]]:+ """``(qualifier, chat-history db)`` pairs across the account's workspaces.++ The default workspace comes first with an empty qualifier so existing+ snapshots keep their bare session ids — renumbering them would churn every+ account-level snapshot (#1799). Each additional registered workspace+ contributes its own chat db under a ``ws:<workspace_id>:`` qualifier, which+ is what makes custom-workspace conversations visible to account-level+ Memory without id collisions against the default workspace.++ The account's own data root supplies the default entry; binding data roots+ are derived the same way :class:`WorkspacePathService` derives them+ (``<content_root>/.deeptutor/data``), with the same symlink refusal — a+ redirected store is skipped rather than raised, so one broken binding+ cannot blank an account-wide scan.+ """+ pairs: list[tuple[str, Path]] = [("", get_path_service().get_chat_history_db())]+ try:+ from deeptutor.services.workspace import get_content_workspace_service++ bindings = get_content_workspace_service().registered_bindings()+ except Exception:+ logger.warning(+ "workspace binding enumeration failed; default workspace only", exc_info=True+ )+ return pairs+ for binding in bindings:+ private_root = (binding.root / ".deeptutor").resolve()+ data_root = private_root / "data"+ if private_root.is_symlink() or data_root.is_symlink():+ logger.warning(+ "workspace %s skipped from memory aggregation: redirected data store",+ binding.workspace_id,+ )+ continue+ pairs.append(+ (+ f"ws:{binding.workspace_id}:",+ PathService(workspace_root=data_root).get_chat_history_db(),+ )+ )+ return pairs++# ── Adapters ─────────────────────────────────────────────────────────@@ -406,109 +452,122 @@ def read_kb_entities() -> list[Entity]:def read_chat_entities() -> list[Entity]:- """One Entity per chat session. ``content`` inlines all turns as- ``user / assistant`` blocks so L2 sees the actual conversation."""- db_path = get_path_service().get_chat_history_db()- if not db_path.exists():- return []+ """One Entity per chat session across the account's workspaces.++ ``content`` inlines all turns as ``user / assistant`` blocks so L2 sees the+ actual conversation. Sessions from the default workspace keep their bare+ session id; sessions from additional workspaces carry a ``ws:<id>:``+ qualifier plus a ``workspace_id`` in metadata, so account-level Memory+ sees every conversation with provenance and no id collisions (#1799).+ """out: list[Entity] = []- try:- with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:- conn.row_factory = sqlite3.Row- sessions = conn.execute(- "SELECT id, title, created_at, updated_at FROM sessions ORDER BY updated_at DESC"- ).fetchall()- for sess in sessions:- sid = sess["id"]- msgs = conn.execute(- "SELECT id, role, content, capability, created_at "- "FROM messages WHERE session_id = ? "- "ORDER BY created_at ASC, id ASC",- (sid,),+ for qualifier, db_path in _iter_chat_dbs():+ if not db_path.exists():+ continue+ try:+ with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:+ conn.row_factory = sqlite3.Row+ sessions = conn.execute(+ "SELECT id, title, created_at, updated_at FROM sessions ORDER BY updated_at DESC").fetchall()- blocks: list[str] = []- for m in msgs:- role = m["role"]- body = (m["content"] or "").strip()- if not body:- continue- blocks.append(f"### {role}\n{body}")- content = "\n\n".join(blocks)- last_msg_id = msgs[-1]["id"] if msgs else 0- out.append(- Entity(- id=sid,- label=sess["title"] or sid,- ts=_iso(sess["updated_at"]),- content=content,- metadata={- "session_id": sid,- "message_count": len(msgs),- },- fingerprint=_sha1(last_msg_id, sess["updated_at"]),+ for sess in sessions:+ sid = sess["id"]+ msgs = conn.execute(+ "SELECT id, role, content, capability, created_at "+ "FROM messages WHERE session_id = ? "+ "ORDER BY created_at ASC, id ASC",+ (sid,),+ ).fetchall()+ blocks: list[str] = []+ for m in msgs:+ role = m["role"]+ body = (m["content"] or "").strip()+ if not body:+ continue+ blocks.append(f"### {role}\n{body}")+ content = "\n\n".join(blocks)+ last_msg_id = msgs[-1]["id"] if msgs else 0+ metadata: dict[str, Any] = {+ "session_id": sid,+ "message_count": len(msgs),+ }+ if qualifier:+ metadata["workspace_id"] = qualifier.removeprefix("ws:").removesuffix(":")+ out.append(+ Entity(+ id=f"{qualifier}{sid}",+ label=sess["title"] or sid,+ ts=_iso(sess["updated_at"]),+ content=content,+ metadata=metadata,+ fingerprint=_sha1(last_msg_id, sess["updated_at"]),+ ))- )- except sqlite3.Error as exc:- logger.warning("chat snapshot scan failed: %s", exc)- return []+ except sqlite3.Error as exc:+ logger.warning("chat snapshot scan failed for %s: %s", db_path, exc)+ continuereturn outdef read_quiz_entities() -> list[Entity]:- """One Entity per recorded quiz attempt (notebook_entries row)."""- db_path = get_path_service().get_chat_history_db()- if not db_path.exists():- return []+ """One Entity per recorded quiz attempt (notebook_entries row), across+ the account's workspaces with the same qualifier scheme as chat (#1799)."""out: list[Entity] = []- try:- with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:- conn.row_factory = sqlite3.Row- rows = conn.execute(- "SELECT id, session_id, turn_id, question_id, question, "- "question_type, options_json, correct_answer, explanation, "- "difficulty, user_answer, is_correct, bookmarked, "- "created_at FROM notebook_entries "- "ORDER BY created_at DESC"- ).fetchall()- for r in rows:- qid = r["question_id"] or f"row_{r['id']}"- entity_id = f"{r['session_id']}:{qid}"- question = (r["question"] or "").strip()- user_answer = (r["user_answer"] or "").strip()- correct = (r["correct_answer"] or "").strip()- explanation = (r["explanation"] or "").strip()- is_correct = bool(int(r["is_correct"] or 0))- content = "\n\n".join(- part- for part in (- f"**Question**: {question}" if question else "",- f"**User answer**: {user_answer}" if user_answer else "",- f"**Correct answer**: {correct}" if correct else "",- f"**Explanation**: {explanation}" if explanation else "",+ for qualifier, db_path in _iter_chat_dbs():+ if not db_path.exists():+ continue+ try:+ with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:+ conn.row_factory = sqlite3.Row+ rows = conn.execute(+ "SELECT id, session_id, turn_id, question_id, question, "+ "question_type, options_json, correct_answer, explanation, "+ "difficulty, user_answer, is_correct, bookmarked, "+ "created_at FROM notebook_entries "+ "ORDER BY created_at DESC"+ ).fetchall()+ for r in rows:+ qid = r["question_id"] or f"row_{r['id']}"+ entity_id = f"{qualifier}{r['session_id']}:{qid}"+ question = (r["question"] or "").strip()+ user_answer = (r["user_answer"] or "").strip()+ correct = (r["correct_answer"] or "").strip()+ explanation = (r["explanation"] or "").strip()+ is_correct = bool(int(r["is_correct"] or 0))+ content = "\n\n".join(+ part+ for part in (+ f"**Question**: {question}" if question else "",+ f"**User answer**: {user_answer}" if user_answer else "",+ f"**Correct answer**: {correct}" if correct else "",+ f"**Explanation**: {explanation}" if explanation else "",+ )+ if part)- if part- )- out.append(- Entity(- id=entity_id,- label=question[:80] or qid,- ts=_iso(r["created_at"]),- content=content,- metadata={- "session_id": r["session_id"],- "turn_id": r["turn_id"],- "question_id": qid,- "question_type": r["question_type"],- "difficulty": r["difficulty"],- "is_correct": is_correct,- "bookmarked": bool(int(r["bookmarked"] or 0)),- },- fingerprint=_sha1(question, user_answer, correct, is_correct),+ metadata: dict[str, Any] = {+ "session_id": r["session_id"],+ "turn_id": r["turn_id"],+ "question_id": qid,+ "question_type": r["question_type"],+ "difficulty": r["difficulty"],+ "is_correct": is_correct,+ "bookmarked": bool(int(r["bookmarked"] or 0)),+ }+ if qualifier:+ metadata["workspace_id"] = qualifier.removeprefix("ws:").removesuffix(":")+ out.append(+ Entity(+ id=entity_id,+ label=question[:80] or qid,+ ts=_iso(r["created_at"]),+ content=content,+ metadata=metadata,+ fingerprint=_sha1(question, user_answer, correct, is_correct),+ ))- )- except sqlite3.Error as exc:- logger.warning("quiz snapshot scan failed: %s", exc)- return []+ except sqlite3.Error as exc:+ logger.warning("quiz snapshot scan failed for %s: %s", db_path, exc)+ continuereturn out@@ -529,39 +588,42 @@ def probe_chat_entities() -> list[EntityStamp]:Skipping ``content`` is the whole point: the full read concatenates everymessage of every session, which is the wrong price to pay for a caller thatonly wants to know what the sessions are called.++ Iterates the same workspace-qualified chat dbs as :func:`read_chat_entities`+ (#1799) — the qualifier expression must stay mirrored exactly."""- db_path = get_path_service().get_chat_history_db()- if not db_path.exists():- return []out: list[EntityStamp] = []- try:- with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:- conn.row_factory = sqlite3.Row- last_msg_id: dict[str, int] = {- row["session_id"]: row["id"]- for row in conn.execute(- "SELECT session_id, id FROM ("- " SELECT session_id, id, ROW_NUMBER() OVER ("- " PARTITION BY session_id ORDER BY created_at DESC, id DESC"- " ) AS rn FROM messages"- ") WHERE rn = 1"- )- }- for sess in conn.execute(- "SELECT id, title, updated_at FROM sessions ORDER BY updated_at DESC"- ):- sid = sess["id"]- out.append(- EntityStamp(- id=sid,- label=sess["title"] or sid,- fingerprint=_sha1(last_msg_id.get(sid, 0), sess["updated_at"]),- ts=_iso(sess["updated_at"]),+ for qualifier, db_path in _iter_chat_dbs():+ if not db_path.exists():+ continue+ try:+ with sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) as conn:+ conn.row_factory = sqlite3.Row+ last_msg_id: dict[str, int] = {+ row["session_id"]: row["id"]+ for row in conn.execute(+ "SELECT session_id, id FROM ("+ " SELECT session_id, id, ROW_NUMBER() OVER ("+ " PARTITION BY session_id ORDER BY created_at DESC, id DESC"+ " ) AS rn FROM messages"+ ") WHERE rn = 1")- )- except sqlite3.Error as exc:- logger.warning("chat snapshot probe failed: %s", exc)- return []+ }+ for sess in conn.execute(+ "SELECT id, title, updated_at FROM sessions ORDER BY updated_at DESC"+ ):+ sid = sess["id"]+ out.append(+ EntityStamp(+ id=f"{qualifier}{sid}",+ label=sess["title"] or sid,+ fingerprint=_sha1(last_msg_id.get(sid, 0), sess["updated_at"]),+ ts=_iso(sess["updated_at"]),+ )+ )+ except sqlite3.Error as exc:+ logger.warning("chat snapshot probe failed for %s: %s", db_path, exc)+ continuereturn outdiff --git a/deeptutor/services/workspace/catalog.py b/deeptutor/services/workspace/catalog.pyindex be98a5c8f..2fca501b5 100644--- a/deeptutor/services/workspace/catalog.py+++ b/deeptutor/services/workspace/catalog.py@@ -10,6 +10,7 @@ from __future__ import annotationsfrom contextlib import contextmanagerfrom datetime import datetime, timezoneimport json+import loggingimport osfrom pathlib import Pathimport re@@ -23,6 +24,8 @@ from deeptutor.multi_user.context import get_current_userfrom deeptutor.services.workspace.models import WorkspaceBinding, WorkspaceErrorfrom deeptutor.utils.secret_files import ensure_private_directory+logger = logging.getLogger(__name__)+_INTERNAL_DIR = ".deeptutor"@@ -134,6 +137,28 @@ class WorkspaceCatalogMixin:self._assert_allowed_root(root)return WorkspaceBinding(row["workspace_id"], root, row["display_name"])+ def registered_bindings(self) -> list[WorkspaceBinding]:+ """All non-default workspaces registered for the current owner.++ Read-only enumeration for account-level aggregation (e.g. Memory L1+ snapshots, #1799). Bindings whose storage moved or grew symlinks are+ skipped rather than raised: one broken registration must not blank an+ account-wide scan.+ """+ out: list[WorkspaceBinding] = []+ for row in self._catalog():+ try:+ binding = self._registered_binding(row)+ except WorkspaceError:+ logger.warning(+ "workspace %s skipped during enumeration: storage unavailable",+ row.get("workspace_id"),+ )+ continue+ if not binding.is_default:+ out.append(binding)+ return out+def _catalog_metadata(self, key: str) -> Any:if not self._catalog_file().exists():return Nonediff --git a/tests/services/memory/test_snapshot_workspace_aggregation.py b/tests/services/memory/test_snapshot_workspace_aggregation.pynew file mode 100644index 000000000..dedc67880--- /dev/null+++ b/tests/services/memory/test_snapshot_workspace_aggregation.py@@ -0,0 +1,253 @@+"""Account-level Memory aggregation across workspaces (#1799).++Memory L1 is documented as account-level shared, but the chat/quiz adapters+used to read only the currently scoped workspace's chat-history DB, so+conversations started inside a custom workspace were invisible to Memory.+These tests pin the aggregation contract:++- the default workspace keeps **bare** session ids — renumbering them would+ churn every existing snapshot;+- additional registered workspaces contribute their sessions under a+ ``ws:<workspace_id>:`` qualifier with workspace provenance in metadata;+- a probe and its full read must agree on identity across workspaces;+- a broken binding is skipped (fail-open) and never blanks the scan.+"""++from __future__ import annotations++from pathlib import Path+import sqlite3+from types import SimpleNamespace++import pytest++from deeptutor.services.memory.snapshot import adapters+from deeptutor.services.path_service import PathService+++class _FakePathService:+ def __init__(self, db_path: Path) -> None:+ self._db_path = db_path++ def get_chat_history_db(self) -> Path:+ return self._db_path+++def _make_db(path: Path) -> sqlite3.Connection:+ conn = sqlite3.connect(path)+ conn.executescript(+ """+ CREATE TABLE sessions (+ id TEXT PRIMARY KEY,+ title TEXT,+ created_at REAL,+ updated_at REAL+ );+ CREATE TABLE messages (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ session_id TEXT,+ role TEXT,+ content TEXT,+ capability TEXT,+ created_at REAL+ );+ CREATE TABLE notebook_entries (+ id INTEGER PRIMARY KEY AUTOINCREMENT,+ session_id TEXT,+ turn_id TEXT,+ question_id TEXT,+ question TEXT,+ question_type TEXT,+ options_json TEXT,+ correct_answer TEXT,+ explanation TEXT,+ difficulty TEXT,+ user_answer TEXT,+ is_correct INTEGER,+ bookmarked INTEGER,+ created_at REAL+ );+ """+ )+ return conn+++def _add_session(+ conn: sqlite3.Connection, sid: str, title: str, *, question: str | None = None+) -> None:+ conn.execute(+ "INSERT INTO sessions (id, title, created_at, updated_at) VALUES (?, ?, 1000.0, 1200.0)",+ (sid, title),+ )+ conn.execute(+ "INSERT INTO messages (session_id, role, content, capability, created_at) "+ "VALUES (?, 'user', ?, 'chat', 1100.0)",+ (sid, f"about {title}"),+ )+ if question is not None:+ conn.execute(+ "INSERT INTO notebook_entries (session_id, turn_id, question_id, question, "+ "question_type, correct_answer, user_answer, is_correct, bookmarked, created_at) "+ "VALUES (?, 't1', 'q1', ?, 'choice', 'A', 'B', 0, 0, 1150.0)",+ (sid, question),+ )+ # The adapters read through a *separate* read-only connection; uncommitted+ # rows would be invisible to it.+ conn.commit()+++def _binding(workspace_id: str, ws_root: Path) -> SimpleNamespace:+ return SimpleNamespace(workspace_id=workspace_id, root=ws_root, display_name=workspace_id)+++def _workspace_data_root(ws_root: Path) -> Path:+ # Mirror WorkspacePathService: data lives under <content_root>/.deeptutor/data.+ return ws_root / ".deeptutor" / "data"+++def _with_bindings(+ monkeypatch: pytest.MonkeyPatch,+ default_db: Path,+ bindings: list[SimpleNamespace],+) -> None:+ monkeypatch.setattr(adapters, "get_path_service", lambda: _FakePathService(default_db))+ monkeypatch.setattr(+ "deeptutor.services.workspace.get_content_workspace_service",+ lambda: SimpleNamespace(registered_bindings=lambda: bindings),+ )+++def default_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> sqlite3.Connection:+ db_path = tmp_path / "default" / "chat_history.db"+ db_path.parent.mkdir(parents=True)+ conn = _make_db(db_path)+ _with_bindings(monkeypatch, db_path, [])+ yield conn+ conn.close()+++def test_default_workspace_ids_stay_bare(default_db: sqlite3.Connection) -> None:+ _add_session(default_db, "s1", "Chain rule")++ entities = adapters.read_chat_entities()++ assert [e.id for e in entities] == ["s1"] # bare id — no snapshot churn+++def test_custom_workspace_sessions_are_aggregated_with_provenance(+ tmp_path: Path, default_db: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch+) -> None:+ ws_root = tmp_path / "ws-thesis"+ db_path = PathService(workspace_root=_workspace_data_root(ws_root)).get_chat_history_db()+ db_path.parent.mkdir(parents=True, exist_ok=True)+ ws_conn = _make_db(db_path)+ try:+ _add_session(default_db, "s-default", "Default chat")+ _add_session(ws_conn, "s1", "Thesis chapter")+ _with_bindings(+ monkeypatch,+ tmp_path / "default" / "chat_history.db",+ [_binding("ws_abc123", ws_root)],+ )++ entities = adapters.read_chat_entities()+ finally:+ ws_conn.close()++ by_id = {e.id: e for e in entities}+ # Both workspaces visible; the same session id in two workspaces never+ # collides because the custom one is qualified.+ assert set(by_id) == {"s-default", "ws:ws_abc123:s1"}+ qualified = by_id["ws:ws_abc123:s1"]+ assert qualified.label == "Thesis chapter"+ assert qualified.metadata["workspace_id"] == "ws_abc123"+ # The default workspace entry stays bare.+ assert by_id["s-default"].metadata.get("workspace_id") is None+++def test_quiz_entities_carry_the_same_qualifier(+ tmp_path: Path, default_db: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch+) -> None:+ ws_root = tmp_path / "ws-quiz"+ db_path = PathService(workspace_root=_workspace_data_root(ws_root)).get_chat_history_db()+ db_path.parent.mkdir(parents=True, exist_ok=True)+ ws_conn = _make_db(db_path)+ try:+ _add_session(ws_conn, "s9", "Quiz chapter", question="What is entropy?")+ _with_bindings(+ monkeypatch,+ tmp_path / "default" / "chat_history.db",+ [_binding("ws_quiz1", ws_root)],+ )++ entities = adapters.read_quiz_entities()+ finally:+ ws_conn.close()++ assert [e.id for e in entities] == ["ws:ws_quiz1:s9:q1"]+ assert entities[0].metadata["workspace_id"] == "ws_quiz1"+++def test_probe_agrees_with_full_read_across_workspaces(+ tmp_path: Path, default_db: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch+) -> None:+ ws_root = tmp_path / "ws-probe"+ db_path = PathService(workspace_root=_workspace_data_root(ws_root)).get_chat_history_db()+ db_path.parent.mkdir(parents=True, exist_ok=True)+ ws_conn = _make_db(db_path)+ try:+ _add_session(default_db, "s-default", "Default chat")+ _add_session(ws_conn, "s1", "Shared id across workspaces")+ _with_bindings(+ monkeypatch,+ tmp_path / "default" / "chat_history.db",+ [_binding("ws_p1", ws_root)],+ )++ full = adapters.read_chat_entities()+ probe = adapters.probe_chat_entities()+ finally:+ ws_conn.close()++ assert sorted(e.id for e in full) == ["s-default", "ws:ws_p1:s1"]+ assert sorted(e.id for e in probe) == ["s-default", "ws:ws_p1:s1"]+++def test_broken_binding_is_skipped_fail_open(+ tmp_path: Path, default_db: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch+) -> None:+ _add_session(default_db, "s1", "Still visible")+ # A binding whose data store does not exist (moved / never provisioned)+ # contributes nothing but must not blank the account-wide scan. (The+ # symlink-refusal branch mirrors WorkspacePathService and cannot be+ # exercised here — creating symlinks on Windows needs privileges.)+ _with_bindings(+ monkeypatch,+ tmp_path / "default" / "chat_history.db",+ [_binding("ws_broken", tmp_path / "ws-broken")],+ )++ entities = adapters.read_chat_entities()++ ids = [e.id for e in entities]+ assert "s1" in ids+ assert not any(i.startswith("ws:") for i in ids)+ missing = PathService(+ workspace_root=_workspace_data_root(tmp_path / "ws-broken")+ ).get_chat_history_db()+ assert not missing.exists()+++def test_enumeration_failure_falls_back_to_default_only(+ tmp_path: Path, default_db: sqlite3.Connection, monkeypatch: pytest.MonkeyPatch+) -> None:+ _add_session(default_db, "s1", "Chain rule")+ monkeypatch.setattr(+ "deeptutor.services.workspace.get_content_workspace_service",+ lambda: (_ for _ in ()).throw(RuntimeError("registry unreadable")),+ )++ entities = adapters.read_chat_entities()++ assert [e.id for e in entities] == ["s1"]