Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

26.5 KB
Raw Blame
"""Workspace → ``Entity`` adapters.

One pure read-only function per surface. Adapters never mutate
workspace state. They read whatever lives under
``data/user/workspace/`` (or, for chat/quiz, the chat history SQLite
DB; for kb-list, the kb config JSON; for the ``partner`` surface, the
per-partner conversation JSONL under ``data/partners/``).

Each adapter returns a ``list[Entity]`` with stable ``id`` and a
deterministic ``fingerprint`` so the diff engine can detect changes
across refreshes.
"""

from __future__ import annotations

from contextlib import closing
from datetime import datetime, timezone
import hashlib
import json
import logging
from pathlib import Path
import sqlite3
from typing import Any

from deeptutor.multi_user.paths import get_account_path_service
from deeptutor.services.memory.paths import Surface
from deeptutor.services.memory.snapshot.entity import Entity, EntityStamp
from deeptutor.services.path_service import get_path_service
from deeptutor.services.workspace.context import WorkspacePathService, WorkspaceScope
from deeptutor.services.workspace.models import WorkspaceError

logger = logging.getLogger(__name__)


# ── Helpers ──────────────────────────────────────────────────────────


def _sha1(*parts: object) -> str:
    h = hashlib.sha1(usedforsecurity=False)
    for part in parts:
        if part is None:
            continue
        if isinstance(part, (dict, list)):
            blob = json.dumps(part, sort_keys=True, ensure_ascii=False)
        else:
            blob = str(part)
        h.update(blob.encode("utf-8"))
        h.update(b"\0")
    return h.hexdigest()[:16]


def _iso(ts: float | int | str | None) -> str:
    if isinstance(ts, str):
        try:
            datetime.fromisoformat(ts.replace("Z", "+00:00"))
            return ts
        except Exception:
            pass
    if isinstance(ts, (int, float)):
        try:
            return datetime.fromtimestamp(float(ts), tz=timezone.utc).isoformat()
        except Exception:
            pass
    return ""


def _iter_chat_dbs() -> list[tuple[str, Path]]:
    """``(qualifier, chat-history db)`` pairs across the account's workspaces.

    The default workspace comes first with an empty qualifier so existing
    snapshots keep their bare session ids — renumbering them would churn every
    account-level snapshot (#1799). Each additional registered workspace
    contributes its own chat db under a ``ws:<workspace_id>:`` qualifier, which
    is what makes custom-workspace conversations visible to account-level
    Memory without id collisions against the default workspace.

    The account's own data root supplies the default entry; binding data roots
    are derived the same way :class:`WorkspacePathService` derives them
    (``<content_root>/.deeptutor/data``), with the same symlink refusal — a
    redirected store is skipped rather than raised, so one broken binding
    cannot blank an account-wide scan.
    """
    account = get_account_path_service()
    pairs: list[tuple[str, Path]] = [("", account.get_chat_history_db())]
    try:
        from deeptutor.services.workspace import get_content_workspace_service

        bindings = get_content_workspace_service().registered_bindings()
    except Exception:
        logger.warning(
            "workspace binding enumeration failed; default workspace only", exc_info=True
        )
        return pairs
    for binding in bindings:
        try:
            paths = WorkspacePathService(
                account,
                WorkspaceScope(binding.workspace_id, account.workspace_root, binding.root),
            )
        except WorkspaceError:
            logger.warning(
                "workspace %s skipped from memory aggregation: redirected data store",
                binding.workspace_id,
            )
            continue
        pairs.append((f"ws:{binding.workspace_id}:", paths.get_chat_history_db()))
    return pairs


# ── Adapters ─────────────────────────────────────────────────────────


def read_notebook_entities() -> list[Entity]:
    ps = get_path_service()
    index_file = ps.get_notebook_index_file()
    if not index_file.exists():
        return []
    try:
        index = json.loads(index_file.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return []
    notebooks = index.get("notebooks") or []
    out: list[Entity] = []
    for nb in notebooks:
        if not isinstance(nb, dict):
            continue
        nb_id = nb.get("id")
        nb_name = nb.get("name") or nb_id
        if not nb_id:
            continue
        nb_file = ps.get_notebook_file(nb_id)
        if not nb_file.exists():
            continue
        try:
            nb_data = json.loads(nb_file.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError) as exc:
            logger.warning("notebook snapshot skipped corrupt file: %s (%s)", nb_file, exc)
            continue
        for r in nb_data.get("records") or []:
            if not isinstance(r, dict):
                continue
            rid = r.get("id")
            if not rid:
                continue
            title = r.get("title", "") or ""
            user_query = r.get("user_query", "") or ""
            output = r.get("output", "") or ""
            summary = r.get("summary", "") or ""
            content = "\n\n".join(
                part
                for part in (
                    f"# {title}" if title else "",
                    f"**User query**: {user_query}" if user_query else "",
                    f"**Summary**: {summary}" if summary else "",
                    output,
                )
                if part
            )
            out.append(
                Entity(
                    id=rid,
                    label=f"{title} · {nb_name}" if title else f"({rid}) · {nb_name}",
                    ts=_iso(r.get("created_at")),
                    content=content,
                    metadata={
                        "notebook_id": nb_id,
                        "notebook_name": nb_name,
                        "record_type": r.get("record_type", ""),
                        "kb_name": r.get("kb_name"),
                    },
                    fingerprint=_sha1(title, user_query, output, summary),
                )
            )
    return out


def read_cowriter_entities() -> list[Entity]:
    docs_dir = get_path_service().get_co_writer_docs_dir()
    if not docs_dir.exists():
        return []
    out: list[Entity] = []
    for entry in sorted(docs_dir.iterdir()):
        manifest = entry / "manifest.json"
        if not manifest.exists():
            continue
        try:
            m = json.loads(manifest.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError) as exc:
            logger.warning("cowriter snapshot skipped corrupt manifest: %s (%s)", manifest, exc)
            continue
        doc_id = m.get("id")
        if not doc_id:
            continue
        title = m.get("title", "") or ""
        content = m.get("content", "") or ""
        out.append(
            Entity(
                id=doc_id,
                label=title or doc_id,
                ts=_iso(m.get("updated_at") or m.get("created_at")),
                content=content,
                metadata={"doc_id": doc_id, "title": title},
                fingerprint=_sha1(title, content),
            )
        )
    return out


def read_book_entities() -> list[Entity]:
    books_dir = get_path_service().get_book_dir()
    if not books_dir.exists():
        return []
    out: list[Entity] = []
    for entry in sorted(books_dir.iterdir()):
        if not entry.is_dir():
            continue
        manifest_path = entry / "manifest.json"
        if not manifest_path.exists():
            continue
        try:
            m = json.loads(manifest_path.read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError) as exc:
            logger.warning("book snapshot skipped corrupt manifest: %s (%s)", manifest_path, exc)
            continue
        book_id = m.get("id")
        if not book_id:
            continue
        title = m.get("title", "") or ""
        description = m.get("description", "") or ""
        # Pull page titles + chapter outline from spine for L2 to chew on.
        spine_path = entry / "spine.json"
        page_titles: list[str] = []
        if spine_path.exists():
            try:
                spine = json.loads(spine_path.read_text(encoding="utf-8"))
                page_titles = [
                    str(p.get("title") or p.get("id") or "")
                    for p in (spine.get("pages") or [])
                    if isinstance(p, dict)
                ]
            except (OSError, json.JSONDecodeError):
                page_titles = []
        content = "\n\n".join(
            part
            for part in (
                f"# {title}",
                description,
                "## Pages\n" + "\n".join(f"- {p}" for p in page_titles) if page_titles else "",
            )
            if part
        )
        out.append(
            Entity(
                id=book_id,
                label=title or book_id,
                ts=_iso(m.get("updated_at") or m.get("created_at")),
                content=content,
                metadata={
                    "book_id": book_id,
                    "page_count": m.get("page_count", 0),
                    "chapter_count": m.get("chapter_count", 0),
                    "knowledge_bases": m.get("knowledge_bases", []),
                    "status": m.get("status", ""),
                },
                fingerprint=_sha1(title, description, m.get("updated_at"), len(page_titles)),
            )
        )
    return out


def _partner_display_name(partner_dir, partner_id: str) -> str:
    """Read ``name`` out of a partner's ``config.yaml`` for tagging.

    Falls back to the directory id when the config is missing or unreadable.
    """
    cfg = partner_dir / "config.yaml"
    if not cfg.exists():
        return partner_id
    try:
        import yaml

        data = yaml.safe_load(cfg.read_text(encoding="utf-8")) or {}
    except Exception:
        return partner_id
    if isinstance(data, dict):
        name = data.get("name")
        if isinstance(name, str) and name.strip():
            return name.strip()
    return partner_id


def _partner_session_entity(path, partner_id: str, partner_name: str) -> Entity | None:
    """Build one Entity from a partner session JSONL file.

    The conversation is inlined as ``user / assistant`` blocks (mirroring
    :func:`read_chat_entities`) so L2 sees the actual dialogue, and the
    originating partner is tagged into both the label and the metadata so
    the consolidator carries provenance into L2/L3.
    """
    records: list[dict] = []
    try:
        with path.open("r", encoding="utf-8") as fh:
            for raw in fh:
                raw = raw.strip()
                if not raw:
                    continue
                try:
                    obj = json.loads(raw)
                except json.JSONDecodeError:
                    continue
                if isinstance(obj, dict):
                    records.append(obj)
    except OSError:
        return None

    blocks: list[str] = []
    for r in records:
        role = r.get("role") or ""
        body = (r.get("content") or "").strip()
        if not body:
            continue
        blocks.append(f"### {role}\n{body}")
    if not blocks:
        return None

    session_key = path.stem
    archived = session_key.startswith("_archived_")
    last = records[-1]
    last_ts = last.get("timestamp", "")
    last_content = last.get("content", "")
    return Entity(
        id=f"{partner_id}:{session_key}",
        label=f"{session_key} · {partner_name}",
        ts=_iso(last_ts),
        content="\n\n".join(blocks),
        metadata={
            "partner_id": partner_id,
            "partner_name": partner_name,
            "session_key": session_key,
            "archived": archived,
            "message_count": len(blocks),
        },
        fingerprint=_sha1(len(blocks), last_ts, last_content),
    )


def read_partner_entities() -> list[Entity]:
    """One Entity per partner conversation *session* (archived + active),
    tagged with the originating partner. Surfaced under the ``partner``
    surface (UI label "伙伴").

    Partner runtimes persist conversations as JSONL under
    ``data/partners/<id>/sessions/*.jsonl`` — a store entirely separate from
    the chat-history SQLite DB the ``chat`` adapter reads. This adapter
    bridges that store into the memory pipeline so partner conversations
    consolidate into L2/L3 like every other surface.

    Admins see the legacy process-wide Partner sessions. Regular users see only
    their own per-user sessions for Partners that are still assigned to them;
    neither side scans another user's relationship-state directory.
    """
    from deeptutor.multi_user.context import get_current_user
    from deeptutor.multi_user.partner_access import assigned_partner_ids
    from deeptutor.multi_user.paths import get_admin_path_service

    user = get_current_user()
    admin_root = get_admin_path_service().workspace_root.resolve()
    # Read-only adapters are also called directly in maintenance/tests where no
    # CurrentUser ContextVar is installed. Preserve the older path-scope guard
    # instead of treating a non-admin workspace as the implicit local admin.
    if user.is_admin and get_path_service().workspace_root.resolve() != admin_root:
        return []
    partners_root = admin_root / "partners"
    if not partners_root.exists():
        return []

    allowed = None if user.is_admin else assigned_partner_ids(user.id)
    out: list[Entity] = []
    for partner_dir in sorted(partners_root.iterdir()):
        if not partner_dir.is_dir():
            continue
        partner_id = partner_dir.name
        if allowed is not None and partner_id not in allowed:
            continue
        sessions_dir = (
            partner_dir / "sessions"
            if user.is_admin
            else partner_dir / "users" / user.id / "sessions"
        )
        if not sessions_dir.is_dir():
            continue
        partner_name = _partner_display_name(partner_dir, partner_id)
        for sess_file in sorted(sessions_dir.glob("*.jsonl")):
            entity = _partner_session_entity(sess_file, partner_id, partner_name)
            if entity is not None:
                out.append(entity)
    return out


def read_kb_entities() -> list[Entity]:
    """KB *list* snapshot — one Entity per registered knowledge base.

    KB queries stay event-driven and live in the trace store; the L2
    consolidator combines both signals.
    """
    kb_root = get_path_service().get_knowledge_bases_root()
    cfg_file = kb_root / "kb_config.json"
    if not cfg_file.exists():
        return []
    try:
        cfg = json.loads(cfg_file.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return []
    kbs = cfg.get("knowledge_bases") or {}
    if not isinstance(kbs, dict):
        return []
    out: list[Entity] = []
    for kb_name, kb_data in sorted(kbs.items()):
        if not isinstance(kb_name, str):
            continue
        if not isinstance(kb_data, dict):
            kb_data = {}
        description = kb_data.get("description", "") or ""
        versions = kb_data.get("index_versions") or []
        sigs = sorted(v.get("signature", "") for v in versions if isinstance(v, dict))
        earliest_created = min(
            (
                v.get("created_at", "")
                for v in versions
                if isinstance(v, dict) and v.get("created_at")
            ),
            default="",
        )
        content = "\n\n".join(
            part
            for part in (
                f"# {kb_name}",
                description,
                f"Index versions: {len(versions)}",
            )
            if part
        )
        out.append(
            Entity(
                id=kb_name,
                label=kb_name,
                ts=earliest_created,
                content=content,
                metadata={
                    "kb_name": kb_name,
                    "version_count": len(versions),
                    "rag_provider": kb_data.get("rag_provider", ""),
                },
                fingerprint=_sha1(description, sigs),
            )
        )
    return out


def read_chat_entities() -> list[Entity]:
    """One Entity per chat session across the account's workspaces.

    ``content`` inlines all turns as ``user / assistant`` blocks so L2 sees the
    actual conversation. Sessions from the default workspace keep their bare
    session id; sessions from additional workspaces carry a ``ws:<id>:``
    qualifier plus a ``workspace_id`` in metadata, so account-level Memory
    sees every conversation with provenance and no id collisions (#1799).
    """
    out: list[Entity] = []
    for qualifier, db_path in _iter_chat_dbs():
        if not db_path.exists():
            continue
        try:
            with closing(sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)) as conn:
                conn.row_factory = sqlite3.Row
                sessions = conn.execute(
                    "SELECT id, title, created_at, updated_at FROM sessions ORDER BY updated_at DESC"
                ).fetchall()
                for sess in sessions:
                    sid = sess["id"]
                    msgs = conn.execute(
                        "SELECT id, role, content, capability, created_at "
                        "FROM messages WHERE session_id = ? "
                        "ORDER BY created_at ASC, id ASC",
                        (sid,),
                    ).fetchall()
                    blocks: list[str] = []
                    for m in msgs:
                        role = m["role"]
                        body = (m["content"] or "").strip()
                        if not body:
                            continue
                        blocks.append(f"### {role}\n{body}")
                    content = "\n\n".join(blocks)
                    last_msg_id = msgs[-1]["id"] if msgs else 0
                    metadata: dict[str, Any] = {
                        "session_id": sid,
                        "message_count": len(msgs),
                    }
                    if qualifier:
                        metadata["workspace_id"] = qualifier.removeprefix("ws:").removesuffix(":")
                    out.append(
                        Entity(
                            id=f"{qualifier}{sid}",
                            label=sess["title"] or sid,
                            ts=_iso(sess["updated_at"]),
                            content=content,
                            metadata=metadata,
                            fingerprint=_sha1(last_msg_id, sess["updated_at"]),
                        )
                    )
        except sqlite3.Error as exc:
            logger.warning("chat snapshot scan failed for %s: %s", db_path, exc)
            continue
    return out


def read_quiz_entities() -> list[Entity]:
    """One Entity per recorded quiz attempt (notebook_entries row), across
    the account's workspaces with the same qualifier scheme as chat (#1799)."""
    out: list[Entity] = []
    for qualifier, db_path in _iter_chat_dbs():
        if not db_path.exists():
            continue
        try:
            with closing(sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)) as conn:
                conn.row_factory = sqlite3.Row
                rows = conn.execute(
                    "SELECT id, session_id, turn_id, question_id, question, "
                    "question_type, options_json, correct_answer, explanation, "
                    "difficulty, user_answer, is_correct, bookmarked, "
                    "created_at FROM notebook_entries "
                    "ORDER BY created_at DESC"
                ).fetchall()
                for r in rows:
                    qid = r["question_id"] or f"row_{r['id']}"
                    entity_id = f"{qualifier}{r['session_id']}:{qid}"
                    question = (r["question"] or "").strip()
                    user_answer = (r["user_answer"] or "").strip()
                    correct = (r["correct_answer"] or "").strip()
                    explanation = (r["explanation"] or "").strip()
                    is_correct = bool(int(r["is_correct"] or 0))
                    content = "\n\n".join(
                        part
                        for part in (
                            f"**Question**: {question}" if question else "",
                            f"**User answer**: {user_answer}" if user_answer else "",
                            f"**Correct answer**: {correct}" if correct else "",
                            f"**Explanation**: {explanation}" if explanation else "",
                        )
                        if part
                    )
                    metadata: dict[str, Any] = {
                        "session_id": r["session_id"],
                        "turn_id": r["turn_id"],
                        "question_id": qid,
                        "question_type": r["question_type"],
                        "difficulty": r["difficulty"],
                        "is_correct": is_correct,
                        "bookmarked": bool(int(r["bookmarked"] or 0)),
                    }
                    if qualifier:
                        metadata["workspace_id"] = qualifier.removeprefix("ws:").removesuffix(":")
                    out.append(
                        Entity(
                            id=entity_id,
                            label=question[:80] or qid,
                            ts=_iso(r["created_at"]),
                            content=content,
                            metadata=metadata,
                            fingerprint=_sha1(question, user_answer, correct, is_correct),
                        )
                    )
        except sqlite3.Error as exc:
            logger.warning("quiz snapshot scan failed for %s: %s", db_path, exc)
            continue
    return out


# ── Probes (identity + stamp, without content) ───────────────────────


def probe_chat_entities() -> list[EntityStamp]:
    """Chat stamps without reading a single message body.

    Deliberately mirrors :func:`read_chat_entities`'s ``id`` / ``label`` /
    ``fingerprint`` expression by expression. In particular the fingerprint is
    ``_sha1(last_msg_id, updated_at)`` where ``last_msg_id`` is the id of the
    last message under ``ORDER BY created_at ASC, id ASC`` — NOT ``MAX(id)``,
    which would diverge whenever ids and timestamps disagree (clock skew, a
    backfilled row). Selecting the first row of the reversed ordering picks the
    same message. Sessions with no messages stamp ``0``, as the full read does.

    Skipping ``content`` is the whole point: the full read concatenates every
    message of every session, which is the wrong price to pay for a caller that
    only wants to know what the sessions are called.

    Iterates the same workspace-qualified chat dbs as :func:`read_chat_entities`
    (#1799) — the qualifier expression must stay mirrored exactly.
    """
    out: list[EntityStamp] = []
    for qualifier, db_path in _iter_chat_dbs():
        if not db_path.exists():
            continue
        try:
            with closing(sqlite3.connect(f"file:{db_path}?mode=ro", uri=True)) as conn:
                conn.row_factory = sqlite3.Row
                last_msg_id: dict[str, int] = {
                    row["session_id"]: row["id"]
                    for row in conn.execute(
                        "SELECT session_id, id FROM ("
                        "  SELECT session_id, id, ROW_NUMBER() OVER ("
                        "    PARTITION BY session_id ORDER BY created_at DESC, id DESC"
                        "  ) AS rn FROM messages"
                        ") WHERE rn = 1"
                    )
                }
                for sess in conn.execute(
                    "SELECT id, title, updated_at FROM sessions ORDER BY updated_at DESC"
                ):
                    sid = sess["id"]
                    out.append(
                        EntityStamp(
                            id=f"{qualifier}{sid}",
                            label=sess["title"] or sid,
                            fingerprint=_sha1(last_msg_id.get(sid, 0), sess["updated_at"]),
                            ts=_iso(sess["updated_at"]),
                        )
                    )
        except sqlite3.Error as exc:
            logger.warning("chat snapshot probe failed for %s: %s", db_path, exc)
            continue
    return out


# ── Dispatch ─────────────────────────────────────────────────────────


_READERS = {
    "notebook": read_notebook_entities,
    "cowriter": read_cowriter_entities,
    "book": read_book_entities,
    "partner": read_partner_entities,
    "kb": read_kb_entities,
    "chat": read_chat_entities,
    "quiz": read_quiz_entities,
}

# Surfaces with a cheap stamps-only path. A surface belongs here only when
# skipping content actually saves something: every other adapter reads a
# bounded amount, so a second code path would be duplication without a payoff
# — and a second chance to drift from its full reader.
_PROBES = {
    "chat": probe_chat_entities,
}


def read_entities(surface: Surface) -> list[Entity]:
    reader = _READERS.get(surface)
    if reader is None:
        return []
    try:
        return reader()
    except Exception as exc:
        logger.warning("snapshot adapter failed surface=%s: %s", surface, exc)
        return []


def read_stamps(surface: Surface) -> list[EntityStamp]:
    """Identity + label + fingerprint + timestamp for every entity on *surface*.

    Uses the surface's probe when it has one and falls back to projecting the
    full read otherwise, so a surface without a probe is simply slower — never
    unsupported. A probe that raises also falls back rather than reporting an
    empty surface, which the diff would otherwise read as "everything was
    deleted".
    """
    probe = _PROBES.get(surface)
    if probe is not None:
        try:
            return probe()
        except Exception as exc:
            logger.warning("snapshot probe failed surface=%s; full read: %s", surface, exc)
    return [
        EntityStamp(id=e.id, label=e.label, fingerprint=e.fingerprint, ts=e.ts)
        for e in read_entities(surface)
    ]


SUPPORTED_SURFACES: tuple[Surface, ...] = tuple(_READERS.keys())  # type: ignore[arg-type,assignment]