Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

14.4 KB
Raw Blame
"""Utilities for turning selected book pages into chat context.

The chat runtime consumes this module from two places:

* the main chat page when a user picks pages via ``@book``;
* the book reader side panel, which automatically references the current page.

Keep this module pure and storage-backed only. It should not call an LLM or
depend on FastAPI so it remains cheap to unit test and reuse.
"""

from __future__ import annotations

from dataclasses import dataclass
import re
from typing import Any, Iterable, Sequence

from .models import Block, BlockStatus, BlockType, Book, Chapter, Page, Spine
from .storage import BookStorage, get_book_storage

DEFAULT_MAX_CONTEXT_CHARS = 32_000
DEFAULT_MAX_PAGE_CHARS = 12_000
DEFAULT_MAX_BLOCK_CHARS = 4_000

_THINK_RE = re.compile(r"<think\b[^>]*>.*?</think>", re.IGNORECASE | re.DOTALL)


@dataclass(frozen=True)
class NormalizedBookReference:
    """One selected book with the page ids that should be sent to chat."""

    book_id: str
    page_ids: list[str]

    def model_dump(self) -> dict[str, Any]:
        return {"book_id": self.book_id, "page_ids": list(self.page_ids)}


from deeptutor.core.source_context import SourceContext as BookContextResult


def normalize_book_references(value: Any) -> list[NormalizedBookReference]:
    """Validate and de-duplicate the public ``book_references`` payload."""

    if not isinstance(value, Sequence) or isinstance(value, (str, bytes, bytearray)):
        return []

    refs: list[NormalizedBookReference] = []
    seen_books: set[str] = set()
    for item in value:
        if not isinstance(item, dict):
            continue
        book_id = str(item.get("book_id") or "").strip()
        raw_page_ids = item.get("page_ids") or []
        if (
            not book_id
            or not isinstance(raw_page_ids, Sequence)
            or isinstance(raw_page_ids, (str, bytes, bytearray))
        ):
            continue

        page_ids: list[str] = []
        seen_pages: set[str] = set()
        for raw_page_id in raw_page_ids:
            page_id = str(raw_page_id or "").strip()
            if not page_id or page_id in seen_pages:
                continue
            seen_pages.add(page_id)
            page_ids.append(page_id)

        if book_id in seen_books:
            existing = next((ref for ref in refs if ref.book_id == book_id), None)
            if existing is not None:
                for page_id in page_ids:
                    if page_id not in existing.page_ids:
                        existing.page_ids.append(page_id)
            continue

        seen_books.add(book_id)
        refs.append(NormalizedBookReference(book_id=book_id, page_ids=page_ids))

    return refs


def build_book_context(
    book_references: Any,
    *,
    storage: BookStorage | None = None,
    max_chars: int = DEFAULT_MAX_CONTEXT_CHARS,
    page_char_limit: int = DEFAULT_MAX_PAGE_CHARS,
    block_char_limit: int = DEFAULT_MAX_BLOCK_CHARS,
) -> BookContextResult:
    """Load selected book pages and serialize them into compact LLM context."""

    refs = normalize_book_references(book_references)
    if not refs:
        return BookContextResult()

    store = storage or get_book_storage()
    warnings: list[str] = []
    sections: list[str] = []

    for ref in refs:
        book = store.load_book(ref.book_id)
        if book is None:
            warnings.append(f"book_not_found:{ref.book_id}")
            continue

        spine = store.load_spine(ref.book_id)
        page_sections: list[str] = []
        for page_id in ref.page_ids:
            page = store.load_page(ref.book_id, page_id)
            if page is None:
                warnings.append(f"page_not_found:{ref.book_id}:{page_id}")
                continue
            chapter = spine.chapter_by_id(page.chapter_id) if spine else None
            page_text = _serialize_page(
                book=book,
                spine=spine,
                chapter=chapter,
                page=page,
                page_char_limit=page_char_limit,
                block_char_limit=block_char_limit,
            )
            if page_text:
                page_sections.append(page_text)

        if page_sections:
            sections.append(_serialize_book_header(book) + "\n\n" + "\n\n".join(page_sections))

    text = _clip("\n\n---\n\n".join(sections), max_chars)
    references = [ref.model_dump() for ref in refs]
    return BookContextResult(text=text, references=references, warnings=warnings)


def _serialize_book_header(book: Book) -> str:
    lines = [f"# Book: {_clean(book.title) or book.id}"]
    if book.description:
        lines.append(f"Description: {_clip(_clean(book.description), 800)}")
    if book.language:
        lines.append(f"Language: {_clean(book.language)}")
    return "\n".join(lines)


def _serialize_page(
    *,
    book: Book,
    spine: Spine | None,
    chapter: Chapter | None,
    page: Page,
    page_char_limit: int,
    block_char_limit: int,
) -> str:
    lines: list[str] = []
    chapter_title = chapter.title if chapter else ""
    lines.append(f"## Page: {_clean(page.title) or page.id}")
    if chapter_title:
        lines.append(f"Chapter: {_clean(chapter_title)}")
    if chapter and chapter.summary:
        lines.append(f"Chapter summary: {_clip(_clean(chapter.summary), 1200)}")
    objectives = page.learning_objectives or (chapter.learning_objectives if chapter else [])
    if objectives:
        lines.append("Learning objectives:")
        lines.extend(f"- {_clip(_clean(obj), 300)}" for obj in objectives if _clean(obj))
    if page.status:
        lines.append(f"Page status: {getattr(page.status, 'value', page.status)}")
    if spine and spine.exploration_summary:
        lines.append(f"Book source summary: {_clip(_clean(spine.exploration_summary), 1000)}")

    block_texts = [
        serialized
        for block in page.blocks
        if (serialized := _serialize_block(block, block_char_limit=block_char_limit))
    ]
    if block_texts:
        lines.append("Page content:")
        lines.extend(block_texts)
    if page.error:
        lines.append(f"Page error: {_clip(_clean(page.error), 600)}")
    return _clip("\n".join(lines), page_char_limit)


def _serialize_block(block: Block, *, block_char_limit: int) -> str:
    block_type = _enum_value(block.type)
    status = _enum_value(block.status)
    if status == BlockStatus.HIDDEN.value:
        return ""

    heading = f"### Block: {_clean(block.title) or block_type}"
    if block_type:
        heading += f" ({block_type})"

    if status != BlockStatus.READY.value:
        details = [heading, f"Status: {status or 'unknown'}"]
        if block.error:
            details.append(f"Error: {_clip(_clean(block.error), 500)}")
        return "\n".join(details)

    payload = block.payload if isinstance(block.payload, dict) else {}
    body = _payload_for_type(block_type, payload)
    if not body and block.error:
        body = f"Error: {_clip(_clean(block.error), 500)}"
    if not body:
        return ""
    return _clip(f"{heading}\n{body}", block_char_limit)


def _payload_for_type(block_type: str, payload: dict[str, Any]) -> str:
    bridge = _join_text_fields(payload, ("bridge_text", "transition_in"))
    main = ""
    if block_type in {BlockType.TEXT.value, BlockType.USER_NOTE.value}:
        main = _join_text_fields(payload, ("content", "body", "text", "markdown"))
    elif block_type == BlockType.CALLOUT.value:
        label = _clean(payload.get("label"))
        body = _clean(payload.get("body"))
        main = "\n".join(part for part in (label, body) if part)
    elif block_type == BlockType.SECTION.value:
        main = _format_section(payload)
    elif block_type == BlockType.QUIZ.value:
        main = _format_quiz(payload)
    elif block_type == BlockType.FLASH_CARDS.value:
        main = _format_flash_cards(payload)
    elif block_type == BlockType.TIMELINE.value:
        main = _format_timeline(payload)
    elif block_type == BlockType.CODE.value:
        main = _format_code(payload)
    elif block_type == BlockType.DEEP_DIVE.value:
        main = _format_deep_dive(payload)
    elif block_type in {
        BlockType.FIGURE.value,
        BlockType.INTERACTIVE.value,
        BlockType.ANIMATION.value,
        BlockType.CONCEPT_GRAPH.value,
    }:
        main = _format_visual_summary(payload)
    else:
        main = _safe_payload_summary(payload)

    return "\n".join(part for part in (bridge, main) if part)


def _format_section(payload: dict[str, Any]) -> str:
    lines: list[str] = []
    if intro := _clean(payload.get("intro")):
        lines.append(intro)
    subsections = payload.get("subsections")
    if isinstance(subsections, list):
        for item in subsections:
            if not isinstance(item, dict):
                continue
            heading = _clean(item.get("heading"))
            body = _clean(item.get("body"))
            focus = _clean(item.get("focus"))
            if heading:
                lines.append(f"#### {heading}")
            if focus:
                lines.append(f"Focus: {focus}")
            if body:
                lines.append(body)
    if takeaway := _clean(payload.get("key_takeaway")):
        lines.append(f"Key takeaway: {takeaway}")
    return "\n".join(lines)


def _format_quiz(payload: dict[str, Any]) -> str:
    questions = payload.get("questions")
    if not isinstance(questions, list):
        return _join_text_fields(payload, ("topic", "question", "explanation"))
    lines: list[str] = []
    for idx, item in enumerate(questions[:8], start=1):
        if not isinstance(item, dict):
            continue
        question = _clean(item.get("question"))
        if not question:
            continue
        lines.append(f"Q{idx}: {question}")
        options = item.get("options")
        if isinstance(options, dict) and options:
            option_text = "; ".join(
                f"{_clean(key)}. {_clean(value)}" for key, value in options.items()
            )
            if option_text:
                lines.append(f"Options: {option_text}")
        if answer := _clean(item.get("correct_answer")):
            lines.append(f"Answer: {answer}")
        if explanation := _clean(item.get("explanation")):
            lines.append(f"Explanation: {explanation}")
    return "\n".join(lines)


def _format_flash_cards(payload: dict[str, Any]) -> str:
    cards = payload.get("cards")
    if not isinstance(cards, list):
        return ""
    lines: list[str] = []
    for idx, item in enumerate(cards[:12], start=1):
        if not isinstance(item, dict):
            continue
        front = _clean(item.get("front"))
        back = _clean(item.get("back"))
        hint = _clean(item.get("hint"))
        if front and back:
            suffix = f" Hint: {hint}" if hint else ""
            lines.append(f"Card {idx}: {front} -> {back}{suffix}")
    return "\n".join(lines)


def _format_timeline(payload: dict[str, Any]) -> str:
    events = payload.get("events")
    if not isinstance(events, list):
        return ""
    lines: list[str] = []
    for item in events[:10]:
        if not isinstance(item, dict):
            continue
        date = _clean(item.get("date"))
        title = _clean(item.get("title"))
        description = _clean(item.get("description"))
        line = " - ".join(part for part in (date, title, description) if part)
        if line:
            lines.append(line)
    return "\n".join(lines)


def _format_code(payload: dict[str, Any]) -> str:
    lines: list[str] = []
    if intent := _clean(payload.get("intent")):
        lines.append(f"Intent: {intent}")
    if explanation := _clean(payload.get("explanation")):
        lines.append(f"Explanation: {explanation}")
    code = _clean(payload.get("code"))
    if code:
        language = _clean(payload.get("language")) or "code"
        lines.append(f"Code ({language}):\n{_clip(code, 1200)}")
    return "\n".join(lines)


def _format_deep_dive(payload: dict[str, Any]) -> str:
    suggestions = payload.get("suggestions")
    if not isinstance(suggestions, list):
        return ""
    lines: list[str] = []
    for item in suggestions[:6]:
        if not isinstance(item, dict):
            continue
        topic = _clean(item.get("topic"))
        rationale = _clean(item.get("rationale"))
        if topic:
            lines.append(f"- {topic}" + (f": {rationale}" if rationale else ""))
    return "\n".join(lines)


def _format_visual_summary(payload: dict[str, Any]) -> str:
    lines = [_join_text_fields(payload, ("description", "summary", "caption", "chart_type"))]
    key_points = payload.get("key_points")
    if isinstance(key_points, list):
        for point in key_points[:8]:
            if text := _clean(point):
                lines.append(f"- {text}")
    return "\n".join(part for part in lines if part)


def _safe_payload_summary(payload: dict[str, Any]) -> str:
    return "\n".join(_iter_safe_strings(payload, max_items=16))


def _iter_safe_strings(value: Any, *, max_items: int) -> Iterable[str]:
    emitted = 0

    def walk(node: Any, key: str = "") -> Iterable[str]:
        nonlocal emitted
        if emitted >= max_items:
            return
        lowered = key.lower()
        if lowered in {"code", "html", "svg", "content", "artifact", "artifacts"}:
            return
        if isinstance(node, str):
            text = _clean(node)
            if text:
                emitted += 1
                yield _clip(text, 500)
            return
        if isinstance(node, dict):
            for child_key, child_value in node.items():
                yield from walk(child_value, str(child_key))
                if emitted >= max_items:
                    break
        elif isinstance(node, list):
            for child in node:
                yield from walk(child, key)
                if emitted >= max_items:
                    break

    yield from walk(value)


def _join_text_fields(payload: dict[str, Any], keys: Sequence[str]) -> str:
    return "\n".join(text for key in keys if (text := _clean(payload.get(key))))


def _enum_value(value: Any) -> str:
    return str(getattr(value, "value", value) or "")


def _clean(value: Any) -> str:
    text = str(value or "")
    text = _THINK_RE.sub("", text)
    text = text.replace("</think>", "").replace("<think>", "")
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def _clip(text: str, limit: int) -> str:
    text = _clean(text)
    if limit <= 0 or len(text) <= limit:
        return text
    head = max(0, limit - 40)
    return text[:head].rstrip() + "\n...[truncated]"


__all__ = [
    "BookContextResult",
    "NormalizedBookReference",
    "build_book_context",
    "normalize_book_references",
]