Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

88.1 KB
Raw Blame
"""
BookEngine
==========

Top-level orchestrator for the Book Engine. Sits **parallel** to
``ChatOrchestrator`` (i.e. it is **not** a ``TurnCapability``) and is the
single public entry point used by the API router, CLI, and SDK.

Lifecycle
---------

::

    create_book(...)         → BookProposal       (Stage 1, requires user confirm)
    confirm_proposal(...)    → Spine              (Stage 2, requires user confirm)
    confirm_spine(...)       → page shells + queued compilation
    compile_page(book, page) → Page               (Stage 3-4, drives BookCompiler)
    resume_book(...)         → re-queue unfinished pages, keeping what exists
    rebuild_book(...)        → discard every page and regenerate from the spine
    list_books(), load_book(), delete_book()

Compilation queue
-----------------

Each book gets a ``_BookRuntime``: an ``asyncio.Queue`` of pending pages, one
background worker, and an **in-flight table** keyed by page id. Every path that
can compile a page — the reader opening one, the worker reaching one, a forced
regenerate — goes through :meth:`BookEngine.compile_page`, which consults that
table so concurrent requests for the same page join a single run instead of
racing into duplicate generations.

The worker trips a breaker (``CONSECUTIVE_PAGE_FAILURE_LIMIT``) when pages fail
for provider-level reasons, pausing the book rather than grinding the remaining
chapters into half-generated debris.

Progress is published to the book's long-lived stream in :mod:`.event_hub`,
never to a request-scoped bus — background compilation must keep streaming
after the call that queued it has returned.
"""

from __future__ import annotations

import asyncio
from dataclasses import dataclass, field
from functools import partial
import logging
import time
from typing import Any

from deeptutor.runtime.stream_bus import StreamBus
from deeptutor.services.workspace.activity import workspace_writer

from . import progress as progress_ops
from .agents.ideation_agent import IdeationAgent
from .agents.source_explorer import SourceExplorer
from .agents.spine_synthesizer import SpineSynthesizer
from .compiler import BookCompiler, CompilerOptions, systemic_failure_reason
from .errors import BookPausedError
from .event_hub import close_book_bus, get_book_bus
from .inputs import IdeationContext, build_book_inputs
from .language import resolve_book_language
from .models import (
    Block,
    BlockStatus,
    BlockType,
    Book,
    BookDepth,
    BookInputs,
    BookProposal,
    BookStatus,
    Chapter,
    ContentType,
    ExplorationReport,
    Page,
    PageLink,
    PageStatus,
    Progress,
    Spine,
)
from .overview_copy import overview_copy
from .storage import BookStorage, get_book_storage
from .streaming import (
    STAGE_COMPILATION,
    STAGE_CRITIQUE,
    STAGE_EXPLORATION,
    STAGE_IDEATION,
    STAGE_OVERVIEW,
    STAGE_SPINE,
    STAGE_SYNTHESIS,
    BookStream,
)

logger = logging.getLogger(__name__)


# ─────────────────────────────────────────────────────────────────────────────
# Per-book runtime state (queues, workers)
# ─────────────────────────────────────────────────────────────────────────────


# A page that fails *entirely* for provider-level reasons (quota exhausted,
# credentials revoked, provider down) will be followed by more of the same, so
# the queue trips a breaker instead of grinding the rest of the book into
# half-generated pages. Ungenerated pages are an asset; half-generated ones are
# debris the user has to clean up.
CONSECUTIVE_PAGE_FAILURE_LIMIT = 2


# Statuses that mean "this page still has work owed to it". PARTIAL is
# deliberately absent: the compiler already ran every block and some failed for
# good, so re-queueing it on each open would re-spend the same model calls
# forever without changing the outcome. Force-regenerate is the way back in.
_UNFINISHED_PAGE_STATUSES = frozenset(
    {
        PageStatus.PENDING,
        PageStatus.PLANNING,
        PageStatus.GENERATING,
        PageStatus.ERROR,
    }
)


# Blocks whose content is a single run of prose the reader can correct in
# place. Everything else carries structured payloads — a section's subsections,
# a quiz's questions, a figure's source — where a plain text box would either
# destroy the structure or edit a field nothing renders. Those stay on
# regenerate.
_EDITABLE_BLOCK_TYPES = frozenset({BlockType.TEXT, BlockType.USER_NOTE, BlockType.CALLOUT})


def _body_key(block: Block) -> str:
    """Which payload key holds a block's prose.

    ``TEXT`` blocks are inconsistent by history: the generator writes ``body``
    while the deterministic overview blocks write ``content``. Respect whatever
    the block already uses so an edit lands where the renderer reads.
    """
    if block.type == BlockType.TEXT and "content" in (block.payload or {}):
        return "content"
    return "body"


def _prune_concept_graph(spine: Spine) -> int:
    """Drop graph nodes whose chapter no longer exists, and dangling edges.

    The concept graph is built once, before the reader ever opens the spine
    editor, and is never rebuilt. Deleting a chapter therefore left its concept
    on the Overview page's map — the book's own front page contradicting its
    table of contents. Pruning is deterministic and costs no model call; the
    remaining graph is still the one that was synthesised, just without the
    parts the reader removed.

    Returns the number of nodes dropped.
    """
    graph = spine.concept_graph
    if not graph.nodes:
        return 0

    live_chapters = {c.id for c in spine.chapters}
    kept = [n for n in graph.nodes if not n.chapter_id or n.chapter_id in live_chapters]
    dropped = len(graph.nodes) - len(kept)
    if not dropped:
        return 0

    kept_ids = {n.id for n in kept}
    graph.nodes = kept
    graph.edges = [e for e in graph.edges if e.src in kept_ids and e.dst in kept_ids]
    return dropped


def _source_quality_summary(
    inputs: BookInputs,
    exploration: ExplorationReport | None,
    *,
    error: str = "",
) -> dict[str, Any]:
    """Compact, persisted source coverage for management views."""

    requested_kbs = list(dict.fromkeys(inputs.knowledge_bases or []))
    covered_kbs = sorted(
        {
            chunk.kb_name
            for chunk in (exploration.chunks if exploration else [])
            if chunk.source == "kb" and chunk.kb_name
        }
    )
    missing_kbs = [name for name in requested_kbs if name not in covered_kbs]
    requested_non_kb = {
        "notebook": len(inputs.notebook_refs or []),
        "chat": len(inputs.chat_selections or []),
        "questions": len(inputs.question_entries or []) + len(inputs.question_categories or []),
    }
    for source in inputs.source_refs:
        kind = source.get("kind", "")
        if kind not in {"knowledge_base", "file", "goal"}:
            requested_non_kb[kind] = requested_non_kb.get(kind, 0) + 1
    coverage = dict(exploration.coverage) if exploration else {}
    warnings: list[str] = []
    if error:
        warnings.append(f"Source exploration failed: {error[:240]}")
    if missing_kbs:
        warnings.append("No retrieved evidence from: " + ", ".join(missing_kbs))
    requested_any = bool(requested_kbs or any(requested_non_kb.values()))
    chunk_count = len(exploration.chunks) if exploration else 0
    if requested_any and chunk_count == 0 and not error:
        warnings.append("Selected sources produced no reusable evidence chunks.")
    return {
        "status": "failed" if error else ("warning" if warnings else "ready"),
        "requested_kbs": requested_kbs,
        "covered_kbs": covered_kbs,
        "missing_kbs": missing_kbs,
        "requested_non_kb": requested_non_kb,
        "coverage": coverage,
        "chunk_count": chunk_count,
        "warnings": warnings,
    }


def _generation_error_category(message: str) -> str:
    text = (message or "").lower()
    if any(token in text for token in ("quota", "insufficient", "credit", "billing")):
        return "quota"
    if any(token in text for token in ("unauthorized", "forbidden", "api key", "credential")):
        return "authentication"
    if any(token in text for token in ("rate limit", "too many requests", "429")):
        return "rate_limit"
    # A block type whose generator needs an optional extra that is not
    # installed. Distinct from a provider fault: nothing will fix itself on a
    # retry, and the answer is either to install the extra or to leave that
    # block type out of the book — so it must not land in "unknown", which is
    # what the whole animation family was doing.
    if any(
        token in text
        for token in (
            "requires the optional",
            "pip install",
            "modulenotfounderror",
            "no module named",
            "not installed",
        )
    ):
        return "missing_dependency"
    if any(token in text for token in ("timeout", "timed out", "connection", "provider")):
        return "provider"
    if any(token in text for token in ("parse", "invalid json", "validation")):
        return "content"
    return "unknown"


def _is_auto_overview(chapter: Chapter) -> bool:
    """Whether *chapter* is the engine-injected overview.

    Checks both markers: ``content_type`` survives a round trip through the
    typed model, while the ``auto_overview`` extra survives a chapter whose
    content type was changed by hand in the spine editor.
    """
    return (
        chapter.content_type == ContentType.OVERVIEW
        or (chapter.__pydantic_extra__ or {}).get("auto_overview") is True
    )


def _coerce_depth(value: str | BookDepth | None) -> BookDepth:
    """Accept anything the API layer might pass; fall back to STANDARD."""
    if isinstance(value, BookDepth):
        return value
    try:
        return BookDepth(str(value or "").strip().lower())
    except ValueError:
        return BookDepth.STANDARD


@dataclass
class _BookRuntime:
    """In-process per-book **scheduling** state.

    Deliberately owns no event stream: streams live in :mod:`.event_hub` and
    outlive any single request, so background compilation keeps broadcasting
    after the call that queued it has returned.
    """

    queue: asyncio.Queue[str] = field(default_factory=asyncio.Queue)
    queued: set[str] = field(default_factory=set)
    # page_id → the task currently compiling it. The single source of truth for
    # "is this page already being built", shared by the foreground
    # ``compile_page`` path and the background worker so they can never race
    # into compiling the same page twice.
    in_flight: dict[str, asyncio.Task[Page]] = field(default_factory=dict)
    worker: asyncio.Task[None] | None = None
    lock: asyncio.Lock = field(default_factory=asyncio.Lock)
    consecutive_page_failures: int = 0


# ─────────────────────────────────────────────────────────────────────────────
# Engine
# ─────────────────────────────────────────────────────────────────────────────


def _with_book_sources(function):
    from functools import wraps
    from inspect import signature

    @wraps(function)
    async def guarded(self, *args, **kwargs):
        from deeptutor.services.workspace.knowledge import learning_source_access

        bound = signature(function).bind(self, *args, **kwargs).arguments
        book_id = bound.get("book_id")
        book = self.storage.load_book(book_id) if book_id else None
        refs = book.knowledge_bases if book else bound.get("knowledge_bases") or []
        with learning_source_access(refs):
            return await function(self, *args, **kwargs)

    return guarded


class BookEngine:
    """Process-wide orchestrator for the Book Engine."""

    def __init__(
        self,
        *,
        storage: BookStorage | None = None,
        compiler_options: CompilerOptions | None = None,
    ) -> None:
        self.storage = storage or get_book_storage()
        self.compiler = BookCompiler(
            storage=self.storage,
            options=compiler_options or CompilerOptions(phase=2),
        )
        self._runtimes: dict[str, _BookRuntime] = {}
        self._global_lock = asyncio.Lock()

    # ── Discovery / lifecycle ────────────────────────────────────────────

    def list_books(self) -> list[Book]:
        books: list[Book] = []
        for book_id in self.storage.list_book_ids():
            book = self.storage.load_book(book_id)
            if book is not None:
                books.append(book)
        books.sort(key=lambda b: b.updated_at, reverse=True)
        return books

    def load_book(self, book_id: str) -> Book | None:
        return self.storage.load_book(book_id)

    def _require_generation_allowed(self, book_id: str, book: Book | None = None) -> Book:
        current = book or self.storage.load_book(book_id)
        if current is None:
            raise ValueError(f"Book {book_id} not found")
        if current.status == BookStatus.PAUSED:
            raise BookPausedError(
                "Book generation is paused. Resume it explicitly before generating content."
            )
        return current

    def reading_summary(self, book: Book) -> dict[str, Any]:
        """How far the reader has got, cheap enough to compute for a whole shelf.

        Reads only ``progress.json`` and the page count already on the manifest
        — deliberately no ``list_pages``, which would turn drawing the library
        into one directory scan per book.
        """
        progress = self.storage.load_progress(book.id)
        total = max(0, book.page_count)
        if progress is None:
            return {
                "current_page_id": "",
                "visited_pages": 0,
                "total_pages": total,
                "percent": 0,
            }
        return {
            "current_page_id": progress.current_page_id,
            "visited_pages": len(set(progress.visited_page_ids)),
            "total_pages": total,
            "percent": round(progress_ops.completion_ratio(progress, total) * 100),
        }

    def load_spine(self, book_id: str) -> Spine | None:
        return self.storage.load_spine(book_id)

    def list_pages(self, book_id: str) -> list[Page]:
        return self.storage.list_pages(book_id)

    def load_page(self, book_id: str, page_id: str) -> Page | None:
        return self.storage.load_page(book_id, page_id)

    def load_progress(self, book_id: str) -> Progress:
        progress = self.storage.load_progress(book_id)
        if progress is None:
            progress = Progress(book_id=book_id)
            self.storage.save_progress(progress)
        return progress

    def delete_book(self, book_id: str) -> bool:
        runtime = self._runtimes.pop(book_id, None)
        if runtime is not None:
            if runtime.worker and not runtime.worker.done():
                runtime.worker.cancel()
            for task in runtime.in_flight.values():
                if not task.done():
                    task.cancel()
        # The book's event stream is the one thing that outlives its requests,
        # so deletion is the only place allowed to close it.
        close_book_bus(book_id)
        return self.storage.delete_book(book_id)

    def set_page_chat_session(self, *, book_id: str, page_id: str, session_id: str) -> Book | None:
        """Persist the chat session associated with a specific book page."""
        book = self.storage.load_book(book_id)
        page = self.storage.load_page(book_id, page_id)
        clean_session_id = (session_id or "").strip()
        if book is None or page is None or not clean_session_id:
            return None

        metadata = dict(book.metadata or {})
        mapping = metadata.get("page_chat_sessions")
        if not isinstance(mapping, dict):
            mapping = {}
        mapping[str(page_id)] = clean_session_id
        metadata["page_chat_sessions"] = mapping
        book.metadata = metadata
        book.updated_at = time.time()
        self.storage.save_book(book)
        self.storage.append_log(
            book_id,
            f"page chat session mapped ({page_id} → {clean_session_id})",
            op="page_chat",
        )
        return book

    @staticmethod
    def _reset_page_for_force_compile(page: Page) -> None:
        """Reset generated block outputs while preserving anything user-authored.

        Notes and hand-edited prose survive a forced regenerate. A reader who
        corrected a sentence and then hit "regenerate page" should not silently
        lose their correction — the two actions have nothing to do with each
        other from their point of view.
        """
        for block in page.blocks:
            if block.type == BlockType.USER_NOTE:
                continue
            if (block.metadata or {}).get("edited_by_user"):
                continue
            preserved_metadata = {
                key: value
                for key, value in (block.metadata or {}).items()
                if key in {"transition_in", "deep_dive_page_id"}
            }
            block.status = BlockStatus.PENDING
            block.payload = {}
            block.error = ""
            block.source_anchors = []
            block.metadata = preserved_metadata
            block.updated_at = time.time()
        page.status = PageStatus.PENDING
        page.error = ""
        page.updated_at = time.time()

    # ── Stage 1: Ideation ────────────────────────────────────────────────

    @workspace_writer
    @_with_book_sources
    async def create_book(
        self,
        *,
        user_intent: str,
        source_refs: list[dict[str, Any]] | None = None,
        chat_session_id: str = "",
        chat_selections: list[dict[str, Any]] | None = None,
        notebook_refs: list[dict[str, Any]] | None = None,
        knowledge_bases: list[str] | None = None,
        question_categories: list[int] | None = None,
        question_entries: list[int] | None = None,
        language: str = "en",
        fallback_language: str = "en",
        depth: str = BookDepth.STANDARD.value,
        stream: StreamBus | None = None,
    ) -> tuple[Book, BookProposal]:
        """Capture inputs, run IdeationAgent, persist DRAFT book + proposal."""
        bus = stream or StreamBus()
        bstream = BookStream(bus)
        language = resolve_book_language(
            user_intent=user_intent,
            requested_language=language,
            fallback_language=fallback_language,
        )

        async with bstream.stage(STAGE_IDEATION):
            await bstream.progress("Capturing inputs…", stage=STAGE_IDEATION)
            book_inputs, ideation_ctx = await build_book_inputs(
                user_intent=user_intent,
                source_refs=source_refs,
                chat_session_id=chat_session_id,
                chat_selections=chat_selections,
                notebook_refs=notebook_refs,
                knowledge_bases=knowledge_bases,
                question_categories=question_categories,
                question_entries=question_entries,
                language=language,
            )

            await bstream.progress(
                "Generating book proposal…",
                stage=STAGE_IDEATION,
            )
            proposal = await self._run_ideation(ideation_ctx, language)

            book = Book(
                title=proposal.title,
                description=proposal.description,
                status=BookStatus.DRAFT,
                proposal=proposal,
                knowledge_bases=book_inputs.knowledge_bases,
                language=language,
                depth=_coerce_depth(depth),
                chapter_count=proposal.estimated_chapters,
            )
            # Capture baseline KB fingerprints immediately so subsequent drift
            # checks have something to compare against. Without this the very
            # first health-check run treats every selected KB as "newly added"
            # and surfaces a spurious drift warning.
            try:
                from .kb_health import fingerprint_kbs

                if book.knowledge_bases:
                    book.kb_fingerprints = fingerprint_kbs(book.knowledge_bases)
            except Exception as exc:  # noqa: BLE001
                logger.debug(f"baseline fingerprint capture skipped: {exc}")
            self.storage.save_book(book)
            self.storage.save_inputs(book.id, book_inputs)
            self.storage.save_progress(Progress(book_id=book.id))
            self.storage.append_log(
                book.id, f"created (status=draft, title='{book.title}')", op="create"
            )

            await bstream.book_event(
                "proposal_ready",
                {
                    "book_id": book.id,
                    "title": book.title,
                    "proposal": proposal.model_dump(),
                },
                stage=STAGE_IDEATION,
            )

        return book, proposal

    async def _run_ideation(self, ctx: IdeationContext, language: str) -> BookProposal:
        agent = IdeationAgent(language=language)
        return await agent.process(ideation_context=ctx)

    # ── Stage 2: Spine ───────────────────────────────────────────────────

    @workspace_writer
    @_with_book_sources
    async def confirm_proposal(
        self,
        *,
        book_id: str,
        edited_proposal: BookProposal | None = None,
        stream: StreamBus | None = None,
    ) -> tuple[Book, Spine]:
        """User confirms (and optionally edits) the proposal → run SpineAgent."""
        book = self.storage.load_book(book_id)
        if book is None:
            raise ValueError(f"Book {book_id} not found")

        if edited_proposal is not None:
            book.proposal = edited_proposal
            book.title = edited_proposal.title or book.title
            book.description = edited_proposal.description or book.description

        bus = stream or get_book_bus(book_id)
        bstream = BookStream(bus)

        proposal = book.proposal or BookProposal(title=book.title)
        inputs = self.storage.load_inputs(book.id) or BookInputs(
            user_intent=book.title or "",
            knowledge_bases=list(book.knowledge_bases),
            language=book.language,
        )

        async with bstream.stage(STAGE_SPINE):
            # ── Sub-stage 1: Source exploration ──────────────────────
            exploration: ExplorationReport | None = None
            try:
                async with bstream.stage(STAGE_EXPLORATION):
                    await bstream.progress(
                        "Exploring your sources in parallel…",
                        stage=STAGE_EXPLORATION,
                    )
                    explorer = SourceExplorer(language=book.language)
                    exploration = await explorer.explore(
                        book_id=book.id,
                        proposal=proposal,
                        inputs=inputs,
                        stream=bus,
                    )
                    self.storage.save_exploration(book.id, exploration)
                    book.metadata = {
                        **(book.metadata or {}),
                        "source_quality": _source_quality_summary(inputs, exploration),
                    }
                    if (book.metadata or {}).get("exploration_failed"):
                        book.metadata = {
                            k: v
                            for k, v in book.metadata.items()
                            if k not in ("exploration_failed", "exploration_error")
                        }
                    await bstream.book_event(
                        "exploration_ready",
                        {
                            "book_id": book.id,
                            "queries": exploration.queries,
                            "coverage": exploration.coverage,
                            "candidate_concepts": exploration.candidate_concepts,
                            "summary": exploration.summary,
                        },
                        stage=STAGE_EXPLORATION,
                    )
            except Exception as exc:
                logger.warning(f"SourceExplorer failed for {book.id}: {exc}")
                exploration = None
                book.metadata = {
                    **(book.metadata or {}),
                    "exploration_failed": True,
                    "exploration_error": str(exc)[:400],
                    "source_quality": _source_quality_summary(
                        inputs,
                        None,
                        error=str(exc),
                    ),
                }
                self.storage.save_book(book)
                self.storage.append_log(
                    book_id,
                    f"source exploration failed, spine built from the proposal alone: {exc}",
                    op="exploration_failed",
                )
                await bstream.book_event(
                    "exploration_failed",
                    {"book_id": book.id, "reason": str(exc)[:400]},
                    stage=STAGE_EXPLORATION,
                )

            # ── Sub-stage 2: Synthesise spine + concept graph ────────
            synthesizer = SpineSynthesizer(language=book.language)

            async def _on_round(label: str, payload: dict[str, Any]) -> None:
                stage = STAGE_CRITIQUE if label.startswith("critique") else STAGE_SYNTHESIS
                # Don't push the full payload; just enough to drive the timeline.
                summary = {
                    "round": label,
                    "chapter_count": len(payload.get("chapters") or [])
                    if isinstance(payload.get("chapters"), list)
                    else 0,
                    "issue_count": len(payload.get("issues") or [])
                    if isinstance(payload.get("issues"), list)
                    else 0,
                    "verdict": payload.get("verdict") or "",
                }
                await bstream.book_event(
                    "spine_round", {"book_id": book.id, **summary}, stage=stage
                )

            async with bstream.stage(STAGE_SYNTHESIS):
                await bstream.progress("Synthesising spine + concept graph…", stage=STAGE_SYNTHESIS)
                spine = await synthesizer.synthesize(
                    book_id=book.id,
                    proposal=proposal,
                    exploration=exploration,
                    on_round=_on_round,
                )

            book.chapter_count = len(spine.chapters)
            book.status = BookStatus.SPINE_READY
            self.storage.save_book(book)
            self.storage.save_spine(spine)
            self.storage.append_log(
                book.id,
                f"spine generated ({len(spine.chapters)} chapters, "
                f"{len(spine.concept_graph.nodes)} concepts)",
                op="spine",
            )

            await bstream.book_event(
                "spine_ready",
                {
                    "book_id": book.id,
                    "chapter_count": len(spine.chapters),
                    "concept_node_count": len(spine.concept_graph.nodes),
                    "concept_edge_count": len(spine.concept_graph.edges),
                    "spine": spine.model_dump(),
                },
                stage=STAGE_SPINE,
            )
        return book, spine

    # ── Stage 2.5: Overview chapter injection ───────────────────────────

    async def _ensure_overview_chapter(
        self,
        spine: Spine,
        book: Book,
        *,
        stream: StreamBus | None,
    ) -> Spine:
        """Ensure exactly one Overview chapter, first in the spine (idempotent).

        Identity, not position, decides whether one already exists. Keying the
        guard on ``chapters[0]`` meant any caller that handed the spine back
        with the overview elsewhere in the list — the spine editor re-appends
        hidden chapters after the user's own — silently got a second one on
        every re-confirm, each with its own page.
        """
        existing = [c for c in spine.chapters if _is_auto_overview(c)]
        if existing:
            keep, *duplicates = existing
            if duplicates:
                logger.warning(
                    f"book {book.id}: dropping {len(duplicates)} duplicate overview chapter(s)"
                )
                dropped = {id(c) for c in duplicates}
                spine.chapters = [c for c in spine.chapters if id(c) not in dropped]
            # Re-seat it at the front and renumber, so a spine that came back
            # out of order still reads correctly.
            spine.chapters = [keep, *[c for c in spine.chapters if c is not keep]]
            for index, chapter in enumerate(spine.chapters):
                chapter.order = index
            return spine

        copy = overview_copy(book.language)
        overview = Chapter(
            title=copy["chapter_title"],
            learning_objectives=list(copy["objectives"]),
            content_type=ContentType.OVERVIEW,
            summary=copy["chapter_summary"],
            order=0,
        )
        # Mark for idempotency on subsequent runs.
        overview.__pydantic_extra__ = overview.__pydantic_extra__ or {}
        overview.__pydantic_extra__["auto_overview"] = True

        spine.chapters = [overview, *spine.chapters]
        for index, chapter in enumerate(spine.chapters):
            chapter.order = index
        return spine

    async def _materialize_overview_page(
        self,
        spine: Spine,
        pages: list[Page],
        book: Book,
        *,
        stream: StreamBus | None,
    ) -> None:
        """Build the Overview page's blocks deterministically."""
        if not spine.chapters:
            return
        overview_chapter = spine.chapters[0]
        if overview_chapter.content_type != ContentType.OVERVIEW:
            return

        overview_page = next((p for p in pages if p.chapter_id == overview_chapter.id), None)
        if overview_page is None or overview_page.status == PageStatus.READY:
            return

        copy = overview_copy(book.language)
        chapter_index = [
            {
                "id": ch.id,
                "title": ch.title,
                "summary": ch.summary,
                "objectives": list(ch.learning_objectives),
                "order": ch.order,
                "content_type": ch.content_type.value,
                "page_id": (ch.page_ids[0] if ch.page_ids else ""),
            }
            for ch in spine.chapters
            if ch.content_type != ContentType.OVERVIEW
        ]

        # 1) Intro text block (pre-rendered, status=READY)
        description = (book.proposal.description if book.proposal else "") or ""
        intro_md = f"# {book.title or copy['untitled_book']}\n\n{description}\n\n" + copy[
            "intro_body"
        ].format(
            concepts=len(spine.concept_graph.nodes),
            chapters=len(chapter_index),
        )
        intro_block = Block(
            type=BlockType.TEXT,
            status=BlockStatus.READY,
            title=copy["intro_title"],
            params={"role": "overview_intro"},
            payload={"content": intro_md, "format": "markdown"},
        )

        # 2) Concept graph block — render deterministically
        from .blocks.concept_graph import render_mermaid

        graph_block = Block(
            type=BlockType.CONCEPT_GRAPH,
            status=BlockStatus.READY,
            title=copy["concept_map_title"],
            params={
                "concept_graph": spine.concept_graph.model_dump(),
                "chapter_index": chapter_index,
            },
            payload={
                "render_type": "concept_graph",
                "code": {
                    "language": "mermaid",
                    "content": render_mermaid(spine.concept_graph),
                },
                "graph": spine.concept_graph.model_dump(),
                "index": {
                    "chapters": chapter_index,
                    "node_to_chapter": {
                        n.id: n.chapter_id for n in spine.concept_graph.nodes if n.chapter_id
                    },
                },
            },
            metadata={
                "node_count": len(spine.concept_graph.nodes),
                "edge_count": len(spine.concept_graph.edges),
            },
        )

        # 3) Chapter index callout — also rendered deterministically
        index_lines = []
        for entry in chapter_index:
            line = f"- **{entry['title']}**"
            if entry.get("summary"):
                line += f" — {entry['summary']}"
            index_lines.append(line)
        index_md = copy["chapter_index_heading"] + "\n\n" + "\n".join(index_lines)
        index_block = Block(
            type=BlockType.TEXT,
            status=BlockStatus.READY,
            title=copy["chapter_index_title"],
            params={"role": "chapter_index"},
            payload={"content": index_md, "format": "markdown"},
        )

        # This page is rebuilt from scratch every time the spine changes, which
        # is the one place ``edited_by_user`` protection cannot reach — the
        # blocks are replaced wholesale rather than reset. Carry the reader's
        # own content across: a hand-edited deterministic block wins over its
        # freshly rendered replacement, and notes are appended.
        edited_by_role: dict[str, Block] = {}
        notes: list[Block] = []
        for existing in overview_page.blocks:
            if existing.type == BlockType.USER_NOTE:
                notes.append(existing)
            elif (existing.metadata or {}).get("edited_by_user"):
                role = str((existing.params or {}).get("role") or existing.type.value)
                edited_by_role[role] = existing

        def _keep(block: Block, role: str) -> Block:
            return edited_by_role.get(role, block)

        overview_page.blocks = [
            _keep(intro_block, "overview_intro"),
            _keep(graph_block, BlockType.CONCEPT_GRAPH.value),
            _keep(index_block, "chapter_index"),
            *notes,
        ]
        overview_page.status = PageStatus.READY
        overview_page.content_type = ContentType.OVERVIEW
        self.storage.save_page(overview_page)

        # Publish to the book's own stream. Guarding on ``stream is not None``
        # silently dropped this event once callers stopped passing a bus.
        await BookStream(stream or get_book_bus(book.id)).book_event(
            "overview_ready",
            {
                "book_id": book.id,
                "page_id": overview_page.id,
                "node_count": len(spine.concept_graph.nodes),
                "chapter_count": len(chapter_index),
            },
            stage=STAGE_OVERVIEW,
        )

    # ── Stage 3: confirm spine + create page shells ─────────────────────

    @workspace_writer
    @_with_book_sources
    async def confirm_spine(
        self,
        *,
        book_id: str,
        edited_spine: Spine | None = None,
        stream: StreamBus | None = None,
        auto_compile: bool = True,
    ) -> list[Page]:
        """User confirms (or edits) the spine → create pending page shells.

        BookEngine v2: automatically injects an **Overview** chapter at order 0
        whose page is fully pre-built (deterministic concept-graph render +
        intro text + chapter index). The rest of the chapters are queued for
        normal compilation.
        """
        book = self.storage.load_book(book_id)
        if book is None:
            raise ValueError(f"Book {book_id} not found")

        spine = edited_spine or self.storage.load_spine(book_id)
        if spine is None:
            raise ValueError(f"No spine for book {book_id}")
        if edited_spine is not None:
            spine.book_id = book_id

        # Chapters the reader deleted must not survive on the concept map.
        dropped = _prune_concept_graph(spine)
        if dropped:
            self.storage.append_log(
                book_id,
                f"pruned {dropped} concept-graph node(s) for deleted chapters",
                op="confirm_spine",
            )

        # ── Inject Overview chapter (idempotent) ─────────────────────
        spine = await self._ensure_overview_chapter(spine, book, stream=stream)

        existing = {p.chapter_id: p for p in self.storage.list_pages(book_id)}
        pages: list[Page] = []
        for chapter in spine.chapters:
            page = existing.get(chapter.id)
            if page is None:
                page = Page(
                    book_id=book_id,
                    chapter_id=chapter.id,
                    title=chapter.title,
                    learning_objectives=list(chapter.learning_objectives),
                    content_type=chapter.content_type,
                    order=chapter.order,
                    status=PageStatus.PENDING,
                )
                self.storage.save_page(page)
                chapter.page_ids = [page.id]
            elif (
                page.order != chapter.order
                or page.title != chapter.title
                or page.content_type != chapter.content_type
            ):
                # A re-confirm after editing the spine has to carry the edits
                # onto pages that already exist, or reordering and renaming
                # chapters changes nothing the reader can see: pages are sorted
                # by ``order`` (storage.list_pages) and titled from the page.
                page.order = chapter.order
                page.title = chapter.title
                page.content_type = chapter.content_type
                page.learning_objectives = list(chapter.learning_objectives)
                page.updated_at = time.time()
                self.storage.save_page(page)
            pages.append(page)
        pages.sort(key=lambda p: (p.order, p.created_at))
        # Persist assigned page ids once. Previously the growing spine was
        # rewritten after every new chapter shell.
        self.storage.save_spine(spine)

        # Build the Overview page eagerly (no LLM, no queue).
        await self._materialize_overview_page(spine, pages, book, stream=stream)

        book.page_count = len(pages)
        book.status = BookStatus.COMPILING
        metadata = {
            k: v
            for k, v in (book.metadata or {}).items()
            if k not in {"pause_reason", "pause_kind"}
        }
        metadata["lazy_compile"] = not auto_compile
        # When this run started, durably. The reader's clock used to count
        # from the first event *their tab* happened to see, so reloading
        # mid-compile restarted it at zero — a number that measured how long
        # the page had been open and claimed to measure the run.
        metadata["compile_started_at"] = time.time()
        book.metadata = metadata
        self.storage.save_book(book)
        self.storage.append_log(
            book_id,
            f"spine confirmed ({len(pages)} page shells, auto_compile={auto_compile})",
            op="confirm_spine",
        )

        if auto_compile:
            await self._enqueue_pending_pages(book_id, pages)
        return pages

    async def _halt_compilation(self, book_id: str) -> None:
        """Stop this book's worker and drain its queue, leaving disk untouched."""
        runtime = self._runtimes.get(book_id)
        if runtime is None:
            return
        current = asyncio.current_task()
        cancelled: list[asyncio.Task[Any]] = []
        async with runtime.lock:
            if (
                runtime.worker is not None
                and runtime.worker is not current
                and not runtime.worker.done()
            ):
                runtime.worker.cancel()
                cancelled.append(runtime.worker)
            runtime.worker = None
            for task in runtime.in_flight.values():
                if task is not current and not task.done():
                    task.cancel()
                    cancelled.append(task)
            runtime.in_flight.clear()
            runtime.queued.clear()
            runtime.consecutive_page_failures = 0
            while not runtime.queue.empty():
                try:
                    runtime.queue.get_nowait()
                except asyncio.QueueEmpty:
                    break
        if cancelled:
            await asyncio.gather(*cancelled, return_exceptions=True)
        async with self._global_lock:
            if self._runtimes.get(book_id) is runtime:
                self._runtimes.pop(book_id, None)

    async def pause_book(
        self,
        *,
        book_id: str,
        reason: str = "Paused by user.",
    ) -> list[Page]:
        """Durably pause generation and cancel all work already in flight.

        The manifest is written first, so even a process termination during
        cancellation cannot make the next ``deeptutor start`` auto-resume the
        book. Generated blocks remain intact; only pages stranded in transient
        in-flight states are reset to ``PENDING`` for an explicit resume.
        """
        book = self.storage.load_book(book_id)
        if book is None:
            raise ValueError(f"Book {book_id} not found")
        if self.storage.load_spine(book_id) is None:
            raise ValueError(f"No spine for book {book_id}")

        book.status = BookStatus.PAUSED
        book.metadata = {
            **(book.metadata or {}),
            "pause_reason": reason,
            "pause_kind": "user",
        }
        book.updated_at = time.time()
        self.storage.save_book(book)

        await self._halt_compilation(book_id)

        pages = self.storage.list_pages(book_id)
        for page in pages:
            if page.status in (PageStatus.PLANNING, PageStatus.GENERATING):
                page.status = PageStatus.PENDING
                page.error = ""
                page.updated_at = time.time()
                self.storage.save_page(page)

        self.storage.append_log(book_id, "compilation paused by user", op="paused")
        await BookStream(get_book_bus(book_id)).book_event(
            "compilation_paused",
            {"book_id": book_id, "reason": reason, "manual": True},
            stage=STAGE_COMPILATION,
        )
        return self.storage.list_pages(book_id)

    async def resume_book(
        self,
        *,
        book_id: str,
        stream: StreamBus | None = None,
    ) -> list[Page]:
        """Re-queue every page that never finished, keeping what already exists.

        The non-destructive counterpart to :meth:`rebuild_book`. Three things
        need it and previously had only "delete everything and start over":

        - a process restart (per-book runtimes live in memory only),
        - a paused book once the user has topped up their quota,
        - a page stranded in ``PLANNING``/``GENERATING`` by a crash.

        Pages that are already ``READY`` are left exactly as they are, so
        resuming costs only the work that is genuinely missing.
        """
        book = self.storage.load_book(book_id)
        if book is None:
            raise ValueError(f"Book {book_id} not found")
        if self.storage.load_spine(book_id) is None:
            raise ValueError(f"No spine for book {book_id}")

        # A page left mid-flight by a crash would otherwise be skipped by the
        # queue's "already generating" checks forever.
        pages = self.storage.list_pages(book_id)
        for page in pages:
            if page.status in (PageStatus.PLANNING, PageStatus.GENERATING):
                page.status = PageStatus.PENDING
                page.updated_at = time.time()
                self.storage.save_page(page)

        pending = [p for p in pages if p.status in _UNFINISHED_PAGE_STATUSES]
        if not pending:
            await self._maybe_finalize_book(book_id)
            return pages

        book.status = BookStatus.COMPILING
        book.metadata = {
            k: v
            for k, v in (book.metadata or {}).items()
            if k not in {"pause_reason", "pause_kind"}
        }
        # Resuming starts a new run: the clock counts this stretch of work,
        # not the wall time since the book was first confirmed.
        book.metadata["compile_started_at"] = time.time()
        book.updated_at = time.time()
        self.storage.save_book(book)
        self.storage.append_log(
            book_id, f"resume requested ({len(pending)} unfinished pages)", op="resume"
        )

        await self._enqueue_pending_pages(book_id, pending)
        return self.storage.list_pages(book_id)

    async def maybe_resume_on_open(self, book_id: str) -> bool:
        """Restart stalled compilation when the reader opens a book.

        ``PathService`` resolves per current user, so a process-wide sweep at
        startup cannot even enumerate other users' books. Opening a book is the
        natural, correctly-scoped trigger instead: it runs as that user and
        only touches the book they actually care about.

        Deliberately does *not* resume a ``PAUSED`` book — that one stopped for
        a reason the user has to clear first, and silently retrying would burn
        the quota they just ran out of. Returns whether work was queued.
        """
        book = self.storage.load_book(book_id)
        if book is None or book.status != BookStatus.COMPILING:
            return False

        if (book.metadata or {}).get("lazy_compile"):
            # The reader asked for chapters to be built as they open them.
            # Resuming would queue the whole book and defeat that choice.
            return False

        runtime = self._runtimes.get(book_id)
        if runtime is not None:
            # Liveness, not queue depth. A queue with items in it and no worker
            # draining them is exactly the state that needs rescuing — reading
            # ``queued`` as proof of progress left such a book wedged at
            # COMPILING for the life of the process.
            if runtime.in_flight:
                return False
            if runtime.worker is not None and not runtime.worker.done():
                return False
        try:
            await self.resume_book(book_id=book_id)
        except ValueError:
            return False
        except Exception:  # noqa: BLE001
            logger.warning(f"auto-resume failed for {book_id}", exc_info=True)
            return False
        return True

    async def rebuild_book(
        self,
        *,
        book_id: str,
        stream: StreamBus | None = None,
        auto_compile: bool = True,
    ) -> list[Page]:
        """Regenerate all pages while preserving the confirmed proposal/spine."""
        book = self.storage.load_book(book_id)
        spine = self.storage.load_spine(book_id)
        if book is None or spine is None:
            raise ValueError(f"Cannot rebuild book – missing book/spine ({book_id})")

        await self._halt_compilation(book_id)

        for page in self.storage.list_pages(book_id):
            self.storage.delete_page(book_id, page.id)
        for chapter in spine.chapters:
            chapter.page_ids = []
        self.storage.save_spine(spine)
        self.storage.save_progress(Progress(book_id=book_id))

        book.status = BookStatus.SPINE_READY
        book.page_count = 0
        book.updated_at = time.time()
        self.storage.save_book(book)
        self.storage.append_log(
            book_id,
            f"rebuild requested (preserve_spine=true, auto_compile={auto_compile})",
            op="rebuild",
        )

        return await self.confirm_spine(
            book_id=book_id,
            edited_spine=spine,
            stream=stream,
            auto_compile=auto_compile,
        )

    # ── Stage 3-4: compile a single page (current page) ──────────────────

    async def compile_page(
        self,
        *,
        book_id: str,
        page_id: str,
        stream: StreamBus | None = None,
        force: bool = False,
    ) -> Page:
        """Compile one page, coalescing concurrent requests for the same page.

        Three call sites can ask for the same page at once — the user opening
        it, the background worker reaching it, and a forced regenerate. Without
        coalescing they each run a full plan-and-generate pass, pay for it, and
        then race to persist, so the loser's output is silently discarded.

        A plain request *joins* the run already in flight. A forced one waits
        for that run to finish and then starts a clean pass, so a regenerate
        never interleaves with the generation it is meant to replace.
        """
        self._require_generation_allowed(book_id)
        runtime = await self._get_or_create_runtime(book_id)

        while True:
            async with runtime.lock:
                running = runtime.in_flight.get(page_id)
                if running is None or running.done():
                    task = asyncio.create_task(
                        self._compile_page_now(
                            book_id=book_id,
                            page_id=page_id,
                            stream=stream,
                            force=force,
                        )
                    )
                    runtime.in_flight[page_id] = task
                    task.add_done_callback(partial(self._release_in_flight, runtime, page_id))
                    break
                if not force:
                    task = running
                    break
                joined = running
            # Outside the lock: let the in-flight pass finish, then loop round
            # and claim the slot for the forced rebuild. ``asyncio.wait`` never
            # re-raises, which is what we want — we only care that it ended.
            await asyncio.wait({joined})

        # Shielded: awaiting a task propagates the awaiter's cancellation *into*
        # that task, so one caller going away — a closed WebSocket, a client
        # that navigated on — would otherwise kill a run that other awaiters and
        # the background worker are relying on. Compilation persists its own
        # output, so it is always right to let it finish; the ways to genuinely
        # stop it (``_halt_compilation``, ``delete_book``) cancel the task
        # directly.
        return await asyncio.shield(task)

    @staticmethod
    def _release_in_flight(runtime: _BookRuntime, page_id: str, task: asyncio.Task[Page]) -> None:
        """Drop *page_id* from the in-flight table once *task* settles.

        Keyed on identity so a task that has already been replaced by a forced
        rebuild cannot evict its successor.
        """
        if runtime.in_flight.get(page_id) is task:
            runtime.in_flight.pop(page_id, None)

    @workspace_writer
    @_with_book_sources
    async def _compile_page_now(
        self,
        *,
        book_id: str,
        page_id: str,
        stream: StreamBus | None = None,
        force: bool = False,
    ) -> Page:
        """Drive the compiler for one page. Always run via :meth:`compile_page`."""
        book = self.storage.load_book(book_id)
        spine = self.storage.load_spine(book_id)
        page = self.storage.load_page(book_id, page_id)
        if book is None or spine is None or page is None:
            raise ValueError(f"Cannot compile page – missing book/spine/page ({book_id}/{page_id})")
        self._require_generation_allowed(book_id, book)
        if page.status == PageStatus.READY and not force:
            return page

        # Overview pages are built deterministically from the spine (intro,
        # concept graph, chapter index). Never run the generic LLM compiler over
        # their hand-built blocks — that would overwrite the deterministic
        # "How to read this book" intro and chapter index with hallucinated
        # prose. Reaching here means force=True or the page is not yet READY, so
        # rebuild deterministically (this also refreshes a changed spine).
        if page.content_type == ContentType.OVERVIEW:
            page.status = PageStatus.PENDING
            self.storage.save_page(page)
            await self._materialize_overview_page(spine, [page], book, stream=stream)
            page = self.storage.load_page(book_id, page_id) or page
            await self._maybe_finalize_book(book_id)
            return page

        if force:
            self._reset_page_for_force_compile(page)
            self.storage.save_page(page)

        chapter = spine.chapter_by_id(page.chapter_id)
        if chapter is None:
            raise ValueError(f"Page {page_id} references unknown chapter {page.chapter_id}")

        bus = stream or get_book_bus(book_id)
        bstream = BookStream(bus)

        try:
            async with bstream.stage(STAGE_COMPILATION):
                page = await self.compiler.compile_page(
                    book_id=book_id,
                    chapter=chapter,
                    page=page,
                    stream=bstream,
                    knowledge_bases=book.knowledge_bases,
                    language=book.language,
                    depth=book.depth.value,
                )
        except Exception as exc:
            self._mark_page_error(page, exc, prefix="Compilation failed")
            raise

        # Refresh KB fingerprints once we successfully ship a READY page.
        # We capture them lazily so a brand-new book gets its baseline as
        # soon as the very first page is compiled.
        if page.status == PageStatus.READY:
            try:
                from .kb_health import refresh_book_fingerprints

                refreshed = self.storage.load_book(book_id)
                if refreshed is not None and not refreshed.kb_fingerprints:
                    refresh_book_fingerprints(book_id, storage=self.storage)
            except Exception as exc:  # noqa: BLE001
                logger.debug(f"fingerprint refresh skipped: {exc}")

        await self._maybe_finalize_book(book_id)
        return page

    # ── Background compilation queue ─────────────────────────────────────

    async def _enqueue_pending_pages(self, book_id: str, pages: list[Page]) -> None:
        while True:
            runtime = await self._get_or_create_runtime(book_id)
            async with runtime.lock:
                # An idle worker retires its runtime under the same two locks in
                # the same order. If it did so between our lookup and this
                # acquisition, we would be queueing into an orphan that nothing
                # drains — take the live runtime and try again.
                async with self._global_lock:
                    if self._runtimes.get(book_id) is not runtime:
                        continue

                runtime.consecutive_page_failures = 0
                for page in pages:
                    if page.status == PageStatus.READY:
                        continue
                    if page.id in runtime.queued:
                        continue
                    runtime.queued.add(page.id)
                    await runtime.queue.put(page.id)
                self._ensure_worker(book_id)
                return

    async def _get_or_create_runtime(self, book_id: str) -> _BookRuntime:
        async with self._global_lock:
            runtime = self._runtimes.get(book_id)
            if runtime is None:
                runtime = _BookRuntime()
                self._runtimes[book_id] = runtime
            return runtime

    def _mark_page_error(self, page: Page | None, exc: Exception, *, prefix: str) -> None:
        """Flip a page stranded in an in-flight state to ``ERROR``.

        The compiler persists ``page.status = PLANNING`` before the LLM
        planning call and ``GENERATING`` before running block generators;
        when either throws, nothing resets the status, so without this the
        page spins forever and is neither retried on resume nor
        force-regenerable in the UI. Saving is best-effort — this runs
        inside exception handlers (including the background worker loop,
        which must survive), so a failing save must not mask the original
        error or kill the worker.
        """
        if page is None or page.status not in (PageStatus.GENERATING, PageStatus.PLANNING):
            return
        page.status = PageStatus.ERROR
        page.error = f"{prefix}: {exc}"
        page.updated_at = time.time()
        try:
            self.storage.save_page(page)
        except Exception:
            logger.warning(f"Failed to persist ERROR status for page {page.id}", exc_info=True)

    def _ensure_worker(self, book_id: str) -> None:
        runtime = self._runtimes.get(book_id)
        if runtime is None:
            return
        if runtime.worker is not None and not runtime.worker.done():
            return
        from deeptutor.services.workspace.activity import acquire_activity

        activity = acquire_activity()
        try:
            runtime.worker = asyncio.create_task(self._worker_loop(book_id))
        except BaseException:
            activity.close()
            raise
        runtime.worker.add_done_callback(lambda _done: activity.close())

    async def _worker_loop(self, book_id: str) -> None:
        from deeptutor.services.workspace.activity import data_activity

        with data_activity():
            await self._worker_loop_active(book_id)

    async def _worker_loop_active(self, book_id: str) -> None:
        runtime = self._runtimes.get(book_id)
        if runtime is None:
            return

        # A task inherits the context it was created in, and this one is created
        # inside whichever request happened to enqueue first. That request may
        # have installed a scoped LLM config (a per-run model pick); without
        # clearing it, every chapter this worker compiles for the rest of its
        # life would silently use that one request's model. Background
        # compilation belongs to the book, not to the call that started it, so
        # it runs against the user's current configuration.
        from deeptutor.services.llm.config import set_scoped_llm_config

        set_scoped_llm_config(None)
        while True:
            try:
                page_id = await asyncio.wait_for(runtime.queue.get(), timeout=2.0)
            except asyncio.TimeoutError:
                async with runtime.lock:
                    if runtime.queue.empty() and not runtime.in_flight:
                        runtime.worker = None
                        async with self._global_lock:
                            if self._runtimes.get(book_id) is runtime:
                                self._runtimes.pop(book_id, None)
                        from deeptutor.runtime.memory_reclaim import schedule_memory_reclaim

                        schedule_memory_reclaim()
                        return
                continue

            # Re-bound every iteration: the except handler below reads `page`,
            # which must be neither unbound (loader raised on the first item —
            # the UnboundLocalError would kill this worker task) nor stale from
            # a previous iteration (the wrong page would be flipped to ERROR).
            page = None
            tripped = False
            try:
                page = self.storage.load_page(book_id, page_id)
                if page is None or page.status == PageStatus.READY:
                    continue
                # Go through the coalescing entry point rather than straight to
                # the compiler: if the reader just opened this very page, both
                # requests must land on one run instead of two.
                compiled = await self.compile_page(book_id=book_id, page_id=page_id)
                tripped = await self._record_page_outcome(runtime, book_id, compiled)
            except asyncio.CancelledError:
                raise
            except BookPausedError:
                if page is not None and page.status in (
                    PageStatus.PLANNING,
                    PageStatus.GENERATING,
                ):
                    page.status = PageStatus.PENDING
                    page.error = ""
                    page.updated_at = time.time()
                    self.storage.save_page(page)
                runtime.worker = None
                return
            except Exception as exc:
                logger.warning(
                    f"Background compilation failed for {book_id}/{page_id}: {exc}",
                    exc_info=True,
                )
                # Reset page status so it can be retried on next resume.
                self._mark_page_error(page, exc, prefix="Background compile failed")
                try:
                    self.storage.append_log(
                        book_id,
                        f"background compile failed for page {page_id}: {exc}",
                        op="compile_error",
                    )
                except Exception:
                    logger.warning(
                        f"Failed to append compile-error log for {book_id}", exc_info=True
                    )
                tripped = await self._record_page_failure(runtime, book_id, str(exc))
            finally:
                runtime.queued.discard(page_id)

            if tripped:
                runtime.worker = None
                return

    # ── Compilation breaker ─────────────────────────────────────────────

    async def _record_page_outcome(self, runtime: _BookRuntime, book_id: str, page: Page) -> bool:
        """Feed a finished background page into the breaker.

        Returns ``True`` when the breaker tripped and the queue was drained.
        """
        reason = systemic_failure_reason(page) if page.status == PageStatus.ERROR else ""
        if not reason:
            # Any page that produced *something* proves the provider is alive.
            runtime.consecutive_page_failures = 0
            return False
        return await self._record_page_failure(runtime, book_id, reason)

    async def _record_page_failure(self, runtime: _BookRuntime, book_id: str, reason: str) -> bool:
        runtime.consecutive_page_failures += 1
        if runtime.consecutive_page_failures < CONSECUTIVE_PAGE_FAILURE_LIMIT:
            return False
        await self._pause_compilation(runtime, book_id, reason)
        return True

    async def _pause_compilation(self, runtime: _BookRuntime, book_id: str, reason: str) -> None:
        """Stop compiling this book and tell the reader why.

        Whatever is already on disk is kept: pausing leaves untouched pages
        ``PENDING`` so ``resume_book`` can pick them up verbatim once the user
        has fixed their quota or credentials.
        """
        failure_count = runtime.consecutive_page_failures
        book = self.storage.load_book(book_id)
        if book is not None:
            book.status = BookStatus.PAUSED
            book.metadata = {
                **(book.metadata or {}),
                "pause_reason": reason,
                "pause_kind": "provider",
            }
            book.updated_at = time.time()
            self.storage.save_book(book)
        await self._halt_compilation(book_id)
        self.storage.append_log(
            book_id,
            f"compilation paused after {failure_count} consecutive provider failures: {reason}",
            op="paused",
        )
        await BookStream(get_book_bus(book_id)).book_event(
            "compilation_paused",
            {"book_id": book_id, "reason": reason},
            stage=STAGE_COMPILATION,
        )

    async def _maybe_finalize_book(self, book_id: str) -> None:
        book = self.storage.load_book(book_id)
        if book is None:
            return
        pages = self.storage.list_pages(book_id)
        if not pages:
            return
        if not any(p.status in _UNFINISHED_PAGE_STATUSES for p in pages):
            # Every page has settled — READY, or PARTIAL with blocks that will
            # not come back. The book is as done as it is going to get.
            if book.status != BookStatus.READY:
                book.status = BookStatus.READY
                book.metadata = {
                    k: v
                    for k, v in (book.metadata or {}).items()
                    if k not in {"pause_reason", "pause_kind"}
                }
                self.storage.save_book(book)
                self.storage.append_log(book_id, "all pages ready → status=READY", op="finalize")
                # Without this the reader never learns the book finished:
                # the last page event fires while the book is still COMPILING.
                await BookStream(get_book_bus(book_id)).book_event(
                    "book_ready",
                    {"book_id": book_id, "page_count": len(pages)},
                    stage=STAGE_COMPILATION,
                )

    # ── Block-level controls (Phase 1: regenerate single block) ─────────

    @_with_book_sources
    async def regenerate_block(
        self,
        *,
        book_id: str,
        page_id: str,
        block_id: str,
        params_override: dict[str, Any] | None = None,
        stream: StreamBus | None = None,
    ) -> Block | None:
        """Re-run a single block generator (e.g. user clicked 'regenerate')."""
        book = self.storage.load_book(book_id)
        spine = self.storage.load_spine(book_id)
        page = self.storage.load_page(book_id, page_id)
        if book is None or spine is None or page is None:
            return None
        self._require_generation_allowed(book_id, book)
        chapter = spine.chapter_by_id(page.chapter_id)
        if chapter is None:
            return None
        block = page.block_by_id(block_id)
        if block is None:
            return None
        if params_override:
            block.params = {**block.params, **params_override}

        bus = stream or get_book_bus(book_id)
        bstream = BookStream(bus)

        from .blocks.base import BlockContext, get_block_registry

        registry = get_block_registry()
        generator = registry.get(block.type)
        if generator is None:
            block.error = f"No generator for {block.type.value}"
            self.storage.save_page(page)
            return block

        ctx = BlockContext(
            book_id=book_id,
            chapter=chapter,
            page=page,
            block=block,
            language=book.language,
            knowledge_bases=book.knowledge_bases,
        )
        # Resolve previous block by planning order so the bridge text can be
        # refreshed alongside the block itself.
        prev_block: Block | None = None
        for candidate in page.blocks:
            if candidate.id == block.id:
                break
            prev_block = candidate

        async with bstream.stage(STAGE_COMPILATION):
            await generator.generate(ctx)
            # The content is machine-written again, so the reader's edit is
            # gone; leaving the flag set would make force-regenerate skip this
            # block for the rest of the book's life.
            if block.metadata:
                block.metadata.pop("edited_by_user", None)
            if block.status.value == "ready":
                await self.compiler.attach_bridge_text(
                    block,
                    chapter=chapter,
                    previous_block=prev_block,
                    language=book.language,
                )
            self.storage.save_page(page)
            self.compiler._finalize_page_status(page)
            self.storage.save_page(page)
            await self._maybe_finalize_book(book_id)
        return block

    # ── Maintenance / health (Phase 4) ─────────────────────────────────

    def kb_drift_report(self, book_id: str) -> dict[str, Any]:
        """Compute and persist the current KB drift report for *book_id*."""
        from .kb_health import mark_drift_on_book

        report = mark_drift_on_book(book_id, storage=self.storage)
        if report is None:
            return {"book_id": book_id, "has_drift": False, "missing": True}
        return report.to_dict()

    def refresh_kb_fingerprints(
        self, book_id: str, *, force: bool = False
    ) -> dict[str, Any] | None:
        from .kb_health import refresh_book_fingerprints

        book = refresh_book_fingerprints(book_id, storage=self.storage, force=force)
        if book is None:
            return None
        return {
            "book_id": book.id,
            "kb_fingerprints": book.kb_fingerprints,
            "stale_page_ids": book.stale_page_ids,
        }

    def log_health(self, book_id: str) -> dict[str, Any]:
        from .kb_health import scan_log_health

        return scan_log_health(book_id, storage=self.storage).to_dict()

    def is_worker_live(self, book_id: str) -> bool:
        """Is something in *this process* actually compiling *book_id* right now?

        ``status == COMPILING`` is a stored fact and outlives the process that
        set it: restart the backend mid-compile and the manifest still claims
        the book is being written while no queue, worker or task exists. The
        runtime table is the only honest answer, so callers that want to tell
        "working" from "abandoned" have to ask it rather than the manifest.
        """

        # `getattr`: embedders and focused tests build the engine with
        # ``__new__`` and only wire ``storage``, and a diagnostics read must
        # not be the thing that raises on them.
        runtime = (getattr(self, "_runtimes", None) or {}).get(book_id)
        if runtime is None:
            return False
        worker = runtime.worker
        if worker is not None and not worker.done():
            return True
        return any(not task.done() for task in runtime.in_flight.values())

    def generation_overview(self, book: Book) -> dict[str, Any]:
        """Manifest-only generation state, cheap enough for the library."""

        source_quality = (book.metadata or {}).get("source_quality")
        compiling = book.status == BookStatus.COMPILING
        working = compiling and self.is_worker_live(book.id)
        return {
            "status": book.status.value,
            "can_resume": book.status
            in {BookStatus.COMPILING, BookStatus.PAUSED, BookStatus.ERROR},
            "pause_reason": str((book.metadata or {}).get("pause_reason") or ""),
            "source_quality": source_quality if isinstance(source_quality, dict) else None,
            # Live now, versus "says compiling but nobody is compiling it".
            "working": working,
            "interrupted": compiling and not working,
            # Epoch seconds this compile run began, so every viewer's clock
            # agrees and survives a reload. 0 when no run has started.
            "started_at": float((book.metadata or {}).get("compile_started_at") or 0.0),
        }

    def generation_summary(
        self,
        book_id: str,
        *,
        book: Book | None = None,
        pages: list[Page] | None = None,
    ) -> dict[str, Any]:
        """Actionable generation, retry, source, and failure diagnostics."""

        book = book or self.storage.load_book(book_id)
        if book is None:
            return {"book_id": book_id, "status": "missing"}
        pages = pages if pages is not None else self.storage.list_pages(book_id)
        page_counts = {status.value: 0 for status in PageStatus}
        failed_blocks = 0
        categories: dict[str, int] = {}
        for page in pages:
            page_counts[page.status.value] += 1
            errors = [page.error] if page.error else []
            for block in page.blocks:
                if block.status == BlockStatus.ERROR:
                    failed_blocks += 1
                    if block.error:
                        errors.append(block.error)
            for error in errors:
                category = _generation_error_category(error)
                categories[category] = categories.get(category, 0) + 1

        overview = self.generation_overview(book)
        # Chapters still owed work. Queued is not the same as broken: while a
        # worker is running these are simply the tail of its queue, and
        # counting them as "retryable" is what made a healthy book announce
        # "3 chapters can be retried" the moment it started — a failure
        # warning on zero failures.
        queued = sum(
            page_counts[status.value]
            for status in (PageStatus.PENDING, PageStatus.PLANNING, PageStatus.GENERATING)
        )
        failed = page_counts[PageStatus.ERROR.value]
        # Owed chapters become the reader's problem only when nothing is going
        # to pick them up: paused, errored, or a compile the process lost.
        stalled = 0 if overview["working"] else queued
        retryable = failed + stalled
        return {
            "book_id": book.id,
            **overview,
            "pages": {"total": len(pages), **page_counts},
            "failed_blocks": failed_blocks,
            "retryable_pages": retryable,
            # Kept apart so the UI can say "2 queued" without an alarm and
            # "1 failed" with one.
            "queued_pages": queued,
            "failed_pages": failed,
            "can_resume": bool(retryable) and bool(overview["can_resume"]),
            "failure_categories": categories,
        }

    # ── Block CRUD operations (Phase 3) ────────────────────────────────

    @_with_book_sources
    async def insert_block(
        self,
        *,
        book_id: str,
        page_id: str,
        block_type: BlockType,
        params: dict[str, Any] | None = None,
        position: int | None = None,
        stream: StreamBus | None = None,
        compile_now: bool = True,
    ) -> Block | None:
        """Insert a fresh PENDING block at *position* (default: end)."""
        spine = self.storage.load_spine(book_id)
        page = self.storage.load_page(book_id, page_id)
        book = self.storage.load_book(book_id)
        if spine is None or page is None or book is None:
            return None
        if compile_now and block_type != BlockType.USER_NOTE:
            self._require_generation_allowed(book_id, book)
        chapter = spine.chapter_by_id(page.chapter_id)
        if chapter is None:
            return None

        merged_params: dict[str, Any] = {
            "chapter_title": chapter.title,
            "chapter_summary": chapter.summary,
            "objectives": chapter.learning_objectives,
            **(params or {}),
        }
        block = Block(type=block_type, status=BlockStatus.PENDING, params=merged_params)
        if position is None or position >= len(page.blocks) or position < 0:
            page.blocks.append(block)
        else:
            page.blocks.insert(position, block)
        self.storage.save_page(page)

        if compile_now and block_type != BlockType.USER_NOTE:
            from .blocks.base import BlockContext, get_block_registry

            generator = get_block_registry().get(block_type)
            if generator is not None:
                ctx = BlockContext(
                    book_id=book_id,
                    chapter=chapter,
                    page=page,
                    block=block,
                    language=book.language,
                    knowledge_bases=book.knowledge_bases,
                )
                bus = stream or get_book_bus(book_id)
                bstream = BookStream(bus)
                async with bstream.stage(STAGE_COMPILATION):
                    await generator.generate(ctx)
                self.compiler._finalize_page_status(page)
                self.storage.save_page(page)
        elif block_type == BlockType.USER_NOTE:
            block.status = BlockStatus.READY
            block.payload = {
                "format": "markdown",
                "body": str(merged_params.get("body") or ""),
                "author": "user",
            }
            self.storage.save_page(page)
        self.storage.append_log(
            book_id,
            f"inserted {block_type.value} block on page {page_id} (pos={position})",
            op="insert_block",
        )
        return block

    async def update_block(
        self,
        *,
        book_id: str,
        page_id: str,
        block_id: str,
        title: str | None = None,
        body: str | None = None,
    ) -> Block | None:
        """Edit a block's prose in place, marking it as the reader's own.

        Deliberately limited to title and body. The point is to let someone fix
        a wrong sentence or write a margin note without gambling on a whole
        regeneration — not to turn the reader into a block-payload editor.

        The ``edited_by_user`` mark matters: :meth:`_reset_page_for_force_compile`
        honours it, so a later "regenerate this page" cannot quietly delete
        something the reader wrote.
        """
        page = self.storage.load_page(book_id, page_id)
        if page is None:
            return None
        block = page.block_by_id(block_id)
        if block is None or block.type not in _EDITABLE_BLOCK_TYPES:
            return None

        if title is not None:
            block.title = title
        if body is not None:
            block.payload = {
                **block.payload,
                _body_key(block): body,
                "format": "markdown",
            }
        block.status = BlockStatus.READY
        block.error = ""
        block.metadata = {**(block.metadata or {}), "edited_by_user": True}
        block.updated_at = time.time()

        self.compiler._finalize_page_status(page)
        self.storage.save_page(page)
        self.storage.append_log(
            book_id, f"edited {block.type.value} block {block_id}", op="edit_block"
        )
        return block

    async def delete_block(self, *, book_id: str, page_id: str, block_id: str) -> bool:
        page = self.storage.load_page(book_id, page_id)
        if page is None:
            return False
        before = len(page.blocks)
        page.blocks = [b for b in page.blocks if b.id != block_id]
        if len(page.blocks) == before:
            return False
        self.compiler._finalize_page_status(page)
        self.storage.save_page(page)
        self.storage.append_log(
            book_id, f"deleted block {block_id} from page {page_id}", op="delete_block"
        )
        return True

    async def move_block(
        self, *, book_id: str, page_id: str, block_id: str, new_position: int
    ) -> bool:
        page = self.storage.load_page(book_id, page_id)
        if page is None:
            return False
        idx = next((i for i, b in enumerate(page.blocks) if b.id == block_id), -1)
        if idx < 0:
            return False
        new_position = max(0, min(len(page.blocks) - 1, new_position))
        block = page.blocks.pop(idx)
        page.blocks.insert(new_position, block)
        self.storage.save_page(page)
        self.storage.append_log(
            book_id,
            f"moved block {block_id} on page {page_id} → pos {new_position}",
            op="move_block",
        )
        return True

    async def change_block_type(
        self,
        *,
        book_id: str,
        page_id: str,
        block_id: str,
        new_type: BlockType,
        params_override: dict[str, Any] | None = None,
        stream: StreamBus | None = None,
    ) -> Block | None:
        self._require_generation_allowed(book_id)
        spine = self.storage.load_spine(book_id)
        page = self.storage.load_page(book_id, page_id)
        if spine is None or page is None:
            return None
        chapter = spine.chapter_by_id(page.chapter_id)
        if chapter is None:
            return None
        block = page.block_by_id(block_id)
        if block is None:
            return None
        block.type = new_type
        block.status = BlockStatus.PENDING
        block.payload = {}
        block.error = ""
        if params_override:
            block.params = {**block.params, **params_override}
        self.storage.save_page(page)
        # Re-run generator immediately
        return await self.regenerate_block(
            book_id=book_id, page_id=page_id, block_id=block_id, stream=stream
        )

    # ── Deep-dive sub-page (Phase 3) ──────────────────────────────────

    async def create_deep_dive_subpage(
        self,
        *,
        book_id: str,
        parent_page_id: str,
        topic: str,
        block_id: str | None = None,
        content_type: ContentType = ContentType.CONCEPT,
        stream: StreamBus | None = None,
    ) -> Page | None:
        """Spawn a child Page that deepens *topic* and link it from the parent."""
        book = self.storage.load_book(book_id)
        spine = self.storage.load_spine(book_id)
        parent = self.storage.load_page(book_id, parent_page_id)
        if book is None or spine is None or parent is None:
            return None
        self._require_generation_allowed(book_id, book)

        # Add a synthetic chapter so the planner has a target
        chapter = Chapter(
            title=f"{topic} (deep dive)",
            learning_objectives=[f"Go deeper into {topic}"],
            content_type=content_type,
            summary=f"Sub-chapter spawned from {parent.title}.",
            order=len(spine.chapters),
        )
        # The compiler needs a chapter to plan against, but this one is not part
        # of the book's structure: the spine editor hides it and a rebuild must
        # not resurrect it as a top-level chapter. Same marker pattern as
        # ``auto_overview``.
        chapter.__pydantic_extra__ = chapter.__pydantic_extra__ or {}
        chapter.__pydantic_extra__["deep_dive"] = True
        spine.chapters.append(chapter)
        self.storage.save_spine(spine)

        sub = Page(
            book_id=book_id,
            chapter_id=chapter.id,
            title=topic,
            learning_objectives=list(chapter.learning_objectives),
            content_type=content_type,
            status=PageStatus.PENDING,
            order=len(self.storage.list_pages(book_id)),
            parent_page_id=parent.id,
        )
        self.storage.save_page(sub)
        chapter.page_ids = [sub.id]
        self.storage.save_spine(spine)

        # Add link from parent → sub
        book.page_count = len(self.storage.list_pages(book_id))
        book.updated_at = time.time()
        self.storage.save_book(book)

        parent.links.append(PageLink(target_page_id=sub.id, relation="deepens", label=topic))
        if block_id:
            block = parent.block_by_id(block_id)
            if block is not None:
                # Keyed by topic: one card offers several, and each can be
                # expanded independently. The old scalar recorded only the
                # first and disabled the others.
                pages_by_topic = dict(block.metadata.get("deep_dive_pages") or {})
                pages_by_topic[topic] = sub.id
                block.metadata = {
                    **block.metadata,
                    "deep_dive_pages": pages_by_topic,
                    # Kept for books written before this change.
                    "deep_dive_page_id": block.metadata.get("deep_dive_page_id") or sub.id,
                }
        self.storage.save_page(parent)

        # Compile the new page now (blocking, so caller gets ready content)
        await self.compile_page(book_id=book_id, page_id=sub.id, stream=stream)
        self.storage.append_log(
            book_id,
            f"deep-dive page {sub.id} spawned from {parent_page_id}: {topic}",
            op="deep_dive",
        )
        return self.storage.load_page(book_id, sub.id)

    # ── Quiz attempts (Phase 3) ────────────────────────────────────────

    def _page_to_chapter(self, book_id: str) -> dict[str, str]:
        return {p.id: p.chapter_id for p in self.storage.list_pages(book_id) if p.chapter_id}

    async def record_quiz_attempt(
        self,
        *,
        book_id: str,
        page_id: str,
        block_id: str,
        question_id: str,
        user_answer: str,
        is_correct: bool | None,
    ) -> Progress:
        progress = progress_ops.record_attempt(
            self.load_progress(book_id),
            page_id=page_id,
            block_id=block_id,
            question_id=question_id,
            user_answer=user_answer,
            is_correct=is_correct,
            page_to_chapter=self._page_to_chapter(book_id),
        )
        self.storage.save_progress(progress)
        return progress

    # ── Reading position / bookmarks ────────────────────────────────────

    def mark_page_visited(self, *, book_id: str, page_id: str) -> Progress:
        """Remember where the reader is, so they can pick the book back up."""
        progress = self.load_progress(book_id)
        if progress_ops.mark_visited(progress, page_id):
            self.storage.save_progress(progress)
        return progress

    def toggle_page_bookmark(self, *, book_id: str, page_id: str) -> Progress:
        progress = self.load_progress(book_id)
        progress_ops.toggle_bookmark(progress, page_id)
        self.storage.save_progress(progress)
        return progress

    async def supplement_for_weakness(
        self,
        *,
        book_id: str,
        page_id: str,
        topic: str,
        stream: StreamBus | None = None,
    ) -> Block | None:
        """Append a pitfall callout + worked explanation + an easier quiz.

        Idempotent per topic: asking twice returns the existing remediation
        rather than stacking a second copy onto the page. Three generated
        blocks per request is expensive enough that a double-click, a retry, or
        a second reader hitting the same wall must not pay for it twice.
        """
        self._require_generation_allowed(book_id)
        page = self.storage.load_page(book_id, page_id)
        if page is None:
            return None

        clean_topic = (topic or "").strip()
        existing = next(
            (
                block
                for block in page.blocks
                if block.params.get("role") == "remediation"
                and str(block.params.get("topic") or "").strip() == clean_topic
            ),
            None,
        )
        if existing is not None:
            return existing

        await self.insert_block(
            book_id=book_id,
            page_id=page_id,
            block_type=BlockType.CALLOUT,
            params={"variant": "common_pitfall", "topic": clean_topic},
            stream=stream,
        )
        # The marker block the idempotency check above looks for.
        await self.insert_block(
            book_id=book_id,
            page_id=page_id,
            block_type=BlockType.TEXT,
            params={"role": "remediation", "topic": clean_topic},
            stream=stream,
        )
        return await self.insert_block(
            book_id=book_id,
            page_id=page_id,
            block_type=BlockType.QUIZ,
            params={"num_questions": 2, "difficulty": "easy", "topic": clean_topic},
            stream=stream,
        )


# ─────────────────────────────────────────────────────────────────────────────
# Singleton accessor
# ─────────────────────────────────────────────────────────────────────────────


_engines: dict[str, BookEngine] = {}


def get_book_engine() -> BookEngine:
    from deeptutor.services.path_service import get_path_service

    key = str(get_path_service().workspace_root.resolve())
    if key not in _engines:
        _engines[key] = BookEngine()
    return _engines[key]


__all__ = ["BookEngine", "get_book_engine"]