Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(server): expose scoped page generation over HTTP (#983)

Raghav Chamadiyad0fe6d0c4fc1
diff --git a/packages/core/src/repowise/core/persistence/crud/pages.py b/packages/core/src/repowise/core/persistence/crud/pages.py
index 8cf021dfb..7113f8874 100644
--- a/packages/core/src/repowise/core/persistence/crud/pages.py
+++ b/packages/core/src/repowise/core/persistence/crud/pages.py
@@ -444,15 +444,25 @@ async def list_pages(
repository_id: str,
*,
page_type: str | None = None,
+ deterministic: bool | None = None,
limit: int = 100,
offset: int = 0,
sort_by: str = "updated_at",
order: str = "desc",
) -> list[Page]:
- """Return pages for a repository, optionally filtered by page_type."""
+ """Return pages for a repository, optionally filtered by page_type.
+
+ ``deterministic=True`` returns only unwritten (template) pages;
+ ``deterministic=False`` returns only model-written ones. ``None`` (default)
+ returns both. A template page is stamped ``provider_name='template'``.
+ """
q = select(Page).where(Page.repository_id == repository_id)
if page_type is not None:
q = q.where(Page.page_type == page_type)
+ if deterministic is True:
+ q = q.where(Page.provider_name == "template")
+ elif deterministic is False:
+ q = q.where(Page.provider_name != "template")
_sort_cols = {
"updated_at": Page.updated_at,
"confidence": Page.confidence,
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index 3d2f60718..b98017603 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -22,7 +22,7 @@ from repowise.core.cancellation import (
get_active_token,
set_active_token,
)
-from repowise.core.docs_mode import DocsMode, docs_mode_state_fields
+from repowise.core.docs_mode import DocsMode, docs_mode_state_fields, resolve_docs_mode
from repowise.core.persistence.crud import (
get_generation_job,
get_repository,
@@ -386,6 +386,31 @@ async def execute_job(
events = create_event_buffer(app_state, job_id)
progress = JobProgressCallback(job_id, session_factory, events)
+ # ---- Scoped generation mode ---------------------------------------
+ # `repowise generate` over HTTP: write an explicit subset of pages via
+ # the shared core engine, not a full re-index. `single_page` is the
+ # legacy per-page regenerate alias for the same path.
+ if mode in ("generate", "single_page"):
+ if llm_client is None:
+ raise RuntimeError(
+ docs_skip_reason or "No LLM provider is configured for this repository."
+ )
+ await _run_generate_job(
+ job_id=job_id,
+ repo_id=repo_id,
+ repo_path=Path(repo_path),
+ config=config,
+ session_factory=session_factory,
+ provider=llm_client,
+ vector_store=vector_store,
+ fts=fts,
+ progress=progress,
+ exclude_patterns=exclude_patterns or [],
+ wiki_style=wiki_style,
+ start=start,
+ )
+ return
+
generate_docs = (
(is_full_resync or is_initial_index)
and llm_client is not None
@@ -419,6 +444,14 @@ async def execute_job(
# This keeps docs fresh without the cost of a full re-index.
incremental_pages: list = []
if mode == "sync" and llm_client is not None:
+ # D3: hand the incremental regen the vector store (so re-rendered
+ # pages are re-embedded, not silently dropped from semantic search)
+ # and the prior pages (so an unchanged page whose prompt still hashes
+ # the same is reused, not re-billed on every sync).
+ from repowise.core.persistence import load_prior_pages
+
+ async with get_session(session_factory) as session:
+ prior_pages = await load_prior_pages(session, repo_id)
incremental_pages = await _incremental_page_regen(
Path(repo_path),
result,
@@ -426,6 +459,8 @@ async def execute_job(
config,
progress,
repo_wiki_style=wiki_style,
+ vector_store=vector_store,
+ prior_pages=prior_pages,
)
# ---- Persist results -----------------------------------------------
@@ -724,6 +759,230 @@ def _persist_initial_index_state(
# ---------------------------------------------------------------------------
+def _build_generate_intent(config: dict) -> Any:
+ """Turn a generate job's config into a :class:`PageSelectionIntent`.
+
+ Understands the new ``selection`` block (all / unwritten / stale / page_ids /
+ path_prefix) and the legacy ``single_page`` shape (``page_id``). Defaults to
+ ``unwritten`` — the index-only upgrade case — when nothing is specified.
+ """
+ from repowise.core.generation.page_selection import PageSelectionIntent
+
+ if config.get("mode") == "single_page":
+ pid = config.get("page_id")
+ return PageSelectionIntent(page_ids=(pid,) if pid else ())
+
+ sel = config.get("selection") or {}
+ kind = sel.get("kind", "unwritten")
+ if kind == "all":
+ return PageSelectionIntent(all_pages=True)
+ if kind == "stale":
+ return PageSelectionIntent(stale=True)
+ if kind == "page_ids":
+ return PageSelectionIntent(page_ids=tuple(sel.get("page_ids") or ()))
+ if kind == "path_prefix":
+ prefix = sel.get("path_prefix")
+ return PageSelectionIntent(path_globs=(prefix,) if prefix else ())
+ return PageSelectionIntent(unwritten=True)
+
+
+def _build_generation_config(repo_path: Path, config: dict, wiki_style: str) -> Any:
+ """Build the ``GenerationConfig`` a generate job (or its estimate) runs with.
+
+ Shared by the executor and the estimate endpoint so both resolve the same
+ style / reasoning / language / concurrency, and therefore the same scope and
+ cost. A per-request ``style`` override (carried in the job config) wins over
+ the repo's default, matching the single-page regenerate contract.
+ """
+ from repowise.core.generation import GenerationConfig
+ from repowise.core.generation.styles import resolve_style
+ from repowise.core.reasoning import resolve_reasoning
+ from repowise.core.repo_config import load_repo_config
+
+ repo_cfg = load_repo_config(repo_path)
+ effective_style = resolve_style(config.get("style") or wiki_style, repo_path=repo_path).name
+ return GenerationConfig(
+ reasoning=resolve_reasoning(config=repo_cfg),
+ wiki_style=effective_style,
+ language=repo_cfg.get("language", "en"),
+ enable_onboarding=bool(repo_cfg.get("enable_onboarding", True)),
+ max_concurrency=int(config.get("concurrency") or 12),
+ )
+
+
+async def _run_generate_job(
+ *,
+ job_id: str,
+ repo_id: str,
+ repo_path: Path,
+ config: dict,
+ session_factory: Any,
+ provider: Any,
+ vector_store: Any | None,
+ fts: Any | None,
+ progress: JobProgressCallback,
+ exclude_patterns: list[str],
+ wiki_style: str,
+ start: float,
+) -> None:
+ """Run one scoped generation job through the shared core engine.
+
+ Rehydrates the graph + git, resolves the requested scope + cascade, writes
+ exactly that subset of pages, and records the terminal job status. The whole
+ generate/persist/heal half is the same code the CLI ``repowise generate``
+ runs, so behaviour matches across the two surfaces.
+ """
+ from repowise.core.generation.scope import resolve_scope
+ from repowise.core.pipeline.scoped_generation import (
+ execute_scoped_generation,
+ rehydrate_repo,
+ )
+
+ gen_config = _build_generation_config(repo_path, config, wiki_style)
+
+ state = _load_state(repo_path)
+ rehydrated = await rehydrate_repo(
+ session_factory,
+ repo_id,
+ repo_path,
+ generation_config=gen_config,
+ exclude_patterns=exclude_patterns,
+ include_submodules=bool(state.get("include_submodules", False)),
+ include_nested_repos=bool(state.get("include_nested_repos", False)),
+ )
+ if rehydrated is None:
+ raise RuntimeError("Repository has no wiki pages yet; run an index first.")
+
+ intent = _build_generate_intent(config)
+ # Explicit selections pull in their container pages by default; the legacy
+ # single-page alias keeps its historical "just this page" behaviour.
+ cascade_mode = config.get("cascade") or (
+ "none" if config.get("mode") == "single_page" else "dependents"
+ )
+ plan = resolve_scope(
+ records=rehydrated.records,
+ intent=intent,
+ cascade_mode=cascade_mode,
+ deps=rehydrated.deps,
+ )
+
+ generated_pages: list = []
+ marked_stale = 0
+ if plan.generate_ids:
+ progress.on_phase_start("generation", len(plan.generate_ids))
+ gen_result = await execute_scoped_generation(
+ session_factory=session_factory,
+ repo_id=repo_id,
+ repo_path=repo_path,
+ rehydrated=rehydrated,
+ plan=plan,
+ provider=provider,
+ generation_config=gen_config,
+ # The resolved server vector store carries its own embedder, exactly
+ # like the sync path — no separate embedder needed here.
+ embedder=None,
+ vector_store=vector_store,
+ fts=fts,
+ progress=progress,
+ # The server derives spend from the generated pages' token counts
+ # rather than a per-repo CostTracker, so none is wired here.
+ cost_tracker=None,
+ concurrency=gen_config.max_concurrency,
+ )
+ generated_pages = gen_result.generated_pages
+ marked_stale = gen_result.marked_stale
+ else:
+ logger.info(
+ "generate_job_empty_scope",
+ job_id=job_id,
+ unknown_page_ids=list(plan.unknown_page_ids),
+ )
+
+ await progress.drain_and_stop()
+
+ elapsed = time.monotonic() - start
+ total_input = sum(getattr(p, "input_tokens", 0) for p in generated_pages)
+ total_output = sum(getattr(p, "output_tokens", 0) for p in generated_pages)
+ pages_generated = len(generated_pages)
+
+ async with get_session(session_factory) as session:
+ job = await get_generation_job(session, job_id)
+ final_config = dict(config)
+ final_config.update(
+ {
+ "total_input_tokens": total_input,
+ "total_output_tokens": total_output,
+ "elapsed_seconds": round(elapsed, 1),
+ "pages_generated": pages_generated,
+ "pages_marked_stale": marked_stale,
+ # Surface requested-but-missing ids on the job record, not just in
+ # logs, so the UI can tell a caller a --page id resolved to nothing.
+ "unknown_page_ids": list(plan.unknown_page_ids),
+ }
+ )
+ if job is not None:
+ job.config_json = json.dumps(final_config)
+ await update_job_status(
+ session,
+ job_id,
+ "completed",
+ completed_pages=pages_generated,
+ total_pages=pages_generated,
+ )
+ total_pages, remaining_templates = await _repo_page_counts(session, repo_id)
+
+ # Keep state.json in step with what the server wrote: the sync baseline, the
+ # page count, and docs_mode — which flips to "llm" only once no template page
+ # remains, mirroring the CLI so a later `repowise update` reads the same mode.
+ try:
+ head = _read_head_sha(repo_path)
+ state = _load_state(repo_path)
+ if head:
+ state["last_sync_commit"] = head
+ state["total_pages"] = total_pages
+ if remaining_templates == 0 and pages_generated and resolve_docs_mode(state) != "llm":
+ state.update(docs_mode_state_fields("llm"))
+ _save_state(repo_path, state)
+ except Exception:
+ logger.debug("state_json_update_failed", job_id=job_id, exc_info=True)
+
+ logger.info(
+ "generate_job_completed",
+ job_id=job_id,
+ pages=pages_generated,
+ marked_stale=marked_stale,
+ elapsed=round(elapsed, 1),
+ )
+
+
+async def _repo_page_counts(session: Any, repo_id: str) -> tuple[int, int]:
+ """Return ``(total_pages, remaining_template_pages)`` for a repo."""
+ from sqlalchemy import func as sa_func
+ from sqlalchemy import select as sa_select
+
+ from repowise.core.persistence.models import Page
+
+ total = int(
+ (
+ await session.execute(
+ sa_select(sa_func.count())
+ .select_from(Page)
+ .where(Page.repository_id == repo_id)
+ )
+ ).scalar_one()
+ )
+ templates = int(
+ (
+ await session.execute(
+ sa_select(sa_func.count())
+ .select_from(Page)
+ .where(Page.repository_id == repo_id, Page.provider_name == "template")
+ )
+ ).scalar_one()
+ )
+ return total, templates
+
+
async def _incremental_page_regen(
repo_path: Path,
result: Any,
@@ -731,6 +990,9 @@ async def _incremental_page_regen(
job_config: dict,
progress: Any | None,
repo_wiki_style: str = "comprehensive",
+ *,
+ vector_store: Any | None = None,
+ prior_pages: dict[str, Any] | None = None,
) -> list:
"""Regenerate only wiki pages affected by recent changes.
@@ -826,7 +1088,18 @@ async def _incremental_page_regen(
language=repo_cfg.get("language", "en"),
)
assembler = ContextAssembler(generation_config)
- generator = PageGenerator(llm_client, assembler, generation_config, repo_path=repo_path)
+ # D3: pass the vector store (re-embed re-rendered pages) and prior pages
+ # (reuse an unchanged page instead of re-billing it), matching the CLI
+ # incremental path. Without these, every sync re-billed the repo-wide
+ # pages and dropped them from semantic search.
+ generator = PageGenerator(
+ llm_client,
+ assembler,
+ generation_config,
+ vector_store=vector_store,
+ prior_pages=prior_pages or {},
+ repo_path=repo_path,
+ )
pages = await generator.generate_all(
affected_parsed,
diff --git a/packages/server/src/repowise/server/routers/pages.py b/packages/server/src/repowise/server/routers/pages.py
index 3896b946b..9222fd0e7 100644
--- a/packages/server/src/repowise/server/routers/pages.py
+++ b/packages/server/src/repowise/server/routers/pages.py
@@ -7,7 +7,7 @@ parameter greedily matches the suffix as part of the page_id.
from __future__ import annotations
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
from pydantic import BaseModel
from sqlalchemy.ext.asyncio import AsyncSession
@@ -27,6 +27,11 @@ router = APIRouter(
async def list_pages(
repo_id: str = Query(..., description="Repository ID"),
page_type: str | None = Query(None, description="Filter by page type"),
+ deterministic: bool | None = Query(
+ None,
+ description="true = only unwritten (template) pages; false = only "
+ "model-written pages; omit for both.",
+ ),
sort_by: str = Query(
"updated_at", description="Sort field: updated_at, confidence, created_at"
),
@@ -40,6 +45,7 @@ async def list_pages(
session,
repo_id,
page_type=page_type,
+ deterministic=deterministic,
limit=limit,
offset=offset,
sort_by=sort_by,
@@ -101,24 +107,44 @@ async def update_page_notes(
@router.post("/lookup/regenerate", status_code=202)
async def regenerate_page_by_query(
+ request: Request,
page_id: str = Query(..., description="Page ID"),
style: str | None = Query(
None,
description="Optional wiki style to regenerate this page in (per-page override).",
),
+ cascade: str = Query(
+ "none",
+ description="What to do with the pages that summarize this one: "
+ "none / dependents / full.",
+ ),
session: AsyncSession = Depends(get_db_session), # noqa: B008
) -> dict:
"""Force-regenerate a single wiki page (page_id as query param).
- An optional ``style`` overrides the repo's default style for this page only
- (D10). It is validated here and carried in the job config; the executor
- resolves it when regenerating.
+ Creates a ``single_page`` generation job and launches it immediately (D1):
+ the previous version created a ``pending`` row and never committed or
+ launched it, so the click did nothing until a 15-minute polling fallback
+ picked it up while the active-job guard blocked syncs.
+
+ An optional ``style`` overrides the repo's default style for this page only,
+ and ``cascade`` controls whether the pages summarizing this one are refreshed
+ too. Both are validated here and carried in the job config; the executor
+ resolves them.
"""
+ from repowise.server.routers.repos import _ensure_no_active_job, _launch_job_task
+
page = await crud.get_page(session, page_id)
if page is None:
raise HTTPException(status_code=404, detail="Page not found")
- job_config: dict = {"mode": "single_page", "page_id": page_id}
+ if cascade not in ("none", "dependents", "full"):
+ raise HTTPException(
+ status_code=400,
+ detail=f"Unknown cascade '{cascade}'. Valid: none, dependents, full.",
+ )
+
+ job_config: dict = {"mode": "single_page", "page_id": page_id, "cascade": cascade}
if style is not None:
from repowise.core.generation.styles import is_known_style, list_styles
@@ -130,12 +156,18 @@ async def regenerate_page_by_query(
)
job_config["style"] = style
+ await _ensure_no_active_job(session, page.repository_id)
+
job = await crud.upsert_generation_job(
session,
repository_id=page.repository_id,
status="pending",
config=job_config,
)
+ # Commit (not just flush) so the background task's separate session sees the
+ # job row, then launch it — the fix for the click that did nothing.
+ await session.commit()
+ _launch_job_task(request, job.id, page.repository_id)
return {"job_id": job.id, "status": "accepted"}
diff --git a/packages/server/src/repowise/server/routers/repos.py b/packages/server/src/repowise/server/routers/repos.py
index 225f9db93..35284ed0e 100644
--- a/packages/server/src/repowise/server/routers/repos.py
+++ b/packages/server/src/repowise/server/routers/repos.py
@@ -7,9 +7,11 @@ import io
import logging
import zipfile
from pathlib import Path, PurePosixPath
+from typing import Literal
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import PlainTextResponse, StreamingResponse
+from pydantic import BaseModel, Field
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -452,6 +454,25 @@ async def get_repo_stats(
)
+async def _ensure_no_active_job(session: AsyncSession, repo_id: str) -> None:
+ """Raise 409 if a pending/running job already holds this repo.
+
+ The active-job guard is repo-wide: overlapping runs share a process-global
+ cancel-token slot, so a second concurrent job is refused rather than started.
+ Shared by every job-launching endpoint.
+ """
+ active = await session.execute(
+ select(GenerationJob.id)
+ .where(GenerationJob.repository_id == repo_id)
+ .where(GenerationJob.status.in_(["pending", "running"]))
+ .limit(1)
+ )
+ if active.scalar_one_or_none() is not None:
+ raise HTTPException(
+ status_code=409, detail="A job is already in progress for this repository"
+ )
+
+
@router.post("/{repo_id}/sync", status_code=202)
async def sync_repo(
repo_id: str,
@@ -467,17 +488,7 @@ async def sync_repo(
if repo is None:
raise HTTPException(status_code=404, detail="Repository not found")
- # Prevent concurrent pipeline runs on the same repo
- active = await session.execute(
- select(GenerationJob.id)
- .where(GenerationJob.repository_id == repo_id)
- .where(GenerationJob.status.in_(["pending", "running"]))
- .limit(1)
- )
- if active.scalar_one_or_none() is not None:
- raise HTTPException(
- status_code=409, detail="A sync job is already in progress for this repository"
- )
+ await _ensure_no_active_job(session, repo_id)
job = await crud.upsert_generation_job(
session,
@@ -507,17 +518,7 @@ async def full_resync(
if repo is None:
raise HTTPException(status_code=404, detail="Repository not found")
- # Prevent concurrent pipeline runs on the same repo
- active = await session.execute(
- select(GenerationJob.id)
- .where(GenerationJob.repository_id == repo_id)
- .where(GenerationJob.status.in_(["pending", "running"]))
- .limit(1)
- )
- if active.scalar_one_or_none() is not None:
- raise HTTPException(
- status_code=409, detail="A sync job is already in progress for this repository"
- )
+ await _ensure_no_active_job(session, repo_id)
job = await crud.upsert_generation_job(
session,
@@ -532,6 +533,196 @@ async def full_resync(
return {"job_id": job.id, "status": "accepted"}
+class GenerateSelectionBody(BaseModel):
+ """Which pages a generate request targets.
+
+ Mirrors the CLI's explicit selection: ``all`` / ``unwritten`` / ``stale``,
+ an explicit ``page_ids`` list, or every page under a ``path_prefix``.
+ """
+
+ kind: Literal["all", "unwritten", "stale", "page_ids", "path_prefix"] = "unwritten"
+ page_ids: list[str] | None = None
+ path_prefix: str | None = None
+
+
+class GenerateRequestBody(BaseModel):
+ """Body for the generate + estimate endpoints."""
+
+ selection: GenerateSelectionBody = Field(default_factory=GenerateSelectionBody)
+ cascade: Literal["none", "dependents", "full"] = "dependents"
+ style: str | None = None
+
+
+def _validate_generate_style(style: str | None) -> None:
+ """Reject an unknown wiki style with a 400 listing the valid ones."""
+ if style is None:
+ return
+ from repowise.core.generation.styles import is_known_style, list_styles
+
+ if not is_known_style(style):
+ valid = ", ".join(s.name for s in list_styles())
+ raise HTTPException(status_code=400, detail=f"Unknown style '{style}'. Valid styles: {valid}.")
+
+
+def _generate_job_config(body: GenerateRequestBody) -> dict:
+ """Build the executor's job config from a validated request body."""
+ selection: dict = {"kind": body.selection.kind}
+ if body.selection.kind == "page_ids":
+ selection["page_ids"] = body.selection.page_ids or []
+ elif body.selection.kind == "path_prefix":
+ selection["path_prefix"] = body.selection.path_prefix
+ config: dict = {"mode": "generate", "selection": selection, "cascade": body.cascade}
+ if body.style is not None:
+ config["style"] = body.style
+ return config
+
+
[email protected]("/{repo_id}/generate", status_code=202)
+async def generate_pages(
+ repo_id: str,
+ request: Request,
+ body: GenerateRequestBody,
+ session: AsyncSession = Depends(get_db_session), # noqa: B008
+) -> dict:
+ """Write a subset of the wiki with a model (the HTTP ``repowise generate``).
+
+ Launches a background ``generate`` job that rehydrates the graph, resolves
+ the requested selection + cascade, and writes exactly those pages via the
+ shared core engine. Returns immediately with a job id to stream.
+ """
+ repo = await crud.get_repository(session, repo_id)
+ if repo is None:
+ raise HTTPException(status_code=404, detail="Repository not found")
+
+ _validate_generate_style(body.style)
+ await _ensure_no_active_job(session, repo_id)
+
+ job = await crud.upsert_generation_job(
+ session,
+ repository_id=repo_id,
+ status="pending",
+ config=_generate_job_config(body),
+ )
+ # Commit (not just flush) so the background task's separate session sees the
+ # job row. See sync_repo comment for rationale.
+ await session.commit()
+ _launch_job_task(request, job.id, repo_id)
+ return {"job_id": job.id, "status": "accepted"}
+
+
[email protected]("/{repo_id}/generate/estimate")
+async def generate_estimate(
+ repo_id: str,
+ request: Request,
+ body: GenerateRequestBody,
+ session: AsyncSession = Depends(get_db_session), # noqa: B008
+) -> dict:
+ """Cost + page counts for a generate selection, including cascade fallout.
+
+ Resolves the exact same scope the job would (rehydrating the graph and
+ re-parsing), so the returned page count and estimate match what a launched
+ job spends. Heavier than the pre-index preflight because it walks the real
+ dependency graph rather than a file count.
+ """
+ repo = await crud.get_repository(session, repo_id)
+ if repo is None:
+ raise HTTPException(status_code=404, detail="Repository not found")
+
+ _validate_generate_style(body.style)
+ repo_path = Path(repo.local_path)
+
+ from repowise.core.generation.scope import resolve_scope
+ from repowise.core.pipeline.scoped_generation import rehydrate_repo
+ from repowise.server.job_executor import (
+ _build_generate_intent,
+ _build_generation_config,
+ _load_state,
+ _repo_exclude_patterns,
+ _repo_wiki_style,
+ )
+
+ exclude_patterns = _repo_exclude_patterns(repo, str(repo_path))
+ wiki_style = _repo_wiki_style(repo, str(repo_path))
+ job_config = _generate_job_config(body)
+ gen_config = _build_generation_config(repo_path, job_config, wiki_style)
+
+ # Price with the repo's configured provider/model, if one resolves.
+ provider_name: str | None = None
+ model_name: str | None = None
+ provider_error: str | None = None
+ try:
+ from repowise.server.provider_config import get_chat_provider_instance
+
+ llm_client = get_chat_provider_instance(repo_path=str(repo_path))
+ provider_name = getattr(llm_client, "provider_name", None)
+ model_name = getattr(llm_client, "model_name", None)
+ except Exception as exc:
+ provider_error = str(exc)
+
+ session_factory = _resolve_repo_session_factory(request.app.state, repo_id)
+ state = _load_state(repo_path)
+ # Read-only preflight: an un-indexed repo (no persisted graph) or one with no
+ # wiki pages yet is a zero estimate, not an error. A launched job would fail
+ # loudly instead; here we just report there is nothing to price.
+ note: str | None = None
+ rehydrated = None
+ try:
+ rehydrated = await rehydrate_repo(
+ session_factory,
+ repo_id,
+ repo_path,
+ generation_config=gen_config,
+ exclude_patterns=exclude_patterns,
+ include_submodules=bool(state.get("include_submodules", False)),
+ include_nested_repos=bool(state.get("include_nested_repos", False)),
+ )
+ except Exception as exc:
+ note = str(exc)
+
+ if rehydrated is None:
+ return {
+ "total_pages": 0,
+ "pages_by_type": {},
+ "pages_to_mark_stale": 0,
+ "unknown_page_ids": [],
+ "provider": {"name": provider_name, "model": model_name, "error": provider_error},
+ "estimate": None,
+ "note": note,
+ }
+
+ plan = resolve_scope(
+ records=rehydrated.records,
+ intent=_build_generate_intent(job_config),
+ cascade_mode=body.cascade,
+ deps=rehydrated.deps,
+ )
+ pages_by_type = {p.page_type: p.count for p in plan.cost_plans}
+ total_pages = sum(pages_by_type.values())
+
+ estimate: dict | None = None
+ if provider_name and model_name and plan.cost_plans:
+ from repowise.core.cost_estimator import estimate_cost
+
+ est = estimate_cost(plan.cost_plans, provider_name, model_name, repo_path=str(repo_path))
+ estimate = {
+ "estimated_cost_usd": round(est.estimated_cost_usd, 4),
+ "cost_low_usd": round(est.cost_range.low, 4) if est.cost_range else None,
+ "cost_high_usd": round(est.cost_range.high, 4) if est.cost_range else None,
+ "estimated_input_tokens": est.estimated_input_tokens,
+ "estimated_output_tokens": est.estimated_output_tokens,
+ "is_calibrated": est.is_calibrated,
+ }
+
+ return {
+ "total_pages": total_pages,
+ "pages_by_type": pages_by_type,
+ "pages_to_mark_stale": len(plan.stale_ids),
+ "unknown_page_ids": list(plan.unknown_page_ids),
+ "provider": {"name": provider_name, "model": model_name, "error": provider_error},
+ "estimate": estimate,
+ }
+
+
@router.post("/{repo_id}/index", status_code=202)
async def index_repo(
repo_id: str,
diff --git a/tests/unit/server/test_generate.py b/tests/unit/server/test_generate.py
new file mode 100644
index 000000000..f1ecd1964
--- /dev/null
+++ b/tests/unit/server/test_generate.py
@@ -0,0 +1,307 @@
+"""Tests for the HTTP scoped-generation surface.
+
+Covers the ``/api/repos/{id}/generate`` + ``/generate/estimate`` endpoints, the
+``generate`` job mode in the executor (the shared core engine), the D1 launch fix
+on the regenerate endpoint, and the D3 wiring on the incremental sync path.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from httpx import AsyncClient
+
+from repowise.core.generation.cascade import build_page_dependencies
+from repowise.core.generation.page_selection import PageRecord, PageSelectionIntent
+from repowise.core.persistence import crud
+from repowise.core.persistence.database import get_session
+from repowise.core.pipeline.scoped_generation import RehydratedRepo, ScopedGenerationResult
+from repowise.server.job_executor import _build_generate_intent
+from tests.unit.server.conftest import create_test_repo
+
+# ---------------------------------------------------------------------------
+# _build_generate_intent
+# ---------------------------------------------------------------------------
+
+
+ ("config", "expected"),
+ [
+ ({"selection": {"kind": "all"}}, PageSelectionIntent(all_pages=True)),
+ ({"selection": {"kind": "unwritten"}}, PageSelectionIntent(unwritten=True)),
+ ({"selection": {"kind": "stale"}}, PageSelectionIntent(stale=True)),
+ (
+ {"selection": {"kind": "page_ids", "page_ids": ["file_page:a.py"]}},
+ PageSelectionIntent(page_ids=("file_page:a.py",)),
+ ),
+ (
+ {"selection": {"kind": "path_prefix", "path_prefix": "src/"}},
+ PageSelectionIntent(path_globs=("src/",)),
+ ),
+ ({}, PageSelectionIntent(unwritten=True)),
+ (
+ {"mode": "single_page", "page_id": "file_page:a.py"},
+ PageSelectionIntent(page_ids=("file_page:a.py",)),
+ ),
+ ],
+)
+def test_build_generate_intent(config: dict, expected: PageSelectionIntent) -> None:
+ assert _build_generate_intent(config) == expected
+
+
+# ---------------------------------------------------------------------------
+# POST /generate + /generate/estimate endpoints
+# ---------------------------------------------------------------------------
+
+
+async def _job_config(session_factory, job_id: str) -> dict:
+ async with get_session(session_factory) as session:
+ job = await crud.get_generation_job(session, job_id)
+ return json.loads(job.config_json) if job.config_json else {}
+
+
+async def test_generate_endpoint_creates_and_launches_job(client: AsyncClient, app) -> None:
+ repo = await create_test_repo(client)
+ with patch("repowise.server.routers.repos._launch_job_task") as launch:
+ resp = await client.post(
+ f"/api/repos/{repo['id']}/generate",
+ json={"selection": {"kind": "unwritten"}, "cascade": "dependents"},
+ )
+ assert resp.status_code == 202
+ job_id = resp.json()["job_id"]
+ launch.assert_called_once()
+
+ cfg = await _job_config(app.state.session_factory, job_id)
+ assert cfg["mode"] == "generate"
+ assert cfg["selection"] == {"kind": "unwritten"}
+ assert cfg["cascade"] == "dependents"
+
+
+async def test_generate_endpoint_page_ids_selection(client: AsyncClient, app) -> None:
+ repo = await create_test_repo(client)
+ with patch("repowise.server.routers.repos._launch_job_task"):
+ resp = await client.post(
+ f"/api/repos/{repo['id']}/generate",
+ json={
+ "selection": {"kind": "page_ids", "page_ids": ["file_page:a.py"]},
+ "cascade": "none",
+ "style": "caveman",
+ },
+ )
+ assert resp.status_code == 202
+ cfg = await _job_config(app.state.session_factory, resp.json()["job_id"])
+ assert cfg["selection"] == {"kind": "page_ids", "page_ids": ["file_page:a.py"]}
+ assert cfg["style"] == "caveman"
+
+
+async def test_generate_endpoint_rejects_unknown_style(client: AsyncClient) -> None:
+ repo = await create_test_repo(client)
+ resp = await client.post(
+ f"/api/repos/{repo['id']}/generate",
+ json={"selection": {"kind": "all"}, "style": "bogus"},
+ )
+ assert resp.status_code == 400
+ assert "style" in resp.json()["detail"].lower()
+
+
+async def test_generate_endpoint_404_unknown_repo(client: AsyncClient) -> None:
+ resp = await client.post("/api/repos/nope/generate", json={})
+ assert resp.status_code == 404
+
+
+async def test_generate_endpoint_409_when_job_active(client: AsyncClient, app) -> None:
+ repo = await create_test_repo(client)
+ async with get_session(app.state.session_factory) as session:
+ await crud.upsert_generation_job(
+ session, repository_id=repo["id"], status="running", config={"mode": "sync"}
+ )
+ await session.commit()
+ resp = await client.post(f"/api/repos/{repo['id']}/generate", json={})
+ assert resp.status_code == 409
+
+
+async def test_generate_estimate_no_pages(client: AsyncClient) -> None:
+ """A repo with no wiki pages returns a zero estimate, not a crash."""
+ repo = await create_test_repo(client)
+ with patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ):
+ resp = await client.post(
+ f"/api/repos/{repo['id']}/generate/estimate",
+ json={"selection": {"kind": "unwritten"}},
+ )
+ assert resp.status_code == 200, resp.text
+ body = resp.json()
+ assert body["total_pages"] == 0
+ assert body["estimate"] is None
+ assert body["provider"]["error"] is not None
+
+
+# ---------------------------------------------------------------------------
+# generate job execution (shared core engine)
+# ---------------------------------------------------------------------------
+
+
+def _fake_rehydrated(template_ids: list[str]) -> RehydratedRepo:
+ records = [
+ PageRecord(
+ page_id=pid,
+ page_type=pid.split(":", 1)[0],
+ target_path=pid.split(":", 1)[1],
+ is_template=True,
+ freshness_status="fresh",
+ )
+ for pid in template_ids
+ ]
+ deps = build_page_dependencies(
+ module_groups=[], scc_groups=[], layer_page_of={}, repo_wide_ids=[]
+ )
+ return RehydratedRepo(
+ graph_builder=MagicMock(),
+ git_meta_map={},
+ parsed_files=[],
+ source_map={},
+ repo_structure=MagicMock(),
+ records=records,
+ kg_ctx=MagicMock(available=False),
+ deps=deps,
+ repo_name="test-repo",
+ )
+
+
+async def _seed_generate_job(session_factory, repo_path: Path, config: dict) -> str:
+ async with session_factory() as session:
+ repo = await crud.upsert_repository(
+ session, name="test-repo", local_path=str(repo_path)
+ )
+ job = await crud.upsert_generation_job(
+ session, repository_id=repo.id, config=config
+ )
+ await session.commit()
+ return job.id
+
+
+async def test_generate_job_runs_scoped_engine(session_factory, tmp_path) -> None:
+ """A generate job resolves the scope and drives the shared core engine."""
+ from repowise.server.job_executor import execute_job
+
+ (tmp_path / ".git").mkdir()
+ job_id = await _seed_generate_job(
+ session_factory,
+ tmp_path,
+ {"mode": "generate", "selection": {"kind": "unwritten"}, "cascade": "none"},
+ )
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ template_ids = ["file_page:a.py", "file_page:b.py"]
+ written = SimpleNamespace(
+ page_id="file_page:a.py", title="a", content="x", input_tokens=10, output_tokens=20
+ )
+ execute_mock = AsyncMock(
+ return_value=ScopedGenerationResult(generated_pages=[written], marked_stale=0)
+ )
+ with (
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ return_value=MagicMock(provider_name="openai", model_name="gpt"),
+ ),
+ patch(
+ "repowise.server.search_helpers.resolve_repo_vector_store",
+ AsyncMock(return_value=None),
+ ),
+ patch(
+ "repowise.core.pipeline.scoped_generation.rehydrate_repo",
+ AsyncMock(return_value=_fake_rehydrated(template_ids)),
+ ),
+ patch(
+ "repowise.core.pipeline.scoped_generation.execute_scoped_generation",
+ execute_mock,
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ execute_mock.assert_awaited_once()
+ plan = execute_mock.await_args.kwargs["plan"]
+ assert plan.generate_ids == set(template_ids)
+
+ async with get_session(session_factory) as session:
+ job = await crud.get_generation_job(session, job_id)
+ assert job.status == "completed"
+ cfg = json.loads(job.config_json)
+ assert cfg["pages_generated"] == 1
+
+
+async def test_generate_job_fails_without_provider(session_factory, tmp_path) -> None:
+ from repowise.server.job_executor import execute_job
+
+ (tmp_path / ".git").mkdir()
+ job_id = await _seed_generate_job(
+ session_factory, tmp_path, {"mode": "generate", "selection": {"kind": "all"}}
+ )
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ with (
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ patch(
+ "repowise.server.search_helpers.resolve_repo_vector_store",
+ AsyncMock(return_value=None),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ async with get_session(session_factory) as session:
+ job = await crud.get_generation_job(session, job_id)
+ assert job.status == "failed"
+
+
+# ---------------------------------------------------------------------------
+# D3: incremental sync hands the regen a vector store + prior pages
+# ---------------------------------------------------------------------------
+
+
+async def test_sync_regen_receives_vector_store_and_prior_pages(session_factory, tmp_path) -> None:
+ from repowise.server.job_executor import execute_job
+
+ (tmp_path / ".git").mkdir()
+ job_id = await _seed_generate_job(session_factory, tmp_path, {"mode": "sync"})
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ fake_result = SimpleNamespace(
+ generated_pages=[], parsed_files=[], file_count=1, symbol_count=1
+ )
+ regen_mock = AsyncMock(return_value=[])
+ with (
+ patch("repowise.server.job_executor.run_pipeline", AsyncMock(return_value=fake_result)),
+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock(return_value=[])),
+ patch("repowise.server.job_executor._incremental_page_regen", regen_mock),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ return_value=MagicMock(provider_name="openai", model_name="gpt"),
+ ),
+ patch(
+ "repowise.server.search_helpers.resolve_repo_vector_store",
+ AsyncMock(return_value=None),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ regen_mock.assert_awaited_once()
+ assert "vector_store" in regen_mock.await_args.kwargs
+ assert "prior_pages" in regen_mock.await_args.kwargs
diff --git a/tests/unit/server/test_pages.py b/tests/unit/server/test_pages.py
index cf3ec27ba..7972d819d 100644
--- a/tests/unit/server/test_pages.py
+++ b/tests/unit/server/test_pages.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+from unittest.mock import patch
+
import pytest
from httpx import AsyncClient
@@ -115,10 +117,66 @@ async def test_update_page_notes_not_found(client: AsyncClient) -> None:
assert resp.status_code == 404
+async def test_list_pages_deterministic_filter(client: AsyncClient, app) -> None:
+ """?deterministic splits template (unwritten) pages from model-written ones."""
+ repo = await create_test_repo(client)
+ repo_id = repo["id"]
+ async with get_session(app.state.session_factory) as session:
+ await crud.upsert_page(
+ session,
+ page_id="file_page:tmpl.py",
+ repository_id=repo_id,
+ page_type="file_page",
+ title="tmpl",
+ content="template page",
+ target_path="tmpl.py",
+ source_hash="",
+ model_name="template",
+ provider_name="template",
+ )
+ await crud.upsert_page(
+ session,
+ page_id="file_page:written.py",
+ repository_id=repo_id,
+ page_type="file_page",
+ title="written",
+ content="model page",
+ target_path="written.py",
+ source_hash="h",
+ model_name="gpt",
+ provider_name="openai",
+ )
+
+ unwritten = await client.get(
+ "/api/pages", params={"repo_id": repo_id, "deterministic": "true"}
+ )
+ assert {p["id"] for p in unwritten.json()} == {"file_page:tmpl.py"}
+
+ written = await client.get(
+ "/api/pages", params={"repo_id": repo_id, "deterministic": "false"}
+ )
+ assert {p["id"] for p in written.json()} == {"file_page:written.py"}
+
+ both = await client.get("/api/pages", params={"repo_id": repo_id})
+ assert len(both.json()) == 2
+
+
+async def test_regenerate_page_launches_job(client: AsyncClient, app) -> None:
+ """D1: the regenerate click commits the job row and launches it immediately."""
+ _, page_id = await _create_page(client, app.state.session_factory)
+ with patch("repowise.server.routers.repos._launch_job_task") as launch:
+ resp = await client.post("/api/pages/lookup/regenerate", params={"page_id": page_id})
+ assert resp.status_code == 202
+ launch.assert_called_once()
+
+
@pytest.mark.asyncio
async def test_regenerate_page_returns_202(client: AsyncClient, app) -> None:
_, page_id = await _create_page(client, app.state.session_factory)
- resp = await client.post("/api/pages/lookup/regenerate", params={"page_id": page_id})
+ with patch("repowise.server.routers.repos._launch_job_task"):
+ resp = await client.post("/api/pages/lookup/regenerate", params={"page_id": page_id})
assert resp.status_code == 202
data = resp.json()
assert "job_id" in data
@@ -136,10 +194,11 @@ async def test_regenerate_page_not_found(client: AsyncClient) -> None:
@pytest.mark.asyncio
async def test_regenerate_page_with_known_style_accepted(client: AsyncClient, app) -> None:
_, page_id = await _create_page(client, app.state.session_factory)
- resp = await client.post(
- "/api/pages/lookup/regenerate",
- params={"page_id": page_id, "style": "caveman"},
- )
+ with patch("repowise.server.routers.repos._launch_job_task"):
+ resp = await client.post(
+ "/api/pages/lookup/regenerate",
+ params={"page_id": page_id, "style": "caveman"},
+ )
assert resp.status_code == 202
assert "job_id" in resp.json()