Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(server): take a repo from registered to fully indexed over the API (#688)

Raghav Chamadiya9d120d03ce1f
diff --git a/packages/cli/src/repowise/cli/cost_estimator/__init__.py b/packages/cli/src/repowise/cli/cost_estimator/__init__.py
index 161003616..8d69a28da 100644
--- a/packages/cli/src/repowise/cli/cost_estimator/__init__.py
+++ b/packages/cli/src/repowise/cli/cost_estimator/__init__.py
@@ -1,35 +1,31 @@
-"""Cost estimation for the repowise generation pipeline.
+"""Backward-compatible re-export of :mod:`repowise.core.cost_estimator`.
-Public API kept stable for existing callers:
-
-- :func:`build_generation_plan` — given parsed_files + graph + config,
- returns a list of :class:`PageTypePlan`. Internally delegates to
- :func:`repowise.core.generation.select_pages` so the estimator and
- the actual generator never disagree.
-- :func:`estimate_cost` — given plans + model, returns a
- :class:`CostEstimate` with a median + low/high range. Optionally
- calibrated against ``.repowise/db.sqlite`` telemetry.
-- :func:`_lookup_cost` — retained for the provider unit tests.
+The estimator moved to core so the server's pre-flight endpoint can share
+the same pricing, heuristics, and calibration as the CLI cost gate. This
+shim keeps the historical ``repowise.cli.cost_estimator`` import path
+working for existing callers.
"""
-from .coverage import (
+from repowise.core.cost_estimator import (
DEFAULT_COVERAGE_OPTIONS,
RECOMMENDED_COVERAGE,
+ CostEstimate,
+ CostRange,
CoverageOption,
+ PageTypePlan,
+ _lookup_cost,
+ build_generation_plan,
compute_coverage_options,
+ estimate_cost,
)
-from .estimator import estimate_cost
-from .plans import build_generation_plan
-from .pricing import _lookup_cost
-from .types import CostEstimate, CostRange, PageTypePlan
__all__ = [
+ "DEFAULT_COVERAGE_OPTIONS",
+ "RECOMMENDED_COVERAGE",
"CostEstimate",
"CostRange",
"CoverageOption",
- "DEFAULT_COVERAGE_OPTIONS",
"PageTypePlan",
- "RECOMMENDED_COVERAGE",
"_lookup_cost",
"build_generation_plan",
"compute_coverage_options",
diff --git a/packages/cli/src/repowise/cli/cost_estimator/calibration.py b/packages/cli/src/repowise/cli/cost_estimator/calibration.py
index 60acb80f9..b9818f0b0 100644
--- a/packages/cli/src/repowise/cli/cost_estimator/calibration.py
+++ b/packages/cli/src/repowise/cli/cost_estimator/calibration.py
@@ -1,55 +1,5 @@
-"""Telemetry-based calibration from prior generation runs.
+"""Backward-compatible re-export; the module lives in core now."""
-When ``.repowise/db.sqlite`` exists and has ``wiki_pages`` rows for the
-target repo, we read actual ``input_tokens`` / ``output_tokens`` per
-page_type and use the per-type average instead of the static
-heuristics. This makes the **second** run of ``repowise init`` /
-``repowise update`` on the same repo highly accurate.
+from repowise.core.cost_estimator.calibration import load_telemetry_averages
-The function is intentionally tolerant of missing tables and schema
-drift — any error short-circuits to ``{}`` so the heuristic path runs.
-"""
-
-from __future__ import annotations
-
-import sqlite3
-from pathlib import Path
-
-import structlog
-
-log = structlog.get_logger(__name__)
-
-
-def load_telemetry_averages(repo_path: Path | str) -> dict[str, tuple[float, float]]:
- """Return ``{page_type: (avg_input, avg_output)}`` from prior runs.
-
- Returns an empty dict when no telemetry is available — the caller
- falls back to static heuristics.
- """
- db_path = Path(repo_path) / ".repowise" / "db.sqlite"
- if not db_path.exists():
- return {}
-
- try:
- with sqlite3.connect(str(db_path)) as conn:
- cur = conn.execute(
- """
- SELECT page_type, AVG(input_tokens), AVG(output_tokens), COUNT(*)
- FROM wiki_pages
- WHERE input_tokens > 0 AND output_tokens > 0
- GROUP BY page_type
- """
- )
- rows = cur.fetchall()
- except sqlite3.Error as exc:
- log.debug("cost_estimator.telemetry_load_failed", error=str(exc))
- return {}
-
- averages: dict[str, tuple[float, float]] = {}
- for page_type, avg_in, avg_out, n in rows:
- # Require at least 3 historical samples per page_type to trust
- # the average — fewer than that is dominated by outliers.
- if n is None or n < 3:
- continue
- averages[str(page_type)] = (float(avg_in or 0.0), float(avg_out or 0.0))
- return averages
+__all__ = ["load_telemetry_averages"]
diff --git a/packages/cli/src/repowise/cli/cost_estimator/heuristics.py b/packages/cli/src/repowise/cli/cost_estimator/heuristics.py
index 2b2e1c770..465a891d1 100644
--- a/packages/cli/src/repowise/cli/cost_estimator/heuristics.py
+++ b/packages/cli/src/repowise/cli/cost_estimator/heuristics.py
@@ -1,40 +1,8 @@
-"""Per-page-type token heuristics.
+"""Backward-compatible re-export; the module lives in core now."""
-These default averages were recalibrated after observing a real run on
-the repowise repo where the prior estimate (4000/2500 per ``file_page``)
-was ~5-8× under-estimating actual input tokens. Real pages include the
-full Jinja2-rendered context: file path + symbol signatures + imports +
-exports + source snippet + dep summaries + RAG context + git metadata.
+from repowise.core.cost_estimator.heuristics import (
+ HEURISTIC_VARIANCE,
+ heuristic_tokens,
+)
-The estimator prefers SQLite telemetry when available
-(see :mod:`calibration`); these heuristics are the cold-start fallback.
-"""
-
-from __future__ import annotations
-
-
-# (input_tokens, output_tokens) per page. Heuristics that bias slightly
-# high — better to over-estimate cost than surprise the user with a
-# bill that's an order of magnitude over the quote.
-_TOKEN_HEURISTICS: dict[str, tuple[int, int]] = {
- "api_contract": (8_000, 3_000),
- "symbol_spotlight": (8_000, 2_500),
- "file_page": (25_000, 5_000),
- "scc_page": (12_000, 4_000),
- "module_page": (45_000, 6_000),
- "repo_overview": (30_000, 5_000),
- "architecture_diagram": (25_000, 5_000),
- "infra_page": (4_000, 2_000),
- "onboarding": (15_000, 3_500),
-}
-
-
-# Variance bracket used to derive the low/high range. Plus/minus 25%
-# captures roughly one standard deviation of observed variance in mixed
-# real-world generation runs.
-HEURISTIC_VARIANCE = 0.25
-
-
-def heuristic_tokens(page_type: str) -> tuple[int, int]:
- """Return ``(input, output)`` tokens for *page_type*; safe default."""
- return _TOKEN_HEURISTICS.get(page_type, (10_000, 3_000))
+__all__ = ["HEURISTIC_VARIANCE", "heuristic_tokens"]
diff --git a/packages/cli/src/repowise/cli/helpers.py b/packages/cli/src/repowise/cli/helpers.py
index c1d9b4b22..803b3ff3d 100644
--- a/packages/cli/src/repowise/cli/helpers.py
+++ b/packages/cli/src/repowise/cli/helpers.py
@@ -632,20 +632,12 @@ def save_distill_commands_enabled(repo_path: Path, *, enabled: bool) -> None:
def config_fingerprint(repo_path: Path) -> str:
"""SHA-256 hex of ``.repowise/config.yaml`` + ``health-rules.json`` content.
- Used by ``repowise update`` and ``repowise init`` to detect config changes
- across runs without relying on filesystem timestamps. Missing files are
- skipped, so an absent config still yields a stable hash.
+ Delegates to the shared core implementation so CLI runs and server jobs
+ compute identical fingerprints for the same on-disk config.
"""
- import hashlib
-
- rw_dir = get_repowise_dir(repo_path)
- h = hashlib.sha256()
- for name in ("config.yaml", "health-rules.json"):
- p = rw_dir / name
- if p.exists():
- h.update(name.encode())
- h.update(p.read_bytes())
- return h.hexdigest()
+ from repowise.core.repo_config import config_fingerprint as _core_fingerprint
+
+ return _core_fingerprint(repo_path)
# ---------------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/cost_estimator/__init__.py b/packages/core/src/repowise/core/cost_estimator/__init__.py
new file mode 100644
index 000000000..f748b5df1
--- /dev/null
+++ b/packages/core/src/repowise/core/cost_estimator/__init__.py
@@ -0,0 +1,39 @@
+"""Cost estimation for the repowise generation pipeline.
+
+Public API kept stable for existing callers:
+
+- :func:`build_generation_plan` — given parsed_files + graph + config,
+ returns a list of :class:`PageTypePlan`. Internally delegates to
+ :func:`repowise.core.generation.select_pages` so the estimator and
+ the actual generator never disagree.
+- :func:`estimate_cost` — given plans + model, returns a
+ :class:`CostEstimate` with a median + low/high range. Optionally
+ calibrated against ``.repowise/db.sqlite`` telemetry.
+- :func:`_lookup_cost` — retained for the provider unit tests.
+"""
+
+from .approx import approximate_generation_plan
+from .coverage import (
+ DEFAULT_COVERAGE_OPTIONS,
+ RECOMMENDED_COVERAGE,
+ CoverageOption,
+ compute_coverage_options,
+)
+from .estimator import estimate_cost
+from .plans import build_generation_plan
+from .pricing import _lookup_cost
+from .types import CostEstimate, CostRange, PageTypePlan
+
+__all__ = [
+ "CostEstimate",
+ "CostRange",
+ "CoverageOption",
+ "DEFAULT_COVERAGE_OPTIONS",
+ "PageTypePlan",
+ "RECOMMENDED_COVERAGE",
+ "_lookup_cost",
+ "approximate_generation_plan",
+ "build_generation_plan",
+ "compute_coverage_options",
+ "estimate_cost",
+]
diff --git a/packages/core/src/repowise/core/cost_estimator/approx.py b/packages/core/src/repowise/core/cost_estimator/approx.py
new file mode 100644
index 000000000..9706a7627
--- /dev/null
+++ b/packages/core/src/repowise/core/cost_estimator/approx.py
@@ -0,0 +1,53 @@
+"""Pre-parse generation-plan approximation.
+
+:func:`build_generation_plan` needs parsed files and a finalized graph, which
+means running the ingestion pipeline first. The server's pre-flight check must
+answer "roughly how many pages, roughly how much" *before* any indexing work,
+from nothing but a file count. This module derives an approximate plan from
+the same selection ratios the real selector converges on, so the pre-flight
+number and the post-index estimate stay in the same ballpark.
+
+Approximation, not selection: counts here can differ from the real plan by a
+handful of pages (module grouping and SCC detection need the graph). The
+variance bracket reported by :func:`~.estimator.estimate_cost` absorbs that.
+"""
+
+from __future__ import annotations
+
+from .types import PageTypePlan
+
+# Observed module-page ratio: curated/community grouping lands near one
+# module page per ~25 selected file pages, floored at 1 and capped so a
+# monorepo estimate does not balloon.
+_FILES_PER_MODULE = 25
+_MAX_MODULE_PAGES = 24
+
+# The curated onboarding collection has 8 fixed slots; slots without
+# enough signal are skipped, so this biases slightly high on tiny repos.
+_ONBOARDING_SLOTS = 8
+
+
+def approximate_generation_plan(
+ file_count: int,
+ *,
+ coverage_pct: float = 0.20,
+ onboarding: bool = True,
+) -> list[PageTypePlan]:
+ """Return an approximate plan for *file_count* documentable files."""
+ file_count = max(0, file_count)
+ file_pages = round(file_count * max(0.0, min(coverage_pct, 1.0)))
+ if file_count > 0:
+ file_pages = max(1, file_pages)
+ module_pages = (
+ min(_MAX_MODULE_PAGES, max(1, file_pages // _FILES_PER_MODULE)) if file_pages else 0
+ )
+
+ plans = [
+ PageTypePlan(page_type="file_page", count=file_pages, level=0),
+ PageTypePlan(page_type="module_page", count=module_pages, level=1),
+ PageTypePlan(page_type="repo_overview", count=1 if file_count else 0, level=2),
+ PageTypePlan(page_type="architecture_diagram", count=1 if file_count else 0, level=2),
+ ]
+ if onboarding and file_count:
+ plans.append(PageTypePlan(page_type="onboarding", count=_ONBOARDING_SLOTS, level=2))
+ return [p for p in plans if p.count > 0]
diff --git a/packages/core/src/repowise/core/cost_estimator/calibration.py b/packages/core/src/repowise/core/cost_estimator/calibration.py
new file mode 100644
index 000000000..60acb80f9
--- /dev/null
+++ b/packages/core/src/repowise/core/cost_estimator/calibration.py
@@ -0,0 +1,55 @@
+"""Telemetry-based calibration from prior generation runs.
+
+When ``.repowise/db.sqlite`` exists and has ``wiki_pages`` rows for the
+target repo, we read actual ``input_tokens`` / ``output_tokens`` per
+page_type and use the per-type average instead of the static
+heuristics. This makes the **second** run of ``repowise init`` /
+``repowise update`` on the same repo highly accurate.
+
+The function is intentionally tolerant of missing tables and schema
+drift — any error short-circuits to ``{}`` so the heuristic path runs.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+import structlog
+
+log = structlog.get_logger(__name__)
+
+
+def load_telemetry_averages(repo_path: Path | str) -> dict[str, tuple[float, float]]:
+ """Return ``{page_type: (avg_input, avg_output)}`` from prior runs.
+
+ Returns an empty dict when no telemetry is available — the caller
+ falls back to static heuristics.
+ """
+ db_path = Path(repo_path) / ".repowise" / "db.sqlite"
+ if not db_path.exists():
+ return {}
+
+ try:
+ with sqlite3.connect(str(db_path)) as conn:
+ cur = conn.execute(
+ """
+ SELECT page_type, AVG(input_tokens), AVG(output_tokens), COUNT(*)
+ FROM wiki_pages
+ WHERE input_tokens > 0 AND output_tokens > 0
+ GROUP BY page_type
+ """
+ )
+ rows = cur.fetchall()
+ except sqlite3.Error as exc:
+ log.debug("cost_estimator.telemetry_load_failed", error=str(exc))
+ return {}
+
+ averages: dict[str, tuple[float, float]] = {}
+ for page_type, avg_in, avg_out, n in rows:
+ # Require at least 3 historical samples per page_type to trust
+ # the average — fewer than that is dominated by outliers.
+ if n is None or n < 3:
+ continue
+ averages[str(page_type)] = (float(avg_in or 0.0), float(avg_out or 0.0))
+ return averages
diff --git a/packages/cli/src/repowise/cli/cost_estimator/coverage.py b/packages/core/src/repowise/core/cost_estimator/coverage.py
similarity index 100%
rename from packages/cli/src/repowise/cli/cost_estimator/coverage.py
rename to packages/core/src/repowise/core/cost_estimator/coverage.py
diff --git a/packages/cli/src/repowise/cli/cost_estimator/estimator.py b/packages/core/src/repowise/core/cost_estimator/estimator.py
similarity index 100%
rename from packages/cli/src/repowise/cli/cost_estimator/estimator.py
rename to packages/core/src/repowise/core/cost_estimator/estimator.py
diff --git a/packages/core/src/repowise/core/cost_estimator/heuristics.py b/packages/core/src/repowise/core/cost_estimator/heuristics.py
new file mode 100644
index 000000000..2b2e1c770
--- /dev/null
+++ b/packages/core/src/repowise/core/cost_estimator/heuristics.py
@@ -0,0 +1,40 @@
+"""Per-page-type token heuristics.
+
+These default averages were recalibrated after observing a real run on
+the repowise repo where the prior estimate (4000/2500 per ``file_page``)
+was ~5-8× under-estimating actual input tokens. Real pages include the
+full Jinja2-rendered context: file path + symbol signatures + imports +
+exports + source snippet + dep summaries + RAG context + git metadata.
+
+The estimator prefers SQLite telemetry when available
+(see :mod:`calibration`); these heuristics are the cold-start fallback.
+"""
+
+from __future__ import annotations
+
+
+# (input_tokens, output_tokens) per page. Heuristics that bias slightly
+# high — better to over-estimate cost than surprise the user with a
+# bill that's an order of magnitude over the quote.
+_TOKEN_HEURISTICS: dict[str, tuple[int, int]] = {
+ "api_contract": (8_000, 3_000),
+ "symbol_spotlight": (8_000, 2_500),
+ "file_page": (25_000, 5_000),
+ "scc_page": (12_000, 4_000),
+ "module_page": (45_000, 6_000),
+ "repo_overview": (30_000, 5_000),
+ "architecture_diagram": (25_000, 5_000),
+ "infra_page": (4_000, 2_000),
+ "onboarding": (15_000, 3_500),
+}
+
+
+# Variance bracket used to derive the low/high range. Plus/minus 25%
+# captures roughly one standard deviation of observed variance in mixed
+# real-world generation runs.
+HEURISTIC_VARIANCE = 0.25
+
+
+def heuristic_tokens(page_type: str) -> tuple[int, int]:
+ """Return ``(input, output)`` tokens for *page_type*; safe default."""
+ return _TOKEN_HEURISTICS.get(page_type, (10_000, 3_000))
diff --git a/packages/cli/src/repowise/cli/cost_estimator/plans.py b/packages/core/src/repowise/core/cost_estimator/plans.py
similarity index 100%
rename from packages/cli/src/repowise/cli/cost_estimator/plans.py
rename to packages/core/src/repowise/core/cost_estimator/plans.py
diff --git a/packages/cli/src/repowise/cli/cost_estimator/pricing.py b/packages/core/src/repowise/core/cost_estimator/pricing.py
similarity index 100%
rename from packages/cli/src/repowise/cli/cost_estimator/pricing.py
rename to packages/core/src/repowise/core/cost_estimator/pricing.py
diff --git a/packages/cli/src/repowise/cli/cost_estimator/types.py b/packages/core/src/repowise/core/cost_estimator/types.py
similarity index 100%
rename from packages/cli/src/repowise/cli/cost_estimator/types.py
rename to packages/core/src/repowise/core/cost_estimator/types.py
diff --git a/packages/core/src/repowise/core/persistence/crud/_shared.py b/packages/core/src/repowise/core/persistence/crud/_shared.py
index 0e2911910..41dee5d4f 100644
--- a/packages/core/src/repowise/core/persistence/crud/_shared.py
+++ b/packages/core/src/repowise/core/persistence/crud/_shared.py
@@ -15,7 +15,9 @@ from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
-_VALID_JOB_STATUSES = frozenset({"pending", "running", "completed", "failed", "paused"})
+_VALID_JOB_STATUSES = frozenset(
+ {"pending", "running", "completed", "failed", "cancelled", "paused"}
+)
_BATCH_SIZE = 500 # max rows per INSERT to stay under SQLite's parameter limit
diff --git a/packages/core/src/repowise/core/persistence/crud/repository.py b/packages/core/src/repowise/core/persistence/crud/repository.py
index 1d380ec3e..86bbdc72e 100644
--- a/packages/core/src/repowise/core/persistence/crud/repository.py
+++ b/packages/core/src/repowise/core/persistence/crud/repository.py
@@ -36,11 +36,18 @@ async def upsert_repository(
default_branch: str = "main",
settings: dict | None = None,
head_commit: str | None = None,
+ repo_id: str | None = None,
) -> Repository:
"""Create or update a repository record.
Lookup is by ``local_path`` (the canonical key for local repositories).
+ ``repo_id`` fixes the primary key when the row is *created*. The server
+ keeps a registry row for each repo in its primary database and the
+ canonical row in the repo-local ``wiki.db``; both must share one id so
+ per-repo session routing keyed by that id resolves consistently. Ignored
+ when a row for ``local_path`` already exists (the existing id wins).
+
``head_commit`` records the git commit the index was built against — the
value the MCP ``_meta`` freshness check compares to the live HEAD. Callers
that already know the synced commit may pass it; otherwise it is read from
@@ -56,7 +63,7 @@ async def upsert_repository(
if repo is None:
repo = Repository(
- id=_new_uuid(),
+ id=repo_id or _new_uuid(),
name=name,
local_path=local_path,
url=url,
@@ -231,7 +238,7 @@ async def update_job_status(
if status == "running" and job.started_at is None:
job.started_at = _now_utc()
- if status in ("completed", "failed"):
+ if status in ("completed", "failed", "cancelled"):
job.finished_at = _now_utc()
await session.flush()
diff --git a/packages/core/src/repowise/core/repo_config.py b/packages/core/src/repowise/core/repo_config.py
index 3e0651496..a6c39b5be 100644
--- a/packages/core/src/repowise/core/repo_config.py
+++ b/packages/core/src/repowise/core/repo_config.py
@@ -57,6 +57,26 @@ def save_repo_config(repo_path: Path | str, config: dict[str, Any]) -> None:
)
+def config_fingerprint(repo_path: Path | str) -> str:
+ """SHA-256 hex of ``.repowise/config.yaml`` + ``health-rules.json`` content.
+
+ Used by ``repowise update`` and the index writers (CLI init, server jobs)
+ to detect config changes across runs without relying on filesystem
+ timestamps. Missing files are skipped, so an absent config still yields a
+ stable hash.
+ """
+ import hashlib
+
+ rw_dir = get_repowise_dir(repo_path)
+ h = hashlib.sha256()
+ for name in ("config.yaml", "health-rules.json"):
+ p = rw_dir / name
+ if p.exists():
+ h.update(name.encode())
+ h.update(p.read_bytes())
+ return h.hexdigest()
+
+
def _read_flat_scalar(text: str, key: str) -> str | None:
"""Read a top-level scalar from config text before YAML bool coercion."""
for line in text.splitlines():
diff --git a/packages/server/src/repowise/server/app.py b/packages/server/src/repowise/server/app.py
index 061d920b5..a5b5cd7d8 100644
--- a/packages/server/src/repowise/server/app.py
+++ b/packages/server/src/repowise/server/app.py
@@ -196,9 +196,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# Store on app state (before scheduler, so scheduler can reference app_state)
app.state.engine = engine
app.state.session_factory = session_factory
+ app.state.db_url = db_url
app.state.fts = fts
app.state.vector_store = vector_store
app.state.background_tasks: set = set() # Strong refs to prevent GC of asyncio tasks
+ app.state.job_tasks = {} # job_id → asyncio.Task (cancel endpoint)
+ app.state.job_cancel_tokens = {} # job_id → CancellationToken
+ app.state.job_events = {} # job_id → JobEventBuffer (SSE message frames)
# Background scheduler (pass app.state so polling can launch jobs)
scheduler = setup_scheduler(session_factory, app_state=app.state)
@@ -345,6 +349,24 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.debug("Workspace detection skipped", exc_info=True)
+ # Re-register per-repo databases for repos added via the API (their data
+ # lives in <repo>/.repowise/wiki.db; the primary DB only holds a registry
+ # row). Runs after workspace detection so already-registered workspace
+ # repos are skipped.
+ try:
+ from repowise.server.repo_db import rediscover_repo_dbs
+
+ rediscovered = await rediscover_repo_dbs(app.state)
+ if rediscovered:
+ logger.info("repo_dbs_rediscovered", extra={"count": rediscovered})
+ # Jobs interrupted by the restart live in those per-repo DBs; the
+ # earlier resets only covered the primary and workspace DBs.
+ stale = await reset_workspace_stale_jobs(app.state)
+ if stale:
+ logger.warning("reset_stale_jobs", extra={"count": stale})
+ except Exception:
+ logger.debug("repo_db_rediscovery_skipped", exc_info=True)
+
logger.info("repowise_server_started", extra={"version": __version__})
yield
diff --git a/packages/server/src/repowise/server/job_events.py b/packages/server/src/repowise/server/job_events.py
new file mode 100644
index 000000000..8f36cd219
--- /dev/null
+++ b/packages/server/src/repowise/server/job_events.py
@@ -0,0 +1,75 @@
+"""In-memory per-job progress event buffers.
+
+The pipeline's ``on_message`` / ``on_phase_start`` callbacks carry the human
+narrative of a run ("Parsing files", "3 files skipped: encoding error") that
+the job row's page counters cannot express. Each running job gets a bounded
+ring buffer here; the SSE stream drains it incrementally and forwards every
+entry to the client alongside the polled progress numbers.
+
+Deliberately in-memory: jobs execute in this same process, the stream is a
+live view, and a bounded deque avoids both a schema migration and per-message
+database writes that would contend with the pipeline's bulk transactions. A
+server restart loses the transcript but also fails the job, so nothing
+consumers rely on outlives the buffer.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import deque
+from dataclasses import dataclass, field
+
+# Ring size per job: enough for the full narrative of a large run's phase
+# starts + warnings without letting a pathological logger grow unbounded.
+_MAX_EVENTS_PER_JOB = 500
+
+# Buffers kept after job completion so late SSE subscribers still see the
+# tail; evicted oldest-first beyond this many jobs.
+_MAX_JOBS = 50
+
+
+@dataclass
+class JobEventBuffer:
+ """Bounded event log + current phase label for one job."""
+
+ events: deque = field(default_factory=lambda: deque(maxlen=_MAX_EVENTS_PER_JOB))
+ next_seq: int = 0
+ phase: str = ""
+
+ def add(self, level: str, text: str) -> None:
+ self.events.append(
+ {
+ "seq": self.next_seq,
+ "ts": time.time(),
+ "level": level,
+ "text": text,
+ "phase": self.phase,
+ }
+ )
+ self.next_seq += 1
+
+ def set_phase(self, phase: str, total: int | None) -> None:
+ self.phase = phase
+ self.add("info", f"Phase started: {phase}" + (f" ({total} items)" if total else ""))
+
+ def since(self, seq: int) -> list[dict]:
+ """Return events with ``seq`` >= *seq* (drained ones are simply gone)."""
+ return [e for e in self.events if e["seq"] >= seq]
+
+
+def get_event_buffers(app_state) -> dict[str, JobEventBuffer]:
+ buffers = getattr(app_state, "job_events", None)
+ if buffers is None:
+ buffers = {}
+ app_state.job_events = buffers
+ return buffers
+
+
+def create_event_buffer(app_state, job_id: str) -> JobEventBuffer:
+ """Create (or return) the buffer for *job_id*, evicting the oldest jobs."""
+ buffers = get_event_buffers(app_state)
+ if job_id not in buffers:
+ while len(buffers) >= _MAX_JOBS:
+ buffers.pop(next(iter(buffers)))
+ buffers[job_id] = JobEventBuffer()
+ return buffers[job_id]
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index d3252e456..bb048f9bd 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -16,6 +16,12 @@ from typing import Any
import structlog
+from repowise.core.cancellation import (
+ CancellationToken,
+ PipelineCancelled,
+ get_active_token,
+ set_active_token,
+)
from repowise.core.persistence.crud import (
get_generation_job,
get_repository,
@@ -23,6 +29,7 @@ from repowise.core.persistence.crud import (
)
from repowise.core.persistence.database import get_session
from repowise.core.pipeline import persist_pipeline_result, run_pipeline
+from repowise.server.job_events import JobEventBuffer, create_event_buffer
logger = structlog.get_logger(__name__)
@@ -119,9 +126,15 @@ class JobProgressCallback:
is sufficient to push live progress to the frontend.
"""
- def __init__(self, job_id: str, session_factory: Any) -> None:
+ def __init__(
+ self,
+ job_id: str,
+ session_factory: Any,
+ events: JobEventBuffer | None = None,
+ ) -> None:
self._job_id = job_id
self._session_factory = session_factory
+ self._events = events
self._completed = 0
self._total: int | None = None
self._phase = ""
@@ -142,6 +155,8 @@ class JobProgressCallback:
def on_phase_start(self, phase: str, total: int | None) -> None:
self._phase = phase
+ if self._events is not None:
+ self._events.set_phase(phase, total)
# Reset per-phase counters so the bar shows progress within the current phase
self._completed = 0
self._total = total
@@ -159,6 +174,8 @@ class JobProgressCallback:
self._sync_job_status()
def on_message(self, level: str, text: str) -> None:
+ if self._events is not None:
+ self._events.add(level, text)
getattr(logger, level, logger.info)(text, job_id=self._job_id, phase=self._phase)
def _sync_job_status(self, *, force: bool = False) -> None:
@@ -236,6 +253,15 @@ class JobProgressCallback:
logger.debug("progress_update_failed", job_id=self._job_id, exc_info=True)
+def get_cancel_tokens(app_state: Any) -> dict[str, CancellationToken]:
+ """Per-job cancellation tokens, keyed by job id (created on demand)."""
+ tokens = getattr(app_state, "job_cancel_tokens", None)
+ if tokens is None:
+ tokens = {}
+ app_state.job_cancel_tokens = tokens
+ return tokens
+
+
async def execute_job(
job_id: str,
app_state: Any,
@@ -250,7 +276,15 @@ async def execute_job(
2. Resolves the LLM provider from server config
3. Runs ``run_pipeline()``
4. Persists all results via ``persist_pipeline_result()``
- 5. Marks the job as ``completed`` (or ``failed`` on error)
+ 5. Marks the job as ``completed`` (or ``failed`` on error,
+ ``cancelled`` on a user cancel)
+
+ Job modes (``config_json.mode``): ``sync`` (default) indexes then
+ regenerates only changed pages; ``full_resync`` regenerates all docs;
+ ``initial_index`` is the first-ever index of a repo triggered from the
+ API — full pipeline with docs plus the ``state.json``/``config.yaml``
+ baseline the CLI writes at ``repowise init``; ``index_only`` refreshes
+ the index/analysis without any LLM work.
In workspace mode, each repo has its own ``wiki.db`` and the route
handler that created this job committed it to a per-repo session
@@ -263,6 +297,18 @@ async def execute_job(
progress: JobProgressCallback | None = None
session_factory = None
+ # Cooperative cancellation: the cancel endpoint flips this token (which
+ # unwinds the CPU-bound loops that poll check_cancelled) and cancels the
+ # asyncio task (which interrupts the awaits in between). The core token
+ # slot is a process global, so when two jobs overlap the later one's token
+ # occupies it and the earlier job's CPU loops poll the wrong token — its
+ # async awaits still cancel, only an in-flight to_thread worker runs on.
+ # Acceptable for the single-active-job norm; worker isolation is the
+ # upgrade path.
+ cancel_token = CancellationToken()
+ get_cancel_tokens(app_state)[job_id] = cancel_token
+ set_active_token(cancel_token)
+
try:
# Resolve required app_state attributes inside the try block so a
# missing attribute (e.g., partially-initialised app_state during
@@ -295,37 +341,56 @@ async def execute_job(
# Resolve the wiki style while ``repo`` is still session-attached.
wiki_style = _repo_wiki_style(repo, repo_path)
config = json.loads(job.config_json) if job.config_json else {}
- is_full_resync = config.get("mode") == "full_resync"
+ mode = config.get("mode") or "sync"
+ is_full_resync = mode == "full_resync"
+ is_initial_index = mode == "initial_index"
+ is_index_only = mode == "index_only"
# Mark running
await update_job_status(session, job_id, "running")
- logger.info(
- "job_started",
- job_id=job_id,
- repo_path=repo_path,
- mode="full_resync" if is_full_resync else "sync",
- )
+ logger.info("job_started", job_id=job_id, repo_path=repo_path, mode=mode)
# ---- Resolve LLM provider -----------------------------------------
llm_client = None
- try:
- from repowise.server.provider_config import get_chat_provider_instance
+ docs_skip_reason: str | None = None
+ if not is_index_only:
+ try:
+ from repowise.server.provider_config import get_chat_provider_instance
- # Pass the repo path so the job reuses the provider/model/key the
- # repo was configured with (``.repowise/config.yaml`` + ``.env``)
- # rather than the server-global default.
- llm_client = get_chat_provider_instance(repo_path=repo_path)
- except Exception as exc:
- logger.warning("no_provider_configured", error=str(exc))
- # Continue without LLM — ingestion + analysis still work
+ # Pass the repo path so the job reuses the provider/model/key the
+ # repo was configured with (``.repowise/config.yaml`` + ``.env``)
+ # rather than the server-global default.
+ llm_client = get_chat_provider_instance(repo_path=repo_path)
+ except Exception as exc:
+ docs_skip_reason = f"no provider configured: {exc}"
+ logger.warning("no_provider_configured", error=str(exc))
+ # Continue without LLM — ingestion + analysis still work
# ---- Run pipeline --------------------------------------------------
- progress = JobProgressCallback(job_id, session_factory)
+ events = create_event_buffer(app_state, job_id)
+ progress = JobProgressCallback(job_id, session_factory, events)
+
+ generate_docs = (
+ (is_full_resync or is_initial_index)
+ and llm_client is not None
+ and bool(config.get("generate_docs", True))
+ )
+ if is_initial_index:
+ # First index of an API-registered repo: make sure the repo-local
+ # data directory exists before the pipeline writes artifacts.
+ (Path(repo_path) / ".repowise").mkdir(parents=True, exist_ok=True)
+ if llm_client is None and config.get("generate_docs", True):
+ progress.on_message(
+ "warning",
+ "No LLM provider configured; indexing without documentation "
+ "generation. Configure a provider and run a full resync to "
+ "generate docs.",
+ )
result = await run_pipeline(
Path(repo_path),
- generate_docs=is_full_resync and llm_client is not None,
+ generate_docs=generate_docs,
llm_client=llm_client,
vector_store=vector_store,
progress=progress,
@@ -338,7 +403,7 @@ async def execute_job(
# then regenerates only the wiki pages affected by recent changes.
# This keeps docs fresh without the cost of a full re-index.
incremental_pages: list = []
- if not is_full_resync and llm_client is not None:
+ if mode == "sync" and llm_client is not None:
incremental_pages = await _incremental_page_regen(
Path(repo_path),
result,
@@ -414,26 +479,22 @@ async def execute_job(
total_pages=pages_generated if pages_generated else result.file_count,
)
- # Update state.json so CLI incremental updates know the new baseline
+ # Update state.json so CLI incremental updates know the new baseline.
+ # An initial index also persists the full baseline (docs flags, run
+ # mode, config) that `repowise init` would have written.
try:
- _state_path = Path(repo_path) / ".repowise" / "state.json"
- import subprocess as _sp
-
- _head_result = _sp.run(
- ["git", "rev-parse", "HEAD"],
- cwd=repo_path,
- capture_output=True,
- text=True,
- timeout=10,
- )
- if _head_result.returncode == 0:
- _head_sha = _head_result.stdout.strip()
- _state_data: dict = {}
- if _state_path.is_file():
- _state_data = json.loads(_state_path.read_text(encoding="utf-8"))
- _state_data["last_sync_commit"] = _head_sha
- _state_path.parent.mkdir(parents=True, exist_ok=True)
- _state_path.write_text(json.dumps(_state_data, indent=2), encoding="utf-8")
+ if is_initial_index:
+ _persist_initial_index_state(
+ Path(repo_path),
+ llm_client=llm_client,
+ docs_enabled=generate_docs,
+ docs_skip_reason=docs_skip_reason,
+ total_pages=len(all_pages),
+ wiki_style=wiki_style,
+ exclude_patterns=exclude_patterns,
+ )
+ else:
+ _stamp_last_sync_commit(Path(repo_path))
except Exception:
logger.debug("state_json_update_failed", job_id=job_id, exc_info=True)
@@ -454,31 +515,191 @@ async def execute_job(
pages=pages_generated,
)
+ except (PipelineCancelled, asyncio.CancelledError):
+ # User-requested cancel: the endpoint flipped our token and/or
+ # cancelled the task. Record the terminal state and swallow — this is
+ # the top of a background task, nothing above us awaits the result.
+ logger.info("job_cancelled", job_id=job_id)
+ await _finalize_job_status(
+ app_state,
+ session_factory,
+ progress,
+ job_id,
+ status="cancelled",
+ error_message="Cancelled by user",
+ )
except Exception as exc:
logger.exception("job_failed", job_id=job_id, error=str(exc))
- # Drain progress updates before writing final "failed" status to prevent
- # a late fire-and-forget progress update from overwriting it with "running".
- if progress is not None:
- try:
- await progress.drain_and_stop()
- except Exception:
- logger.debug("drain_failed_on_error_path", job_id=job_id, exc_info=True)
- # If we failed before resolving session_factory, fall back to the one
- # on app_state (best-effort) so the row never stays stuck in pending.
- recovery_factory = session_factory or getattr(app_state, "session_factory", None)
- if recovery_factory is not None:
- try:
- async with get_session(recovery_factory) as session:
- await update_job_status(
- session,
- job_id,
- "failed",
- error_message=str(exc)[:500],
- )
- except Exception:
- logger.exception("job_status_update_failed", job_id=job_id)
- else:
- logger.error("job_status_update_skipped_no_session", job_id=job_id)
+ await _finalize_job_status(
+ app_state,
+ session_factory,
+ progress,
+ job_id,
+ status="failed",
+ error_message=str(exc)[:500],
+ )
+ finally:
+ get_cancel_tokens(app_state).pop(job_id, None)
+ # Disarm only if the global slot still holds our token; a later job
+ # may have replaced it, and its token must not be clobbered. Reset to
+ # None rather than the captured previous token — that one may belong
+ # to a job that already finished (possibly cancelled), and re-arming
+ # it would poison the next check_cancelled() poll.
+ if get_active_token() is cancel_token:
+ set_active_token(None)
+
+
+async def _finalize_job_status(
+ app_state: Any,
+ session_factory: Any,
+ progress: JobProgressCallback | None,
+ job_id: str,
+ *,
+ status: str,
+ error_message: str,
+) -> None:
+ """Best-effort terminal status write shared by the failure and cancel paths.
+
+ Drains in-flight progress updates first so a late fire-and-forget write
+ can't overwrite the terminal status with "running". Falls back to the
+ app-level session factory when the job's own factory was never resolved,
+ so the row never stays stuck in pending.
+ """
+ if progress is not None:
+ try:
+ await progress.drain_and_stop()
+ except Exception:
+ logger.debug("drain_failed_on_error_path", job_id=job_id, exc_info=True)
+ recovery_factory = session_factory or getattr(app_state, "session_factory", None)
+ if recovery_factory is None:
+ logger.error("job_status_update_skipped_no_session", job_id=job_id)
+ return
+ try:
+ async with get_session(recovery_factory) as session:
+ await update_job_status(
+ session,
+ job_id,
+ status,
+ error_message=error_message,
+ )
+ except Exception:
+ logger.exception("job_status_update_failed", job_id=job_id)
+
+
+# ---------------------------------------------------------------------------
+# state.json / config.yaml writers
+# ---------------------------------------------------------------------------
+
+
+def _read_head_sha(repo_path: Path) -> str | None:
+ import subprocess as _sp
+
+ try:
+ result = _sp.run(
+ ["git", "rev-parse", "HEAD"],
+ cwd=str(repo_path),
+ capture_output=True,
+ text=True,
+ timeout=10,
+ )
+ except Exception:
+ return None
+ return result.stdout.strip() if result.returncode == 0 else None
+
+
+def _load_state(repo_path: Path) -> dict:
+ state_path = repo_path / ".repowise" / "state.json"
+ if state_path.is_file():
+ return json.loads(state_path.read_text(encoding="utf-8"))
+ return {}
+
+
+def _save_state(repo_path: Path, state: dict) -> None:
+ state_path = repo_path / ".repowise" / "state.json"
+ state_path.parent.mkdir(parents=True, exist_ok=True)
+ state_path.write_text(json.dumps(state, indent=2), encoding="utf-8")
+
+
+def _stamp_last_sync_commit(repo_path: Path) -> None:
+ """Record the synced HEAD so CLI incremental updates know the baseline."""
+ head = _read_head_sha(repo_path)
+ if not head:
+ return
+ state = _load_state(repo_path)
+ state["last_sync_commit"] = head
+ _save_state(repo_path, state)
+
+
+def _persist_initial_index_state(
+ repo_path: Path,
+ *,
+ llm_client: Any,
+ docs_enabled: bool,
+ docs_skip_reason: str | None,
+ total_pages: int,
+ wiki_style: str,
+ exclude_patterns: list[str],
+) -> None:
+ """Write the ``repowise init`` baseline after a first API-driven index.
+
+ Mirrors what the CLI persists at the end of ``init``: a complete
+ ``state.json`` (sync baseline, docs flags, run mode, store-format stamp,
+ config fingerprint) and a ``config.yaml`` recording the provider/model
+ and style the run used, so later CLI ``update`` runs and server jobs
+ resolve the same configuration.
+ """
+ from repowise.core.generation.styles import DEFAULT_STYLE
+ from repowise.core.repo_config import (
+ config_fingerprint,
+ load_repo_config,
+ save_repo_config,
+ )
+
+ # ---- config.yaml (written first: the fingerprint below covers it) ----
+ config = load_repo_config(repo_path)
+ if llm_client is not None:
+ config["provider"] = getattr(llm_client, "provider_name", "") or config.get("provider")
+ config["model"] = getattr(llm_client, "model_name", "") or config.get("model")
+ if wiki_style and wiki_style != DEFAULT_STYLE and not config.get("wiki_style"):
+ config["wiki_style"] = wiki_style
+ if exclude_patterns and not config.get("exclude_patterns"):
+ config["exclude_patterns"] = exclude_patterns
+ try:
+ save_repo_config(repo_path, config)
+ except Exception:
+ logger.debug("config_yaml_write_failed", repo_path=str(repo_path), exc_info=True)
+
+ # ---- state.json ----
+ state = _load_state(repo_path)
+ head = _read_head_sha(repo_path)
+ if head:
+ state["last_sync_commit"] = head
+ state["docs_enabled"] = docs_enabled
+ if docs_skip_reason and not docs_enabled:
+ state["docs_skip_reason"] = docs_skip_reason
+ state["run_mode"] = "standard"
+ state["git_tier"] = "full"
+ state["include_submodules"] = False
+ state["total_pages"] = total_pages
+ if llm_client is not None:
+ state["provider"] = getattr(llm_client, "provider_name", "")
+ state["model"] = getattr(llm_client, "model_name", "")
+
+ try:
+ from importlib.metadata import version as _dist_version
+
+ from repowise.core.upgrade import stamp as _stamp_store_version
+
+ try:
+ _pkg_version: str | None = _dist_version("repowise")
+ except Exception:
+ _pkg_version = None
+ _stamp_store_version(state, package_version=_pkg_version)
+ except Exception:
+ logger.debug("store_version_stamp_failed", repo_path=str(repo_path), exc_info=True)
+
+ state["config_fingerprint"] = config_fingerprint(repo_path)
+ _save_state(repo_path, state)
# ---------------------------------------------------------------------------
diff --git a/packages/server/src/repowise/server/repo_db.py b/packages/server/src/repowise/server/repo_db.py
new file mode 100644
index 000000000..dfc507594
--- /dev/null
+++ b/packages/server/src/repowise/server/repo_db.py
@@ -0,0 +1,277 @@
+"""Per-repo database registration for API-managed repositories.
+
+A repository's canonical store is ``<repo>/.repowise/wiki.db`` — the same
+file the CLI creates and a later ``repowise serve`` run from inside that repo
+expects to find. Before this module, a repo registered via ``POST /api/repos``
+while the server was pointed at some other database got all of its pipeline
+output (pages, graph, jobs) written into that ambient database instead:
+invisible to the CLI and polluting whatever DB the server happened to open.
+
+The fix mirrors what workspace mode already does at startup: open a dedicated
+engine/session factory on the repo's own ``wiki.db`` and register it in
+``app_state.workspace_sessions`` so :func:`repowise.server.deps.resolve_session_factory`
+routes every ``repo_id``-scoped request (and the job executor) to the right
+file. The map predates non-workspace registration, hence its name; it is
+simply "repo_id to its own database" and works identically in both modes.
+
+The server's primary database keeps a lightweight registry row (same repo id)
+so repos can be listed and re-registered across restarts.
+"""
+
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+from sqlalchemy import select
+
+from repowise.core.persistence.crud import upsert_repository
+from repowise.core.persistence.database import (
+ create_engine,
+ create_session_factory,
+ get_session,
+ init_db,
+)
+from repowise.core.persistence.models import Repository
+from repowise.core.persistence.search import FullTextSearch
+
+logger = logging.getLogger(__name__)
+
+
+def repo_db_file(local_path: str | Path) -> Path:
+ """Return the repo-local database file path (may not exist yet)."""
+ return Path(local_path).resolve() / ".repowise" / "wiki.db"
+
+
+def is_primary_db_for(app_state, local_path: str | Path) -> bool:
+ """True when the server's primary database *is* this repo's wiki.db.
+
+ In that case no per-repo engine is needed — the ambient session factory
+ already reads and writes the canonical file.
+ """
+ db_url: str = getattr(app_state, "db_url", "") or ""
+ return repo_db_file(local_path).as_posix() in db_url.replace("\\", "/")
+
+
+def _state_dict(app_state, name: str) -> dict:
+ """Fetch-or-create a dict attribute on app.state (test apps skip lifespan)."""
+ existing = getattr(app_state, name, None)
+ if existing is None:
+ existing = {}
+ setattr(app_state, name, existing)
+ return existing
+
+
+def _state_list(app_state, name: str) -> list:
+ existing = getattr(app_state, name, None)
+ if existing is None:
+ existing = []
+ setattr(app_state, name, existing)
+ return existing
+
+
+async def ensure_repo_registration(
+ app_state,
+ *,
+ local_path: str,
+ name: str,
+ url: str = "",
+ default_branch: str = "main",
+ settings: dict | None = None,
+ repo_id: str | None = None,
+):
+ """Open (creating if needed) a repo's own wiki.db and register it for routing.
+
+ Ensures the ``Repository`` row exists in the repo-local database — created
+ with ``repo_id`` when given, so the primary-DB registry row and the
+ canonical row share one id — and registers the session factory plus a
+ per-repo FTS instance on ``app_state``. Idempotent: an already-registered
+ repo returns its existing factory.
+
+ Returns ``(session_factory, repo_id)`` for the canonical row.
+ """
+ resolved = str(Path(local_path).resolve())
+
+ if is_primary_db_for(app_state, resolved):
+ # Server is already serving this repo's own DB as its primary.
+ async with get_session(app_state.session_factory) as session:
+ rid = await _ensure_repo_row(
+ session,
+ name=name,
+ local_path=resolved,
+ url=url,
+ default_branch=default_branch,
+ settings=settings,
+ repo_id=repo_id,
+ )
+ return app_state.session_factory, rid
+
+ sessions = _state_dict(app_state, "workspace_sessions")
+ fts_map = _state_dict(app_state, "workspace_fts")
+ engines = _state_list(app_state, "workspace_engines")
+
+ db_file = repo_db_file(resolved)
+ db_file.parent.mkdir(parents=True, exist_ok=True)
+
+ engine = create_engine(f"sqlite+aiosqlite:///{db_file.as_posix()}")
+ await init_db(engine)
+ factory = create_session_factory(engine)
+
+ async with get_session(factory) as session:
+ rid = await _ensure_repo_row(
+ session,
+ name=name,
+ local_path=resolved,
+ url=url,
+ default_branch=default_branch,
+ settings=settings,
+ repo_id=repo_id,
+ )
+
+ if rid in sessions:
+ # Registered earlier in this process — keep the original engine and
+ # drop the redundant one we just opened.
+ await engine.dispose()
+ return sessions[rid], rid
+
+ sessions[rid] = factory
+ engines.append(engine)
+ try:
+ repo_fts = FullTextSearch(engine)
+ await repo_fts.ensure_index()
+ fts_map[rid] = repo_fts
+ except Exception:
+ logger.debug("repo_fts_init_failed", extra={"repo_id": rid}, exc_info=True)
+
+ logger.info("repo_db_registered", extra={"repo_id": rid, "db": str(db_file)})
+ return factory, rid
+
+
+async def _ensure_repo_row(
+ session,
+ *,
+ name: str,
+ local_path: str,
+ url: str,
+ default_branch: str,
+ settings: dict | None,
+ repo_id: str | None,
+) -> str:
+ """Create the canonical repo row if missing; never clobber an existing one.
+
+ Startup rediscovery and job launches route through here, so an existing
+ row's name/url/branch must survive untouched — only ``POST /api/repos``
+ (which updates fields explicitly) may overwrite them.
+ """
+ from repowise.core.persistence.crud import get_repository_by_path
+
+ repo = await get_repository_by_path(session, local_path)
+ if repo is None:
+ repo = await upsert_repository(
+ session,
+ name=name,
+ local_path=local_path,
+ url=url,
+ default_branch=default_branch,
+ settings=settings,
+ repo_id=repo_id,
+ )
+ return repo.id
+
+
+async def upsert_registry_row(
+ session,
+ *,
+ repo_id: str,
+ name: str,
+ local_path: str,
+ url: str = "",
+ default_branch: str = "main",
+ settings: dict | None = None,
+) -> None:
+ """Mirror the canonical repo row into the server's primary database.
+
+ The registry row is what makes an externally-stored repo listable and,
+ on the next server start, re-registerable. If a row for the same path
+ exists under a different id (e.g. registered before the repo was ever
+ indexed by the CLI), it is replaced so routing stays keyed by one id.
+ """
+ import json as _json
+
+ result = await session.execute(select(Repository).where(Repository.local_path == local_path))
+ existing = result.scalar_one_or_none()
+ if existing is not None and existing.id != repo_id:
+ await session.delete(existing)
+ await session.flush()
+ existing = None
+
+ if existing is None:
+ await upsert_repository(
+ session,
+ name=name,
+ local_path=local_path,
+ url=url,
+ default_branch=default_branch,
+ settings=settings,
+ repo_id=repo_id,
+ )
+ else:
+ existing.name = name
+ existing.url = url
+ existing.default_branch = default_branch
+ if settings is not None:
+ existing.settings_json = _json.dumps(settings)
+ await session.flush()
+
+
+async def rediscover_repo_dbs(app_state) -> int:
+ """Re-register per-repo databases recorded in the primary DB's registry.
+
+ Called at startup after workspace detection. For every repository row in
+ the primary database whose ``local_path`` carries its own ``wiki.db``
+ (and is not the primary DB itself, nor already registered by the
+ workspace loop), open an engine and register it for routing.
+ """
+ import sqlite3
+
+ count = 0
+ sessions = _state_dict(app_state, "workspace_sessions")
+
+ async with get_session(app_state.session_factory) as session:
+ result = await session.execute(select(Repository.id, Repository.local_path))
+ rows = result.all()
+
+ for registry_id, local_path in rows:
+ if not local_path or registry_id in sessions:
+ continue
+ db_file = repo_db_file(local_path)
+ if not db_file.is_file() or is_primary_db_for(app_state, local_path):
+ continue
+ # Read the canonical id without opening a full async engine first.
+ try:
+ with sqlite3.connect(str(db_file)) as conn:
+ row = conn.execute(
+ "SELECT id, name FROM repositories WHERE local_path = ? LIMIT 1",
+ (str(Path(local_path).resolve()),),
+ ).fetchone()
+ if row is None:
+ row = conn.execute("SELECT id, name FROM repositories LIMIT 1").fetchone()
+ except Exception:
+ continue
+ canonical_id = row[0] if row else None
+ if canonical_id and canonical_id != registry_id:
+ logger.warning(
+ "repo_db_id_mismatch",
+ extra={"registry_id": registry_id, "canonical_id": canonical_id},
+ )
+ try:
+ await ensure_repo_registration(
+ app_state,
+ local_path=local_path,
+ name=(row[1] if row else None) or Path(local_path).name,
+ repo_id=registry_id,
+ )
+ count += 1
+ except Exception:
+ logger.debug("repo_db_rediscovery_failed", extra={"path": local_path}, exc_info=True)
+ return count
diff --git a/packages/server/src/repowise/server/routers/dead_code.py b/packages/server/src/repowise/server/routers/dead_code.py
index 32cbe5448..696035815 100644
--- a/packages/server/src/repowise/server/routers/dead_code.py
+++ b/packages/server/src/repowise/server/routers/dead_code.py
@@ -2,7 +2,7 @@
from __future__ import annotations
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
from sqlalchemy.ext.asyncio import AsyncSession
from repowise.core.analysis.dead_code.risk_factors import effective_safe_to_delete
@@ -53,16 +53,45 @@ async def list_dead_code(
@router.post("/api/repos/{repo_id}/dead-code/analyze", status_code=202)
async def analyze_dead_code(
repo_id: str,
+ request: Request,
session: AsyncSession = Depends(get_db_session), # noqa: B008
) -> dict:
"""Trigger a fresh dead code analysis.
- Returns 202 immediately. The analysis runs asynchronously.
+ Runs an index-only pipeline job (no LLM work) — dead-code detection is
+ part of the analysis stage, so a re-index refreshes the findings. Returns
+ 202 with the job id; poll or stream it like any other job.
"""
+ from sqlalchemy import select
+
+ from repowise.core.persistence.models import GenerationJob
+ from repowise.server.routers.repos import _launch_job_task
+
repo = await crud.get_repository(session, repo_id)
if repo is None:
raise HTTPException(status_code=404, detail="Repository not found")
- return {"status": "analyzing", "repository_id": repo_id}
+
+ active = await session.execute(
+ select(GenerationJob.id)
+ .where(GenerationJob.repository_id == repo_id)
+ .where(GenerationJob.status.in_(["pending", "running"]))
+ .limit(1)
+ )
+ if active.scalar_one_or_none() is not None:
+ raise HTTPException(
+ status_code=409, detail="A job is already in progress for this repository"
+ )
+
+ job = await crud.upsert_generation_job(
+ session,
+ repository_id=repo_id,
+ status="pending",
+ config={"mode": "index_only", "source": "dead_code_analyze"},
+ )
+ # Commit so the background task's separate session can see the row.
+ await session.commit()
+ _launch_job_task(request, job.id, repo_id)
+ return {"job_id": job.id, "status": "accepted", "repository_id": repo_id}
@router.get(
diff --git a/packages/server/src/repowise/server/routers/jobs.py b/packages/server/src/repowise/server/routers/jobs.py
index 98f99c675..1b496588d 100644
--- a/packages/server/src/repowise/server/routers/jobs.py
+++ b/packages/server/src/repowise/server/routers/jobs.py
@@ -120,13 +120,15 @@ async def get_job(job_id: str, request: Request) -> JobResponse:
async def cancel_job(job_id: str, request: Request) -> JobResponse:
"""Cancel a pending or running generation job.
- Marks the job as ``failed`` with a cancellation message. The background
- task itself is not interrupted — but in practice this unblocks the
- active-job guard in /repos/{id}/sync so the user can start a new sync
- immediately. Useful when a job is stuck in ``pending`` because the
- background task crashed before it could record a failure.
+ Actually stops the work: flips the job's cooperative cancellation token
+ (unwinding the CPU-bound pipeline phases) and cancels the background
+ asyncio task (interrupting in-flight awaits, including LLM calls), then
+ marks the job ``cancelled``. Also unblocks the active-job guard in
+ /repos/{id}/sync when a job is stuck in ``pending`` because its
+ background task never started.
"""
- factory, job = await _find_job_factory(request.app.state, job_id)
+ app_state = request.app.state
+ factory, job = await _find_job_factory(app_state, job_id)
if job is None or factory is None:
raise HTTPException(status_code=404, detail="Job not found")
if job.status not in ("pending", "running"):
@@ -134,11 +136,22 @@ async def cancel_job(job_id: str, request: Request) -> JobResponse:
status_code=409,
detail=f"Cannot cancel a job in '{job.status}' state",
)
+
+ # Signal the running pipeline first, then record the terminal state. The
+ # executor also writes "cancelled" when it unwinds; both writes are
+ # idempotent.
+ token = getattr(app_state, "job_cancel_tokens", {}).get(job_id)
+ if token is not None:
+ token.cancel()
+ task = getattr(app_state, "job_tasks", {}).get(job_id)
+ if task is not None and not task.done():
+ task.cancel()
+
async with get_session(factory) as session:
await crud.update_job_status(
session,
job_id,
- "failed",
+ "cancelled",
error_message="Cancelled by user",
)
job = await crud.get_generation_job(session, job_id)
@@ -150,8 +163,12 @@ async def cancel_job(job_id: str, request: Request) -> JobResponse:
async def stream_job(job_id: str, request: Request) -> StreamingResponse:
"""SSE progress stream for a generation job.
- Emits ``event: progress`` every second until the job completes or fails,
- then emits ``event: done`` and closes.
+ Emits ``event: progress`` every second until the job reaches a terminal
+ state, then emits ``event: done`` and closes. Pipeline messages recorded
+ by the executor (phase starts, per-file warnings) are interleaved as
+ ``event: message`` frames, and each progress frame carries the current
+ ``phase`` label, so clients can render "Parsing files 1200/4000" instead
+ of bare numbers.
"""
factory, _ = await _find_job_factory(request.app.state, job_id)
if factory is None:
@@ -160,6 +177,7 @@ async def stream_job(job_id: str, request: Request) -> StreamingResponse:
factory = request.app.state.session_factory
async def event_generator():
+ next_seq = 0
while True:
# Check if client disconnected
if await request.is_disconnected():
@@ -183,6 +201,14 @@ async def stream_job(job_id: str, request: Request) -> StreamingResponse:
async with get_session(factory) as cost_session:
actual_cost_usd = await cost_session.scalar(cost_q)
+ # Drain any new pipeline messages from the in-memory buffer.
+ buffer = getattr(request.app.state, "job_events", {}).get(job_id)
+ phase = buffer.phase if buffer is not None else ""
+ if buffer is not None:
+ for event in buffer.since(next_seq):
+ next_seq = event["seq"] + 1
+ yield f"event: message\ndata: {json.dumps(event)}\n\n"
+
progress = {
"job_id": job.id,
"status": job.status,
@@ -190,12 +216,14 @@ async def stream_job(job_id: str, request: Request) -> StreamingResponse:
"total_pages": job.total_pages,
"failed_pages": job.failed_pages,
"current_level": job.current_level,
+ "phase": phase,
"actual_cost_usd": actual_cost_usd,
+ "error_message": job.error_message,
}
data = json.dumps(progress)
yield f"event: progress\ndata: {data}\n\n"
- if job.status in ("completed", "failed"):
+ if job.status in ("completed", "failed", "cancelled"):
yield f"event: done\ndata: {data}\n\n"
return
diff --git a/packages/server/src/repowise/server/routers/repos.py b/packages/server/src/repowise/server/routers/repos.py
index e57729281..b4edd0205 100644
--- a/packages/server/src/repowise/server/routers/repos.py
+++ b/packages/server/src/repowise/server/routers/repos.py
@@ -38,18 +38,104 @@ router = APIRouter(
@router.post("", response_model=RepoResponse, status_code=201)
async def create_repo(
body: RepoCreate,
+ request: Request,
session: AsyncSession = Depends(get_db_session), # noqa: B008
) -> RepoResponse:
- """Register a new repository (or update if same local_path exists)."""
- repo = await crud.upsert_repository(
- session,
- name=body.name,
+ """Register a new repository (or update if same local_path exists).
+
+ The repository's data lives in its own ``<repo>/.repowise/wiki.db`` (the
+ same store the CLI uses); the server's primary database only keeps a
+ registry row so the repo stays listed across restarts. With ``index``
+ (the default) the first full index — docs included, when a provider is
+ configured — is enqueued immediately; the created job's id is returned
+ as ``initial_job_id`` so clients can attach to its progress stream.
+ """
+ if not body.index:
+ # Metadata-only registration (kept for API compatibility and tests):
+ # the row lands in the ambient DB; per-repo storage is established
+ # when the repo is first indexed (here with index=true, or later via
+ # POST /api/repos/{id}/index).
+ repo = await crud.upsert_repository(
+ session,
+ name=body.name,
+ local_path=body.local_path,
+ url=body.url,
+ default_branch=body.default_branch,
+ settings=body.settings,
+ )
+ return RepoResponse.from_orm(repo)
+
+ from repowise.server.repo_db import ensure_repo_registration, upsert_registry_row
+
+ app_state = request.app.state
+
+ # Canonical row in the repo-local DB; the factory routes all later access.
+ repo_factory, repo_id = await ensure_repo_registration(
+ app_state,
local_path=body.local_path,
+ name=body.name,
url=body.url,
default_branch=body.default_branch,
settings=body.settings,
)
- return RepoResponse.from_orm(repo)
+
+ # Apply any metadata updates to the canonical row (registration itself
+ # never clobbers an existing row).
+ async with get_session(repo_factory) as repo_session:
+ repo = await crud.get_repository(repo_session, repo_id)
+ if repo is not None:
+ repo.name = body.name
+ repo.url = body.url
+ repo.default_branch = body.default_branch
+ if body.settings is not None:
+ import json as _json
+
+ repo.settings_json = _json.dumps(body.settings)
+ await repo_session.flush()
+ response = RepoResponse.from_orm(repo)
+ else: # pragma: no cover — the row was created two lines above
+ raise HTTPException(status_code=500, detail="Repository registration failed")
+
+ # Registry row in the primary DB (skip when the repo IS the primary DB).
+ if repo_factory is not app_state.session_factory:
+ await upsert_registry_row(
+ session,
+ repo_id=repo_id,
+ name=body.name,
+ local_path=body.local_path,
+ url=body.url,
+ default_branch=body.default_branch,
+ settings=body.settings,
+ )
+ await session.commit()
+
+ response.initial_job_id = await _enqueue_index_job(request, repo_factory, repo_id)
+ return response
+
+
+async def _enqueue_index_job(request: Request, session_factory, repo_id: str) -> str | None:
+ """Create and launch an ``initial_index`` job unless one is already active."""
+ async with get_session(session_factory) as session:
+ active = await session.execute(
+ select(GenerationJob.id)
+ .where(GenerationJob.repository_id == repo_id)
+ .where(GenerationJob.status.in_(["pending", "running"]))
+ .limit(1)
+ )
+ if active.scalar_one_or_none() is not None:
+ return None
+ job = await crud.upsert_generation_job(
+ session,
+ repository_id=repo_id,
+ status="pending",
+ config={"mode": "initial_index"},
+ )
+ # Commit (not just flush) so the background task's separate session
+ # can see the job row.
+ await session.commit()
+ job_id = job.id
+ _launch_job_task(request, job_id, repo_id)
+ return job_id
@router.get("", response_model=list[RepoResponse])
@@ -229,6 +315,7 @@ async def update_repo(
@router.delete("/{repo_id}")
async def delete_repo(
repo_id: str,
+ request: Request,
session: AsyncSession = Depends(get_db_session), # noqa: B008
fts=Depends(get_fts), # noqa: B008
) -> dict:
@@ -240,15 +327,30 @@ async def delete_repo(
# Collect page IDs before CASCADE deletes the Page rows
page_ids = await crud.list_page_ids(session, repo_id)
- # Clean up FTS index (FTS5 virtual table has no FK cascade).
- # fts is always initialized in the lifespan before the server accepts
- # requests, so this guard is purely defensive.
- if fts is not None:
- await fts.delete_many(page_ids)
+ # Clean up FTS index (FTS5 virtual table has no FK cascade). Use the
+ # repo's own FTS instance when it lives in a per-repo database.
+ repo_fts = getattr(request.app.state, "workspace_fts", {}).get(repo_id) or fts
+ if repo_fts is not None:
+ await repo_fts.delete_many(page_ids)
# Delete repository — CASCADE handles all child ORM tables
await crud.delete_repository(session, repo_id)
+ # Drop per-repo routing and the primary-DB registry row, if any, so the
+ # repo neither lingers in listings nor resurrects on the next restart.
+ app_state = request.app.state
+ ws_sessions = getattr(app_state, "workspace_sessions", None) or {}
+ if repo_id in ws_sessions:
+ ws_sessions.pop(repo_id, None)
+ getattr(app_state, "workspace_fts", {}).pop(repo_id, None)
+ try:
+ async with get_session(app_state.session_factory) as primary:
+ registry = await crud.get_repository(primary, repo_id)
+ if registry is not None:
+ await crud.delete_repository(primary, repo_id)
+ except Exception:
+ logger.debug("registry_row_delete_failed", extra={"repo_id": repo_id})
+
return {"ok": True, "deleted_pages": len(page_ids)}
@@ -401,6 +503,133 @@ async def full_resync(
return {"job_id": job.id, "status": "accepted"}
[email protected]("/{repo_id}/index", status_code=202)
+async def index_repo(
+ repo_id: str,
+ request: Request,
+ session: AsyncSession = Depends(get_db_session), # noqa: B008
+) -> dict:
+ """Run the first full index (docs included) for a registered repository.
+
+ Unlike ``/sync`` and ``/full-resync``, this endpoint also establishes the
+ repo-local database and writes the ``repowise init`` baseline
+ (``state.json``, ``config.yaml``), so a repo that was merely registered
+ becomes fully indexed and CLI-compatible. Safe to call on an already
+ indexed repo — it behaves like a full rebuild.
+ """
+ from repowise.server.repo_db import ensure_repo_registration
+
+ repo = await crud.get_repository(session, repo_id)
+ if repo is None:
+ raise HTTPException(status_code=404, detail="Repository not found")
+
+ factory, canonical_id = await ensure_repo_registration(
+ request.app.state,
+ local_path=repo.local_path,
+ name=repo.name,
+ url=repo.url,
+ default_branch=repo.default_branch,
+ repo_id=repo.id,
+ )
+ job_id = await _enqueue_index_job(request, factory, canonical_id)
+ if job_id is None:
+ raise HTTPException(
+ status_code=409, detail="A job is already in progress for this repository"
+ )
+ return {"job_id": job_id, "status": "accepted"}
+
+
[email protected]("/{repo_id}/preflight")
+async def preflight_index(
+ repo_id: str,
+ request: Request,
+ coverage_pct: float = Query(0.20, ge=0.0, le=1.0),
+ session: AsyncSession = Depends(get_db_session), # noqa: B008
+) -> dict:
+ """Pre-index readiness check: provider connectivity + rough cost estimate.
+
+ Mirrors the CLI's pre-generation gate — a live provider smoke test plus a
+ page-count/cost estimate — so the UI can surface the expected spend and a
+ broken API key *before* launching an index job. The estimate is derived
+ from a fast file walk (no parsing), so page counts are approximate; the
+ reported range absorbs the variance.
+ """
+ repo = await crud.get_repository(session, repo_id)
+ if repo is None:
+ raise HTTPException(status_code=404, detail="Repository not found")
+
+ repo_path = repo.local_path
+ from repowise.server.job_executor import _repo_exclude_patterns
+
+ exclude_patterns = _repo_exclude_patterns(repo, repo_path)
+
+ # ---- Provider smoke test (same probe the CLI uses at init) ----
+ provider_ok = False
+ provider_name: str | None = None
+ model_name: str | None = None
+ provider_error: str | None = None
+ llm_client = None
+ try:
+ from repowise.server.provider_config import get_chat_provider_instance
+
+ llm_client = get_chat_provider_instance(repo_path=repo_path)
+ provider_name = getattr(llm_client, "provider_name", None)
+ model_name = getattr(llm_client, "model_name", None)
+ except Exception as exc:
+ provider_error = str(exc)
+
+ if llm_client is not None:
+ try:
+ await llm_client.generate("You are a test.", "Reply with OK.", max_tokens=50)
+ provider_ok = True
+ except Exception as exc:
+ provider_error = str(exc)
+
+ # ---- File count + cost estimate ----
+ def _count_files() -> int:
+ from repowise.core.ingestion import FileTraverser
+
+ traverser = FileTraverser(
+ Path(repo_path),
+ extra_exclude_patterns=exclude_patterns or None,
+ )
+ return sum(1 for _ in traverser.traverse())
+
+ try:
+ file_count = await asyncio.to_thread(_count_files)
+ except Exception:
+ logger.exception("preflight_file_count_failed", extra={"repo_id": repo_id})
+ file_count = 0
+
+ estimate: dict | None = None
+ if provider_name and model_name:
+ from repowise.core.cost_estimator import approximate_generation_plan, estimate_cost
+
+ plans = approximate_generation_plan(file_count, coverage_pct=coverage_pct)
+ est = estimate_cost(plans, provider_name, model_name, repo_path=repo_path)
+ estimate = {
+ "total_pages": est.total_pages,
+ "estimated_cost_usd": round(est.estimated_cost_usd, 4),
+ "cost_low_usd": round(est.cost_range.low, 4) if est.cost_range else None,
+ "cost_high_usd": round(est.cost_range.high, 4) if est.cost_range else None,
+ "estimated_input_tokens": est.estimated_input_tokens,
+ "estimated_output_tokens": est.estimated_output_tokens,
+ "is_calibrated": est.is_calibrated,
+ "coverage_pct": coverage_pct,
+ }
+
+ return {
+ "provider": {
+ "ok": provider_ok,
+ "name": provider_name,
+ "model": model_name,
+ "error": provider_error,
+ },
+ "file_count": file_count,
+ "estimate": estimate,
+ }
+
+
def _resolve_repo_session_factory(app_state, repo_id: str):
"""Backward-compatible alias for :func:`deps.resolve_session_factory`.
@@ -431,7 +660,7 @@ def _launch_job_task(request: Request, job_id: str, repo_id: str) -> None:
app_state = request.app.state
session_factory = _resolve_repo_session_factory(app_state, repo_id)
- async def _mark_failed(reason: str) -> None:
+ async def _mark_terminal(status: str, reason: str) -> None:
try:
from repowise.core.persistence.crud import update_job_status
@@ -439,7 +668,7 @@ def _launch_job_task(request: Request, job_id: str, repo_id: str) -> None:
await update_job_status(
session,
job_id,
- "failed",
+ status,
error_message=reason[:500],
)
except Exception:
@@ -454,16 +683,25 @@ def _launch_job_task(request: Request, job_id: str, repo_id: str) -> None:
logger.exception("create_task_failed", extra={"job_id": job_id})
# Schedule the failure-marking on the running loop; we're already in
# an async request handler so a fresh task is fine.
- asyncio.create_task(_mark_failed(f"Failed to launch background task: {exc}"))
+ asyncio.create_task(_mark_terminal("failed", f"Failed to launch background task: {exc}"))
return
bg_tasks: set[asyncio.Task] = app_state.background_tasks # type: ignore[assignment]
bg_tasks.add(task)
+ # Track by job id so the cancel endpoint can interrupt the task itself.
+ job_tasks = getattr(app_state, "job_tasks", None)
+ if job_tasks is None:
+ job_tasks = {}
+ app_state.job_tasks = job_tasks
+ job_tasks[job_id] = task
def _on_done(t: asyncio.Task) -> None:
bg_tasks.discard(t)
+ job_tasks.pop(job_id, None)
if t.cancelled():
- asyncio.create_task(_mark_failed("Job task was cancelled"))
+ # execute_job normally records "cancelled" itself; this covers a
+ # cancel that landed before its try block was entered.
+ asyncio.create_task(_mark_terminal("cancelled", "Cancelled by user"))
return
exc = t.exception()
if exc is not None:
@@ -471,7 +709,7 @@ def _launch_job_task(request: Request, job_id: str, repo_id: str) -> None:
# execute_job already tries to mark failed in its except block,
# but if that itself raised we must still ensure the row is
# not left in pending/running.
- asyncio.create_task(_mark_failed(f"Background task crashed: {exc}"))
+ asyncio.create_task(_mark_terminal("failed", f"Background task crashed: {exc}"))
task.add_done_callback(_on_done)
diff --git a/packages/server/src/repowise/server/routers/workspace.py b/packages/server/src/repowise/server/routers/workspace.py
index fd2cc9e43..0fb1a1991 100644
--- a/packages/server/src/repowise/server/routers/workspace.py
+++ b/packages/server/src/repowise/server/routers/workspace.py
@@ -782,18 +782,36 @@ async def sync_workspace(
repo_path = (ws_root_path / entry.path).resolve()
db_path = repo_path / ".repowise" / "wiki.db"
- # Not indexed yet → no row to write a job against. Surface this
- # as "skipped" so the UI can show a clear "run `repowise update
- # --repo <alias>` from the CLI" hint. (A future enhancement is
- # to support remote first-time indexing; see Phase C.)
+ # Not indexed yet → establish the repo-local database and run the
+ # first full index through the same job machinery, instead of
+ # bouncing the user to the CLI.
if not db_path.exists():
+ try:
+ from repowise.server.repo_db import ensure_repo_registration
+ from repowise.server.routers.repos import _enqueue_index_job
+
+ factory, new_repo_id = await ensure_repo_registration(
+ request.app.state,
+ local_path=str(repo_path),
+ name=entry.alias,
+ )
+ job_id = await _enqueue_index_job(request, factory, new_repo_id)
+ except Exception as exc:
+ results.append(
+ WorkspaceSyncResult(
+ alias=entry.alias,
+ status="error",
+ reason=f"first-time index failed to start: {exc}",
+ )
+ )
+ continue
results.append(
WorkspaceSyncResult(
alias=entry.alias,
- status="skipped",
- reason="not indexed yet (run `repowise update --repo "
- + entry.alias
- + "` from the CLI)",
+ repo_id=new_repo_id,
+ status="accepted" if job_id else "skipped",
+ reason=None if job_id else "a job is already running for this repo",
+ job_id=job_id,
)
)
continue
diff --git a/packages/server/src/repowise/server/schemas/repository.py b/packages/server/src/repowise/server/schemas/repository.py
index 4403783bc..a5807e882 100644
--- a/packages/server/src/repowise/server/schemas/repository.py
+++ b/packages/server/src/repowise/server/schemas/repository.py
@@ -15,6 +15,10 @@ class RepoCreate(BaseModel):
url: str = ""
default_branch: str = "main"
settings: dict | None = None
+ # Enqueue the first full index immediately after registration. On by
+ # default so adding a repo from the UI takes it straight to indexed;
+ # pass false to register metadata only.
+ index: bool = True
@field_validator("local_path")
@classmethod
@@ -56,6 +60,9 @@ class RepoResponse(BaseModel):
is_primary: bool | None = None
docs_enabled: bool | None = None
docs_skip_reason: str | None = None
+ # Set on POST /api/repos responses when registration auto-enqueued the
+ # first index; clients attach to /api/jobs/{id}/stream with it.
+ initial_job_id: str | None = None
@classmethod
def from_orm(cls, obj: object) -> RepoResponse:
diff --git a/tests/unit/server/conftest.py b/tests/unit/server/conftest.py
index b269862b2..adcf9347c 100644
--- a/tests/unit/server/conftest.py
+++ b/tests/unit/server/conftest.py
@@ -165,6 +165,7 @@ async def create_test_repo(client: AsyncClient, tmp_path: Path | None = None) ->
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "test-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/test-repo",
diff --git a/tests/unit/server/test_architecture_api.py b/tests/unit/server/test_architecture_api.py
index e235d4909..124dcd107 100644
--- a/tests/unit/server/test_architecture_api.py
+++ b/tests/unit/server/test_architecture_api.py
@@ -24,6 +24,7 @@ async def _create_repo(client: AsyncClient) -> dict:
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "test-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/test-repo",
diff --git a/tests/unit/server/test_architecture_builder.py b/tests/unit/server/test_architecture_builder.py
index 58814994b..6a3d2842f 100644
--- a/tests/unit/server/test_architecture_builder.py
+++ b/tests/unit/server/test_architecture_builder.py
@@ -25,6 +25,7 @@ async def _create_repo(client: AsyncClient) -> dict:
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "test-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/test-repo",
@@ -307,6 +308,7 @@ async def test_auto_migrate_kg_file_to_db(client: AsyncClient, app) -> None:
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "migrate-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/migrate-repo",
diff --git a/tests/unit/server/test_c4.py b/tests/unit/server/test_c4.py
index 43a81df75..7f8690de7 100644
--- a/tests/unit/server/test_c4.py
+++ b/tests/unit/server/test_c4.py
@@ -25,6 +25,7 @@ async def create_test_repo(client: AsyncClient) -> dict:
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "test-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/test-repo",
diff --git a/tests/unit/server/test_dead_code.py b/tests/unit/server/test_dead_code.py
index 60c7239ee..cd7c087b6 100644
--- a/tests/unit/server/test_dead_code.py
+++ b/tests/unit/server/test_dead_code.py
@@ -133,11 +133,30 @@ async def test_dead_code_summary(client: AsyncClient, app) -> None:
@pytest.mark.asyncio
-async def test_analyze_dead_code(client: AsyncClient) -> None:
+async def test_analyze_dead_code(client: AsyncClient, monkeypatch) -> None:
+ # Stub the executor: this test covers the endpoint contract, not the pipeline.
+ async def _noop_execute(job_id, app_state, session_factory_override=None):
+ return None
+
+ from repowise.server.routers import repos as repos_module
+
+ monkeypatch.setattr(repos_module, "execute_job", _noop_execute)
+
repo = await create_test_repo(client)
resp = await client.post(f"/api/repos/{repo['id']}/dead-code/analyze")
assert resp.status_code == 202
- assert resp.json()["status"] == "analyzing"
+ data = resp.json()
+ assert data["status"] == "accepted"
+ assert data["job_id"]
+
+ # The analysis runs as an index-only job through the normal job machinery.
+ job_resp = await client.get(f"/api/jobs/{data['job_id']}")
+ assert job_resp.status_code == 200
+ assert job_resp.json()["config"]["mode"] == "index_only"
+
+ # A second trigger while the job is still pending is rejected.
+ resp2 = await client.post(f"/api/repos/{repo['id']}/dead-code/analyze")
+ assert resp2.status_code == 409
@pytest.mark.asyncio
diff --git a/tests/unit/server/test_initial_index.py b/tests/unit/server/test_initial_index.py
new file mode 100644
index 000000000..c5ee1b5ae
--- /dev/null
+++ b/tests/unit/server/test_initial_index.py
@@ -0,0 +1,331 @@
+"""Tests for API-driven first-time indexing.
+
+Covers repo-local database registration on create, the explicit
+``POST /api/repos/{id}/index`` trigger, the pre-flight check, real job
+cancellation, the ``initial_index`` executor mode's state.json baseline,
+and the job event buffer that feeds the SSE message stream.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import sqlite3
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from httpx import AsyncClient
+
+from repowise.core.cancellation import PipelineCancelled
+from repowise.core.persistence.crud import (
+ get_generation_job,
+ upsert_generation_job,
+ upsert_repository,
+)
+from repowise.server.job_events import JobEventBuffer, create_event_buffer
+from repowise.server.job_executor import execute_job
+
+
+def _make_fake_repo(tmp_path: Path, name: str = "fresh-repo") -> Path:
+ repo_dir = tmp_path / name
+ repo_dir.mkdir()
+ (repo_dir / ".git").mkdir()
+ (repo_dir / "main.py").write_text("print('hi')\n", encoding="utf-8")
+ (repo_dir / "util.py").write_text("def f():\n return 1\n", encoding="utf-8")
+ return repo_dir
+
+
+async def _noop_execute(job_id, app_state, session_factory_override=None):
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Repo-local DB registration
+# ---------------------------------------------------------------------------
+
+
+async def test_create_with_index_uses_repo_local_db(
+ client: AsyncClient, app, tmp_path, monkeypatch
+) -> None:
+ """POST /api/repos with index=true stores the repo in its own wiki.db."""
+ from repowise.server.routers import repos as repos_module
+
+ monkeypatch.setattr(repos_module, "execute_job", _noop_execute)
+ repo_dir = _make_fake_repo(tmp_path)
+
+ resp = await client.post(
+ "/api/repos",
+ json={"name": "fresh-repo", "local_path": str(repo_dir), "index": True},
+ )
+ assert resp.status_code == 201
+ data = resp.json()
+ repo_id = data["id"]
+ assert data["initial_job_id"]
+
+ # The canonical store is the repo-local wiki.db, with the job row inside.
+ db_file = repo_dir / ".repowise" / "wiki.db"
+ assert db_file.is_file()
+ with sqlite3.connect(str(db_file)) as conn:
+ row = conn.execute("SELECT id FROM repositories LIMIT 1").fetchone()
+ assert row[0] == repo_id
+ job_row = conn.execute("SELECT config_json FROM generation_jobs LIMIT 1").fetchone()
+ assert json.loads(job_row[0])["mode"] == "initial_index"
+
+ # Routed reads resolve through the registered per-repo factory.
+ assert repo_id in app.state.workspace_sessions
+ get_resp = await client.get(f"/api/repos/{repo_id}")
+ assert get_resp.status_code == 200
+
+ # The primary DB carries a registry row with the same id, and the repo
+ # appears exactly once in listings.
+ from sqlalchemy import select
+
+ from repowise.core.persistence.models import Repository
+
+ async with app.state.session_factory() as session:
+ result = await session.execute(select(Repository.id))
+ assert [r[0] for r in result.all()] == [repo_id]
+
+ list_resp = await client.get("/api/repos")
+ assert [r["id"] for r in list_resp.json()] == [repo_id]
+
+
+async def test_index_endpoint_registers_and_enqueues(
+ client: AsyncClient, app, tmp_path, monkeypatch
+) -> None:
+ """POST /{id}/index takes a metadata-only repo to a repo-local index job."""
+ from repowise.server.routers import repos as repos_module
+
+ monkeypatch.setattr(repos_module, "execute_job", _noop_execute)
+ repo_dir = _make_fake_repo(tmp_path)
+
+ created = await client.post(
+ "/api/repos",
+ json={"name": "fresh-repo", "local_path": str(repo_dir), "index": False},
+ )
+ repo_id = created.json()["id"]
+ assert not (repo_dir / ".repowise" / "wiki.db").exists()
+
+ resp = await client.post(f"/api/repos/{repo_id}/index")
+ assert resp.status_code == 202
+ assert resp.json()["job_id"]
+ assert (repo_dir / ".repowise" / "wiki.db").is_file()
+ assert repo_id in app.state.workspace_sessions
+
+ # Active-job guard: a second trigger is rejected while the first pends.
+ resp2 = await client.post(f"/api/repos/{repo_id}/index")
+ assert resp2.status_code == 409
+
+
+# ---------------------------------------------------------------------------
+# Pre-flight
+# ---------------------------------------------------------------------------
+
+
+async def test_preflight_reports_provider_and_estimate(
+ client: AsyncClient, tmp_path, monkeypatch
+) -> None:
+ repo_dir = _make_fake_repo(tmp_path)
+ created = await client.post(
+ "/api/repos",
+ json={"name": "fresh-repo", "local_path": str(repo_dir), "index": False},
+ )
+ repo_id = created.json()["id"]
+
+ fake_provider = SimpleNamespace(
+ provider_name="mock",
+ model_name="claude-sonnet-4-6",
+ generate=AsyncMock(return_value="OK"),
+ )
+ with patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ return_value=fake_provider,
+ ):
+ resp = await client.post(f"/api/repos/{repo_id}/preflight")
+
+ assert resp.status_code == 200
+ data = resp.json()
+ assert data["provider"] == {
+ "ok": True,
+ "name": "mock",
+ "model": "claude-sonnet-4-6",
+ "error": None,
+ }
+ assert data["file_count"] >= 2
+ assert data["estimate"]["total_pages"] >= 1
+ assert data["estimate"]["estimated_cost_usd"] > 0
+
+
+async def test_preflight_surfaces_provider_failure(client: AsyncClient, tmp_path) -> None:
+ repo_dir = _make_fake_repo(tmp_path)
+ created = await client.post(
+ "/api/repos",
+ json={"name": "fresh-repo", "local_path": str(repo_dir), "index": False},
+ )
+ repo_id = created.json()["id"]
+
+ with patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("No active provider configured"),
+ ):
+ resp = await client.post(f"/api/repos/{repo_id}/preflight")
+
+ assert resp.status_code == 200
+ data = resp.json()
+ assert data["provider"]["ok"] is False
+ assert "No active provider" in data["provider"]["error"]
+ assert data["estimate"] is None
+
+
+# ---------------------------------------------------------------------------
+# Cancellation
+# ---------------------------------------------------------------------------
+
+
+async def test_cancel_stops_task_and_marks_cancelled(
+ client: AsyncClient, app, session_factory, tmp_path
+) -> None:
+ """POST /cancel flips the token, cancels the task, and records 'cancelled'."""
+ async with session_factory() as session:
+ repo = await upsert_repository(session, name="r", local_path=str(tmp_path))
+ job = await upsert_generation_job(session, repository_id=repo.id, status="running")
+ await session.commit()
+ job_id = job.id
+
+ from repowise.core.cancellation import CancellationToken
+
+ token = CancellationToken()
+ blocker = asyncio.Event()
+ task = asyncio.create_task(blocker.wait())
+ app.state.job_cancel_tokens = {job_id: token}
+ app.state.job_tasks = {job_id: task}
+
+ resp = await client.post(f"/api/jobs/{job_id}/cancel")
+ assert resp.status_code == 200
+ assert resp.json()["status"] == "cancelled"
+ assert token.cancelled
+
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ # Terminal state: a second cancel is a 409.
+ resp2 = await client.post(f"/api/jobs/{job_id}/cancel")
+ assert resp2.status_code == 409
+
+
+async def test_execute_job_records_cancelled_status(session_factory, tmp_path) -> None:
+ """A pipeline unwound by PipelineCancelled ends as 'cancelled', not 'failed'."""
+ async with session_factory() as session:
+ repo = await upsert_repository(session, name="r", local_path=str(tmp_path))
+ job = await upsert_generation_job(
+ session, repository_id=repo.id, config={"mode": "full_resync"}
+ )
+ await session.commit()
+ job_id = job.id
+
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+
+ with (
+ patch(
+ "repowise.server.job_executor.run_pipeline",
+ AsyncMock(side_effect=PipelineCancelled()),
+ ),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ async with session_factory() as session:
+ refreshed = await get_generation_job(session, job_id)
+ assert refreshed.status == "cancelled"
+ assert refreshed.finished_at is not None
+ assert app_state.job_cancel_tokens == {}
+
+
+# ---------------------------------------------------------------------------
+# initial_index executor mode
+# ---------------------------------------------------------------------------
+
+
+async def test_execute_job_initial_index_writes_state_baseline(session_factory, tmp_path) -> None:
+ repo_dir = tmp_path / "repo"
+ repo_dir.mkdir()
+ async with session_factory() as session:
+ repo = await upsert_repository(session, name="r", local_path=str(repo_dir))
+ job = await upsert_generation_job(
+ session, repository_id=repo.id, config={"mode": "initial_index"}
+ )
+ await session.commit()
+ job_id = job.id
+
+ app_state = SimpleNamespace(session_factory=session_factory, fts=None, vector_store=None)
+ result = SimpleNamespace(generated_pages=[], parsed_files=[], file_count=2, symbol_count=3)
+
+ with (
+ patch("repowise.server.job_executor.run_pipeline", AsyncMock(return_value=result)),
+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ async with session_factory() as session:
+ refreshed = await get_generation_job(session, job_id)
+ assert refreshed.status == "completed"
+
+ state = json.loads((repo_dir / ".repowise" / "state.json").read_text(encoding="utf-8"))
+ assert state["docs_enabled"] is False
+ assert "no provider configured" in state["docs_skip_reason"]
+ assert state["run_mode"] == "standard"
+ assert state["git_tier"] == "full"
+ assert state["config_fingerprint"]
+ assert state["store_format_version"]
+
+ # The no-provider warning reached the event buffer for the SSE stream.
+ events = app_state.job_events[job_id].since(0)
+ assert any("No LLM provider configured" in e["text"] for e in events)
+
+
+# ---------------------------------------------------------------------------
+# Job event buffer
+# ---------------------------------------------------------------------------
+
+
+def test_job_event_buffer_sequences_and_phases() -> None:
+ buf = JobEventBuffer()
+ buf.set_phase("parse", 100)
+ buf.add("warning", "1 file skipped")
+ buf.set_phase("generation", None)
+
+ events = buf.since(0)
+ assert [e["seq"] for e in events] == [0, 1, 2]
+ assert events[0]["text"] == "Phase started: parse (100 items)"
+ assert events[1]["level"] == "warning"
+ assert events[1]["phase"] == "parse"
+ assert events[2]["phase"] == "generation"
+ assert buf.phase == "generation"
+
+ # Incremental drain: only new events come back.
+ assert [e["seq"] for e in buf.since(2)] == [2]
+
+
+def test_event_buffer_registry_evicts_oldest() -> None:
+ state = SimpleNamespace()
+ for i in range(55):
+ create_event_buffer(state, f"job-{i}")
+ assert len(state.job_events) == 50
+ assert "job-0" not in state.job_events
+ assert "job-54" in state.job_events
diff --git a/tests/unit/server/test_refactoring.py b/tests/unit/server/test_refactoring.py
index 063e9a615..9cf8f3640 100644
--- a/tests/unit/server/test_refactoring.py
+++ b/tests/unit/server/test_refactoring.py
@@ -21,6 +21,7 @@ async def create_test_repo(client: AsyncClient) -> dict:
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "test-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/test-repo",
@@ -349,6 +350,7 @@ async def _seed_enrich_repo(client: AsyncClient, app, *, enabled: bool) -> tuple
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "enrich-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/enrich-repo",
diff --git a/tests/unit/server/test_repos.py b/tests/unit/server/test_repos.py
index ea02844bd..ca2b3a633 100644
--- a/tests/unit/server/test_repos.py
+++ b/tests/unit/server/test_repos.py
@@ -21,6 +21,7 @@ async def test_create_repo(client: AsyncClient) -> None:
resp = await client.post(
"/api/repos",
json={
+ "index": False,
"name": "my-repo",
"local_path": str(repo_dir),
"url": "https://github.com/example/my-repo",
diff --git a/tests/unit/server/test_workspace_sidebar.py b/tests/unit/server/test_workspace_sidebar.py
index 89e20ec6b..30d649d9e 100644
--- a/tests/unit/server/test_workspace_sidebar.py
+++ b/tests/unit/server/test_workspace_sidebar.py
@@ -214,10 +214,18 @@ async def workspace_app_with_workspace_router(workspace_app):
return (app, *rest)
-async def test_workspace_sync_endpoint_skips_unindexed(
- workspace_app_with_workspace_router, client: AsyncClient
+async def test_workspace_sync_endpoint_indexes_unindexed(
+ workspace_app_with_workspace_router, client: AsyncClient, monkeypatch
):
- app, ws_root, ws_config, repo_id = workspace_app_with_workspace_router
+ _app, ws_root, _ws_config, repo_id = workspace_app_with_workspace_router
+
+ # Stub the executor: this test covers the fan-out decisions, not the pipeline.
+ async def _noop_execute(job_id, app_state, session_factory_override=None):
+ return None
+
+ from repowise.server.routers import repos as repos_module
+
+ monkeypatch.setattr(repos_module, "execute_job", _noop_execute)
# Repo path resolution needs the fake wiki.db to point at the right
# repo_id; the endpoint uses sqlite3.connect on disk to discover it.
# We seeded a placeholder; replace with a real schema containing the
@@ -238,10 +246,12 @@ async def test_workspace_sync_endpoint_skips_unindexed(
assert resp.status_code == 202, resp.text
body = resp.json()
- # frontend has no wiki.db → skipped with helpful reason.
+ # frontend has no wiki.db → a repo-local DB is created and the first
+ # full index is enqueued through the normal job machinery.
aliases = {r["alias"]: r for r in body["results"]}
- assert aliases["frontend"]["status"] == "skipped"
- assert "not indexed" in aliases["frontend"]["reason"]
+ assert aliases["frontend"]["status"] == "accepted"
+ assert aliases["frontend"]["job_id"]
+ assert (ws_root / "frontend" / ".repowise" / "wiki.db").is_file()
# backend got a job (or was accepted) — exact value depends on
# whether the dummy job table is wired, but it must not error.
assert aliases["backend"]["status"] in {"accepted", "error", "skipped"}