Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(persistence): split crud.py into a per-domain crud/ package (#256)
Swati Ahujaa9aaa9efe284
diff --git a/packages/core/src/repowise/core/persistence/crud.py b/packages/core/src/repowise/core/persistence/crud.pydeleted file mode 100644index 1cadc0051..000000000--- a/packages/core/src/repowise/core/persistence/crud.py+++ /dev/null@@ -1,2861 +0,0 @@-"""Async CRUD operations for the repowise persistence layer.--All functions accept an AsyncSession as the first argument; the caller owns-transaction boundaries. Functions that complete a logical unit of work call-``await session.flush()`` to write changes to the transaction buffer — the-caller must ``await session.commit()`` (or use the ``get_session`` context-manager from database.py).--Versioning contract for upsert_page:- First upsert → inserts Page (version=1). No PageVersion created.- Second upsert → archives existing Page as PageVersion, then updates Page- in place (version increments). created_at is preserved.-"""--from __future__ import annotations--import json-from datetime import UTC, datetime-from typing import Any--from sqlalchemy import delete, func, select, text-from sqlalchemy.ext.asyncio import AsyncSession--from repowise.core.analysis.decision_provenance import compute_confidence, rank_for_source--from .decision_graph import sync_decision_node_links-from .models import (- ChatMessage,- Conversation,- CoverageFile,- DeadCodeFinding,- DecisionEvidence,- DecisionRecord,- ExternalSystem,- GenerationJob,- GitMetadata,- GraphEdge,- GraphMetric,- GraphNode,- HealthFileMetric,- HealthFinding,- HealthSnapshot,- KnowledgeGraphLayer,- KnowledgeGraphTourStep,- Page,- PageVersion,- Repository,- WebhookEvent,- WikiSymbol,- _new_uuid,- _now_utc,-)--# ----------------------------------------------------------------------------# Internal helpers-# -----------------------------------------------------------------------------_VALID_JOB_STATUSES = frozenset({"pending", "running", "completed", "failed", "paused"})--_BATCH_SIZE = 500 # max rows per INSERT to stay under SQLite's parameter limit---def _parse_dt(ts: str) -> datetime:- """Parse an ISO-8601 UTC string to a timezone-aware datetime."""- ts = ts.replace("Z", "+00:00")- dt = datetime.fromisoformat(ts)- if dt.tzinfo is None:- dt = dt.replace(tzinfo=UTC)- return dt---# ----------------------------------------------------------------------------# Repository CRUD-# ------------------------------------------------------------------------------async def upsert_repository(- session: AsyncSession,- *,- name: str,- local_path: str,- url: str = "",- default_branch: str = "main",- settings: dict | None = None,-) -> Repository:- """Create or update a repository record.-- Lookup is by ``local_path`` (the canonical key for local repositories).- """- result = await session.execute(select(Repository).where(Repository.local_path == local_path))- repo = result.scalar_one_or_none()-- if repo is None:- repo = Repository(- id=_new_uuid(),- name=name,- local_path=local_path,- url=url,- default_branch=default_branch,- settings_json=json.dumps(settings or {}),- )- session.add(repo)- else:- repo.name = name- repo.url = url- repo.default_branch = default_branch- if settings is not None:- repo.settings_json = json.dumps(settings)- repo.updated_at = _now_utc()-- await session.flush()- return repo---async def get_repository(session: AsyncSession, repo_id: str) -> Repository | None:- """Return a Repository by primary key, or None."""- return await session.get(Repository, repo_id)---async def get_repository_by_path(session: AsyncSession, local_path: str) -> Repository | None:- """Return a Repository by local_path, or None."""- result = await session.execute(select(Repository).where(Repository.local_path == local_path))- return result.scalar_one_or_none()---async def delete_repository(session: AsyncSession, repo_id: str) -> bool:- """Delete a repository and all cascaded children.-- Returns True if deleted, False if not found.-- NOTE: The caller should clean up the FTS index *before* calling this,- since the CASCADE will delete Page rows and we lose the page IDs.- """- repo = await session.get(Repository, repo_id)- if repo is None:- return False- await session.delete(repo)- await session.flush()- return True---async def list_page_ids(session: AsyncSession, repository_id: str) -> list[str]:- """Return all page IDs for a repository (lightweight, ID-only query)."""- result = await session.execute(select(Page.id).where(Page.repository_id == repository_id))- return list(result.scalars().all())---# ----------------------------------------------------------------------------# GenerationJob CRUD-# ------------------------------------------------------------------------------async def upsert_generation_job(- session: AsyncSession,- *,- repository_id: str,- status: str = "pending",- provider_name: str = "",- model_name: str = "",- total_pages: int = 0,- config: dict | None = None,- job_id: str | None = None,-) -> GenerationJob:- """Insert a new GenerationJob (jobs are append-only)."""- job = GenerationJob(- id=job_id or _new_uuid(),- repository_id=repository_id,- status=status,- provider_name=provider_name,- model_name=model_name,- total_pages=total_pages,- config_json=json.dumps(config or {}),- )- session.add(job)- await session.flush()- return job---async def get_generation_job(session: AsyncSession, job_id: str) -> GenerationJob | None:- """Return a GenerationJob by primary key, or None."""- return await session.get(GenerationJob, job_id)---async def update_job_status(- session: AsyncSession,- job_id: str,- status: str,- *,- completed_pages: int | None = None,- failed_pages: int | None = None,- current_level: int | None = None,- total_pages: int | None = None,- error_message: str | None = None,-) -> GenerationJob:- """Update the mutable fields of a GenerationJob.-- Raises:- ValueError: If *status* is not a recognised value.- LookupError: If *job_id* does not exist.- """- if status not in _VALID_JOB_STATUSES:- raise ValueError(- f"Unknown job status {status!r}. Valid values: {sorted(_VALID_JOB_STATUSES)}"- )-- job = await session.get(GenerationJob, job_id)- if job is None:- raise LookupError(f"No GenerationJob with id={job_id!r}")-- job.status = status- job.updated_at = _now_utc()-- if completed_pages is not None:- job.completed_pages = completed_pages- if failed_pages is not None:- job.failed_pages = failed_pages- if current_level is not None:- job.current_level = current_level- if total_pages is not None:- job.total_pages = total_pages- if error_message is not None:- job.error_message = error_message-- if status == "running" and job.started_at is None:- job.started_at = _now_utc()- if status in ("completed", "failed"):- job.finished_at = _now_utc()-- await session.flush()- return job---# ----------------------------------------------------------------------------# Page CRUD (with versioning)-# ------------------------------------------------------------------------------async def upsert_page(- session: AsyncSession,- *,- page_id: str,- repository_id: str,- page_type: str,- title: str,- content: str,- summary: str = "",- target_path: str,- source_hash: str,- model_name: str,- provider_name: str,- input_tokens: int = 0,- output_tokens: int = 0,- cached_tokens: int = 0,- generation_level: int = 0,- confidence: float = 1.0,- freshness_status: str = "fresh",- metadata: dict | None = None,- created_at: datetime | None = None,- updated_at: datetime | None = None,-) -> Page:- """Insert or update a wiki page, creating a PageVersion snapshot on update.-- First call → inserts Page at version=1.- Subsequent → archives the current Page as a PageVersion, then updates the- Page in-place (version += 1, created_at preserved).- """- now = _now_utc()- page_created_at = created_at or now- page_updated_at = updated_at or now- meta_json = json.dumps(metadata or {})-- existing_result = await session.execute(select(Page).where(Page.id == page_id))- existing = existing_result.scalar_one_or_none()-- if existing is not None:- # Archive the current state before overwriting- snapshot = PageVersion(- id=_new_uuid(),- page_id=existing.id,- repository_id=existing.repository_id,- version=existing.version,- page_type=existing.page_type,- title=existing.title,- content=existing.content,- source_hash=existing.source_hash,- model_name=existing.model_name,- provider_name=existing.provider_name,- input_tokens=existing.input_tokens,- output_tokens=existing.output_tokens,- confidence=existing.confidence,- archived_at=now,- )- session.add(snapshot)-- # Update Page in place (preserves created_at)- existing.page_type = page_type- existing.title = title- existing.content = content- existing.summary = summary- existing.target_path = target_path- existing.source_hash = source_hash- existing.model_name = model_name- existing.provider_name = provider_name- existing.input_tokens = input_tokens- existing.output_tokens = output_tokens- existing.cached_tokens = cached_tokens- existing.generation_level = generation_level- existing.version = existing.version + 1- existing.confidence = confidence- existing.freshness_status = freshness_status- existing.metadata_json = meta_json- existing.updated_at = page_updated_at-- await session.flush()- return existing- else:- page = Page(- id=page_id,- repository_id=repository_id,- page_type=page_type,- title=title,- content=content,- summary=summary,- target_path=target_path,- source_hash=source_hash,- model_name=model_name,- provider_name=provider_name,- input_tokens=input_tokens,- output_tokens=output_tokens,- cached_tokens=cached_tokens,- generation_level=generation_level,- version=1,- confidence=confidence,- freshness_status=freshness_status,- metadata_json=meta_json,- created_at=page_created_at,- updated_at=page_updated_at,- )- session.add(page)- await session.flush()- return page---async def load_prior_pages(- session: AsyncSession,- repository_id: str,-) -> dict[str, Any]:- """Return a ``page_id → PriorPage`` map for cross-run cache reuse.-- Loads every existing wiki page for the repository so the generator can- short-circuit the LLM call when the freshly rendered prompt produces a- matching ``source_hash`` under the same model. Returns an empty dict if- nothing has been generated yet.- """- # Import lazily — keeps persistence independent of generation models at- # module-load time.- from repowise.core.generation.page_generator import PriorPage-- result = await session.execute(select(Page).where(Page.repository_id == repository_id))- prior: dict[str, Any] = {}- for row in result.scalars():- prior[row.id] = PriorPage(- source_hash=row.source_hash,- model_name=row.model_name,- content=row.content,- input_tokens=row.input_tokens,- output_tokens=row.output_tokens,- cached_tokens=row.cached_tokens,- )- return prior---async def upsert_page_from_generated(- session: AsyncSession,- generated_page: object, # repowise.core.generation.models.GeneratedPage- repository_id: str,-) -> Page:- """Convenience wrapper that unpacks a GeneratedPage dataclass.-- This keeps the CRUD layer independent of the generation models at the- import level while still providing a clean API for callers that have a- GeneratedPage in hand.- """- gp = generated_page # type alias for brevity- return await upsert_page(- session,- page_id=gp.page_id, # type: ignore[attr-defined]- repository_id=repository_id,- page_type=gp.page_type, # type: ignore[attr-defined]- title=gp.title, # type: ignore[attr-defined]- content=gp.content, # type: ignore[attr-defined]- summary=getattr(gp, "summary", "") or "",- target_path=gp.target_path, # type: ignore[attr-defined]- source_hash=gp.source_hash, # type: ignore[attr-defined]- model_name=gp.model_name, # type: ignore[attr-defined]- provider_name=gp.provider_name, # type: ignore[attr-defined]- input_tokens=gp.input_tokens, # type: ignore[attr-defined]- output_tokens=gp.output_tokens, # type: ignore[attr-defined]- cached_tokens=gp.cached_tokens, # type: ignore[attr-defined]- generation_level=gp.generation_level, # type: ignore[attr-defined]- confidence=gp.confidence, # type: ignore[attr-defined]- freshness_status=gp.freshness_status, # type: ignore[attr-defined]- metadata=gp.metadata, # type: ignore[attr-defined]- created_at=_parse_dt(gp.created_at), # type: ignore[attr-defined]- updated_at=_parse_dt(gp.updated_at), # type: ignore[attr-defined]- )---async def get_page(session: AsyncSession, page_id: str) -> Page | None:- """Return a Page by its page_id, or None."""- return await session.get(Page, page_id)---async def list_pages(- session: AsyncSession,- repository_id: str,- *,- page_type: str | None = None,- limit: int = 100,- offset: int = 0,- sort_by: str = "updated_at",- order: str = "desc",-) -> list[Page]:- """Return pages for a repository, optionally filtered by page_type."""- q = select(Page).where(Page.repository_id == repository_id)- if page_type is not None:- q = q.where(Page.page_type == page_type)- _sort_cols = {- "updated_at": Page.updated_at,- "confidence": Page.confidence,- "created_at": Page.created_at,- }- sort_col = _sort_cols.get(sort_by, Page.updated_at)- q = q.order_by(sort_col.asc() if order == "asc" else sort_col.desc())- q = q.limit(limit).offset(offset)- result = await session.execute(q)- return list(result.scalars().all())---async def get_page_versions(- session: AsyncSession,- page_id: str,- *,- limit: int = 50,-) -> list[PageVersion]:- """Return historical versions of a page, newest first."""- result = await session.execute(- select(PageVersion)- .where(PageVersion.page_id == page_id)- .order_by(PageVersion.version.desc())- .limit(limit)- )- return list(result.scalars().all())---async def get_stale_pages(- session: AsyncSession,- repository_id: str,-) -> list[Page]:- """Return pages with freshness_status in ('stale', 'expired')."""- result = await session.execute(- select(Page).where(- Page.repository_id == repository_id,- Page.freshness_status.in_(["stale", "expired"]),- )- )- return list(result.scalars().all())---# ----------------------------------------------------------------------------# Graph CRUD (batch)-# ------------------------------------------------------------------------------async def batch_upsert_graph_nodes(- session: AsyncSession,- repository_id: str,- nodes: list[dict],-) -> None:- """Upsert graph nodes for a repository in batches of up to 500.-- Each element of *nodes* is a dict with keys matching GraphNode fields- (excluding id and repository_id which are set here).-- Uses SELECT-then-INSERT/UPDATE for dialect portability.- """- for node_data in nodes:- node_id = node_data.get("node_id", "")- result = await session.execute(- select(GraphNode).where(- GraphNode.repository_id == repository_id,- GraphNode.node_id == node_id,- )- )- existing = result.scalar_one_or_none()-- if existing is not None:- for key, val in node_data.items():- if key not in ("id", "repository_id", "created_at") and hasattr(existing, key):- setattr(existing, key, val)- else:- session.add(- GraphNode(- id=_new_uuid(),- repository_id=repository_id,- **{k: v for k, v in node_data.items() if k not in ("id", "repository_id")},- )- )-- await session.flush()---async def batch_upsert_graph_edges(- session: AsyncSession,- repository_id: str,- edges: list[dict],-) -> None:- """Upsert graph edges for a repository.-- Each element of *edges* should have ``source_node_id``, ``target_node_id``,- ``edge_type``, and optionally ``imported_names_json`` and ``confidence``.-- The unique constraint is (repository_id, source, target, edge_type),- allowing multiple edge types between the same pair of nodes.- """- for edge_data in edges:- source = edge_data.get("source_node_id", "")- target = edge_data.get("target_node_id", "")- edge_type = edge_data.get("edge_type", "imports")- result = await session.execute(- select(GraphEdge).where(- GraphEdge.repository_id == repository_id,- GraphEdge.source_node_id == source,- GraphEdge.target_node_id == target,- GraphEdge.edge_type == edge_type,- )- )- existing = result.scalar_one_or_none()-- if existing is not None:- imported = edge_data.get("imported_names_json")- if imported is not None:- existing.imported_names_json = imported- confidence = edge_data.get("confidence")- if confidence is not None:- existing.confidence = confidence- else:- session.add(- GraphEdge(- id=_new_uuid(),- repository_id=repository_id,- source_node_id=source,- target_node_id=target,- imported_names_json=edge_data.get("imported_names_json", "[]"),- edge_type=edge_type,- confidence=edge_data.get("confidence", 1.0),- )- )-- await session.flush()---async def batch_upsert_graph_metrics(- session: AsyncSession,- repository_id: str,- metrics: dict[str, dict],-) -> None:- """Materialize the file-level metrics snapshot into ``graph_metrics``.-- *metrics* maps ``node_id`` → a dict with ``pagerank``, ``betweenness``,- ``community_id``, ``in_degree``, ``out_degree``. Additive to- ``graph_nodes`` — this is the snapshot read back by- ``GraphBuilder.load_metrics_from_sql`` on large repos. SELECT-then-write- for dialect portability (SQLite + Postgres).- """- fields = ("pagerank", "betweenness", "community_id", "in_degree", "out_degree")- for node_id, m in metrics.items():- result = await session.execute(- select(GraphMetric).where(- GraphMetric.repository_id == repository_id,- GraphMetric.node_id == node_id,- )- )- existing = result.scalar_one_or_none()- if existing is not None:- for key in fields:- if key in m:- setattr(existing, key, m[key])- else:- session.add(- GraphMetric(- id=_new_uuid(),- repository_id=repository_id,- node_id=node_id,- pagerank=float(m.get("pagerank", 0.0)),- betweenness=float(m.get("betweenness", 0.0)),- community_id=int(m.get("community_id", 0)),- in_degree=int(m.get("in_degree", 0)),- out_degree=int(m.get("out_degree", 0)),- )- )-- await session.flush()---async def get_graph_metrics(- session: AsyncSession,- repository_id: str,-) -> dict[str, dict]:- """Read the materialized ``graph_metrics`` snapshot as ``node_id → metrics``."""- result = await session.execute(- select(GraphMetric).where(GraphMetric.repository_id == repository_id)- )- return {- row.node_id: {- "pagerank": row.pagerank,- "betweenness": row.betweenness,- "community_id": row.community_id,- "in_degree": row.in_degree,- "out_degree": row.out_degree,- }- for row in result.scalars().all()- }---async def get_all_graph_nodes(- session: AsyncSession,- repository_id: str,-) -> list[dict]:- """Read every persisted graph node as a list of plain dicts.-- Used to rehydrate an in-memory :class:`GraphBuilder` from SQL without- re-parsing or re-resolving the graph (see- ``repowise.core.pipeline.upgrade.rehydrate_graph_builder``). Each dict- carries ``node_id`` plus the file/symbol attributes that the NetworkX node- needs for traversal and rendering.- """- result = await session.execute(- select(GraphNode).where(GraphNode.repository_id == repository_id)- )- return [- {- "node_id": row.node_id,- "node_type": row.node_type,- "language": row.language,- "symbol_count": row.symbol_count,- "has_error": row.has_error,- "is_test": row.is_test,- "is_entry_point": row.is_entry_point,- "kind": row.kind,- "name": row.name,- "qualified_name": row.qualified_name,- "file_path": row.file_path,- "start_line": row.start_line,- "end_line": row.end_line,- "visibility": row.visibility,- "signature": row.signature,- "parent_symbol_id": row.parent_symbol_id,- }- for row in result.scalars().all()- ]---async def get_all_graph_edges(- session: AsyncSession,- repository_id: str,-) -> list[dict]:- """Read every persisted graph edge as a list of plain dicts.-- Companion to :func:`get_all_graph_nodes` for graph rehydration. The- ``imported_names_json`` column is decoded back into a list so the- rehydrated edge matches the in-memory shape produced during ingestion.- """- import json as _json-- result = await session.execute(- select(GraphEdge).where(GraphEdge.repository_id == repository_id)- )- edges: list[dict] = []- for row in result.scalars().all():- try:- imported_names = _json.loads(row.imported_names_json or "[]")- except (ValueError, TypeError):- imported_names = []- edges.append(- {- "source_node_id": row.source_node_id,- "target_node_id": row.target_node_id,- "edge_type": row.edge_type,- "confidence": row.confidence,- "imported_names": imported_names,- }- )- return edges---# ----------------------------------------------------------------------------# ExternalSystem CRUD-# ------------------------------------------------------------------------------async def bulk_upsert_external_systems(- session: AsyncSession,- repository_id: str,- systems: list[dict],-) -> dict[tuple[str, str], int]:- """Upsert external systems for a repository.-- Each element of *systems* is a dict with keys matching ``ExternalSystem``- columns (excluding ``id``, ``repository_id``, ``created_at``).-- Returns a mapping of ``(name, declared_in)`` → row ``id`` for both newly- inserted and existing rows, so callers can link ``graph_nodes`` to the- persisted external system without an extra round-trip.- """- id_map: dict[tuple[str, str], int] = {}- for sys_data in systems:- name = sys_data.get("name", "")- declared_in = sys_data.get("declared_in", "")- if not name:- continue- result = await session.execute(- select(ExternalSystem).where(- ExternalSystem.repository_id == repository_id,- ExternalSystem.name == name,- ExternalSystem.declared_in == declared_in,- )- )- existing = result.scalar_one_or_none()- if existing is not None:- for key, val in sys_data.items():- if key not in ("id", "repository_id", "created_at") and hasattr(existing, key):- setattr(existing, key, val)- id_map[(name, declared_in)] = existing.id- else:- row = ExternalSystem(- repository_id=repository_id,- **{k: v for k, v in sys_data.items() if k not in ("id", "repository_id")},- )- session.add(row)- await session.flush()- id_map[(name, declared_in)] = row.id- await session.flush()- return id_map---async def link_graph_nodes_to_external_systems(- session: AsyncSession,- repository_id: str,- name_to_id: dict[str, int],-) -> int:- """Resolve ``external:{name}`` graph nodes to their ExternalSystem row.-- ``name_to_id`` should be a flat map of dep name → ExternalSystem id- (collapse multi-manifest entries by picking any id — the C4 renderer- only needs ``name``/``category`` which are the same across rows).-- Returns the number of graph_nodes updated.- """- if not name_to_id:- return 0- prefix = "external:"- result = await session.execute(- select(GraphNode).where(- GraphNode.repository_id == repository_id,- GraphNode.node_id.like(f"{prefix}%"),- )- )- updated = 0- for node in result.scalars():- suffix = node.node_id[len(prefix) :]- # Try the full suffix first, then the first segment (handles e.g.- # ``external:fastapi.responses`` → ``fastapi``).- sys_id = name_to_id.get(suffix)- if sys_id is None and "." in suffix:- sys_id = name_to_id.get(suffix.split(".", 1)[0])- if sys_id is None and "/" in suffix:- sys_id = name_to_id.get(suffix.split("/", 1)[0])- if sys_id is not None and node.external_system_id != sys_id:- node.external_system_id = sys_id- updated += 1- await session.flush()- return updated---async def list_external_systems(session: AsyncSession, repository_id: str) -> list[ExternalSystem]:- """List all external systems for a repository, ordered by name."""- result = await session.execute(- select(ExternalSystem)- .where(ExternalSystem.repository_id == repository_id)- .order_by(ExternalSystem.name)- )- return list(result.scalars())---# ----------------------------------------------------------------------------# WikiSymbol CRUD (batch)-# ------------------------------------------------------------------------------async def batch_upsert_symbols(- session: AsyncSession,- repository_id: str,- symbols: list, # list[ingestion.models.Symbol]-) -> None:- """Upsert ingestion Symbol objects into the wiki_symbols table.-- Accepts ingestion.models.Symbol dataclass instances (duck-typed).- """- for sym in symbols:- symbol_id = getattr(sym, "id", None) or f"{sym.file_path}::{sym.name}"- result = await session.execute(- select(WikiSymbol).where(- WikiSymbol.repository_id == repository_id,- WikiSymbol.symbol_id == symbol_id,- )- )- existing = result.scalar_one_or_none()-- if existing is not None:- existing.name = sym.name- existing.qualified_name = getattr(sym, "qualified_name", sym.name)- existing.kind = sym.kind- existing.signature = getattr(sym, "signature", "")- existing.start_line = getattr(sym, "start_line", 0)- existing.end_line = getattr(sym, "end_line", 0)- existing.docstring = getattr(sym, "docstring", None)- existing.visibility = getattr(sym, "visibility", "public")- existing.is_async = getattr(sym, "is_async", False)- existing.complexity_estimate = getattr(sym, "complexity_estimate", 0)- existing.language = getattr(sym, "language", "")- existing.parent_name = getattr(sym, "parent_name", None)- existing.updated_at = _now_utc()- else:- session.add(- WikiSymbol(- id=_new_uuid(),- repository_id=repository_id,- file_path=getattr(sym, "file_path", ""),- symbol_id=symbol_id,- name=sym.name,- qualified_name=getattr(sym, "qualified_name", sym.name),- kind=sym.kind,- signature=getattr(sym, "signature", ""),- start_line=getattr(sym, "start_line", 0),- end_line=getattr(sym, "end_line", 0),- docstring=getattr(sym, "docstring", None),- visibility=getattr(sym, "visibility", "public"),- is_async=getattr(sym, "is_async", False),- complexity_estimate=getattr(sym, "complexity_estimate", 0),- language=getattr(sym, "language", ""),- parent_name=getattr(sym, "parent_name", None),- )- )-- await session.flush()---# ----------------------------------------------------------------------------# WebhookEvent CRUD-# ------------------------------------------------------------------------------async def store_webhook_event(- session: AsyncSession,- *,- provider: str,- event_type: str,- payload: dict,- repository_id: str | None = None,- delivery_id: str = "",-) -> WebhookEvent:- """Append a new WebhookEvent record."""- event = WebhookEvent(- id=_new_uuid(),- repository_id=repository_id,- provider=provider,- event_type=event_type,- delivery_id=delivery_id,- payload_json=json.dumps(payload),- processed=False,- )- session.add(event)- await session.flush()- return event---async def mark_webhook_processed(- session: AsyncSession, event_id: str, *, job_id: str | None = None-) -> None:- """Mark a WebhookEvent as processed and optionally link it to a job."""- event = await session.get(WebhookEvent, event_id)- if event is None:- raise LookupError(f"No WebhookEvent with id={event_id!r}")- event.processed = True- if job_id is not None:- event.job_id = job_id- await session.flush()---# ----------------------------------------------------------------------------# GitMetadata CRUD-# ------------------------------------------------------------------------------async def upsert_git_metadata(- session: AsyncSession,- *,- repository_id: str,- file_path: str,- **kwargs: object,-) -> GitMetadata:- """Create or update a single GitMetadata row."""- result = await session.execute(- select(GitMetadata).where(- GitMetadata.repository_id == repository_id,- GitMetadata.file_path == file_path,- )- )- existing = result.scalar_one_or_none()-- if existing is not None:- for key, val in kwargs.items():- if hasattr(existing, key):- setattr(existing, key, val)- existing.updated_at = _now_utc()- else:- existing = GitMetadata(- id=_new_uuid(),- repository_id=repository_id,- file_path=file_path,- **{k: v for k, v in kwargs.items() if hasattr(GitMetadata, k)},- )- session.add(existing)-- await session.flush()- return existing---async def get_git_metadata(- session: AsyncSession, repository_id: str, file_path: str-) -> GitMetadata | None:- """Return GitMetadata for a specific file, or None."""- result = await session.execute(- select(GitMetadata).where(- GitMetadata.repository_id == repository_id,- GitMetadata.file_path == file_path,- )- )- return result.scalar_one_or_none()---async def get_git_metadata_bulk(- session: AsyncSession, repository_id: str, file_paths: list[str]-) -> dict[str, GitMetadata]:- """Return a dict of file_path → GitMetadata for the given paths."""- if not file_paths:- return {}- result = await session.execute(- select(GitMetadata).where(- GitMetadata.repository_id == repository_id,- GitMetadata.file_path.in_(file_paths),- )- )- return {gm.file_path: gm for gm in result.scalars().all()}---async def get_all_git_metadata(session: AsyncSession, repository_id: str) -> dict[str, GitMetadata]:- """Return all GitMetadata rows for a repository."""- result = await session.execute(- select(GitMetadata).where(GitMetadata.repository_id == repository_id)- )- return {gm.file_path: gm for gm in result.scalars().all()}---async def upsert_git_metadata_bulk(- session: AsyncSession,- repository_id: str,- metadata_list: list[dict],-) -> None:- """Bulk upsert git metadata rows in batches."""- for i in range(0, len(metadata_list), _BATCH_SIZE):- batch = metadata_list[i : i + _BATCH_SIZE]- for meta in batch:- file_path = meta.get("file_path", "")- result = await session.execute(- select(GitMetadata).where(- GitMetadata.repository_id == repository_id,- GitMetadata.file_path == file_path,- )- )- existing = result.scalar_one_or_none()-- if existing is not None:- for key, val in meta.items():- if key not in ("id", "repository_id") and hasattr(existing, key):- setattr(existing, key, val)- existing.updated_at = _now_utc()- else:- session.add(- GitMetadata(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in meta.items()- if k not in ("id", "repository_id") and hasattr(GitMetadata, k)- },- )- )- await session.flush()---async def recompute_git_percentiles(- session: AsyncSession,- repository_id: str,-) -> int:- """Recompute churn_percentile + is_hotspot using a SQL PERCENT_RANK window function.-- Called after incremental updates so that percentile rankings stay fresh- without a full ``repowise init``. Returns the number of rows updated.-- Primary ranking signal is temporal_hotspot_score (exponentially decayed churn);- commit_count_90d is the tiebreak. Works on both SQLite (3.25+) and PostgreSQL.- """- # First check how many rows exist so we can return the count without an- # extra query after the UPDATE.- count_result = await session.execute(- select(GitMetadata).where(GitMetadata.repository_id == repository_id)- )- rows = count_result.scalars().all()- if not rows:- return 0-- sql = """-WITH ranked AS (- SELECT id, PERCENT_RANK() OVER (- PARTITION BY repository_id- ORDER BY COALESCE(temporal_hotspot_score, 0.0), commit_count_90d- ) AS prank- FROM git_metadata- WHERE repository_id = :repo_id-)-UPDATE git_metadata-SET churn_percentile = (SELECT prank FROM ranked WHERE ranked.id = git_metadata.id),- is_hotspot = ((SELECT prank FROM ranked WHERE ranked.id = git_metadata.id) >= 0.75- AND git_metadata.commit_count_90d > 0)-WHERE repository_id = :repo_id;-"""- await session.execute(text(sql), {"repo_id": repository_id})- await session.flush()- return len(rows)---# ----------------------------------------------------------------------------# DeadCodeFinding CRUD-# ------------------------------------------------------------------------------async def save_dead_code_findings(- session: AsyncSession,- repository_id: str,- findings: list[dict],-) -> None:- """Persist dead code findings, replacing any existing open findings for the repo."""- # Delete existing open findings for this repo before saving new ones- existing = await session.execute(- select(DeadCodeFinding).where(- DeadCodeFinding.repository_id == repository_id,- DeadCodeFinding.status == "open",- )- )- for row in existing.scalars().all():- await session.delete(row)-- for i in range(0, len(findings), _BATCH_SIZE):- batch = findings[i : i + _BATCH_SIZE]- for finding in batch:- # Accept both DeadCodeFindingData-like objects and plain dicts- if hasattr(finding, "kind"):- data = {- "kind": str(finding.kind.value)- if hasattr(finding.kind, "value")- else str(finding.kind),- "file_path": finding.file_path,- "symbol_name": finding.symbol_name,- "symbol_kind": finding.symbol_kind,- "confidence": finding.confidence,- "reason": finding.reason,- "last_commit_at": finding.last_commit_at,- "commit_count_90d": finding.commit_count_90d,- "lines": finding.lines,- "package": finding.package,- "evidence_json": json.dumps(- finding.evidence if hasattr(finding, "evidence") else []- ),- "safe_to_delete": finding.safe_to_delete,- "primary_owner": finding.primary_owner,- "age_days": finding.age_days,- }- else:- data = dict(finding)- if "evidence" in data:- data["evidence_json"] = json.dumps(data.pop("evidence"))-- session.add(- DeadCodeFinding(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in data.items()- if k not in ("id", "repository_id") and hasattr(DeadCodeFinding, k)- },- )- )- await session.flush()---async def get_dead_code_findings(- session: AsyncSession,- repository_id: str,- *,- kind: str | None = None,- min_confidence: float = 0.0,- status: str = "open",-) -> list[DeadCodeFinding]:- """Return dead code findings filtered by kind, confidence, and status."""- q = select(DeadCodeFinding).where(- DeadCodeFinding.repository_id == repository_id,- DeadCodeFinding.status == status,- DeadCodeFinding.confidence >= min_confidence,- )- if kind is not None:- q = q.where(DeadCodeFinding.kind == kind)- q = q.order_by(DeadCodeFinding.confidence.desc())- result = await session.execute(q)- return list(result.scalars().all())---async def update_dead_code_status(- session: AsyncSession,- finding_id: str,- status: str,- note: str | None = None,-) -> DeadCodeFinding | None:- """Update the status (and optional note) of a dead code finding."""- finding = await session.get(DeadCodeFinding, finding_id)- if finding is None:- return None- finding.status = status- if note is not None:- finding.note = note- await session.flush()- return finding---async def get_dead_code_summary(session: AsyncSession, repository_id: str) -> dict:- """Return aggregate dead code statistics."""- result = await session.execute(- select(DeadCodeFinding).where(- DeadCodeFinding.repository_id == repository_id,- DeadCodeFinding.status == "open",- )- )- findings = list(result.scalars().all())-- summary: dict[str, int] = {"high": 0, "medium": 0, "low": 0}- total_lines = 0- by_kind: dict[str, int] = {}-- for f in findings:- if f.confidence >= 0.7:- summary["high"] += 1- elif f.confidence >= 0.4:- summary["medium"] += 1- else:- summary["low"] += 1- total_lines += f.lines- by_kind[f.kind] = by_kind.get(f.kind, 0) + 1-- return {- "total_findings": len(findings),- "confidence_summary": summary,- "deletable_lines": sum(f.lines for f in findings if f.safe_to_delete),- "total_lines": total_lines,- "by_kind": by_kind,- }---# ----------------------------------------------------------------------------# DecisionRecord CRUD-# -----------------------------------------------------------------------------_VALID_DECISION_STATUSES = frozenset({"proposed", "active", "deprecated", "superseded"})---async def upsert_decision(- session: AsyncSession,- *,- repository_id: str,- title: str,- status: str = "proposed",- context: str = "",- decision: str = "",- rationale: str = "",- alternatives: list[str] | None = None,- consequences: list[str] | None = None,- affected_files: list[str] | None = None,- affected_modules: list[str] | None = None,- tags: list[str] | None = None,- source: str = "cli",- evidence_commits: list[str] | None = None,- evidence_file: str | None = None,- evidence_line: int | None = None,- confidence: float = 1.0,- verification: str = "unverified",- last_code_change: datetime | None = None,- staleness_score: float = 0.0,- superseded_by: str | None = None,- decision_id: str | None = None,-) -> DecisionRecord:- """Create or update a decision record.-- Dedup key: ``(repository_id, title, source, evidence_file)``.- """- # Normalise text fields — LLM extractors may return explicit None- rationale = rationale or ""- context = context or ""- decision = decision or ""-- # Build the WHERE clause — evidence_file may be NULL- q = select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.title == title,- DecisionRecord.source == source,- )- if evidence_file is not None:- q = q.where(DecisionRecord.evidence_file == evidence_file)- else:- q = q.where(DecisionRecord.evidence_file.is_(None))-- result = await session.execute(q)- existing = result.scalar_one_or_none()-- if existing is not None:- existing.status = status- existing.context = context- existing.decision = decision- existing.rationale = rationale- existing.alternatives_json = json.dumps(alternatives or [])- existing.consequences_json = json.dumps(consequences or [])- existing.affected_files_json = json.dumps(affected_files or [])- existing.affected_modules_json = json.dumps(affected_modules or [])- existing.tags_json = json.dumps(tags or [])- existing.evidence_commits_json = json.dumps(evidence_commits or [])- existing.evidence_line = evidence_line- existing.confidence = confidence- existing.verification = verification- existing.last_code_change = last_code_change- existing.staleness_score = staleness_score- existing.superseded_by = superseded_by- existing.updated_at = _now_utc()- await session.flush()- return existing-- rec = DecisionRecord(- id=decision_id or _new_uuid(),- repository_id=repository_id,- title=title,- status=status,- context=context,- decision=decision,- rationale=rationale,- alternatives_json=json.dumps(alternatives or []),- consequences_json=json.dumps(consequences or []),- affected_files_json=json.dumps(affected_files or []),- affected_modules_json=json.dumps(affected_modules or []),- tags_json=json.dumps(tags or []),- evidence_commits_json=json.dumps(evidence_commits or []),- source=source,- evidence_file=evidence_file,- evidence_line=evidence_line,- confidence=confidence,- verification=verification,- last_code_change=last_code_change,- staleness_score=staleness_score,- superseded_by=superseded_by,- )- session.add(rec)- await session.flush()- return rec---async def get_decision(session: AsyncSession, decision_id: str) -> DecisionRecord | None:- """Return a DecisionRecord by primary key, or None."""- return await session.get(DecisionRecord, decision_id)---async def list_decisions(- session: AsyncSession,- repository_id: str,- *,- status: str | None = None,- source: str | None = None,- tag: str | None = None,- module: str | None = None,- include_proposed: bool = True,- limit: int = 100,- offset: int = 0,-) -> list[DecisionRecord]:- """Return decision records with optional filters."""- q = select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)- if status is not None:- q = q.where(DecisionRecord.status == status)- elif not include_proposed:- q = q.where(DecisionRecord.status != "proposed")- if source is not None:- q = q.where(DecisionRecord.source == source)- if tag is not None:- # Match exact tag value in JSON array, not substring.- # JSON arrays store as '["tag1", "tag2"]', so we match '"tag"'- q = q.where(DecisionRecord.tags_json.contains(f'"{tag}"'))- if module is not None:- # Match exact module path in JSON array- q = q.where(DecisionRecord.affected_modules_json.contains(f'"{module}"'))- q = q.order_by(DecisionRecord.created_at.desc()).limit(limit).offset(offset)- result = await session.execute(q)- return list(result.scalars().all())---async def update_decision_metadata(- session: AsyncSession,- decision_id: str,- *,- affected_modules: list[str] | None = None,- affected_files: list[str] | None = None,-) -> DecisionRecord | None:- """Patch the module/file linkage on a decision record.-- Each argument left as ``None`` is preserved. Pass an empty list to clear.- Returns the updated record, or ``None`` if the id was not found.- """- rec = await session.get(DecisionRecord, decision_id)- if rec is None:- return None- if affected_modules is not None:- rec.affected_modules_json = json.dumps(affected_modules)- if affected_files is not None:- rec.affected_files_json = json.dumps(affected_files)- rec.updated_at = _now_utc()- await session.flush()- return rec---async def update_decision_status(- session: AsyncSession,- decision_id: str,- status: str,- *,- superseded_by: str | None = None,-) -> DecisionRecord | None:- """Update the status of a decision record.-- Raises ValueError for invalid statuses. Returns None if not found.- """- if status not in _VALID_DECISION_STATUSES:- raise ValueError(- f"Unknown decision status {status!r}. Valid values: {sorted(_VALID_DECISION_STATUSES)}"- )- rec = await session.get(DecisionRecord, decision_id)- if rec is None:- return None- rec.status = status- if superseded_by is not None:- rec.superseded_by = superseded_by- rec.updated_at = _now_utc()- await session.flush()- return rec---async def update_decision_by_id(- session: AsyncSession,- decision_id: str,- **fields: Any,-) -> DecisionRecord | None:- """Update content fields of a decision record by ID (partial update).-- Accepts keyword arguments for any updatable field:- title, context, decision, rationale, alternatives, consequences,- affected_files, affected_modules, tags, evidence_file, evidence_line,- confidence.-- JSON list fields (alternatives, consequences, affected_files,- affected_modules, tags) accept Python lists and are serialized to JSON.-- Returns None if the decision is not found.- """- rec = await session.get(DecisionRecord, decision_id)- if rec is None:- return None-- _json_fields = {- "alternatives": "alternatives_json",- "consequences": "consequences_json",- "affected_files": "affected_files_json",- "affected_modules": "affected_modules_json",- "tags": "tags_json",- }- _scalar_fields = {- "title",- "context",- "decision",- "rationale",- "evidence_file",- "evidence_line",- "confidence",- }-- for key, value in fields.items():- if key in _json_fields:- setattr(rec, _json_fields[key], json.dumps(value))- elif key in _scalar_fields:- setattr(rec, key, value)-- rec.updated_at = _now_utc()- await session.flush()- return rec---async def delete_decision(session: AsyncSession, decision_id: str) -> bool:- """Delete a decision record. Returns True if deleted, False if not found."""- rec = await session.get(DecisionRecord, decision_id)- if rec is None:- return False- await session.delete(rec)- await session.flush()- return True---def _normalize_title(title: str) -> str:- """Normalize a decision title for cross-source dedup comparison."""- import re as _re-- t = title.lower().strip()- t = _re.sub(r"[^a-z0-9\s]", "", t)- t = _re.sub(r"\s+", " ", t)- return t---def _evidence_quote(d: dict) -> str:- """Pick the verbatim span recorded as this evidence row's source quote.-- Prefers the LLM/parser-supplied ``source_quote``; falls back to the- decision/rationale text so an evidence row is never empty.- """- return (d.get("source_quote") or d.get("decision") or d.get("rationale") or "").strip()---async def _upsert_decision_evidence(- session: AsyncSession,- decision_id: str,- *,- source: str,- source_rank: int,- evidence_file: str | None,- evidence_line: int | None,- evidence_commit: str | None,- source_quote: str,- confidence: float,- verification: str,-) -> None:- """Insert or update one evidence row, idempotent on its natural key.-- Natural key is ``(decision_id, source, evidence_file, evidence_commit)``.- NULLs are matched explicitly (SQLite treats NULLs as distinct in a unique- constraint), so re-indexing the same source converges instead of- duplicating, while an incremental update adds genuinely new evidence.- """- q = select(DecisionEvidence).where(- DecisionEvidence.decision_id == decision_id,- DecisionEvidence.source == source,- )- q = (- q.where(DecisionEvidence.evidence_file == evidence_file)- if evidence_file is not None- else q.where(DecisionEvidence.evidence_file.is_(None))- )- q = (- q.where(DecisionEvidence.evidence_commit == evidence_commit)- if evidence_commit is not None- else q.where(DecisionEvidence.evidence_commit.is_(None))- )- existing = (await session.execute(q)).scalar_one_or_none()-- if existing is not None:- existing.source_rank = source_rank- existing.evidence_line = evidence_line- existing.source_quote = source_quote- existing.confidence = confidence- existing.verification = verification- return-- session.add(- DecisionEvidence(- decision_id=decision_id,- source=source,- source_rank=source_rank,- evidence_file=evidence_file,- evidence_line=evidence_line,- evidence_commit=evidence_commit,- source_quote=source_quote,- confidence=confidence,- verification=verification,- )- )---def _best_verification(values: list[str]) -> str:- """Reduce per-evidence verdicts to the strongest: exact > fuzzy > unverified."""- if "exact" in values:- return "exact"- if "fuzzy" in values:- return "fuzzy"- return "unverified"---async def list_decision_evidence(- session: AsyncSession,- decision_id: str,-) -> list[DecisionEvidence]:- """Return all evidence rows for a decision, highest source rank first."""- result = await session.execute(- select(DecisionEvidence)- .where(DecisionEvidence.decision_id == decision_id)- .order_by(DecisionEvidence.source_rank.desc(), DecisionEvidence.created_at.asc())- )- return list(result.scalars().all())---def _first_commit(d: dict) -> str | None:- commits = d.get("evidence_commits") or []- return commits[0] if commits else None---async def bulk_upsert_decisions(- session: AsyncSession,- repository_id: str,- decisions: list[dict],- *,- vector_store: Any | None = None,-) -> list[str]:- """Upsert decisions, accreting provenance instead of discarding losers.-- Decisions with near-identical normalized titles are merged into a single- :class:`DecisionRecord`; every contributing source becomes a- :class:`DecisionEvidence` row. The record's headline fields come from the- highest-``source_rank`` contributor, and its confidence is recomputed from- the best rank + the number of independently corroborating sources + the- strongest surviving verification verdict.-- Idempotent: a full re-index converges (evidence rows upsert on their- natural key); an incremental update adds new evidence and re-derives the- headline + confidence from the union.-- When *vector_store* (the shared page-generator store) is supplied, a- Phase-2C semantic pass augments the cheap normalized-title match: an- incoming group that matched no existing record by title is looked up in the- store, and if its nearest ``decision:`` neighbour clears the cosine- threshold it is folded into that record as additional evidence — so- paraphrases ("Use Redis" vs "Adopt Redis cache") collapse into one record.- Every touched record is (re-)embedded into the store, so decisions are- matchable next run *and* discoverable via ``search_codebase``.-- Returns the ids of every record touched (created or updated) this call, so- a caller can run the Phase-3 supersession/conflict detection over just the- records that changed.- """- # Group incoming decisions by normalized title.- groups: dict[str, list[dict]] = {}- for d in decisions:- norm = _normalize_title(d.get("title", ""))- if not norm:- continue- groups.setdefault(norm, []).append(d)-- if not groups:- return []-- # Map existing records (this repo) by normalized title so cross-run merges- # land on the same row. On a title collision keep the most authoritative- # existing row as canonical.- existing_rows = await session.execute(- select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)- )- existing_by_norm: dict[str, DecisionRecord] = {}- for rec in existing_rows.scalars().all():- norm = _normalize_title(rec.title)- prior = existing_by_norm.get(norm)- if prior is None or rank_for_source(rec.source) > rank_for_source(prior.source):- existing_by_norm[norm] = rec-- # Phase 2C semantic dedup runs against the shared vector store. ``id_to_rec``- # lets a store hit (which returns a decision id) resolve back to the live- # record, and is grown as records are created so paraphrases *within* one- # batch also collapse.- id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}- if vector_store is not None:- from repowise.core.analysis.decision_semantic_match import (- find_duplicate_decision,- upsert_decision_vector,- )-- touched_ids: list[str] = []-- for norm, members in groups.items():- # Headline candidate: highest source rank, tie-break by confidence.- headline = max(- members,- key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),- )- rec = existing_by_norm.get(norm)-- # No title match → ask the store whether a semantically-equivalent- # decision already exists, and fold into it if so (cheap title dedup- # stays the first pass; this only runs on the residual).- if rec is None and vector_store is not None:- match_id = await find_duplicate_decision(- vector_store,- title=headline.get("title", ""),- decision=headline.get("decision") or "",- )- if match_id is not None and match_id in id_to_rec:- rec = id_to_rec[match_id]- existing_by_norm[norm] = rec-- if rec is None:- rec = DecisionRecord(- id=_new_uuid(),- repository_id=repository_id,- title=headline.get("title", ""),- status=headline.get("status", "proposed"),- context=headline.get("context") or "",- decision=headline.get("decision") or "",- rationale=headline.get("rationale") or "",- alternatives_json=json.dumps(headline.get("alternatives") or []),- consequences_json=json.dumps(headline.get("consequences") or []),- affected_files_json=json.dumps(headline.get("affected_files") or []),- affected_modules_json=json.dumps(headline.get("affected_modules") or []),- tags_json=json.dumps(headline.get("tags") or []),- evidence_commits_json=json.dumps(headline.get("evidence_commits") or []),- source=headline.get("source", "cli"),- evidence_file=headline.get("evidence_file"),- evidence_line=headline.get("evidence_line"),- confidence=headline.get("confidence", 0.5),- )- session.add(rec)- await session.flush()- existing_by_norm[norm] = rec- id_to_rec[rec.id] = rec- elif rank_for_source(headline.get("source", "")) >= rank_for_source(rec.source):- # A new contributor at least as authoritative as the current- # headline → promote its fields (provenance still accretes below).- rec.title = headline.get("title", rec.title)- rec.status = headline.get("status", rec.status)- rec.context = headline.get("context") or rec.context- rec.decision = headline.get("decision") or rec.decision- rec.rationale = headline.get("rationale") or rec.rationale- rec.alternatives_json = json.dumps(headline.get("alternatives") or [])- rec.consequences_json = json.dumps(headline.get("consequences") or [])- rec.affected_files_json = json.dumps(headline.get("affected_files") or [])- rec.affected_modules_json = json.dumps(headline.get("affected_modules") or [])- rec.tags_json = json.dumps(headline.get("tags") or [])- rec.evidence_commits_json = json.dumps(headline.get("evidence_commits") or [])- rec.source = headline.get("source", rec.source)- rec.evidence_file = headline.get("evidence_file")- rec.evidence_line = headline.get("evidence_line")-- # Accrete one evidence row per contributing source occurrence.- for d in members:- src = d.get("source", "cli")- await _upsert_decision_evidence(- session,- rec.id,- source=src,- source_rank=rank_for_source(src),- evidence_file=d.get("evidence_file"),- evidence_line=d.get("evidence_line"),- evidence_commit=_first_commit(d),- source_quote=_evidence_quote(d),- confidence=d.get("confidence", 0.5),- verification=d.get("verification", "unverified"),- )-- # Re-derive headline confidence + verification from the FULL evidence- # set (existing + just-added), so corroboration accrues across runs.- evidence = await list_decision_evidence(session, rec.id)- if evidence:- distinct_sources = {e.source for e in evidence}- top_rank = max(e.source_rank for e in evidence)- best_ver = _best_verification([e.verification for e in evidence])- rec.confidence = compute_confidence(top_rank, len(distinct_sources), best_ver)- rec.verification = best_ver- rec.updated_at = _now_utc()- touched_ids.append(rec.id)-- # Mirror the JSON file/module arrays into first-class decision→code- # links so the graph is traversable both directions (Phase 3A). The- # JSON stays the cheap read cache; these rows are the queryable truth.- await sync_decision_node_links(- session,- repository_id,- rec.id,- files=json.loads(rec.affected_files_json or "[]"),- modules=json.loads(rec.affected_modules_json or "[]"),- )-- # (Re-)embed the record into the shared store so it's matchable by- # later groups in this batch + future runs, and discoverable via- # search_codebase. Best-effort — never blocks the SQL upsert.- if vector_store is not None:- await upsert_decision_vector(- vector_store,- rec.id,- title=rec.title,- decision=rec.decision or "",- evidence_file=rec.evidence_file,- )-- await session.flush()- return touched_ids---async def recompute_decision_staleness(- session: AsyncSession,- repository_id: str,- git_meta_map: dict[str, dict],-) -> int:- """Recompute staleness_score for all active decisions. Returns update count."""- result = await session.execute(- select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.status.in_(["active", "proposed"]),- )- )- decisions = list(result.scalars().all())-- now = _now_utc()- updated = 0- for dec in decisions:- affected = json.loads(dec.affected_files_json)- if not affected:- continue-- from repowise.core.analysis.decision_extractor import DecisionExtractor-- decision_text = f"{dec.title} {dec.decision} {dec.rationale}"- new_score = DecisionExtractor.compute_staleness(- dec.created_at,- affected,- git_meta_map,- decision_text=decision_text,- )- if abs(new_score - dec.staleness_score) > 0.01:- dec.staleness_score = round(new_score, 3)- dec.updated_at = now- updated += 1-- if updated:- await session.flush()- return updated---async def get_stale_decisions(- session: AsyncSession,- repository_id: str,- threshold: float = 0.5,-) -> list[DecisionRecord]:- """Return active decisions with staleness_score >= threshold."""- result = await session.execute(- select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- DecisionRecord.status.in_(["active"]),- DecisionRecord.staleness_score >= threshold,- )- )- return list(result.scalars().all())---async def get_decision_health_summary(- session: AsyncSession,- repository_id: str,-) -> dict:- """Return decision health: counts by status, stale decisions, ungoverned hotspots."""- result = await session.execute(- select(DecisionRecord).where(- DecisionRecord.repository_id == repository_id,- )- )- all_decisions = list(result.scalars().all())-- counts = {"active": 0, "proposed": 0, "deprecated": 0, "superseded": 0, "stale": 0}- stale_decisions: list[DecisionRecord] = []- proposed_decisions: list[DecisionRecord] = []-- # Collect all governed files from active decisions- governed_files: set[str] = set()- for d in all_decisions:- counts[d.status] = counts.get(d.status, 0) + 1- if d.status == "active":- if d.staleness_score >= 0.5:- counts["stale"] += 1- stale_decisions.append(d)- for fp in json.loads(d.affected_files_json):- governed_files.add(fp)- elif d.status == "proposed":- proposed_decisions.append(d)-- # Find ungoverned hotspots- hotspot_result = await session.execute(- select(GitMetadata.file_path).where(- GitMetadata.repository_id == repository_id,- GitMetadata.is_hotspot == True, # noqa: E712- )- )- hotspot_files = {row[0] for row in hotspot_result.all()}- ungoverned = sorted(hotspot_files - governed_files)-- # Phase 3B: surface contradictory active decisions (conflicts_with edges).- from .decision_graph import list_conflict_edges-- by_id = {d.id: d for d in all_decisions}- conflicts: list[dict] = []- for edge in await list_conflict_edges(session, repository_id):- src = by_id.get(edge.src_decision_id)- dst = by_id.get(edge.dst_decision_id)- if src is None or dst is None:- continue- conflicts.append(- {- "src": {"id": src.id, "title": src.title, "status": src.status},- "dst": {"id": dst.id, "title": dst.title, "status": dst.status},- "confidence": edge.confidence,- "evidence": edge.evidence,- }- )- counts["conflicts"] = len(conflicts)-- return {- "summary": counts,- "stale_decisions": stale_decisions,- "proposed_awaiting_review": proposed_decisions,- "ungoverned_hotspots": ungoverned,- "conflicts": conflicts,- }---# ----------------------------------------------------------------------------# Conversation CRUD-# ------------------------------------------------------------------------------async def create_conversation(- session: AsyncSession,- *,- repository_id: str,- title: str = "New conversation",-) -> Conversation:- conv = Conversation(repository_id=repository_id, title=title)- session.add(conv)- await session.flush()- return conv---async def get_conversation(session: AsyncSession, conversation_id: str) -> Conversation | None:- return await session.get(Conversation, conversation_id)---async def list_conversations(- session: AsyncSession, repository_id: str, *, limit: int = 50-) -> list[Conversation]:- result = await session.execute(- select(Conversation)- .where(Conversation.repository_id == repository_id)- .order_by(Conversation.updated_at.desc())- .limit(limit)- )- return list(result.scalars().all())---async def update_conversation_title(- session: AsyncSession, conversation_id: str, title: str-) -> Conversation | None:- conv = await session.get(Conversation, conversation_id)- if conv:- conv.title = title- conv.updated_at = _now_utc()- await session.flush()- return conv---async def delete_conversation(session: AsyncSession, conversation_id: str) -> bool:- conv = await session.get(Conversation, conversation_id)- if conv is None:- return False- await session.delete(conv)- await session.flush()- return True---async def touch_conversation(session: AsyncSession, conversation_id: str) -> None:- """Update the updated_at timestamp of a conversation."""- conv = await session.get(Conversation, conversation_id)- if conv:- conv.updated_at = _now_utc()- await session.flush()---# ----------------------------------------------------------------------------# ChatMessage CRUD-# ------------------------------------------------------------------------------async def create_chat_message(- session: AsyncSession,- *,- conversation_id: str,- role: str,- content: dict,-) -> ChatMessage:- msg = ChatMessage(- conversation_id=conversation_id,- role=role,- content_json=json.dumps(content),- )- session.add(msg)- await session.flush()- return msg---async def list_chat_messages(session: AsyncSession, conversation_id: str) -> list[ChatMessage]:- result = await session.execute(- select(ChatMessage)- .where(ChatMessage.conversation_id == conversation_id)- .order_by(ChatMessage.created_at.asc())- )- return list(result.scalars().all())---async def count_chat_messages(session: AsyncSession, conversation_id: str) -> int:- result = await session.execute(- select(func.count())- .select_from(ChatMessage)- .where(ChatMessage.conversation_id == conversation_id)- )- return result.scalar() or 0---# ----------------------------------------------------------------------------# Graph read-side queries (Phase 5 — MCP graph tools)-# ------------------------------------------------------------------------------async def get_graph_node(- session: AsyncSession,- repository_id: str,- node_id: str,-) -> GraphNode | None:- """Look up a single GraphNode by its ``node_id`` (file path or symbol ID)."""- result = await session.execute(- select(GraphNode).where(- GraphNode.repository_id == repository_id,- GraphNode.node_id == node_id,- )- )- return result.scalar_one_or_none()---async def get_graph_edges_for_node(- session: AsyncSession,- repository_id: str,- node_id: str,- *,- direction: str = "both",- edge_types: list[str] | None = None,- limit: int = 50,-) -> list[GraphEdge]:- """Return edges adjacent to *node_id*.-- Parameters- ----------- direction:- ``"callers"`` → inbound edges (target == node_id),- ``"callees"`` → outbound edges (source == node_id),- ``"both"`` → union of both.- edge_types:- Optional filter, e.g. ``["calls"]`` or ``["extends", "implements"]``.- limit:- Max edges per direction.- """- results: list[GraphEdge] = []-- if direction in ("callers", "both"):- q = select(GraphEdge).where(- GraphEdge.repository_id == repository_id,- GraphEdge.target_node_id == node_id,- )- if edge_types:- q = q.where(GraphEdge.edge_type.in_(edge_types))- q = q.limit(limit)- res = await session.execute(q)- results.extend(res.scalars().all())-- if direction in ("callees", "both"):- q = select(GraphEdge).where(- GraphEdge.repository_id == repository_id,- GraphEdge.source_node_id == node_id,- )- if edge_types:- q = q.where(GraphEdge.edge_type.in_(edge_types))- q = q.limit(limit)- res = await session.execute(q)- results.extend(res.scalars().all())-- return results---async def get_graph_nodes_by_ids(- session: AsyncSession,- repository_id: str,- node_ids: list[str],-) -> dict[str, GraphNode]:- """Batch-lookup GraphNodes by node_id. Returns ``{node_id: GraphNode}``."""- if not node_ids:- return {}- # Process in batches to stay under SQLite parameter limits- out: dict[str, GraphNode] = {}- for i in range(0, len(node_ids), _BATCH_SIZE):- batch = node_ids[i : i + _BATCH_SIZE]- result = await session.execute(- select(GraphNode).where(- GraphNode.repository_id == repository_id,- GraphNode.node_id.in_(batch),- )- )- for node in result.scalars().all():- out[node.node_id] = node- return out---async def get_community_members(- session: AsyncSession,- repository_id: str,- community_id: int,- *,- node_type: str = "file",- limit: int = 50,-) -> list[GraphNode]:- """Return all nodes in a community, ordered by PageRank descending."""- result = await session.execute(- select(GraphNode)- .where(- GraphNode.repository_id == repository_id,- GraphNode.node_type == node_type,- GraphNode.community_id == community_id,- )- .order_by(GraphNode.pagerank.desc())- .limit(limit)- )- return list(result.scalars().all())---async def get_all_file_metrics(- session: AsyncSession,- repository_id: str,-) -> list[GraphNode]:- """Return all file-type GraphNodes (for percentile computation)."""- result = await session.execute(- select(GraphNode).where(- GraphNode.repository_id == repository_id,- GraphNode.node_type == "file",- )- )- return list(result.scalars().all())---async def get_cross_community_edges(- session: AsyncSession,- repository_id: str,- community_id: int,-) -> list[dict]:- """Count edges crossing from *community_id* to other communities.-- Returns a list of ``{"target_community_id": int, "edge_count": int}``.- Uses a join through ``graph_nodes`` to resolve target community.- """- # Alias for the target node lookup- target_node = GraphNode.__table__.alias("tn")- source_node = GraphNode.__table__.alias("sn")-- q = (- select(- target_node.c.community_id.label("target_community_id"),- func.count().label("edge_count"),- )- .select_from(GraphEdge.__table__)- .join(- source_node,- (GraphEdge.__table__.c.source_node_id == source_node.c.node_id)- & (GraphEdge.__table__.c.repository_id == source_node.c.repository_id),- )- .join(- target_node,- (GraphEdge.__table__.c.target_node_id == target_node.c.node_id)- & (GraphEdge.__table__.c.repository_id == target_node.c.repository_id),- )- .where(- GraphEdge.__table__.c.repository_id == repository_id,- source_node.c.community_id == community_id,- target_node.c.community_id != community_id,- # Only count file-level edges for meaningful community crossing- source_node.c.node_type == "file",- target_node.c.node_type == "file",- )- .group_by(target_node.c.community_id)- .order_by(func.count().desc())- )- result = await session.execute(q)- return [- {"target_community_id": row.target_community_id, "edge_count": row.edge_count}- for row in result.all()- ]---async def get_top_entry_points(- session: AsyncSession,- repository_id: str,- *,- min_score: float = 0.3,- limit: int = 20,-) -> list[GraphNode]:- """Return symbol nodes with stored entry_point_score >= *min_score*.-- Scores are stored inside ``community_meta_json``. Since the count of- symbol nodes is typically < 5000, an in-memory filter is acceptable.- """- result = await session.execute(- select(GraphNode).where(- GraphNode.repository_id == repository_id,- GraphNode.node_type == "symbol",- )- )- all_symbols = result.scalars().all()-- scored: list[tuple[float, GraphNode]] = []- for node in all_symbols:- try:- meta = json.loads(node.community_meta_json or "{}")- except (json.JSONDecodeError, TypeError):- continue- score = meta.get("entry_point_score")- if score is not None and score >= min_score:- scored.append((score, node))-- scored.sort(key=lambda x: x[0], reverse=True)- return [node for _, node in scored[:limit]]---async def get_node_degree_counts(- session: AsyncSession,- repository_id: str,- node_id: str,-) -> dict[str, int]:- """Return in-degree and out-degree for a node from edge counts."""- in_result = await session.execute(- select(func.count())- .select_from(GraphEdge)- .where(- GraphEdge.repository_id == repository_id,- GraphEdge.target_node_id == node_id,- )- )- out_result = await session.execute(- select(func.count())- .select_from(GraphEdge)- .where(- GraphEdge.repository_id == repository_id,- GraphEdge.source_node_id == node_id,- )- )- return {- "in_degree": in_result.scalar() or 0,- "out_degree": out_result.scalar() or 0,- }---# ----------------------------------------------------------------------------# Code Health CRUD-# ------------------------------------------------------------------------------async def save_health_findings(- session: AsyncSession,- repository_id: str,- findings: list[Any],-) -> None:- """Replace open health findings for *repository_id* with *findings*.-- Mirrors ``save_dead_code_findings`` — delete-then-insert. Accepts- either ``HealthFindingData`` dataclasses or plain dicts.- """- existing = await session.execute(- select(HealthFinding).where(- HealthFinding.repository_id == repository_id,- HealthFinding.status == "open",- )- )- for row in existing.scalars().all():- await session.delete(row)-- for i in range(0, len(findings), _BATCH_SIZE):- batch = findings[i : i + _BATCH_SIZE]- for f in batch:- if hasattr(f, "biomarker_type"):- severity = f.severity- severity_str = str(severity.value) if hasattr(severity, "value") else str(severity)- data = {- "file_path": f.file_path,- "biomarker_type": f.biomarker_type,- "severity": severity_str,- "function_name": f.function_name,- "line_start": f.line_start,- "line_end": f.line_end,- "details_json": json.dumps(f.details or {}),- "health_impact": float(f.health_impact),- "reason": f.reason or "",- }- else:- data = dict(f)- if "details" in data:- data["details_json"] = json.dumps(data.pop("details") or {})-- session.add(- HealthFinding(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in data.items()- if k not in ("id", "repository_id") and hasattr(HealthFinding, k)- },- )- )- await session.flush()---_GOVERNANCE_BIOMARKER_TYPES = frozenset(- {"ungoverned_hotspot", "stale_governance", "contradictory_decision"}-)---async def replace_governance_findings(- session: AsyncSession,- repository_id: str,- findings: list[Any],-) -> None:- """Idempotent additive write of governance-layer health findings.-- Deletes any existing ``health_findings`` rows whose ``biomarker_type``- is one of ``ungoverned_hotspot``, ``stale_governance``, or- ``contradictory_decision`` for *repository_id*, then inserts the new- *findings* in batches.-- This function deliberately does **not** recompute ``HealthFileMetric.score``- — that pass has already completed in the upstream health-analysis phase.- Governance findings surface through the findings layer (``get_risk``- ``top_biomarkers``, ``get_context`` health block) rather than the numeric- score. A second score-recomputation pass would require re-loading the full- per-file results table; the conservative choice is to leave scores as-is- and let findings carry the governance signal.-- Composable with ``save_health_findings``: the delete is scoped to only- the three governance biomarker types, so structural findings written by- ``save_health_findings`` are untouched.-- Accepts ``HealthFindingData`` dataclasses or plain dicts (same protocol- as ``save_health_findings``).- """- # Delete existing governance findings for this repo only.- existing = await session.execute(- select(HealthFinding).where(- HealthFinding.repository_id == repository_id,- HealthFinding.biomarker_type.in_(list(_GOVERNANCE_BIOMARKER_TYPES)),- )- )- for row in existing.scalars().all():- await session.delete(row)- await session.flush()-- if not findings:- return-- for i in range(0, len(findings), _BATCH_SIZE):- batch = findings[i : i + _BATCH_SIZE]- for f in batch:- if hasattr(f, "biomarker_type"):- severity = f.severity- severity_str = str(severity.value) if hasattr(severity, "value") else str(severity)- data = {- "file_path": f.file_path,- "biomarker_type": f.biomarker_type,- "severity": severity_str,- "function_name": f.function_name,- "line_start": f.line_start,- "line_end": f.line_end,- "details_json": json.dumps(f.details or {}),- "health_impact": float(f.health_impact),- "reason": f.reason or "",- }- else:- data = dict(f)- if "details" in data:- data["details_json"] = json.dumps(data.pop("details") or {})-- session.add(- HealthFinding(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in data.items()- if k not in ("id", "repository_id") and hasattr(HealthFinding, k)- },- )- )- await session.flush()---async def save_health_metrics(- session: AsyncSession,- repository_id: str,- metrics: list[Any],-) -> None:- """Replace per-file health metrics for *repository_id*.-- Delete-then-insert (matches the findings writer). The unique- constraint on (repository_id, file_path) means we cannot leave- stale rows around without an upsert dance — delete-and-insert keeps- it simple and aligns with how dead-code findings are written.- """- existing = await session.execute(- select(HealthFileMetric).where(HealthFileMetric.repository_id == repository_id)- )- for row in existing.scalars().all():- await session.delete(row)- await session.flush()-- for i in range(0, len(metrics), _BATCH_SIZE):- batch = metrics[i : i + _BATCH_SIZE]- for m in batch:- if hasattr(m, "file_path"):- data = {- "file_path": m.file_path,- "score": float(m.score),- "max_ccn": int(m.max_ccn),- "max_nesting": int(m.max_nesting),- "nloc": int(m.nloc),- "duplication_pct": m.duplication_pct,- "has_test_file": bool(m.has_test_file),- "line_coverage_pct": m.line_coverage_pct,- "branch_coverage_pct": m.branch_coverage_pct,- "module": m.module,- }- else:- data = dict(m)-- session.add(- HealthFileMetric(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in data.items()- if k not in ("id", "repository_id") and hasattr(HealthFileMetric, k)- },- )- )- await session.flush()---async def get_health_findings(- session: AsyncSession,- repository_id: str,- *,- biomarker_type: str | None = None,- min_severity: str | None = None,- file_path: str | None = None,- status: str = "open",-) -> list[HealthFinding]:- q = select(HealthFinding).where(- HealthFinding.repository_id == repository_id,- HealthFinding.status == status,- )- if biomarker_type is not None:- q = q.where(HealthFinding.biomarker_type == biomarker_type)- if file_path is not None:- q = q.where(HealthFinding.file_path == file_path)- if min_severity is not None:- # Severity order: low < medium < high < critical- order = {"low": 0, "medium": 1, "high": 2, "critical": 3}- threshold = order.get(min_severity, 0)- allowed = [k for k, v in order.items() if v >= threshold]- q = q.where(HealthFinding.severity.in_(allowed))- q = q.order_by(HealthFinding.health_impact.desc())- result = await session.execute(q)- return list(result.scalars().all())---async def get_health_metrics(- session: AsyncSession,- repository_id: str,- *,- file_paths: list[str] | None = None,-) -> list[HealthFileMetric]:- q = select(HealthFileMetric).where(HealthFileMetric.repository_id == repository_id)- if file_paths is not None:- q = q.where(HealthFileMetric.file_path.in_(file_paths))- q = q.order_by(HealthFileMetric.score.asc())- result = await session.execute(q)- return list(result.scalars().all())---async def get_health_summary(session: AsyncSession, repository_id: str) -> dict:- """Aggregate KPIs over the per-file metrics table."""- metrics = await get_health_metrics(session, repository_id)- if not metrics:- return {- "file_count": 0,- "average_health": 10.0,- "worst_performer_path": None,- "worst_performer_score": None,- "open_findings": 0,- }- total_nloc = sum(max(m.nloc, 1) for m in metrics)- if total_nloc:- avg = sum(m.score * max(m.nloc, 1) for m in metrics) / total_nloc- else:- avg = sum(m.score for m in metrics) / len(metrics)- worst = min(metrics, key=lambda r: r.score)- findings_count = await session.execute(- select(func.count())- .select_from(HealthFinding)- .where(- HealthFinding.repository_id == repository_id,- HealthFinding.status == "open",- )- )- return {- "file_count": len(metrics),- "average_health": round(avg, 2),- "worst_performer_path": worst.file_path,- "worst_performer_score": round(worst.score, 2),- "open_findings": findings_count.scalar() or 0,- }---async def update_health_finding_status(- session: AsyncSession,- finding_id: str,- status: str,-) -> HealthFinding | None:- f = await session.get(HealthFinding, finding_id)- if f is None:- return None- f.status = status- await session.flush()- return f---# Rolling history kept per repo. Older snapshots are deleted on insert.-# 50 entries gives Phase 4's `--trend` flag (last 10) plus the 5-back-# Declining-Health baseline plenty of headroom.-HEALTH_SNAPSHOT_RETENTION: int = 50---async def save_health_snapshot(- session: AsyncSession,- repository_id: str,- *,- hotspot_health: float,- average_health: float,- worst_performer_path: str | None,- worst_performer_score: float | None,- per_file_scores: dict[str, float] | None = None,- taken_at: datetime | None = None,-) -> HealthSnapshot:- """Append a snapshot; prune oldest rows past ``HEALTH_SNAPSHOT_RETENTION``.-- Returns the inserted row. Per-file scores are stored compactly as- ``{path: score}`` JSON (no per-finding detail — that lives in- ``HealthFinding`` rows; snapshots are a thin history layer).- """- snap = HealthSnapshot(- id=_new_uuid(),- repository_id=repository_id,- taken_at=taken_at or _now_utc(),- hotspot_health=float(hotspot_health),- average_health=float(average_health),- worst_performer_path=worst_performer_path,- worst_performer_score=(- float(worst_performer_score) if worst_performer_score is not None else None- ),- per_file_scores_json=json.dumps(per_file_scores or {}, separators=(",", ":")),- )- session.add(snap)- await session.flush()-- # Prune older-than-retention rows. We keep the *N* newest by- # ``taken_at``; ties are broken by id (UUIDs are random but stable).- rows = await session.execute(- select(HealthSnapshot)- .where(HealthSnapshot.repository_id == repository_id)- .order_by(HealthSnapshot.taken_at.desc(), HealthSnapshot.id.desc())- )- history = list(rows.scalars().all())- if len(history) > HEALTH_SNAPSHOT_RETENTION:- for row in history[HEALTH_SNAPSHOT_RETENTION:]:- await session.delete(row)- await session.flush()- return snap---async def list_health_snapshots(- session: AsyncSession,- repository_id: str,- *,- limit: int | None = None,-) -> list[HealthSnapshot]:- """Return snapshots **oldest-first** (the shape ``trends.diff_snapshots``- expects). Pass ``limit`` to cap the most recent N (still returned- oldest-first for stable iteration)."""- q = (- select(HealthSnapshot)- .where(HealthSnapshot.repository_id == repository_id)- .order_by(HealthSnapshot.taken_at.asc(), HealthSnapshot.id.asc())- )- result = await session.execute(q)- rows = list(result.scalars().all())- if limit is not None and len(rows) > limit:- rows = rows[-limit:]- return rows---async def upsert_health_findings(- session: AsyncSession,- repository_id: str,- findings: list[Any],- *,- file_paths: list[str],-) -> None:- """Replace open findings **only for the given file paths**.-- Used by the incremental ``repowise update`` path so unchanged files- keep their findings instead of being wiped on every partial re-index.- """- if not file_paths:- return- existing = await session.execute(- select(HealthFinding).where(- HealthFinding.repository_id == repository_id,- HealthFinding.status == "open",- HealthFinding.file_path.in_(file_paths),- )- )- for row in existing.scalars().all():- await session.delete(row)- await session.flush()-- for i in range(0, len(findings), _BATCH_SIZE):- batch = findings[i : i + _BATCH_SIZE]- for f in batch:- if hasattr(f, "biomarker_type"):- severity = f.severity- severity_str = str(severity.value) if hasattr(severity, "value") else str(severity)- data = {- "file_path": f.file_path,- "biomarker_type": f.biomarker_type,- "severity": severity_str,- "function_name": f.function_name,- "line_start": f.line_start,- "line_end": f.line_end,- "details_json": json.dumps(f.details or {}),- "health_impact": float(f.health_impact),- "reason": f.reason or "",- }- else:- data = dict(f)- if "details" in data:- data["details_json"] = json.dumps(data.pop("details") or {})-- session.add(- HealthFinding(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in data.items()- if k not in ("id", "repository_id") and hasattr(HealthFinding, k)- },- )- )- await session.flush()---async def upsert_health_metrics(- session: AsyncSession,- repository_id: str,- metrics: list[Any],-) -> None:- """Upsert per-file metrics; unchanged files in the table stay put.-- Sibling of ``save_health_metrics`` (which delete-then-inserts the- whole repo). Used by the incremental analysis path so a partial- re-index never wipes metric rows for files that weren't touched.- """- if not metrics:- return- paths = [m.file_path if hasattr(m, "file_path") else m["file_path"] for m in metrics]- existing = await session.execute(- select(HealthFileMetric).where(- HealthFileMetric.repository_id == repository_id,- HealthFileMetric.file_path.in_(paths),- )- )- by_path = {row.file_path: row for row in existing.scalars().all()}-- for m in metrics:- if hasattr(m, "file_path"):- data = {- "file_path": m.file_path,- "score": float(m.score),- "max_ccn": int(m.max_ccn),- "max_nesting": int(m.max_nesting),- "nloc": int(m.nloc),- "duplication_pct": m.duplication_pct,- "has_test_file": bool(m.has_test_file),- "line_coverage_pct": m.line_coverage_pct,- "branch_coverage_pct": m.branch_coverage_pct,- "module": m.module,- }- else:- data = dict(m)-- row = by_path.get(data["file_path"])- if row is not None:- for k, v in data.items():- if k in ("id", "repository_id") or not hasattr(HealthFileMetric, k):- continue- setattr(row, k, v)- else:- session.add(- HealthFileMetric(- id=_new_uuid(),- repository_id=repository_id,- **{- k: v- for k, v in data.items()- if k not in ("id", "repository_id") and hasattr(HealthFileMetric, k)- },- )- )- await session.flush()---# ----------------------------------------------------------------------------# Coverage CRUD-# ------------------------------------------------------------------------------async def save_coverage_files(- session: AsyncSession,- repository_id: str,- files: list[Any],- *,- source_format: str,- ingested_commit_sha: str | None = None,-) -> None:- """Replace coverage rows for *repository_id* with *files*.-- Mirrors the delete-then-insert pattern used by the health writers.- *files* is a list of ``FileCoverage`` dataclasses (or dicts with the- same shape).- """- existing = await session.execute(- select(CoverageFile).where(CoverageFile.repository_id == repository_id)- )- for row in existing.scalars().all():- await session.delete(row)- await session.flush()-- for i in range(0, len(files), _BATCH_SIZE):- batch = files[i : i + _BATCH_SIZE]- for f in batch:- if hasattr(f, "file_path"):- data = {- "file_path": f.file_path,- "line_coverage_pct": float(f.line_coverage_pct),- "branch_coverage_pct": (- float(f.branch_coverage_pct) if f.branch_coverage_pct is not None else None- ),- "covered_lines_json": json.dumps(list(f.covered_lines or [])),- "total_coverable_lines": int(f.total_coverable_lines or 0),- }- else:- data = dict(f)- if "covered_lines" in data:- data["covered_lines_json"] = json.dumps(list(data.pop("covered_lines") or []))-- session.add(- CoverageFile(- id=_new_uuid(),- repository_id=repository_id,- source_format=source_format,- ingested_commit_sha=ingested_commit_sha,- **{- k: v- for k, v in data.items()- if k- not in (- "id",- "repository_id",- "source_format",- "ingested_commit_sha",- )- and hasattr(CoverageFile, k)- },- )- )- await session.flush()---async def load_coverage_for_repo(- session: AsyncSession,- repository_id: str,- *,- file_paths: list[str] | None = None,-) -> list[CoverageFile]:- q = select(CoverageFile).where(CoverageFile.repository_id == repository_id)- if file_paths is not None:- q = q.where(CoverageFile.file_path.in_(file_paths))- result = await session.execute(q)- return list(result.scalars().all())---async def get_coverage_summary(- session: AsyncSession,- repository_id: str,-) -> dict[str, Any]:- """Repo-level coverage aggregate. Returns an empty shape when no rows."""- rows = await load_coverage_for_repo(session, repository_id)- if not rows:- return {- "file_count": 0,- "covered_lines": 0,- "total_lines": 0,- "line_coverage_pct": None,- "branch_coverage_pct": None,- "source_format": None,- "ingested_at": None,- "ingested_commit_sha": None,- }- covered = 0- total = 0- branch_pcts: list[float] = []- branch_weights: list[int] = []- for r in rows:- covered += round(r.line_coverage_pct / 100.0 * r.total_coverable_lines)- total += r.total_coverable_lines- if r.branch_coverage_pct is not None:- branch_pcts.append(r.branch_coverage_pct)- branch_weights.append(max(r.total_coverable_lines, 1))- line_pct = (covered / total * 100.0) if total else 0.0- branch_pct: float | None- if branch_pcts:- wsum = sum(branch_weights)- branch_pct = sum(p * w for p, w in zip(branch_pcts, branch_weights, strict=True)) / wsum- else:- branch_pct = None- latest = max(rows, key=lambda r: r.ingested_at)- return {- "file_count": len(rows),- "covered_lines": covered,- "total_lines": total,- "line_coverage_pct": round(line_pct, 2),- "branch_coverage_pct": round(branch_pct, 2) if branch_pct is not None else None,- "source_format": latest.source_format,- "ingested_at": latest.ingested_at,- "ingested_commit_sha": latest.ingested_commit_sha,- }---# ----------------------------------------------------------------------------# Knowledge Graph layers & tour steps-# ------------------------------------------------------------------------------async def upsert_kg_layers(session: AsyncSession, repo_id: str, layers: list[dict]) -> None:- """Replace all KG layers for a repo (delete + bulk insert)."""- await session.execute(- delete(KnowledgeGraphLayer).where(KnowledgeGraphLayer.repository_id == repo_id)- )- for i, layer in enumerate(layers):- session.add(- KnowledgeGraphLayer(- repository_id=repo_id,- layer_id=layer["id"],- name=layer["name"],- description=layer.get("description", ""),- node_ids_json=json.dumps(layer.get("nodeIds", layer.get("node_ids", []))),- display_order=layer.get("display_order", i),- )- )- await session.flush()---async def get_kg_layers(session: AsyncSession, repo_id: str) -> list[KnowledgeGraphLayer]:- """Fetch all KG layers ordered by display_order."""- result = await session.execute(- select(KnowledgeGraphLayer)- .where(KnowledgeGraphLayer.repository_id == repo_id)- .order_by(KnowledgeGraphLayer.display_order)- )- return list(result.scalars())---async def upsert_kg_tour_steps(session: AsyncSession, repo_id: str, steps: list[dict]) -> None:- """Replace all KG tour steps for a repo (delete + bulk insert)."""- await session.execute(- delete(KnowledgeGraphTourStep).where(KnowledgeGraphTourStep.repository_id == repo_id)- )- for step in steps:- session.add(- KnowledgeGraphTourStep(- repository_id=repo_id,- step_order=step["order"],- title=step["title"],- description=step.get("description", ""),- node_ids_json=json.dumps(step.get("nodeIds", step.get("node_ids", []))),- )- )- await session.flush()---async def get_kg_tour_steps(session: AsyncSession, repo_id: str) -> list[KnowledgeGraphTourStep]:- """Fetch all KG tour steps ordered by step_order."""- result = await session.execute(- select(KnowledgeGraphTourStep)- .where(KnowledgeGraphTourStep.repository_id == repo_id)- .order_by(KnowledgeGraphTourStep.step_order)- )- return list(result.scalars())diff --git a/packages/core/src/repowise/core/persistence/crud/__init__.py b/packages/core/src/repowise/core/persistence/crud/__init__.pynew file mode 100644index 000000000..950535cf0--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/__init__.py@@ -0,0 +1,29 @@+"""Async CRUD operations for the repowise persistence layer.++All functions accept an AsyncSession as the first argument; the caller owns+transaction boundaries. Functions that complete a logical unit of work call+``await session.flush()`` to write changes to the transaction buffer — the+caller must ``await session.commit()`` (or use the ``get_session`` context+manager from database.py).++Versioning contract for upsert_page:+ First upsert → inserts Page (version=1). No PageVersion created.+ Second upsert → archives existing Page as a PageVersion, then updates Page+ in place (version increments). created_at is preserved.++This module is a façade: the implementations live in per-domain submodules+(repository, pages, graph, external_systems, git, analysis, decisions, chat,+knowledge_graph). Import names from here — sub-module layout may change.+"""++from __future__ import annotations++from .analysis import * # noqa: F403+from .chat import * # noqa: F403+from .decisions import * # noqa: F403+from .external_systems import * # noqa: F403+from .git import * # noqa: F403+from .graph import * # noqa: F403+from .knowledge_graph import * # noqa: F403+from .pages import * # noqa: F403+from .repository import * # noqa: F403diff --git a/packages/core/src/repowise/core/persistence/crud/_shared.py b/packages/core/src/repowise/core/persistence/crud/_shared.pynew file mode 100644index 000000000..617afcfe3--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/_shared.py@@ -0,0 +1,66 @@+"""Shared internals for the ``crud`` package.++Holds the datetime parser, batch-size constant, the job-status whitelist, and+the generic SELECT-then-INSERT/UPDATE loop (:func:`_batch_upsert`) that the+per-domain batch upserts delegate to. Private to the package.+"""++from __future__ import annotations++from collections.abc import Callable, Iterable+from datetime import UTC, datetime+from typing import Any++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++_VALID_JOB_STATUSES = frozenset({"pending", "running", "completed", "failed", "paused"})++_BATCH_SIZE = 500 # max rows per INSERT to stay under SQLite's parameter limit+++def _parse_dt(ts: str) -> datetime:+ """Parse an ISO-8601 UTC string to a timezone-aware datetime."""+ ts = ts.replace("Z", "+00:00")+ dt = datetime.fromisoformat(ts)+ if dt.tzinfo is None:+ dt = dt.replace(tzinfo=UTC)+ return dt+++async def _batch_upsert(+ session: AsyncSession,+ model: type[Any],+ items: Iterable[Any],+ *,+ key_fn: Callable[[Any], tuple[Any, ...]],+ update_fn: Callable[[Any, Any], None],+ insert_fn: Callable[[Any], Any],+ batch_size: int | None = None,+) -> None:+ """Generic SELECT-then-INSERT/UPDATE loop shared by the batch upserts.++ For each item, look up the existing row with the filter expressions from+ ``key_fn(item)``; if found, mutate it via ``update_fn(existing, item)``,+ otherwise ``session.add(insert_fn(item))``.++ Flushes once after the whole sequence, or once per ``batch_size`` chunk when+ *batch_size* is given (matching the SELECT-then-write semantics the callers+ relied on before this was extracted — including no flush at all on an empty+ batched sequence).+ """+ if batch_size is None:+ chunks: list[list[Any]] = [list(items)]+ else:+ materialized = list(items)+ chunks = [materialized[i : i + batch_size] for i in range(0, len(materialized), batch_size)]+ for chunk in chunks:+ for item in chunk:+ existing = (+ await session.execute(select(model).where(*key_fn(item)))+ ).scalar_one_or_none()+ if existing is not None:+ update_fn(existing, item)+ else:+ session.add(insert_fn(item))+ await session.flush()diff --git a/packages/core/src/repowise/core/persistence/crud/analysis.py b/packages/core/src/repowise/core/persistence/crud/analysis.pynew file mode 100644index 000000000..f5c1b982e--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/analysis.py@@ -0,0 +1,765 @@+"""CRUD operations for the analysis domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json+from datetime import datetime+from typing import Any++from sqlalchemy import func, select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ CoverageFile,+ DeadCodeFinding,+ HealthFileMetric,+ HealthFinding,+ HealthSnapshot,+ _new_uuid,+ _now_utc,+)+from ._shared import _BATCH_SIZE++# ---------------------------------------------------------------------------+# DeadCodeFinding CRUD+# ---------------------------------------------------------------------------+++async def save_dead_code_findings(+ session: AsyncSession,+ repository_id: str,+ findings: list[dict],+) -> None:+ """Persist dead code findings, replacing any existing open findings for the repo."""+ # Delete existing open findings for this repo before saving new ones+ existing = await session.execute(+ select(DeadCodeFinding).where(+ DeadCodeFinding.repository_id == repository_id,+ DeadCodeFinding.status == "open",+ )+ )+ for row in existing.scalars().all():+ await session.delete(row)++ for i in range(0, len(findings), _BATCH_SIZE):+ batch = findings[i : i + _BATCH_SIZE]+ for finding in batch:+ # Accept both DeadCodeFindingData-like objects and plain dicts+ if hasattr(finding, "kind"):+ data = {+ "kind": str(finding.kind.value)+ if hasattr(finding.kind, "value")+ else str(finding.kind),+ "file_path": finding.file_path,+ "symbol_name": finding.symbol_name,+ "symbol_kind": finding.symbol_kind,+ "confidence": finding.confidence,+ "reason": finding.reason,+ "last_commit_at": finding.last_commit_at,+ "commit_count_90d": finding.commit_count_90d,+ "lines": finding.lines,+ "package": finding.package,+ "evidence_json": json.dumps(+ finding.evidence if hasattr(finding, "evidence") else []+ ),+ "safe_to_delete": finding.safe_to_delete,+ "primary_owner": finding.primary_owner,+ "age_days": finding.age_days,+ }+ else:+ data = dict(finding)+ if "evidence" in data:+ data["evidence_json"] = json.dumps(data.pop("evidence"))++ session.add(+ DeadCodeFinding(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in data.items()+ if k not in ("id", "repository_id") and hasattr(DeadCodeFinding, k)+ },+ )+ )+ await session.flush()+++async def get_dead_code_findings(+ session: AsyncSession,+ repository_id: str,+ *,+ kind: str | None = None,+ min_confidence: float = 0.0,+ status: str = "open",+) -> list[DeadCodeFinding]:+ """Return dead code findings filtered by kind, confidence, and status."""+ q = select(DeadCodeFinding).where(+ DeadCodeFinding.repository_id == repository_id,+ DeadCodeFinding.status == status,+ DeadCodeFinding.confidence >= min_confidence,+ )+ if kind is not None:+ q = q.where(DeadCodeFinding.kind == kind)+ q = q.order_by(DeadCodeFinding.confidence.desc())+ result = await session.execute(q)+ return list(result.scalars().all())+++async def update_dead_code_status(+ session: AsyncSession,+ finding_id: str,+ status: str,+ note: str | None = None,+) -> DeadCodeFinding | None:+ """Update the status (and optional note) of a dead code finding."""+ finding = await session.get(DeadCodeFinding, finding_id)+ if finding is None:+ return None+ finding.status = status+ if note is not None:+ finding.note = note+ await session.flush()+ return finding+++async def get_dead_code_summary(session: AsyncSession, repository_id: str) -> dict:+ """Return aggregate dead code statistics."""+ result = await session.execute(+ select(DeadCodeFinding).where(+ DeadCodeFinding.repository_id == repository_id,+ DeadCodeFinding.status == "open",+ )+ )+ findings = list(result.scalars().all())++ summary: dict[str, int] = {"high": 0, "medium": 0, "low": 0}+ total_lines = 0+ by_kind: dict[str, int] = {}++ for f in findings:+ if f.confidence >= 0.7:+ summary["high"] += 1+ elif f.confidence >= 0.4:+ summary["medium"] += 1+ else:+ summary["low"] += 1+ total_lines += f.lines+ by_kind[f.kind] = by_kind.get(f.kind, 0) + 1++ return {+ "total_findings": len(findings),+ "confidence_summary": summary,+ "deletable_lines": sum(f.lines for f in findings if f.safe_to_delete),+ "total_lines": total_lines,+ "by_kind": by_kind,+ }+++# ---------------------------------------------------------------------------+# Code Health CRUD+# ---------------------------------------------------------------------------+++async def save_health_findings(+ session: AsyncSession,+ repository_id: str,+ findings: list[Any],+) -> None:+ """Replace open health findings for *repository_id* with *findings*.++ Mirrors ``save_dead_code_findings`` — delete-then-insert. Accepts+ either ``HealthFindingData`` dataclasses or plain dicts.+ """+ existing = await session.execute(+ select(HealthFinding).where(+ HealthFinding.repository_id == repository_id,+ HealthFinding.status == "open",+ )+ )+ for row in existing.scalars().all():+ await session.delete(row)++ for i in range(0, len(findings), _BATCH_SIZE):+ batch = findings[i : i + _BATCH_SIZE]+ for f in batch:+ if hasattr(f, "biomarker_type"):+ severity = f.severity+ severity_str = str(severity.value) if hasattr(severity, "value") else str(severity)+ data = {+ "file_path": f.file_path,+ "biomarker_type": f.biomarker_type,+ "severity": severity_str,+ "function_name": f.function_name,+ "line_start": f.line_start,+ "line_end": f.line_end,+ "details_json": json.dumps(f.details or {}),+ "health_impact": float(f.health_impact),+ "reason": f.reason or "",+ }+ else:+ data = dict(f)+ if "details" in data:+ data["details_json"] = json.dumps(data.pop("details") or {})++ session.add(+ HealthFinding(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in data.items()+ if k not in ("id", "repository_id") and hasattr(HealthFinding, k)+ },+ )+ )+ await session.flush()+++_GOVERNANCE_BIOMARKER_TYPES = frozenset(+ {"ungoverned_hotspot", "stale_governance", "contradictory_decision"}+)+++async def replace_governance_findings(+ session: AsyncSession,+ repository_id: str,+ findings: list[Any],+) -> None:+ """Idempotent additive write of governance-layer health findings.++ Deletes any existing ``health_findings`` rows whose ``biomarker_type``+ is one of ``ungoverned_hotspot``, ``stale_governance``, or+ ``contradictory_decision`` for *repository_id*, then inserts the new+ *findings* in batches.++ This function deliberately does **not** recompute ``HealthFileMetric.score``+ — that pass has already completed in the upstream health-analysis phase.+ Governance findings surface through the findings layer (``get_risk``+ ``top_biomarkers``, ``get_context`` health block) rather than the numeric+ score. A second score-recomputation pass would require re-loading the full+ per-file results table; the conservative choice is to leave scores as-is+ and let findings carry the governance signal.++ Composable with ``save_health_findings``: the delete is scoped to only+ the three governance biomarker types, so structural findings written by+ ``save_health_findings`` are untouched.++ Accepts ``HealthFindingData`` dataclasses or plain dicts (same protocol+ as ``save_health_findings``).+ """+ # Delete existing governance findings for this repo only.+ existing = await session.execute(+ select(HealthFinding).where(+ HealthFinding.repository_id == repository_id,+ HealthFinding.biomarker_type.in_(list(_GOVERNANCE_BIOMARKER_TYPES)),+ )+ )+ for row in existing.scalars().all():+ await session.delete(row)+ await session.flush()++ if not findings:+ return++ for i in range(0, len(findings), _BATCH_SIZE):+ batch = findings[i : i + _BATCH_SIZE]+ for f in batch:+ if hasattr(f, "biomarker_type"):+ severity = f.severity+ severity_str = str(severity.value) if hasattr(severity, "value") else str(severity)+ data = {+ "file_path": f.file_path,+ "biomarker_type": f.biomarker_type,+ "severity": severity_str,+ "function_name": f.function_name,+ "line_start": f.line_start,+ "line_end": f.line_end,+ "details_json": json.dumps(f.details or {}),+ "health_impact": float(f.health_impact),+ "reason": f.reason or "",+ }+ else:+ data = dict(f)+ if "details" in data:+ data["details_json"] = json.dumps(data.pop("details") or {})++ session.add(+ HealthFinding(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in data.items()+ if k not in ("id", "repository_id") and hasattr(HealthFinding, k)+ },+ )+ )+ await session.flush()+++async def save_health_metrics(+ session: AsyncSession,+ repository_id: str,+ metrics: list[Any],+) -> None:+ """Replace per-file health metrics for *repository_id*.++ Delete-then-insert (matches the findings writer). The unique+ constraint on (repository_id, file_path) means we cannot leave+ stale rows around without an upsert dance — delete-and-insert keeps+ it simple and aligns with how dead-code findings are written.+ """+ existing = await session.execute(+ select(HealthFileMetric).where(HealthFileMetric.repository_id == repository_id)+ )+ for row in existing.scalars().all():+ await session.delete(row)+ await session.flush()++ for i in range(0, len(metrics), _BATCH_SIZE):+ batch = metrics[i : i + _BATCH_SIZE]+ for m in batch:+ if hasattr(m, "file_path"):+ data = {+ "file_path": m.file_path,+ "score": float(m.score),+ "max_ccn": int(m.max_ccn),+ "max_nesting": int(m.max_nesting),+ "nloc": int(m.nloc),+ "duplication_pct": m.duplication_pct,+ "has_test_file": bool(m.has_test_file),+ "line_coverage_pct": m.line_coverage_pct,+ "branch_coverage_pct": m.branch_coverage_pct,+ "module": m.module,+ }+ else:+ data = dict(m)++ session.add(+ HealthFileMetric(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in data.items()+ if k not in ("id", "repository_id") and hasattr(HealthFileMetric, k)+ },+ )+ )+ await session.flush()+++async def get_health_findings(+ session: AsyncSession,+ repository_id: str,+ *,+ biomarker_type: str | None = None,+ min_severity: str | None = None,+ file_path: str | None = None,+ status: str = "open",+) -> list[HealthFinding]:+ q = select(HealthFinding).where(+ HealthFinding.repository_id == repository_id,+ HealthFinding.status == status,+ )+ if biomarker_type is not None:+ q = q.where(HealthFinding.biomarker_type == biomarker_type)+ if file_path is not None:+ q = q.where(HealthFinding.file_path == file_path)+ if min_severity is not None:+ # Severity order: low < medium < high < critical+ order = {"low": 0, "medium": 1, "high": 2, "critical": 3}+ threshold = order.get(min_severity, 0)+ allowed = [k for k, v in order.items() if v >= threshold]+ q = q.where(HealthFinding.severity.in_(allowed))+ q = q.order_by(HealthFinding.health_impact.desc())+ result = await session.execute(q)+ return list(result.scalars().all())+++async def get_health_metrics(+ session: AsyncSession,+ repository_id: str,+ *,+ file_paths: list[str] | None = None,+) -> list[HealthFileMetric]:+ q = select(HealthFileMetric).where(HealthFileMetric.repository_id == repository_id)+ if file_paths is not None:+ q = q.where(HealthFileMetric.file_path.in_(file_paths))+ q = q.order_by(HealthFileMetric.score.asc())+ result = await session.execute(q)+ return list(result.scalars().all())+++async def get_health_summary(session: AsyncSession, repository_id: str) -> dict:+ """Aggregate KPIs over the per-file metrics table."""+ metrics = await get_health_metrics(session, repository_id)+ if not metrics:+ return {+ "file_count": 0,+ "average_health": 10.0,+ "worst_performer_path": None,+ "worst_performer_score": None,+ "open_findings": 0,+ }+ total_nloc = sum(max(m.nloc, 1) for m in metrics)+ if total_nloc:+ avg = sum(m.score * max(m.nloc, 1) for m in metrics) / total_nloc+ else:+ avg = sum(m.score for m in metrics) / len(metrics)+ worst = min(metrics, key=lambda r: r.score)+ findings_count = await session.execute(+ select(func.count())+ .select_from(HealthFinding)+ .where(+ HealthFinding.repository_id == repository_id,+ HealthFinding.status == "open",+ )+ )+ return {+ "file_count": len(metrics),+ "average_health": round(avg, 2),+ "worst_performer_path": worst.file_path,+ "worst_performer_score": round(worst.score, 2),+ "open_findings": findings_count.scalar() or 0,+ }+++async def update_health_finding_status(+ session: AsyncSession,+ finding_id: str,+ status: str,+) -> HealthFinding | None:+ f = await session.get(HealthFinding, finding_id)+ if f is None:+ return None+ f.status = status+ await session.flush()+ return f+++# Rolling history kept per repo. Older snapshots are deleted on insert.+# 50 entries gives Phase 4's `--trend` flag (last 10) plus the 5-back+# Declining-Health baseline plenty of headroom.+HEALTH_SNAPSHOT_RETENTION: int = 50+++async def save_health_snapshot(+ session: AsyncSession,+ repository_id: str,+ *,+ hotspot_health: float,+ average_health: float,+ worst_performer_path: str | None,+ worst_performer_score: float | None,+ per_file_scores: dict[str, float] | None = None,+ taken_at: datetime | None = None,+) -> HealthSnapshot:+ """Append a snapshot; prune oldest rows past ``HEALTH_SNAPSHOT_RETENTION``.++ Returns the inserted row. Per-file scores are stored compactly as+ ``{path: score}`` JSON (no per-finding detail — that lives in+ ``HealthFinding`` rows; snapshots are a thin history layer).+ """+ snap = HealthSnapshot(+ id=_new_uuid(),+ repository_id=repository_id,+ taken_at=taken_at or _now_utc(),+ hotspot_health=float(hotspot_health),+ average_health=float(average_health),+ worst_performer_path=worst_performer_path,+ worst_performer_score=(+ float(worst_performer_score) if worst_performer_score is not None else None+ ),+ per_file_scores_json=json.dumps(per_file_scores or {}, separators=(",", ":")),+ )+ session.add(snap)+ await session.flush()++ # Prune older-than-retention rows. We keep the *N* newest by+ # ``taken_at``; ties are broken by id (UUIDs are random but stable).+ rows = await session.execute(+ select(HealthSnapshot)+ .where(HealthSnapshot.repository_id == repository_id)+ .order_by(HealthSnapshot.taken_at.desc(), HealthSnapshot.id.desc())+ )+ history = list(rows.scalars().all())+ if len(history) > HEALTH_SNAPSHOT_RETENTION:+ for row in history[HEALTH_SNAPSHOT_RETENTION:]:+ await session.delete(row)+ await session.flush()+ return snap+++async def list_health_snapshots(+ session: AsyncSession,+ repository_id: str,+ *,+ limit: int | None = None,+) -> list[HealthSnapshot]:+ """Return snapshots **oldest-first** (the shape ``trends.diff_snapshots``+ expects). Pass ``limit`` to cap the most recent N (still returned+ oldest-first for stable iteration)."""+ q = (+ select(HealthSnapshot)+ .where(HealthSnapshot.repository_id == repository_id)+ .order_by(HealthSnapshot.taken_at.asc(), HealthSnapshot.id.asc())+ )+ result = await session.execute(q)+ rows = list(result.scalars().all())+ if limit is not None and len(rows) > limit:+ rows = rows[-limit:]+ return rows+++async def upsert_health_findings(+ session: AsyncSession,+ repository_id: str,+ findings: list[Any],+ *,+ file_paths: list[str],+) -> None:+ """Replace open findings **only for the given file paths**.++ Used by the incremental ``repowise update`` path so unchanged files+ keep their findings instead of being wiped on every partial re-index.+ """+ if not file_paths:+ return+ existing = await session.execute(+ select(HealthFinding).where(+ HealthFinding.repository_id == repository_id,+ HealthFinding.status == "open",+ HealthFinding.file_path.in_(file_paths),+ )+ )+ for row in existing.scalars().all():+ await session.delete(row)+ await session.flush()++ for i in range(0, len(findings), _BATCH_SIZE):+ batch = findings[i : i + _BATCH_SIZE]+ for f in batch:+ if hasattr(f, "biomarker_type"):+ severity = f.severity+ severity_str = str(severity.value) if hasattr(severity, "value") else str(severity)+ data = {+ "file_path": f.file_path,+ "biomarker_type": f.biomarker_type,+ "severity": severity_str,+ "function_name": f.function_name,+ "line_start": f.line_start,+ "line_end": f.line_end,+ "details_json": json.dumps(f.details or {}),+ "health_impact": float(f.health_impact),+ "reason": f.reason or "",+ }+ else:+ data = dict(f)+ if "details" in data:+ data["details_json"] = json.dumps(data.pop("details") or {})++ session.add(+ HealthFinding(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in data.items()+ if k not in ("id", "repository_id") and hasattr(HealthFinding, k)+ },+ )+ )+ await session.flush()+++async def upsert_health_metrics(+ session: AsyncSession,+ repository_id: str,+ metrics: list[Any],+) -> None:+ """Upsert per-file metrics; unchanged files in the table stay put.++ Sibling of ``save_health_metrics`` (which delete-then-inserts the+ whole repo). Used by the incremental analysis path so a partial+ re-index never wipes metric rows for files that weren't touched.+ """+ if not metrics:+ return+ paths = [m.file_path if hasattr(m, "file_path") else m["file_path"] for m in metrics]+ existing = await session.execute(+ select(HealthFileMetric).where(+ HealthFileMetric.repository_id == repository_id,+ HealthFileMetric.file_path.in_(paths),+ )+ )+ by_path = {row.file_path: row for row in existing.scalars().all()}++ for m in metrics:+ if hasattr(m, "file_path"):+ data = {+ "file_path": m.file_path,+ "score": float(m.score),+ "max_ccn": int(m.max_ccn),+ "max_nesting": int(m.max_nesting),+ "nloc": int(m.nloc),+ "duplication_pct": m.duplication_pct,+ "has_test_file": bool(m.has_test_file),+ "line_coverage_pct": m.line_coverage_pct,+ "branch_coverage_pct": m.branch_coverage_pct,+ "module": m.module,+ }+ else:+ data = dict(m)++ row = by_path.get(data["file_path"])+ if row is not None:+ for k, v in data.items():+ if k in ("id", "repository_id") or not hasattr(HealthFileMetric, k):+ continue+ setattr(row, k, v)+ else:+ session.add(+ HealthFileMetric(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in data.items()+ if k not in ("id", "repository_id") and hasattr(HealthFileMetric, k)+ },+ )+ )+ await session.flush()+++# ---------------------------------------------------------------------------+# Coverage CRUD+# ---------------------------------------------------------------------------+++async def save_coverage_files(+ session: AsyncSession,+ repository_id: str,+ files: list[Any],+ *,+ source_format: str,+ ingested_commit_sha: str | None = None,+) -> None:+ """Replace coverage rows for *repository_id* with *files*.++ Mirrors the delete-then-insert pattern used by the health writers.+ *files* is a list of ``FileCoverage`` dataclasses (or dicts with the+ same shape).+ """+ existing = await session.execute(+ select(CoverageFile).where(CoverageFile.repository_id == repository_id)+ )+ for row in existing.scalars().all():+ await session.delete(row)+ await session.flush()++ for i in range(0, len(files), _BATCH_SIZE):+ batch = files[i : i + _BATCH_SIZE]+ for f in batch:+ if hasattr(f, "file_path"):+ data = {+ "file_path": f.file_path,+ "line_coverage_pct": float(f.line_coverage_pct),+ "branch_coverage_pct": (+ float(f.branch_coverage_pct) if f.branch_coverage_pct is not None else None+ ),+ "covered_lines_json": json.dumps(list(f.covered_lines or [])),+ "total_coverable_lines": int(f.total_coverable_lines or 0),+ }+ else:+ data = dict(f)+ if "covered_lines" in data:+ data["covered_lines_json"] = json.dumps(list(data.pop("covered_lines") or []))++ session.add(+ CoverageFile(+ id=_new_uuid(),+ repository_id=repository_id,+ source_format=source_format,+ ingested_commit_sha=ingested_commit_sha,+ **{+ k: v+ for k, v in data.items()+ if k+ not in (+ "id",+ "repository_id",+ "source_format",+ "ingested_commit_sha",+ )+ and hasattr(CoverageFile, k)+ },+ )+ )+ await session.flush()+++async def load_coverage_for_repo(+ session: AsyncSession,+ repository_id: str,+ *,+ file_paths: list[str] | None = None,+) -> list[CoverageFile]:+ q = select(CoverageFile).where(CoverageFile.repository_id == repository_id)+ if file_paths is not None:+ q = q.where(CoverageFile.file_path.in_(file_paths))+ result = await session.execute(q)+ return list(result.scalars().all())+++async def get_coverage_summary(+ session: AsyncSession,+ repository_id: str,+) -> dict[str, Any]:+ """Repo-level coverage aggregate. Returns an empty shape when no rows."""+ rows = await load_coverage_for_repo(session, repository_id)+ if not rows:+ return {+ "file_count": 0,+ "covered_lines": 0,+ "total_lines": 0,+ "line_coverage_pct": None,+ "branch_coverage_pct": None,+ "source_format": None,+ "ingested_at": None,+ "ingested_commit_sha": None,+ }+ covered = 0+ total = 0+ branch_pcts: list[float] = []+ branch_weights: list[int] = []+ for r in rows:+ covered += round(r.line_coverage_pct / 100.0 * r.total_coverable_lines)+ total += r.total_coverable_lines+ if r.branch_coverage_pct is not None:+ branch_pcts.append(r.branch_coverage_pct)+ branch_weights.append(max(r.total_coverable_lines, 1))+ line_pct = (covered / total * 100.0) if total else 0.0+ branch_pct: float | None+ if branch_pcts:+ wsum = sum(branch_weights)+ branch_pct = sum(p * w for p, w in zip(branch_pcts, branch_weights, strict=True)) / wsum+ else:+ branch_pct = None+ latest = max(rows, key=lambda r: r.ingested_at)+ return {+ "file_count": len(rows),+ "covered_lines": covered,+ "total_lines": total,+ "line_coverage_pct": round(line_pct, 2),+ "branch_coverage_pct": round(branch_pct, 2) if branch_pct is not None else None,+ "source_format": latest.source_format,+ "ingested_at": latest.ingested_at,+ "ingested_commit_sha": latest.ingested_commit_sha,+ }diff --git a/packages/core/src/repowise/core/persistence/crud/chat.py b/packages/core/src/repowise/core/persistence/crud/chat.pynew file mode 100644index 000000000..d7fc30e5b--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/chat.py@@ -0,0 +1,118 @@+"""CRUD operations for the chat domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json++from sqlalchemy import func, select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ ChatMessage,+ Conversation,+ _now_utc,+)++# ---------------------------------------------------------------------------+# Conversation CRUD+# ---------------------------------------------------------------------------+++async def create_conversation(+ session: AsyncSession,+ *,+ repository_id: str,+ title: str = "New conversation",+) -> Conversation:+ conv = Conversation(repository_id=repository_id, title=title)+ session.add(conv)+ await session.flush()+ return conv+++async def get_conversation(session: AsyncSession, conversation_id: str) -> Conversation | None:+ return await session.get(Conversation, conversation_id)+++async def list_conversations(+ session: AsyncSession, repository_id: str, *, limit: int = 50+) -> list[Conversation]:+ result = await session.execute(+ select(Conversation)+ .where(Conversation.repository_id == repository_id)+ .order_by(Conversation.updated_at.desc())+ .limit(limit)+ )+ return list(result.scalars().all())+++async def update_conversation_title(+ session: AsyncSession, conversation_id: str, title: str+) -> Conversation | None:+ conv = await session.get(Conversation, conversation_id)+ if conv:+ conv.title = title+ conv.updated_at = _now_utc()+ await session.flush()+ return conv+++async def delete_conversation(session: AsyncSession, conversation_id: str) -> bool:+ conv = await session.get(Conversation, conversation_id)+ if conv is None:+ return False+ await session.delete(conv)+ await session.flush()+ return True+++async def touch_conversation(session: AsyncSession, conversation_id: str) -> None:+ """Update the updated_at timestamp of a conversation."""+ conv = await session.get(Conversation, conversation_id)+ if conv:+ conv.updated_at = _now_utc()+ await session.flush()+++# ---------------------------------------------------------------------------+# ChatMessage CRUD+# ---------------------------------------------------------------------------+++async def create_chat_message(+ session: AsyncSession,+ *,+ conversation_id: str,+ role: str,+ content: dict,+) -> ChatMessage:+ msg = ChatMessage(+ conversation_id=conversation_id,+ role=role,+ content_json=json.dumps(content),+ )+ session.add(msg)+ await session.flush()+ return msg+++async def list_chat_messages(session: AsyncSession, conversation_id: str) -> list[ChatMessage]:+ result = await session.execute(+ select(ChatMessage)+ .where(ChatMessage.conversation_id == conversation_id)+ .order_by(ChatMessage.created_at.asc())+ )+ return list(result.scalars().all())+++async def count_chat_messages(session: AsyncSession, conversation_id: str) -> int:+ result = await session.execute(+ select(func.count())+ .select_from(ChatMessage)+ .where(ChatMessage.conversation_id == conversation_id)+ )+ return result.scalar() or 0diff --git a/packages/core/src/repowise/core/persistence/crud/decisions.py b/packages/core/src/repowise/core/persistence/crud/decisions.pynew file mode 100644index 000000000..2c5460f09--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/decisions.py@@ -0,0 +1,691 @@+"""CRUD operations for the decisions domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json+from datetime import datetime+from typing import Any++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.decision_provenance import compute_confidence, rank_for_source++from ..decision_graph import sync_decision_node_links+from ..models import (+ DecisionEvidence,+ DecisionRecord,+ GitMetadata,+ _new_uuid,+ _now_utc,+)++# ---------------------------------------------------------------------------+# DecisionRecord CRUD+# ---------------------------------------------------------------------------++_VALID_DECISION_STATUSES = frozenset({"proposed", "active", "deprecated", "superseded"})+++async def upsert_decision(+ session: AsyncSession,+ *,+ repository_id: str,+ title: str,+ status: str = "proposed",+ context: str = "",+ decision: str = "",+ rationale: str = "",+ alternatives: list[str] | None = None,+ consequences: list[str] | None = None,+ affected_files: list[str] | None = None,+ affected_modules: list[str] | None = None,+ tags: list[str] | None = None,+ source: str = "cli",+ evidence_commits: list[str] | None = None,+ evidence_file: str | None = None,+ evidence_line: int | None = None,+ confidence: float = 1.0,+ verification: str = "unverified",+ last_code_change: datetime | None = None,+ staleness_score: float = 0.0,+ superseded_by: str | None = None,+ decision_id: str | None = None,+) -> DecisionRecord:+ """Create or update a decision record.++ Dedup key: ``(repository_id, title, source, evidence_file)``.+ """+ # Normalise text fields — LLM extractors may return explicit None+ rationale = rationale or ""+ context = context or ""+ decision = decision or ""++ # Build the WHERE clause — evidence_file may be NULL+ q = select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.title == title,+ DecisionRecord.source == source,+ )+ if evidence_file is not None:+ q = q.where(DecisionRecord.evidence_file == evidence_file)+ else:+ q = q.where(DecisionRecord.evidence_file.is_(None))++ result = await session.execute(q)+ existing = result.scalar_one_or_none()++ if existing is not None:+ existing.status = status+ existing.context = context+ existing.decision = decision+ existing.rationale = rationale+ existing.alternatives_json = json.dumps(alternatives or [])+ existing.consequences_json = json.dumps(consequences or [])+ existing.affected_files_json = json.dumps(affected_files or [])+ existing.affected_modules_json = json.dumps(affected_modules or [])+ existing.tags_json = json.dumps(tags or [])+ existing.evidence_commits_json = json.dumps(evidence_commits or [])+ existing.evidence_line = evidence_line+ existing.confidence = confidence+ existing.verification = verification+ existing.last_code_change = last_code_change+ existing.staleness_score = staleness_score+ existing.superseded_by = superseded_by+ existing.updated_at = _now_utc()+ await session.flush()+ return existing++ rec = DecisionRecord(+ id=decision_id or _new_uuid(),+ repository_id=repository_id,+ title=title,+ status=status,+ context=context,+ decision=decision,+ rationale=rationale,+ alternatives_json=json.dumps(alternatives or []),+ consequences_json=json.dumps(consequences or []),+ affected_files_json=json.dumps(affected_files or []),+ affected_modules_json=json.dumps(affected_modules or []),+ tags_json=json.dumps(tags or []),+ evidence_commits_json=json.dumps(evidence_commits or []),+ source=source,+ evidence_file=evidence_file,+ evidence_line=evidence_line,+ confidence=confidence,+ verification=verification,+ last_code_change=last_code_change,+ staleness_score=staleness_score,+ superseded_by=superseded_by,+ )+ session.add(rec)+ await session.flush()+ return rec+++async def get_decision(session: AsyncSession, decision_id: str) -> DecisionRecord | None:+ """Return a DecisionRecord by primary key, or None."""+ return await session.get(DecisionRecord, decision_id)+++async def list_decisions(+ session: AsyncSession,+ repository_id: str,+ *,+ status: str | None = None,+ source: str | None = None,+ tag: str | None = None,+ module: str | None = None,+ include_proposed: bool = True,+ limit: int = 100,+ offset: int = 0,+) -> list[DecisionRecord]:+ """Return decision records with optional filters."""+ q = select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)+ if status is not None:+ q = q.where(DecisionRecord.status == status)+ elif not include_proposed:+ q = q.where(DecisionRecord.status != "proposed")+ if source is not None:+ q = q.where(DecisionRecord.source == source)+ if tag is not None:+ # Match exact tag value in JSON array, not substring.+ # JSON arrays store as '["tag1", "tag2"]', so we match '"tag"'+ q = q.where(DecisionRecord.tags_json.contains(f'"{tag}"'))+ if module is not None:+ # Match exact module path in JSON array+ q = q.where(DecisionRecord.affected_modules_json.contains(f'"{module}"'))+ q = q.order_by(DecisionRecord.created_at.desc()).limit(limit).offset(offset)+ result = await session.execute(q)+ return list(result.scalars().all())+++async def update_decision_metadata(+ session: AsyncSession,+ decision_id: str,+ *,+ affected_modules: list[str] | None = None,+ affected_files: list[str] | None = None,+) -> DecisionRecord | None:+ """Patch the module/file linkage on a decision record.++ Each argument left as ``None`` is preserved. Pass an empty list to clear.+ Returns the updated record, or ``None`` if the id was not found.+ """+ rec = await session.get(DecisionRecord, decision_id)+ if rec is None:+ return None+ if affected_modules is not None:+ rec.affected_modules_json = json.dumps(affected_modules)+ if affected_files is not None:+ rec.affected_files_json = json.dumps(affected_files)+ rec.updated_at = _now_utc()+ await session.flush()+ return rec+++async def update_decision_status(+ session: AsyncSession,+ decision_id: str,+ status: str,+ *,+ superseded_by: str | None = None,+) -> DecisionRecord | None:+ """Update the status of a decision record.++ Raises ValueError for invalid statuses. Returns None if not found.+ """+ if status not in _VALID_DECISION_STATUSES:+ raise ValueError(+ f"Unknown decision status {status!r}. Valid values: {sorted(_VALID_DECISION_STATUSES)}"+ )+ rec = await session.get(DecisionRecord, decision_id)+ if rec is None:+ return None+ rec.status = status+ if superseded_by is not None:+ rec.superseded_by = superseded_by+ rec.updated_at = _now_utc()+ await session.flush()+ return rec+++async def update_decision_by_id(+ session: AsyncSession,+ decision_id: str,+ **fields: Any,+) -> DecisionRecord | None:+ """Update content fields of a decision record by ID (partial update).++ Accepts keyword arguments for any updatable field:+ title, context, decision, rationale, alternatives, consequences,+ affected_files, affected_modules, tags, evidence_file, evidence_line,+ confidence.++ JSON list fields (alternatives, consequences, affected_files,+ affected_modules, tags) accept Python lists and are serialized to JSON.++ Returns None if the decision is not found.+ """+ rec = await session.get(DecisionRecord, decision_id)+ if rec is None:+ return None++ _json_fields = {+ "alternatives": "alternatives_json",+ "consequences": "consequences_json",+ "affected_files": "affected_files_json",+ "affected_modules": "affected_modules_json",+ "tags": "tags_json",+ }+ _scalar_fields = {+ "title",+ "context",+ "decision",+ "rationale",+ "evidence_file",+ "evidence_line",+ "confidence",+ }++ for key, value in fields.items():+ if key in _json_fields:+ setattr(rec, _json_fields[key], json.dumps(value))+ elif key in _scalar_fields:+ setattr(rec, key, value)++ rec.updated_at = _now_utc()+ await session.flush()+ return rec+++async def delete_decision(session: AsyncSession, decision_id: str) -> bool:+ """Delete a decision record. Returns True if deleted, False if not found."""+ rec = await session.get(DecisionRecord, decision_id)+ if rec is None:+ return False+ await session.delete(rec)+ await session.flush()+ return True+++def _normalize_title(title: str) -> str:+ """Normalize a decision title for cross-source dedup comparison."""+ import re as _re++ t = title.lower().strip()+ t = _re.sub(r"[^a-z0-9\s]", "", t)+ t = _re.sub(r"\s+", " ", t)+ return t+++def _evidence_quote(d: dict) -> str:+ """Pick the verbatim span recorded as this evidence row's source quote.++ Prefers the LLM/parser-supplied ``source_quote``; falls back to the+ decision/rationale text so an evidence row is never empty.+ """+ return (d.get("source_quote") or d.get("decision") or d.get("rationale") or "").strip()+++async def _upsert_decision_evidence(+ session: AsyncSession,+ decision_id: str,+ *,+ source: str,+ source_rank: int,+ evidence_file: str | None,+ evidence_line: int | None,+ evidence_commit: str | None,+ source_quote: str,+ confidence: float,+ verification: str,+) -> None:+ """Insert or update one evidence row, idempotent on its natural key.++ Natural key is ``(decision_id, source, evidence_file, evidence_commit)``.+ NULLs are matched explicitly (SQLite treats NULLs as distinct in a unique+ constraint), so re-indexing the same source converges instead of+ duplicating, while an incremental update adds genuinely new evidence.+ """+ q = select(DecisionEvidence).where(+ DecisionEvidence.decision_id == decision_id,+ DecisionEvidence.source == source,+ )+ q = (+ q.where(DecisionEvidence.evidence_file == evidence_file)+ if evidence_file is not None+ else q.where(DecisionEvidence.evidence_file.is_(None))+ )+ q = (+ q.where(DecisionEvidence.evidence_commit == evidence_commit)+ if evidence_commit is not None+ else q.where(DecisionEvidence.evidence_commit.is_(None))+ )+ existing = (await session.execute(q)).scalar_one_or_none()++ if existing is not None:+ existing.source_rank = source_rank+ existing.evidence_line = evidence_line+ existing.source_quote = source_quote+ existing.confidence = confidence+ existing.verification = verification+ return++ session.add(+ DecisionEvidence(+ decision_id=decision_id,+ source=source,+ source_rank=source_rank,+ evidence_file=evidence_file,+ evidence_line=evidence_line,+ evidence_commit=evidence_commit,+ source_quote=source_quote,+ confidence=confidence,+ verification=verification,+ )+ )+++def _best_verification(values: list[str]) -> str:+ """Reduce per-evidence verdicts to the strongest: exact > fuzzy > unverified."""+ if "exact" in values:+ return "exact"+ if "fuzzy" in values:+ return "fuzzy"+ return "unverified"+++async def list_decision_evidence(+ session: AsyncSession,+ decision_id: str,+) -> list[DecisionEvidence]:+ """Return all evidence rows for a decision, highest source rank first."""+ result = await session.execute(+ select(DecisionEvidence)+ .where(DecisionEvidence.decision_id == decision_id)+ .order_by(DecisionEvidence.source_rank.desc(), DecisionEvidence.created_at.asc())+ )+ return list(result.scalars().all())+++def _first_commit(d: dict) -> str | None:+ commits = d.get("evidence_commits") or []+ return commits[0] if commits else None+++async def bulk_upsert_decisions(+ session: AsyncSession,+ repository_id: str,+ decisions: list[dict],+ *,+ vector_store: Any | None = None,+) -> list[str]:+ """Upsert decisions, accreting provenance instead of discarding losers.++ Decisions with near-identical normalized titles are merged into a single+ :class:`DecisionRecord`; every contributing source becomes a+ :class:`DecisionEvidence` row. The record's headline fields come from the+ highest-``source_rank`` contributor, and its confidence is recomputed from+ the best rank + the number of independently corroborating sources + the+ strongest surviving verification verdict.++ Idempotent: a full re-index converges (evidence rows upsert on their+ natural key); an incremental update adds new evidence and re-derives the+ headline + confidence from the union.++ When *vector_store* (the shared page-generator store) is supplied, a+ Phase-2C semantic pass augments the cheap normalized-title match: an+ incoming group that matched no existing record by title is looked up in the+ store, and if its nearest ``decision:`` neighbour clears the cosine+ threshold it is folded into that record as additional evidence — so+ paraphrases ("Use Redis" vs "Adopt Redis cache") collapse into one record.+ Every touched record is (re-)embedded into the store, so decisions are+ matchable next run *and* discoverable via ``search_codebase``.++ Returns the ids of every record touched (created or updated) this call, so+ a caller can run the Phase-3 supersession/conflict detection over just the+ records that changed.+ """+ # Group incoming decisions by normalized title.+ groups: dict[str, list[dict]] = {}+ for d in decisions:+ norm = _normalize_title(d.get("title", ""))+ if not norm:+ continue+ groups.setdefault(norm, []).append(d)++ if not groups:+ return []++ # Map existing records (this repo) by normalized title so cross-run merges+ # land on the same row. On a title collision keep the most authoritative+ # existing row as canonical.+ existing_rows = await session.execute(+ select(DecisionRecord).where(DecisionRecord.repository_id == repository_id)+ )+ existing_by_norm: dict[str, DecisionRecord] = {}+ for rec in existing_rows.scalars().all():+ norm = _normalize_title(rec.title)+ prior = existing_by_norm.get(norm)+ if prior is None or rank_for_source(rec.source) > rank_for_source(prior.source):+ existing_by_norm[norm] = rec++ # Phase 2C semantic dedup runs against the shared vector store. ``id_to_rec``+ # lets a store hit (which returns a decision id) resolve back to the live+ # record, and is grown as records are created so paraphrases *within* one+ # batch also collapse.+ id_to_rec: dict[str, DecisionRecord] = {rec.id: rec for rec in existing_by_norm.values()}+ if vector_store is not None:+ from repowise.core.analysis.decision_semantic_match import (+ find_duplicate_decision,+ upsert_decision_vector,+ )++ touched_ids: list[str] = []++ for norm, members in groups.items():+ # Headline candidate: highest source rank, tie-break by confidence.+ headline = max(+ members,+ key=lambda d: (rank_for_source(d.get("source", "")), d.get("confidence", 0.0)),+ )+ rec = existing_by_norm.get(norm)++ # No title match → ask the store whether a semantically-equivalent+ # decision already exists, and fold into it if so (cheap title dedup+ # stays the first pass; this only runs on the residual).+ if rec is None and vector_store is not None:+ match_id = await find_duplicate_decision(+ vector_store,+ title=headline.get("title", ""),+ decision=headline.get("decision") or "",+ )+ if match_id is not None and match_id in id_to_rec:+ rec = id_to_rec[match_id]+ existing_by_norm[norm] = rec++ if rec is None:+ rec = DecisionRecord(+ id=_new_uuid(),+ repository_id=repository_id,+ title=headline.get("title", ""),+ status=headline.get("status", "proposed"),+ context=headline.get("context") or "",+ decision=headline.get("decision") or "",+ rationale=headline.get("rationale") or "",+ alternatives_json=json.dumps(headline.get("alternatives") or []),+ consequences_json=json.dumps(headline.get("consequences") or []),+ affected_files_json=json.dumps(headline.get("affected_files") or []),+ affected_modules_json=json.dumps(headline.get("affected_modules") or []),+ tags_json=json.dumps(headline.get("tags") or []),+ evidence_commits_json=json.dumps(headline.get("evidence_commits") or []),+ source=headline.get("source", "cli"),+ evidence_file=headline.get("evidence_file"),+ evidence_line=headline.get("evidence_line"),+ confidence=headline.get("confidence", 0.5),+ )+ session.add(rec)+ await session.flush()+ existing_by_norm[norm] = rec+ id_to_rec[rec.id] = rec+ elif rank_for_source(headline.get("source", "")) >= rank_for_source(rec.source):+ # A new contributor at least as authoritative as the current+ # headline → promote its fields (provenance still accretes below).+ rec.title = headline.get("title", rec.title)+ rec.status = headline.get("status", rec.status)+ rec.context = headline.get("context") or rec.context+ rec.decision = headline.get("decision") or rec.decision+ rec.rationale = headline.get("rationale") or rec.rationale+ rec.alternatives_json = json.dumps(headline.get("alternatives") or [])+ rec.consequences_json = json.dumps(headline.get("consequences") or [])+ rec.affected_files_json = json.dumps(headline.get("affected_files") or [])+ rec.affected_modules_json = json.dumps(headline.get("affected_modules") or [])+ rec.tags_json = json.dumps(headline.get("tags") or [])+ rec.evidence_commits_json = json.dumps(headline.get("evidence_commits") or [])+ rec.source = headline.get("source", rec.source)+ rec.evidence_file = headline.get("evidence_file")+ rec.evidence_line = headline.get("evidence_line")++ # Accrete one evidence row per contributing source occurrence.+ for d in members:+ src = d.get("source", "cli")+ await _upsert_decision_evidence(+ session,+ rec.id,+ source=src,+ source_rank=rank_for_source(src),+ evidence_file=d.get("evidence_file"),+ evidence_line=d.get("evidence_line"),+ evidence_commit=_first_commit(d),+ source_quote=_evidence_quote(d),+ confidence=d.get("confidence", 0.5),+ verification=d.get("verification", "unverified"),+ )++ # Re-derive headline confidence + verification from the FULL evidence+ # set (existing + just-added), so corroboration accrues across runs.+ evidence = await list_decision_evidence(session, rec.id)+ if evidence:+ distinct_sources = {e.source for e in evidence}+ top_rank = max(e.source_rank for e in evidence)+ best_ver = _best_verification([e.verification for e in evidence])+ rec.confidence = compute_confidence(top_rank, len(distinct_sources), best_ver)+ rec.verification = best_ver+ rec.updated_at = _now_utc()+ touched_ids.append(rec.id)++ # Mirror the JSON file/module arrays into first-class decision→code+ # links so the graph is traversable both directions (Phase 3A). The+ # JSON stays the cheap read cache; these rows are the queryable truth.+ await sync_decision_node_links(+ session,+ repository_id,+ rec.id,+ files=json.loads(rec.affected_files_json or "[]"),+ modules=json.loads(rec.affected_modules_json or "[]"),+ )++ # (Re-)embed the record into the shared store so it's matchable by+ # later groups in this batch + future runs, and discoverable via+ # search_codebase. Best-effort — never blocks the SQL upsert.+ if vector_store is not None:+ await upsert_decision_vector(+ vector_store,+ rec.id,+ title=rec.title,+ decision=rec.decision or "",+ evidence_file=rec.evidence_file,+ )++ await session.flush()+ return touched_ids+++async def recompute_decision_staleness(+ session: AsyncSession,+ repository_id: str,+ git_meta_map: dict[str, dict],+) -> int:+ """Recompute staleness_score for all active decisions. Returns update count."""+ result = await session.execute(+ select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.status.in_(["active", "proposed"]),+ )+ )+ decisions = list(result.scalars().all())++ now = _now_utc()+ updated = 0+ for dec in decisions:+ affected = json.loads(dec.affected_files_json)+ if not affected:+ continue++ from repowise.core.analysis.decision_extractor import DecisionExtractor++ decision_text = f"{dec.title} {dec.decision} {dec.rationale}"+ new_score = DecisionExtractor.compute_staleness(+ dec.created_at,+ affected,+ git_meta_map,+ decision_text=decision_text,+ )+ if abs(new_score - dec.staleness_score) > 0.01:+ dec.staleness_score = round(new_score, 3)+ dec.updated_at = now+ updated += 1++ if updated:+ await session.flush()+ return updated+++async def get_stale_decisions(+ session: AsyncSession,+ repository_id: str,+ threshold: float = 0.5,+) -> list[DecisionRecord]:+ """Return active decisions with staleness_score >= threshold."""+ result = await session.execute(+ select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ DecisionRecord.status.in_(["active"]),+ DecisionRecord.staleness_score >= threshold,+ )+ )+ return list(result.scalars().all())+++async def get_decision_health_summary(+ session: AsyncSession,+ repository_id: str,+) -> dict:+ """Return decision health: counts by status, stale decisions, ungoverned hotspots."""+ result = await session.execute(+ select(DecisionRecord).where(+ DecisionRecord.repository_id == repository_id,+ )+ )+ all_decisions = list(result.scalars().all())++ counts = {"active": 0, "proposed": 0, "deprecated": 0, "superseded": 0, "stale": 0}+ stale_decisions: list[DecisionRecord] = []+ proposed_decisions: list[DecisionRecord] = []++ # Collect all governed files from active decisions+ governed_files: set[str] = set()+ for d in all_decisions:+ counts[d.status] = counts.get(d.status, 0) + 1+ if d.status == "active":+ if d.staleness_score >= 0.5:+ counts["stale"] += 1+ stale_decisions.append(d)+ for fp in json.loads(d.affected_files_json):+ governed_files.add(fp)+ elif d.status == "proposed":+ proposed_decisions.append(d)++ # Find ungoverned hotspots+ hotspot_result = await session.execute(+ select(GitMetadata.file_path).where(+ GitMetadata.repository_id == repository_id,+ GitMetadata.is_hotspot == True, # noqa: E712+ )+ )+ hotspot_files = {row[0] for row in hotspot_result.all()}+ ungoverned = sorted(hotspot_files - governed_files)++ # Phase 3B: surface contradictory active decisions (conflicts_with edges).+ from ..decision_graph import list_conflict_edges++ by_id = {d.id: d for d in all_decisions}+ conflicts: list[dict] = []+ for edge in await list_conflict_edges(session, repository_id):+ src = by_id.get(edge.src_decision_id)+ dst = by_id.get(edge.dst_decision_id)+ if src is None or dst is None:+ continue+ conflicts.append(+ {+ "src": {"id": src.id, "title": src.title, "status": src.status},+ "dst": {"id": dst.id, "title": dst.title, "status": dst.status},+ "confidence": edge.confidence,+ "evidence": edge.evidence,+ }+ )+ counts["conflicts"] = len(conflicts)++ return {+ "summary": counts,+ "stale_decisions": stale_decisions,+ "proposed_awaiting_review": proposed_decisions,+ "ungoverned_hotspots": ungoverned,+ "conflicts": conflicts,+ }diff --git a/packages/core/src/repowise/core/persistence/crud/external_systems.py b/packages/core/src/repowise/core/persistence/crud/external_systems.pynew file mode 100644index 000000000..97107fda7--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/external_systems.py@@ -0,0 +1,183 @@+"""CRUD operations for the external systems domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++from typing import Any++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ ExternalSystem,+ GraphNode,+ WikiSymbol,+ _new_uuid,+ _now_utc,+)+from ._shared import _batch_upsert++# ---------------------------------------------------------------------------+# ExternalSystem CRUD+# ---------------------------------------------------------------------------+++async def bulk_upsert_external_systems(+ session: AsyncSession,+ repository_id: str,+ systems: list[dict],+) -> dict[tuple[str, str], int]:+ """Upsert external systems for a repository.++ Each element of *systems* is a dict with keys matching ``ExternalSystem``+ columns (excluding ``id``, ``repository_id``, ``created_at``).++ Returns a mapping of ``(name, declared_in)`` → row ``id`` for both newly+ inserted and existing rows, so callers can link ``graph_nodes`` to the+ persisted external system without an extra round-trip.+ """+ id_map: dict[tuple[str, str], int] = {}+ for sys_data in systems:+ name = sys_data.get("name", "")+ declared_in = sys_data.get("declared_in", "")+ if not name:+ continue+ result = await session.execute(+ select(ExternalSystem).where(+ ExternalSystem.repository_id == repository_id,+ ExternalSystem.name == name,+ ExternalSystem.declared_in == declared_in,+ )+ )+ existing = result.scalar_one_or_none()+ if existing is not None:+ for key, val in sys_data.items():+ if key not in ("id", "repository_id", "created_at") and hasattr(existing, key):+ setattr(existing, key, val)+ id_map[(name, declared_in)] = existing.id+ else:+ row = ExternalSystem(+ repository_id=repository_id,+ **{k: v for k, v in sys_data.items() if k not in ("id", "repository_id")},+ )+ session.add(row)+ await session.flush()+ id_map[(name, declared_in)] = row.id+ await session.flush()+ return id_map+++async def link_graph_nodes_to_external_systems(+ session: AsyncSession,+ repository_id: str,+ name_to_id: dict[str, int],+) -> int:+ """Resolve ``external:{name}`` graph nodes to their ExternalSystem row.++ ``name_to_id`` should be a flat map of dep name → ExternalSystem id+ (collapse multi-manifest entries by picking any id — the C4 renderer+ only needs ``name``/``category`` which are the same across rows).++ Returns the number of graph_nodes updated.+ """+ if not name_to_id:+ return 0+ prefix = "external:"+ result = await session.execute(+ select(GraphNode).where(+ GraphNode.repository_id == repository_id,+ GraphNode.node_id.like(f"{prefix}%"),+ )+ )+ updated = 0+ for node in result.scalars():+ suffix = node.node_id[len(prefix) :]+ # Try the full suffix first, then the first segment (handles e.g.+ # ``external:fastapi.responses`` → ``fastapi``).+ sys_id = name_to_id.get(suffix)+ if sys_id is None and "." in suffix:+ sys_id = name_to_id.get(suffix.split(".", 1)[0])+ if sys_id is None and "/" in suffix:+ sys_id = name_to_id.get(suffix.split("/", 1)[0])+ if sys_id is not None and node.external_system_id != sys_id:+ node.external_system_id = sys_id+ updated += 1+ await session.flush()+ return updated+++async def list_external_systems(session: AsyncSession, repository_id: str) -> list[ExternalSystem]:+ """List all external systems for a repository, ordered by name."""+ result = await session.execute(+ select(ExternalSystem)+ .where(ExternalSystem.repository_id == repository_id)+ .order_by(ExternalSystem.name)+ )+ return list(result.scalars())+++# ---------------------------------------------------------------------------+# WikiSymbol CRUD (batch)+# ---------------------------------------------------------------------------+++def _symbol_id(sym: Any) -> str:+ return getattr(sym, "id", None) or f"{sym.file_path}::{sym.name}"+++def _update_wiki_symbol(existing: WikiSymbol, sym: Any) -> None:+ existing.name = sym.name+ existing.qualified_name = getattr(sym, "qualified_name", sym.name)+ existing.kind = sym.kind+ existing.signature = getattr(sym, "signature", "")+ existing.start_line = getattr(sym, "start_line", 0)+ existing.end_line = getattr(sym, "end_line", 0)+ existing.docstring = getattr(sym, "docstring", None)+ existing.visibility = getattr(sym, "visibility", "public")+ existing.is_async = getattr(sym, "is_async", False)+ existing.complexity_estimate = getattr(sym, "complexity_estimate", 0)+ existing.language = getattr(sym, "language", "")+ existing.parent_name = getattr(sym, "parent_name", None)+ existing.updated_at = _now_utc()+++async def batch_upsert_symbols(+ session: AsyncSession,+ repository_id: str,+ symbols: list, # list[ingestion.models.Symbol]+) -> None:+ """Upsert ingestion Symbol objects into the wiki_symbols table.++ Accepts ingestion.models.Symbol dataclass instances (duck-typed).+ """+ await _batch_upsert(+ session,+ WikiSymbol,+ symbols,+ key_fn=lambda sym: (+ WikiSymbol.repository_id == repository_id,+ WikiSymbol.symbol_id == _symbol_id(sym),+ ),+ update_fn=_update_wiki_symbol,+ insert_fn=lambda sym: WikiSymbol(+ id=_new_uuid(),+ repository_id=repository_id,+ file_path=getattr(sym, "file_path", ""),+ symbol_id=_symbol_id(sym),+ name=sym.name,+ qualified_name=getattr(sym, "qualified_name", sym.name),+ kind=sym.kind,+ signature=getattr(sym, "signature", ""),+ start_line=getattr(sym, "start_line", 0),+ end_line=getattr(sym, "end_line", 0),+ docstring=getattr(sym, "docstring", None),+ visibility=getattr(sym, "visibility", "public"),+ is_async=getattr(sym, "is_async", False),+ complexity_estimate=getattr(sym, "complexity_estimate", 0),+ language=getattr(sym, "language", ""),+ parent_name=getattr(sym, "parent_name", None),+ ),+ )diff --git a/packages/core/src/repowise/core/persistence/crud/git.py b/packages/core/src/repowise/core/persistence/crud/git.pynew file mode 100644index 000000000..fccbaea9c--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/git.py@@ -0,0 +1,167 @@+"""CRUD operations for the git domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++from sqlalchemy import select, text+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ GitMetadata,+ _new_uuid,+ _now_utc,+)+from ._shared import _BATCH_SIZE, _batch_upsert++# ---------------------------------------------------------------------------+# GitMetadata CRUD+# ---------------------------------------------------------------------------+++async def upsert_git_metadata(+ session: AsyncSession,+ *,+ repository_id: str,+ file_path: str,+ **kwargs: object,+) -> GitMetadata:+ """Create or update a single GitMetadata row."""+ result = await session.execute(+ select(GitMetadata).where(+ GitMetadata.repository_id == repository_id,+ GitMetadata.file_path == file_path,+ )+ )+ existing = result.scalar_one_or_none()++ if existing is not None:+ for key, val in kwargs.items():+ if hasattr(existing, key):+ setattr(existing, key, val)+ existing.updated_at = _now_utc()+ else:+ existing = GitMetadata(+ id=_new_uuid(),+ repository_id=repository_id,+ file_path=file_path,+ **{k: v for k, v in kwargs.items() if hasattr(GitMetadata, k)},+ )+ session.add(existing)++ await session.flush()+ return existing+++async def get_git_metadata(+ session: AsyncSession, repository_id: str, file_path: str+) -> GitMetadata | None:+ """Return GitMetadata for a specific file, or None."""+ result = await session.execute(+ select(GitMetadata).where(+ GitMetadata.repository_id == repository_id,+ GitMetadata.file_path == file_path,+ )+ )+ return result.scalar_one_or_none()+++async def get_git_metadata_bulk(+ session: AsyncSession, repository_id: str, file_paths: list[str]+) -> dict[str, GitMetadata]:+ """Return a dict of file_path → GitMetadata for the given paths."""+ if not file_paths:+ return {}+ result = await session.execute(+ select(GitMetadata).where(+ GitMetadata.repository_id == repository_id,+ GitMetadata.file_path.in_(file_paths),+ )+ )+ return {gm.file_path: gm for gm in result.scalars().all()}+++async def get_all_git_metadata(session: AsyncSession, repository_id: str) -> dict[str, GitMetadata]:+ """Return all GitMetadata rows for a repository."""+ result = await session.execute(+ select(GitMetadata).where(GitMetadata.repository_id == repository_id)+ )+ return {gm.file_path: gm for gm in result.scalars().all()}+++def _update_git_metadata(existing: GitMetadata, meta: dict) -> None:+ for key, val in meta.items():+ if key not in ("id", "repository_id") and hasattr(existing, key):+ setattr(existing, key, val)+ existing.updated_at = _now_utc()+++async def upsert_git_metadata_bulk(+ session: AsyncSession,+ repository_id: str,+ metadata_list: list[dict],+) -> None:+ """Bulk upsert git metadata rows in batches."""+ await _batch_upsert(+ session,+ GitMetadata,+ metadata_list,+ key_fn=lambda meta: (+ GitMetadata.repository_id == repository_id,+ GitMetadata.file_path == meta.get("file_path", ""),+ ),+ update_fn=_update_git_metadata,+ insert_fn=lambda meta: GitMetadata(+ id=_new_uuid(),+ repository_id=repository_id,+ **{+ k: v+ for k, v in meta.items()+ if k not in ("id", "repository_id") and hasattr(GitMetadata, k)+ },+ ),+ batch_size=_BATCH_SIZE,+ )+++async def recompute_git_percentiles(+ session: AsyncSession,+ repository_id: str,+) -> int:+ """Recompute churn_percentile + is_hotspot using a SQL PERCENT_RANK window function.++ Called after incremental updates so that percentile rankings stay fresh+ without a full ``repowise init``. Returns the number of rows updated.++ Primary ranking signal is temporal_hotspot_score (exponentially decayed churn);+ commit_count_90d is the tiebreak. Works on both SQLite (3.25+) and PostgreSQL.+ """+ # First check how many rows exist so we can return the count without an+ # extra query after the UPDATE.+ count_result = await session.execute(+ select(GitMetadata).where(GitMetadata.repository_id == repository_id)+ )+ rows = count_result.scalars().all()+ if not rows:+ return 0++ sql = """+WITH ranked AS (+ SELECT id, PERCENT_RANK() OVER (+ PARTITION BY repository_id+ ORDER BY COALESCE(temporal_hotspot_score, 0.0), commit_count_90d+ ) AS prank+ FROM git_metadata+ WHERE repository_id = :repo_id+)+UPDATE git_metadata+SET churn_percentile = (SELECT prank FROM ranked WHERE ranked.id = git_metadata.id),+ is_hotspot = ((SELECT prank FROM ranked WHERE ranked.id = git_metadata.id) >= 0.75+ AND git_metadata.commit_count_90d > 0)+WHERE repository_id = :repo_id;+"""+ await session.execute(text(sql), {"repo_id": repository_id})+ await session.flush()+ return len(rows)diff --git a/packages/core/src/repowise/core/persistence/crud/graph.py b/packages/core/src/repowise/core/persistence/crud/graph.pynew file mode 100644index 000000000..977d429a7--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/graph.py@@ -0,0 +1,474 @@+"""CRUD operations for the graph domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json++from sqlalchemy import func, select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ GraphEdge,+ GraphMetric,+ GraphNode,+ _new_uuid,+)+from ._shared import _BATCH_SIZE, _batch_upsert++# ---------------------------------------------------------------------------+# Graph CRUD (batch)+# ---------------------------------------------------------------------------++_METRIC_FIELDS = ("pagerank", "betweenness", "community_id", "in_degree", "out_degree")+++def _update_graph_node(existing: GraphNode, node_data: dict) -> None:+ for key, val in node_data.items():+ if key not in ("id", "repository_id", "created_at") and hasattr(existing, key):+ setattr(existing, key, val)+++def _update_graph_edge(existing: GraphEdge, edge_data: dict) -> None:+ imported = edge_data.get("imported_names_json")+ if imported is not None:+ existing.imported_names_json = imported+ confidence = edge_data.get("confidence")+ if confidence is not None:+ existing.confidence = confidence+++def _update_graph_metric(existing: GraphMetric, m: dict) -> None:+ for key in _METRIC_FIELDS:+ if key in m:+ setattr(existing, key, m[key])+++async def batch_upsert_graph_nodes(+ session: AsyncSession,+ repository_id: str,+ nodes: list[dict],+) -> None:+ """Upsert graph nodes for a repository in batches of up to 500.++ Each element of *nodes* is a dict with keys matching GraphNode fields+ (excluding id and repository_id which are set here).++ Uses SELECT-then-INSERT/UPDATE for dialect portability.+ """+ await _batch_upsert(+ session,+ GraphNode,+ nodes,+ key_fn=lambda n: (+ GraphNode.repository_id == repository_id,+ GraphNode.node_id == n.get("node_id", ""),+ ),+ update_fn=_update_graph_node,+ insert_fn=lambda n: GraphNode(+ id=_new_uuid(),+ repository_id=repository_id,+ **{k: v for k, v in n.items() if k not in ("id", "repository_id")},+ ),+ )+++async def batch_upsert_graph_edges(+ session: AsyncSession,+ repository_id: str,+ edges: list[dict],+) -> None:+ """Upsert graph edges for a repository.++ Each element of *edges* should have ``source_node_id``, ``target_node_id``,+ ``edge_type``, and optionally ``imported_names_json`` and ``confidence``.++ The unique constraint is (repository_id, source, target, edge_type),+ allowing multiple edge types between the same pair of nodes.+ """+ await _batch_upsert(+ session,+ GraphEdge,+ edges,+ key_fn=lambda e: (+ GraphEdge.repository_id == repository_id,+ GraphEdge.source_node_id == e.get("source_node_id", ""),+ GraphEdge.target_node_id == e.get("target_node_id", ""),+ GraphEdge.edge_type == e.get("edge_type", "imports"),+ ),+ update_fn=_update_graph_edge,+ insert_fn=lambda e: GraphEdge(+ id=_new_uuid(),+ repository_id=repository_id,+ source_node_id=e.get("source_node_id", ""),+ target_node_id=e.get("target_node_id", ""),+ imported_names_json=e.get("imported_names_json", "[]"),+ edge_type=e.get("edge_type", "imports"),+ confidence=e.get("confidence", 1.0),+ ),+ )+++async def batch_upsert_graph_metrics(+ session: AsyncSession,+ repository_id: str,+ metrics: dict[str, dict],+) -> None:+ """Materialize the file-level metrics snapshot into ``graph_metrics``.++ *metrics* maps ``node_id`` → a dict with ``pagerank``, ``betweenness``,+ ``community_id``, ``in_degree``, ``out_degree``. Additive to+ ``graph_nodes`` — this is the snapshot read back by+ ``GraphBuilder.load_metrics_from_sql`` on large repos. SELECT-then-write+ for dialect portability (SQLite + Postgres).+ """+ await _batch_upsert(+ session,+ GraphMetric,+ list(metrics.items()),+ key_fn=lambda kv: (+ GraphMetric.repository_id == repository_id,+ GraphMetric.node_id == kv[0],+ ),+ update_fn=lambda existing, kv: _update_graph_metric(existing, kv[1]),+ insert_fn=lambda kv: GraphMetric(+ id=_new_uuid(),+ repository_id=repository_id,+ node_id=kv[0],+ pagerank=float(kv[1].get("pagerank", 0.0)),+ betweenness=float(kv[1].get("betweenness", 0.0)),+ community_id=int(kv[1].get("community_id", 0)),+ in_degree=int(kv[1].get("in_degree", 0)),+ out_degree=int(kv[1].get("out_degree", 0)),+ ),+ )+++async def get_graph_metrics(+ session: AsyncSession,+ repository_id: str,+) -> dict[str, dict]:+ """Read the materialized ``graph_metrics`` snapshot as ``node_id → metrics``."""+ result = await session.execute(+ select(GraphMetric).where(GraphMetric.repository_id == repository_id)+ )+ return {+ row.node_id: {+ "pagerank": row.pagerank,+ "betweenness": row.betweenness,+ "community_id": row.community_id,+ "in_degree": row.in_degree,+ "out_degree": row.out_degree,+ }+ for row in result.scalars().all()+ }+++async def get_all_graph_nodes(+ session: AsyncSession,+ repository_id: str,+) -> list[dict]:+ """Read every persisted graph node as a list of plain dicts.++ Used to rehydrate an in-memory :class:`GraphBuilder` from SQL without+ re-parsing or re-resolving the graph (see+ ``repowise.core.pipeline.upgrade.rehydrate_graph_builder``). Each dict+ carries ``node_id`` plus the file/symbol attributes that the NetworkX node+ needs for traversal and rendering.+ """+ result = await session.execute(+ select(GraphNode).where(GraphNode.repository_id == repository_id)+ )+ return [+ {+ "node_id": row.node_id,+ "node_type": row.node_type,+ "language": row.language,+ "symbol_count": row.symbol_count,+ "has_error": row.has_error,+ "is_test": row.is_test,+ "is_entry_point": row.is_entry_point,+ "kind": row.kind,+ "name": row.name,+ "qualified_name": row.qualified_name,+ "file_path": row.file_path,+ "start_line": row.start_line,+ "end_line": row.end_line,+ "visibility": row.visibility,+ "signature": row.signature,+ "parent_symbol_id": row.parent_symbol_id,+ }+ for row in result.scalars().all()+ ]+++async def get_all_graph_edges(+ session: AsyncSession,+ repository_id: str,+) -> list[dict]:+ """Read every persisted graph edge as a list of plain dicts.++ Companion to :func:`get_all_graph_nodes` for graph rehydration. The+ ``imported_names_json`` column is decoded back into a list so the+ rehydrated edge matches the in-memory shape produced during ingestion.+ """+ result = await session.execute(+ select(GraphEdge).where(GraphEdge.repository_id == repository_id)+ )+ edges: list[dict] = []+ for row in result.scalars().all():+ try:+ imported_names = json.loads(row.imported_names_json or "[]")+ except (ValueError, TypeError):+ imported_names = []+ edges.append(+ {+ "source_node_id": row.source_node_id,+ "target_node_id": row.target_node_id,+ "edge_type": row.edge_type,+ "confidence": row.confidence,+ "imported_names": imported_names,+ }+ )+ return edges+++# ---------------------------------------------------------------------------+# Graph read-side queries (Phase 5 — MCP graph tools)+# ---------------------------------------------------------------------------+++async def get_graph_node(+ session: AsyncSession,+ repository_id: str,+ node_id: str,+) -> GraphNode | None:+ """Look up a single GraphNode by its ``node_id`` (file path or symbol ID)."""+ result = await session.execute(+ select(GraphNode).where(+ GraphNode.repository_id == repository_id,+ GraphNode.node_id == node_id,+ )+ )+ return result.scalar_one_or_none()+++async def get_graph_edges_for_node(+ session: AsyncSession,+ repository_id: str,+ node_id: str,+ *,+ direction: str = "both",+ edge_types: list[str] | None = None,+ limit: int = 50,+) -> list[GraphEdge]:+ """Return edges adjacent to *node_id*.++ Parameters+ ----------+ direction:+ ``"callers"`` → inbound edges (target == node_id),+ ``"callees"`` → outbound edges (source == node_id),+ ``"both"`` → union of both.+ edge_types:+ Optional filter, e.g. ``["calls"]`` or ``["extends", "implements"]``.+ limit:+ Max edges per direction.+ """+ results: list[GraphEdge] = []++ if direction in ("callers", "both"):+ q = select(GraphEdge).where(+ GraphEdge.repository_id == repository_id,+ GraphEdge.target_node_id == node_id,+ )+ if edge_types:+ q = q.where(GraphEdge.edge_type.in_(edge_types))+ q = q.limit(limit)+ res = await session.execute(q)+ results.extend(res.scalars().all())++ if direction in ("callees", "both"):+ q = select(GraphEdge).where(+ GraphEdge.repository_id == repository_id,+ GraphEdge.source_node_id == node_id,+ )+ if edge_types:+ q = q.where(GraphEdge.edge_type.in_(edge_types))+ q = q.limit(limit)+ res = await session.execute(q)+ results.extend(res.scalars().all())++ return results+++async def get_graph_nodes_by_ids(+ session: AsyncSession,+ repository_id: str,+ node_ids: list[str],+) -> dict[str, GraphNode]:+ """Batch-lookup GraphNodes by node_id. Returns ``{node_id: GraphNode}``."""+ if not node_ids:+ return {}+ # Process in batches to stay under SQLite parameter limits+ out: dict[str, GraphNode] = {}+ for i in range(0, len(node_ids), _BATCH_SIZE):+ batch = node_ids[i : i + _BATCH_SIZE]+ result = await session.execute(+ select(GraphNode).where(+ GraphNode.repository_id == repository_id,+ GraphNode.node_id.in_(batch),+ )+ )+ for node in result.scalars().all():+ out[node.node_id] = node+ return out+++async def get_community_members(+ session: AsyncSession,+ repository_id: str,+ community_id: int,+ *,+ node_type: str = "file",+ limit: int = 50,+) -> list[GraphNode]:+ """Return all nodes in a community, ordered by PageRank descending."""+ result = await session.execute(+ select(GraphNode)+ .where(+ GraphNode.repository_id == repository_id,+ GraphNode.node_type == node_type,+ GraphNode.community_id == community_id,+ )+ .order_by(GraphNode.pagerank.desc())+ .limit(limit)+ )+ return list(result.scalars().all())+++async def get_all_file_metrics(+ session: AsyncSession,+ repository_id: str,+) -> list[GraphNode]:+ """Return all file-type GraphNodes (for percentile computation)."""+ result = await session.execute(+ select(GraphNode).where(+ GraphNode.repository_id == repository_id,+ GraphNode.node_type == "file",+ )+ )+ return list(result.scalars().all())+++async def get_cross_community_edges(+ session: AsyncSession,+ repository_id: str,+ community_id: int,+) -> list[dict]:+ """Count edges crossing from *community_id* to other communities.++ Returns a list of ``{"target_community_id": int, "edge_count": int}``.+ Uses a join through ``graph_nodes`` to resolve target community.+ """+ # Alias for the target node lookup+ target_node = GraphNode.__table__.alias("tn")+ source_node = GraphNode.__table__.alias("sn")++ q = (+ select(+ target_node.c.community_id.label("target_community_id"),+ func.count().label("edge_count"),+ )+ .select_from(GraphEdge.__table__)+ .join(+ source_node,+ (GraphEdge.__table__.c.source_node_id == source_node.c.node_id)+ & (GraphEdge.__table__.c.repository_id == source_node.c.repository_id),+ )+ .join(+ target_node,+ (GraphEdge.__table__.c.target_node_id == target_node.c.node_id)+ & (GraphEdge.__table__.c.repository_id == target_node.c.repository_id),+ )+ .where(+ GraphEdge.__table__.c.repository_id == repository_id,+ source_node.c.community_id == community_id,+ target_node.c.community_id != community_id,+ # Only count file-level edges for meaningful community crossing+ source_node.c.node_type == "file",+ target_node.c.node_type == "file",+ )+ .group_by(target_node.c.community_id)+ .order_by(func.count().desc())+ )+ result = await session.execute(q)+ return [+ {"target_community_id": row.target_community_id, "edge_count": row.edge_count}+ for row in result.all()+ ]+++async def get_top_entry_points(+ session: AsyncSession,+ repository_id: str,+ *,+ min_score: float = 0.3,+ limit: int = 20,+) -> list[GraphNode]:+ """Return symbol nodes with stored entry_point_score >= *min_score*.++ Scores are stored inside ``community_meta_json``. Since the count of+ symbol nodes is typically < 5000, an in-memory filter is acceptable.+ """+ result = await session.execute(+ select(GraphNode).where(+ GraphNode.repository_id == repository_id,+ GraphNode.node_type == "symbol",+ )+ )+ all_symbols = result.scalars().all()++ scored: list[tuple[float, GraphNode]] = []+ for node in all_symbols:+ try:+ meta = json.loads(node.community_meta_json or "{}")+ except (json.JSONDecodeError, TypeError):+ continue+ score = meta.get("entry_point_score")+ if score is not None and score >= min_score:+ scored.append((score, node))++ scored.sort(key=lambda x: x[0], reverse=True)+ return [node for _, node in scored[:limit]]+++async def get_node_degree_counts(+ session: AsyncSession,+ repository_id: str,+ node_id: str,+) -> dict[str, int]:+ """Return in-degree and out-degree for a node from edge counts."""+ in_result = await session.execute(+ select(func.count())+ .select_from(GraphEdge)+ .where(+ GraphEdge.repository_id == repository_id,+ GraphEdge.target_node_id == node_id,+ )+ )+ out_result = await session.execute(+ select(func.count())+ .select_from(GraphEdge)+ .where(+ GraphEdge.repository_id == repository_id,+ GraphEdge.source_node_id == node_id,+ )+ )+ return {+ "in_degree": in_result.scalar() or 0,+ "out_degree": out_result.scalar() or 0,+ }diff --git a/packages/core/src/repowise/core/persistence/crud/knowledge_graph.py b/packages/core/src/repowise/core/persistence/crud/knowledge_graph.pynew file mode 100644index 000000000..4c3562d78--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/knowledge_graph.py@@ -0,0 +1,78 @@+"""CRUD operations for the knowledge graph domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json++from sqlalchemy import delete, select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ KnowledgeGraphLayer,+ KnowledgeGraphTourStep,+)++# ---------------------------------------------------------------------------+# Knowledge Graph layers & tour steps+# ---------------------------------------------------------------------------+++async def upsert_kg_layers(session: AsyncSession, repo_id: str, layers: list[dict]) -> None:+ """Replace all KG layers for a repo (delete + bulk insert)."""+ await session.execute(+ delete(KnowledgeGraphLayer).where(KnowledgeGraphLayer.repository_id == repo_id)+ )+ for i, layer in enumerate(layers):+ session.add(+ KnowledgeGraphLayer(+ repository_id=repo_id,+ layer_id=layer["id"],+ name=layer["name"],+ description=layer.get("description", ""),+ node_ids_json=json.dumps(layer.get("nodeIds", layer.get("node_ids", []))),+ display_order=layer.get("display_order", i),+ )+ )+ await session.flush()+++async def get_kg_layers(session: AsyncSession, repo_id: str) -> list[KnowledgeGraphLayer]:+ """Fetch all KG layers ordered by display_order."""+ result = await session.execute(+ select(KnowledgeGraphLayer)+ .where(KnowledgeGraphLayer.repository_id == repo_id)+ .order_by(KnowledgeGraphLayer.display_order)+ )+ return list(result.scalars())+++async def upsert_kg_tour_steps(session: AsyncSession, repo_id: str, steps: list[dict]) -> None:+ """Replace all KG tour steps for a repo (delete + bulk insert)."""+ await session.execute(+ delete(KnowledgeGraphTourStep).where(KnowledgeGraphTourStep.repository_id == repo_id)+ )+ for step in steps:+ session.add(+ KnowledgeGraphTourStep(+ repository_id=repo_id,+ step_order=step["order"],+ title=step["title"],+ description=step.get("description", ""),+ node_ids_json=json.dumps(step.get("nodeIds", step.get("node_ids", []))),+ )+ )+ await session.flush()+++async def get_kg_tour_steps(session: AsyncSession, repo_id: str) -> list[KnowledgeGraphTourStep]:+ """Fetch all KG tour steps ordered by step_order."""+ result = await session.execute(+ select(KnowledgeGraphTourStep)+ .where(KnowledgeGraphTourStep.repository_id == repo_id)+ .order_by(KnowledgeGraphTourStep.step_order)+ )+ return list(result.scalars())diff --git a/packages/core/src/repowise/core/persistence/crud/pages.py b/packages/core/src/repowise/core/persistence/crud/pages.pynew file mode 100644index 000000000..1d01a0078--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/pages.py@@ -0,0 +1,258 @@+"""CRUD operations for the pages domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json+from datetime import datetime+from typing import Any++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ Page,+ PageVersion,+ _new_uuid,+ _now_utc,+)+from ._shared import _parse_dt++# ---------------------------------------------------------------------------+# Page CRUD (with versioning)+# ---------------------------------------------------------------------------+++async def upsert_page(+ session: AsyncSession,+ *,+ page_id: str,+ repository_id: str,+ page_type: str,+ title: str,+ content: str,+ summary: str = "",+ target_path: str,+ source_hash: str,+ model_name: str,+ provider_name: str,+ input_tokens: int = 0,+ output_tokens: int = 0,+ cached_tokens: int = 0,+ generation_level: int = 0,+ confidence: float = 1.0,+ freshness_status: str = "fresh",+ metadata: dict | None = None,+ created_at: datetime | None = None,+ updated_at: datetime | None = None,+) -> Page:+ """Insert or update a wiki page, creating a PageVersion snapshot on update.++ First call → inserts Page at version=1.+ Subsequent → archives the current Page as a PageVersion, then updates the+ Page in-place (version += 1, created_at preserved).+ """+ now = _now_utc()+ page_created_at = created_at or now+ page_updated_at = updated_at or now+ meta_json = json.dumps(metadata or {})++ existing_result = await session.execute(select(Page).where(Page.id == page_id))+ existing = existing_result.scalar_one_or_none()++ if existing is not None:+ # Archive the current state before overwriting+ snapshot = PageVersion(+ id=_new_uuid(),+ page_id=existing.id,+ repository_id=existing.repository_id,+ version=existing.version,+ page_type=existing.page_type,+ title=existing.title,+ content=existing.content,+ source_hash=existing.source_hash,+ model_name=existing.model_name,+ provider_name=existing.provider_name,+ input_tokens=existing.input_tokens,+ output_tokens=existing.output_tokens,+ confidence=existing.confidence,+ archived_at=now,+ )+ session.add(snapshot)++ # Update Page in place (preserves created_at)+ existing.page_type = page_type+ existing.title = title+ existing.content = content+ existing.summary = summary+ existing.target_path = target_path+ existing.source_hash = source_hash+ existing.model_name = model_name+ existing.provider_name = provider_name+ existing.input_tokens = input_tokens+ existing.output_tokens = output_tokens+ existing.cached_tokens = cached_tokens+ existing.generation_level = generation_level+ existing.version = existing.version + 1+ existing.confidence = confidence+ existing.freshness_status = freshness_status+ existing.metadata_json = meta_json+ existing.updated_at = page_updated_at++ await session.flush()+ return existing+ else:+ page = Page(+ id=page_id,+ repository_id=repository_id,+ page_type=page_type,+ title=title,+ content=content,+ summary=summary,+ target_path=target_path,+ source_hash=source_hash,+ model_name=model_name,+ provider_name=provider_name,+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ cached_tokens=cached_tokens,+ generation_level=generation_level,+ version=1,+ confidence=confidence,+ freshness_status=freshness_status,+ metadata_json=meta_json,+ created_at=page_created_at,+ updated_at=page_updated_at,+ )+ session.add(page)+ await session.flush()+ return page+++async def load_prior_pages(+ session: AsyncSession,+ repository_id: str,+) -> dict[str, Any]:+ """Return a ``page_id → PriorPage`` map for cross-run cache reuse.++ Loads every existing wiki page for the repository so the generator can+ short-circuit the LLM call when the freshly rendered prompt produces a+ matching ``source_hash`` under the same model. Returns an empty dict if+ nothing has been generated yet.+ """+ # Import lazily — keeps persistence independent of generation models at+ # module-load time.+ from repowise.core.generation.page_generator import PriorPage++ result = await session.execute(select(Page).where(Page.repository_id == repository_id))+ prior: dict[str, Any] = {}+ for row in result.scalars():+ prior[row.id] = PriorPage(+ source_hash=row.source_hash,+ model_name=row.model_name,+ content=row.content,+ input_tokens=row.input_tokens,+ output_tokens=row.output_tokens,+ cached_tokens=row.cached_tokens,+ )+ return prior+++async def upsert_page_from_generated(+ session: AsyncSession,+ generated_page: object, # repowise.core.generation.models.GeneratedPage+ repository_id: str,+) -> Page:+ """Convenience wrapper that unpacks a GeneratedPage dataclass.++ This keeps the CRUD layer independent of the generation models at the+ import level while still providing a clean API for callers that have a+ GeneratedPage in hand.+ """+ gp = generated_page # type alias for brevity+ return await upsert_page(+ session,+ page_id=gp.page_id, # type: ignore[attr-defined]+ repository_id=repository_id,+ page_type=gp.page_type, # type: ignore[attr-defined]+ title=gp.title, # type: ignore[attr-defined]+ content=gp.content, # type: ignore[attr-defined]+ summary=getattr(gp, "summary", "") or "",+ target_path=gp.target_path, # type: ignore[attr-defined]+ source_hash=gp.source_hash, # type: ignore[attr-defined]+ model_name=gp.model_name, # type: ignore[attr-defined]+ provider_name=gp.provider_name, # type: ignore[attr-defined]+ input_tokens=gp.input_tokens, # type: ignore[attr-defined]+ output_tokens=gp.output_tokens, # type: ignore[attr-defined]+ cached_tokens=gp.cached_tokens, # type: ignore[attr-defined]+ generation_level=gp.generation_level, # type: ignore[attr-defined]+ confidence=gp.confidence, # type: ignore[attr-defined]+ freshness_status=gp.freshness_status, # type: ignore[attr-defined]+ metadata=gp.metadata, # type: ignore[attr-defined]+ created_at=_parse_dt(gp.created_at), # type: ignore[attr-defined]+ updated_at=_parse_dt(gp.updated_at), # type: ignore[attr-defined]+ )+++async def get_page(session: AsyncSession, page_id: str) -> Page | None:+ """Return a Page by its page_id, or None."""+ return await session.get(Page, page_id)+++async def list_pages(+ session: AsyncSession,+ repository_id: str,+ *,+ page_type: str | None = None,+ limit: int = 100,+ offset: int = 0,+ sort_by: str = "updated_at",+ order: str = "desc",+) -> list[Page]:+ """Return pages for a repository, optionally filtered by page_type."""+ q = select(Page).where(Page.repository_id == repository_id)+ if page_type is not None:+ q = q.where(Page.page_type == page_type)+ _sort_cols = {+ "updated_at": Page.updated_at,+ "confidence": Page.confidence,+ "created_at": Page.created_at,+ }+ sort_col = _sort_cols.get(sort_by, Page.updated_at)+ q = q.order_by(sort_col.asc() if order == "asc" else sort_col.desc())+ q = q.limit(limit).offset(offset)+ result = await session.execute(q)+ return list(result.scalars().all())+++async def get_page_versions(+ session: AsyncSession,+ page_id: str,+ *,+ limit: int = 50,+) -> list[PageVersion]:+ """Return historical versions of a page, newest first."""+ result = await session.execute(+ select(PageVersion)+ .where(PageVersion.page_id == page_id)+ .order_by(PageVersion.version.desc())+ .limit(limit)+ )+ return list(result.scalars().all())+++async def get_stale_pages(+ session: AsyncSession,+ repository_id: str,+) -> list[Page]:+ """Return pages with freshness_status in ('stale', 'expired')."""+ result = await session.execute(+ select(Page).where(+ Page.repository_id == repository_id,+ Page.freshness_status.in_(["stale", "expired"]),+ )+ )+ return list(result.scalars().all())diff --git a/packages/core/src/repowise/core/persistence/crud/repository.py b/packages/core/src/repowise/core/persistence/crud/repository.pynew file mode 100644index 000000000..ef148c908--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/repository.py@@ -0,0 +1,224 @@+"""CRUD operations for the repository domain (repowise persistence layer).++Split out of the former monolithic ``crud.py``; ``crud/__init__.py`` re-exports+every public name, so existing imports are unaffected.+"""++from __future__ import annotations++import json++from sqlalchemy import select+from sqlalchemy.ext.asyncio import AsyncSession++from ..models import (+ GenerationJob,+ Page,+ Repository,+ WebhookEvent,+ _new_uuid,+ _now_utc,+)+from ._shared import _VALID_JOB_STATUSES++# ---------------------------------------------------------------------------+# Repository CRUD+# ---------------------------------------------------------------------------+++async def upsert_repository(+ session: AsyncSession,+ *,+ name: str,+ local_path: str,+ url: str = "",+ default_branch: str = "main",+ settings: dict | None = None,+) -> Repository:+ """Create or update a repository record.++ Lookup is by ``local_path`` (the canonical key for local repositories).+ """+ result = await session.execute(select(Repository).where(Repository.local_path == local_path))+ repo = result.scalar_one_or_none()++ if repo is None:+ repo = Repository(+ id=_new_uuid(),+ name=name,+ local_path=local_path,+ url=url,+ default_branch=default_branch,+ settings_json=json.dumps(settings or {}),+ )+ session.add(repo)+ else:+ repo.name = name+ repo.url = url+ repo.default_branch = default_branch+ if settings is not None:+ repo.settings_json = json.dumps(settings)+ repo.updated_at = _now_utc()++ await session.flush()+ return repo+++async def get_repository(session: AsyncSession, repo_id: str) -> Repository | None:+ """Return a Repository by primary key, or None."""+ return await session.get(Repository, repo_id)+++async def get_repository_by_path(session: AsyncSession, local_path: str) -> Repository | None:+ """Return a Repository by local_path, or None."""+ result = await session.execute(select(Repository).where(Repository.local_path == local_path))+ return result.scalar_one_or_none()+++async def delete_repository(session: AsyncSession, repo_id: str) -> bool:+ """Delete a repository and all cascaded children.++ Returns True if deleted, False if not found.++ NOTE: The caller should clean up the FTS index *before* calling this,+ since the CASCADE will delete Page rows and we lose the page IDs.+ """+ repo = await session.get(Repository, repo_id)+ if repo is None:+ return False+ await session.delete(repo)+ await session.flush()+ return True+++async def list_page_ids(session: AsyncSession, repository_id: str) -> list[str]:+ """Return all page IDs for a repository (lightweight, ID-only query)."""+ result = await session.execute(select(Page.id).where(Page.repository_id == repository_id))+ return list(result.scalars().all())+++# ---------------------------------------------------------------------------+# GenerationJob CRUD+# ---------------------------------------------------------------------------+++async def upsert_generation_job(+ session: AsyncSession,+ *,+ repository_id: str,+ status: str = "pending",+ provider_name: str = "",+ model_name: str = "",+ total_pages: int = 0,+ config: dict | None = None,+ job_id: str | None = None,+) -> GenerationJob:+ """Insert a new GenerationJob (jobs are append-only)."""+ job = GenerationJob(+ id=job_id or _new_uuid(),+ repository_id=repository_id,+ status=status,+ provider_name=provider_name,+ model_name=model_name,+ total_pages=total_pages,+ config_json=json.dumps(config or {}),+ )+ session.add(job)+ await session.flush()+ return job+++async def get_generation_job(session: AsyncSession, job_id: str) -> GenerationJob | None:+ """Return a GenerationJob by primary key, or None."""+ return await session.get(GenerationJob, job_id)+++async def update_job_status(+ session: AsyncSession,+ job_id: str,+ status: str,+ *,+ completed_pages: int | None = None,+ failed_pages: int | None = None,+ current_level: int | None = None,+ total_pages: int | None = None,+ error_message: str | None = None,+) -> GenerationJob:+ """Update the mutable fields of a GenerationJob.++ Raises:+ ValueError: If *status* is not a recognised value.+ LookupError: If *job_id* does not exist.+ """+ if status not in _VALID_JOB_STATUSES:+ raise ValueError(+ f"Unknown job status {status!r}. Valid values: {sorted(_VALID_JOB_STATUSES)}"+ )++ job = await session.get(GenerationJob, job_id)+ if job is None:+ raise LookupError(f"No GenerationJob with id={job_id!r}")++ job.status = status+ job.updated_at = _now_utc()++ if completed_pages is not None:+ job.completed_pages = completed_pages+ if failed_pages is not None:+ job.failed_pages = failed_pages+ if current_level is not None:+ job.current_level = current_level+ if total_pages is not None:+ job.total_pages = total_pages+ if error_message is not None:+ job.error_message = error_message++ if status == "running" and job.started_at is None:+ job.started_at = _now_utc()+ if status in ("completed", "failed"):+ job.finished_at = _now_utc()++ await session.flush()+ return job+++# ---------------------------------------------------------------------------+# WebhookEvent CRUD+# ---------------------------------------------------------------------------+++async def store_webhook_event(+ session: AsyncSession,+ *,+ provider: str,+ event_type: str,+ payload: dict,+ repository_id: str | None = None,+ delivery_id: str = "",+) -> WebhookEvent:+ """Append a new WebhookEvent record."""+ event = WebhookEvent(+ id=_new_uuid(),+ repository_id=repository_id,+ provider=provider,+ event_type=event_type,+ delivery_id=delivery_id,+ payload_json=json.dumps(payload),+ processed=False,+ )+ session.add(event)+ await session.flush()+ return event+++async def mark_webhook_processed(+ session: AsyncSession, event_id: str, *, job_id: str | None = None+) -> None:+ """Mark a WebhookEvent as processed and optionally link it to a job."""+ event = await session.get(WebhookEvent, event_id)+ if event is None:+ raise LookupError(f"No WebhookEvent with id={event_id!r}")+ event.processed = True+ if job_id is not None:+ event.job_id = job_id+ await session.flush()diff --git a/tests/unit/persistence/test_crud_smoke.py b/tests/unit/persistence/test_crud_smoke.pynew file mode 100644index 000000000..888a8c622--- /dev/null+++ b/tests/unit/persistence/test_crud_smoke.py@@ -0,0 +1,133 @@+"""Round-trip smoke tests for the batch upserts routed through ``_batch_upsert``.++These guard the generic SELECT-then-INSERT/UPDATE helper that+``crud/_shared.py`` introduced when ``crud.py`` was split into a package: every+batch upsert must still insert on first write and update in place on the second,+and the batched git upsert must converge across the 500-row chunk boundary.+``test_crud.py`` already covers the graph-node/edge/symbol insert paths; this+file fills the gaps (metrics, git bulk, the update paths, and batching).+"""++from __future__ import annotations++import pytest+from sqlalchemy import select++from repowise.core.persistence.crud import (+ batch_upsert_graph_edges,+ batch_upsert_graph_metrics,+ batch_upsert_symbols,+ get_all_git_metadata,+ get_graph_metrics,+ upsert_git_metadata_bulk,+)+from repowise.core.persistence.models import GraphEdge, WikiSymbol+from tests.unit.persistence.helpers import insert_repo+++class _Sym:+ """Duck-typed stand-in for ingestion.models.Symbol."""++ def __init__(self, file_path: str, name: str, kind: str = "function", **extra):+ self.file_path = file_path+ self.name = name+ self.kind = kind+ self.id = f"{file_path}::{name}"+ for k, v in extra.items():+ setattr(self, k, v)+++async def test_batch_upsert_graph_metrics_inserts_then_updates(async_session):+ repo = await insert_repo(async_session)+ await batch_upsert_graph_metrics(+ async_session,+ repo.id,+ {"a.py": {"pagerank": 0.5, "betweenness": 0.1, "community_id": 2, "in_degree": 3}},+ )+ got = await get_graph_metrics(async_session, repo.id)+ assert got["a.py"]["pagerank"] == 0.5+ assert got["a.py"]["community_id"] == 2+ assert got["a.py"]["in_degree"] == 3++ # Second write updates in place — only supplied fields change.+ await batch_upsert_graph_metrics(+ async_session, repo.id, {"a.py": {"pagerank": 0.9, "out_degree": 7}}+ )+ got = await get_graph_metrics(async_session, repo.id)+ assert got["a.py"]["pagerank"] == 0.9 # updated+ assert got["a.py"]["out_degree"] == 7 # updated+ assert got["a.py"]["community_id"] == 2 # preserved (not in second payload)+ # exactly one row for the node_id+ rows = await get_graph_metrics(async_session, repo.id)+ assert len(rows) == 1+++async def test_batch_upsert_graph_edges_updates_existing(async_session):+ repo = await insert_repo(async_session)+ edges = [{"source_node_id": "a", "target_node_id": "b", "edge_type": "imports"}]+ await batch_upsert_graph_edges(async_session, repo.id, edges)+ # Re-upsert the same key with new optional fields → in-place update.+ await batch_upsert_graph_edges(+ async_session,+ repo.id,+ [+ {+ "source_node_id": "a",+ "target_node_id": "b",+ "edge_type": "imports",+ "imported_names_json": '["foo"]',+ "confidence": 0.5,+ }+ ],+ )+ result = await async_session.execute(+ select(GraphEdge).where(GraphEdge.repository_id == repo.id)+ )+ rows = list(result.scalars().all())+ assert len(rows) == 1+ assert rows[0].imported_names_json == '["foo"]'+ assert rows[0].confidence == 0.5+++async def test_batch_upsert_symbols_updates_existing(async_session):+ repo = await insert_repo(async_session)+ await batch_upsert_symbols(async_session, repo.id, [_Sym("m.py", "f", signature="()")])+ await batch_upsert_symbols(+ async_session, repo.id, [_Sym("m.py", "f", signature="(x)", kind="method")]+ )+ result = await async_session.execute(+ select(WikiSymbol).where(WikiSymbol.repository_id == repo.id)+ )+ rows = list(result.scalars().all())+ assert len(rows) == 1+ assert rows[0].signature == "(x)"+ assert rows[0].kind == "method"+++async def test_upsert_git_metadata_bulk_inserts_updates_and_batches(async_session):+ repo = await insert_repo(async_session)+ # 1200 rows spans the 500-row chunk boundary (flush per chunk).+ rows = [{"file_path": f"f{i}.py", "commit_count_total": 1} for i in range(1200)]+ await upsert_git_metadata_bulk(async_session, repo.id, rows)+ got = await get_all_git_metadata(async_session, repo.id)+ assert len(got) == 1200+ assert got["f0.py"].commit_count_total == 1++ # Re-upsert one path → in-place update, no duplicate row.+ await upsert_git_metadata_bulk(+ async_session, repo.id, [{"file_path": "f0.py", "commit_count_total": 9}]+ )+ got = await get_all_git_metadata(async_session, repo.id)+ assert len(got) == 1200+ assert got["f0.py"].commit_count_total == 9+++async def test_upsert_git_metadata_bulk_empty_is_noop(async_session):+ repo = await insert_repo(async_session)+ await upsert_git_metadata_bulk(async_session, repo.id, [])+ assert await get_all_git_metadata(async_session, repo.id) == {}