Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
refactor(workspace): split the single-repo update into stages (#4046)
Raghav Chamadiya1060ff20586e
diff --git a/packages/core/src/repowise/core/workspace/update/repo.py b/packages/core/src/repowise/core/workspace/update/repo.pyindex 9b0aa3fbc..edaa9a3f9 100644--- a/packages/core/src/repowise/core/workspace/update/repo.py+++ b/packages/core/src/repowise/core/workspace/update/repo.py@@ -7,7 +7,7 @@ import loggingimport sqlite3from dataclasses import dataclassfrom pathlib import Path-from typing import Any+from typing import Any, NamedTuplefrom ...ingestion.git_indexer import git_history_stalefrom ...ingestion.git_indexer._constants import _DEFAULT_COMMIT_LIMIT@@ -33,50 +33,13 @@ async def _merged_repo_excludes(from ...persistence.database import get_configured_db_urlconfigured_url = get_configured_db_url()- settings_json: str | None = None-if configured_url is None:- db_path = repo_path / ".repowise" / "wiki.db"- if db_path.is_file():- try:- with sqlite3.connect(str(db_path)) as conn:- row = conn.execute("SELECT settings_json FROM repositories LIMIT 1").fetchone()- if row and row[0]:- settings_json = row[0]- except Exception:- pass+ settings_json = _local_settings_json(repo_path)else:- from ...persistence import (- create_engine,- create_session_factory,- get_session,- init_db,- )- from ...persistence.crud import get_repository_by_path-- try:- engine = create_engine(configured_url)- try:- await init_db(engine)- sf = create_session_factory(engine)- async with get_session(sf) as session:- repo = await get_repository_by_path(session, str(repo_path))- if repo is not None and repo.settings_json:- settings_json = repo.settings_json- finally:- await engine.dispose()- except Exception:- pass+ settings_json = await _configured_settings_json(configured_url, repo_path)if settings_json:- try:- settings = _json.loads(settings_json)- if isinstance(settings, dict):- for value in settings.get("exclude_patterns") or []:- if isinstance(value, str) and value not in patterns:- patterns.append(value)- except Exception:- pass+ _merge_settings_excludes(patterns, settings_json)for pattern in extra_exclude_patterns or []:if pattern not in patterns:@@ -84,6 +47,60 @@ async def _merged_repo_excludes(return patterns+def _local_settings_json(repo_path: Path) -> str | None:+ """``settings_json`` from the repo-local ``wiki.db``, or ``None``."""+ db_path = repo_path / ".repowise" / "wiki.db"+ if not db_path.is_file():+ return None+ try:+ with sqlite3.connect(str(db_path)) as conn:+ row = conn.execute("SELECT settings_json FROM repositories LIMIT 1").fetchone()+ if row and row[0]:+ return row[0]+ except Exception:+ pass+ return None+++async def _configured_settings_json(configured_url: str, repo_path: Path) -> str | None:+ """``settings_json`` of this repo's row in the configured database, or ``None``."""+ from ...persistence import (+ create_engine,+ create_session_factory,+ get_session,+ init_db,+ )+ from ...persistence.crud import get_repository_by_path++ settings_json: str | None = None+ try:+ engine = create_engine(configured_url)+ try:+ await init_db(engine)+ sf = create_session_factory(engine)+ async with get_session(sf) as session:+ repo = await get_repository_by_path(session, str(repo_path))+ if repo is not None and repo.settings_json:+ settings_json = repo.settings_json+ finally:+ await engine.dispose()+ except Exception:+ pass+ return settings_json+++def _merge_settings_excludes(patterns: list[str], settings_json: str) -> None:+ """Append the string ``exclude_patterns`` of *settings_json* not already in *patterns*."""+ try:+ settings = _json.loads(settings_json)+ if isinstance(settings, dict):+ for value in settings.get("exclude_patterns") or []:+ if isinstance(value, str) and value not in patterns:+ patterns.append(value)+ except Exception:+ pass++@dataclassclass RepoUpdateResult:"""Result of updating a single repo within a workspace."""@@ -135,45 +152,19 @@ async def _incremental_repo_update(Raises on failure — the caller falls back to the full pipeline."""- from ...ingestion.change_detector import ChangeDetector, merge_file_diffs- from ...pipeline.incremental import (- DocDriftUpdate,- persist_incremental_index,- rebuild_graph_and_git,- run_doc_drift_partial,- run_partial_analysis,- )+ from ...pipeline.incremental import persist_incremental_index, rebuild_graph_and_gitfrom ...pipeline.phases.git import drop_transient_git_signalsalias = repo_path.name- head = get_head_commit(repo_path) or "HEAD"-- detector = ChangeDetector(repo_path)- working_tree_diffs: list = []- working_tree_paths: list[str] | None = None- if include_working_tree:- working_tree_diffs = detector.get_working_tree_changes()- dirty = {fd.path for fd in working_tree_diffs}- # Re-diff anything the last run indexed from the working tree that no- # longer diverges from HEAD, so a reverted edit or a deleted untracked- # file stops being served. See ``stale_working_tree_diffs``.- working_tree_diffs += detector.stale_working_tree_diffs(- state.get("working_tree_paths") or [], dirty- )- working_tree_paths = sorted(dirty)-- file_diffs = merge_file_diffs(- detector.get_changed_files(base_ref, head),- working_tree_diffs,+ file_diffs, working_tree_paths = _changed_file_diffs(+ repo_path, state, base_ref, include_working_tree)if not file_diffs and not force_full_git:- # New commits but nothing the index cares about changed (merge/empty- # commits, or every change excluded). Report success so the caller+ # Nothing the index cares about changed: report success so the caller# bumps ``last_sync_commit`` instead of re-diffing forever.return RepoUpdateResult(alias=alias, updated=True, working_tree_paths=working_tree_paths)- # Per-repo config, like the single-repo update path. The workspace-level- # ``exclude_patterns`` (when provided) apply on top.+ # Per-repo config; workspace-level ``exclude_patterns`` apply on top.from ...repo_config import load_repo_configcfg = load_repo_config(repo_path)@@ -204,47 +195,8 @@ async def _incremental_repo_update(log=_log.info,)- # Stored per-file git fields: ``git_meta_map`` covers only the changed files,- # and an empty dict reads as "no commits in 90 days" to the dead-code analyzer.- from ...pipeline.incremental import (- load_stored_coverage_map,- load_stored_function_mod_p80,- load_stored_git_meta,- load_stored_performance_callers,- )-- stored_git_meta = await load_stored_git_meta(repo_path, log=_log.info)- # Repo-wide p80, so the Function Hotspot gate scores against the full repo.- stored_function_mod_p80 = await load_stored_function_mod_p80(repo_path, log=_log.info)- performance_changed_paths = {diff.path for diff in file_diffs}- performance_changed_paths.update(- diff.old_path for diff in file_diffs if getattr(diff, "old_path", None)- )- stored_performance_callers = await load_stored_performance_callers(- repo_path, performance_changed_paths, log=_log.info- )- # Without a coverage_map the changed files would lose their stored coverage.- stored_coverage_map = await load_stored_coverage_map(repo_path, log=_log.info)-- partial_health_report, dead_code_report = run_partial_analysis(- repo_path,- graph_builder,- git_meta_map,- parsed_files,- file_diffs,- source_map=source_map,- stored_git_meta=stored_git_meta,- stored_performance_callers=stored_performance_callers,- repo_function_mod_p80=stored_function_mod_p80,- coverage_map=stored_coverage_map,- log=_log.info,- )- doc_drift_report = run_doc_drift_partial(- graph_builder,- source_map,- repo_path=repo_path,- log=_log.info,- update=DocDriftUpdate.from_file_diffs(base_ref, file_diffs),+ partial_health_report, dead_code_report, doc_drift_report = await _partial_analysis(+ repo_path, base_ref, file_diffs, parsed_files, source_map, graph_builder, git_meta_map)# Drop the transient, non-serializable ``BlameIndex`` before persistence.@@ -286,27 +238,124 @@ async def _incremental_repo_update(log=_log.info,)- kg_state: dict[str, Any] | None = None- if kg is not None:- from ...analysis.knowledge_graph import build_kg_state, save_knowledge_graph_json-- try:- save_knowledge_graph_json(repo_path, kg)- kg_state = build_kg_state(kg)- except Exception:- _log.warning("knowledge-graph.json export failed for %s", alias, exc_info=True)-return RepoUpdateResult(alias=alias,updated=True,file_count=file_count,symbol_count=sum(len(pf.symbols) for pf in parsed_files),- kg_state=kg_state,+ kg_state=_export_kg_state(repo_path, alias, kg),working_tree_paths=working_tree_paths,git_history_refreshed=bool(full_git_summaries) and "Git persist" not in failed_steps,)+def _changed_file_diffs(+ repo_path: Path, state: dict[str, Any], base_ref: str, include_working_tree: bool+) -> tuple[list, list[str] | None]:+ """The ``base_ref..HEAD`` diffs, plus working-tree ones when asked.++ Returns ``(file_diffs, working_tree_paths)``; the paths are ``None`` on the+ commit-anchored path.+ """+ from ...ingestion.change_detector import ChangeDetector, merge_file_diffs++ head = get_head_commit(repo_path) or "HEAD"++ detector = ChangeDetector(repo_path)+ working_tree_diffs: list = []+ working_tree_paths: list[str] | None = None+ if include_working_tree:+ working_tree_diffs = detector.get_working_tree_changes()+ dirty = {fd.path for fd in working_tree_diffs}+ # Re-diff what the last run indexed from the working tree that no longer+ # diverges from HEAD, so a reverted edit or deleted untracked file drops.+ working_tree_diffs += detector.stale_working_tree_diffs(+ state.get("working_tree_paths") or [], dirty+ )+ working_tree_paths = sorted(dirty)++ file_diffs = merge_file_diffs(+ detector.get_changed_files(base_ref, head),+ working_tree_diffs,+ )+ return file_diffs, working_tree_paths+++async def _partial_analysis(+ repo_path: Path,+ base_ref: str,+ file_diffs: list,+ parsed_files: list,+ source_map: dict,+ graph_builder: Any,+ git_meta_map: dict,+) -> tuple[Any, Any, Any]:+ """Partial health, dead-code and doc-drift reports over the changed files.++ Returns ``(partial_health_report, dead_code_report, doc_drift_report)``.+ """+ from ...pipeline.incremental import (+ DocDriftUpdate,+ load_stored_coverage_map,+ load_stored_function_mod_p80,+ load_stored_git_meta,+ load_stored_performance_callers,+ run_doc_drift_partial,+ run_partial_analysis,+ )++ # Stored per-file git fields: ``git_meta_map`` covers only the changed files,+ # and an empty dict reads as "no commits in 90 days" to the dead-code analyzer.+ stored_git_meta = await load_stored_git_meta(repo_path, log=_log.info)+ # Repo-wide p80, so the Function Hotspot gate scores against the full repo.+ stored_function_mod_p80 = await load_stored_function_mod_p80(repo_path, log=_log.info)+ performance_changed_paths = {diff.path for diff in file_diffs}+ performance_changed_paths.update(+ diff.old_path for diff in file_diffs if getattr(diff, "old_path", None)+ )+ stored_performance_callers = await load_stored_performance_callers(+ repo_path, performance_changed_paths, log=_log.info+ )+ # Without a coverage_map the changed files would lose their stored coverage.+ stored_coverage_map = await load_stored_coverage_map(repo_path, log=_log.info)++ partial_health_report, dead_code_report = run_partial_analysis(+ repo_path,+ graph_builder,+ git_meta_map,+ parsed_files,+ file_diffs,+ source_map=source_map,+ stored_git_meta=stored_git_meta,+ stored_performance_callers=stored_performance_callers,+ repo_function_mod_p80=stored_function_mod_p80,+ coverage_map=stored_coverage_map,+ log=_log.info,+ )+ doc_drift_report = run_doc_drift_partial(+ graph_builder,+ source_map,+ repo_path=repo_path,+ log=_log.info,+ update=DocDriftUpdate.from_file_diffs(base_ref, file_diffs),+ )+ return partial_health_report, dead_code_report, doc_drift_report+++def _export_kg_state(repo_path: Path, alias: str, kg: Any) -> dict[str, Any] | None:+ """Write ``knowledge-graph.json`` and return its state.json block, or ``None``."""+ if kg is None:+ return None+ from ...analysis.knowledge_graph import build_kg_state, save_knowledge_graph_json++ try:+ save_knowledge_graph_json(repo_path, kg)+ return build_kg_state(kg)+ except Exception:+ _log.warning("knowledge-graph.json export failed for %s", alias, exc_info=True)+ return None++async def _has_persisted_repo_index(repo_path: Path) -> bool:"""Return whether this repo has persisted index data.@@ -355,12 +404,7 @@ async def update_single_repo_index(incremental failure run the full ingestion pipeline instead (index-only —no wiki pages)."""- from ...repo_config import (- changed_config_dependencies,- config_dependency_fingerprints,- config_fingerprint,- load_repo_config,- )+ from ...repo_config import load_repo_configalias = repo_path.namestate = read_repo_state(repo_path)@@ -370,14 +414,60 @@ async def update_single_repo_index(repo_commit_depth = int(repo_config.get("commit_limit", commit_depth))repo_follow_renames = bool(repo_config.get("follow_renames", False))- # Config drift check, mirroring the single-repo update path: a changed- # config.yaml / health-rules.json invalidates persisted health scores and- # exclude handling, so the incremental (changed-files-only) path must not- # run. The full pipeline below re-derives everything under the new config;- # the workspace pass stamps the new fingerprint into state.json afterwards.- # A missing stored fingerprint (legacy state) is NOT drift — same as the- # single-repo path, which only stamps it — so legacy repos don't pay a- # surprise full re-index.+ drift = _drift_requirements(*_config_changes(repo_path, state, repo_config))+ has_persisted_index = await _has_persisted_repo_index(repo_path)+ if drift.requires_full_reindex and has_persisted_index:+ _log.info(+ "workspace_update: %s config fingerprint drifted — full re-index "+ "so health scores reflect the new config",+ alias,+ )++ if (+ not drift.requires_full_reindex+ and base_ref+ and has_persisted_index+ and commit_exists(repo_path, str(base_ref))+ ):+ incremental_result = await _try_incremental_update(+ repo_path, state, str(base_ref), exclude_patterns, include_working_tree+ )+ if incremental_result is not None:+ return incremental_result++ return await _full_index_update(+ repo_path,+ state,+ merged_excludes,+ repo_commit_depth,+ repo_follow_renames,+ drift,+ progress,+ )+++class _ConfigDrift(NamedTuple):+ """What a config change demands of this update."""++ requires_full_reindex: bool+ require_git_success: bool+ require_health_success: bool+++def _config_changes(+ repo_path: Path, state: dict[str, Any], repo_config: dict[str, Any]+) -> tuple[bool, set[str] | None]:+ """Return ``(config_changed, changed_dependencies)`` against the stored fingerprints.++ A missing stored fingerprint (legacy state) is not drift: the single-repo+ path only stamps it, so legacy repos do not pay a surprise full re-index.+ """+ from ...repo_config import (+ changed_config_dependencies,+ config_dependency_fingerprints,+ config_fingerprint,+ )+stored_fp = state.get("config_fingerprint")config_changed = stored_fp is not None and stored_fp != config_fingerprint(repo_path)current_dependency_fps = config_dependency_fingerprints(repo_path, config=repo_config)@@ -388,9 +478,17 @@ async def update_single_repo_index(if config_changedelse set())- # Workspace indexing has no generation layer. Only settings that affect- # its persisted graph/history/health stores require a full re-index;- # formatting, distill/MCP, and generation-only edits are state updates.+ return config_changed, changed_dependencies+++def _drift_requirements(+ config_changed: bool, changed_dependencies: set[str] | None+) -> _ConfigDrift:+ """Which persisted stores a config change invalidates.++ Workspace indexing has no generation layer, so formatting, distill/MCP and+ generation-only edits are state updates, not a full re-index.+ """requires_full_reindex = config_changed and (changed_dependencies is Noneor bool((changed_dependencies or set()) - {"state_only", "generation"})@@ -406,50 +504,58 @@ async def update_single_repo_index(& (changed_dependencies or set())))- has_persisted_index = await _has_persisted_repo_index(repo_path)- if requires_full_reindex and has_persisted_index:- _log.info(- "workspace_update: %s config fingerprint drifted — full re-index "- "so health scores reflect the new config",- alias,+ return _ConfigDrift(requires_full_reindex, require_git_success, require_health_success)+++async def _try_incremental_update(+ repo_path: Path,+ state: dict[str, Any],+ base_ref: str,+ exclude_patterns: list[str] | None,+ include_working_tree: bool,+) -> RepoUpdateResult | None:+ """The incremental update's result, or ``None`` to fall back to the full pipeline."""+ try:+ return await _incremental_repo_update(+ repo_path,+ state=state,+ base_ref=base_ref,+ exclude_patterns=exclude_patterns,+ include_working_tree=include_working_tree,+ force_full_git=git_history_stale(state),+ )+ except Exception:+ _log.warning(+ "Incremental update failed for %s — falling back to the full pipeline",+ repo_path,+ exc_info=True,)+ return None- if (- not requires_full_reindex- and base_ref- and has_persisted_index- and commit_exists(repo_path, str(base_ref))- ):- try:- incremental_result = await _incremental_repo_update(- repo_path,- state=state,- base_ref=str(base_ref),- exclude_patterns=exclude_patterns,- include_working_tree=include_working_tree,- force_full_git=git_history_stale(state),- )- if incremental_result is not None:- return incremental_result- except Exception:- _log.warning(- "Incremental update failed for %s — falling back to the full pipeline",- repo_path,- exc_info=True,- )+async def _full_index_update(+ repo_path: Path,+ state: dict[str, Any],+ merged_excludes: list[str],+ commit_depth: int,+ follow_renames: bool,+ drift: _ConfigDrift,+ progress: Any | None,+) -> RepoUpdateResult:+ """Run the full index-only pipeline; a failure becomes an error result."""+ alias = repo_path.nametry:from ...pipeline.full_index import index_repo_fullresult = await index_repo_full(repo_path,- commit_depth=repo_commit_depth,+ commit_depth=commit_depth,exclude_patterns=merged_excludes,include_submodules=bool(state.get("include_submodules", False)),include_nested_repos=bool(state.get("include_nested_repos", False)),- follow_renames=repo_follow_renames,- require_git_success=require_git_success,- require_health_success=require_health_success,+ follow_renames=follow_renames,+ require_git_success=drift.require_git_success,+ require_health_success=drift.require_health_success,progress=progress,)