Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat: extract pipeline API from CLI + release v0.1.30 (#25)

Raghav Chamadiya012482d0dd63
diff --git a/.claude/settings.local.json b/.claude/settings.local.json
deleted file mode 100644
index 8597de264..000000000
--- a/.claude/settings.local.json
+++ /dev/null
@@ -1,198 +0,0 @@
-{
- "permissions": {
- "allow": [
- "Bash(uv sync:*)",
- "Read(//c/Users/ragha/AppData/Roaming/uv/**)",
- "Read(//c/Users/ragha/.local/bin//**)",
- "Bash(/c/Users/ragha/AppData/Roaming/uv/bin/uv.exe sync:*)",
- "Bash(cmd.exe /c \"echo %PATH%\")",
- "Bash(tr ';' '\\\\n')",
- "Bash(powershell.exe -Command \"uv --version\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv sync --all-packages 2>&1\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/providers/ -v 2>&1\")",
- "Bash(xargs cat:*)",
- "Bash(python /c/Users/ragha/Desktop/repowise/.venv/Scripts/python.exe -c \"import tree_sitter_typescript; print\\(dir\\(tree_sitter_typescript\\)\\)\")",
- "Bash(/c/Users/ragha/Desktop/repowise/.venv/Scripts/python.exe -c \"import tree_sitter_typescript; print\\(dir\\(tree_sitter_typescript\\)\\)\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv sync --all-packages 2>&1 | tail -5\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run python -c \\\\\"from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder; print\\('imports OK'\\)\\\\\" 2>&1\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/ingestion/ tests/integration/ -v --tb=short 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run python -c \\\\\"from pathlib import Path; p = Path\\('packages/core/src/repowise/core/ingestion/parser.py'\\); print\\(p.parent.parent.parent.parent.parent / 'queries'\\)\\\\\"\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/ingestion/test_graph.py::TestPageRank -v --tb=long 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv sync --all-packages 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/ -v --tb=short -q 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/ -q 2>&1\" 2>&1)",
- "Bash(find C:UsersraghaDesktoprepowise -maxdepth 3 -type d -name *generation* -o -type d -name *generate*)",
- "Bash(wc -l C:UsersraghaDesktoprepowisepackagescoresrcrepowisecoreingestion*.py)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/persistence/ -v 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/integration/test_persistence.py -v 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/ -v --tb=short 2>&1\" 2>&1)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/ -q 2>&1 | tail -20\" 2>&1)",
- "Bash(powershell.exe -Command \"uv run pytest tests/unit/cli/ -v 2>&1\")",
- "Bash(powershell.exe -Command \"uv run pytest tests/integration/test_cli.py -v 2>&1\")",
- "Bash(powershell.exe -Command \"uv run pytest tests/integration/test_cli.py::TestInitFullMock -v 2>&1\")",
- "Bash(powershell.exe -Command \"uv run pytest tests/ -q 2>&1\")",
- "Bash(powershell.exe -Command \"uv run pytest tests/unit/generation/test_page_generator.py::test_generate_file_page_provider_name -v 2>&1\")",
- "Bash(sqlite3:*)",
- "Bash(python3:*)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/generation/ -v --tb=short 2>&1 | tail -60\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/ -v --tb=short -q 2>&1 | tail -30\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/cli/test_cost_estimator.py -v --tb=short 2>&1 | tail -30\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/generation/ tests/unit/persistence/ tests/unit/server/ tests/unit/cli/test_commands.py -q --tb=short 2>&1 | tail -15\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/generation/test_templates.py tests/unit/generation/test_context_assembler.py tests/unit/generation/test_page_generator.py -v --tb=short 2>&1 | tail -30\")",
- "Bash(powershell.exe -File \"C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise\\\\run_test.ps1\" 2>&1)",
- "Bash(xargs grep:*)",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run repowise export --help 2>&1\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run repowise export 'C:\\\\Users\\\\ragha\\\\Desktop\\\\interview-coach' 2>&1\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/ingestion/test_traverser.py -v 2>&1\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run pytest tests/unit/cli/ tests/unit/server/ -v --tb=short -q 2>&1 | tail -30\")",
- "Bash(dir /s /b \"C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise\\\\packages\\\\server\\\\src\")",
- "Bash(findstr \"models.py\")",
- "Bash(find C:/Users/ragha/Desktop/repowise/packages -name *.py)",
- "Bash(python -m py_compile packages/core/src/repowise/core/analysis/decision_extractor.py)",
- "Bash(python -m py_compile packages/cli/src/repowise/cli/commands/init_cmd.py)",
- "Bash(python -m py_compile packages/core/src/repowise/core/ingestion/traverser.py)",
- "Bash(python -m py_compile packages/core/src/repowise/core/ingestion/git_indexer.py)",
- "Bash(git -C \"C:/Users/ragha/Desktop/repowise\" status)",
- "Bash(pip install:*)",
- "mcp__repowise__get_overview",
- "mcp__repowise__get_module_docs",
- "mcp__repowise__get_file_docs",
- "mcp__repowise__get_symbol",
- "mcp__repowise__get_file_history",
- "mcp__repowise__get_co_changes",
- "mcp__repowise__search_codebase",
- "mcp__repowise__get_architecture_diagram",
- "mcp__repowise__get_dependency_path",
- "mcp__repowise__get_dead_code",
- "mcp__repowise__get_decisions",
- "mcp__repowise__get_why",
- "mcp__repowise__get_codebase_ownership",
- "Bash(grep -r \"13\\\\|thirteen\\\\|16\\\\|sixteen\" C:UsersraghaDesktoprepowisepackages --include=*.py --include=*.md)",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/ -x -v --timeout=30 2>&1 | head -150\")",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/ -x -v 2>&1 | head -200\")",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/ -v --ignore=tests/integration/test_dead_code_integration.py 2>&1 | tail -80\")",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/unit/ -v --ignore=tests/integration 2>&1 | tail -80\")",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/unit/server/test_mcp.py -v 2>&1 | tail -50\")",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/integration/test_cli.py tests/unit/persistence/ tests/unit/server/ -v 2>&1 | tail -30\")",
- "Bash(git add:*)",
- "Bash(git commit:*)",
- "Bash(powershell.exe -Command \"uv run python -m pytest tests/unit/persistence/test_models.py -v 2>&1 | tail -20\")",
- "Bash(powershell -Command \"uv run pytest tests/unit/server/test_mcp.py -v --tb=short 2>&1 | head -100\")",
- "Bash(powershell -Command \"uv run pytest tests/integration/test_mcp.py -v --tb=short 2>&1 | head -50\")",
- "Bash(powershell -Command \"uv run pytest tests/unit/server/test_mcp.py tests/integration/test_mcp.py -v --tb=short 2>&1\")",
- "mcp__repowise__get_context",
- "mcp__repowise__get_risk",
- "Bash(powershell -Command \"uv run pytest tests/unit/server/test_mcp.py::test_search_codebase tests/integration/test_mcp.py::test_mcp_search_flow -v --tb=short 2>&1\")",
- "Read(//c/Users/ragha/Desktop/interview-coach/.repowise/**)",
- "Read(//c/Users/ragha/.claude/projects/**)",
- "Bash(powershell -Command \"uv run python -c ''import lancedb; print\\(\"\"lancedb installed:\"\", lancedb.__version__\\)'' 2>&1\")",
- "Bash(powershell -Command \"uv run python -c \"\"import lancedb; print\\(lancedb.__version__\\)\"\"\")",
- "Bash(powershell -Command \"cd C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise; uv run repowise reindex --help\" 2>&1)",
- "Bash(powershell -Command \"cd C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise; uv pip install lancedb\" 2>&1)",
- "Bash(uv run:*)",
- "Bash(xargs wc:*)",
- "Bash(git mv:*)",
- "Bash(python:*)",
- "Bash(find /c/Users/ragha/Desktop/repowise/src -type f -name *stage* -o -name *pipeline*)",
- "Bash(find /c/Users/ragha/Desktop/repowise/packages -type f -name *.py)",
- "Bash(find C:UsersraghaDesktoprepowisepackageswebsrcapp -name *chat* -o -path */chat/*)",
- "Bash(powershell.exe -Command \"uv run pytest tests/unit/test_git_indexer.py -v 2>&1\")",
- "Bash(powershell.exe -Command \"uv run pytest tests/unit/ -v 2>&1\")",
- "Bash(powershell -Command \"uv run python -c ''import sqlite3; conn = sqlite3.connect\\(\"\".repowise/wiki.db\"\"\\); c = conn.cursor\\(\\); c.execute\\(\"\"SELECT page_type, COUNT\\(*\\) FROM pages GROUP BY page_type ORDER BY COUNT\\(*\\) DESC\"\"\\); [print\\(r\\) for r in c.fetchall\\(\\)]''\")",
- "Bash(powershell.exe -Command \"cd 'C:\\\\Users\\\\ragha\\\\Desktop\\\\repowise'; uv run python -m pytest tests/unit/server/test_mcp.py -v --tb=short 2>&1 | tail -40\")",
- "Bash(powershell.exe -Command \"uv run pytest tests/unit/test_dead_code.py tests/unit/server/test_dead_code.py tests/unit/server/test_mcp.py -x -q 2>&1\" 2>&1)",
- "Bash(powershell.exe -NoProfile -Command 'Stop-Process -Id 30856,40916,5320,8576,37672,45336 -Force -ErrorAction SilentlyContinue')",
- "Bash(powershell.exe -NoProfile -Command 'Get-Process git -ErrorAction SilentlyContinue | Format-Table Id, CPU, StartTime -AutoSize')",
- "Bash(powershell.exe -NoProfile -Command 'Get-Process git -ErrorAction SilentlyContinue | Stop-Process -Force; Get-Process python -ErrorAction SilentlyContinue | Where-Object {$_.Id -notin @\\(13000,17328,39408,41112,45468,45912\\)} | Stop-Process -Force')",
- "Bash(powershell.exe -NoProfile -Command \"uv run repowise update --since 59ca5c8\" 2>&1)",
- "Bash(find /c/Users/ragha/Desktop/repowise/docs -type f \\\\\\(-name *.md -o -name *.txt \\\\\\))",
- "Bash(grep -r \"def get_\" /c/Users/ragha/Desktop/repowise/packages/server/src/repowise/server/mcp_server/*.py)",
- "Bash(npm install:*)",
- "Bash(npm run:*)",
- "Bash(ls:*)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web -name *search* -type f)",
- "Bash(grep -E \"\\\\.\\(md|txt|adr\\)$\")",
- "Bash(grep -r hybrid /Users/swati.ahuja/Desktop/wise/repowise/packages/core --include=*.py)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/tests -name *search* -o -name *vector*)",
- "Bash(grep -r \"get_vector_store\\\\|get_fts\\\\|vector_store\\\\|_state\\\\._vector_store\" /Users/swati.ahuja/Desktop/wise/repowise/packages/server/src/repowise/server --include=*.py)",
- "Bash(grep -rn \"transaction\\\\|consistency\\\\|crash\\\\|atomic\\\\|two-phase\" /Users/swati.ahuja/Desktop/wise/repowise/packages/core/src/repowise/core/persistence/*.py)",
- "Bash(git push:*)",
- "Bash(source /Users/swati.ahuja/Desktop/wise/repowise/venv/bin/activate)",
- "Bash(curl -s http://localhost:7337/api/repos)",
- "Bash(curl -s http://localhost:7337/health)",
- "Bash(curl -s \"http://localhost:7337/api/pages?repo_id=e0779e597a3142e59e3dd57435f0ef1b\")",
- "Bash(curl -s http://localhost:3000)",
- "Bash(curl -s http://localhost:3000/api/repos)",
- "Bash(curl -s http://localhost:3000/health)",
- "Bash(curl -s \"http://localhost:3000/api/pages?repo_id=e0779e597a3142e59e3dd57435f0ef1b&limit=5\")",
- "Bash(curl -s \"http://localhost:3000/api/repos/e0779e597a3142e59e3dd57435f0ef1b/stats\")",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src/components/graph -type f -name *.ts -o -name *.tsx -o -name *.css -o -name *.json)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src/app -name *.tsx)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/server /Users/swati.ahuja/Desktop/wise/repowise/packages/core -type f -name *.py)",
- "Bash(curl -s http://localhost:3000/api/graph/e0779e597a3142e59e3dd57435f0ef1b/dead-nodes)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise -name *.db -o -name *.sqlite -o -name *.sqlite3)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/server -name \"__pycache__\" -type d -exec rm -rf {} +)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/server -name \"*.pyc\" -delete)",
- "Bash(curl -s http://localhost:7337/api/graph/e0779e597a3142e59e3dd57435f0ef1b/dead-nodes)",
- "Bash(kill 67304)",
- "Bash(xargs kill:*)",
- "Bash(.venv/bin/python -c \"import repowise.server.routers.graph; print\\(repowise.server.routers.graph.__file__\\)\")",
- "Bash(# Apply the same fix to the installed copy in venv/\ncp packages/server/src/repowise/server/routers/graph.py venv/lib/python3.13/site-packages/repowise/server/routers/graph.py && rm -f venv/lib/python3.13/site-packages/repowise/server/routers/__pycache__/graph.cpython-313.pyc && echo \"Copied and cleared cache\")",
- "Bash(grep -r \"from.*tabs\\\\|import.*Tabs\\\\|<Tabs\" /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src --include=*.tsx)",
- "Bash(grep -E \"\\\\.py$\")",
- "Bash(grep -r use-elk-layout /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src --include=*.tsx --include=*.ts)",
- "Bash(.venv/bin/python -m pytest tests/providers/test_mock_provider.py -x)",
- "Bash(.venv/bin/python -m pytest tests/unit/server/ -x)",
- "Bash(.venv/bin/python -m pytest --tb=short -q)",
- "Bash(.venv/bin/python -m pytest tests/unit/test_providers/ tests/unit/test_persistence/ -v 2>&1)",
- "Bash(.venv/bin/python -m pytest tests/unit/test_providers/test_anthropic_provider.py tests/unit/test_providers/test_gemini_provider.py tests/unit/test_providers/test_openai_provider.py tests/unit/test_persistence/test_openai_embedder.py -v 2>&1)",
- "Bash(.venv/bin/python -c \"import anthropic; print\\(anthropic.__version__\\)\")",
- "Bash(.venv/bin/python -c \"import openai; print\\(openai.__version__\\)\")",
- "Bash(.venv/bin/python -c \"import google.genai; print\\(''ok''\\)\")",
- "Bash(/Users/swati.ahuja/Desktop/wise/repowise/venv/bin/python -m pytest tests/ -x --tb=short -q)",
- "Bash(/Users/swati.ahuja/Desktop/wise/repowise/venv/bin/python -m pytest tests/ --tb=short -q)",
- "Bash(find . -path */bin/pytest -o -path */bin/python*)",
- "Bash(/Users/swati.ahuja/Desktop/wise/repowise/venv/bin/python -m pytest tests/unit/server/test_repos.py::test_create_repo -v --tb=long)",
- "Bash(/Users/swati.ahuja/Desktop/wise/repowise/venv/bin/python -m pytest tests/providers/test_mock_provider.py::TestBaseProviderInterface::test_generate_returns_generated_response -v --tb=long)",
- "Bash(/Users/swati.ahuja/Desktop/wise/repowise/venv/bin/python -m pytest tests/unit/cli/test_commands.py::TestErrorCases::test_init_no_provider -v --tb=long)",
- "Bash(./.venv/bin/pytest tests/unit/server/test_mcp.py -x -q)",
- "Bash(/Users/swati.ahuja/Desktop/wise/repowise/venv/bin/python -m pytest tests/unit/persistence/test_vector_store.py::test_mock_embedder_returns_unit_vectors -v --tb=long)",
- "Bash(source venv/bin/activate)",
- "Bash(pip uninstall:*)",
- "Bash(ruff check:*)",
- "Bash(node:*)",
- "Bash(pip show:*)",
- "Read(//Users/swati.ahuja/**)",
- "Bash(head -20 grep -rn \"wiki.db\\\\|db_path\\\\|database_url\\\\|engine\\\\|DB_URL\" /Users/swati.ahuja/Desktop/wise/repowise/packages/server/src/repowise/server/)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/venv/lib -path *repowise_core* -o -path *repowise/core*)",
- "Bash(git rm:*)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise -name *.db -o -name .repowise -type d)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web -name tailwind.config.* -o -name globals.css -o -name theme* -o -name *variables.css)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src -name *.tsx -o -name *.ts)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web -name repowise* -o -name logo*)",
- "Bash(mkdir -p /Users/swati.ahuja/Desktop/wise/repowise/packages/web/public)",
- "Bash(cp \"/Users/swati.ahuja/Desktop/wise/repowise/packages/web/src/repowise-logo .png\" /Users/swati.ahuja/Desktop/wise/repowise/packages/web/public/repowise-logo.png)",
- "Bash(npx next:*)",
- "Bash(grep -r 'GET /api/repos' /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src --include=*.ts --include=*.tsx)",
- "Bash(grep -r \"useSWR\\\\|useQuery\\\\|setInterval\\\\|useEffect\" /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src --include=*.ts --include=*.tsx -l)",
- "Bash(grep -r /api/repos/ /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src --include=*.ts --include=*.tsx)",
- "Bash(grep -E \"\\\\.\\(ts|tsx\\)$\")",
- "Bash(grep -r \"localStorage\\\\|useContext\\\\|Context\" /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src/lib --include=*.ts --include=*.tsx -l)",
- "Bash(wc -l /Users/swati.ahuja/Desktop/wise/repowise/repowise-hosted-frontend/src/components/landing/*.tsx)",
- "Bash(grep:*)",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web -name \"swr\\\\|next.config\\\\|package.json\")",
- "Bash(find /Users/swati.ahuja/Desktop/wise/repowise/packages/web/src/lib/api -type f -name *.ts)",
- "Bash(wc:*)",
- "Bash(find packages/web/src/components/graph packages/web/src/components/docs packages/web/src/components/wiki packages/web/src/components/git packages/web/src/components/dead-code packages/web/src/components/decisions packages/web/src/components/coverage packages/web/src/components/symbols packages/web/src/components/jobs -name *.tsx -o -name *.ts)",
- "Bash(find packages/web/src/lib -name *.ts -o -name *.tsx)",
- "Bash(npx tsc:*)",
- "Bash(kill %1)",
- "Bash(lsof -ti:3002)",
- "WebSearch"
- ]
- },
- "enableAllProjectMcpServers": true,
- "enabledMcpjsonServers": [
- "repowise"
- ]
-}
diff --git a/.gitignore b/.gitignore
index 33f7c4348..7bcdd87f2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,11 @@
+# Hosted backend (private, not part of the open-source project)
+backend/
+hosted_backend_plan/
+migrations/
+
+# Outreach / marketing data
+repowise_outreach_targets.csv
+
# IDE/tool caches
.sfdx/
diff --git a/packages/cli/src/repowise/cli/__init__.py b/packages/cli/src/repowise/cli/__init__.py
index e6726bc60..d284de9dc 100644
--- a/packages/cli/src/repowise/cli/__init__.py
+++ b/packages/cli/src/repowise/cli/__init__.py
@@ -6,4 +6,4 @@ git signals, dead code detection, architectural decisions, and
AI-generated documentation.
"""
-__version__ = "0.1.29"
+__version__ = "0.1.30"
diff --git a/packages/cli/src/repowise/cli/commands/init_cmd.py b/packages/cli/src/repowise/cli/commands/init_cmd.py
index ba3b92e31..c8d165901 100644
--- a/packages/cli/src/repowise/cli/commands/init_cmd.py
+++ b/packages/cli/src/repowise/cli/commands/init_cmd.py
@@ -28,6 +28,22 @@ from repowise.cli.helpers import (
)
+# ---------------------------------------------------------------------------
+# Helpers (kept in this file; _resolve_embedder also imported by other cmds)
+# ---------------------------------------------------------------------------
+
+
+def _resolve_embedder(embedder_flag: str | None) -> str:
+ """Auto-detect embedder from env vars, or use the flag value."""
+ if embedder_flag:
+ return embedder_flag
+ if os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY"):
+ return "gemini"
+ if os.environ.get("OPENAI_API_KEY"):
+ return "openai"
+ return "mock"
+
+
def _register_mcp_with_claude(console_obj: Any, repo_path: Path) -> None:
"""Register the repowise MCP server with Claude Desktop and Claude Code."""
from repowise.cli.mcp_config import register_with_claude_code, register_with_claude_desktop
@@ -104,16 +120,19 @@ async def _write_claude_md_async(repo_path: Path) -> None:
ClaudeMdGenerator().write(repo_path, data)
-async def _persist_index_only(
+# ---------------------------------------------------------------------------
+# Persistence — saves PipelineResult to SQLite
+# ---------------------------------------------------------------------------
+
+
+async def _persist_result(
+ result: Any,
repo_path: Path,
- repo_name: str,
- graph_builder: Any,
- parsed_files: list[Any],
- git_metadata_list: list[dict],
- dead_code_report: Any,
- decision_report: Any = None,
) -> None:
- """Persist graph, symbols, git metadata, dead code, and decisions — no pages."""
+ """Persist a PipelineResult to the local SQLite database.
+
+ Handles both index-only (no pages) and full (with pages + FTS) modes.
+ """
from repowise.cli.helpers import get_db_url_for_repo
from repowise.core.persistence import (
batch_upsert_graph_edges,
@@ -138,16 +157,23 @@ async def _persist_index_only(
async with get_session(sf) as session:
repo = await upsert_repository(
session,
- name=repo_name,
+ name=result.repo_name,
local_path=str(repo_path),
)
repo_id = repo.id
+ # Pages (if generated)
+ if result.generated_pages:
+ from repowise.core.persistence import upsert_page_from_generated
+
+ for page in result.generated_pages:
+ await upsert_page_from_generated(session, page, repo_id)
+
# Graph nodes
- graph = graph_builder.graph()
- pr = graph_builder.pagerank()
- bc = graph_builder.betweenness_centrality()
- cd = graph_builder.community_detection()
+ graph = result.graph_builder.graph()
+ pr = result.graph_builder.pagerank()
+ bc = result.graph_builder.betweenness_centrality()
+ cd = result.graph_builder.community_detection()
nodes = []
for node_path in graph.nodes:
data = graph.nodes[node_path]
@@ -183,7 +209,7 @@ async def _persist_index_only(
# Symbols
all_symbols = []
- for pf in parsed_files:
+ for pf in result.parsed_files:
for sym in pf.symbols:
sym.file_path = pf.file_info.path
all_symbols.append(sym)
@@ -191,15 +217,15 @@ async def _persist_index_only(
await batch_upsert_symbols(session, repo_id, all_symbols)
# Git metadata
- if git_metadata_list:
- await upsert_git_metadata_bulk(session, repo_id, git_metadata_list)
+ if result.git_metadata_list:
+ await upsert_git_metadata_bulk(session, repo_id, result.git_metadata_list)
# Dead code findings
- if dead_code_report and dead_code_report.findings:
- await save_dead_code_findings(session, repo_id, dead_code_report.findings)
+ if result.dead_code_report and result.dead_code_report.findings:
+ await save_dead_code_findings(session, repo_id, result.dead_code_report.findings)
# Decision records
- if decision_report and decision_report.decisions:
+ if result.decision_report and result.decision_report.decisions:
import dataclasses as _dc
from repowise.core.persistence.crud import bulk_upsert_decisions
@@ -207,21 +233,24 @@ async def _persist_index_only(
await bulk_upsert_decisions(
session,
repo_id,
- [_dc.asdict(d) for d in decision_report.decisions],
+ [_dc.asdict(d) for d in result.decision_report.decisions],
)
+ # FTS indexing (only when pages were generated)
+ if result.generated_pages:
+ from repowise.core.persistence import FullTextSearch
+
+ fts = FullTextSearch(engine)
+ await fts.ensure_index()
+ for page in result.generated_pages:
+ await fts.index(page.page_id, page.title, page.content)
+
await engine.dispose()
-def _resolve_embedder(embedder_flag: str | None) -> str:
- """Auto-detect embedder from env vars, or use the flag value."""
- if embedder_flag:
- return embedder_flag
- if os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY"):
- return "gemini"
- if os.environ.get("OPENAI_API_KEY"):
- return "openai"
- return "mock"
+# ---------------------------------------------------------------------------
+# CLI command
+# ---------------------------------------------------------------------------
@click.command("init")
@@ -315,6 +344,7 @@ def init_command(
"""
from repowise.cli.ui import (
BRAND,
+ RichProgressCallback,
build_completion_panel,
format_elapsed,
interactive_advanced_config,
@@ -333,12 +363,9 @@ def init_command(
raise click.ClickException(f"Not a directory: {repo_path}")
ensure_repowise_dir(repo_path)
-
- # Load saved API keys from .repowise/.env (won't overwrite existing env vars)
load_dotenv(repo_path)
- # Suppress all library/structlog output so the interactive UX stays clean.
- # Progress bars and summary lines are the only output the user needs.
+ # Suppress library/structlog output — progress bars are the only output needed.
import logging
logging.getLogger("httpx").setLevel(logging.ERROR)
@@ -365,7 +392,6 @@ def init_command(
if mode == "index_only":
index_only = True
elif mode == "advanced":
- # Provider setup first, then advanced config
provider_name, model = interactive_provider_select(console, model, repo_path=repo_path)
adv = interactive_advanced_config(console)
commit_limit = adv["commit_limit"]
@@ -376,7 +402,6 @@ def init_command(
exclude = adv["exclude"]
test_run = adv["test_run"]
else:
- # Full mode — just provider setup
provider_name, model = interactive_provider_select(console, model, repo_path=repo_path)
# Merge exclude_patterns from config.yaml and --exclude/-x flags
@@ -386,7 +411,6 @@ def init_command(
# Resolve commit limit: CLI flag → config.yaml → default (500)
resolved_commit_limit: int = commit_limit or config.get("commit_limit") or 500
resolved_commit_limit = max(1, min(resolved_commit_limit, 5000))
- # Persist to config so `repowise update` picks it up automatically
if commit_limit is not None:
config["commit_limit"] = resolved_commit_limit
@@ -395,15 +419,13 @@ def init_command(
if follow_renames:
config["follow_renames"] = True
- embedder = _resolve_embedder(embedder_name)
+ embedder_name_resolved = _resolve_embedder(embedder_name)
- # Compute phase counts for headers
- total_phases = 3 if index_only else 4
+ # ---- Resolve provider ----
+ provider = None
+ decision_provider = None
if index_only:
- provider = None
- # Still try to resolve a provider for decision extraction (no page generation)
- decision_provider = None
try:
if (
provider_name
@@ -420,7 +442,7 @@ def init_command(
):
decision_provider = resolve_provider(provider_name, model, repo_path)
except Exception:
- pass # No provider available — inline markers only
+ pass
has_provider = decision_provider is not None
if is_interactive:
@@ -433,9 +455,7 @@ def init_command(
f"Decision extraction provider: [cyan]{decision_provider.provider_name}[/cyan]"
)
else:
- # Non-interactive path: resolve provider from flags/env
if not is_interactive and provider_name is None and sys.stdin.isatty():
- # Fallback for TTY without interactive mode (shouldn't happen, but safety)
from repowise.cli.ui import interactive_provider_select as _ips
provider_name, model = _ips(console, model)
@@ -446,9 +466,9 @@ def init_command(
console.print(
f" Provider: [cyan]{provider.provider_name}[/cyan] / Model: [cyan]{provider.model_name}[/cyan]"
)
- console.print(f" Embedder: [cyan]{embedder}[/cyan]")
+ console.print(f" Embedder: [cyan]{embedder_name_resolved}[/cyan]")
- # Validate API key with a lightweight call before ingesting
+ # Validate provider connection
from repowise.core.providers.llm.base import ProviderError
with console.status(" Verifying provider connection…", spinner="dots"):
@@ -458,25 +478,11 @@ def init_command(
raise click.ClickException(f"Provider validation failed: {exc}") from exc
console.print(" [green]✓[/green] Provider connection verified")
- # ---- Phase 1: Ingestion ----
- print_phase_header(
- console,
- 1,
- total_phases,
- "Ingestion",
- "Parsing source files and building the dependency graph",
- )
- from concurrent.futures import ThreadPoolExecutor, as_completed
-
- from repowise.core.generation import GenerationConfig
- from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
-
- config = GenerationConfig(max_concurrency=concurrency)
+ # ---- Phase 1 & 2: Ingestion + Analysis (always) ----
+ total_phases = 3 if index_only else 4
+ llm_client = provider if not index_only else decision_provider
- git_future = None
- git_summary = None
- git_metadata_list: list[dict] = []
- git_meta_map: dict[str, dict] = {}
+ from repowise.core.pipeline import run_pipeline
with Progress(
SpinnerColumn(),
@@ -485,295 +491,256 @@ def init_command(
TextColumn("{task.completed}/{task.total}"),
TimeElapsedColumn(),
console=console,
- ) as progress:
- task_traverse = progress.add_task("Traversing files...", total=None)
- task_parse = progress.add_task("Parsing files...", total=None, visible=False)
- task_graph = progress.add_task("Building dependency graph...", total=1, visible=False)
- task_git = progress.add_task("Indexing file history...", total=None, visible=False)
- task_cochange = progress.add_task("Analyzing co-changes...", total=None, visible=False)
-
- # Start git indexing in a background thread immediately — it uses
- # git ls-files for its own file list, fully independent of traversal.
- git_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="git-idx")
- try:
- from repowise.core.ingestion.git_indexer import GitIndexer
+ ) as progress_bar:
+ callback = RichProgressCallback(progress_bar, console)
- git_indexer = GitIndexer(
+ # Always run ingestion + analysis first (generate_docs=False).
+ # Generation happens separately after cost confirmation.
+ result = run_async(
+ run_pipeline(
repo_path,
- commit_limit=resolved_commit_limit,
+ commit_depth=resolved_commit_limit,
follow_renames=resolved_follow_renames,
+ skip_tests=skip_tests,
+ skip_infra=skip_infra,
+ exclude_patterns=exclude_patterns if exclude_patterns else None,
+ generate_docs=False,
+ llm_client=llm_client,
+ concurrency=concurrency,
+ test_run=test_run,
+ progress=callback,
)
+ )
- def _run_git_indexing() -> tuple:
- def on_start(total: int) -> None:
- progress.update(task_git, total=total, visible=True)
+ # ---- Phase 3: Generation (full mode only) ----
+ if not index_only:
+ print_phase_header(
+ console,
+ 3,
+ total_phases,
+ "Generation",
+ f"Generating wiki pages with {provider.provider_name} / {provider.model_name}",
+ )
- def on_file_done() -> None:
- progress.advance(task_git)
+ # Cost estimation
+ from repowise.core.generation import GenerationConfig
- def on_co_change_start(total: int) -> None:
- progress.update(task_cochange, total=total, visible=True)
+ gen_config = GenerationConfig(max_concurrency=concurrency)
+ plans = build_generation_plan(
+ result.parsed_files, result.graph_builder, gen_config, skip_tests, skip_infra
+ )
+ est = estimate_cost(plans, provider.provider_name, provider.model_name)
+
+ table = Table(title="Generation Plan", border_style=BRAND)
+ table.add_column("Page Type", style="cyan")
+ table.add_column("Count", justify="right")
+ table.add_column("Level", justify="right")
+ for plan in est.plans:
+ table.add_row(plan.page_type, str(plan.count), str(plan.level))
+ table.add_section()
+ table.add_row("[bold]Total[/bold]", f"[bold]{est.total_pages}[/bold]", "")
+ console.print(table)
- def on_commit_done() -> None:
- progress.advance(task_cochange)
+ console.print(
+ f" Estimated tokens: ~{est.estimated_input_tokens + est.estimated_output_tokens:,} "
+ f"(${est.estimated_cost_usd:.2f} USD)"
+ )
+ console.print()
- return run_async(
- git_indexer.index_repo(
- "",
- on_start=on_start,
- on_file_done=on_file_done,
- on_commit_done=on_commit_done,
- on_co_change_start=on_co_change_start,
- )
- )
+ if dry_run:
+ console.print("[yellow]Dry run — no pages generated.[/yellow]")
+ return
+
+ if (
+ est.estimated_cost_usd > 2.00
+ and not yes
+ and not click.confirm(" Estimated cost exceeds $2.00. Continue?")
+ ):
+ console.print("[yellow]Aborted.[/yellow]")
+ return
- git_future = git_executor.submit(_run_git_indexing)
- except Exception as exc:
- progress.console.print(f"[yellow]Git indexing skipped: {exc}[/yellow]")
-
- # Traverse: walk directory tree (sequential), then process files
- # with parallel I/O (stat + header reads are the bottleneck on Windows).
- traverser = FileTraverser(repo_path, extra_exclude_patterns=exclude_patterns or None)
- all_paths = list(traverser._walk()) # fast directory enumeration
- progress.update(task_traverse, total=len(all_paths))
-
- file_infos: list[Any] = []
- with ThreadPoolExecutor(max_workers=8) as io_pool:
- futures = [io_pool.submit(traverser._build_file_info, p) for p in all_paths]
- for future in as_completed(futures):
- try:
- result = future.result()
- except Exception:
- result = None
- if result is not None:
- file_infos.append(result)
- progress.advance(task_traverse)
-
- repo_structure = traverser.get_repo_structure(file_infos)
-
- # Filter
- if skip_tests:
- file_infos = [fi for fi in file_infos if not fi.is_test]
- if skip_infra:
- file_infos = [
- fi
- for fi in file_infos
- if fi.language not in ("dockerfile", "makefile", "terraform", "shell")
- ]
-
- # Parse (sequential — GraphBuilder is not thread-safe)
- progress.update(task_parse, total=len(file_infos), visible=True)
- parser = ASTParser()
- parsed_files: list[Any] = []
- source_map: dict[str, bytes] = {}
- graph_builder = GraphBuilder()
-
- for fi in file_infos:
+ # Build embedder + vector store
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+ from repowise.core.providers.embedding.base import MockEmbedder
+
+ embedder_impl: Any
+ if embedder_name_resolved == "gemini":
try:
- source = Path(fi.abs_path).read_bytes()
- parsed = parser.parse_file(fi, source)
- parsed_files.append(parsed)
- source_map[fi.path] = source
- graph_builder.add_file(parsed)
+ from repowise.core.providers.embedding.gemini import GeminiEmbedder
+
+ embedder_impl = GeminiEmbedder()
except Exception:
- pass # skip unparseable files
- progress.advance(task_parse)
- progress.update(task_parse, completed=len(file_infos))
+ embedder_impl = MockEmbedder()
+ elif embedder_name_resolved == "openai":
+ try:
+ from repowise.core.providers.embedding.openai import OpenAIEmbedder
- # Build graph
- progress.update(task_graph, visible=True)
- graph_builder.build()
+ embedder_impl = OpenAIEmbedder()
+ except Exception:
+ embedder_impl = MockEmbedder()
+ else:
+ embedder_impl = MockEmbedder()
- # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask)
+ lance_dir = repo_path / ".repowise" / "lancedb"
try:
- from repowise.core.generation.editor_files.tech_stack import detect_tech_stack
+ from repowise.core.persistence.vector_store import LanceDBVectorStore
- tech_items = detect_tech_stack(repo_path)
- graph_builder.add_framework_edges([item.name for item in tech_items])
- except Exception:
- pass # framework edge detection is best-effort
-
- progress.update(task_graph, completed=1)
-
- # Wait for git indexing to complete — show a spinner so the user
- # doesn't think it's stuck after the co-change bar hits 100%.
- if git_future is not None:
- task_git_finalize = progress.add_task(
- "Finalizing git analysis...",
- total=None,
- visible=True,
+ lance_dir.mkdir(parents=True, exist_ok=True)
+ vector_store: Any = LanceDBVectorStore(str(lance_dir), embedder=embedder_impl)
+ except ImportError:
+ vector_store = InMemoryVectorStore(embedder_impl)
+
+ # Run generation via the pipeline's generation function
+ from repowise.core.pipeline import run_generation
+
+ with Progress(
+ SpinnerColumn(),
+ TextColumn("[progress.description]{task.description}"),
+ BarColumn(),
+ TextColumn("{task.completed}/{task.total}"),
+ TimeElapsedColumn(),
+ console=console,
+ ) as gen_progress:
+ gen_callback = RichProgressCallback(gen_progress, console)
+
+ generated_pages = run_async(
+ run_generation(
+ repo_path=repo_path,
+ parsed_files=result.parsed_files,
+ source_map=result.source_map,
+ graph_builder=result.graph_builder,
+ repo_structure=result.repo_structure,
+ git_meta_map=result.git_meta_map,
+ llm_client=provider,
+ embedder=embedder_impl,
+ vector_store=vector_store,
+ concurrency=concurrency,
+ progress=gen_callback,
+ )
)
- try:
- git_summary, git_metadata_list = git_future.result()
- git_meta_map = {m["file_path"]: m for m in git_metadata_list}
- graph_builder.add_co_change_edges(git_meta_map)
- except Exception as exc:
- progress.console.print(f"[yellow]Git indexing failed: {exc}[/yellow]")
- progress.update(task_git_finalize, visible=False)
- git_executor.shutdown(wait=False)
+ result.generated_pages = generated_pages
+ console.print(f" [green]✓[/green] Generated [bold]{len(generated_pages)}[/bold] pages")
- if traverser._oversized_skip_count:
- console.print(
- f" Skipped [yellow]{traverser._oversized_skip_count}[/yellow] oversized files "
- f"(>{traverser.max_file_size_bytes // 1024} KB)"
- )
- console.print(f" [green]✓[/green] Ingested [bold]{len(parsed_files)}[/bold] files")
- if git_summary:
- console.print(
- f" [green]✓[/green] Git: [bold]{git_summary.files_indexed}[/bold] files "
- f"· {git_summary.hotspots} hotspots · {git_summary.stable_files} stable "
- f"({git_summary.duration_seconds:.1f}s)"
+ # ---- Persistence ----
+ if index_only:
+ print_phase_header(console, 3, total_phases, "Persistence", "Saving to database")
+ else:
+ print_phase_header(
+ console, 4, total_phases, "Persistence", "Saving to database and building search index"
)
- # ---- Test-run: limit to top 10 files by PageRank ----
- if test_run and not index_only:
- import networkx as nx
+ with console.status(" Persisting to database…", spinner="dots"):
+ run_async(_persist_result(result, repo_path))
+ console.print(" [green]✓[/green] Database updated")
- graph = graph_builder.graph()
+ # ---- Post-run: config, state, MCP, CLAUDE.md ----
+ if commit_limit is not None:
+ cfg = load_config(repo_path)
+ cfg["commit_limit"] = resolved_commit_limit
try:
- ranks = nx.pagerank(graph)
- except Exception:
- ranks = {}
- parsed_files = sorted(
- parsed_files,
- key=lambda pf: ranks.get(pf.file_info.path, 0),
- reverse=True,
- )[:10]
- console.print(f"[yellow]Test run: limiting to {len(parsed_files)} files[/yellow]")
-
- # ---- Phase 2: Analysis ----
- print_phase_header(
- console,
- 2,
- total_phases,
- "Analysis",
- "Dead code detection and architectural decision extraction",
- )
+ import yaml # type: ignore[import-untyped]
- dead_code_report = None
- try:
- from repowise.core.analysis.dead_code import DeadCodeAnalyzer
+ cfg_path = repo_path / ".repowise" / "config.yaml"
+ cfg_path.write_text(
+ yaml.dump(cfg, default_flow_style=False, sort_keys=False),
+ encoding="utf-8",
+ )
+ except ImportError:
+ pass
- with console.status(" Detecting dead code…", spinner="dots"):
- analyzer = DeadCodeAnalyzer(graph_builder.graph(), git_meta_map)
- dead_code_report = analyzer.analyze()
- unreachable = sum(
- 1 for f in dead_code_report.findings if f.kind.value == "unreachable_file"
- )
- unused_exports = sum(
- 1 for f in dead_code_report.findings if f.kind.value == "unused_export"
- )
- console.print(
- f" [green]✓[/green] Dead code: [yellow]{unreachable}[/yellow] unreachable files "
- f"· [yellow]{unused_exports}[/yellow] unused exports "
- f"(~{dead_code_report.deletable_lines:,} lines)"
- )
- except Exception as exc:
- console.print(f" [yellow]Dead code detection skipped: {exc}[/yellow]")
+ from repowise.cli.mcp_config import save_mcp_config, save_root_mcp_config
- decision_report = None
- try:
- from repowise.core.analysis.decision_extractor import DecisionExtractor
-
- # Use decision_provider in index-only mode (provider is None but
- # decision_provider may still be set for decision extraction)
- _decision_llm = provider or (decision_provider if index_only else None)
- extractor = DecisionExtractor(
- repo_path=repo_path,
- provider=_decision_llm,
- graph=graph_builder.graph(),
- git_meta_map=git_meta_map,
- parsed_files=parsed_files,
- )
- with console.status(" Extracting architectural decisions…", spinner="dots"):
- decision_report = run_async(extractor.extract_all())
- inline = decision_report.by_source.get("inline_marker", 0)
- readme = decision_report.by_source.get("readme_mining", 0)
- git_arch = decision_report.by_source.get("git_archaeology", 0)
- console.print(
- f" [green]✓[/green] Decisions: [green]{inline}[/green] inline "
- f"· [yellow]{readme}[/yellow] from docs · [yellow]{git_arch}[/yellow] from git"
- )
- except Exception as exc:
- console.print(f" [yellow]Decision extraction skipped: {exc}[/yellow]")
+ save_mcp_config(repo_path)
+ save_root_mcp_config(repo_path)
+ _register_mcp_with_claude(console, repo_path)
- # ---- Index-only: persist and show completion panel ----
- if index_only:
- print_phase_header(console, 3, total_phases, "Persistence", "Saving to database")
+ _maybe_generate_claude_md(console, repo_path, no_claude_md=no_claude_md)
- with console.status(" Persisting index…", spinner="dots"):
- run_async(
- _persist_index_only(
- repo_path=repo_path,
- repo_name=repo_path.name,
- graph_builder=graph_builder,
- parsed_files=parsed_files,
- git_metadata_list=git_metadata_list,
- dead_code_report=dead_code_report,
- decision_report=decision_report,
+ # ---- State + config (full mode only) ----
+ if not index_only and provider:
+ async def _count_db_pages() -> int:
+ from sqlalchemy import func as sa_func
+ from sqlalchemy import select as sa_select
+
+ from repowise.cli.helpers import get_db_url_for_repo as _get_url
+ from repowise.core.persistence import create_engine, create_session_factory, get_session
+ from repowise.core.persistence.models import Page, Repository
+
+ _engine = create_engine(_get_url(repo_path))
+ _sf = create_session_factory(_engine)
+ async with get_session(_sf) as _sess:
+ repo_result = await _sess.execute(
+ sa_select(Repository.id).where(Repository.local_path == str(repo_path))
)
- )
- # Persist commit_limit to config so `repowise update` picks it up
- if commit_limit is not None:
- cfg = load_config(repo_path)
- cfg["commit_limit"] = resolved_commit_limit
- try:
- import yaml # type: ignore[import-untyped]
-
- cfg_path = repo_path / ".repowise" / "config.yaml"
- cfg_path.write_text(
- yaml.dump(cfg, default_flow_style=False, sort_keys=False),
- encoding="utf-8",
+ _repo_id = repo_result.scalar_one_or_none()
+ if _repo_id is None:
+ await _engine.dispose()
+ return len(result.generated_pages or [])
+
+ count_result = await _sess.execute(
+ sa_select(sa_func.count())
+ .select_from(Page)
+ .where(Page.repository_id == _repo_id)
)
- except ImportError:
- pass # No yaml — commit_limit will use default next time
-
- # MCP config
- from repowise.cli.mcp_config import (
- save_mcp_config,
- save_root_mcp_config,
+ count = count_result.scalar_one()
+ await _engine.dispose()
+ return count
+
+ head = get_head_commit(repo_path)
+ state = load_state(repo_path)
+ state["last_sync_commit"] = head
+ state["total_pages"] = run_async(_count_db_pages())
+ state["provider"] = provider.provider_name
+ state["model"] = provider.model_name
+ total_tokens = sum(p.total_tokens for p in (result.generated_pages or []))
+ state["total_tokens"] = total_tokens
+ save_state(repo_path, state)
+
+ save_config(
+ repo_path,
+ provider.provider_name,
+ provider.model_name,
+ embedder_name_resolved,
+ exclude_patterns=exclude_patterns if exclude_patterns else None,
+ commit_limit=resolved_commit_limit if commit_limit is not None else None,
)
- save_mcp_config(repo_path)
- save_root_mcp_config(repo_path)
- _register_mcp_with_claude(console, repo_path)
-
- # .claude/CLAUDE.md (index-only: structural data is available)
- _maybe_generate_claude_md(console, repo_path, no_claude_md=no_claude_md)
-
- elapsed = time.monotonic() - start
+ # ---- Completion panel ----
+ elapsed = time.monotonic() - start
- # Collect stats for the completion panel
- _graph = graph_builder.graph()
- _langs = {fi.language for fi in file_infos if hasattr(fi, "language") and fi.language}
- _dc_unreachable = sum(
- 1
- for f in (dead_code_report.findings if dead_code_report else [])
- if f.kind.value == "unreachable_file"
- )
- _dc_unused = sum(
- 1
- for f in (dead_code_report.findings if dead_code_report else [])
- if f.kind.value == "unused_export"
- )
- _n_decisions = sum(decision_report.by_source.values()) if decision_report else 0
+ _graph = result.graph_builder.graph()
+ _dc_unreachable = sum(
+ 1
+ for f in (result.dead_code_report.findings if result.dead_code_report else [])
+ if f.kind.value == "unreachable_file"
+ )
+ _dc_unused = sum(
+ 1
+ for f in (result.dead_code_report.findings if result.dead_code_report else [])
+ if f.kind.value == "unused_export"
+ )
+ _n_decisions = sum(result.decision_report.by_source.values()) if result.decision_report else 0
+ if index_only:
metrics: list[tuple[str, str]] = [
- ("Files indexed", str(len(parsed_files))),
- ("Symbols", f"{sum(len(pf.symbols) for pf in parsed_files):,}"),
- ("Languages", str(len(_langs))),
+ ("Files indexed", str(result.file_count)),
+ ("Symbols", f"{result.symbol_count:,}"),
+ ("Languages", str(len(result.languages))),
("Elapsed", format_elapsed(elapsed)),
("", ""),
- ("Graph", f"{_graph.number_of_nodes()} nodes · {_graph.number_of_edges()} edges"),
+ (
+ "Graph",
+ f"{_graph.number_of_nodes()} nodes · {_graph.number_of_edges()} edges",
+ ),
("Dead code", f"{_dc_unreachable} unreachable · {_dc_unused} unused exports"),
("Decisions", str(_n_decisions)),
]
- if git_summary:
+ if result.git_summary:
metrics.append(
(
"Git history",
- f"{git_summary.files_indexed} files · {git_summary.hotspots} hotspots",
+ f"{result.git_summary.files_indexed} files · {result.git_summary.hotspots} hotspots",
)
)
@@ -788,346 +755,29 @@ def init_command(
build_completion_panel("repowise index complete", metrics, next_steps=next_steps)
)
console.print()
- return
-
- # ---- Phase 3: Generation ----
- print_phase_header(
- console,
- 3,
- total_phases,
- "Generation",
- f"Generating wiki pages with {provider.provider_name} / {provider.model_name}",
- )
-
- plans = build_generation_plan(parsed_files, graph_builder, config, skip_tests, skip_infra)
- est = estimate_cost(plans, provider.provider_name, provider.model_name)
-
- table = Table(title="Generation Plan", border_style=BRAND)
- table.add_column("Page Type", style="cyan")
- table.add_column("Count", justify="right")
- table.add_column("Level", justify="right")
- for plan in est.plans:
- table.add_row(plan.page_type, str(plan.count), str(plan.level))
- table.add_section()
- table.add_row("[bold]Total[/bold]", f"[bold]{est.total_pages}[/bold]", "")
- console.print(table)
-
- console.print(
- f" Estimated tokens: ~{est.estimated_input_tokens + est.estimated_output_tokens:,} "
- f"(${est.estimated_cost_usd:.2f} USD)"
- )
- console.print()
-
- if dry_run:
- console.print("[yellow]Dry run — no pages generated.[/yellow]")
- return
-
- if (
- est.estimated_cost_usd > 2.00
- and not yes
- and not click.confirm(" Estimated cost exceeds $2.00. Continue?")
- ):
- console.print("[yellow]Aborted.[/yellow]")
- return
-
- # ---- Generate pages ----
- from repowise.core.generation import ContextAssembler, JobSystem, PageGenerator
- from repowise.core.persistence.vector_store import InMemoryVectorStore
- from repowise.core.providers.embedding.base import MockEmbedder
-
- assembler = ContextAssembler(config)
-
- # Build vector store for RAG context (B1)
- embedder_impl: Any
- embedder_resolved = _resolve_embedder(embedder_name)
- if embedder_resolved == "gemini":
- try:
- from repowise.core.providers.embedding.gemini import GeminiEmbedder
-
- embedder_impl = GeminiEmbedder()
- except Exception:
- embedder_impl = MockEmbedder()
- elif embedder_resolved == "openai":
- try:
- from repowise.core.providers.embedding.openai import OpenAIEmbedder
-
- embedder_impl = OpenAIEmbedder()
- except Exception:
- embedder_impl = MockEmbedder()
else:
- embedder_impl = MockEmbedder()
- # Try LanceDB for persistent vector storage, fall back to in-memory
- lance_dir = repo_path / ".repowise" / "lancedb"
- try:
- from repowise.core.persistence.vector_store import LanceDBVectorStore
-
- lance_dir.mkdir(parents=True, exist_ok=True)
- vector_store = LanceDBVectorStore(str(lance_dir), embedder=embedder_impl)
- except ImportError:
- vector_store = InMemoryVectorStore(embedder_impl)
-
- generator = PageGenerator(provider, assembler, config, vector_store=vector_store)
-
- job_system = None
- jobs_dir = Path(config.jobs_dir)
- if not jobs_dir.is_absolute():
- jobs_dir = repo_path / jobs_dir
- job_system = JobSystem(jobs_dir)
-
- repo_name = repo_path.name
-
- with Progress(
- SpinnerColumn(),
- TextColumn("[progress.description]{task.description}"),
- BarColumn(),
- TextColumn("{task.completed}/{task.total}"),
- TimeElapsedColumn(),
- console=console,
- ) as gen_progress:
- task_gen = gen_progress.add_task("Generating pages...", total=est.total_pages)
-
- _pages_done = 0
-
- def on_page_done(page_type: str) -> None:
- nonlocal _pages_done
- _pages_done += 1
- # Update total if actual count exceeds estimate
- current_total = gen_progress.tasks[task_gen].total or 0
- if _pages_done > current_total:
- gen_progress.update(task_gen, total=_pages_done + 5)
- gen_progress.advance(task_gen)
- gen_progress.update(task_gen, description=f"Generating pages... [{page_type}]")
-
- generated_pages = run_async(
- generator.generate_all(
- parsed_files,
- source_map,
- graph_builder,
- repo_structure,
- repo_name,
- job_system=job_system,
- on_page_done=on_page_done,
- git_meta_map=git_meta_map if git_meta_map else None,
- )
- )
-
- console.print(f" [green]✓[/green] Generated [bold]{len(generated_pages)}[/bold] pages")
-
- # ---- Phase 4: Persistence ----
- print_phase_header(
- console,
- 4,
- total_phases,
- "Persistence",
- "Saving to database and building search index",
- )
-
- async def _persist() -> None:
- from repowise.cli.helpers import get_db_url_for_repo
- from repowise.core.persistence import (
- FullTextSearch,
- batch_upsert_graph_edges,
- batch_upsert_graph_nodes,
- batch_upsert_symbols,
- create_engine,
- create_session_factory,
- get_session,
- init_db,
- upsert_page_from_generated,
- upsert_repository,
- )
- from repowise.core.persistence.crud import (
- save_dead_code_findings,
- upsert_git_metadata_bulk,
- )
-
- url = get_db_url_for_repo(repo_path)
- engine = create_engine(url)
- await init_db(engine)
- sf = create_session_factory(engine)
-
- async with get_session(sf) as session:
- repo = await upsert_repository(
- session,
- name=repo_name,
- local_path=str(repo_path),
- )
- repo_id = repo.id
-
- for page in generated_pages:
- await upsert_page_from_generated(session, page, repo_id)
-
- # Graph nodes
- graph = graph_builder.graph()
- pr = graph_builder.pagerank()
- bc = graph_builder.betweenness_centrality()
- cd = graph_builder.community_detection()
- nodes = []
- for node_path in graph.nodes:
- data = graph.nodes[node_path]
- nodes.append(
- {
- "node_id": node_path,
- "symbol_count": data.get("symbol_count", 0),
- "has_error": data.get("has_error", False),
- "is_test": data.get("is_test", False),
- "is_entry_point": data.get("is_entry_point", False),
- "language": data.get("language", "unknown"),
- "pagerank": pr.get(node_path, 0.0),
- "betweenness": bc.get(node_path, 0.0),
- "community_id": cd.get(node_path, 0),
- }
- )
- if nodes:
- await batch_upsert_graph_nodes(session, repo_id, nodes)
-
- # Graph edges
- edges = []
- for u, v, data in graph.edges(data=True):
- edges.append(
- {
- "source_node_id": u,
- "target_node_id": v,
- "imported_names_json": json.dumps(data.get("imported_names", [])),
- "edge_type": data.get("edge_type", "imports"),
- }
- )
- if edges:
- await batch_upsert_graph_edges(session, repo_id, edges)
-
- # Symbols — pass actual Symbol objects (duck-typed)
- all_symbols = []
- for pf in parsed_files:
- for sym in pf.symbols:
- # Attach file_path to symbol for batch_upsert_symbols
- sym.file_path = pf.file_info.path
- all_symbols.append(sym)
- if all_symbols:
- await batch_upsert_symbols(session, repo_id, all_symbols)
-
- # Git metadata
- if git_metadata_list:
- await upsert_git_metadata_bulk(session, repo_id, git_metadata_list)
-
- # Dead code findings
- if dead_code_report and dead_code_report.findings:
- await save_dead_code_findings(session, repo_id, dead_code_report.findings)
-
- # Decision records
- if decision_report and decision_report.decisions:
- import dataclasses as _dc
-
- from repowise.core.persistence.crud import bulk_upsert_decisions
-
- await bulk_upsert_decisions(
- session,
- repo_id,
- [_dc.asdict(d) for d in decision_report.decisions],
+ total_tokens = sum(p.total_tokens for p in (result.generated_pages or []))
+ metrics = [
+ ("Pages generated", str(len(result.generated_pages or []))),
+ ("Total tokens", f"{total_tokens:,}"),
+ ("Provider", f"{provider.provider_name} / {provider.model_name}"),
+ ("Elapsed", format_elapsed(elapsed)),
+ ("", ""),
+ ("Dead code", f"{_dc_unreachable} unreachable · {_dc_unused} unused exports"),
+ ("Decisions", str(_n_decisions)),
+ ]
+ if result.git_summary:
+ metrics.append(
+ (
+ "Git history",
+ f"{result.git_summary.files_indexed} files · {result.git_summary.hotspots} hotspots",
)
-
- # FTS indexing
- fts = FullTextSearch(engine)
- await fts.ensure_index()
- for page in generated_pages:
- await fts.index(page.page_id, page.title, page.content)
-
- await engine.dispose()
-
- with console.status(" Persisting to database…", spinner="dots"):
- run_async(_persist())
- console.print(" [green]✓[/green] Database updated")
-
- # ---- CLAUDE.md ----
- _maybe_generate_claude_md(console, repo_path, no_claude_md=no_claude_md)
-
- # ---- State ----
- # Query actual DB page count (not just current job's pages)
- async def _count_db_pages() -> int:
- from sqlalchemy import func as sa_func
- from sqlalchemy import select as sa_select
-
- from repowise.cli.helpers import get_db_url_for_repo as _get_url
- from repowise.core.persistence import create_engine, create_session_factory, get_session
- from repowise.core.persistence.models import Page, Repository
-
- _engine = create_engine(_get_url(repo_path))
- _sf = create_session_factory(_engine)
- async with get_session(_sf) as _sess:
- repo_result = await _sess.execute(
- sa_select(Repository.id).where(Repository.local_path == str(repo_path))
)
- _repo_id = repo_result.scalar_one_or_none()
- if _repo_id is None:
- await _engine.dispose()
- return len(generated_pages) # fallback
-
- result = await _sess.execute(
- sa_select(sa_func.count()).select_from(Page).where(Page.repository_id == _repo_id)
- )
- count = result.scalar_one()
- await _engine.dispose()
- return count
-
- head = get_head_commit(repo_path)
- state = load_state(repo_path)
- state["last_sync_commit"] = head
- state["total_pages"] = run_async(_count_db_pages())
- state["provider"] = provider.provider_name
- state["model"] = provider.model_name
- total_tokens = sum(p.total_tokens for p in generated_pages)
- state["total_tokens"] = total_tokens
- save_state(repo_path, state)
-
- save_config(
- repo_path,
- provider.provider_name,
- provider.model_name,
- embedder,
- exclude_patterns=exclude_patterns if exclude_patterns else None,
- commit_limit=resolved_commit_limit if commit_limit is not None else None,
- )
-
- # ---- Completion ----
- from repowise.cli.mcp_config import (
- format_setup_instructions,
- save_mcp_config,
- save_root_mcp_config,
- )
-
- save_mcp_config(repo_path)
- save_root_mcp_config(repo_path)
- _register_mcp_with_claude(console, repo_path)
-
- elapsed = time.monotonic() - start
- _dc_unreachable = sum(
- 1
- for f in (dead_code_report.findings if dead_code_report else [])
- if f.kind.value == "unreachable_file"
- )
- _dc_unused = sum(
- 1
- for f in (dead_code_report.findings if dead_code_report else [])
- if f.kind.value == "unused_export"
- )
- _n_decisions = sum(decision_report.by_source.values()) if decision_report else 0
-
- metrics = [
- ("Pages generated", str(len(generated_pages))),
- ("Total tokens", f"{total_tokens:,}"),
- ("Provider", f"{provider.provider_name} / {provider.model_name}"),
- ("Elapsed", format_elapsed(elapsed)),
- ("", ""),
- ("Dead code", f"{_dc_unreachable} unreachable · {_dc_unused} unused exports"),
- ("Decisions", str(_n_decisions)),
- ]
- if git_summary:
- metrics.append(
- ("Git history", f"{git_summary.files_indexed} files · {git_summary.hotspots} hotspots")
- )
+ from repowise.cli.mcp_config import format_setup_instructions
- console.print()
- console.print(build_completion_panel("repowise init complete", metrics))
- console.print()
- console.print(format_setup_instructions(repo_path))
- console.print()
+ console.print()
+ console.print(build_completion_panel("repowise init complete", metrics))
+ console.print()
+ console.print(format_setup_instructions(repo_path))
+ console.print()
diff --git a/packages/cli/src/repowise/cli/ui.py b/packages/cli/src/repowise/cli/ui.py
index 42b29c25d..3c500d1d4 100644
--- a/packages/cli/src/repowise/cli/ui.py
+++ b/packages/cli/src/repowise/cli/ui.py
@@ -528,3 +528,57 @@ def format_elapsed(seconds: float) -> str:
s = int(seconds) % 60
return f"{m}m {s}s"
return f"{seconds:.1f}s"
+
+
+# ---------------------------------------------------------------------------
+# Rich progress callback — implements core ProgressCallback protocol
+# ---------------------------------------------------------------------------
+
+_PHASE_LABELS: dict[str, str] = {
+ "traverse": "Traversing files...",
+ "parse": "Parsing files...",
+ "graph": "Building dependency graph...",
+ "git": "Indexing file history...",
+ "co_change": "Analyzing co-changes...",
+ "dead_code": "Detecting dead code...",
+ "decisions": "Extracting decisions...",
+ "generation": "Generating pages...",
+}
+
+
+class RichProgressCallback:
+ """Adapter that implements ``repowise.core.pipeline.ProgressCallback``
+ using a Rich ``Progress`` instance for terminal display.
+
+ Usage::
+
+ from rich.progress import Progress
+ with Progress(...) as progress_bar:
+ callback = RichProgressCallback(progress_bar, console)
+ result = run_async(run_pipeline(..., progress=callback))
+ """
+
+ def __init__(self, progress: Any, console: Console) -> None:
+ self._progress = progress
+ self._console = console
+ self._tasks: dict[str, Any] = {}
+
+ def on_phase_start(self, phase: str, total: int | None) -> None:
+ label = _PHASE_LABELS.get(phase, f"{phase}...")
+ # If phase already has a task, update its total and make visible
+ if phase in self._tasks:
+ self._progress.update(self._tasks[phase], total=total, visible=True)
+ else:
+ self._tasks[phase] = self._progress.add_task(label, total=total, visible=True)
+
+ def on_item_done(self, phase: str) -> None:
+ if phase in self._tasks:
+ self._progress.advance(self._tasks[phase])
+
+ def on_message(self, level: str, text: str) -> None:
+ style_map = {"info": OK, "warning": WARN, "error": ERR}
+ style = style_map.get(level, "")
+ if style:
+ self._progress.console.print(f" [{style}]{text}[/{style}]")
+ else:
+ self._progress.console.print(f" {text}")
diff --git a/packages/core/src/repowise/core/__init__.py b/packages/core/src/repowise/core/__init__.py
index 354cada42..4c4b7108f 100644
--- a/packages/core/src/repowise/core/__init__.py
+++ b/packages/core/src/repowise/core/__init__.py
@@ -6,4 +6,4 @@ generation engine. This is the heart of repowise — all other packages depend o
Namespace package: repowise.core is part of the repowise namespace.
"""
-__version__ = "0.1.29"
+__version__ = "0.1.30"
diff --git a/packages/core/src/repowise/core/pipeline/__init__.py b/packages/core/src/repowise/core/pipeline/__init__.py
new file mode 100644
index 000000000..d4fa9eb29
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/__init__.py
@@ -0,0 +1,22 @@
+"""repowise pipeline — programmatic API for running the indexing pipeline.
+
+Usage::
+
+ import asyncio
+ from pathlib import Path
+ from repowise.core.pipeline import run_pipeline
+
+ result = asyncio.run(run_pipeline(Path("/path/to/repo"), generate_docs=False))
+ print(f"Indexed {result.file_count} files, {result.symbol_count} symbols")
+"""
+
+from .orchestrator import PipelineResult, run_generation, run_pipeline
+from .progress import LoggingProgressCallback, ProgressCallback
+
+__all__ = [
+ "LoggingProgressCallback",
+ "PipelineResult",
+ "ProgressCallback",
+ "run_generation",
+ "run_pipeline",
+]
diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.py
new file mode 100644
index 000000000..4618fcab9
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py
@@ -0,0 +1,567 @@
+"""Programmatic pipeline orchestrator for repowise.
+
+Provides ``run_pipeline()`` — the single entry point for running the full
+repowise indexing/analysis/generation pipeline without any CLI dependencies.
+
+This module has **zero** imports from ``repowise.cli``, ``click``, or ``rich``.
+All progress reporting is done through the optional ``ProgressCallback`` protocol.
+
+Callers:
+ - CLI (``init_cmd.py``) — passes a Rich-backed ProgressCallback, persists to SQLite
+ - Modal worker (Phase 2) — passes LoggingProgressCallback, serializes to files
+ - Tests — passes None, inspects PipelineResult in memory
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import structlog
+
+from repowise.core.pipeline.progress import ProgressCallback
+
+logger = structlog.get_logger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Result dataclass
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class PipelineResult:
+ """All outputs from a pipeline run, held in memory.
+
+ The caller decides how to persist — SQLite, files for upload, or nothing.
+ """
+
+ # Ingestion
+ parsed_files: list[Any]
+ """List of ``ParsedFile`` objects from the AST parser."""
+
+ file_infos: list[Any]
+ """All traversed ``FileInfo`` objects (pre-filter)."""
+
+ repo_structure: Any
+ """``RepoStructure`` — monorepo detection result."""
+
+ source_map: dict[str, bytes]
+ """Mapping of relative file path → raw source bytes."""
+
+ # Graph
+ graph_builder: Any
+ """``GraphBuilder`` instance — call ``.graph()``, ``.pagerank()``, etc."""
+
+ # Git
+ git_metadata_list: list[dict]
+ """Raw metadata dicts ready for ``upsert_git_metadata_bulk``."""
+
+ git_meta_map: dict[str, dict]
+ """File path → git metadata dict."""
+
+ git_summary: Any | None
+ """``GitIndexSummary`` or None if git indexing was skipped."""
+
+ # Analysis
+ dead_code_report: Any | None
+ """``DeadCodeReport`` or None."""
+
+ decision_report: Any | None
+ """``DecisionExtractionReport`` or None."""
+
+ # Generation (None when generate_docs=False)
+ generated_pages: list[Any] | None
+ """List of ``GeneratedPage`` objects, or None if docs weren't generated."""
+
+ # Stats
+ repo_name: str
+ file_count: int
+ symbol_count: int
+ languages: set[str] = field(default_factory=set)
+ elapsed_seconds: float = 0.0
+
+
+# ---------------------------------------------------------------------------
+# Pipeline
+# ---------------------------------------------------------------------------
+
+
+async def run_pipeline(
+ repo_path: Path | str,
+ *,
+ commit_depth: int = 500,
+ follow_renames: bool = False,
+ skip_tests: bool = False,
+ skip_infra: bool = False,
+ exclude_patterns: list[str] | None = None,
+ generate_docs: bool = False,
+ llm_client: Any | None = None,
+ embedder: Any | None = None,
+ vector_store: Any | None = None,
+ concurrency: int = 5,
+ test_run: bool = False,
+ progress: ProgressCallback | None = None,
+) -> PipelineResult:
+ """Run the repowise indexing/analysis/generation pipeline.
+
+ Parameters
+ ----------
+ repo_path:
+ Path to an already-cloned repository on disk.
+ commit_depth:
+ Maximum commits to analyse per file (1–5000). Default 500.
+ follow_renames:
+ Use ``git log --follow`` to track files across renames.
+ skip_tests:
+ Exclude test files from parsing.
+ skip_infra:
+ Exclude infrastructure files (Dockerfile, Makefile, etc.) from parsing.
+ exclude_patterns:
+ Additional gitignore-style exclusion patterns.
+ generate_docs:
+ When True, run LLM page generation (requires *llm_client*).
+ llm_client:
+ A configured ``BaseProvider`` instance for LLM calls. Required when
+ *generate_docs* is True, optional otherwise (used for decision extraction).
+ embedder:
+ A configured embedder instance for vector embeddings. Falls back to
+ ``MockEmbedder`` when None.
+ vector_store:
+ A pre-constructed vector store (e.g. ``LanceDBVectorStore``). Falls
+ back to ``InMemoryVectorStore`` when None and *generate_docs* is True.
+ concurrency:
+ Maximum concurrent LLM calls during generation.
+ test_run:
+ Limit generation to top 10 files by PageRank (for quick validation).
+ progress:
+ Optional callback for progress reporting. Pass None for silent operation.
+
+ Returns
+ -------
+ PipelineResult
+ All pipeline outputs held in memory.
+ """
+ repo_path = Path(repo_path).resolve()
+ start = time.monotonic()
+
+ commit_depth = max(1, min(commit_depth, 5000))
+
+ # ---- Phase 1: Ingestion ------------------------------------------------
+ if progress:
+ progress.on_message("info", "Phase 1: Ingestion")
+
+ parsed_files, file_infos, repo_structure, source_map, graph_builder = (
+ await _run_ingestion(
+ repo_path,
+ exclude_patterns=exclude_patterns,
+ skip_tests=skip_tests,
+ skip_infra=skip_infra,
+ progress=progress,
+ )
+ )
+
+ # Git indexing (runs concurrently with ingestion in the CLI, but here
+ # we start it after traversal since we're already async)
+ git_summary, git_metadata_list, git_meta_map = await _run_git_indexing(
+ repo_path,
+ commit_depth=commit_depth,
+ follow_renames=follow_renames,
+ progress=progress,
+ )
+
+ # Add co-change edges to the graph
+ if git_meta_map:
+ graph_builder.add_co_change_edges(git_meta_map)
+
+ if progress:
+ progress.on_message(
+ "info",
+ f"Ingested {len(parsed_files)} files"
+ + (
+ f" · Git: {git_summary.files_indexed} files"
+ if git_summary and git_summary.files_indexed
+ else ""
+ ),
+ )
+
+ # Test-run: limit to top 10 files by PageRank
+ if test_run and generate_docs:
+ try:
+ import networkx as nx
+
+ ranks = nx.pagerank(graph_builder.graph())
+ except Exception:
+ ranks = {}
+ parsed_files = sorted(
+ parsed_files,
+ key=lambda pf: ranks.get(pf.file_info.path, 0),
+ reverse=True,
+ )[:10]
+ if progress:
+ progress.on_message("warning", f"Test run: limiting to {len(parsed_files)} files")
+
+ # ---- Phase 2: Analysis --------------------------------------------------
+ if progress:
+ progress.on_message("info", "Phase 2: Analysis")
+
+ dead_code_report = await _run_dead_code_analysis(
+ graph_builder, git_meta_map, progress=progress
+ )
+
+ decision_report = await _run_decision_extraction(
+ repo_path,
+ llm_client=llm_client,
+ graph_builder=graph_builder,
+ git_meta_map=git_meta_map,
+ parsed_files=parsed_files,
+ progress=progress,
+ )
+
+ # ---- Phase 3: Generation (optional) ------------------------------------
+ generated_pages: list[Any] | None = None
+ if generate_docs and llm_client is not None:
+ if progress:
+ progress.on_message("info", "Phase 3: Generation")
+
+ generated_pages = await run_generation(
+ repo_path=repo_path,
+ parsed_files=parsed_files,
+ source_map=source_map,
+ graph_builder=graph_builder,
+ repo_structure=repo_structure,
+ git_meta_map=git_meta_map,
+ llm_client=llm_client,
+ embedder=embedder,
+ vector_store=vector_store,
+ concurrency=concurrency,
+ progress=progress,
+ )
+
+ # ---- Build result -------------------------------------------------------
+ elapsed = time.monotonic() - start
+ languages = {fi.language for fi in file_infos if hasattr(fi, "language") and fi.language}
+ symbol_count = sum(len(pf.symbols) for pf in parsed_files)
+
+ return PipelineResult(
+ parsed_files=parsed_files,
+ file_infos=file_infos,
+ repo_structure=repo_structure,
+ source_map=source_map,
+ graph_builder=graph_builder,
+ git_metadata_list=git_metadata_list,
+ git_meta_map=git_meta_map,
+ git_summary=git_summary,
+ dead_code_report=dead_code_report,
+ decision_report=decision_report,
+ generated_pages=generated_pages,
+ repo_name=repo_path.name,
+ file_count=len(parsed_files),
+ symbol_count=symbol_count,
+ languages=languages,
+ elapsed_seconds=elapsed,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Phase helpers (private)
+# ---------------------------------------------------------------------------
+
+
+async def _run_ingestion(
+ repo_path: Path,
+ *,
+ exclude_patterns: list[str] | None,
+ skip_tests: bool,
+ skip_infra: bool,
+ progress: ProgressCallback | None,
+) -> tuple[list[Any], list[Any], Any, dict[str, bytes], Any]:
+ """Traverse, parse, and build the dependency graph.
+
+ Returns (parsed_files, file_infos, repo_structure, source_map, graph_builder).
+ """
+ from repowise.core.ingestion import ASTParser, FileTraverser, GraphBuilder
+
+ traverser = FileTraverser(repo_path, extra_exclude_patterns=exclude_patterns or None)
+
+ # Walk directory tree
+ all_paths = list(traverser._walk())
+ if progress:
+ progress.on_phase_start("traverse", len(all_paths))
+
+ # Parallel stat + header reads (I/O bound)
+ file_infos: list[Any] = []
+ with ThreadPoolExecutor(max_workers=8) as io_pool:
+ futures = [io_pool.submit(traverser._build_file_info, p) for p in all_paths]
+ for future in as_completed(futures):
+ try:
+ result = future.result()
+ except Exception:
+ result = None
+ if result is not None:
+ file_infos.append(result)
+ if progress:
+ progress.on_item_done("traverse")
+
+ repo_structure = traverser.get_repo_structure(file_infos)
+
+ # Filter
+ if skip_tests:
+ file_infos = [fi for fi in file_infos if not fi.is_test]
+ if skip_infra:
+ file_infos = [
+ fi
+ for fi in file_infos
+ if fi.language not in ("dockerfile", "makefile", "terraform", "shell")
+ ]
+
+ # Parse (sequential — GraphBuilder is not thread-safe)
+ if progress:
+ progress.on_phase_start("parse", len(file_infos))
+
+ parser = ASTParser()
+ parsed_files: list[Any] = []
+ source_map: dict[str, bytes] = {}
+ graph_builder = GraphBuilder()
+
+ for fi in file_infos:
+ try:
+ source = Path(fi.abs_path).read_bytes()
+ parsed = parser.parse_file(fi, source)
+ parsed_files.append(parsed)
+ source_map[fi.path] = source
+ graph_builder.add_file(parsed)
+ except Exception:
+ pass # skip unparseable files
+ if progress:
+ progress.on_item_done("parse")
+
+ # Build graph
+ if progress:
+ progress.on_phase_start("graph", 1)
+ graph_builder.build()
+
+ # Add framework-aware synthetic edges (conftest, Django, FastAPI, Flask)
+ try:
+ from repowise.core.generation.editor_files.tech_stack import detect_tech_stack
+
+ tech_items = detect_tech_stack(repo_path)
+ graph_builder.add_framework_edges([item.name for item in tech_items])
+ except Exception:
+ pass # framework edge detection is best-effort
+
+ if progress:
+ progress.on_item_done("graph")
+
+ # Report oversized file skips
+ if traverser._oversized_skip_count and progress:
+ progress.on_message(
+ "warning",
+ f"Skipped {traverser._oversized_skip_count} oversized files "
+ f"(>{traverser.max_file_size_bytes // 1024} KB)",
+ )
+
+ return parsed_files, file_infos, repo_structure, source_map, graph_builder
+
+
+async def _run_git_indexing(
+ repo_path: Path,
+ *,
+ commit_depth: int,
+ follow_renames: bool,
+ progress: ProgressCallback | None,
+) -> tuple[Any | None, list[dict], dict[str, dict]]:
+ """Run git history indexing.
+
+ Returns (git_summary, git_metadata_list, git_meta_map).
+ """
+ try:
+ from repowise.core.ingestion.git_indexer import GitIndexer
+
+ git_indexer = GitIndexer(
+ repo_path,
+ commit_limit=commit_depth,
+ follow_renames=follow_renames,
+ )
+
+ def _on_start(total: int) -> None:
+ if progress:
+ progress.on_phase_start("git", total)
+
+ def _on_file_done() -> None:
+ if progress:
+ progress.on_item_done("git")
+
+ def _on_co_change_start(total: int) -> None:
+ if progress:
+ progress.on_phase_start("co_change", total)
+
+ def _on_commit_done() -> None:
+ if progress:
+ progress.on_item_done("co_change")
+
+ git_summary, git_metadata_list = await git_indexer.index_repo(
+ "",
+ on_start=_on_start,
+ on_file_done=_on_file_done,
+ on_co_change_start=_on_co_change_start,
+ on_commit_done=_on_commit_done,
+ )
+ git_meta_map = {m["file_path"]: m for m in git_metadata_list}
+ return git_summary, git_metadata_list, git_meta_map
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Git indexing skipped: {exc}")
+ return None, [], {}
+
+
+async def _run_dead_code_analysis(
+ graph_builder: Any,
+ git_meta_map: dict[str, dict],
+ *,
+ progress: ProgressCallback | None,
+) -> Any | None:
+ """Run dead code detection (pure graph traversal, no LLM)."""
+ try:
+ from repowise.core.analysis.dead_code import DeadCodeAnalyzer
+
+ if progress:
+ progress.on_phase_start("dead_code", None)
+
+ analyzer = DeadCodeAnalyzer(graph_builder.graph(), git_meta_map)
+ report = analyzer.analyze()
+
+ if progress:
+ unreachable = sum(1 for f in report.findings if f.kind.value == "unreachable_file")
+ unused_exports = sum(1 for f in report.findings if f.kind.value == "unused_export")
+ progress.on_message(
+ "info",
+ f"Dead code: {unreachable} unreachable files "
+ f"· {unused_exports} unused exports (~{report.deletable_lines:,} lines)",
+ )
+
+ return report
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Dead code detection skipped: {exc}")
+ return None
+
+
+async def _run_decision_extraction(
+ repo_path: Path,
+ *,
+ llm_client: Any | None,
+ graph_builder: Any,
+ git_meta_map: dict[str, dict],
+ parsed_files: list[Any],
+ progress: ProgressCallback | None,
+) -> Any | None:
+ """Extract architectural decisions from source and git history."""
+ try:
+ from repowise.core.analysis.decision_extractor import DecisionExtractor
+
+ if progress:
+ progress.on_phase_start("decisions", None)
+
+ extractor = DecisionExtractor(
+ repo_path=repo_path,
+ provider=llm_client,
+ graph=graph_builder.graph(),
+ git_meta_map=git_meta_map,
+ parsed_files=parsed_files,
+ )
+ report = await extractor.extract_all()
+
+ if progress:
+ inline = report.by_source.get("inline_marker", 0)
+ readme = report.by_source.get("readme_mining", 0)
+ git_arch = report.by_source.get("git_archaeology", 0)
+ progress.on_message(
+ "info",
+ f"Decisions: {inline} inline · {readme} from docs · {git_arch} from git",
+ )
+
+ return report
+ except Exception as exc:
+ if progress:
+ progress.on_message("warning", f"Decision extraction skipped: {exc}")
+ return None
+
+
+async def run_generation(
+ *,
+ repo_path: Path,
+ parsed_files: list[Any],
+ source_map: dict[str, bytes],
+ graph_builder: Any,
+ repo_structure: Any,
+ git_meta_map: dict[str, dict],
+ llm_client: Any,
+ embedder: Any | None,
+ vector_store: Any | None,
+ concurrency: int,
+ progress: ProgressCallback | None,
+) -> list[Any]:
+ """Run LLM-powered page generation.
+
+ Returns a list of ``GeneratedPage`` objects.
+ """
+ from repowise.core.generation import (
+ ContextAssembler,
+ GenerationConfig,
+ JobSystem,
+ PageGenerator,
+ )
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+ from repowise.core.providers.embedding.base import MockEmbedder
+
+ config = GenerationConfig(max_concurrency=concurrency)
+ assembler = ContextAssembler(config)
+
+ # Resolve embedder and vector store
+ embedder_impl = embedder if embedder is not None else MockEmbedder()
+
+ if vector_store is None:
+ vector_store = InMemoryVectorStore(embedder_impl)
+
+ generator = PageGenerator(llm_client, assembler, config, vector_store=vector_store)
+
+ # Job system — use a temp-like dir under repo_path for checkpoints
+ jobs_dir = repo_path / ".repowise" / "jobs"
+ jobs_dir.mkdir(parents=True, exist_ok=True)
+ job_system = JobSystem(jobs_dir)
+
+ repo_name = repo_path.name
+
+ # Track generation progress
+ _pages_done = 0
+
+ def on_page_done(page_type: str) -> None:
+ nonlocal _pages_done
+ _pages_done += 1
+ if progress:
+ progress.on_item_done("generation")
+
+ if progress:
+ progress.on_phase_start("generation", None)
+
+ generated_pages = await generator.generate_all(
+ parsed_files,
+ source_map,
+ graph_builder,
+ repo_structure,
+ repo_name,
+ job_system=job_system,
+ on_page_done=on_page_done,
+ git_meta_map=git_meta_map if git_meta_map else None,
+ )
+
+ if progress:
+ progress.on_message("info", f"Generated {len(generated_pages)} pages")
+
+ return generated_pages
diff --git a/packages/core/src/repowise/core/pipeline/progress.py b/packages/core/src/repowise/core/pipeline/progress.py
new file mode 100644
index 000000000..57f5c03e8
--- /dev/null
+++ b/packages/core/src/repowise/core/pipeline/progress.py
@@ -0,0 +1,46 @@
+"""Progress reporting protocol for the repowise pipeline.
+
+Decouples pipeline execution from UI concerns. The CLI implements this
+with Rich progress bars; Modal uses structured logging; tests pass None.
+
+Phase names (stable strings used across all implementations):
+ traverse, parse, graph, git, co_change, dead_code, decisions, generation
+"""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+@runtime_checkable
+class ProgressCallback(Protocol):
+ """Protocol for pipeline progress reporting."""
+
+ def on_phase_start(self, phase: str, total: int | None) -> None:
+ """Called when a pipeline phase begins. *total* may be None for indeterminate phases."""
+ ...
+
+ def on_item_done(self, phase: str) -> None:
+ """Called after one unit of work completes within a phase."""
+ ...
+
+ def on_message(self, level: str, text: str) -> None:
+ """Emit a free-form message. *level* is 'info', 'warning', or 'error'."""
+ ...
+
+
+class LoggingProgressCallback:
+ """Emits progress as structured log messages. Suitable for headless workers (Modal)."""
+
+ def on_phase_start(self, phase: str, total: int | None) -> None:
+ logger.info("phase_start", phase=phase, total=total)
+
+ def on_item_done(self, phase: str) -> None:
+ logger.debug("item_done", phase=phase)
+
+ def on_message(self, level: str, text: str) -> None:
+ getattr(logger, level, logger.info)(text)
diff --git a/packages/server/src/repowise/server/__init__.py b/packages/server/src/repowise/server/__init__.py
index 4c5017cc2..d2ad5496f 100644
--- a/packages/server/src/repowise/server/__init__.py
+++ b/packages/server/src/repowise/server/__init__.py
@@ -7,4 +7,4 @@ Provides:
- Background job scheduler (APScheduler)
"""
-__version__ = "0.1.29"
+__version__ = "0.1.30"
diff --git a/pyproject.toml b/pyproject.toml
index 82172a751..ed2c3d0c4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "repowise"
-version = "0.1.29"
+version = "0.1.30"
description = "Codebase intelligence for developers and AI — generates and maintains a structured wiki for any codebase"
readme = "README.md"
requires-python = ">=3.11"
@@ -125,6 +125,7 @@ packages = [
"repowise.core.generation.editor_files",
"repowise.core.ingestion",
"repowise.core.ingestion.parsers",
+ "repowise.core.pipeline",
"repowise.core.persistence",
"repowise.core.providers",
"repowise.core.providers.llm",
@@ -194,7 +195,7 @@ known-first-party = ["repowise"]
# mypy — strict type checking
# ---------------------------------------------------------------------------
[tool.mypy]
-python_version = "0.1.29"
+python_version = "0.1.30"
warn_unused_configs = true
show_error_codes = true
namespace_packages = true
diff --git a/uv.lock b/uv.lock
index 61de49090..54a2b857b 100644
--- a/uv.lock
+++ b/uv.lock
@@ -2336,7 +2336,7 @@ wheels = [
[[package]]
name = "repowise"
-version = "0.1.25"
+version = "0.1.29"
source = { editable = "." }
dependencies = [
{ name = "aiosqlite" },