Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix: decouple embedding concurrency from generation (#163)

zxyasfasb79f4590d017
diff --git a/packages/core/src/repowise/core/generation/models.py b/packages/core/src/repowise/core/generation/models.py
index ca65093cd..4b2690eea 100644
--- a/packages/core/src/repowise/core/generation/models.py
+++ b/packages/core/src/repowise/core/generation/models.py
@@ -62,6 +62,8 @@ class GenerationConfig:
temperature: Sampling temperature (0.3 for consistent docs).
token_budget: Context tokens fed to LLM (not output).
max_concurrency: asyncio.Semaphore size for parallel calls.
+ embed_concurrency: asyncio.Semaphore size for vector-store writes.
+ Defaults to max_concurrency.
cache_enabled: In-memory SHA256 prompt deduplication.
staleness_threshold_days: Days before a page is considered stale.
expiry_threshold_days: Days before a page is considered expired.
@@ -73,6 +75,7 @@ class GenerationConfig:
temperature: float = 0.3
token_budget: int = 48000
max_concurrency: int = 5
+ embed_concurrency: int | None = None
cache_enabled: bool = True
staleness_threshold_days: int = 7
expiry_threshold_days: int = 30
@@ -84,6 +87,11 @@ class GenerationConfig:
large_file_source_pct: float = 0.4 # use structural summary when source tokens > budget * this
language: str = "en"
+ def __post_init__(self) -> None:
+ if self.embed_concurrency is None:
+ object.__setattr__(self, "embed_concurrency", self.max_concurrency)
+
+
# ---------------------------------------------------------------------------
# GeneratedPage
# ---------------------------------------------------------------------------
diff --git a/packages/core/src/repowise/core/generation/page_generator.py b/packages/core/src/repowise/core/generation/page_generator.py
index ce8c29197..750956c19 100644
--- a/packages/core/src/repowise/core/generation/page_generator.py
+++ b/packages/core/src/repowise/core/generation/page_generator.py
@@ -458,6 +458,7 @@ class PageGenerator:
all_pages: list[GeneratedPage] = []
semaphore = asyncio.Semaphore(self._config.max_concurrency)
+ embed_semaphore = asyncio.Semaphore(self._config.embed_concurrency or 1)
# Summaries of completed pages: target_path → brief summary text (for dep context)
completed_page_summaries: dict[str, str] = {}
@@ -497,13 +498,15 @@ class PageGenerator:
job_system.update_level(job_id, level)
async def guarded_named(page_id: str, coro: Any) -> Any:
- async with semaphore:
- try:
+ try:
+ async with semaphore:
result = await coro
- # Embed page for RAG (B1)
- if self._vector_store is not None and isinstance(result, GeneratedPage):
- try:
- page_summary = _extract_summary(result.content)
+
+ # Embed page for RAG (B1)
+ if self._vector_store is not None and isinstance(result, GeneratedPage):
+ try:
+ page_summary = _extract_summary(result.content)
+ async with embed_semaphore:
await self._vector_store.embed_and_upsert(
result.page_id,
result.content,
@@ -514,27 +517,27 @@ class PageGenerator:
"summary": page_summary,
},
)
- except Exception as e:
- log.debug("rag.embed_failed", page_id=result.page_id, error=str(e))
- # Store summary for dependency context (B2)
- if isinstance(result, GeneratedPage):
- completed_page_summaries[result.target_path] = _extract_summary(
- result.content
- )
- # Report progress immediately (not batched after gather)
- if on_page_done is not None:
- on_page_done(result.page_type)
- return result
- except Exception as exc:
- if job_system is not None and job_id is not None:
- job_system.fail_page(job_id, page_id, str(exc))
- log.error(
- "page_generation_failed",
- page_id=page_id,
- level=level,
- error=str(exc),
+ except Exception as e:
+ log.debug("rag.embed_failed", page_id=result.page_id, error=str(e))
+ # Store summary for dependency context (B2)
+ if isinstance(result, GeneratedPage):
+ completed_page_summaries[result.target_path] = _extract_summary(
+ result.content
)
- return exc # return as value so gather works
+ # Report progress immediately (not batched after gather)
+ if on_page_done is not None:
+ on_page_done(result.page_type)
+ return result
+ except Exception as exc:
+ if job_system is not None and job_id is not None:
+ job_system.fail_page(job_id, page_id, str(exc))
+ log.error(
+ "page_generation_failed",
+ page_id=page_id,
+ level=level,
+ error=str(exc),
+ )
+ return exc # return as value so gather works
tasks = [guarded_named(pid, c) for pid, c in named_coros]
results = await asyncio.gather(*tasks)
diff --git a/tests/integration/test_generation_pipeline.py b/tests/integration/test_generation_pipeline.py
index 387a0dff6..ea320986f 100644
--- a/tests/integration/test_generation_pipeline.py
+++ b/tests/integration/test_generation_pipeline.py
@@ -6,7 +6,11 @@ resulting GeneratedPage list and job checkpoint.
from __future__ import annotations
+import asyncio
+import time
+from datetime import UTC, datetime
from pathlib import Path
+from typing import Any
import pytest
@@ -16,17 +20,136 @@ from repowise.core.generation.models import GenerationConfig
from repowise.core.generation.page_generator import PageGenerator
from repowise.core.ingestion.graph import GraphBuilder
from repowise.core.ingestion.models import (
+ FileInfo,
PackageInfo,
ParsedFile,
RepoStructure,
+ Symbol,
)
from repowise.core.ingestion.parser import ASTParser
from repowise.core.ingestion.traverser import FileTraverser
+from repowise.core.providers.llm.base import GeneratedResponse
from repowise.core.providers.llm.mock import MockProvider
SAMPLE_REPO = Path(__file__).parents[1] / "fixtures" / "sample_repo"
+class _TrackingProvider(MockProvider):
+ def __init__(self, delay: float = 0.01) -> None:
+ super().__init__()
+ self.delay = delay
+ self.file_page_starts: list[float] = []
+
+ async def generate(
+ self,
+ system_prompt: str,
+ user_prompt: str,
+ max_tokens: int = 4096,
+ temperature: float = 0.3,
+ request_id: str | None = None,
+ ) -> GeneratedResponse:
+ if "Required sections: ## Overview, ## Public API" in system_prompt:
+ self.file_page_starts.append(time.perf_counter())
+ await asyncio.sleep(self.delay)
+ return await super().generate(
+ system_prompt,
+ user_prompt,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ request_id=request_id,
+ )
+
+
+class _SlowVectorStore:
+ def __init__(self, delay: float = 0.05) -> None:
+ self.delay = delay
+ self.active_embeds = 0
+ self.max_active_embeds = 0
+ self.file_page_embed_finishes: list[float] = []
+
+ async def embed_and_upsert(self, page_id: str, text: str, metadata: dict[str, Any]) -> None:
+ self.active_embeds += 1
+ self.max_active_embeds = max(self.max_active_embeds, self.active_embeds)
+ try:
+ await asyncio.sleep(self.delay)
+ if metadata.get("page_type") == "file_page":
+ self.file_page_embed_finishes.append(time.perf_counter())
+ finally:
+ self.active_embeds -= 1
+
+ async def list_page_ids(self) -> set[str]:
+ return set()
+
+ async def get_page_summary_by_path(self, path: str) -> None:
+ return None
+
+ async def search(self, query: str, limit: int = 3) -> list[Any]:
+ return []
+
+
+def _make_concurrency_fixture() -> tuple[list[ParsedFile], dict[str, bytes], RepoStructure]:
+ parsed_files: list[ParsedFile] = []
+ source_map: dict[str, bytes] = {}
+
+ for index in range(4):
+ path = f"pkg/module_{index}.py"
+ source = f"def function_{index}() -> int:\n return {index}\n".encode()
+ file_info = FileInfo(
+ path=path,
+ abs_path=f"/repo/{path}",
+ language="python",
+ size_bytes=len(source),
+ git_hash=f"hash-{index}",
+ last_modified=datetime(2026, 1, 1, tzinfo=UTC),
+ is_test=False,
+ is_config=False,
+ is_api_contract=False,
+ is_entry_point=index == 0,
+ )
+ symbol = Symbol(
+ id=f"{path}::function_{index}",
+ name=f"function_{index}",
+ qualified_name=f"pkg.module_{index}.function_{index}",
+ kind="function",
+ signature=f"def function_{index}() -> int:",
+ start_line=1,
+ end_line=2,
+ docstring=None,
+ visibility="public",
+ language="python",
+ )
+ parsed_files.append(
+ ParsedFile(
+ file_info=file_info,
+ symbols=[symbol],
+ imports=[],
+ exports=[symbol.name],
+ docstring=None,
+ parse_errors=[],
+ content_hash=f"content-{index}",
+ )
+ )
+ source_map[path] = source
+
+ repo_structure = RepoStructure(
+ is_monorepo=False,
+ packages=[
+ PackageInfo(
+ name="pkg",
+ path="pkg",
+ language="python",
+ entry_points=["pkg/module_0.py"],
+ manifest_file="pyproject.toml",
+ )
+ ],
+ root_language_distribution={"python": 1.0},
+ total_files=len(parsed_files),
+ total_loc=sum(len(source.splitlines()) for source in source_map.values()),
+ entry_points=["pkg/module_0.py"],
+ )
+ return parsed_files, source_map, repo_structure
+
+
# ---------------------------------------------------------------------------
# Pipeline fixture (class-scoped so it runs once per test class)
# ---------------------------------------------------------------------------
@@ -106,6 +229,44 @@ async def pipeline_result(tmp_path_factory):
}
+async def test_embedding_latency_does_not_gate_llm_concurrency():
+ parsed_files, source_map, repo_structure = _make_concurrency_fixture()
+ builder = GraphBuilder()
+ for parsed in parsed_files:
+ builder.add_file(parsed)
+ builder.build()
+
+ config = GenerationConfig(
+ max_tokens=256,
+ token_budget=1000,
+ max_concurrency=2,
+ embed_concurrency=1,
+ file_page_top_percentile=1.0,
+ top_symbol_percentile=0.01,
+ max_pages_pct=1.0,
+ cache_enabled=False,
+ )
+ provider = _TrackingProvider()
+ vector_store = _SlowVectorStore()
+ assembler = ContextAssembler(config)
+ generator = PageGenerator(provider, assembler, config, vector_store=vector_store)
+
+ pages = await generator.generate_all(
+ parsed_files,
+ source_map,
+ builder,
+ repo_structure,
+ "concurrency_repo",
+ )
+
+ file_pages = [page for page in pages if page.page_type == "file_page"]
+ assert len(file_pages) == 4
+ assert len(provider.file_page_starts) >= 4
+ assert vector_store.file_page_embed_finishes
+ assert vector_store.max_active_embeds == 1
+ assert provider.file_page_starts[2] < vector_store.file_page_embed_finishes[0]
+
+
# ---------------------------------------------------------------------------
# Test class
# ---------------------------------------------------------------------------
diff --git a/tests/unit/generation/test_models.py b/tests/unit/generation/test_models.py
index b51cc63b3..2096bb511 100644
--- a/tests/unit/generation/test_models.py
+++ b/tests/unit/generation/test_models.py
@@ -146,8 +146,14 @@ def test_generation_config_defaults():
assert config.temperature == 0.3
assert config.token_budget == 48000
assert config.max_concurrency == 5
+ assert config.embed_concurrency == 5
assert config.cache_enabled is True
assert config.staleness_threshold_days == 7
assert config.expiry_threshold_days == 30
assert config.top_symbol_percentile == 0.10
assert config.large_file_source_pct == 0.4
+
+
+def test_generation_config_embed_concurrency_defaults_to_max_concurrency():
+ config = GenerationConfig(max_concurrency=3)
+ assert config.embed_concurrency == 3