Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(generation): give each cost tier its own progress total (#1607)
Raghav Chamadiyae42e8bfaa540
diff --git a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py b/packages/core/src/repowise/core/generation/page_generator/orchestrate.pyindex 4c41da491..a5896d79f 100644--- a/packages/core/src/repowise/core/generation/page_generator/orchestrate.py+++ b/packages/core/src/repowise/core/generation/page_generator/orchestrate.py@@ -12,6 +12,7 @@ purely a structural split to satisfy the project's 400-line ceiling.from __future__ import annotationsimport asyncio+import contextlibfrom collections.abc import Callablefrom pathlib import Pathfrom typing import TYPE_CHECKING, Any@@ -43,6 +44,43 @@ if TYPE_CHECKING:log = structlog.get_logger(__name__)+def _free_page_types() -> frozenset[str]:+ """Page types rendered from a template, with no provider call.++ Deferred import on purpose: ``cost_estimator`` reaches back into+ ``generation.selection``, so importing it at module scope from inside the+ generation package risks a partially-initialised cycle. Reused rather than+ restated because that set is already the repo's answer to "does this page+ cost tokens", and a second copy here is what would drift the first time a+ page type changes tier.+ """+ from repowise.core.cost_estimator import STRUCTURAL_PAGE_TYPES++ return frozenset(STRUCTURAL_PAGE_TYPES)+++def _count_free_ids(page_ids: Any) -> int:+ """How many of *page_ids* are free-tier, read off the id's type prefix.++ Page ids are ``"<page_type>:<target>"``, so the prefix is the tier without+ needing the pages themselves — which matters here because these are+ completed ids recovered from a prior run, not objects in hand.+ """+ free = _free_page_types()+ return sum(1 for page_id in page_ids if str(page_id).split(":", 1)[0] in free)+++def _announce_paid(on_subphase: Any, total: int) -> None:+ """Start the model-backed bar, or leave it off a run that has none.++ A deterministic run makes no model calls at all, and a bar sitting at 0/0+ for its whole duration is an unanswered question rather than an answer.+ """+ if total > 0 and on_subphase is not None:+ with contextlib.suppress(Exception):+ on_subphase("generation.llm", total)++class _GenerationRun:"""Mutable per-call state for one ``generate_all`` invocation."""@@ -435,9 +473,12 @@ class _GenerationRun:# still an upper bound (a page may gate-skip at build time), matching# the contract the caller reconciles against the real completed count.if self.only_page_ids is not None:- remaining = len(self.only_page_ids - self.completed_ids)+ remaining_ids = self.only_page_ids - self.completed_ids+ remaining = len(remaining_ids)+ free_remaining = _count_free_ids(remaining_ids)if self.on_total_known is not None:- self.on_total_known(remaining)+ self.on_total_known(free_remaining)+ _announce_paid(self.on_subphase, remaining - free_remaining)if self.job_system is not None and self.job_id is not None:self.job_system.start_job(self.job_id, remaining)return@@ -466,9 +507,21 @@ class _GenerationRun:+ counts["infra_page"]+ onboarding_page_count)- remaining_total = max(0, estimated_total - len(self.completed_ids))+ # Split the total across the two progress bars. One bar counting both+ # tiers reads as frozen: the free levels run first, so it reaches ~97%+ # in minutes and then crawls for the rest of the run while the ~95+ # model-backed pages finish. The paid count is the remainder rather+ # than a second sum, so the two always add back to the total this+ # already reported and the job system still books the whole run.+ free_total = sum(counts.get(page_type, 0) for page_type in _free_page_types())+ done_free = _count_free_ids(self.completed_ids)+ remaining_free = max(0, free_total - done_free)+ remaining_paid = max(+ 0, (estimated_total - free_total) - (len(self.completed_ids) - done_free)+ )if self.on_total_known is not None:- self.on_total_known(remaining_total)+ self.on_total_known(remaining_free)+ _announce_paid(self.on_subphase, remaining_paid)if self.job_system is not None and self.job_id is not None:self.job_system.start_job(self.job_id, estimated_total)diff --git a/packages/core/src/repowise/core/pipeline/phases/generation.py b/packages/core/src/repowise/core/pipeline/phases/generation.pyindex 8ec070920..4d2f9bb53 100644--- a/packages/core/src/repowise/core/pipeline/phases/generation.py+++ b/packages/core/src/repowise/core/pipeline/phases/generation.py@@ -114,34 +114,26 @@ async def run_generation(_pages_done += 1if not progress:return- # Onboarding keeps its own named step, and also advances the overall- # bar. It used to do only the former while ``_announce_total`` counted- # onboarding slots into the overall total, so that bar's denominator- # included pages that could never tick it and it never reached 100%- # on its own — ``on_phase_done`` forcing it there at the end is what- # hid the arithmetic.+ # Onboarding keeps its own named step as well as advancing its tier's+ # bar, so it stays visible as a distinct step in the terminal.if page_type == "onboarding":progress.on_item_done("onboarding")- progress.on_item_done("generation")- # One bar counting 4,229 items where ~4,134 are free template renders- # and ~95 are paid model calls reads as frozen: the cheap levels run- # first, so it sprints to 97% and then crawls for another quarter of an- # hour with the cost figure sitting still. The paid work gets its own- # counter so that stretch has something visibly moving in it.+ # Each page advances the bar for its own tier. One bar counting 4,229+ # items where ~4,134 are free template renders and ~95 are paid model+ # calls reads as frozen: the cheap levels run first, so it sprints to+ # 97% and then crawls for another quarter of an hour. Split, the free+ # bar finishes and hides while the paid one counts the stretch that is+ # actually still running.#- # Counted rather than announced up front: the per-tier totals are known- # only inside the level builders, which this phase does not own. An- # indeterminate counter states what is true without inventing a- # denominator.- #- # Counts by page *type*, not by whether a call was made, so a resume or- # update run that reuses a cached page still counts it here. The tier- # is right; on a heavily-reused run the count overstates the calls.- if page_type not in _FREE_PAGE_TYPES:+ # Routed by page *type*, not by whether a call was made, so a resume or+ # update run that reuses a cached page still counts it against the paid+ # tier. That matches how ``_announce_total`` derives the denominators,+ # which is what keeps the bar reaching 100%.+ if page_type in _FREE_PAGE_TYPES:+ progress.on_item_done("generation")+ else:_llm_pages_done += 1- if _llm_pages_done == 1:- progress.on_phase_start("generation.llm", None)progress.on_item_done("generation.llm")# Push live cost update if the callback supports itdiff --git a/tests/unit/generation/test_announce_total.py b/tests/unit/generation/test_announce_total.pyindex 932e45dca..2b057d0a5 100644--- a/tests/unit/generation/test_announce_total.py+++ b/tests/unit/generation/test_announce_total.py@@ -1,10 +1,17 @@-"""The progress total announced by ``_GenerationRun._announce_total``.+"""The progress totals announced by ``_GenerationRun._announce_total``.Regression for issue #922: the up-front total omitted the level-8 onboardingpages (the non-promoted slots), so the bar could report more pages completed-than its total (e.g. "43 of 41"). The estimate must now include one slot per+than its total (e.g. "43 of 41"). The estimate must include one slot perregistered onboarding spec that has not already been generated, gated by``config.enable_onboarding``.++The total is now announced as two, one per cost tier — free template renders+on the main bar, model-backed pages on ``generation.llm`` — because a single+bar spanning both runs the cheap levels first and then sits at ~97% for the+rest of the run. #922's requirement is unchanged and asserted here against+their sum: onboarding slots must still be counted, they are simply counted on+the paid bar now."""from __future__ import annotations@@ -36,6 +43,7 @@ def _selection() -> SimpleNamespace:def _fake_run(*, enable_onboarding: bool, completed_ids: set[str]) -> SimpleNamespace:announced: list[int] = []+ paid: list[tuple[str, int]] = []return SimpleNamespace(selection=_selection(),kg_ctx=SimpleNamespace(available=False),@@ -45,12 +53,19 @@ def _fake_run(*, enable_onboarding: bool, completed_ids: set[str]) -> SimpleName# to len(only_page_ids), tested in test_deterministic_generation).only_page_ids=None,on_total_known=announced.append,+ on_subphase=lambda name, total: paid.append((name, total)),job_system=None,job_id=None,_announced=announced,+ _paid=paid,)+def _paid_total(run: SimpleNamespace) -> int:+ """The count announced for the model-backed bar, or 0 when it never starts."""+ return sum(total for name, total in run._paid if name == "generation.llm")++def test_total_includes_onboarding_pages() -> None:specs = _onboarding.iter_specs()assert specs, "expected onboarding subkinds to be registered on import"@@ -58,17 +73,35 @@ def test_total_includes_onboarding_pages() -> None:run = _fake_run(enable_onboarding=True, completed_ids=set())_GenerationRun._announce_total(run)- (total,) = run._announced- # 1 file page + one page per registered onboarding slot.- assert total == 1 + len(specs)+ (free_total,) = run._announced+ # 1 file page + one page per registered onboarding slot, across both bars.+ assert free_total + _paid_total(run) == 1 + len(specs)+++def test_the_tiers_are_split_by_cost_not_by_level() -> None:+ """The free bar must carry only pages that cost nothing.++ A slot landing on the free bar would leave it unable to reach the+ denominator, which is the arithmetic this split exists to fix.+ """+ specs = _onboarding.iter_specs()+ run = _fake_run(enable_onboarding=True, completed_ids=set())+ _GenerationRun._announce_total(run)++ (free_total,) = run._announced+ assert free_total == 1, "only the file page is free"+ assert _paid_total(run) == len(specs)def test_onboarding_disabled_excludes_those_pages() -> None:run = _fake_run(enable_onboarding=False, completed_ids=set())_GenerationRun._announce_total(run)- (total,) = run._announced- assert total == 1+ (free_total,) = run._announced+ assert free_total == 1+ # Nothing model-backed remains, so that bar is never started rather than+ # sitting at 0 for the whole run.+ assert run._paid == []def test_already_generated_onboarding_pages_are_not_recounted() -> None:@@ -77,6 +110,9 @@ def test_already_generated_onboarding_pages_are_not_recounted() -> None:run = _fake_run(enable_onboarding=True, completed_ids=done)_GenerationRun._announce_total(run)- (total,) = run._announced- # The one already-completed slot drops out of the remaining total.- assert total == 1 + len(specs) - 1+ (free_total,) = run._announced+ # The one already-completed slot drops out of the remaining total, and it+ # drops out of the paid tier specifically — the free bar is untouched by a+ # resumed onboarding page.+ assert free_total == 1+ assert _paid_total(run) == len(specs) - 1diff --git a/tests/unit/pipeline/test_generation_progress_split.py b/tests/unit/pipeline/test_generation_progress_split.pyindex b7a766a68..ee80bdc74 100644--- a/tests/unit/pipeline/test_generation_progress_split.py+++ b/tests/unit/pipeline/test_generation_progress_split.py@@ -96,15 +96,11 @@ def test_free_page_types_are_the_ones_that_cost_nothing() -> None:assert "onboarding" not in _FREE_PAGE_TYPES-def test_paid_pages_get_their_own_counter(monkeypatch, tmp_path: Path) -> None:+def test_each_tier_advances_only_its_own_bar(monkeypatch, tmp_path: Path) -> None:progress = _run(monkeypatch, tmp_path, ["file_page"] * 5 + ["module_page"] * 3)- # The free pages must not appear on the paid counter...assert progress.items.count("generation.llm") == 3- # ...and the paid counter must not be announced before the first paid page,- # or it sits at 0 on screen through the entire free stretch.- assert ("generation.llm", None) in progress.started- assert progress.started.index(("generation.llm", None)) > 0+ assert progress.items.count("generation") == 5def test_a_run_with_no_paid_pages_shows_no_paid_counter(monkeypatch, tmp_path: Path) -> None:@@ -112,24 +108,19 @@ def test_a_run_with_no_paid_pages_shows_no_paid_counter(monkeypatch, tmp_path: Pwould be an unanswered question, not an answer."""progress = _run(monkeypatch, tmp_path, ["file_page"] * 4)- assert "generation.llm" not in [phase for phase, _ in progress.started]assert "generation.llm" not in progress.items-def test_every_page_still_advances_the_overall_bar(monkeypatch, tmp_path: Path) -> None:- """The overall total counts onboarding, so the overall bar must too.+def test_onboarding_counts_as_paid_and_keeps_its_own_step(monkeypatch, tmp_path: Path) -> None:+ """Onboarding slots cost a model call, so they belong to the paid tier.- It previously routed onboarding pages *only* to the onboarding task while- ``_announce_total`` counted them into the overall total, so that bar could- never reach its own denominator — ``on_phase_done`` forcing it to 100% at- the end is what hid the arithmetic.+ Routing them to the free bar would leave that bar unable to reach the+ denominator ``_announce_total`` derives from the same tier split."""- page_types = ["file_page", "module_page", "onboarding", "onboarding"]- progress = _run(monkeypatch, tmp_path, page_types)+ progress = _run(monkeypatch, tmp_path, ["file_page", "module_page", "onboarding", "onboarding"])- announced = dict(progress.started)["generation"]- assert announced == len(page_types)- assert progress.items.count("generation") == len(page_types)+ assert progress.items.count("generation") == 1+ assert progress.items.count("generation.llm") == 3# Onboarding keeps its own named step as well.assert progress.items.count("onboarding") == 2