Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

9.1 KB
Raw Blame
"""The parse pool's worker count is bounded (issue #1394).

Every worker is a fresh spawned interpreter holding ~50 MB of private memory,
so sizing the pool from the host's core count made peak memory a function of
the machine rather than the repo.
"""

import subprocess
import sys

import pytest

from repowise.core.pipeline import parse_pool
from repowise.core.pipeline.phases import ingestion as ing


class TestParsePoolWorkers:
    """The cap, the floor, and the env override."""

    def test_the_cap_is_eight(self):
        """Pin the literal.

        Every other assertion here compares the helper's output to
        ``_MAX_PARSE_WORKERS``, which a revert of the constant would satisfy
        while reintroducing the 32-worker fleet this exists to prevent.
        """
        assert ing._MAX_PARSE_WORKERS == 8

    def test_cap_applies_to_host_core_count(self, monkeypatch):
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
        assert ing.parse_pool_workers(5000) == 8

    def test_below_cap_is_left_alone(self, monkeypatch):
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 4)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 4, raising=False)
        assert ing.parse_pool_workers(5000) == 4

    def test_never_exceeds_the_files_to_parse(self, monkeypatch):
        """A three-file update has no use for eight interpreters."""
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
        assert ing.parse_pool_workers(3) == 3

    def test_empty_work_list_degrades_to_one(self):
        assert ing.parse_pool_workers(0) == 1
        assert ing.parse_pool_workers(-1) == 1

    def test_unknown_core_count_degrades_to_a_default(self, monkeypatch):
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: None)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: None, raising=False)
        assert ing.parse_pool_workers(5000) == 4

    def test_affinity_aware_count_wins_over_host_count(self, monkeypatch):
        """A container pinned to 2 CPUs must not size its pool from the host."""
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 2, raising=False)
        assert ing.parse_pool_workers(5000) == 2

    def test_missing_process_cpu_count_falls_back(self, monkeypatch):
        """Python 3.11/3.12 have no ``process_cpu_count``."""
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.delattr(ing.os, "process_cpu_count", raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 3)
        assert ing.parse_pool_workers(5000) == 3

    @pytest.mark.parametrize("value,expected", [("2", 2), ("16", 16), ("1", 1)])
    def test_env_override_wins_in_both_directions(self, monkeypatch, value, expected):
        monkeypatch.setenv(ing._PARSE_WORKERS_ENV, value)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
        assert ing.parse_pool_workers(5000) == expected

    def test_env_override_still_bounded_by_pending_files(self, monkeypatch):
        monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "64")
        assert ing.parse_pool_workers(5) == 5

    @pytest.mark.parametrize("value", ["nonsense", "0", "-4", "3.5"])
    def test_unusable_env_value_is_ignored_not_fatal(self, monkeypatch, value):
        """A bad env value must never fail an otherwise healthy parse."""
        monkeypatch.setenv(ing._PARSE_WORKERS_ENV, value)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
        assert ing.parse_pool_workers(5000) == 8

    def test_unusable_env_value_says_so(self, monkeypatch):
        """The warning is the only feedback a misconfigured user gets."""
        monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "-4")
        warned: list[tuple] = []
        monkeypatch.setattr(
            ing.logger, "warning", lambda event, **kw: warned.append((event, kw))
        )
        ing.parse_pool_workers(5000)
        assert warned and warned[0][0] == "invalid_parse_workers_env"
        assert warned[0][1]["value"] == "-4"

    def test_empty_env_value_means_unset(self, monkeypatch):
        """``export REPOWISE_PARSE_WORKERS=`` from an undefined shell variable
        is not a misconfiguration worth warning about."""
        monkeypatch.setenv(ing._PARSE_WORKERS_ENV, "")
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 32)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 32, raising=False)
        warned: list[tuple] = []
        monkeypatch.setattr(
            ing.logger, "warning", lambda event, **kw: warned.append((event, kw))
        )
        assert ing.parse_pool_workers(5000) == 8
        assert not warned

    def test_affinity_unavailable_falls_back_to_host_count(self, monkeypatch):
        """``process_cpu_count`` exists but returns None when there is no mask."""
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: None, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 3)
        assert ing.parse_pool_workers(5000) == 3


class TestBothCallSitesAreBounded:
    """The bound has to reach the executor on the init *and* resume paths.

    Grepping the source is what this guards: both paths parse through
    ``_parse_stage``, and its pool is sized by the shared helper, never by
    ``os.cpu_count()``.
    """

    def test_pool_uses_the_shared_helper(self):
        import inspect

        src = inspect.getsource(ing.parse_guarded)
        assert "parse_pool_workers(" in src, "the parse pool must be sized via the shared helper"
        assert "os.cpu_count()" not in src, "the pool must not be sized from the host core count"

    @pytest.mark.parametrize("func", ["_run_ingestion", "reparse_for_resume"])
    def test_call_site_parses_through_the_shared_stage(self, func):
        import inspect

        src = inspect.getsource(getattr(ing, func))
        assert "_parse_stage(" in src, f"{func} must parse via the shared stage"
        assert "os.cpu_count()" not in src, f"{func} must not size its pool from the host core count"

    async def test_the_bound_reaches_the_real_executor(self, tmp_path, monkeypatch):
        """Record what ``ProcessPoolExecutor`` is really constructed with.

        The source-text assertions above cannot catch a regression spelled a
        different way; this pins the argument itself on a real ingestion run.
        """
        monkeypatch.delenv(ing._PARSE_WORKERS_ENV, raising=False)
        monkeypatch.setattr(ing.os, "cpu_count", lambda: 64)
        monkeypatch.setattr(ing.os, "process_cpu_count", lambda: 64, raising=False)

        for i in range(12):
            (tmp_path / f"mod_{i}.py").write_text(f"def f_{i}(x):\n    return x + {i}\n")

        seen: list[int | None] = []
        real = parse_pool.ProcessPoolExecutor

        def recording(*args, **kwargs):
            seen.append(kwargs.get("max_workers"))
            # Honour the bound under test but keep the pool tiny: this is a
            # unit test, not a benchmark, and spawning eight interpreters to
            # parse twelve files would dominate the suite's runtime.
            kwargs["max_workers"] = 1
            return real(*args, **kwargs)

        monkeypatch.setattr(parse_pool, "ProcessPoolExecutor", recording)

        parsed, *_ = await ing._run_ingestion(
            tmp_path,
            exclude_patterns=None,
            skip_tests=False,
            skip_infra=False,
            progress=None,
        )

        assert parsed, "ingestion parsed nothing, so the run proves nothing"
        assert seen, "no process pool was constructed, so the bound was never exercised"
        assert seen == [8], f"expected the capped count to reach the executor, got {seen}"


class TestWorkerImportWeight:
    """A spawned worker re-imports this module, so its imports are per-worker cost.

    A string constant imported from the persistence package once pulled
    SQLAlchemy into every worker: +31 MB and +209 modules each.
    """

    def test_worker_module_does_not_load_persistence(self):
        script = (
            "import sys, repowise.core.pipeline.phases.ingestion; "
            "print('\\n'.join(sorted(m for m in sys.modules if m == 'sqlalchemy' "
            "or m.startswith(('sqlalchemy.', 'repowise.core.persistence')))))"
        )
        done = subprocess.run(
            [sys.executable, "-c", script], capture_output=True, text=True, check=False
        )
        assert done.returncode == 0, done.stderr
        assert done.stdout.split() == [], f"parse worker imports: {done.stdout.split()[:10]}"