Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

21.6 KB
Raw Blame
"""Tests for LLM-enriched knowledge graph layers and tour generation."""

from __future__ import annotations

from dataclasses import dataclass, field
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock

import pytest

from repowise.core.analysis.knowledge_graph import KnowledgeGraphResult
from repowise.core.generation.knowledge_graph import (
    _backfill_summaries,
    _parse_json_response,
    build_deterministic_tour,
    enrich_knowledge_graph,
    enrich_knowledge_graph_structural,
    finalize_knowledge_graph,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


@dataclass
class FakeKGResult:
    project: dict = field(default_factory=dict)
    nodes: list[dict] = field(default_factory=list)
    edges: list[dict] = field(default_factory=list)
    layers: list[dict] = field(default_factory=list)
    tour: list[dict] = field(default_factory=list)
    fingerprint: str = ""


def _make_llm_client(response_content: str = "{}"):
    client = AsyncMock()
    client.generate.return_value = SimpleNamespace(
        content=response_content,
        input_tokens=100,
        output_tokens=50,
    )
    return client


def _make_graph_builder(pagerank: dict[str, float] | None = None):
    builder = MagicMock()
    builder.pagerank.return_value = pagerank or {}
    builder.betweenness_centrality.return_value = {}
    return builder


def _make_repo_structure(entry_points: list[str] | None = None):
    return SimpleNamespace(
        is_monorepo=False,
        entry_points=entry_points or ["src/main.py"],
        total_files=10,
        total_loc=1000,
    )


def _make_kg_skeleton(layers: list[dict] | None = None, nodes: list[dict] | None = None):
    if layers is None:
        layers = [
            {"id": "layer:core", "name": "src/core", "description": "", "nodeIds": ["file:src/core.py", "file:src/utils.py"]},
            {"id": "layer:cli", "name": "cli", "description": "", "nodeIds": ["file:src/main.py"]},
        ]
    if nodes is None:
        nodes = [
            {"id": "file:src/core.py", "type": "file", "filePath": "src/core.py", "summary": ""},
            {"id": "file:src/utils.py", "type": "file", "filePath": "src/utils.py", "summary": ""},
            {"id": "file:src/main.py", "type": "file", "filePath": "src/main.py", "summary": ""},
        ]
    return FakeKGResult(
        layers=layers,
        nodes=nodes,
    )


# ---------------------------------------------------------------------------
# Layer enrichment tests
# ---------------------------------------------------------------------------


class TestEnrichLayers:
    @pytest.mark.asyncio
    async def test_enriches_layer_names(self):
        llm = _make_llm_client(
            '{"layers": [{"id": "layer:core", "name": "Core Pipeline", "description": "Handles data flow"}, '
            '{"id": "layer:cli", "name": "CLI Interface", "description": "Command line entry"}]}'
        )
        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder({"src/core.py": 0.5, "src/main.py": 0.3})
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert result.layers[0]["name"] == "Core Pipeline"
        assert result.layers[0]["description"] == "Handles data flow"
        assert result.layers[1]["name"] == "CLI Interface"

    @pytest.mark.asyncio
    async def test_naming_prompt_counts_each_layers_files_by_folder(self):
        layers = [
            {
                "id": "layer:config",
                "name": "config",
                "description": "",
                "nodeIds": [
                    "file:packages/web/a.tsx",
                    "file:packages/web/b.tsx",
                    "file:.github/workflows/ci.yml",
                    "file:README.md",
                ],
            },
            {"id": "layer:cli", "name": "cli", "description": "", "nodeIds": ["file:src/main.py"]},
        ]
        llm = _make_llm_client('{"layers": []}')
        await enrich_knowledge_graph(
            _make_kg_skeleton(layers=layers), llm, _make_graph_builder(), _make_repo_structure(), []
        )
        prompt = llm.generate.call_args_list[0].args[1]
        assert "Files by folder: packages/web 2, .github/workflows 1, (root) 1" in prompt
        # A single-folder layer keeps its one folder and the existing fields.
        cli = prompt.split('Community "layer:cli"')[1]
        assert "File count: 1" in cli
        assert "Files by folder: src 1\n" in cli
        assert "Top files: src/main.py" in cli

    @pytest.mark.asyncio
    async def test_positional_index_responses_are_ignored(self):
        # Regression: a model answering with list positions instead of layer
        # ids must not rename anything (positional joins once shuffled every
        # layer name when later batches restarted their indices at 0).
        llm = _make_llm_client(
            '{"layers": [{"index": 0, "name": "Wrong Name", "description": "wrong"}, '
            '{"index": 1, "name": "Also Wrong", "description": "wrong"}]}'
        )
        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder()
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert result.layers[0]["name"] == "src/core"  # heuristic preserved
        assert result.layers[1]["name"] == "cli"

    @pytest.mark.asyncio
    async def test_cross_batch_ids_do_not_clobber_other_layers(self):
        # Eight layers → two batches of five and three. Batch 2's response
        # echoes batch 1's ids; those answers must be dropped, not applied.
        layers = [
            {"id": f"layer:l{i}", "name": f"heuristic-{i}", "description": "",
             "nodeIds": [f"file:src/f{i}.py"]}
            for i in range(8)
        ]
        nodes = [
            {"id": f"file:src/f{i}.py", "type": "file", "filePath": f"src/f{i}.py", "summary": ""}
            for i in range(8)
        ]
        responses = [
            # Batch 1 (l0..l4): correct ids.
            '{"layers": [{"id": "layer:l0", "name": "Named Zero", "description": "d0"}]}',
            # Batch 2 (l5..l7): answers with batch-1 ids — all must be skipped.
            '{"layers": [{"id": "layer:l0", "name": "Clobbered", "description": "x"}, '
            '{"id": "layer:l1", "name": "Clobbered", "description": "x"}]}',
            # Tour call.
            '{"tour": []}',
        ]
        call_count = 0

        async def _side_effect(*args, **kwargs):
            nonlocal call_count
            content = responses[min(call_count, len(responses) - 1)]
            call_count += 1
            return SimpleNamespace(content=content, input_tokens=10, output_tokens=10)

        llm = AsyncMock()
        llm.generate.side_effect = _side_effect

        skeleton = _make_kg_skeleton(layers=layers, nodes=nodes)
        builder = _make_graph_builder()
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert result.layers[0]["name"] == "Named Zero"
        assert all(layer["name"] != "Clobbered" for layer in result.layers)
        assert result.layers[5]["name"] == "heuristic-5"  # untouched by bad batch

    @pytest.mark.asyncio
    async def test_names_stay_unique_across_batches(self):
        # Batches are named independently; a name one batch already used must
        # be offered as taken to the next and, if reused anyway, qualified.
        layers = [
            {"id": f"layer:l{i}", "name": f"heuristic-{i}", "description": "",
             "nodeIds": [f"file:src/f{i}.py"]}
            for i in range(6)
        ]
        nodes = [
            {"id": f"file:src/f{i}.py", "type": "file", "filePath": f"src/f{i}.py", "summary": ""}
            for i in range(6)
        ]
        responses = iter([
            '{"layers": [{"id": "layer:l0", "name": "Shared Widgets", "description": "d"}]}',
            '{"layers": [{"id": "layer:l5", "name": "shared widgets", "description": "d"}]}',
            '{"tour": []}',
        ])
        prompts: list[str] = []

        async def _side_effect(system, user, **kwargs):
            prompts.append(user)
            return SimpleNamespace(content=next(responses), input_tokens=1, output_tokens=1)

        llm = AsyncMock()
        llm.generate.side_effect = _side_effect

        skeleton = _make_kg_skeleton(layers=layers, nodes=nodes)
        result = await enrich_knowledge_graph(
            skeleton, llm, _make_graph_builder(), _make_repo_structure(), []
        )

        assert "Names already given" not in prompts[0]
        assert "Shared Widgets" in prompts[1]
        assert result.layers[0]["name"] == "Shared Widgets"
        assert result.layers[5]["name"] == "shared widgets (l5)"
        names = [layer["name"].casefold() for layer in result.layers]
        assert len(names) == len(set(names))

    @pytest.mark.asyncio
    async def test_unknown_id_is_skipped(self):
        llm = _make_llm_client(
            '{"layers": [{"id": "layer:nonexistent", "name": "Ghost", "description": "x"}]}'
        )
        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder()
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert all(layer["name"] != "Ghost" for layer in result.layers)

    @pytest.mark.asyncio
    async def test_falls_back_on_llm_failure(self):
        llm = AsyncMock()
        llm.generate.side_effect = RuntimeError("LLM is down")
        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder()
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert result.layers[0]["name"] == "src/core"  # heuristic label preserved

    @pytest.mark.asyncio
    async def test_falls_back_on_invalid_json(self):
        llm = _make_llm_client("not json at all {{{ broken")
        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder()
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert result.layers[0]["name"] == "src/core"  # heuristic preserved

    @pytest.mark.asyncio
    async def test_empty_layers_no_crash(self):
        llm = _make_llm_client('{"tour": []}')
        skeleton = _make_kg_skeleton(layers=[])
        builder = _make_graph_builder()
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert result.layers == []


# ---------------------------------------------------------------------------
# Tour generation tests
# ---------------------------------------------------------------------------


class TestTourGeneration:
    @pytest.mark.asyncio
    async def test_tour_generated_from_llm(self):
        tour_response = (
            '{"tour": [{"order": 1, "title": "Start Here", '
            '"description": "Begin with the CLI entry point.", '
            '"files": ["src/main.py"]}]}'
        )
        call_count = 0

        async def _side_effect(*args, **kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                return SimpleNamespace(content='{"layers": []}', input_tokens=10, output_tokens=10)
            return SimpleNamespace(content=tour_response, input_tokens=100, output_tokens=50)

        llm = AsyncMock()
        llm.generate.side_effect = _side_effect

        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder({"src/main.py": 0.5})
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert len(result.tour) == 1
        assert result.tour[0]["order"] == 1
        assert result.tour[0]["title"] == "Start Here"
        assert "file:src/main.py" in result.tour[0]["nodeIds"]

    @pytest.mark.asyncio
    async def test_tour_fallback_on_failure(self):
        call_count = 0

        async def _side_effect(*args, **kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                return SimpleNamespace(content='{"layers": []}', input_tokens=10, output_tokens=10)
            raise RuntimeError("Tour generation failed")

        llm = AsyncMock()
        llm.generate.side_effect = _side_effect

        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder({"src/main.py": 0.5, "src/core.py": 0.3})
        repo = _make_repo_structure()

        result = await enrich_knowledge_graph(skeleton, llm, builder, repo, [])
        assert len(result.tour) >= 1  # deterministic fallback


# ---------------------------------------------------------------------------
# Deterministic tour fallback tests
# ---------------------------------------------------------------------------


class TestDeterministicTour:
    def test_basic_tour(self):
        pagerank = {"src/main.py": 0.5, "src/core.py": 0.3, "src/utils.py": 0.1}
        entry_points = ["src/main.py"]
        layers = [
            {"name": "Core", "nodeIds": ["file:src/core.py", "file:src/utils.py"]},
            {"name": "CLI", "nodeIds": ["file:src/main.py"]},
        ]
        tour = build_deterministic_tour(pagerank, entry_points, layers)
        assert len(tour) >= 2
        assert tour[0]["order"] == 1
        assert tour[0]["title"] == "Entry Point"
        assert "file:src/main.py" in tour[0]["nodeIds"]

    def test_no_entry_points(self):
        pagerank = {"src/core.py": 0.5}
        layers = [{"name": "Core", "nodeIds": ["file:src/core.py"]}]
        tour = build_deterministic_tour(pagerank, [], layers)
        assert len(tour) >= 1
        assert tour[0]["title"] == "Core"

    def test_empty_layers(self):
        tour = build_deterministic_tour({}, ["main.py"], [])
        assert len(tour) == 1
        assert tour[0]["title"] == "Entry Point"


# ---------------------------------------------------------------------------
# Summary backfill tests
# ---------------------------------------------------------------------------


class TestSummaryBackfill:
    def test_populates_summaries(self):
        kg = _make_kg_skeleton()
        pages = [
            SimpleNamespace(target_path="src/core.py", summary="Core business logic module"),
        ]
        _backfill_summaries(kg, pages)
        core_node = next(n for n in kg.nodes if n["filePath"] == "src/core.py")
        assert core_node["summary"] == "Core business logic module"

    def test_does_not_overwrite_existing(self):
        kg = _make_kg_skeleton(nodes=[
            {"id": "file:src/core.py", "type": "file", "filePath": "src/core.py", "summary": "Existing summary"},
        ])
        pages = [
            SimpleNamespace(target_path="src/core.py", summary="New summary"),
        ]
        _backfill_summaries(kg, pages)
        assert kg.nodes[0]["summary"] == "Existing summary"

    def test_handles_no_pages(self):
        kg = _make_kg_skeleton()
        _backfill_summaries(kg, [])
        assert all(n["summary"] == "" for n in kg.nodes)


# ---------------------------------------------------------------------------
# JSON parsing tests
# ---------------------------------------------------------------------------


class TestParseJsonResponse:
    def test_plain_json(self):
        result = _parse_json_response('{"layers": []}')
        assert result == {"layers": []}

    def test_markdown_fenced(self):
        result = _parse_json_response('```json\n{"layers": []}\n```')
        assert result == {"layers": []}

    def test_json_with_preamble(self):
        result = _parse_json_response('Here is the result:\n{"layers": []}')
        assert result == {"layers": []}

    def test_invalid_returns_none(self):
        result = _parse_json_response("not json at all")
        assert result is None

    def test_empty_string(self):
        result = _parse_json_response("")
        assert result is None


# ---------------------------------------------------------------------------
# Structural / finalize split (the init overlap seam)
# ---------------------------------------------------------------------------


class TestEnrichmentSplit:
    """The orchestrator runs the page-independent half (layer naming + tour)
    concurrently with page generation, then finalizes with the pages. These
    pin that the split is behavior-preserving vs the serial wrapper and that
    the structural half never depends on generated pages."""

    @pytest.mark.asyncio
    async def test_structural_returns_layers_and_tour_without_pages(self):
        llm = _make_llm_client(
            '{"layers": [{"id": "layer:core", "name": "Core Pipeline", "description": "d"}], '
            '"tour": [{"order": 1, "title": "Start", "description": "s", "files": ["src/main.py"]}]}'
        )
        skeleton = _make_kg_skeleton()
        builder = _make_graph_builder({"src/core.py": 0.5, "src/main.py": 0.3})
        repo = _make_repo_structure()

        enriched_layers, tour = await enrich_knowledge_graph_structural(
            skeleton, llm, builder, repo, []
        )
        # Layer naming applied…
        assert enriched_layers[0]["name"] == "Core Pipeline"
        # …tour produced…
        assert tour and tour[0]["title"] == "Start"
        # …but the structural half has no pages, so no summary was backfilled
        # and it did not assign layers/tour onto the skeleton (finalize's job).
        assert all(n["summary"] == "" for n in skeleton.nodes)

    def test_finalize_backfills_assigns_and_returns_skeleton(self):
        skeleton = _make_kg_skeleton()
        enriched_layers = [{"id": "layer:core", "name": "Renamed", "description": "d", "nodeIds": []}]
        tour = [{"order": 1, "title": "T", "description": "", "nodeIds": []}]
        pages = [SimpleNamespace(target_path="src/core.py", summary="Core logic")]

        result = finalize_knowledge_graph(skeleton, enriched_layers, tour, pages)

        assert result is skeleton
        assert result.layers == enriched_layers
        assert result.tour == tour
        core = next(n for n in result.nodes if n["filePath"] == "src/core.py")
        assert core["summary"] == "Core logic"

    @pytest.mark.asyncio
    async def test_split_matches_serial_wrapper(self):
        # Equivalence: structural + finalize must produce the same skeleton the
        # single-call wrapper does, given identical (deterministic) LLM output.
        response = (
            '{"layers": [{"id": "layer:core", "name": "Core Pipeline", "description": "d"}, '
            '{"id": "layer:cli", "name": "CLI", "description": "c"}], '
            '"tour": [{"order": 1, "title": "Start", "description": "s", "files": ["src/main.py"]}]}'
        )
        pages = [SimpleNamespace(target_path="src/core.py", summary="Core logic")]
        builder = _make_graph_builder({"src/core.py": 0.5, "src/main.py": 0.3})
        repo = _make_repo_structure()

        wrapper_skel = _make_kg_skeleton()
        wrapped = await enrich_knowledge_graph(
            wrapper_skel, _make_llm_client(response), builder, repo, [], pages
        )

        split_skel = _make_kg_skeleton()
        layers, tour = await enrich_knowledge_graph_structural(
            split_skel, _make_llm_client(response), builder, repo, []
        )
        split = finalize_knowledge_graph(split_skel, layers, tour, pages)

        assert split.layers == wrapped.layers
        assert split.tour == wrapped.tour
        assert [n["summary"] for n in split.nodes] == [n["summary"] for n in wrapped.nodes]

    @pytest.mark.asyncio
    async def test_generation_snapshot_insulated_from_concurrent_layer_naming(self):
        # The orchestrator hands generation a to_dict() snapshot of the skeleton
        # and lets structural enrichment run concurrently. Enrichment renames
        # layers in place, so the snapshot generation holds must be a copy that
        # a later rename cannot mutate.
        result = KnowledgeGraphResult(
            layers=[{"id": "layer:core", "name": "heuristic-core", "description": "", "nodeIds": ["file:src/core.py"]}],
            nodes=[{"id": "file:src/core.py", "type": "file", "filePath": "src/core.py", "summary": ""}],
        )
        snapshot = result.to_dict()

        llm = _make_llm_client(
            '{"layers": [{"id": "layer:core", "name": "Renamed Core", "description": "d"}]}'
        )
        await enrich_knowledge_graph_structural(
            result, llm, _make_graph_builder({"src/core.py": 0.5}), _make_repo_structure(), []
        )

        # In-place rename landed on the live skeleton…
        assert result.layers[0]["name"] == "Renamed Core"
        # …but the snapshot generation already holds is untouched.
        assert snapshot["layers"][0]["name"] == "heuristic-core"


def test_only_file_ids_become_tour_files() -> None:
    """A layer's member list can hold ids that name no file.

    ``module:``, ``layer:`` and ``dir:`` are all real namespaces here, and none
    is a known prefix — so each parses as a bare path and the permissive
    helper handed it back as a file. It then got looked up in PageRank and
    offered as a tour stop, which is a path that does not exist.
    """
    from repowise.core.generation.knowledge_graph import build_deterministic_tour

    layers = [
        {
            "id": "layer:api",
            "name": "API",
            "nodeIds": [
                "module:core-ingestion",
                "layer:api",
                "dir:packages/core",
                "file:packages/core/api.py",
            ],
        }
    ]
    steps = build_deterministic_tour(
        pagerank={"packages/core/api.py": 0.9, "module:core-ingestion": 0.99},
        entry_points=[],
        layers=layers,
    )
    named = [nid for step in steps for nid in step.get("nodeIds", [])]
    assert named == ["file:packages/core/api.py"], named