Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

30.4 KB
Raw Blame
import unittest

from lib import fusion, render, schema


def make_item(item_id: str, source: str, url: str, title: str, rank_score: float) -> schema.SourceItem:
    return schema.SourceItem(
        item_id=item_id,
        source=source,
        title=title,
        body=title,
        url=url,
        relevance_hint=rank_score,
        snippet=title,
        metadata={
            "local_relevance": rank_score,
            "freshness": 80,
            "engagement_score": 5,
            "source_quality": 0.7,
        },
    )


class FusionV3Tests(unittest.TestCase):
    def test_weighted_rrf_merges_duplicate_urls(self):
        plan = schema.QueryPlan(
            intent="breaking_news",
            freshness_mode="strict_recent",
            cluster_mode="story",
            raw_topic="test",
            subqueries=[
                schema.SubQuery(label="primary", search_query="test", ranking_query="What happened in test?", sources=["reddit", "x"], weight=0.7),
                schema.SubQuery(label="reaction", search_query="test reaction", ranking_query="What are the reactions to test?", sources=["x"], weight=0.3),
            ],
            source_weights={"reddit": 0.4, "x": 0.6},
        )
        shared = "https://example.com/shared"
        streams = {
            ("primary", "reddit"): [make_item("r1", "reddit", shared, "Shared item", 0.8)],
            ("primary", "x"): [make_item("x1", "x", shared, "Shared item", 0.9)],
            ("reaction", "x"): [make_item("x2", "x", "https://example.com/unique", "Unique item", 0.7)],
        }
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=10)
        self.assertEqual(2, len(candidates))
        merged = next(candidate for candidate in candidates if candidate.url == shared)
        self.assertEqual({"primary"}, set(merged.subquery_labels))
        self.assertEqual(2, len(merged.native_ranks))
        self.assertEqual({"reddit", "x"}, set(merged.sources))
        self.assertEqual(2, len(merged.source_items))

    def test_diversify_pool_guarantees_min_per_qualifying_source(self):
        """Every qualifying source (local_relevance >= 0.25) gets at least 2
        items in the fused pool.

        Dominant sources (x, tiktok) get high weights, so pure-RRF truncation
        would squeeze out low-weight sources entirely.  The diversity guarantee
        must reserve at least 2 slots per qualifying active source.  All sources
        here have rank_score=0.8 (well above the 0.25 threshold), so every
        source qualifies for reserved slots.
        """
        sources = ["reddit", "hackernews", "x", "tiktok", "bluesky", "youtube"]
        # Heavily skewed weights: x and tiktok dominate.
        weights = {
            "x": 3.0,
            "tiktok": 2.5,
            "reddit": 0.5,
            "hackernews": 0.4,
            "bluesky": 0.3,
            "youtube": 0.3,
        }
        plan = schema.QueryPlan(
            intent="concept",
            freshness_mode="relaxed",
            cluster_mode="concept",
            raw_topic="RAG",
            subqueries=[
                schema.SubQuery(
                    label="primary",
                    search_query="RAG",
                    ranking_query="What is RAG?",
                    sources=sources,
                    weight=1.0,
                ),
            ],
            source_weights=weights,
        )
        streams: dict[tuple[str, str], list[schema.SourceItem]] = {}
        for src in sources:
            items = []
            for rank in range(4):
                items.append(
                    make_item(
                        item_id=f"{src}_{rank}",
                        source=src,
                        url=f"https://{src}.example.com/{rank}",
                        title=f"{src} item {rank}",
                        rank_score=0.8,
                    )
                )
            streams[("primary", src)] = items

        candidates = fusion.weighted_rrf(streams, plan, pool_limit=12)
        self.assertEqual(12, len(candidates))

        source_counts: dict[str, int] = {}
        for c in candidates:
            source_counts[c.source] = source_counts.get(c.source, 0) + 1

        for src in sources:
            self.assertGreaterEqual(
                source_counts.get(src, 0),
                2,
                f"Source '{src}' has {source_counts.get(src, 0)} items, expected >= 2",
            )

    def test_diversify_pool_denies_slots_for_low_relevance_source(self):
        """Sources with best local_relevance < 0.25 do not get reserved slots.

        Create two sources: 'x' with local_relevance=0.5 (qualifies) and
        'reddit' with local_relevance=0.1 (below threshold). With a tight
        pool_limit, the high-relevance source gets reserved slots while
        the low-relevance source must compete on RRF merit alone.
        """
        plan = schema.QueryPlan(
            intent="concept",
            freshness_mode="relaxed",
            cluster_mode="concept",
            raw_topic="test",
            subqueries=[
                schema.SubQuery(
                    label="primary",
                    search_query="test",
                    ranking_query="What is test?",
                    sources=["x", "reddit"],
                    weight=1.0,
                ),
            ],
            source_weights={"x": 1.0, "reddit": 1.0},
        )

        # x items: high relevance (0.5) -- qualifies for diversity reservation
        x_items = [
            make_item(f"x_{i}", "x", f"https://x.example.com/{i}", f"x item {i}", 0.5)
            for i in range(4)
        ]

        # reddit items: low relevance (0.1) -- below threshold, no reserved slots
        reddit_items = [
            make_item(f"r_{i}", "reddit", f"https://reddit.example.com/{i}", f"reddit item {i}", 0.1)
            for i in range(4)
        ]

        streams = {
            ("primary", "x"): x_items,
            ("primary", "reddit"): reddit_items,
        }

        # pool_limit=3: x gets 2 reserved + 1 more by RRF. Reddit has no
        # reserved slots, so it must out-score x items in the remainder.
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=3)
        self.assertEqual(3, len(candidates))

        # x must have at least 2 (reserved slots)
        x_count = sum(1 for c in candidates if c.source == "x")
        self.assertGreaterEqual(x_count, 2, "x should have at least 2 reserved slots")

    def test_diversify_pool_no_reservation_when_all_below_threshold(self):
        """When all sources are below the relevance threshold, no reserved slots
        are granted. The pool is filled purely by RRF score order."""
        plan = schema.QueryPlan(
            intent="concept",
            freshness_mode="relaxed",
            cluster_mode="concept",
            raw_topic="test",
            subqueries=[
                schema.SubQuery(
                    label="primary",
                    search_query="test",
                    ranking_query="What is test?",
                    sources=["x", "reddit", "hackernews"],
                    weight=1.0,
                ),
            ],
            # Give x a much higher weight so its items get higher RRF scores
            source_weights={"x": 3.0, "reddit": 0.3, "hackernews": 0.3},
        )

        streams: dict[tuple[str, str], list[schema.SourceItem]] = {}
        # All sources below threshold (local_relevance = 0.1)
        for src in ["x", "reddit", "hackernews"]:
            items = [
                make_item(f"{src}_{i}", src, f"https://{src}.example.com/{i}", f"{src} item {i}", 0.1)
                for i in range(4)
            ]
            streams[("primary", src)] = items

        candidates = fusion.weighted_rrf(streams, plan, pool_limit=4)
        self.assertEqual(4, len(candidates))

        # With no diversity reservation and x having 3x the weight,
        # x should dominate the top slots purely on RRF score
        source_counts: dict[str, int] = {}
        for c in candidates:
            source_counts[c.source] = source_counts.get(c.source, 0) + 1

        # x has 3x weight so its RRF scores are ~3x higher than reddit/hn.
        # All 4 x items should beat all reddit/hackernews items.
        self.assertEqual(
            source_counts.get("x", 0),
            4,
            f"Expected x to take all 4 slots on pure RRF merit, got {source_counts}",
        )

    def test_diversify_pool_threshold_boundary(self):
        """Source with best local_relevance exactly at the threshold (0.25)
        qualifies for reserved slots."""
        plan = schema.QueryPlan(
            intent="concept",
            freshness_mode="relaxed",
            cluster_mode="concept",
            raw_topic="boundary",
            subqueries=[
                schema.SubQuery(
                    label="primary",
                    search_query="boundary",
                    ranking_query="What is boundary?",
                    sources=["x", "reddit"],
                    weight=1.0,
                ),
            ],
            # Give x much higher weight so it would dominate without reservation
            source_weights={"x": 5.0, "reddit": 0.1},
        )

        x_items = [
            make_item(f"x_{i}", "x", f"https://x.example.com/{i}", f"x item {i}", 0.8)
            for i in range(6)
        ]

        # reddit at exactly the threshold
        reddit_items = [
            make_item(f"r_{i}", "reddit", f"https://reddit.example.com/{i}", f"reddit item {i}", 0.25)
            for i in range(3)
        ]

        streams = {
            ("primary", "x"): x_items,
            ("primary", "reddit"): reddit_items,
        }

        candidates = fusion.weighted_rrf(streams, plan, pool_limit=6)
        self.assertEqual(6, len(candidates))

        reddit_count = sum(1 for c in candidates if c.source == "reddit")
        self.assertGreaterEqual(
            reddit_count,
            2,
            f"reddit (local_relevance=0.25, at threshold) should get 2 reserved slots, got {reddit_count}",
        )


def make_item_with_author(
    item_id: str, source: str, url: str, title: str, rank_score: float, author: str | None = None,
) -> schema.SourceItem:
    return schema.SourceItem(
        item_id=item_id,
        source=source,
        title=title,
        body=title,
        url=url,
        author=author,
        relevance_hint=rank_score,
        snippet=title,
        metadata={
            "local_relevance": rank_score,
            "freshness": 80,
            "engagement_score": 5,
            "source_quality": 0.7,
        },
    )


class TestPerAuthorCap(unittest.TestCase):
    """Per-author cap: no single author should have more than 3 items in fused pool."""

    def _make_plan(self, sources: list[str]) -> schema.QueryPlan:
        return schema.QueryPlan(
            intent="breaking_news",
            freshness_mode="strict_recent",
            cluster_mode="story",
            raw_topic="test",
            subqueries=[
                schema.SubQuery(
                    label="primary",
                    search_query="test",
                    ranking_query="test",
                    sources=sources,
                    weight=1.0,
                ),
            ],
            source_weights={s: 1.0 for s in sources},
        )

    def test_author_with_8_items_capped_to_3(self):
        """@grok scenario: 8 items from the same author, only best 3 survive."""
        plan = self._make_plan(["x"])
        items = [
            make_item_with_author(
                f"x_{i}", "x", f"https://x.com/{i}", f"grok summary {i}", 0.7, author="@grok",
            )
            for i in range(8)
        ]
        streams = {("primary", "x"): items}
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
        grok_count = sum(
            1 for c in candidates
            if any(si.author == "@grok" for si in c.source_items)
        )
        self.assertLessEqual(grok_count, 3, f"@grok should be capped at 3, got {grok_count}")

    def test_author_with_3_items_all_kept(self):
        """Author with exactly 3 items should keep all of them."""
        plan = self._make_plan(["x"])
        items = [
            make_item_with_author(
                f"x_{i}", "x", f"https://x.com/{i}", f"author3 post {i}", 0.7, author="@author3",
            )
            for i in range(3)
        ]
        streams = {("primary", "x"): items}
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
        count = sum(
            1 for c in candidates
            if any(si.author == "@author3" for si in c.source_items)
        )
        self.assertEqual(count, 3)

    def test_items_without_author_not_capped(self):
        """Items with no author field should never be dropped by the cap."""
        plan = self._make_plan(["reddit"])
        items = [
            make_item_with_author(
                f"r_{i}", "reddit", f"https://reddit.com/{i}", f"post {i}", 0.7, author=None,
            )
            for i in range(6)
        ]
        streams = {("primary", "reddit"): items}
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
        self.assertEqual(len(candidates), 6)

    def test_multiple_authors_capped_independently(self):
        """Two prolific authors each get capped to 3 independently."""
        plan = self._make_plan(["x"])
        items = []
        for i in range(5):
            items.append(make_item_with_author(
                f"grok_{i}", "x", f"https://x.com/grok/{i}", f"grok {i}", 0.7, author="@grok",
            ))
        for i in range(5):
            items.append(make_item_with_author(
                f"spam_{i}", "x", f"https://x.com/spam/{i}", f"spam {i}", 0.6, author="@spammer",
            ))
        streams = {("primary", "x"): items}
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
        grok_count = sum(1 for c in candidates if any(si.author == "@grok" for si in c.source_items))
        spam_count = sum(1 for c in candidates if any(si.author == "@spammer" for si in c.source_items))
        self.assertLessEqual(grok_count, 3)
        self.assertLessEqual(spam_count, 3)

    def test_cap_keeps_best_items_by_rrf_order(self):
        """The cap should keep the first (highest-ranked) items per author."""
        plan = self._make_plan(["x"])
        # Items with decreasing relevance scores so ranking is deterministic
        items = [
            make_item_with_author(
                f"x_{i}", "x", f"https://x.com/{i}", f"post {i}", 0.9 - (i * 0.05), author="@prolific",
            )
            for i in range(5)
        ]
        streams = {("primary", "x"): items}
        candidates = fusion.weighted_rrf(streams, plan, pool_limit=20)
        kept_ids = [c.item_id for c in candidates if any(si.author == "@prolific" for si in c.source_items)]
        # The top 3 items (x_0, x_1, x_2) should be kept
        self.assertEqual(["x_0", "x_1", "x_2"], kept_ids)


class TestUrlNormalization(unittest.TestCase):
    def test_strips_www(self):
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://www.reddit.com/r/test"),
            _normalize_url("https://reddit.com/r/test"),
        )

    def test_strips_old_prefix(self):
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://old.reddit.com/r/test"),
            _normalize_url("https://reddit.com/r/test"),
        )

    def test_strips_mobile_prefix(self):
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://m.youtube.com/watch?v=abc"),
            _normalize_url("https://youtube.com/watch?v=abc"),
        )

    def test_strips_utm_params(self):
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://example.com/page?utm_source=twitter&id=5"),
            _normalize_url("https://example.com/page?id=5"),
        )

    def test_strips_trailing_slash(self):
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://example.com/page/"),
            _normalize_url("https://example.com/page"),
        )

    def test_preserves_non_tracking_params(self):
        from lib.fusion import _normalize_url
        result = _normalize_url("https://example.com/page?id=5&sort=new")
        self.assertIn("id=5", result)
        self.assertIn("sort=new", result)

    def test_case_insensitive(self):
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://Reddit.com/r/test"),
            _normalize_url("https://reddit.com/r/test"),
        )

    def test_path_case_preserved(self):
        # CR-009: the path is case-sensitive; only the host lowercases.
        from lib.fusion import _normalize_url
        self.assertNotEqual(
            _normalize_url("https://example.com/Page"),
            _normalize_url("https://example.com/page"),
        )

    def test_query_case_preserved(self):
        # CR-009: the query is case-sensitive; only the host lowercases.
        from lib.fusion import _normalize_url
        self.assertNotEqual(
            _normalize_url("https://youtube.com/watch?v=ABC123"),
            _normalize_url("https://youtube.com/watch?v=abc123"),
        )

    def test_uppercase_utm_params_stripped(self):
        # Tracking params strip case-insensitively, so an uppercase
        # UTM param does not poison the dedup key.
        from lib.fusion import _normalize_url
        self.assertEqual(
            _normalize_url("https://example.com/page?UTM_SOURCE=x"),
            _normalize_url("https://example.com/page"),
        )
        self.assertEqual(
            _normalize_url("https://example.com/page?UTM_SOURCE=x"),
            _normalize_url("https://example.com/page?utm_source=y"),
        )

if __name__ == "__main__":
    unittest.main()


class OutOfWindowSortTests(unittest.TestCase):
    def _candidate(self, name: str, published_at: str | None, confidence: str, rrf: float) -> schema.Candidate:
        item = schema.SourceItem(
            item_id=name,
            source="youtube",
            title=name,
            body="body",
            url=f"https://youtube.com/watch?v={name}",
            published_at=published_at,
            date_confidence=confidence,
        )
        return schema.Candidate(
            candidate_id=name,
            item_id=name,
            source="youtube",
            title=name,
            url=item.url,
            snippet="snippet",
            subquery_labels=["primary"],
            native_ranks={"primary:youtube": 1},
            local_relevance=0.9,
            freshness=90,
            engagement=60.0,
            source_quality=0.85,
            rrf_score=rrf,
            source_items=[item],
        )

    def test_out_of_window_sorts_below_in_window(self):
        stale = self._candidate("stale", "2025-10-15", "low", rrf=0.9)
        fresh = self._candidate("fresh", "2026-07-20", "high", rrf=0.01)
        ordered = sorted([stale, fresh], key=fusion._candidate_sort_key)
        self.assertEqual(["fresh", "stale"], [c.candidate_id for c in ordered])

    def test_undated_candidate_keeps_its_place(self):
        undated = self._candidate("undated", None, "low", rrf=0.9)
        dated = self._candidate("dated", "2026-07-20", "high", rrf=0.01)
        ordered = sorted([dated, undated], key=fusion._candidate_sort_key)
        self.assertEqual(["undated", "dated"], [c.candidate_id for c in ordered])

    def test_adapter_high_confidence_outside_window_is_still_stale(self):
        """Adapters may supply date_confidence='high' for old dates.

        The window membership must be derived from the actual date compared to
        the run window, not solely from adapter-provided date_confidence.
        """
        item = schema.SourceItem(
            item_id="old_job",
            source="jobs",
            title="Old job posting",
            body="body",
            url="https://example.com/job",
            published_at="2025-10-15",
            date_confidence="high",
        )
        stale_with_high_confidence = schema.Candidate(
            candidate_id="stale_high",
            item_id="old_job",
            source="jobs",
            title="Old job posting",
            url="https://example.com/job",
            snippet="snippet",
            subquery_labels=["primary"],
            native_ranks={"primary:jobs": 1},
            local_relevance=0.9,
            freshness=90,
            engagement=60.0,
            source_quality=0.85,
            rrf_score=0.9,
            source_items=[item],
            metadata={"range_from": "2026-06-15", "range_to": "2026-07-15"},
        )
        fresh = self._candidate("fresh", "2026-07-10", "high", rrf=0.01)
        fresh.metadata["range_from"] = "2026-06-15"
        fresh.metadata["range_to"] = "2026-07-15"

        self.assertTrue(schema.candidate_out_of_window(stale_with_high_confidence))
        self.assertFalse(schema.candidate_out_of_window(fresh))

        ordered = sorted([stale_with_high_confidence, fresh], key=fusion._candidate_sort_key)
        self.assertEqual(["fresh", "stale_high"], [c.candidate_id for c in ordered])


def _comments(n: int) -> list[dict]:
    return [{"score": 100 - i, "author": f"u{i}", "excerpt": f"comment {i}"} for i in range(n)]


class FusionEnrichedCopyTests(unittest.TestCase):
    """Same thread, two subquery streams, per-stream ids collide (R1 / R1):
    the candidate must keep the copy that carries the enrichment."""

    def _plan(self):
        return schema.QueryPlan(
            intent="breaking_news",
            freshness_mode="strict_recent",
            cluster_mode="story",
            raw_topic="kanye west",
            subqueries=[
                schema.SubQuery(label="primary", search_query="kanye west", ranking_query="kanye west", sources=["reddit"], weight=1.0),
                schema.SubQuery(label="russia", search_query="kanye russia", ranking_query="kanye russia", sources=["reddit"], weight=0.8),
            ],
            source_weights={},
        )

    def test_enriched_second_copy_is_kept(self):
        url = "https://www.reddit.com/r/Music/comments/1vy0ilk/kanye_wests_soldout_russia/"
        bare = make_item("R1", "reddit", url, "Kanye West's Russia shows canceled", 0.6)
        rich = make_item("R1", "reddit", url, "Kanye West's Russia shows canceled", 0.3)
        rich.metadata["top_comments"] = _comments(10)
        rich.metadata["comment_insights"] = ["Putin doesn't care"]
        rich.engagement = {"score": 20859, "num_comments": 741}
        bare.engagement = {"score": 18941, "num_comments": 713}
        streams = {("primary", "reddit"): [bare], ("russia", "reddit"): [rich]}

        candidates = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)

        self.assertEqual(1, len(candidates))
        cand = candidates[0]
        self.assertEqual(1, len(cand.source_items))
        self.assertEqual(10, len(cand.source_items[0].metadata["top_comments"]))
        self.assertEqual(["Putin doesn't care"], cand.source_items[0].metadata["comment_insights"])
        self.assertEqual(20859, cand.source_items[0].engagement["score"])
        self.assertEqual({"primary:reddit", "russia:reddit"}, set(cand.native_ranks))

    def test_enriched_first_copy_is_retained(self):
        url = "https://www.reddit.com/r/Music/comments/1vy0ilk/kanye_wests_soldout_russia/"
        rich = make_item("R1", "reddit", url, "Kanye West's Russia shows canceled", 0.6)
        rich.metadata["top_comments"] = _comments(4)
        bare = make_item("R1", "reddit", url, "Kanye West's Russia shows canceled", 0.3)
        streams = {("primary", "reddit"): [rich], ("russia", "reddit"): [bare]}

        candidates = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)

        self.assertEqual(4, len(candidates[0].source_items[0].metadata["top_comments"]))

    def test_distinct_urls_with_colliding_ids_stay_separate(self):
        a = make_item("R1", "reddit", "https://www.reddit.com/r/Kanye/comments/aaa/one/", "Thread one", 0.6)
        b = make_item("R1", "reddit", "https://www.reddit.com/r/Kanye/comments/bbb/two/", "Thread two", 0.5)
        streams = {("primary", "reddit"): [a], ("russia", "reddit"): [b]}

        candidates = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)

        self.assertEqual(2, len(candidates))

    def test_collapse_duplicate_urls_merges_enrichment(self):
        url = "https://www.reddit.com/r/Music/comments/1vy0ilk/kanye/"
        bare = make_item("R1", "reddit", url, "Kanye", 0.6)
        rich = make_item("R1", "reddit", url, "Kanye", 0.3)
        rich.metadata["top_comments"] = _comments(3)
        other = make_item("R2", "reddit", "https://www.reddit.com/r/Kanye/comments/zzz/other/", "Other", 0.4)

        out = fusion.collapse_duplicate_urls([bare, rich, other])

        self.assertEqual(2, len(out))
        self.assertIs(out[0], bare)
        self.assertEqual(3, len(out[0].metadata["top_comments"]))
        self.assertIs(out[1], other)


def _hn_story(story_id: str, author: str, points: int, comments: list[dict], rank: float) -> schema.SourceItem:
    item = make_item(story_id, "hackernews", "https://sqlite.org/releaselog/3_54_0.html", "SQLite 3.54.0 Released", rank)
    item.author = author
    item.engagement = {"points": points, "comments": len(comments)}
    item.metadata["hn_url"] = f"https://news.ycombinator.com/item?id={story_id}"
    item.metadata["top_comments"] = comments
    return item


class HackerNewsDuplicateSubmissionTests(unittest.TestCase):
    """Two HN submissions of one article URL are two stories, not two copies:
    each keeps its own points, comment count, discussion link, and comments."""

    def _plan(self):
        return schema.QueryPlan(
            intent="breaking_news",
            freshness_mode="strict_recent",
            cluster_mode="story",
            raw_topic="sqlite",
            subqueries=[
                schema.SubQuery(label="primary", search_query="sqlite", ranking_query="sqlite", sources=["hackernews", "reddit"], weight=1.0),
                schema.SubQuery(label="release", search_query="sqlite release", ranking_query="sqlite release", sources=["hackernews"], weight=0.8),
            ],
            source_weights={},
        )

    def _stories(self):
        comment = {"score": 0, "author": "ciupicri", "excerpt": "How nice of them to support Windows XP until now."}
        first = _hn_story("50040360", "thunderbong", 11, [], 0.6)
        second = _hn_story("50032781", "lawrencejgd", 6, [comment], 0.5)
        return first, second

    def _assert_unmerged(self, items):
        hn = {item.item_id: item for item in items if item.source == "hackernews"}
        self.assertEqual({"50040360", "50032781"}, set(hn))
        self.assertEqual({"points": 11, "comments": 0}, hn["50040360"].engagement)
        self.assertEqual([], hn["50040360"].metadata["top_comments"])
        self.assertEqual("https://news.ycombinator.com/item?id=50040360", hn["50040360"].metadata["hn_url"])
        self.assertEqual({"points": 6, "comments": 1}, hn["50032781"].engagement)
        self.assertEqual("ciupicri", hn["50032781"].metadata["top_comments"][0]["author"])

    def test_weighted_rrf_keeps_each_submission_engagement(self):
        first, second = self._stories()
        streams = {("primary", "hackernews"): [first], ("release", "hackernews"): [second]}

        candidates = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)

        # Same article URL still clusters into one candidate.
        self.assertEqual(1, len(candidates))
        self._assert_unmerged(candidates[0].source_items)

    def test_same_submission_from_two_streams_still_merges(self):
        bare = _hn_story("50032781", "lawrencejgd", 6, [], 0.6)
        rich = _hn_story("50032781", "lawrencejgd", 6, [{"score": 0, "author": "ciupicri", "excerpt": "x"}], 0.4)
        streams = {("primary", "hackernews"): [bare], ("release", "hackernews"): [rich]}

        candidates = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)

        self.assertEqual(1, len(candidates[0].source_items))
        self.assertEqual("ciupicri", candidates[0].source_items[0].metadata["top_comments"][0]["author"])

    def test_hn_and_reddit_of_same_article_share_a_candidate(self):
        first, _ = self._stories()
        reddit = make_item("R1", "reddit", "https://sqlite.org/releaselog/3_54_0.html", "SQLite 3.54.0 released", 0.5)
        streams = {("primary", "hackernews"): [first], ("primary", "reddit"): [reddit]}

        candidates = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)

        self.assertEqual(1, len(candidates))
        self.assertEqual(["hackernews", "reddit"], sorted(item.source for item in candidates[0].source_items))

    def _evidence_line(self, candidate):
        lines = render._render_candidate(candidate, "1.")
        evidence = [line for line in lines if "Evidence:" in line]
        self.assertEqual(1, len(evidence), lines)
        return evidence[0]

    def test_card_evidence_comes_from_primary_submission(self):
        # The sibling submission has the longer snippet; the card shows the
        # primary story's metrics, so its Evidence must be the primary's text.
        first, second = self._stories()
        second.snippet = "How nice of them to support Windows XP until now, says ciupicri in a long comment."
        streams = {("primary", "hackernews"): [first], ("release", "hackernews"): [second]}

        candidate = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)[0]
        primary = schema.candidate_primary_item(candidate)

        self.assertEqual("50040360", primary.item_id)
        self.assertEqual(primary.snippet, candidate.snippet)
        evidence = self._evidence_line(candidate)
        self.assertIn("SQLite 3.54.0 Released", evidence)
        self.assertNotIn("ciupicri", evidence)

    def test_card_evidence_matches_primary_when_sibling_scores_higher(self):
        # A later sibling that wins the primary-score comparison must not put
        # its text beside the first-attached submission's metrics either.
        first, second = self._stories()
        first.metadata["local_relevance"] = 0.3
        second.snippet = "How nice of them to support Windows XP until now, says ciupicri."
        streams = {("primary", "hackernews"): [first], ("release", "hackernews"): [second]}

        candidate = fusion.weighted_rrf(streams, self._plan(), pool_limit=10)[0]
        primary = schema.candidate_primary_item(candidate)

        self.assertEqual(primary.snippet, candidate.snippet)
        self.assertIn(primary.snippet, self._evidence_line(candidate))

    def test_collapse_duplicate_urls_keeps_distinct_submissions(self):
        first, second = self._stories()

        out = fusion.collapse_duplicate_urls([first, second])

        self._assert_unmerged(out)