Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

8.2 KB
Raw Blame
"""Tests for scripts/lib/reddit_shreddit.py — keyless shreddit comment scrape."""

from pathlib import Path
from unittest import mock

from lib import reddit_shreddit as rs

FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "reddit_shreddit_comments_sample.html"


def _html():
    return FIXTURE.read_text(encoding="utf-8")


# The partial after a DOM round-trip: HTML attribute names are case-insensitive,
# so a serializer writes ``thingId`` as ``thingid``. Values keep their case.
LOWERCASED_ATTR_HTML = """
<shreddit-comment-tree-stats total-comments="2" sort="TOP"></shreddit-comment-tree-stats>
<shreddit-comment created="2026-08-09T12:45:51+0000" author="AliceCooper" thingid="t1_aaa"
  permalink="/r/videos/comments/x1/comment/aaa/" score="1136" depth="0">
  <div id="t1_aaa-post-rtjson-content"><div><p>First body about $750 pending.</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-08-09T12:46:00+0000" author="bob" thingid="t1_bbb"
  permalink="/r/videos/comments/x1/comment/bbb/" score="42" depth="0">
  <div id="t1_bbb-post-rtjson-content"><div><p>Second body.</p></div></div>
</shreddit-comment>
"""


class TestExtractPostRef:
    def test_extracts_sub_and_id(self):
        ref = rs.extract_post_ref("https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/")
        assert ref == ("Rakuten", "1taeiw0")

    def test_non_thread_url_returns_none(self):
        assert rs.extract_post_ref("https://www.reddit.com/r/Rakuten/") is None
        assert rs.extract_post_ref("") is None

    def test_svc_url_shape(self):
        # sort=top guarantees the highest-scored comments land on page 1.
        assert rs._svc_url("Rakuten", "1taeiw0") == (
            "https://www.reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0?sort=top"
        )


class TestParseComments:
    """parse_comments reads <shreddit-comment> elements into scored dicts."""

    def test_happy_path(self):
        comments = rs.parse_comments(_html())
        assert len(comments) >= 1
        for c in comments:
            assert isinstance(c["score"], int)
            assert c["author"] and c["author"] not in ("[deleted]", "[removed]")
            assert c["body"]

    def test_sorted_by_score_desc(self):
        scores = [c["score"] for c in rs.parse_comments(_html())]
        assert scores == sorted(scores, reverse=True)

    def test_deleted_and_removed_filtered(self):
        authors = [c["author"] for c in rs.parse_comments(_html())]
        assert "[deleted]" not in authors and "[removed]" not in authors

    def test_negative_score_retained(self):
        scores = [c["score"] for c in rs.parse_comments(_html())]
        assert -7 in scores  # synthetic downvoted-but-real comment

    def test_limit_honored(self):
        assert len(rs.parse_comments(_html(), limit=2)) == 2

    def test_body_text_extracted(self):
        bodies = [c["body"] for c in rs.parse_comments(_html())]
        assert any("$750" in b or "pending" in b for b in bodies)

    def test_comment_url_built(self):
        comments = rs.parse_comments(_html())
        assert [c["url"] for c in comments] == [
            f"https://reddit.com/r/Rakuten/comments/1taeiw0/comment/{comment_id}/"
            for comment_id in (
                "ol8tp8n", "olcy1iv", "olaqzjk", "omlbiqg", "ol8undb", "ol8w8w6", "synthneg"
            )
        ]

    def test_empty_html_returns_empty(self):
        assert rs.parse_comments("") == []
        assert rs.parse_comments("<html>no comments here</html>") == []

    def test_lowercased_attribute_names_parsed(self):
        comments = rs.parse_comments(LOWERCASED_ATTR_HTML)
        assert [c["score"] for c in comments] == [1136, 42]
        assert comments[0]["author"] == "AliceCooper"  # values keep their case
        assert "$750" in comments[0]["body"]

class TestBotFilter:
    """Bot comments occupy top-comment slots without carrying community signal."""

    @staticmethod
    def _comment_html(author, thing_id="t1_botfilter1", score=999):
        return (
            f'<shreddit-comment author="{author}" thingId="{thing_id}" '
            f'score="{score}" permalink="/r/test/comments/1/x/{thing_id}/">'
            f'</shreddit-comment>'
            f'<div id="{thing_id}-post-rtjson-content">'
            f'<p>I will be messaging you in 3 days to remind you of this link.</p>'
            f'</div>'
        )

    def test_known_bots_dropped(self):
        for bot in ("RemindMeBot", "AutoModerator", "sneakpeekbot"):
            assert rs.parse_comments(self._comment_html(bot)) == [], bot

    def test_bot_match_is_case_insensitive(self):
        assert rs.parse_comments(self._comment_html("remindmebot")) == []

    def test_separator_suffix_bots_dropped(self):
        for bot in ("some-random-bot", "subreddit_bot"):
            assert rs.parse_comments(self._comment_html(bot)) == [], bot

    def test_camelcase_bots_dropped(self):
        # The separator-free convention is the common one on Reddit.
        for bot in ("WikiTextBot", "RepostSleuthBot", "RemindMeBot2"):
            assert rs.parse_comments(self._comment_html(bot)) == [], bot

    def test_human_authors_kept(self):
        # Names merely ending in "bot" are people, not bots. The capital B in
        # the camelCase rule is what separates "WikiTextBot" from "Talbot".
        for human in ("Talbot", "abbot", "u_bothell_local", "MSRS-",
                      "TheBotanist", "Botany101", "Robotics_fan"):
            out = rs.parse_comments(self._comment_html(human))
            assert len(out) == 1, human
            assert out[0]["author"] == human

    def test_bot_does_not_displace_human_from_slot(self):
        # The reported failure was a bot *taking a slot*, not merely appearing:
        # it outscores the humans, so it wins the ranking before truncation.
        html = (self._comment_html("RemindMeBot", thing_id="t1_bot", score=999)
                + self._comment_html("real_person", thing_id="t1_human", score=5))
        out = rs.parse_comments(html, limit=1)
        assert [c["author"] for c in out] == ["real_person"]

    def test_is_bot_author_handles_blank(self):
        assert rs._is_bot_author("") is False
        assert rs._is_bot_author(None) is False


class TestTotalComments:
    def test_reads_total(self):
        assert rs._total_comments(_html()) == 14

    def test_missing_returns_none(self):
        assert rs._total_comments("<html></html>") is None


class TestFetchComments:
    """fetch_comments wires URL -> svc fetch -> parse, never raising."""

    def test_happy_path(self):
        url = "https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/"
        with mock.patch.object(rs.http, "get_text", return_value=_html()) as m:
            out = rs.fetch_comments(url)
        # svc endpoint, not .json
        assert "/svc/shreddit/comments/" in m.call_args[0][0]
        assert ".json" not in m.call_args[0][0]
        assert out["num_comments"] == 14
        assert len(out["top_comments"]) >= 1
        first = out["top_comments"][0]
        assert {"score", "date", "author", "excerpt", "url"} <= set(first.keys())
        assert isinstance(out["comment_insights"], list)

    def test_bad_url_returns_empty(self):
        out = rs.fetch_comments("https://www.reddit.com/r/Rakuten/")
        assert out["top_comments"] == [] and out["num_comments"] is None

    def test_fetch_failure_returns_empty(self):
        url = "https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/"
        with mock.patch.object(rs.http, "get_text", return_value=None):
            out = rs.fetch_comments(url)
        assert out["top_comments"] == [] and out["num_comments"] is None


class TestEnrichmentBudget:
    """Busy topics enrich more threads and carry more comments per thread."""

    def test_enrich_limits_by_depth(self):
        assert rs.ENRICH_LIMITS == {"quick": 4, "default": 8, "deep": 12}

    def test_parse_comments_returns_up_to_twelve(self):
        html = "".join(
            f'<shreddit-comment author="user{i}" thingId="t1_c{i}" score="{100 - i}" '
            f'permalink="/r/test/comments/1/x/t1_c{i}/"></shreddit-comment>'
            f'<div id="t1_c{i}-post-rtjson-content"><p>comment number {i} body text</p></div>'
            for i in range(30)
        )
        out = rs.parse_comments(html)
        assert len(out) == rs.MAX_COMMENTS == 12
        assert [c["score"] for c in out] == list(range(100, 88, -1))