AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
"""Tests for scripts/lib/reddit_shreddit.py — keyless shreddit comment scrape."""
from pathlib import Path
from unittest import mock
from lib import reddit_shreddit as rs
FIXTURE = Path(__file__).resolve().parent.parent / "fixtures" / "reddit_shreddit_comments_sample.html"
def _html():
return FIXTURE.read_text(encoding="utf-8")
# The partial after a DOM round-trip: HTML attribute names are case-insensitive,
# so a serializer writes ``thingId`` as ``thingid``. Values keep their case.
LOWERCASED_ATTR_HTML = """
<shreddit-comment-tree-stats total-comments="2" sort="TOP"></shreddit-comment-tree-stats>
<shreddit-comment created="2026-08-09T12:45:51+0000" author="AliceCooper" thingid="t1_aaa"
permalink="/r/videos/comments/x1/comment/aaa/" score="1136" depth="0">
<div id="t1_aaa-post-rtjson-content"><div><p>First body about $750 pending.</p></div></div>
</shreddit-comment>
<shreddit-comment created="2026-08-09T12:46:00+0000" author="bob" thingid="t1_bbb"
permalink="/r/videos/comments/x1/comment/bbb/" score="42" depth="0">
<div id="t1_bbb-post-rtjson-content"><div><p>Second body.</p></div></div>
</shreddit-comment>
"""
class TestExtractPostRef:
def test_extracts_sub_and_id(self):
ref = rs.extract_post_ref("https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/")
assert ref == ("Rakuten", "1taeiw0")
def test_non_thread_url_returns_none(self):
assert rs.extract_post_ref("https://www.reddit.com/r/Rakuten/") is None
assert rs.extract_post_ref("") is None
def test_svc_url_shape(self):
# sort=top guarantees the highest-scored comments land on page 1.
assert rs._svc_url("Rakuten", "1taeiw0") == (
"https://www.reddit.com/svc/shreddit/comments/r/Rakuten/t3_1taeiw0?sort=top"
)
class TestParseComments:
"""parse_comments reads <shreddit-comment> elements into scored dicts."""
def test_happy_path(self):
comments = rs.parse_comments(_html())
assert len(comments) >= 1
for c in comments:
assert isinstance(c["score"], int)
assert c["author"] and c["author"] not in ("[deleted]", "[removed]")
assert c["body"]
def test_sorted_by_score_desc(self):
scores = [c["score"] for c in rs.parse_comments(_html())]
assert scores == sorted(scores, reverse=True)
def test_deleted_and_removed_filtered(self):
authors = [c["author"] for c in rs.parse_comments(_html())]
assert "[deleted]" not in authors and "[removed]" not in authors
def test_negative_score_retained(self):
scores = [c["score"] for c in rs.parse_comments(_html())]
assert -7 in scores # synthetic downvoted-but-real comment
def test_limit_honored(self):
assert len(rs.parse_comments(_html(), limit=2)) == 2
def test_body_text_extracted(self):
bodies = [c["body"] for c in rs.parse_comments(_html())]
assert any("$750" in b or "pending" in b for b in bodies)
def test_comment_url_built(self):
comments = rs.parse_comments(_html())
assert [c["url"] for c in comments] == [
f"https://reddit.com/r/Rakuten/comments/1taeiw0/comment/{comment_id}/"
for comment_id in (
"ol8tp8n", "olcy1iv", "olaqzjk", "omlbiqg", "ol8undb", "ol8w8w6", "synthneg"
)
]
def test_empty_html_returns_empty(self):
assert rs.parse_comments("") == []
assert rs.parse_comments("<html>no comments here</html>") == []
def test_lowercased_attribute_names_parsed(self):
comments = rs.parse_comments(LOWERCASED_ATTR_HTML)
assert [c["score"] for c in comments] == [1136, 42]
assert comments[0]["author"] == "AliceCooper" # values keep their case
assert "$750" in comments[0]["body"]
class TestBotFilter:
"""Bot comments occupy top-comment slots without carrying community signal."""
@staticmethod
def _comment_html(author, thing_id="t1_botfilter1", score=999):
return (
f'<shreddit-comment author="{author}" thingId="{thing_id}" '
f'score="{score}" permalink="/r/test/comments/1/x/{thing_id}/">'
f'</shreddit-comment>'
f'<div id="{thing_id}-post-rtjson-content">'
f'<p>I will be messaging you in 3 days to remind you of this link.</p>'
f'</div>'
)
def test_known_bots_dropped(self):
for bot in ("RemindMeBot", "AutoModerator", "sneakpeekbot"):
assert rs.parse_comments(self._comment_html(bot)) == [], bot
def test_bot_match_is_case_insensitive(self):
assert rs.parse_comments(self._comment_html("remindmebot")) == []
def test_separator_suffix_bots_dropped(self):
for bot in ("some-random-bot", "subreddit_bot"):
assert rs.parse_comments(self._comment_html(bot)) == [], bot
def test_camelcase_bots_dropped(self):
# The separator-free convention is the common one on Reddit.
for bot in ("WikiTextBot", "RepostSleuthBot", "RemindMeBot2"):
assert rs.parse_comments(self._comment_html(bot)) == [], bot
def test_human_authors_kept(self):
# Names merely ending in "bot" are people, not bots. The capital B in
# the camelCase rule is what separates "WikiTextBot" from "Talbot".
for human in ("Talbot", "abbot", "u_bothell_local", "MSRS-",
"TheBotanist", "Botany101", "Robotics_fan"):
out = rs.parse_comments(self._comment_html(human))
assert len(out) == 1, human
assert out[0]["author"] == human
def test_bot_does_not_displace_human_from_slot(self):
# The reported failure was a bot *taking a slot*, not merely appearing:
# it outscores the humans, so it wins the ranking before truncation.
html = (self._comment_html("RemindMeBot", thing_id="t1_bot", score=999)
+ self._comment_html("real_person", thing_id="t1_human", score=5))
out = rs.parse_comments(html, limit=1)
assert [c["author"] for c in out] == ["real_person"]
def test_is_bot_author_handles_blank(self):
assert rs._is_bot_author("") is False
assert rs._is_bot_author(None) is False
class TestTotalComments:
def test_reads_total(self):
assert rs._total_comments(_html()) == 14
def test_missing_returns_none(self):
assert rs._total_comments("<html></html>") is None
class TestFetchComments:
"""fetch_comments wires URL -> svc fetch -> parse, never raising."""
def test_happy_path(self):
url = "https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/"
with mock.patch.object(rs.http, "get_text", return_value=_html()) as m:
out = rs.fetch_comments(url)
# svc endpoint, not .json
assert "/svc/shreddit/comments/" in m.call_args[0][0]
assert ".json" not in m.call_args[0][0]
assert out["num_comments"] == 14
assert len(out["top_comments"]) >= 1
first = out["top_comments"][0]
assert {"score", "date", "author", "excerpt", "url"} <= set(first.keys())
assert isinstance(out["comment_insights"], list)
def test_bad_url_returns_empty(self):
out = rs.fetch_comments("https://www.reddit.com/r/Rakuten/")
assert out["top_comments"] == [] and out["num_comments"] is None
def test_fetch_failure_returns_empty(self):
url = "https://www.reddit.com/r/Rakuten/comments/1taeiw0/title/"
with mock.patch.object(rs.http, "get_text", return_value=None):
out = rs.fetch_comments(url)
assert out["top_comments"] == [] and out["num_comments"] is None
class TestEnrichmentBudget:
"""Busy topics enrich more threads and carry more comments per thread."""
def test_enrich_limits_by_depth(self):
assert rs.ENRICH_LIMITS == {"quick": 4, "default": 8, "deep": 12}
def test_parse_comments_returns_up_to_twelve(self):
html = "".join(
f'<shreddit-comment author="user{i}" thingId="t1_c{i}" score="{100 - i}" '
f'permalink="/r/test/comments/1/x/t1_c{i}/"></shreddit-comment>'
f'<div id="t1_c{i}-post-rtjson-content"><p>comment number {i} body text</p></div>'
for i in range(30)
)
out = rs.parse_comments(html)
assert len(out) == rs.MAX_COMMENTS == 12
assert [c["score"] for c in out] == list(range(100, 88, -1))