AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(hackernews): attach top comments by calling enrich_top_stories (#1170)
Mohammad Hijjawi17c2e0a24b60
diff --git a/changelog.d/1168.fixed.md b/changelog.d/1168.fixed.mdnew file mode 100644index 00000000..6b2427a6--- /dev/null+++ b/changelog.d/1168.fixed.md@@ -0,0 +1 @@+Hacker News stories now get their top comments attached again: the pipeline calls `hackernews.enrich_top_stories` after parsing search results, so HN items no longer reach normalization with empty `top_comments`.diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.pyindex 5129719e..b78b96f7 100644--- a/skills/last30days/scripts/lib/hackernews.py+++ b/skills/last30days/scripts/lib/hackernews.py@@ -378,8 +378,12 @@ def enrich_top_stories(_log(f"Enriching top {len(to_enrich)} stories with comments")with ThreadPoolExecutor(max_workers=5) as executor:+ # submit_with_context, not executor.submit: a plain submit drops the+ # pipeline's capture_failures() sink, so failed or rate-limited item+ # fetches would vanish and the HN source would look fully enriched.futures = {- executor.submit(+ http.submit_with_context(+ executor,_fetch_item_comments,items[idx]["id"],): idxdiff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.pyindex 6e6b6d56..f341c349 100644--- a/skills/last30days/scripts/lib/pipeline.py+++ b/skills/last30days/scripts/lib/pipeline.py@@ -5579,10 +5579,11 @@ def _retrieve_stream_impl(return items, _result_outcome_artifact(source, result)if source == "hackernews":result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)- return (- hackernews.parse_hackernews_response(result, query=subquery.search_query),- _result_outcome_artifact(source, result),- )+ items = hackernews.parse_hackernews_response(result, query=subquery.search_query)+ # Attach top-level comments to the highest-point stories (bounded by+ # hackernews.ENRICH_LIMITS per depth).+ items = hackernews.enrich_top_stories(items, depth=depth)+ return items, _result_outcome_artifact(source, result)if source == "stocktwits":# Pass raw_topic so symbol detection sees the full topic, not the# narrowed per-subquery search_query (same rationale as reddit).diff --git a/tests/eval/fixtures/breaking-event/http.json b/tests/eval/fixtures/breaking-event/http.jsonindex 1e749cd2..e5b077ad 100644--- a/tests/eval/fixtures/breaking-event/http.json+++ b/tests/eval/fixtures/breaking-event/http.json@@ -78,6 +78,42 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/breaking-event-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/breaking-event-hn-2",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/breaking-event-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/eval/fixtures/comparison/http.json b/tests/eval/fixtures/comparison/http.jsonindex 12659240..12c45408 100644--- a/tests/eval/fixtures/comparison/http.json+++ b/tests/eval/fixtures/comparison/http.json@@ -78,6 +78,42 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/comparison-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/comparison-hn-2",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/comparison-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/eval/fixtures/emerging-event/http.json b/tests/eval/fixtures/emerging-event/http.jsonindex fee5e300..df49bbdb 100644--- a/tests/eval/fixtures/emerging-event/http.json+++ b/tests/eval/fixtures/emerging-event/http.json@@ -78,6 +78,42 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/emerging-event-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/emerging-event-hn-2",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/emerging-event-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/eval/fixtures/niche/http.json b/tests/eval/fixtures/niche/http.jsonindex fde62f72..d96976ff 100644--- a/tests/eval/fixtures/niche/http.json+++ b/tests/eval/fixtures/niche/http.json@@ -78,6 +78,42 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/niche-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/niche-hn-2",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/niche-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/eval/fixtures/non-english-cjk/http.json b/tests/eval/fixtures/non-english-cjk/http.jsonindex f9b3c754..6435376c 100644--- a/tests/eval/fixtures/non-english-cjk/http.json+++ b/tests/eval/fixtures/non-english-cjk/http.json@@ -78,6 +78,30 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/non-english-cjk-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/non-english-cjk-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/eval/fixtures/person/http.json b/tests/eval/fixtures/person/http.jsonindex 43def758..e0f4a52f 100644--- a/tests/eval/fixtures/person/http.json+++ b/tests/eval/fixtures/person/http.json@@ -78,6 +78,42 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/person-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/person-hn-2",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/person-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/eval/fixtures/tech-product/http.json b/tests/eval/fixtures/tech-product/http.jsonindex 29af6322..02845a96 100644--- a/tests/eval/fixtures/tech-product/http.json+++ b/tests/eval/fixtures/tech-product/http.json@@ -78,6 +78,42 @@"nbHits": 3}}+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/tech-product-hn-1",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/tech-product-hn-2",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }+ },+ {+ "request": {+ "method": "GET",+ "url": "https://hn.algolia.com/api/v1/items/tech-product-hn-3",+ "raw": false+ },+ "response": {+ "value": {+ "children": []+ }+ }}]}diff --git a/tests/test_hackernews.py b/tests/test_hackernews.pyindex e5c6678a..f7df333e 100644--- a/tests/test_hackernews.py+++ b/tests/test_hackernews.py@@ -504,5 +504,114 @@ def test_fetch_item_comments_preserves_absent_and_zero_points(mock_request, poinassert result["comments"][0]["points"] is points+# === Pipeline wiring ===+++@patch("lib.hackernews.http.request")+def test_pipeline_hackernews_stream_attaches_top_comments(mock_request):+ """The HN stream must enrich top stories with comments (#1168)."""+ from lib import pipeline, schema++ def fake_request(method, url, **kwargs):+ if url.startswith(hackernews.ALGOLIA_ITEM_URL):+ return {+ "children": [+ {"author": "alice", "text": "<p>Great launch.</p>", "points": 7},+ ]+ }+ return {+ "hits": [+ create_mock_hit(object_id="1", title="Widget launch", points=300),+ create_mock_hit(object_id="2", title="Widget review", points=50),+ ]+ }++ mock_request.side_effect = fake_request+ subquery = schema.SubQuery(+ label="primary",+ search_query="widget",+ ranking_query="widget",+ sources=["hackernews"],+ )++ items, _ = pipeline._retrieve_stream_impl(+ topic="widget",+ subquery=subquery,+ source="hackernews",+ config={},+ depth="quick",+ date_range=("2026-08-24", "2026-09-23"),+ runtime=schema.ProviderRuntime(+ reasoning_provider="none",+ planner_model="none",+ rerank_model="none",+ ),+ mock=False,+ )++ by_id = {item["id"]: item for item in items}+ assert by_id["1"]["top_comments"] == [+ {"author": "alice", "text": "Great launch.", "points": 7}+ ]+ assert by_id["1"]["comment_insights"] == ["Great launch."]+ item_urls = [+ call.args[1] for call in mock_request.call_args_list+ if call.args[1].startswith(hackernews.ALGOLIA_ITEM_URL)+ ]+ assert sorted(item_urls) == [+ f"{hackernews.ALGOLIA_ITEM_URL}/1",+ f"{hackernews.ALGOLIA_ITEM_URL}/2",+ ]+++@patch("lib.http.time.sleep")+@patch("lib.http.urllib.request.urlopen")+@patch("lib.hackernews.search_hackernews")+def test_pipeline_hackernews_reports_failed_comment_enrichment(+ mock_search, mock_urlopen, _mock_sleep+):+ """Rate-limited item fetches keep the stories but surface in the outcome."""+ import urllib.error++ from lib import pipeline, schema++ mock_search.return_value = {+ "hits": [+ create_mock_hit(object_id="1", title="Widget launch", points=300),+ create_mock_hit(object_id="2", title="Widget review", points=50),+ ]+ }+ mock_urlopen.side_effect = urllib.error.HTTPError(+ hackernews.ALGOLIA_ITEM_URL, 429, "Too Many Requests", {}, None+ )+ subquery = schema.SubQuery(+ label="primary",+ search_query="widget",+ ranking_query="widget",+ sources=["hackernews"],+ )++ items, artifact = pipeline._retrieve_stream(+ topic="widget",+ subquery=subquery,+ source="hackernews",+ config={},+ depth="quick",+ date_range=("2026-08-24", "2026-09-23"),+ runtime=schema.ProviderRuntime(+ reasoning_provider="none",+ planner_model="none",+ rerank_model="none",+ ),+ mock=False,+ )++ assert sorted(item["id"] for item in items) == ["1", "2"]+ assert all(item["top_comments"] == [] for item in items)+ assert "_source_outcome" not in artifact+ assert "2 sub-requests rate-limited (HTTP 429)" in artifact["_source_outcome_detail"]+ assert artifact["_source_outcome_detail_state"] == schema.RATE_LIMITED++if __name__ == "__main__":pytest.main([__file__, "-v"])