Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix(hackernews): attach top comments by calling enrich_top_stories (#1170)

Mohammad Hijjawi17c2e0a24b60
diff --git a/changelog.d/1168.fixed.md b/changelog.d/1168.fixed.md
new file mode 100644
index 00000000..6b2427a6
--- /dev/null
+++ b/changelog.d/1168.fixed.md
@@ -0,0 +1 @@
+Hacker News stories now get their top comments attached again: the pipeline calls `hackernews.enrich_top_stories` after parsing search results, so HN items no longer reach normalization with empty `top_comments`.
diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.py
index 5129719e..b78b96f7 100644
--- a/skills/last30days/scripts/lib/hackernews.py
+++ b/skills/last30days/scripts/lib/hackernews.py
@@ -378,8 +378,12 @@ def enrich_top_stories(
_log(f"Enriching top {len(to_enrich)} stories with comments")
with ThreadPoolExecutor(max_workers=5) as executor:
+ # submit_with_context, not executor.submit: a plain submit drops the
+ # pipeline's capture_failures() sink, so failed or rate-limited item
+ # fetches would vanish and the HN source would look fully enriched.
futures = {
- executor.submit(
+ http.submit_with_context(
+ executor,
_fetch_item_comments,
items[idx]["id"],
): idx
diff --git a/skills/last30days/scripts/lib/pipeline.py b/skills/last30days/scripts/lib/pipeline.py
index 6e6b6d56..f341c349 100644
--- a/skills/last30days/scripts/lib/pipeline.py
+++ b/skills/last30days/scripts/lib/pipeline.py
@@ -5579,10 +5579,11 @@ def _retrieve_stream_impl(
return items, _result_outcome_artifact(source, result)
if source == "hackernews":
result = hackernews.search_hackernews(subquery.search_query, from_date, to_date, depth=depth)
- return (
- hackernews.parse_hackernews_response(result, query=subquery.search_query),
- _result_outcome_artifact(source, result),
- )
+ items = hackernews.parse_hackernews_response(result, query=subquery.search_query)
+ # Attach top-level comments to the highest-point stories (bounded by
+ # hackernews.ENRICH_LIMITS per depth).
+ items = hackernews.enrich_top_stories(items, depth=depth)
+ return items, _result_outcome_artifact(source, result)
if source == "stocktwits":
# Pass raw_topic so symbol detection sees the full topic, not the
# narrowed per-subquery search_query (same rationale as reddit).
diff --git a/tests/eval/fixtures/breaking-event/http.json b/tests/eval/fixtures/breaking-event/http.json
index 1e749cd2..e5b077ad 100644
--- a/tests/eval/fixtures/breaking-event/http.json
+++ b/tests/eval/fixtures/breaking-event/http.json
@@ -78,6 +78,42 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/breaking-event-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/breaking-event-hn-2",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/breaking-event-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/eval/fixtures/comparison/http.json b/tests/eval/fixtures/comparison/http.json
index 12659240..12c45408 100644
--- a/tests/eval/fixtures/comparison/http.json
+++ b/tests/eval/fixtures/comparison/http.json
@@ -78,6 +78,42 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/comparison-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/comparison-hn-2",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/comparison-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/eval/fixtures/emerging-event/http.json b/tests/eval/fixtures/emerging-event/http.json
index fee5e300..df49bbdb 100644
--- a/tests/eval/fixtures/emerging-event/http.json
+++ b/tests/eval/fixtures/emerging-event/http.json
@@ -78,6 +78,42 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/emerging-event-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/emerging-event-hn-2",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/emerging-event-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/eval/fixtures/niche/http.json b/tests/eval/fixtures/niche/http.json
index fde62f72..d96976ff 100644
--- a/tests/eval/fixtures/niche/http.json
+++ b/tests/eval/fixtures/niche/http.json
@@ -78,6 +78,42 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/niche-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/niche-hn-2",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/niche-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/eval/fixtures/non-english-cjk/http.json b/tests/eval/fixtures/non-english-cjk/http.json
index f9b3c754..6435376c 100644
--- a/tests/eval/fixtures/non-english-cjk/http.json
+++ b/tests/eval/fixtures/non-english-cjk/http.json
@@ -78,6 +78,30 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/non-english-cjk-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/non-english-cjk-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/eval/fixtures/person/http.json b/tests/eval/fixtures/person/http.json
index 43def758..e0f4a52f 100644
--- a/tests/eval/fixtures/person/http.json
+++ b/tests/eval/fixtures/person/http.json
@@ -78,6 +78,42 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/person-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/person-hn-2",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/person-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/eval/fixtures/tech-product/http.json b/tests/eval/fixtures/tech-product/http.json
index 29af6322..02845a96 100644
--- a/tests/eval/fixtures/tech-product/http.json
+++ b/tests/eval/fixtures/tech-product/http.json
@@ -78,6 +78,42 @@
"nbHits": 3
}
}
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/tech-product-hn-1",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/tech-product-hn-2",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
+ },
+ {
+ "request": {
+ "method": "GET",
+ "url": "https://hn.algolia.com/api/v1/items/tech-product-hn-3",
+ "raw": false
+ },
+ "response": {
+ "value": {
+ "children": []
+ }
+ }
}
]
}
diff --git a/tests/test_hackernews.py b/tests/test_hackernews.py
index e5c6678a..f7df333e 100644
--- a/tests/test_hackernews.py
+++ b/tests/test_hackernews.py
@@ -504,5 +504,114 @@ def test_fetch_item_comments_preserves_absent_and_zero_points(mock_request, poin
assert result["comments"][0]["points"] is points
+# === Pipeline wiring ===
+
+
+@patch("lib.hackernews.http.request")
+def test_pipeline_hackernews_stream_attaches_top_comments(mock_request):
+ """The HN stream must enrich top stories with comments (#1168)."""
+ from lib import pipeline, schema
+
+ def fake_request(method, url, **kwargs):
+ if url.startswith(hackernews.ALGOLIA_ITEM_URL):
+ return {
+ "children": [
+ {"author": "alice", "text": "<p>Great launch.</p>", "points": 7},
+ ]
+ }
+ return {
+ "hits": [
+ create_mock_hit(object_id="1", title="Widget launch", points=300),
+ create_mock_hit(object_id="2", title="Widget review", points=50),
+ ]
+ }
+
+ mock_request.side_effect = fake_request
+ subquery = schema.SubQuery(
+ label="primary",
+ search_query="widget",
+ ranking_query="widget",
+ sources=["hackernews"],
+ )
+
+ items, _ = pipeline._retrieve_stream_impl(
+ topic="widget",
+ subquery=subquery,
+ source="hackernews",
+ config={},
+ depth="quick",
+ date_range=("2026-08-24", "2026-09-23"),
+ runtime=schema.ProviderRuntime(
+ reasoning_provider="none",
+ planner_model="none",
+ rerank_model="none",
+ ),
+ mock=False,
+ )
+
+ by_id = {item["id"]: item for item in items}
+ assert by_id["1"]["top_comments"] == [
+ {"author": "alice", "text": "Great launch.", "points": 7}
+ ]
+ assert by_id["1"]["comment_insights"] == ["Great launch."]
+ item_urls = [
+ call.args[1] for call in mock_request.call_args_list
+ if call.args[1].startswith(hackernews.ALGOLIA_ITEM_URL)
+ ]
+ assert sorted(item_urls) == [
+ f"{hackernews.ALGOLIA_ITEM_URL}/1",
+ f"{hackernews.ALGOLIA_ITEM_URL}/2",
+ ]
+
+
+@patch("lib.http.time.sleep")
+@patch("lib.http.urllib.request.urlopen")
+@patch("lib.hackernews.search_hackernews")
+def test_pipeline_hackernews_reports_failed_comment_enrichment(
+ mock_search, mock_urlopen, _mock_sleep
+):
+ """Rate-limited item fetches keep the stories but surface in the outcome."""
+ import urllib.error
+
+ from lib import pipeline, schema
+
+ mock_search.return_value = {
+ "hits": [
+ create_mock_hit(object_id="1", title="Widget launch", points=300),
+ create_mock_hit(object_id="2", title="Widget review", points=50),
+ ]
+ }
+ mock_urlopen.side_effect = urllib.error.HTTPError(
+ hackernews.ALGOLIA_ITEM_URL, 429, "Too Many Requests", {}, None
+ )
+ subquery = schema.SubQuery(
+ label="primary",
+ search_query="widget",
+ ranking_query="widget",
+ sources=["hackernews"],
+ )
+
+ items, artifact = pipeline._retrieve_stream(
+ topic="widget",
+ subquery=subquery,
+ source="hackernews",
+ config={},
+ depth="quick",
+ date_range=("2026-08-24", "2026-09-23"),
+ runtime=schema.ProviderRuntime(
+ reasoning_provider="none",
+ planner_model="none",
+ rerank_model="none",
+ ),
+ mock=False,
+ )
+
+ assert sorted(item["id"] for item in items) == ["1", "2"]
+ assert all(item["top_comments"] == [] for item in items)
+ assert "_source_outcome" not in artifact
+ assert "2 sub-requests rate-limited (HTTP 429)" in artifact["_source_outcome_detail"]
+ assert artifact["_source_outcome_detail_state"] == schema.RATE_LIMITED
+
+
if __name__ == "__main__":
pytest.main([__file__, "-v"])