AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
fix(hackernews): stop dropping every HN comment on an unmeetable score floor (#890)
Terry Li53fb6104c97f
diff --git a/changelog.d/889.fixed.md b/changelog.d/889.fixed.mdnew file mode 100644index 00000000..d314c8ec--- /dev/null+++ b/changelog.d/889.fixed.md@@ -0,0 +1 @@+Hacker News comments no longer vanish from every per-source path. HN comments arrive as `{author, text, points}` while downstream readers key on `score`/`excerpt`, and `_normalize_hackernews` stored them raw, so `render._top_comments_list` filtered `(c.get("score") or 0) >= 5` against a key that was never present and rejected the entire source. HN comments are now remapped like the YouTube and TikTok ones, and the HN floor is 0 because the Algolia API returns `points: null` for every comment child, which makes any positive threshold unmeetable. A comment with no vote signal renders without a fabricated "(0 points)".diff --git a/skills/last30days/scripts/lib/hackernews.py b/skills/last30days/scripts/lib/hackernews.pyindex 53bb7b31..5129719e 100644--- a/skills/last30days/scripts/lib/hackernews.py+++ b/skills/last30days/scripts/lib/hackernews.py@@ -339,7 +339,7 @@ def _fetch_item_comments(object_id: str, max_comments: int = 5) -> Dict[str, Anycomments.append({"author": c.get("author", ""),"text": excerpt,- "points": c.get("points") or 0,+ "points": c.get("points"),})# First sentence as insightfirst_sentence = text.split(". ")[0].split("\n")[0][:200]diff --git a/skills/last30days/scripts/lib/normalize.py b/skills/last30days/scripts/lib/normalize.pyindex 62e64a8f..cc3eacad 100644--- a/skills/last30days/scripts/lib/normalize.py+++ b/skills/last30days/scripts/lib/normalize.py@@ -40,14 +40,24 @@ def normalize_source_items("reddit": _normalize_reddit,"x": _normalize_x,"youtube": _normalize_youtube,- "tiktok": lambda s, i, idx, fd, td: _normalize_shortform_video(s, i, idx, fd, td, "TK", "TikTok post"),- "instagram": lambda s, i, idx, fd, td: _normalize_shortform_video(s, i, idx, fd, td, "IG", "Instagram reel"),+ "tiktok": lambda s, i, idx, fd, td: _normalize_shortform_video(+ s, i, idx, fd, td, "TK", "TikTok post"+ ),+ "instagram": lambda s, i, idx, fd, td: _normalize_shortform_video(+ s, i, idx, fd, td, "IG", "Instagram reel"+ ),"hackernews": _normalize_hackernews,"stocktwits": _normalize_stocktwits,"dripstack": _normalize_dripstack,- "bluesky": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "BS", "Bluesky post"),- "truthsocial": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TS", "Truth Social post"),- "threads": lambda s, i, idx, fd, td: _normalize_microblog(s, i, idx, fd, td, "TH", "Threads post"),+ "bluesky": lambda s, i, idx, fd, td: _normalize_microblog(+ s, i, idx, fd, td, "BS", "Bluesky post"+ ),+ "truthsocial": lambda s, i, idx, fd, td: _normalize_microblog(+ s, i, idx, fd, td, "TS", "Truth Social post"+ ),+ "threads": lambda s, i, idx, fd, td: _normalize_microblog(+ s, i, idx, fd, td, "TH", "Threads post"+ ),"xquik": _normalize_x,"pinterest": _normalize_pinterest,"polymarket": _normalize_polymarket,@@ -65,7 +75,10 @@ def normalize_source_items(normalizer = normalizers.get(source)if normalizer is None:raise ValueError(f"Unsupported source: {source}")- normalized = [normalizer(source, item, index, from_date, to_date) for index, item in enumerate(items)]+ normalized = [+ normalizer(source, item, index, from_date, to_date)+ for index, item in enumerate(items)+ ]if source == "jobs":# A careers board is a snapshot of CURRENTLY OPEN roles. An open posting# is current evidence regardless of when it was posted, so date-windowing@@ -74,7 +87,9 @@ def normalize_source_items(# Keep the full board; recency is annotated, not used to drop.return normalizedrequire_date = source == "grounding"- filtered = filter_by_date_range(normalized, from_date, to_date, require_date=require_date)+ filtered = filter_by_date_range(+ normalized, from_date, to_date, require_date=require_date+ )if filtered:return filteredif freshness_mode == "evergreen_ok" and source == "youtube":@@ -88,6 +103,8 @@ def _remap_comments(raw: list[Any],score_keys: tuple[str, ...],excerpt_keys: tuple[str, ...],+ *,+ preserve_absent_score: bool = False,) -> list[dict[str, Any]]:"""Normalize comments from any source into the shared Reddit-compatible shape.@@ -95,19 +112,29 @@ def _remap_comments(entity_extract, rerank) all expect `score` and `excerpt`. This helper mapsper-source field names (YT: likes/text, TikTok: digg_count/text) onto thatshape while preserving author/date/url passthrough.++ Sources that distinguish an absent vote from a measured zero can opt into+ preserving the absent value as ``None``."""out: list[dict[str, Any]] = []for raw_c in raw:if not isinstance(raw_c, dict):continue- score = _first_present(raw_c, score_keys, default=0)+ score = _first_present(+ raw_c,+ score_keys,+ default=None if preserve_absent_score else 0,+ )excerpt = _first_present(raw_c, excerpt_keys, default="")- try:- score_int = int(score or 0)- except (TypeError, ValueError):- score_int = 0+ if score is None and preserve_absent_score:+ normalized_score = None+ else:+ try:+ normalized_score = int(score or 0)+ except (TypeError, ValueError):+ normalized_score = 0entry: dict[str, Any] = {- "score": score_int,+ "score": normalized_score,"excerpt": str(excerpt or "")[:400],"author": str(raw_c.get("author") or ""),"date": str(raw_c.get("date") or ""),@@ -145,7 +172,9 @@ def _domain_from_url(url: str) -> str | None:return domain or None-def _date_confidence(item: dict[str, Any], from_date: str, to_date: str, default: str = "low") -> str:+def _date_confidence(+ item: dict[str, Any], from_date: str, to_date: str, default: str = "low"+) -> str:if item.get("date_confidence"):return str(item["date_confidence"])date_value = item.get("date")@@ -211,7 +240,7 @@ def _normalize_stocktwits(relevance_hint=item.get("relevance", 0.7),why_relevant=str(item.get("why_relevant") or ""),snippet=str(item.get("snippet") or "")[:400],- metadata=meta, # carries sentiment + symbol-level bull/bear aggregate+ metadata=meta, # carries sentiment + symbol-level bull/bear aggregate)@@ -234,7 +263,9 @@ def _normalize_dripstack(item_id=str(item.get("id") or f"DS{index + 1}"),source=source,title=str(item.get("title") or ""),- body=str(item.get("body") or "") or str(item.get("snippet") or "") or str(item.get("title") or ""),+ body=str(item.get("body") or "")+ or str(item.get("snippet") or "")+ or str(item.get("title") or ""),url=str(item.get("url") or ""),author=str(item.get("author") or "") or None,container=str(meta.get("publication_slug") or "") or None,@@ -326,7 +357,9 @@ def _normalize_jobs(title = str(item.get("title") or "").strip()department = str(item.get("department") or "").strip()location = str(item.get("location") or "").strip()- body = "\n".join(part for part in [title, department, location, description] if part)+ body = "\n".join(+ part for part in [title, department, location, description] if part+ )provider = str(item.get("provider") or "").strip()return _source_item(item_id=str(item.get("id") or f"J{index + 1}"),@@ -345,12 +378,15 @@ def _normalize_jobs(metadata={"provider": provider,"department": department,- "departments": item.get("departments") or ([department] if department else []),+ "departments": item.get("departments")+ or ([department] if department else []),"location": location,"offices": item.get("offices") or [],"board_token": item.get("board_token") or "","source_url": item.get("source_url") or "",- "source_domain": item.get("source_domain") or _domain_from_url(str(item.get("url") or "")) or "",+ "source_domain": item.get("source_domain")+ or _domain_from_url(str(item.get("url") or ""))+ or "",},)@@ -471,10 +507,23 @@ def _normalize_hackernews(from_date: str,to_date: str,) -> schema.SourceItem:- top_comments = item.get("top_comments") or []- comment_text = _join_comment_excerpts(top_comments, "text")+ # HN comments arrive as {author, text, points}; downstream code keys on+ # score/excerpt, so remap here exactly as the YouTube and TikTok normalisers+ # do. Without this the per-source floor in render._top_comments_list reads a+ # `score` that is never present and rejects every HN comment.+ top_comments = _remap_comments(+ item.get("top_comments") or [],+ score_keys=("points", "score"),+ excerpt_keys=("text", "excerpt"),+ preserve_absent_score=True,+ )+ comment_text = _join_comment_excerpts(top_comments, "excerpt")title = str(item.get("title") or "").strip()- body = "\n".join(part for part in [title, str(item.get("text") or "").strip(), comment_text] if part)+ body = "\n".join(+ part+ for part in [title, str(item.get("text") or "").strip(), comment_text]+ if part+ )return _source_item(item_id=str(item.get("id") or f"HN{index + 1}"),source=source,@@ -666,7 +715,8 @@ def _normalize_trustpilot(return _source_item(item_id=str(item.get("id") or f"TP{index + 1}"),source=source,- title=title or (f"{name} on Trustpilot" if name else f"Trustpilot reviews {index + 1}"),+ title=title+ or (f"{name} on Trustpilot" if name else f"Trustpilot reviews {index + 1}"),body=body,url=str(item.get("url") or ""),author=name or None,@@ -703,7 +753,11 @@ def _normalize_polymarket(item_id=str(item.get("event_id") or item.get("id") or f"PM{index + 1}"),source=source,title=title or question or f"Polymarket event {index + 1}",- body="\n".join(part for part in [title, question, str(item.get("price_movement") or "")] if part),+ body="\n".join(+ part+ for part in [title, question, str(item.get("price_movement") or "")]+ if part+ ),url=str(item.get("url") or ""),author=None,container="Polymarket",@@ -723,7 +777,6 @@ def _normalize_polymarket()-def _normalize_github(source: str,item: dict[str, Any],@@ -759,6 +812,7 @@ def _normalize_github(},)+def _normalize_grounding(source: str,item: dict[str, Any],diff --git a/skills/last30days/scripts/lib/render.py b/skills/last30days/scripts/lib/render.pyindex 1dcf41db..39e78b56 100644--- a/skills/last30days/scripts/lib/render.py+++ b/skills/last30days/scripts/lib/render.py@@ -93,16 +93,16 @@ def _format_discovery_engagement(if value]if metric_parts:- parts.append(f"{SOURCE_LABELS.get(source, source.title())}: {', '.join(metric_parts)}")+ parts.append(+ f"{SOURCE_LABELS.get(source, source.title())}: {', '.join(metric_parts)}"+ )return " · ".join(parts) or "No native engagement counters reported"def render_discovery(report: schema.DiscoveryReport) -> str:"""Render a compact topic-per-section discovery brief."""title = (- f"# Trending discovery: {report.domain}"- if report.domain- else "# Trending now"+ f"# Trending discovery: {report.domain}" if report.domain else "# Trending now")lines = [*_render_badge(),@@ -112,55 +112,70 @@ def render_discovery(report: schema.DiscoveryReport) -> str:f"Feeds: {', '.join(report.plan.sources)}",]if report.plan.subreddits:- lines.append("Communities: " + ", ".join(f"r/{sub}" for sub in report.plan.subreddits))+ lines.append(+ "Communities: " + ", ".join(f"r/{sub}" for sub in report.plan.subreddits)+ )lines.append("")if not report.topics:if report.outcome == "nothing-solid":- lines.extend([- "**Nothing solid this window.** No topic cleared the confidence "- "floor - not enough cross-source confirmation or engagement to "- "call anything a trend, and ranked noise would be worse than an "- "honest empty result.",- "",- ])- if report.weak_signal:- lines.extend([- f"Closest weak signal: {report.weak_signal} (sub-floor; "- "single-source or too little engagement).",+ lines.extend(+ [+ "**Nothing solid this window.** No topic cleared the confidence "+ "floor - not enough cross-source confirmation or engagement to "+ "call anything a trend, and ranked noise would be worse than an "+ "honest empty result.","",- ])+ ]+ )+ if report.weak_signal:+ lines.extend(+ [+ f"Closest weak signal: {report.weak_signal} (sub-floor; "+ "single-source or too little engagement).",+ "",+ ]+ )else:lines.extend(["No trending topic clusters survived this sweep.", ""])for topic in report.topics:momentum = "New this week" if topic.momentum == "new-this-week" else "Building"confirmation = (f" · confirmed across {topic.corroboration_count} sources"- if topic.corroboration_count >= 2 else ""+ if topic.corroboration_count >= 2+ else "")- lines.extend([- f"## {topic.rank}. {topic.name}",- "",- f"**Momentum:** {momentum} · velocity {topic.velocity_score:,.2f}{confirmation}",- "",- topic.why_spiking,- "",- ])- if topic.top_comment:- lines.extend([- f"**Community voice:** {topic.top_comment}",+ lines.extend(+ [+ f"## {topic.rank}. {topic.name}","",- ])- if topic.podcast_angle:- lines.extend([- f"**Podcast angle:** {topic.podcast_angle}",+ f"**Momentum:** {momentum} · velocity {topic.velocity_score:,.2f}{confirmation}","",- ])- if topic.x_article_angle:- lines.extend([- f"**X article angle:** {topic.x_article_angle}",+ topic.why_spiking,"",- ])+ ]+ )+ if topic.top_comment:+ lines.extend(+ [+ f"**Community voice:** {topic.top_comment}",+ "",+ ]+ )+ if topic.podcast_angle:+ lines.extend(+ [+ f"**Podcast angle:** {topic.podcast_angle}",+ "",+ ]+ )+ if topic.x_article_angle:+ lines.extend(+ [+ f"**X article angle:** {topic.x_article_angle}",+ "",+ ]+ )pipeline_notes: list[str] = []if topic.previously_surfaced_count > 0:# previously_surfaced_count is PRIOR appearances, so this@@ -173,16 +188,20 @@ def render_discovery(report: schema.DiscoveryReport) -> str:# so no date is rendered here.pipeline_notes.append("marked covered")if pipeline_notes:- lines.extend([- f"**Pipeline:** {', '.join(pipeline_notes)}",+ lines.extend(+ [+ f"**Pipeline:** {', '.join(pipeline_notes)}",+ "",+ ]+ )+ lines.extend(+ [+ f"**Evidence:** {_format_discovery_engagement(topic.engagement_by_source)}",+ "",+ f"**Research next:** `{topic.command}`","",- ])- lines.extend([- f"**Evidence:** {_format_discovery_engagement(topic.engagement_by_source)}",- "",- f"**Research next:** `{topic.command}`",- "",- ])+ ]+ )if report.warnings:lines.extend(["### Coverage notes", ""])@@ -190,6 +209,7 @@ def render_discovery(report: schema.DiscoveryReport) -> str:lines.append("")return "\n".join(lines).rstrip() + "\n"+SOURCE_LABELS = {"reddit": "Reddit","youtube": "YouTube",@@ -321,7 +341,9 @@ def render_library_search(label = "Saved brief" if match.source_kind == "brief" else "Store sighting"engagement = ""if match.engagement is not None:- engagement = f"; {_format_library_engagement(match.engagement)} engagement"+ engagement = (+ f"; {_format_library_engagement(match.engagement)} engagement"+ )lines.append(f"- **{label}:** {match.headline}{engagement}")if match.snippet and match.snippet != match.headline:lines.append(f" {match.snippet}")@@ -374,13 +396,15 @@ def _render_ranked_clusters(}solid_clusters = _clusters_clearing_relevance_floor(report, clusters)if clusters and not solid_clusters:- lines.extend([- "**Nothing solid this window.**",- "",- "No recent evidence cluster cleared the relevance floor. "- "Do not infer findings or quote community comments from this run.",- "",- ])+ lines.extend(+ [+ "**Nothing solid this window.**",+ "",+ "No recent evidence cluster cleared the relevance floor. "+ "Do not infer findings or quote community comments from this run.",+ "",+ ]+ )for index, cluster in enumerate(solid_clusters, start=1):lines.append(f"### {index}. {cluster.title} "@@ -398,7 +422,9 @@ def _render_ranked_clusters(candidate = candidate_by_id.get(candidate_id)if not candidate:continue- lines.extend(_render_candidate(candidate, prefix=f"{rep_index}.", report=report))+ lines.extend(+ _render_candidate(candidate, prefix=f"{rep_index}.", report=report)+ )lines.append("")return lines@@ -428,7 +454,9 @@ def _clusters_clearing_relevance_floor(for candidate_id in cluster.candidate_idsif candidate_id in candidate_by_id]- if members and not any(_best_take_relevance_ok(candidate) for candidate in members):+ if members and not any(+ _best_take_relevance_ok(candidate) for candidate in members+ ):continuesolid.append(cluster)return solid@@ -440,9 +468,7 @@ def _candidates_in_clusters() -> list[schema.Candidate]:"""Return ranked candidates belonging to the supplied visible clusters."""candidate_ids = {- candidate_id- for cluster in clusters- for candidate_id in cluster.candidate_ids+ candidate_id for cluster in clusters for candidate_id in cluster.candidate_ids}return [candidate@@ -513,7 +539,9 @@ def _render_corpus_section(report: schema.Report, limit: int = 8) -> list[str]:if path:lines.append(f" - File: `{_defang_corpus_sentinels(path)}`")if candidate.snippet:- lines.append(f" - {_defang_corpus_sentinels(_truncate(candidate.snippet, 300))}")+ lines.append(+ f" - {_defang_corpus_sentinels(_truncate(candidate.snippet, 300))}"+ )lines.append(PRIVATE_CORPUS_END)return lines@@ -562,7 +590,9 @@ def _render_freshness_verdicts(report: schema.Report) -> list[str]:# The verifier's detail carries the formatted movement for stale# rows and the reason a claim could not be re-checked otherwise.claim += f" ({verdict.detail.replace('|', chr(92) + '|')})"- evidence_label = verdict.evidence_timestamp or verdict.source_timestamp or "source"+ evidence_label = (+ verdict.evidence_timestamp or verdict.source_timestamp or "source"+ )evidence = (f"[{evidence_label}]({verdict.evidence_url})"if verdict.evidence_url@@ -584,10 +614,12 @@ def _clusters_for_register(clusters = list(report.clusters)if audience.emphasis_weights:clusters.sort(- key=lambda cluster: -cluster.score- * max(- (audience.emphasis_for(source) for source in cluster.sources),- default=1.0,+ key=lambda cluster: (+ -cluster.score+ * max(+ (audience.emphasis_for(source) for source in cluster.sources),+ default=1.0,+ )))return clusters[: audience.budget_for("clusters", fallback_limit)]@@ -623,10 +655,16 @@ def _render_registered_sections(# The preset's source emphasis must reach the lead section's own# ranking: a creator register surfaces TikTok/IG/YouTube takes ahead# of equally-rated HN or GitHub ones.- source_weight=(audience.emphasis_for if audience.emphasis_weights else None),+ source_weight=(+ audience.emphasis_for if audience.emphasis_weights else None+ ),)if not best_takes:- best_takes = ["## Best Takes", "", "- No qualifying takes surfaced in this run."]+ best_takes = [+ "## Best Takes",+ "",+ "- No qualifying takes surfaced in this run.",+ ]top_comments = _render_top_comments(report,@@ -691,7 +729,9 @@ def render_compact("",*_assistant_safety_lines(),f"- Date range: {report.range_from} to {report.range_to}",- f"- Sources: {len(non_empty)} active ({', '.join(_source_label(s) for s in non_empty)})" if non_empty else "- Sources: none",+ f"- Sources: {len(non_empty)} active ({', '.join(_source_label(s) for s in non_empty)})"+ if non_empty+ else "- Sources: none","",]drill_context = _render_drill_context(report)@@ -703,11 +743,13 @@ def render_compact(freshness_warning = _assess_data_freshness(report)if freshness_warning:- lines.extend([- "## Freshness",- f"- {freshness_warning}",- "",- ])+ lines.extend(+ [+ "## Freshness",+ f"- {freshness_warning}",+ "",+ ]+ )if report.warnings:lines.append("## Warnings")@@ -730,7 +772,9 @@ def render_compact(# verbatim as user output. The envelope comments give the model an# unambiguous scope for "pass through verbatim" (the PASS-THROUGH FOOTER# block below) vs "synthesize from" (this block).- lines.append("<!-- EVIDENCE FOR SYNTHESIS: read this, do not emit verbatim. Transform into `What I learned:` prose per LAW 2. -->")+ lines.append(+ "<!-- EVIDENCE FOR SYNTHESIS: read this, do not emit verbatim. Transform into `What I learned:` prose per LAW 2. -->"+ )lines.append("")# Echo the synthesis contract early so it survives tail truncation (#726).lines.extend(_render_synthesis_directive())@@ -786,7 +830,11 @@ def render_compact(lines.extend(_render_source_coverage(report))else:- lines.extend(_render_registered_sections(evidence_report, audience, fun_params, cluster_limit))+ lines.extend(+ _render_registered_sections(+ evidence_report, audience, fun_params, cluster_limit+ )+ )corpus_section = _render_corpus_section(report)if corpus_section:lines.extend(["", *corpus_section])@@ -812,7 +860,9 @@ def render_compact(footer = _render_emoji_footer(report, save_path)if footer:lines.append("")- lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")+ lines.append(+ "<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->"+ )lines.extend(footer)lines.append("<!-- END PASS-THROUGH FOOTER -->")@@ -872,16 +922,18 @@ def render_for_html(lines.extend(["", *hiring_block])if not synthesis_md and audience.name not in {"default", "eli5"}:fun_params = _FUN_LEVELS.get(fun_level, _FUN_LEVELS["medium"])- lines.extend([- "",- *_render_registered_sections(- evidence_report,- audience,- fun_params,- 8,- include_source_diagnostics=False,- ),- ])+ lines.extend(+ [+ "",+ *_render_registered_sections(+ evidence_report,+ audience,+ fun_params,+ 8,+ include_source_diagnostics=False,+ ),+ ]+ )corpus_section = _render_corpus_section(report)if corpus_section:lines.extend(["", *corpus_section])@@ -944,9 +996,13 @@ def collect_html_warnings(report: schema.Report) -> list[str]:"""notes: list[str] = []if _render_degraded_run_warning(report):- notes.append("Run was missing pre-flight resolution. Re-run with `--plan` for richer results.")+ notes.append(+ "Run was missing pre-flight resolution. Re-run with `--plan` for richer results."+ )elif _render_pre_research_warning(report):- notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")+ notes.append(+ "Pre-research was skipped, so results may be thinner than a resolved run."+ )freshness_warning = _assess_data_freshness(report)if freshness_warning:notes.append(freshness_warning)@@ -987,10 +1043,14 @@ def _render_html_data_quality_note(report: schema.Report) -> str | None:notes: list[str] = []degraded_warning = _render_degraded_run_warning(report)if degraded_warning:- notes.append("This run was missing pre-flight resolution. Re-run with `--plan` for richer results.")+ notes.append(+ "This run was missing pre-flight resolution. Re-run with `--plan` for richer results."+ )pre_research_warning = _render_pre_research_warning(report)if pre_research_warning and not degraded_warning:- notes.append("Pre-research was skipped, so results may be thinner than a resolved run.")+ notes.append(+ "Pre-research was skipped, so results may be thinner than a resolved run."+ )freshness_warning = _assess_data_freshness(report)if freshness_warning:notes.append(freshness_warning)@@ -1026,10 +1086,14 @@ def _dedupe_notes(notes: list[str]) -> list[str]:return out-def _append_html_footer(lines: list[str], report: schema.Report, save_path: str | None) -> None:+def _append_html_footer(+ lines: list[str], report: schema.Report, save_path: str | None+) -> None:footer = _render_emoji_footer(report, save_path)lines.append("")- lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")+ lines.append(+ "<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->"+ )lines.extend(footer)lines.append("<!-- END PASS-THROUGH FOOTER -->")@@ -1164,7 +1228,7 @@ def _render_pre_research_warning(report: schema.Report) -> list[str]:f"rerun `/last30days {report.topic}`. The skill will resolve handles","and communities before calling the engine this time, producing richer results.","",- "If this topic really is abstract (e.g. \"AI regulation\") and doesn't need",+ 'If this topic really is abstract (e.g. "AI regulation") and doesn\'t need',"handle resolution, add `--auto-resolve` to the engine command or ignore this","warning - the current results are the keyword-search fallback.",]@@ -1202,7 +1266,7 @@ def _render_degraded_run_warning(report: schema.Report) -> list[str]:"<!-- USER-VISIBLE BANNER: emit verbatim before synthesis per LAW 5 / LAW 7. -->","## DEGRADED RUN WARNING","",- f"⚠️ This run was called BARE on a named-entity topic (\"{report.topic}\"):",+ f'⚠️ This run was called BARE on a named-entity topic ("{report.topic}"):',"no `--plan` JSON, no pre-flight handle resolution. The engine ran","deterministic fallback retrieval, which is the cron/headless path -","weaker than the path where the hosting reasoning model authors its own",@@ -1238,6 +1302,7 @@ def _parse_comparison_entities(topic: str) -> list[str] | None:if not topic:return Nonefrom . import planner+entities = planner._comparison_entities(topic)return entities if len(entities) >= 2 else None@@ -1279,7 +1344,7 @@ def _render_comparison_scaffold(topic: str) -> list[str]:fill_instructions = ("Fill each cell based on the research above. Keep cells short (5-15 words). ""Use ' - ' (hyphen with spaces) not em-dashes. Write N/A for axes that do not apply to this topic class. "- "Ground the \"What it is\" row in first-party positioning fetched during this run's research when "+ 'Ground the "What it is" row in first-party positioning fetched during this run\'s research when '"available - describe each entity as it pitches itself today, never from memory. ""This scaffold matches the April 9 launch-video exemplar shape.")@@ -1359,12 +1424,14 @@ def render_comparison_multi(fun_params = _FUN_LEVELS.get(fun_level, _FUN_LEVELS["medium"])for label, report in entity_reports:- lines.extend(_render_entity_evidence_block(- label=label,- report=report,- cluster_limit=cluster_limit,- fun_params=fun_params,- ))+ lines.extend(+ _render_entity_evidence_block(+ label=label,+ report=report,+ cluster_limit=cluster_limit,+ fun_params=fun_params,+ )+ )lines.append("<!-- END EVIDENCE FOR SYNTHESIS -->")lines.append("")@@ -1383,7 +1450,9 @@ def render_comparison_multi(footer = _render_emoji_footer(main_report, save_path)if footer:lines.append("")- lines.append("<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->")+ lines.append(+ "<!-- PASS-THROUGH FOOTER: emit verbatim in the model response per LAW 5. -->"+ )lines.extend(footer)lines.append("<!-- END PASS-THROUGH FOOTER -->")@@ -1420,15 +1489,20 @@ def _render_resolved_entities_block(x_display = f"@{x_handle}" if x_handle else "-"subs_display = (- ", ".join(f"r/{s}" for s in subs[:5]) + (- f" (+{len(subs) - 5})" if len(subs) > 5 else ""+ (+ ", ".join(f"r/{s}" for s in subs[:5])+ + (f" (+{len(subs) - 5})" if len(subs) > 5 else ""))- ) if subs else "-"+ if subs+ else "-"+ )gh_display = f"@{gh_user}" if gh_user else "-"if gh_repos:- gh_display += f" ({', '.join(gh_repos[:3])}" + (- f" +{len(gh_repos) - 3}" if len(gh_repos) > 3 else ""- ) + ")"+ gh_display += (+ f" ({', '.join(gh_repos[:3])}"+ + (f" +{len(gh_repos) - 3}" if len(gh_repos) > 3 else "")+ + ")"+ )context_display = _truncate(context, 120) if context else "-"out.append(@@ -1467,12 +1541,14 @@ def _render_entity_evidence_block(out.append("### Ranked Evidence Clusters")out.append("")if requested_clusters and not visible_clusters:- out.extend([- "**Nothing solid this window.**",- "",- "No recent evidence cluster cleared the relevance floor.",- "",- ])+ out.extend(+ [+ "**Nothing solid this window.**",+ "",+ "No recent evidence cluster cleared the relevance floor.",+ "",+ ]+ )for index, cluster in enumerate(visible_clusters, start=1):out.append(f"#### {index}. {cluster.title} "@@ -1490,7 +1566,11 @@ def _render_entity_evidence_block(candidate = candidate_by_id.get(candidate_id)if not candidate:continue- out.extend(_render_candidate(candidate, prefix=f"{rep_index}.", report=evidence_report))+ out.extend(+ _render_candidate(+ candidate, prefix=f"{rep_index}.", report=evidence_report+ )+ )out.append("")comparison_candidates = _candidates_for_auxiliary_sections(@@ -1571,7 +1651,9 @@ def render_full(report: schema.Report) -> str:"",*_assistant_safety_lines(),f"- Date range: {report.range_from} to {report.range_to}",- f"- Sources: {len(non_empty)} active ({', '.join(_source_label(s) for s in non_empty)})" if non_empty else "- Sources: none",+ f"- Sources: {len(non_empty)} active ({', '.join(_source_label(s) for s in non_empty)})"+ if non_empty+ else "- Sources: none","",]@@ -1621,8 +1703,25 @@ def render_full(report: schema.Report) -> str:# ALL items by source (flat dump, v2-style)lines.append("## All Items by Source")lines.append("")- source_order = ["reddit", "x", "youtube", "tiktok", "instagram", "threads", "pinterest",- "hackernews", "bluesky", "truthsocial", "polymarket", "grounding", "xiaohongshu", "github", "digg", "perplexity", "jobs"]+ source_order = [+ "reddit",+ "x",+ "youtube",+ "tiktok",+ "instagram",+ "threads",+ "pinterest",+ "hackernews",+ "bluesky",+ "truthsocial",+ "polymarket",+ "grounding",+ "xiaohongshu",+ "github",+ "digg",+ "perplexity",+ "jobs",+ ]for source in source_order:items = evidence_report.items_by_source.get(source, [])if not items:@@ -1631,7 +1730,9 @@ def render_full(report: schema.Report) -> str:lines.append("")for item in items:score = item.local_rank_score if item.local_rank_score is not None else 0- lines.append(f"**{item.item_id}** (score:{score:.0f}) {item.author or ''} ({item.published_at or 'date unknown'}) [{_format_item_engagement(item)}]")+ lines.append(+ f"**{item.item_id}** (score:{score:.0f}) {item.author or ''} ({item.published_at or 'date unknown'}) [{_format_item_engagement(item)}]"+ )lines.append(f" {item.title}")if item.url:lines.append(f" {item.url}")@@ -1649,8 +1750,13 @@ def render_full(report: schema.Report) -> str:excerpt = tc.get("excerpt", tc.get("text", ""))tc_score = tc.get("score", "")attribution = _comment_attribution(item.source, tc.get("author"))+ vote_part = (+ f" ({tc_score} {vote_label})"+ if tc_score is not None and tc_score != ""+ else ""+ )lines.append(- f" Top comment {attribution} ({tc_score} {vote_label}): "+ f" Top comment {attribution}{vote_part}: "f"{_format_untrusted_evidence(excerpt, 200, continuation_indent=' ')}")# Digg: inline X-post quotes attached to the cluster.@@ -1667,7 +1773,9 @@ def render_full(report: schema.Report) -> str:# Transcript highlights for YouTubehighlights = item.metadata.get("transcript_highlights", [])if highlights:- lines.append(" Highlights (auto-generated transcript; may contain transcription errors):")+ lines.append(+ " Highlights (auto-generated transcript; may contain transcription errors):"+ )for hl in highlights[:5]:lines.append(f' - "{_format_untrusted_evidence(hl, 200, continuation_indent=" ")}"'@@ -1675,7 +1783,9 @@ def render_full(report: schema.Report) -> str:# Full transcript snippet for YouTubetranscript = item.metadata.get("transcript_snippet", "")if transcript and len(transcript) > 100:- lines.append(f" <details><summary>Transcript ({len(transcript.split())} words; auto-generated — may contain transcription errors)</summary>")+ lines.append(+ f" <details><summary>Transcript ({len(transcript.split())} words; auto-generated — may contain transcription errors)</summary>"+ )lines.append(f" {_format_untrusted_evidence(transcript, 5000, continuation_indent=' ')}")@@ -1689,7 +1799,11 @@ def render_full(report: schema.Report) -> str:odds_parts = []for name, price in outcome_prices:if isinstance(price, (int, float)):- pct = f"{price * 100:.0f}%" if price >= 0.1 else f"{price * 100:.1f}%"+ pct = (+ f"{price * 100:.0f}%"+ if price >= 0.1+ else f"{price * 100:.1f}%"+ )odds_parts.append(f"{name}: {pct}")if odds_parts:lines.append(f" Odds: {' | '.join(odds_parts)}")@@ -1721,8 +1835,19 @@ def _format_item_engagement(item: schema.SourceItem) -> str:if not eng:return ""parts = []- for key in ["score", "likes", "views", "points", "reposts", "replies", "comments",- "play_count", "digg_count", "share_count", "num_comments"]:+ for key in [+ "score",+ "likes",+ "views",+ "points",+ "reposts",+ "replies",+ "comments",+ "play_count",+ "digg_count",+ "share_count",+ "num_comments",+ ]:val = eng.get(key)if val is not None and val != 0:parts.append(f"{val} {key}")@@ -1731,7 +1856,10 @@ def _format_item_engagement(item: schema.SourceItem) -> str:def render_context(report: schema.Report, cluster_limit: int = 6) -> str:evidence_report = schema.without_sources(report, {"corpus"})- candidate_by_id = {candidate.candidate_id: candidate for candidate in evidence_report.ranked_candidates}+ candidate_by_id = {+ candidate.candidate_id: candidate+ for candidate in evidence_report.ranked_candidates+ }requested_clusters = evidence_report.clusters[:cluster_limit]visible_clusters = _clusters_clearing_relevance_floor(evidence_report,@@ -1771,7 +1899,9 @@ def render_context(report: schema.Report, cluster_limit: int = 6) -> str:if no_solid_evidence:lines.append("- Nothing solid this window.")for cluster in visible_clusters:- lines.append(f"- {cluster.title} [{', '.join(_source_label(source) for source in cluster.sources)}]")+ lines.append(+ f"- {cluster.title} [{', '.join(_source_label(source) for source in cluster.sources)}]"+ )for candidate_id in _qualifying_representative_ids(cluster,candidate_by_id,@@ -1822,7 +1952,9 @@ def render_brief(report: schema.Report, cluster_limit: int = 8) -> str:"",*_assistant_safety_lines(),f"- Date range: {report.range_from} to {report.range_to}",- f"- Sources: {len(non_empty)} active ({', '.join(_source_label(s) for s in non_empty)})" if non_empty else "- Sources: none",+ f"- Sources: {len(non_empty)} active ({', '.join(_source_label(s) for s in non_empty)})"+ if non_empty+ else "- Sources: none","",]drill_context = _render_drill_context(report)@@ -1854,8 +1986,12 @@ def render_brief(report: schema.Report, cluster_limit: int = 8) -> str:lines.extend(["**Nothing solid this window.**", ""])for i, cluster in enumerate(visible_clusters, start=1):source_tags = ", ".join(_source_label(s) for s in cluster.sources)- qualifier = f" [{cluster.uncertainty.replace('-', ' ')}]" if cluster.uncertainty else ""- lines.append(f"### {i}. {cluster.title} (score {cluster.score:.0f}, {source_tags}){qualifier}")+ qualifier = (+ f" [{cluster.uncertainty.replace('-', ' ')}]" if cluster.uncertainty else ""+ )+ lines.append(+ f"### {i}. {cluster.title} (score {cluster.score:.0f}, {source_tags}){qualifier}"+ )for cid in _qualifying_representative_ids(cluster,candidate_by_id,@@ -1874,7 +2010,11 @@ def render_brief(report: schema.Report, cluster_limit: int = 8) -> str:lines.append("")hooks = sorted(- (c for c in qualifying_candidates if c.fun_score is not None and c.fun_score >= 70),+ (+ c+ for c in qualifying_candidates+ if c.fun_score is not None and c.fun_score >= 70+ ),key=lambda c: -(c.fun_score or 0),)if hooks:@@ -1893,7 +2033,8 @@ def render_brief(report: schema.Report, cluster_limit: int = 8) -> str:attribution = source_labelreason = (f" — {candidate.fun_explanation}"- if candidate.fun_explanation and candidate.fun_explanation != "heuristic-fallback"+ if candidate.fun_explanation+ and candidate.fun_explanation != "heuristic-fallback"else "")lines.append(@@ -1907,7 +2048,11 @@ def render_brief(report: schema.Report, cluster_limit: int = 8) -> str:lines.append("## Topic Tensions")lines.append("")for cluster in tensions[:cluster_limit]:- label = cluster.uncertainty.replace("-", " ").title() if cluster.uncertainty else ""+ label = (+ cluster.uncertainty.replace("-", " ").title()+ if cluster.uncertainty+ else ""+ )source_tags = ", ".join(_source_label(s) for s in cluster.sources)lines.append(f"- **{cluster.title}** [{label}]: {source_tags}")lines.append("")@@ -2022,7 +2167,7 @@ def _render_hiring_signals(out.append("- Strategic single-role signals (judge novelty yourself - a founding ""or first-of-function role can outweigh a whole department; in synthesis, "- "distinguish \"new bets\" from \"doubling down\"):"+ 'distinguish "new bets" from "doubling down"):')for cand in strategic[:8]:title = cand.get("title") or "Job posting"@@ -2081,8 +2226,13 @@ def _render_candidate(vote_label = _vote_label_for(primary.source) if primary else "upvotes"source = primary.source if primary else Noneattribution = _comment_attribution(source, tc.get("author"))+ vote_part = (+ f" ({score} {vote_label})"+ if score is not None and score != ""+ else ""+ )lines.append(- f" - {attribution} ({score} {vote_label}): "+ f" - {attribution}{vote_part}: "f"{_format_untrusted_evidence(excerpt.strip(), 240)}")for post in _digg_posts_for(primary):@@ -2092,7 +2242,9 @@ def _render_candidate(lines.append(f" - Insight: {_format_untrusted_evidence(insight, 220)}")highlights = _transcript_highlights(primary)if highlights:- lines.append(" - Highlights (auto-generated transcript; may contain transcription errors):")+ lines.append(+ " - Highlights (auto-generated transcript; may contain transcription errors):"+ )for hl in highlights:lines.append(f' - "{_format_untrusted_evidence(hl, 200)}"')return lines@@ -2129,13 +2281,23 @@ def _shorten_polymarket_title(title: str) -> str:t = t[5:].strip()# Drop "by <Month> <Day>" or "by <Month> <Day>, <Year>" tail- t = re.sub(r"\s+by\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d+(?:,\s*\d{4})?$", "", t, flags=re.IGNORECASE)+ t = re.sub(+ r"\s+by\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d+(?:,\s*\d{4})?$",+ "",+ t,+ flags=re.IGNORECASE,+ )# Drop "in <Year>" tail (e.g. "separate in 2026")t = re.sub(r"\s+in\s+\d{4}$", "", t, flags=re.IGNORECASE)# Drop "by <Year>" tailt = re.sub(r"\s+by\s+\d{4}$", "", t, flags=re.IGNORECASE)# Drop "before <Month> <Day>" tail- t = re.sub(r"\s+before\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d+$", "", t, flags=re.IGNORECASE)+ t = re.sub(+ r"\s+before\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+\d+$",+ "",+ t,+ flags=re.IGNORECASE,+ )# Pattern: "<Subject> visit <Place>" -> "<Place> visit"m = re.match(r"^(.+?)\s+visit\s+(?:the\s+)?(.+)$", t, flags=re.IGNORECASE)@@ -2151,12 +2313,14 @@ def _shorten_polymarket_title(title: str) -> str:t = " ".join(words[:6])# Drop a leading article so the descriptor doesn't read "an Anthropic Claude..."- t = re.sub(r"^(?:a|an|the)\s+", "", t, flags=re.I)+ t = re.sub(r"^(?:a|an|the)\s+", "", t, flags=re.IGNORECASE)return t-def _polymarket_top_markets(items: list[schema.SourceItem], limit: int = 3) -> list[str]:+def _polymarket_top_markets(+ items: list[schema.SourceItem], limit: int = 3+) -> list[str]:"""Build short summary strings for the top Polymarket markets by volume.Returns list like: ['UK visit 5.5%', 'Israel visit 8%', 'blocked from entering 36%']@@ -2178,9 +2342,15 @@ def _polymarket_top_markets(items: list[schema.SourceItem], limit: int = 3) -> lif not isinstance(lead_price, (int, float)):continue- pct = f"{lead_price * 100:.0f}%" if lead_price >= 0.1 else f"{lead_price * 100:.1f}%"+ pct = (+ f"{lead_price * 100:.0f}%"+ if lead_price >= 0.1+ else f"{lead_price * 100:.1f}%"+ )- descriptor = _shorten_polymarket_title(item.metadata.get("question") or item.title or "")+ descriptor = _shorten_polymarket_title(+ item.metadata.get("question") or item.title or ""+ )if not descriptor:continue@@ -2345,8 +2515,7 @@ def _site_name_for_url(url: str) -> str:# urlparse needs a scheme to resolve the netloc; prepend http:// if missing.parsed = urlparse(u if "://" in u else f"http://{u}")host = (parsed.netloc or parsed.path.split("/", 1)[0]).lower()- if host.startswith("www."):- host = host[4:]+ host = host.removeprefix("www.")if not host:return u[:40]if host in _SITE_NAMES:@@ -2385,27 +2554,80 @@ def _format_web_line_sources(items: list[schema.SourceItem], limit: int = 8) -># Report.items_by_source. Order here is the render order._FOOTER_SOURCES: list[tuple[str, str, str, str, list[tuple[str, str]]]] = [# (source_key, emoji, display_name, item_word_singular, [(engagement_key, word)])- ("reddit", "🟠", "Reddit", "thread", [("score", "upvotes"), ("num_comments", "comments")]),- ("x", "🔵", "X", "post", [("likes", "likes"), ("reposts", "reposts")]),- ("youtube", "🔴", "YouTube", "video", [("views", "views")]), # transcripts appended below in _build_source_footer_lines- ("tiktok", "🎵", "TikTok", "video", [("views", "views"), ("likes", "likes")]),- ("instagram", "📸", "Instagram", "reel", [("views", "views"), ("likes", "likes")]),- ("threads", "🧵", "Threads", "post", [("likes", "likes"), ("replies", "replies")]),- ("pinterest", "📌", "Pinterest", "pin", [("saves", "saves"), ("comments", "comments")]),- ("hackernews", "🟡", "HN", "story", [("points", "points"), ("comments", "comments")]),- ("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),- ("truthsocial", "🇺🇸", "Truth Social", "post", [("likes", "likes"), ("reposts", "reposts")]),- ("linkedin", "👔", "LinkedIn", "post", [("likes", "likes"), ("comments", "comments")]),- ("github", "🐙", "GitHub", "item", [("stars", "stars"), ("merged_prs", "merged"), ("reactions", "reactions"), ("comments", "comments")]),- ("digg", "⛏️", "Digg", "cluster", [("postCount", "posts"), ("uniqueAuthors", "authors")]),- ("arxiv", "📄", "arXiv", "paper", []),- ("techmeme", "📰", "Techmeme", "headline", []),- ("trustpilot", "⭐", "Trustpilot", "review", [("reviews", "reviews")]),+ (+ "reddit",+ "🟠",+ "Reddit",+ "thread",+ [("score", "upvotes"), ("num_comments", "comments")],+ ),+ ("x", "🔵", "X", "post", [("likes", "likes"), ("reposts", "reposts")]),+ (+ "youtube",+ "🔴",+ "YouTube",+ "video",+ [("views", "views")],+ ), # transcripts appended below in _build_source_footer_lines+ ("tiktok", "🎵", "TikTok", "video", [("views", "views"), ("likes", "likes")]),+ ("instagram", "📸", "Instagram", "reel", [("views", "views"), ("likes", "likes")]),+ ("threads", "🧵", "Threads", "post", [("likes", "likes"), ("replies", "replies")]),+ (+ "pinterest",+ "📌",+ "Pinterest",+ "pin",+ [("saves", "saves"), ("comments", "comments")],+ ),+ (+ "hackernews",+ "🟡",+ "HN",+ "story",+ [("points", "points"), ("comments", "comments")],+ ),+ ("bluesky", "🦋", "Bluesky", "post", [("likes", "likes"), ("reposts", "reposts")]),+ (+ "truthsocial",+ "🇺🇸",+ "Truth Social",+ "post",+ [("likes", "likes"), ("reposts", "reposts")],+ ),+ (+ "linkedin",+ "👔",+ "LinkedIn",+ "post",+ [("likes", "likes"), ("comments", "comments")],+ ),+ (+ "github",+ "🐙",+ "GitHub",+ "item",+ [+ ("stars", "stars"),+ ("merged_prs", "merged"),+ ("reactions", "reactions"),+ ("comments", "comments"),+ ],+ ),+ (+ "digg",+ "⛏️",+ "Digg",+ "cluster",+ [("postCount", "posts"), ("uniqueAuthors", "authors")],+ ),+ ("arxiv", "📄", "arXiv", "paper", []),+ ("techmeme", "📰", "Techmeme", "headline", []),+ ("trustpilot", "⭐", "Trustpilot", "review", [("reviews", "reviews")]),# Jobs must appear so a scoped --hiring-signals run (jobs-only) still emits# the LAW 5 footer; without it the footer was dropped entirely.- ("jobs", "💼", "Jobs", "role", []),- ("perplexity", "🧠", "Perplexity", "result", [("citations", "citations")]),- ("corpus", "🔒", "Your files", "file", []),+ ("jobs", "💼", "Jobs", "role", []),+ ("perplexity", "🧠", "Perplexity", "result", [("citations", "citations")]),+ ("corpus", "🔒", "Your files", "file", []),]@@ -2422,7 +2644,9 @@ def _sum_engagement(items: list[schema.SourceItem], key: str) -> int:return total-def _footer_line_for_source(emoji: str, label: str, count: int, item_word: str, stats: str) -> str:+def _footer_line_for_source(+ emoji: str, label: str, count: int, item_word: str, stats: str+) -> str:count_str = f"{count:,}" if count >= 1000 else str(count)plural = f"{item_word}s" if count != 1 else item_wordif stats:@@ -2455,8 +2679,12 @@ def _build_source_footer_lines(report: schema.Report) -> list[str]:# case that needs to be loud is the one previously omitted from the footer.if source_key == "youtube":with_transcripts = sum(- 1 for it in items- if (it.metadata.get("transcript_highlights") or it.metadata.get("transcript_snippet"))+ 1+ for it in items+ if (+ it.metadata.get("transcript_highlights")+ or it.metadata.get("transcript_snippet")+ ))parts.append(f"{with_transcripts}/{len(items)} with transcripts")stats = " │ ".join(parts)@@ -2516,7 +2744,15 @@ def _top_voices_footer_line(report: schema.Report) -> str | None:"""handle_items = {source: report.items_by_source.get(source) or []- for source in ("x", "bluesky", "truthsocial", "youtube", "tiktok", "instagram", "threads")+ for source in (+ "x",+ "bluesky",+ "truthsocial",+ "youtube",+ "tiktok",+ "instagram",+ "threads",+ )}handle_counts: Counter[str] = Counter()for items in handle_items.values():@@ -2645,7 +2881,9 @@ def _assess_data_freshness(report: schema.Report) -> str | None:item.published_at,reference_date=report.range_to,)- ) is not None and _days_ago <= 7+ )+ is not None+ and _days_ago <= 7]if len(recent_items) < 3:return f"Limited recent data: only {len(recent_items)} of {len(dated_items)} dated items are from the last 7 days."@@ -2680,22 +2918,27 @@ def _format_actor(item: schema.SourceItem | None) -> str | None:# Per-source engagement display fields: list of (field_name, label) tuples.ENGAGEMENT_DISPLAY: dict[str, list[tuple[str, str]]] = {- "reddit": [("score", "pts"), ("num_comments", "cmt")],- "x": [("likes", "likes"), ("reposts", "rt"), ("replies", "re")],- "youtube": [("views", "views"), ("likes", "likes"), ("comments", "cmt")],- "tiktok": [("views", "views"), ("likes", "likes"), ("comments", "cmt")],- "instagram": [("views", "views"), ("likes", "likes"), ("comments", "cmt")],- "threads": [("likes", "likes"), ("replies", "re")],- "pinterest": [("saves", "saves"), ("comments", "cmt")],- "hackernews": [("points", "pts"), ("comments", "cmt")],- "bluesky": [("likes", "likes"), ("reposts", "rt"), ("replies", "re")],- "truthsocial": [("likes", "likes"), ("reposts", "rt"), ("replies", "re")],- "linkedin": [("likes", "likes"), ("comments", "cmt")],- "polymarket": [],- "github": [("stars", "stars"), ("merged_prs", "merged"), ("reactions", "react"), ("comments", "cmt")],- "perplexity": [("citations", "cite")],- "digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],- "trustpilot": [("reviews", "reviews")],+ "reddit": [("score", "pts"), ("num_comments", "cmt")],+ "x": [("likes", "likes"), ("reposts", "rt"), ("replies", "re")],+ "youtube": [("views", "views"), ("likes", "likes"), ("comments", "cmt")],+ "tiktok": [("views", "views"), ("likes", "likes"), ("comments", "cmt")],+ "instagram": [("views", "views"), ("likes", "likes"), ("comments", "cmt")],+ "threads": [("likes", "likes"), ("replies", "re")],+ "pinterest": [("saves", "saves"), ("comments", "cmt")],+ "hackernews": [("points", "pts"), ("comments", "cmt")],+ "bluesky": [("likes", "likes"), ("reposts", "rt"), ("replies", "re")],+ "truthsocial": [("likes", "likes"), ("reposts", "rt"), ("replies", "re")],+ "linkedin": [("likes", "likes"), ("comments", "cmt")],+ "polymarket": [],+ "github": [+ ("stars", "stars"),+ ("merged_prs", "merged"),+ ("reactions", "react"),+ ("comments", "cmt"),+ ],+ "perplexity": [("citations", "cite")],+ "digg": [("postCount", "posts"), ("uniqueAuthors", "auth")],+ "trustpilot": [("reviews", "reviews")],}@@ -2765,7 +3008,9 @@ def _top_actor_summary(source: str, items: list[schema.SourceItem]) -> str | Nonreturn f"{label}: {', '.join(actors)}"-def _top_actors_for_source(source: str, items: list[schema.SourceItem], limit: int = 3) -> list[str]:+def _top_actors_for_source(+ source: str, items: list[schema.SourceItem], limit: int = 3+) -> list[str]:counts: Counter[str] = Counter()for item in items:actor = _stats_actor(item)@@ -2774,7 +3019,9 @@ def _top_actors_for_source(source: str, items: list[schema.SourceItem], limit: ireturn [actor for actor, _ in counts.most_common(limit)]-def _top_voices_overall(items_by_source: dict[str, list[schema.SourceItem]], limit: int = 5) -> list[str]:+def _top_voices_overall(+ items_by_source: dict[str, list[schema.SourceItem]], limit: int = 5+) -> list[str]:counts: Counter[str] = Counter()for items in items_by_source.values():for item in items:@@ -2824,7 +3071,10 @@ _TOP_COMMENT_MIN_SCORE: dict[str, int] = {"youtube": 50,"tiktok": 500,"instagram": 5,- "hackernews": 5,+ # Zero, not a tuned floor: the Algolia items endpoint returns points=null+ # for every comment child (only stories carry points), so any positive+ # threshold here rejects the entire source rather than filtering it.+ "hackernews": 0,}_TOP_COMMENT_VOTE_LABEL: dict[str, str] = {"reddit": "upvotes",@@ -2866,11 +3116,13 @@ def _comment_attribution(source: str | None, author: str | None) -> str:# Some sources (YouTube/TikTok) already store the author with a leading '@';# strip it before re-prefixing so we don't emit '@@handle'.if prefix and author.startswith(prefix):- author = author[len(prefix):]+ author = author[len(prefix) :]return f"{prefix}{author}" if prefix else author-def _top_comments_list(item: schema.SourceItem | None, limit: int = 3, min_score: int | None = None) -> list[dict]:+def _top_comments_list(+ item: schema.SourceItem | None, limit: int = 3, min_score: int | None = None+) -> list[dict]:"""Return up to `limit` top comments with score at or above the source's minimum.If `min_score` is passed explicitly it overrides the per-source default;@@ -2939,7 +3191,6 @@ def _source_label(source: str) -> str:return SOURCE_LABELS.get(source, source.replace("_", " ").title())-def _best_take_relevance_ok(candidate) -> bool:"""Exclude off-topic-but-viral candidates from Best Takes.@@ -2983,7 +3234,8 @@ def _render_best_takes(source_weight=None,):eligible = [- c for c in candidates+ c+ for c in candidatesif c.fun_score is not Noneand c.fun_score >= _BEST_TAKE_FUNNY_FLOORand _best_take_relevance_ok(c)@@ -2994,7 +3246,8 @@ def _render_best_takes(# threshold, so emphasis reorders takes without inventing eligibility.rank_key = ((lambda pair: -pair[1] * source_weight(pair[0].source))- if source_weight else (lambda pair: -pair[1])+ if source_weight+ else (lambda pair: -pair[1]))# Carry the effective score forward so the display loop doesn't recompute it.gems = [(c, eff) for c, eff in sorted(scored, key=rank_key) if eff >= threshold]@@ -3003,17 +3256,54 @@ def _render_best_takes(lines = ["## Best Takes", ""]for candidate, effective in gems[:limit]:text = candidate.title.strip()+ selected_comment_item = None+ hackernews_take = Nonefor item in candidate.source_items:for comment in item.metadata.get("top_comments", [])[:3]:- body = (comment.get("body") or comment.get("text") or "") if isinstance(comment, dict) else str(comment)+ body = (+ (+ comment.get("body")+ or comment.get("text")+ or (+ comment.get("excerpt")+ if item.source == "hackernews"+ else ""+ )+ or ""+ )+ if isinstance(comment, dict)+ else str(comment)+ )body = body.strip()- if body and len(body) < len(text) and len(body) > 10:+ if not body or len(body) <= 10:+ continue+ if item.source == "hackernews" and (+ hackernews_take is None or len(body) < len(hackernews_take[0])+ ):+ hackernews_take = (body, item)+ elif hackernews_take is None and len(body) < len(text):text = body- source_label = _source_label(candidate.source)- author = candidate.source_items[0].author if candidate.source_items else None- attribution = f"@{author} on {source_label}" if author and candidate.source in ("x", "tiktok", "instagram", "threads") else f"{source_label}"- if author and candidate.source == "reddit":- container = candidate.source_items[0].container if candidate.source_items else None+ selected_comment_item = item+ if hackernews_take is not None:+ text, selected_comment_item = hackernews_take+ attribution_item = (+ selected_comment_item+ or (candidate.source_items[0] if candidate.source_items else None)+ )+ attribution_source = (+ selected_comment_item.source+ if selected_comment_item is not None+ else candidate.source+ )+ source_label = _source_label(attribution_source)+ author = attribution_item.author if attribution_item else None+ attribution = (+ f"@{author} on {source_label}"+ if author and attribution_source in ("x", "tiktok", "instagram", "threads")+ else f"{source_label}"+ )+ if author and attribution_source == "reddit":+ container = attribution_item.container if attribution_item else Noneattribution = f"r/{container} comment" if container else "Reddit"# fun: is the LLM humor score; flag when crowd votes materially lifted# this item's ranking, so a lower-fun item ranking above a higher-fun one@@ -3021,7 +3311,12 @@ def _render_best_takes(crowd_boost = effective - (candidate.fun_score or 0.0)crowd_tag = " +crowd" if crowd_boost >= 5.0 else ""score_tag = f"(fun:{candidate.fun_score:.0f}{crowd_tag})"- reason = f" -- {candidate.fun_explanation}" if candidate.fun_explanation and candidate.fun_explanation != "heuristic-fallback" else ""+ reason = (+ f" -- {candidate.fun_explanation}"+ if candidate.fun_explanation+ and candidate.fun_explanation != "heuristic-fallback"+ else ""+ )lines.append(f'- "{_format_untrusted_evidence(text, 280, continuation_indent=" ")}" 'f"-- {attribution} {score_tag}{reason}"@@ -3051,7 +3346,8 @@ def _render_top_comments(scored: list[tuple[float, schema.Candidate, schema.SourceItem, dict, str]] = []candidate_pool = report.ranked_candidates if candidates is None else candidatesfloor_candidates = [- cand for cand in candidate_pool+ cand+ for cand in candidate_poolif _best_take_relevance_ok(cand)and (cand.local_relevance or 0.0) >= relevance.RELEVANCE_FLOOR]@@ -3061,7 +3357,10 @@ def _render_top_comments(continue# Skip comments from off-topic threads when enough candidates clear the# floor; sparse niche topics still surface their best comments (#641).- if apply_relevance_floor and (cand.local_relevance or 0.0) < relevance.RELEVANCE_FLOOR:+ if (+ apply_relevance_floor+ and (cand.local_relevance or 0.0) < relevance.RELEVANCE_FLOOR+ ):continuefor item in cand.source_items:# Pass min_score=0 here: the cross-platform list deliberately does@@ -3072,7 +3371,9 @@ def _render_top_comments(for tc in _top_comments_list(item, min_score=0):if not isinstance(tc, dict):continue- body = (tc.get("excerpt") or tc.get("text") or tc.get("body") or "").strip()+ body = (+ tc.get("excerpt") or tc.get("text") or tc.get("body") or ""+ ).strip()if len(body) < 12:continuekey = body[:60].lower()@@ -3081,7 +3382,9 @@ def _render_top_comments(seen.add(key)# Blend vote strength (60%) with thread relevance (40%) so comments# from on-topic threads rank above off-topic viral comments.- vote_strength = signals.normalized_comment_vote(cand.source, tc.get("score"))+ vote_strength = signals.normalized_comment_vote(+ item.source, tc.get("score")+ )strength = 0.6 * vote_strength + 0.4 * (cand.local_relevance or 0.0)scored.append((strength, cand, item, tc, body))if len(scored) < 2:@@ -3097,7 +3400,7 @@ def _render_top_comments(# among same-rank picks. The model still makes the final quotable pick.by_source: dict[str, list] = {}for row in scored:- by_source.setdefault(row[1].source, []).append(row)+ by_source.setdefault(row[2].source, []).append(row)for src_rows in by_source.values():src_rows.sort(key=lambda row: -row[0])ordered: list = []@@ -3107,15 +3410,20 @@ def _render_top_comments(tier.sort(key=lambda row: -row[0]) # among same-rank picks, strongest firstordered.extend(tier)lines = ["## Top Community Comments", ""]- for _strength, cand, _item, tc, body in ordered[:limit]:+ for _strength, cand, item, tc, body in ordered[:limit]:score = tc.get("score", "")- vote_label = _vote_label_for(cand.source)- attribution = _comment_attribution(cand.source, tc.get("author"))+ vote_label = _vote_label_for(item.source)+ attribution = _comment_attribution(item.source, tc.get("author"))url = tc.get("url") or cand.url or ""url_part = f" — {url}" if url else ""+ vote_part = (+ f" ({score} {vote_label})"+ if score is not None and score != ""+ else ""+ )lines.append(f'- "{_format_untrusted_evidence(body, 240, continuation_indent=" ")}" '- f"— {attribution} ({score} {vote_label}){url_part}"+ f"— {attribution}{vote_part}{url_part}")return linesdiff --git a/tests/test_hackernews.py b/tests/test_hackernews.pyindex 8ab57ba6..e5c6678a 100644--- a/tests/test_hackernews.py+++ b/tests/test_hackernews.py@@ -484,5 +484,25 @@ def test_engagement_score_zero_values():assert engagement["points"] == 0assert engagement["comments"] == 0+[email protected]("points", [None, 0])+@patch("lib.hackernews.http.request")+def test_fetch_item_comments_preserves_absent_and_zero_points(mock_request, points):+ """The adapter must distinguish an unmeasured score from a measured zero."""+ mock_request.return_value = {+ "children": [+ {+ "author": "alice",+ "text": "A useful comment.",+ "points": points,+ }+ ]+ }++ result = hackernews._fetch_item_comments("123")++ assert result["comments"][0]["points"] is points++if __name__ == "__main__":pytest.main([__file__, "-v"])diff --git a/tests/test_hackernews_comment_floor.py b/tests/test_hackernews_comment_floor.pynew file mode 100644index 00000000..5853acdc--- /dev/null+++ b/tests/test_hackernews_comment_floor.py@@ -0,0 +1,333 @@+"""Hacker News comments must survive the per-source top-comment floor.++HN comments arrive as ``{author, text, points}`` while every downstream reader+keys on ``score``/``excerpt``. Before the fix, ``_normalize_hackernews`` stored+them raw, so ``render._top_comments_list`` evaluated ``(c.get("score") or 0) >= 5``+against a key that was never present and rejected the entire source.+"""++from unittest.mock import patch++from lib import hackernews, normalize, render, schema++FROM_DATE = "2026-06-26"+TO_DATE = "2026-07-26"+++def _hn_item():+ return {+ "id": "42",+ "title": "Show HN: a thing",+ "url": "https://example.test/thing",+ "hn_url": "https://news.ycombinator.com/item?id=42",+ "author": "pg",+ "date": "2026-07-20",+ "engagement": {"points": 420, "comments": 2},+ "top_comments": [+ {+ "author": "alice",+ "text": "The single clearest explanation I have read.",+ "points": 93,+ },+ {+ "author": "bob",+ "text": "Counterpoint: the benchmark skips the hard cases.",+ "points": None,+ },+ ],+ }+++def _reddit_item():+ return {+ "id": "r1",+ "title": "a reddit thread",+ "url": "https://reddit.test/r1",+ "author": "carol",+ "date": "2026-07-20",+ "engagement": {"upvotes": 300, "comments": 2},+ "top_comments": [+ {+ "author": "alice",+ "excerpt": "The single clearest explanation I have read.",+ "score": 93,+ },+ {+ "author": "bob",+ "excerpt": "Counterpoint: the benchmark skips the hard cases.",+ "score": 51,+ },+ ],+ }+++def test_hn_comments_are_remapped_to_the_shared_shape():+ item = normalize._normalize_hackernews(+ "hackernews", _hn_item(), 0, FROM_DATE, TO_DATE+ )+ comments = item.metadata["top_comments"]+ assert comments, "HN comments should survive normalisation"+ for comment in comments:+ assert "score" in comment, (+ f"expected the shared score key, got {sorted(comment)}"+ )+ assert "excerpt" in comment, (+ f"expected the shared excerpt key, got {sorted(comment)}"+ )+ assert comments[0]["score"] == 93, "points must carry through as score"+++def test_hn_comments_clear_the_per_source_floor():+ item = normalize._normalize_hackernews(+ "hackernews", _hn_item(), 0, FROM_DATE, TO_DATE+ )+ assert render._top_comments_list(item), (+ "HN comments must not be filtered out by a floor keyed on a field the "+ "source never populated"+ )+++def test_hn_floor_is_zero_because_hn_has_no_per_comment_points():+ """The Algolia items endpoint returns points=null for every comment child.++ Any positive threshold therefore rejects the whole source rather than+ filtering it, so this constant is load-bearing rather than a tuning knob.+ """+ assert render._TOP_COMMENT_MIN_SCORE["hackernews"] == 0+++def test_reddit_control_is_unchanged():+ item = normalize._normalize_reddit("reddit", _reddit_item(), 0, FROM_DATE, TO_DATE)+ assert len(render._top_comments_list(item)) == 2+++def _candidate(source, item):+ return schema.Candidate(+ candidate_id="c1",+ item_id=item.item_id,+ source=source,+ title=item.title,+ url=item.url,+ snippet="",+ subquery_labels=[],+ native_ranks={},+ local_relevance=0.9,+ freshness=1,+ engagement=100,+ source_quality=0.9,+ rrf_score=1.0,+ source_items=[item],+ final_score=50.0,+ explanation="llm-rerank",+ )+++def _report(item, candidate):+ return schema.Report(+ topic="HN comments",+ range_from=FROM_DATE,+ range_to=TO_DATE,+ generated_at=f"{TO_DATE}T00:00:00+00:00",+ provider_runtime=schema.ProviderRuntime(+ reasoning_provider="local",+ planner_model="local",+ rerank_model="local",+ ),+ query_plan=schema.QueryPlan(+ intent="general",+ freshness_mode="balanced_recent",+ cluster_mode="story",+ raw_topic="HN comments",+ subqueries=[+ schema.SubQuery(+ label="primary",+ search_query="HN comments",+ ranking_query="What are the top HN comments?",+ sources=["hackernews"],+ )+ ],+ source_weights={"hackernews": 1.0},+ ),+ clusters=[],+ ranked_candidates=[candidate],+ items_by_source={"hackernews": [item]},+ errors_by_source={},+ )+++def _render_vote_paths(points):+ raw = _hn_item()+ with patch("lib.hackernews.http.request") as request:+ request.return_value = {+ "children": [+ {+ "author": "alice",+ "text": raw["top_comments"][0]["text"],+ "points": 93,+ },+ {+ "author": "bob",+ "text": raw["top_comments"][1]["text"],+ "points": points,+ },+ ]+ }+ raw["top_comments"] = hackernews._fetch_item_comments("42")["comments"]+ hn = normalize._normalize_hackernews(+ "hackernews", raw, 0, FROM_DATE, TO_DATE+ )+ candidate = _candidate("hackernews", hn)+ report = _report(hn, candidate)+ return hn.metadata["top_comments"][1]["score"], {+ "full": render.render_full(report),+ "candidate": "\n".join(render._render_candidate(candidate, prefix="1.")),+ "top_comments": "\n".join(render._render_top_comments(report)),+ }+++def test_absent_hn_vote_is_omitted_across_comment_renderers():+ """A missing vote must not display a fabricated numeric measurement."""+ normalized_score, rendered_paths = _render_vote_paths(None)+ assert normalized_score is None+ for path, rendered in rendered_paths.items():+ assert "(0 points)" not in rendered, f"{path}:\n{rendered}"+ assert "(93 points)" in rendered, f"{path} dropped a real vote count"+++def test_explicit_zero_hn_vote_is_rendered_across_comment_renderers():+ """A measured zero remains visible in every zero-admitting comment path."""+ normalized_score, rendered_paths = _render_vote_paths(0)+ assert normalized_score == 0+ for path, rendered in rendered_paths.items():+ assert "(0 points)" in rendered, f"{path} hid an explicit numeric zero"+++def test_best_takes_uses_normalized_hn_comment_excerpt():+ """Best Takes should display HN comment text, not only the story title."""+ first_raw = _hn_item()+ first_raw["title"] = "Show HN: a deliberately long story title for testing"+ first_raw["top_comments"] = [+ {"author": "alice", "text": "A sharp HN take.", "points": None}+ ]+ second_raw = _hn_item()+ second_raw["id"] = "43"+ second_raw["title"] = "Ask HN: another deliberately long story title for testing"+ second_raw["top_comments"] = [+ {"author": "bob", "text": "Another good take.", "points": None}+ ]+ first = _candidate(+ "hackernews",+ normalize._normalize_hackernews(+ "hackernews", first_raw, 0, FROM_DATE, TO_DATE+ ),+ )+ second = _candidate(+ "hackernews",+ normalize._normalize_hackernews(+ "hackernews", second_raw, 1, FROM_DATE, TO_DATE+ ),+ )+ first.fun_score = 80.0+ second.fun_score = 80.0++ rendered = "\n".join(+ render._render_best_takes([first, second], threshold=70.0, vote_weight=0.0)+ )+ assert "A sharp HN take." in rendered+ assert "Another good take." in rendered+++def test_best_takes_uses_hn_comment_longer_than_story_title():+ """HN comment excerpts remain the take even when longer than the title."""+ first_raw = _hn_item()+ first_raw["title"] = "Short story"+ first_raw["top_comments"] = [+ {+ "author": "alice",+ "text": "This longer Hacker News comment is the actual sharp take.",+ "points": None,+ }+ ]+ second_raw = _hn_item()+ second_raw["id"] = "43"+ second_raw["top_comments"] = [+ {"author": "bob", "text": "Another good take.", "points": None}+ ]+ first = _candidate(+ "hackernews",+ normalize._normalize_hackernews(+ "hackernews", first_raw, 0, FROM_DATE, TO_DATE+ ),+ )+ second = _candidate(+ "hackernews",+ normalize._normalize_hackernews(+ "hackernews", second_raw, 1, FROM_DATE, TO_DATE+ ),+ )+ first.fun_score = 80.0+ second.fun_score = 80.0++ rendered = "\n".join(+ render._render_best_takes([first, second], threshold=70.0, vote_weight=0.0)+ )+ assert "This longer Hacker News comment is the actual sharp take." in rendered+ assert '"Short story"' not in rendered+++def test_best_takes_uses_hn_excerpt_retained_by_fused_candidate():+ """A non-HN representative must not hide its retained HN comment."""+ reddit = normalize._normalize_reddit(+ "reddit", _reddit_item(), 0, FROM_DATE, TO_DATE+ )+ hn_raw = _hn_item()+ hn_raw["top_comments"] = [+ {"author": "alice", "text": "The retained HN take.", "points": 93}+ ]+ hn = normalize._normalize_hackernews(+ "hackernews", hn_raw, 0, FROM_DATE, TO_DATE+ )+ fused = _candidate("reddit", reddit)+ fused.source_items = [reddit, hn]+ assert fused.source != "hackernews"+ control_raw = _reddit_item()+ control_raw["id"] = "r2"+ control_raw["title"] = "a separate reddit thread"+ control = _candidate(+ "reddit",+ normalize._normalize_reddit(+ "reddit", control_raw, 1, FROM_DATE, TO_DATE+ ),+ )+ fused.fun_score = 80.0+ control.fun_score = 80.0++ with patch(+ "lib.render.signals.normalized_comment_vote",+ wraps=render.signals.normalized_comment_vote,+ ) as normalized_vote:+ top_comments = "\n".join(+ render._render_top_comments(+ _report(reddit, fused),+ candidates=[fused, control],+ )+ )+ rendered = "\n".join(+ render._render_best_takes([fused, control], threshold=70.0, vote_weight=0.0)+ )+ assert "The retained HN take." in rendered+ fused_line = next(+ line for line in rendered.splitlines() if "The retained HN take." in line+ )+ assert "-- Hacker News " in fused_line+ assert "Reddit" not in fused_line+ assert "r/" not in fused_line+ retained_line = next(+ line for line in top_comments.splitlines() if "The retained HN take." in line+ )+ assert "— alice (93 points)" in retained_line+ assert "upvotes" not in retained_line+ assert "u/alice" not in retained_line+ assert any(+ call.args == ("hackernews", 93) for call in normalized_vote.call_args_list+ )diff --git a/tests/test_registers.py b/tests/test_registers.pyindex 751e4857..43c07deb 100644--- a/tests/test_registers.py+++ b/tests/test_registers.py@@ -203,7 +203,7 @@ def test_default_register_is_byte_identical_when_omitted(monkeypatch):assert implicit == explicitassert hashlib.sha256(implicit.encode()).hexdigest() == (- "8ca92207d49f09bc08ac4aba60c573cc832a23b1c78ad7cf90a5ca24bfbafe50"+ "fe85fb29fb787ed71f7cc1b6fcce48e07e4dfba4e2d4d16a0eceb9d9152bce1c")