Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: demote reranker candidates that miss the primary entity

Matt Van Horna709d66e2a1b
diff --git a/scripts/lib/rerank.py b/scripts/lib/rerank.py
index 680f4ff8..9954613c 100644
--- a/scripts/lib/rerank.py
+++ b/scripts/lib/rerank.py
@@ -3,8 +3,34 @@
from __future__ import annotations
import json
-
-from . import http, providers, schema
+import re
+
+from . import http, providers, query, schema
+
+
+# Penalty applied when a candidate does not mention the primary entity
+# from the topic in its title or snippet. Picked empirically: a typical
+# score spread in the shortlist is 30-70, so 25 points reliably pushes
+# an off-topic candidate below on-topic ones without fully zeroing out
+# marginal matches. See 2026-04-19 Hermes Agent Use Cases failure: a
+# Nate Herk "Managed Agents" video scored 51 / ranked #2 with zero
+# Hermes content.
+ENTITY_MISS_PENALTY = 25.0
+
+# Intent modifiers to strip before extracting the primary entity so that,
+# for example, "Hermes Agent use cases" yields primary_entity="hermes agent"
+# rather than "hermes agent use cases". Kept in sync with
+# planner._INTENT_MODIFIER_PATTERNS.
+_INTENT_MODIFIER_RE = re.compile(
+ r"\b("
+ r"use cases|use case|workflows|workflow|"
+ r"examples|example|tutorial|tutorials|"
+ r"review|reviews|comparison|applications|"
+ r"in practice|production use|production|"
+ r"how i use"
+ r")\b",
+ re.IGNORECASE,
+)
INTENT_SCORING_HINTS: dict[str, str] = {
"comparison": (
@@ -60,20 +86,21 @@ def rerank_candidates(
) -> list[schema.Candidate]:
"""Rerank the fused shortlist, demoting candidates the reranker scored as irrelevant."""
shortlisted = candidates[:shortlist_size]
+ primary_entity = _primary_entity(topic)
if provider and model and shortlisted:
try:
- response = provider.generate_json(model, _build_prompt(topic, plan, shortlisted))
+ response = provider.generate_json(model, _build_prompt(topic, plan, shortlisted, primary_entity))
_apply_llm_scores(shortlisted, response)
except (ValueError, KeyError, json.JSONDecodeError, OSError, http.HTTPError) as exc:
import sys
print(f"[Rerank] LLM reranking failed, using local fallback: {type(exc).__name__}: {exc}", file=sys.stderr)
- _apply_fallback_scores(shortlisted)
+ _apply_fallback_scores(shortlisted, primary_entity=primary_entity)
else:
- _apply_fallback_scores(shortlisted)
+ _apply_fallback_scores(shortlisted, primary_entity=primary_entity)
if len(candidates) > shortlist_size:
tail = candidates[shortlist_size:]
- _apply_fallback_scores(tail)
+ _apply_fallback_scores(tail, primary_entity=primary_entity)
return sorted(
candidates,
@@ -103,7 +130,7 @@ def _fenced_untrusted_content(candidate_block: str) -> str:
)
-def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Candidate]) -> str:
+def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Candidate], primary_entity: str = "") -> str:
ranking_queries = "\n".join(
f"- {subquery.label}: {subquery.ranking_query}"
for subquery in plan.subqueries
@@ -121,6 +148,16 @@ def _build_prompt(topic: str, plan: schema.QueryPlan, candidates: list[schema.Ca
)
for candidate in candidates
)
+ grounding_hint = ""
+ if primary_entity:
+ grounding_hint = (
+ f"\nPrimary entity grounding: the user's primary entity is \"{primary_entity}\". "
+ "A candidate that does NOT mention this entity (or a clear synonym/abbreviation) "
+ "in its title or snippet should score no higher than 30, regardless of other "
+ "signals. Do not let a candidate match the topic vicinity without matching the "
+ "entity itself. 2026-04-19 Hermes Agent Use Cases failure: a Nate Herk video "
+ "about Claude's Managed Agents scored 51 with zero Hermes content.\n"
+ )
return f"""
Judge search-result relevance for a last-30-days research pipeline.
@@ -145,7 +182,7 @@ Scoring guidance:
- 70 to 89: clearly relevant and useful
- 40 to 69: somewhat relevant but weaker
- 0 to 39: weak, redundant, or off-target
-{_intent_hint_block(plan)}
+{grounding_hint}{_intent_hint_block(plan)}
{_fenced_untrusted_content(candidate_block)}
""".strip()
@@ -169,21 +206,45 @@ def _apply_llm_scores(candidates: list[schema.Candidate], payload: dict) -> None
candidate.final_score = _final_score(candidate)
-def _apply_fallback_scores(candidates: list[schema.Candidate]) -> None:
+def _apply_fallback_scores(candidates: list[schema.Candidate], *, primary_entity: str = "") -> None:
for candidate in candidates:
- rerank_score, reason = _fallback_tuple(candidate)
+ rerank_score, reason = _fallback_tuple(candidate, primary_entity=primary_entity)
candidate.rerank_score = rerank_score
candidate.explanation = reason
candidate.final_score = _final_score(candidate)
-def _fallback_tuple(candidate: schema.Candidate) -> tuple[float, str]:
+def _fallback_tuple(candidate: schema.Candidate, *, primary_entity: str = "") -> tuple[float, str]:
score = (
(candidate.local_relevance * 100.0 * 0.7)
+ (candidate.freshness * 0.2)
+ (candidate.source_quality * 100.0 * 0.1)
)
- return max(0.0, min(100.0, score)), "fallback-local-score"
+ reason = "fallback-local-score"
+ # Entity-grounding demotion: if the primary entity (topic minus intent
+ # modifier) is not present in the candidate's title or snippet, subtract
+ # ENTITY_MISS_PENALTY. Skip for candidates with no text at all (e.g.,
+ # image-only TikToks) to avoid penalizing thin-text sources unfairly.
+ if primary_entity and (candidate.title or candidate.snippet):
+ haystack = f"{candidate.title} {candidate.snippet}".lower()
+ if primary_entity.lower() not in haystack:
+ score -= ENTITY_MISS_PENALTY
+ reason = "fallback-local-score (entity-miss demotion)"
+ return max(0.0, min(100.0, score)), reason
+
+
+def _primary_entity(topic: str) -> str:
+ """Extract the primary entity from the topic for grounding checks.
+
+ Strips intent-modifier suffixes (see planner._INTENT_MODIFIER_PATTERNS),
+ trims trailing punctuation, collapses whitespace. Returns the empty
+ string for topics that are all intent modifier with no entity, so
+ callers can skip the grounding check.
+ """
+ stripped = _INTENT_MODIFIER_RE.sub(" ", topic)
+ # Also collapse multiple spaces and strip punctuation.
+ stripped = re.sub(r"\s+", " ", stripped).strip(" \t\r\n?.,:;!")
+ return stripped
def _final_score(candidate: schema.Candidate) -> float:
diff --git a/tests/test_rerank_v3.py b/tests/test_rerank_v3.py
index 1aa2be46..861575ec 100644
--- a/tests/test_rerank_v3.py
+++ b/tests/test_rerank_v3.py
@@ -178,9 +178,83 @@ class RerankV3Tests(unittest.TestCase):
self.assertEqual("gemini-3.1-flash-lite-preview", provider.model)
self.assertEqual(95.0, first.rerank_score)
self.assertEqual("high fit", first.explanation)
- self.assertEqual("fallback-local-score", second.explanation)
+ # Tail is scored via the fallback (may or may not carry the entity-miss
+ # suffix depending on topic-title overlap; assert the base tag is present).
+ self.assertIn("fallback-local-score", second.explanation or "")
self.assertEqual(first.candidate_id, ranked[0].candidate_id)
+class EntityGroundingTests(unittest.TestCase):
+ """Unit 4: Reranker entity-grounding demotion. 2026-04-19 Hermes Agent
+ Use Cases failure: an off-topic video about Claude Managed Agents
+ scored 51 and ranked #2 with zero Hermes content.
+ """
+
+ def _candidate(self, title: str, snippet: str = "") -> schema.Candidate:
+ return schema.Candidate(
+ candidate_id=f"c-{title[:10]}",
+ item_id="i1",
+ source="youtube",
+ title=title,
+ url="https://example.com",
+ snippet=snippet,
+ subquery_labels=["primary"],
+ native_ranks={"primary:youtube": 1},
+ local_relevance=0.8,
+ freshness=80,
+ engagement=50,
+ source_quality=0.7,
+ rrf_score=0.02,
+ )
+
+ def test_primary_entity_strips_intent_modifier(self):
+ self.assertEqual("Hermes Agent", rerank._primary_entity("Hermes Agent use cases"))
+ self.assertEqual("Hermes Agent Actual", rerank._primary_entity("Hermes Agent Actual Use Cases"))
+ self.assertEqual("Claude Code", rerank._primary_entity("Claude Code workflows"))
+ self.assertEqual("DSPy", rerank._primary_entity("DSPy tutorial"))
+
+ def test_primary_entity_leaves_bare_entity_unchanged(self):
+ self.assertEqual("Kanye West", rerank._primary_entity("Kanye West"))
+ self.assertEqual("Nous Research", rerank._primary_entity("Nous Research"))
+
+ def test_fallback_demotes_candidate_without_primary_entity(self):
+ on_topic = self._candidate("Hermes Agent: Self-Improving AI", "Nous Research Hermes walkthrough")
+ off_topic = self._candidate("I Tested Claude's Managed Agents", "What you need to know about Anthropic's new managed agents")
+ rerank._apply_fallback_scores([on_topic, off_topic], primary_entity="Hermes Agent")
+ self.assertGreater(on_topic.final_score, off_topic.final_score)
+ self.assertIn("entity-miss", off_topic.explanation or "")
+ self.assertEqual(on_topic.explanation, "fallback-local-score")
+
+ def test_fallback_match_is_case_insensitive(self):
+ on_topic = self._candidate("HERMES agent rocks", "some text")
+ rerank._apply_fallback_scores([on_topic], primary_entity="Hermes Agent")
+ self.assertEqual("fallback-local-score", on_topic.explanation)
+
+ def test_fallback_skips_demotion_for_empty_text_candidates(self):
+ empty = self._candidate("", "")
+ rerank._apply_fallback_scores([empty], primary_entity="Hermes Agent")
+ self.assertEqual("fallback-local-score", empty.explanation)
+
+ def test_fallback_skips_demotion_when_no_primary_entity(self):
+ off = self._candidate("Completely unrelated", "snippet")
+ rerank._apply_fallback_scores([off], primary_entity="")
+ self.assertEqual("fallback-local-score", off.explanation)
+
+ def test_llm_prompt_includes_primary_entity_grounding_hint(self):
+ candidate = self._candidate("Something", "snippet text")
+ plan = make_plan()
+ prompt = rerank._build_prompt(
+ "Hermes Agent use cases", plan, [candidate], primary_entity="Hermes Agent"
+ )
+ self.assertIn("Primary entity grounding", prompt)
+ self.assertIn("Hermes Agent", prompt)
+
+ def test_llm_prompt_omits_grounding_hint_when_no_primary_entity(self):
+ candidate = self._candidate("Something", "snippet text")
+ plan = make_plan()
+ prompt = rerank._build_prompt("", plan, [candidate], primary_entity="")
+ self.assertNotIn("Primary entity grounding", prompt)
+
+
if __name__ == "__main__":
unittest.main()