Skip to content

AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary

fix: repair remaining test oracles and manual comparison outcomes (#1226)

Ilia Alshanetskyfedb8a94edbd
diff --git a/changelog.d/1226.fixed.md b/changelog.d/1226.fixed.md
new file mode 100644
index 00000000..57cc1fcf
--- /dev/null
+++ b/changelog.d/1226.fixed.md
@@ -0,0 +1,3 @@
+Reject YouTube SSH host aliases beginning with a dash while preserving valid aliases and SSH option termination.
+
+The manual engine comparison tool now uses the canonical engine path, counts nested entity reports, reports command failures separately from source errors, and exits unsuccessfully for failed or partial comparisons, including failures retained only in source status.
diff --git a/skills/last30days/scripts/lib/youtube_yt.py b/skills/last30days/scripts/lib/youtube_yt.py
index 3234d7d1..9bfb10f5 100644
--- a/skills/last30days/scripts/lib/youtube_yt.py
+++ b/skills/last30days/scripts/lib/youtube_yt.py
@@ -296,7 +296,7 @@ def is_ytdlp_installed() -> bool:
# Host aliases must be plain hostnames / SSH config aliases — no flags, no
# shell metacharacters. Rejects any value that could be reinterpreted by ssh
# (or the surrounding shell) as something other than a destination.
-_SSH_HOST_ALIAS_RE = re.compile(r"^[a-zA-Z0-9._-]+$")
+_SSH_HOST_ALIAS_RE = re.compile(r"^(?!-)[a-zA-Z0-9._-]+$")
def _ytdlp_ssh_host() -> Optional[str]:
diff --git a/tests/e2e_comparison.py b/tests/e2e_comparison.py
index c346f346..199b9828 100644
--- a/tests/e2e_comparison.py
+++ b/tests/e2e_comparison.py
@@ -5,6 +5,9 @@ Usage:
python3 tests/e2e_comparison.py [--v2-script PATH]
Outputs a markdown comparison table with per-query metrics.
+Passed comparisons exit 0; failed or partial comparisons exit 1, including
+otherwise successful commands that report source errors.
+Comparison totals sum the metrics of each entity report.
"""
import json
@@ -14,7 +17,7 @@ import time
from pathlib import Path
REPO = Path(__file__).resolve().parents[1]
-V3_SCRIPT = str(REPO / "scripts" / "last30days.py")
+V3_SCRIPT = str(REPO / "skills" / "last30days" / "scripts" / "last30days.py")
# v2.9.5 from plugin cache (main branch equivalent)
V2_SCRIPT = str(
@@ -41,6 +44,26 @@ def _load_queries() -> list[tuple[str, str]]:
QUERIES = _load_queries()
+def _v3_metrics(data: dict) -> dict:
+ items_by_source = data.get("items_by_source", {})
+ errors = list(data.get("errors_by_source", {}))
+ for source, outcome in data.get("source_status", {}).items():
+ if outcome.get("state") in {
+ "partial", "rate-limited", "auth-failed", "payment-required",
+ "unreachable", "timeout", "schema-drift", "error",
+ } and source not in errors:
+ errors.append(source)
+ return {
+ "sources": sum(1 for items in items_by_source.values() if items),
+ "total_items": sum(len(items) for items in items_by_source.values()),
+ "candidates": len(data.get("ranked_candidates", [])),
+ "clusters": len(data.get("clusters", [])),
+ "intent": data["query_plan"].get("intent", "?"),
+ "subqueries": len(data["query_plan"].get("subqueries", [])),
+ "errors": errors,
+ }
+
+
def run_query(script: str, topic: str, timeout: int = 180) -> dict:
"""Run a query and return parsed JSON + timing."""
start = time.time()
@@ -54,7 +77,7 @@ def run_query(script: str, topic: str, timeout: int = 180) -> dict:
elapsed = time.time() - start
if result.returncode != 0:
return {
- "error": result.stderr[:200],
+ "error": result.stderr[:200] or f"engine exited with status {result.returncode}",
"elapsed": elapsed,
"sources": 0,
"candidates": 0,
@@ -63,20 +86,23 @@ def run_query(script: str, topic: str, timeout: int = 180) -> dict:
}
data = json.loads(result.stdout)
- # v3 shape
- if "query_plan" in data:
- items_by_source = data.get("items_by_source", {})
+ if data.get("comparison"):
+ reports = [(entry["entity"], _v3_metrics(entry["report"])) for entry in data["reports"]]
+ if not reports:
+ raise ValueError("comparison output contains no reports")
return {
"elapsed": elapsed,
- "sources": sum(1 for v in items_by_source.values() if v),
- "total_items": sum(len(v) for v in items_by_source.values()),
- "candidates": len(data.get("ranked_candidates", [])),
- "clusters": len(data.get("clusters", [])),
- "intent": data["query_plan"].get("intent", "?"),
- "subqueries": len(data["query_plan"].get("subqueries", [])),
- "errors": list(data.get("errors_by_source", {}).keys()),
+ **{key: sum(metrics[key] for _, metrics in reports) for key in (
+ "sources", "total_items", "candidates", "clusters", "subqueries",
+ )},
+ "intent": "comparison",
+ "errors": [f"{entity}: {error}" for entity, metrics in reports for error in metrics["errors"]],
}
+ # v3 shape
+ if "query_plan" in data:
+ return {"elapsed": elapsed, **_v3_metrics(data)}
+
# v2 shape
sources_with_items = 0
total_items = 0
@@ -118,7 +144,7 @@ def run_query(script: str, topic: str, timeout: int = 180) -> dict:
}
-def main():
+def main() -> int:
v2_script = V2_SCRIPT
if len(sys.argv) > 2 and sys.argv[1] == "--v2-script":
v2_script = sys.argv[2]
@@ -160,18 +186,20 @@ def main():
})
# Print comparison table
- print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 errors | v2 errors |")
- print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|")
+ print("| Query | Intent | v3 sources | v2 sources | v3 items | v2 items | v3 time | v2 time | v3 source errors | v2 source errors | v3 command error | v2 command error |")
+ print("|-------|--------|-----------|-----------|---------|---------|---------|---------|-----------|-----------|-----------|-----------|")
for r in results:
v3, v2 = r["v3"], r["v2"]
v3_err = ", ".join(v3.get("errors", [])) or "-"
v2_err = ", ".join(v2.get("errors", [])) or "-"
+ v3_command_error = str(v3.get("error", "-")).replace("|", "\\|").replace("\n", " ")
+ v2_command_error = str(v2.get("error", "-")).replace("|", "\\|").replace("\n", " ")
print(
f"| {r['topic'][:45]} | {v3.get('intent', '?')} | "
f"{v3.get('sources', 0)} | {v2.get('sources', 0)} | "
f"{v3.get('total_items', 0)} | {v2.get('total_items', 0)} | "
f"{v3.get('elapsed', 0):.1f}s | {v2.get('elapsed', 0):.1f}s | "
- f"{v3_err} | {v2_err} |"
+ f"{v3_err} | {v2_err} | {v3_command_error} | {v2_command_error} |"
)
# Summary
@@ -184,6 +212,14 @@ def main():
v2_total_time = sum(r["v2"].get("elapsed", 0) for r in results)
v3_errors = sum(len(r["v3"].get("errors", [])) for r in results)
v2_errors = sum(len(r["v2"].get("errors", [])) for r in results)
+ v3_command_errors = sum("error" in r["v3"] for r in results)
+ v2_command_errors = sum("error" in r["v2"] for r in results)
+ command_errors = v3_command_errors + v2_command_errors
+ outcome = (
+ "failed" if command_errors == 2 * len(results)
+ else "partial" if command_errors or v3_errors or v2_errors
+ else "passed"
+ )
print("## Summary")
print()
@@ -193,10 +229,13 @@ def main():
print(f"| Total items retrieved | {v3_total_items} | {v2_total_items} | {v3_total_items - v2_total_items:+d} |")
print(f"| Total wall time | {v3_total_time:.1f}s | {v2_total_time:.1f}s | {v3_total_time - v2_total_time:+.1f}s |")
print(f"| Source errors | {v3_errors} | {v2_errors} | {v3_errors - v2_errors:+d} |")
+ print(f"| Command errors | {v3_command_errors} | {v2_command_errors} | {v3_command_errors - v2_command_errors:+d} |")
print(f"| Avg sources/query | {v3_total_sources/len(results):.1f} | {v2_total_sources/len(results):.1f} | |")
print(f"| Avg items/query | {v3_total_items/len(results):.1f} | {v2_total_items/len(results):.1f} | |")
print(f"| Avg time/query | {v3_total_time/len(results):.1f}s | {v2_total_time/len(results):.1f}s | |")
+ print(f"\nOutcome: {outcome}")
+ return 0 if outcome == "passed" else 1
if __name__ == "__main__":
- main()
+ sys.exit(main())
diff --git a/tests/test_amazon.py b/tests/test_amazon.py
index 10c4fe66..73880e0c 100644
--- a/tests/test_amazon.py
+++ b/tests/test_amazon.py
@@ -327,9 +327,13 @@ class TestEnrichmentLane:
def test_reviews_attach_to_the_right_product(self):
out, status = amazon.enrich_with_reviews(
self._products(3), depth="default",
- fetcher=lambda url: {"records": [review_record(2, 5, review_id=url)]},
+ fetcher=lambda url: {"records": [review_record(2, 5, review_text=f"Review for {url}")]},
)
assert all(p.get("top_comments") for p in out)
+ assert {p["asin"]: p["top_comments"][0]["excerpt"] for p in out} == {
+ f"B00000000{i}": f"Review for https://www.amazon.com/dp/B00000000{i}"
+ for i in range(3)
+ }
assert out[0]["product_rating_count"] == 459
assert status is None
diff --git a/tests/test_categories.py b/tests/test_categories.py
index 57c96ca4..8f9340f0 100644
--- a/tests/test_categories.py
+++ b/tests/test_categories.py
@@ -86,11 +86,9 @@ class DetectCategoryEdgeCases(unittest.TestCase):
self.assertIsNone(detect_category(None))
def test_first_match_wins_image_gen_before_chat_model(self):
- # "gpt image 2" contains "gpt image" (ai_image_generation) and the
- # substring "gpt" could resemble gpt-N chat-model patterns. The
- # narrower category wins because it is declared earlier.
+ self.assertEqual(detect_category("gpt-5"), "ai_chat_model")
self.assertEqual(
- detect_category("gpt image 2 review"),
+ detect_category("gpt image and gpt-5"),
"ai_image_generation",
)
diff --git a/tests/test_cjk.py b/tests/test_cjk.py
index 5d24aad9..e703c61b 100644
--- a/tests/test_cjk.py
+++ b/tests/test_cjk.py
@@ -65,12 +65,11 @@ class TestChineseRelevance(_BigramBase):
self.assertGreater(contiguous, scattered)
def test_english_phrase_bonus_stays_space_sensitive(self):
- # has_cjk gate: English must NOT gain a bonus from space-stripped
- # concatenation (no "reacthooks" false phrase match).
q = relevance.PreparedQuery("react hooks")
- # "reacthooks" contiguous-without-space should not trigger a CJK-style retry
- score = relevance.token_overlap_relevance(q, "myreacthooks bundle")
- self.assertLessEqual(score, 1.0) # sanity; behavior identical to pre-change
+ joined = relevance.token_overlap_relevance(q, "react myreacthooks bundle")
+ absent = relevance.token_overlap_relevance(q, "react unrelated bundle")
+ self.assertEqual(joined, 0.58)
+ self.assertEqual(absent, 0.58)
class TestChineseDedupe(_BigramBase):
diff --git a/tests/test_competitor_fanout.py b/tests/test_competitor_fanout.py
index 67b74a1f..9c5b4fe9 100644
--- a/tests/test_competitor_fanout.py
+++ b/tests/test_competitor_fanout.py
@@ -105,24 +105,29 @@ class FanoutOrchestratorTests(unittest.TestCase):
"""Comparison mode must not inherit a prior run's ytsearch cache."""
from lib import youtube_yt
- youtube_yt._search_cache[("prior", 8, "2026-01-01")] = {"items": []}
- cleared = []
+ stale_key = ("prior", 8, "2026-01-01")
+ youtube_yt._search_cache[stale_key] = {"items": []}
+ observed = {}
+ barrier = threading.Barrier(2, timeout=3)
- def main_runner():
- cleared.append("main" in youtube_yt._search_cache or len(youtube_yt._search_cache) == 0)
- return _fake_report("OpenAI")
+ def runner(topic):
+ observed[topic] = stale_key in youtube_yt._search_cache
+ barrier.wait()
+ return _fake_report(topic)
with mock.patch.object(
youtube_yt, "reset_search_cache", wraps=youtube_yt.reset_search_cache
) as reset_mock, redirect_stderr(io.StringIO()):
- fanout.run_competitor_fanout(
+ results = fanout.run_competitor_fanout(
main_topic="OpenAI",
- main_runner=main_runner,
+ main_runner=lambda: runner("OpenAI"),
competitors=["Anthropic"],
- competitor_runner=lambda e: _fake_report(e),
+ competitor_runner=runner,
)
- reset_mock.assert_called()
- self.assertNotIn(("prior", 8, "2026-01-01"), youtube_yt._search_cache)
+ reset_mock.assert_called_once_with()
+ self.assertEqual(observed, {"OpenAI": False, "Anthropic": False})
+ self.assertEqual([label for label, _ in results], ["OpenAI", "Anthropic"])
+ self.assertNotIn(stale_key, youtube_yt._search_cache)
def test_sub_runs_execute_in_parallel(self):
"""Wall clock should be closer to max(latency) than sum(latency)."""
diff --git a/tests/test_competitors_routing.py b/tests/test_competitors_routing.py
index 7214b981..b42dd524 100644
--- a/tests/test_competitors_routing.py
+++ b/tests/test_competitors_routing.py
@@ -8,8 +8,10 @@ from __future__ import annotations
import io
import json
+import sys
import unittest
from contextlib import redirect_stderr
+from unittest.mock import patch
import last30days as cli
from lib import competitors, planner, render
@@ -135,23 +137,17 @@ class TestEntityCapAlignment(unittest.TestCase):
self.assertIn("primary", [sq.label for sq in plan.subqueries])
def test_empty_plan_alone_errors_clearly(self):
- args = _parse("Weber grills", "--competitors-plan", "{}")
- enabled, count, explicit = cli.resolve_competitors_args(args)
- plan = cli.parse_competitors_plan(args.competitors_plan)
- topic, enabled, count, explicit = cli.apply_vs_competitor_routing(
- "Weber grills",
- competitors_flag=args.competitors,
- comp_enabled=enabled,
- comp_count=count,
- comp_explicit=explicit,
- comp_plan=plan,
- )
- self.assertTrue(enabled)
- self.assertEqual(explicit, [])
- # Guard in _main: plan present, no peers, not discover-N
- self.assertIsNone(args.competitors)
- self.assertTrue(args.competitors_plan)
- self.assertFalse(explicit)
+ stderr = io.StringIO()
+ with patch.object(sys, "argv", ["last30days", "Weber grills", "--mock", "--competitors-plan", "{}"]), patch.object(
+ cli.env, "get_config", return_value={}
+ ), patch("lib.competitors.discover_competitors") as discover, patch(
+ "lib.fanout.run_competitor_fanout"
+ ) as fanout, patch.object(cli.pipeline, "run") as research, redirect_stderr(stderr):
+ self.assertEqual(cli.main(), 2)
+ self.assertIn("--competitors-plan has no usable peer entries", stderr.getvalue())
+ discover.assert_not_called()
+ fanout.assert_not_called()
+ research.assert_not_called()
def test_over_max_warns_and_names_dropped(self):
# main + 6 peers = max; 8th entity dropped with warning
diff --git a/tests/test_dedupe_v3.py b/tests/test_dedupe_v3.py
index 32866fcf..1da701c6 100644
--- a/tests/test_dedupe_v3.py
+++ b/tests/test_dedupe_v3.py
@@ -96,10 +96,10 @@ class TestTokenJaccard(unittest.TestCase):
self.assertAlmostEqual(result, 1.0 / 3.0) # {big} / {big, cat, dog}
def test_filters_single_char_tokens(self):
- # Single char tokens like "I" are filtered (len > 1)
- result = dedupe.token_jaccard("I am great", "I am terrible")
- # "am" is len 2, "great"/"terrible" are content
- self.assertGreater(result, 0.0)
+ self.assertAlmostEqual(
+ dedupe.token_jaccard("x shared alpha", "x shared beta"), 1.0 / 3.0
+ )
+ self.assertEqual(dedupe.token_jaccard("x alpha", "x beta"), 0.0)
# ---------------------------------------------------------------------------
# hybrid_similarity
diff --git a/tests/test_e2e_comparison.py b/tests/test_e2e_comparison.py
new file mode 100644
index 00000000..4f7577e7
--- /dev/null
+++ b/tests/test_e2e_comparison.py
@@ -0,0 +1,297 @@
+"""Offline regression checks for the opt-in manual engine comparison."""
+
+import importlib.util
+import json
+import subprocess
+import sys
+from pathlib import Path
+from unittest import mock
+
+import pytest
+
+import last30days as cli
+from lib import pipeline, schema
+
+
+def build_report(topic, source_counts, *, errors=None):
+ items_by_source = {
+ source: [schema.SourceItem(
+ item_id=f"{topic}-{source}-{index}", source=source, title=topic,
+ body="fixture evidence", url=f"https://example.invalid/{source}/{index}",
+ ) for index in range(count)]
+ for source, count in source_counts.items()
+ }
+ candidates = [schema.Candidate(
+ candidate_id=item.item_id, item_id=item.item_id, source=source,
+ title=item.title, url=item.url, snippet=item.body,
+ subquery_labels=[topic], native_ranks={source: index + 1},
+ local_relevance=0.8, freshness=1, engagement=1, source_quality=0.8,
+ rrf_score=0.1,
+ ) for source, items in items_by_source.items() for index, item in enumerate(items)]
+ return schema.Report(
+ topic=topic, range_from="2026-09-06", range_to="2026-10-06",
+ generated_at="2026-10-06T00:00:00Z",
+ provider_runtime=schema.ProviderRuntime("local", "fixture", "fixture"),
+ query_plan=schema.QueryPlan(
+ intent="concept", freshness_mode="balanced_recent", cluster_mode="topic",
+ raw_topic=topic,
+ subqueries=[schema.SubQuery(topic, topic, topic, list(source_counts))],
+ source_weights={source: 1.0 for source in source_counts},
+ ),
+ clusters=[schema.Cluster(
+ cluster_id=topic, title=topic,
+ candidate_ids=[candidate.candidate_id for candidate in candidates],
+ representative_ids=[candidates[0].candidate_id] if candidates else [],
+ sources=list(source_counts), score=0.8,
+ )],
+ ranked_candidates=candidates, items_by_source=items_by_source,
+ errors_by_source=errors or {},
+ source_status={source: schema.SourceOutcome(
+ source=source, state="ok" if items else "no-results", items_returned=len(items),
+ ) for source, items in items_by_source.items()},
+ )
+
+
+def load_comparison_module():
+ path = Path(__file__).with_name("e2e_comparison.py")
+ spec = importlib.util.spec_from_file_location("e2e_comparison_module", path)
+ module = importlib.util.module_from_spec(spec)
+ assert spec and spec.loader
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_current_engine_uses_canonical_path():
+ module = load_comparison_module()
+ assert Path(module.V3_SCRIPT) == (
+ Path(__file__).resolve().parents[1]
+ / "skills" / "last30days" / "scripts" / "last30days.py"
+ )
+ assert Path(module.V3_SCRIPT).is_file()
+
+
[email protected]("source_error", [False, True])
+def test_comparison_output_aggregates_nested_reports_and_errors(tmp_path, capsys, source_error):
+ module = load_comparison_module()
+ reports = [
+ ("alpha", build_report("alpha", {"reddit": 2, "web": 1})),
+ ("beta", build_report("beta", {"reddit": 1, "x": 0},
+ errors={"x": "fixture timeout"} if source_error else None)),
+ ]
+ if source_error:
+ reports[1][1].source_status["x"] = schema.SourceOutcome(source="x", state="timeout")
+ payload = cli.emit_comparison_output(reports, "json", json_profile="raw")
+ result = subprocess.CompletedProcess([], 0, payload, "")
+ reference = tmp_path / "reference.py"
+ reference.touch()
+ with mock.patch.object(module.subprocess, "run", return_value=result), mock.patch.object(
+ module, "QUERIES", [("alpha vs beta", "comparison")]
+ ), mock.patch.object(sys, "argv", ["e2e_comparison", "--v2-script", str(reference)]):
+ metrics = module.run_query("fixture.py", "alpha vs beta")
+ assert metrics["sources"] == 3
+ assert metrics["total_items"] == 4
+ assert metrics["candidates"] == 4
+ assert metrics["clusters"] == 2
+ assert metrics["intent"] == "comparison"
+ assert metrics["subqueries"] == 2
+ assert metrics["errors"] == (["beta: x"] if source_error else [])
+ assert module.main() == int(source_error)
+ output = capsys.readouterr().out
+ assert "| Total items retrieved | 4 | 4 | +0 |" in output
+ assert "| Total sources with items | 3 | 3 | +0 |" in output
+ assert f"Outcome: {'partial' if source_error else 'passed'}" in output
+ if source_error:
+ assert "beta: x" in output
+
+
[email protected]("comparison", [False, True])
+ "partial", "rate-limited", "auth-failed", "payment-required",
+ "unreachable", "timeout", "schema-drift", "error",
+])
+def test_source_status_failures_are_partial_without_legacy_errors(tmp_path, capsys, comparison, state):
+ module = load_comparison_module()
+ report = build_report("alpha", {"reddit": 1 if state in {"partial", "auth-failed"} else 0, "web": 1})
+ bundle = schema.RetrievalBundle()
+ for source, items in report.items_by_source.items():
+ bundle.add_items("alpha", source, items)
+ bundle.record_failure("reddit", state, "fixture retrieval failure")
+ report.source_status = pipeline._finalize_source_status(bundle.source_status, report.items_by_source)
+ report.warnings = pipeline._warnings(
+ report.items_by_source, report.ranked_candidates, {}, {"reddit": "fixture retrieval failure"},
+ )
+ assert report.errors_by_source == {}
+ assert report.source_status["reddit"].state == state
+ payload = (
+ cli.emit_comparison_output([("alpha", report)], "json", json_profile="raw")
+ if comparison else json.dumps(schema.to_dict(report))
+ )
+ result = subprocess.CompletedProcess([], 0, payload, "")
+ reference = tmp_path / "reference.py"
+ reference.touch()
+ with mock.patch.object(module.subprocess, "run", return_value=result), mock.patch.object(
+ module, "QUERIES", [("fixture topic", "concept")]
+ ), mock.patch.object(sys, "argv", ["e2e_comparison", "--v2-script", str(reference)]):
+ metrics = module.run_query("fixture.py", "fixture topic")
+ assert metrics["errors"] == (["alpha: reddit"] if comparison else ["reddit"])
+ assert module.main() == 1
+ output = capsys.readouterr().out
+ assert "| Source errors | 1 | 1 | +0 |" in output
+ assert "Outcome: partial" in output
+
+
[email protected]("state", ["ok", "no-results", "skipped-unconfigured"])
+def test_benign_source_status_and_warnings_remain_successful(tmp_path, capsys, state):
+ module = load_comparison_module()
+ report = build_report("alpha", {"reddit": 1 if state == "ok" else 0, "web": 1})
+ report.source_status["reddit"] = schema.SourceOutcome(
+ source="reddit", state=state, items_returned=len(report.items_by_source["reddit"]),
+ attempted=state != "skipped-unconfigured", detail="Informational coverage note",
+ lane_failure_state="timeout" if state == "ok" else None,
+ )
+ report.warnings = ["Evidence is thin for this topic.", "Informational coverage note"]
+ result = subprocess.CompletedProcess([], 0, json.dumps(schema.to_dict(report)), "")
+ reference = tmp_path / "reference.py"
+ reference.touch()
+ with mock.patch.object(module.subprocess, "run", return_value=result), mock.patch.object(
+ module, "QUERIES", [("fixture topic", "concept")]
+ ), mock.patch.object(sys, "argv", ["e2e_comparison", "--v2-script", str(reference)]):
+ assert module.main() == 0
+ output = capsys.readouterr().out
+ assert "| Source errors | 0 | 0 | +0 |" in output
+ assert "Outcome: passed" in output
+
+
+def test_source_error_is_counted_once_when_both_channels_report_it():
+ module = load_comparison_module()
+ report = build_report("alpha", {"reddit": 0, "web": 1}, errors={"reddit": "fixture timeout"})
+ report.source_status["reddit"] = schema.SourceOutcome(source="reddit", state="timeout")
+ result = subprocess.CompletedProcess([], 0, json.dumps(schema.to_dict(report)), "")
+ with mock.patch.object(module.subprocess, "run", return_value=result):
+ assert module.run_query("fixture.py", "fixture topic")["errors"] == ["reddit"]
+
+
+def test_legacy_v2_success_retains_source_and_item_counts():
+ module = load_comparison_module()
+ payload = {"mode": "legacy", "reddit": [{"id": "one"}, {"id": "two"}], "web": [{"id": "three"}]}
+ result = subprocess.CompletedProcess([], 0, json.dumps(payload), "")
+ with mock.patch.object(module.subprocess, "run", return_value=result):
+ metrics = module.run_query("fixture.py", "fixture topic")
+ assert metrics["sources"] == 2
+ assert metrics["total_items"] == 3
+ assert metrics["candidates"] == 3
+ assert metrics["intent"] == "legacy"
+ assert metrics["errors"] == []
+
+
[email protected]("comparison", [False, True])
+def test_process_exits_nonzero_for_partial_source_status(tmp_path, comparison):
+ report = build_report("alpha", {"reddit": 1})
+ bundle = schema.RetrievalBundle()
+ bundle.add_items("alpha", "reddit", report.items_by_source["reddit"])
+ bundle.record_failure("reddit", "rate-limited", "fixture retrieval failure")
+ report.source_status = pipeline._finalize_source_status(bundle.source_status, report.items_by_source)
+ payload = (
+ cli.emit_comparison_output([("alpha", report)], "json", json_profile="raw")
+ if comparison else json.dumps(schema.to_dict(report))
+ )
+ runner = tmp_path / "comparison_runner.py"
+ tool = Path(__file__).with_name("e2e_comparison.py").resolve()
+ reference = tmp_path / "reference.py"
+ reference.touch()
+ runner.write_text(
+ "import runpy, subprocess\n"
+ "from unittest.mock import patch\n"
+ f"result = subprocess.CompletedProcess([], 0, {payload!r}, '')\n"
+ "with patch('subprocess.run', return_value=result):\n"
+ f" runpy.run_path({str(tool)!r}, run_name='__main__')\n"
+ )
+ result = subprocess.run(
+ [sys.executable, str(runner), "--v2-script", str(reference)],
+ capture_output=True, text=True, timeout=10,
+ )
+ assert result.returncode == 1, result.stderr
+ assert "Outcome: partial" in result.stdout
+ expected_source = "alpha: reddit" if comparison else "reddit"
+ assert expected_source in result.stdout
+
+
[email protected]("source_errors,expected_exit,outcome", [
+ ({}, 0, "passed"), ({"reddit": "rate-limited"}, 1, "partial"),
+])
+def test_reports_source_outcomes(tmp_path, capsys, source_errors, expected_exit, outcome):
+ module = load_comparison_module()
+ engine = tmp_path / "fixture_engine.py"
+ payload = {
+ "query_plan": {"intent": "concept", "subqueries": [{}]},
+ "items_by_source": {"reddit": [{"id": "fixture"}]},
+ "ranked_candidates": [{"id": "fixture"}],
+ "errors_by_source": source_errors,
+ }
+ engine.write_text("print(" + repr(json.dumps(payload)) + ")\n")
+ with mock.patch.object(module, "V3_SCRIPT", str(engine)), mock.patch.object(
+ module, "QUERIES", [("fixture topic", "concept")]
+ ), mock.patch.object(sys, "argv", ["e2e_comparison", "--v2-script", str(engine)]):
+ assert module.main() == expected_exit
+ output = capsys.readouterr().out
+ assert "| Total items retrieved | 1 | 1 | +0 |" in output
+ assert "| Command errors | 0 | 0 | +0 |" in output
+ assert f"| Source errors | {len(source_errors)} | {len(source_errors)} | +0 |" in output
+ assert f"Outcome: {outcome}" in output
+
+
[email protected]("failed_engines,outcome", [(2, "failed"), (1, "partial")])
+def test_command_failures_are_visible_and_fail(tmp_path, capsys, failed_engines, outcome):
+ module = load_comparison_module()
+ reference = tmp_path / "reference.py"
+ reference.touch()
+ failure = subprocess.CompletedProcess([], 3, "", "engine-failure-sentinel")
+ success = subprocess.CompletedProcess([], 0, '{"reddit":[{"id":"fixture"}]}', "")
+ outcomes = [failure, failure if failed_engines == 2 else success]
+ with mock.patch.object(module.subprocess, "run", side_effect=outcomes) as run, mock.patch.object(
+ module, "QUERIES", [("fixture topic", "concept")]
+ ), mock.patch.object(sys, "argv", ["e2e_comparison", "--v2-script", str(reference)]):
+ assert module.main() == 1
+ assert run.call_count == 2
+ output = capsys.readouterr().out
+ other_errors = failed_engines - 1
+ assert f"| Command errors | 1 | {other_errors} | {1 - other_errors:+d} |" in output
+ assert "engine-failure-sentinel" in output
+ assert f"Outcome: {outcome}" in output
+
+
[email protected]("result,expected_error", [
+ (subprocess.CompletedProcess([], 4, "", ""), "engine exited with status 4"),
+ (subprocess.CompletedProcess([], 0, "not-json", ""), "Expecting value"),
+ (subprocess.CompletedProcess([], 0, '{"comparison":true,"reports":[]}', ""), "comparison output contains no reports"),
+ (subprocess.TimeoutExpired("fixture", 7), "timeout"),
+])
+def test_invalid_output_and_timeout_are_command_errors(result, expected_error):
+ module = load_comparison_module()
+ behavior = {"side_effect": result} if isinstance(result, Exception) else {"return_value": result}
+ with mock.patch.object(module.subprocess, "run", **behavior) as run:
+ report = module.run_query("fixture.py", "fixture topic", timeout=7)
+ assert expected_error in report["error"]
+ assert report["sources"] == 0
+ assert report["candidates"] == 0
+ assert run.call_args.kwargs["timeout"] == 7
+
+
+def test_process_exits_nonzero_when_engines_fail(tmp_path):
+ runner = tmp_path / "comparison_runner.py"
+ tool = Path(__file__).with_name("e2e_comparison.py").resolve()
+ reference = tmp_path / "reference.py"
+ reference.touch()
+ runner.write_text(
+ "import runpy, subprocess\n"
+ "from unittest.mock import patch\n"
+ "failure = subprocess.CompletedProcess([], 3, '', 'fixture-failure')\n"
+ "with patch('subprocess.run', return_value=failure):\n"
+ f" runpy.run_path({str(tool)!r}, run_name='__main__')\n"
+ )
+ result = subprocess.run(
+ [sys.executable, str(runner), "--v2-script", str(reference)],
+ capture_output=True, text=True, timeout=10,
+ )
+ assert result.returncode == 1, result.stderr
+ assert "Outcome: failed" in result.stdout
diff --git a/tests/test_footer_nudge_suppression.py b/tests/test_footer_nudge_suppression.py
index abc15ed8..d32bb20b 100644
--- a/tests/test_footer_nudge_suppression.py
+++ b/tests/test_footer_nudge_suppression.py
@@ -69,6 +69,8 @@ class FooterNudgeSuppressionTests(unittest.TestCase):
def test_bare_run_emits_web_promo(self):
result = self._run(topic="OpenAI")
+ self.assertEqual(result.returncode, 0, msg=result.stderr)
+ self.assertRegex(result.stdout, r"(?m)^# last30days v[^\n]+: OpenAI$")
combined = result.stdout + result.stderr
# Mock mode still shows the promo when nothing indicates a hosting
# model is driving. Check both streams since the UI may emit to stderr.
@@ -81,6 +83,9 @@ class FooterNudgeSuppressionTests(unittest.TestCase):
'{"Anthropic":{"x_handle":"AnthropicAI","subreddits":["ClaudeAI"]}}',
topic="OpenAI",
)
+ self.assertEqual(result.returncode, 0, msg=result.stderr)
+ self.assertRegex(result.stdout, r"(?m)^# last30days v[^\n]+: OpenAI vs Anthropic$")
+ self.assertIn("- Comparison mode: 2 entities (OpenAI, Anthropic)", result.stdout)
combined = result.stdout + result.stderr
self.assertNotIn(
"unlock native grounded web search",
@@ -96,6 +101,9 @@ class FooterNudgeSuppressionTests(unittest.TestCase):
'"sources":["grounding"]}],"source_weights":{"grounding":1.0}}'
)
result = self._run("--plan", plan, topic="OpenAI")
+ self.assertEqual(result.returncode, 0, msg=result.stderr)
+ self.assertRegex(result.stdout, r"(?m)^# last30days v[^\n]+: OpenAI$")
+ self.assertIn("## Stats", result.stdout)
combined = result.stdout + result.stderr
self.assertNotIn(
"unlock native grounded web search",
diff --git a/tests/test_github.py b/tests/test_github.py
index 1dea7035..720e3503 100644
--- a/tests/test_github.py
+++ b/tests/test_github.py
@@ -270,7 +270,22 @@ class TestParseGithubResponse(unittest.TestCase):
self.assertEqual(items[0]["title"], "In window")
def test_sorts_by_relevance(self):
- items = github.parse_github_response(self._RAW_ENVELOPE)
+ envelope = {
+ "items": [
+ {"html_url": "https://github.com/foo/bar/issues/1",
+ "title": "Unrelated work", "comments": 0,
+ "reactions": {"total_count": 0}},
+ {"html_url": "https://github.com/foo/bar/issues/2",
+ "title": "foo", "comments": 100,
+ "reactions": {"total_count": 500}},
+ ],
+ "context": {"core": "foo"},
+ }
+ items = github.parse_github_response(envelope)
+ self.assertEqual(
+ [item["url"] for item in items],
+ ["https://github.com/foo/bar/issues/2", "https://github.com/foo/bar/issues/1"],
+ )
scores = [i.get("relevance", 0) for i in items]
self.assertEqual(scores, sorted(scores, reverse=True))
diff --git a/tests/test_hackernews.py b/tests/test_hackernews.py
index f2b79552..d1f5398a 100644
--- a/tests/test_hackernews.py
+++ b/tests/test_hackernews.py
@@ -412,18 +412,18 @@ def test_parse_hackernews_response_relevance_scoring():
def test_parse_hackernews_response_engagement_boost():
"""Test that high-engagement items get relevance boost."""
- response = {
- "hits": [
- create_mock_hit(object_id="1", points=500, num_comments=200), # High engagement
- create_mock_hit(object_id="2", points=10, num_comments=5), # Low engagement
- ]
- }
-
- items = hackernews.parse_hackernews_response(response, query="test")
-
- # Verify engagement is captured
- assert items[0]["engagement"]["points"] == 500
- assert items[1]["engagement"]["points"] == 10
+ high = hackernews.parse_hackernews_response(
+ {"hits": [create_mock_hit(object_id="story", points=500, num_comments=5)]},
+ query="",
+ )[0]
+ low = hackernews.parse_hackernews_response(
+ {"hits": [create_mock_hit(object_id="story", points=10, num_comments=5)]},
+ query="",
+ )[0]
+
+ assert high["engagement"]["points"] == 500
+ assert low["engagement"]["points"] == 10
+ assert high["relevance"] > low["relevance"]
def test_parse_hackernews_response_prefix_filtering():
diff --git a/tests/test_http_v3.py b/tests/test_http_v3.py
index 589a8e20..0339465a 100644
--- a/tests/test_http_v3.py
+++ b/tests/test_http_v3.py
@@ -295,12 +295,11 @@ class TestDNSResolutionRetry(unittest.TestCase):
mock_urlopen.side_effect = err
with self.assertRaises(http.HTTPError):
- http.request("GET", "http://nonexistent.example", retries=3)
+ http.request("GET", "http://nonexistent.example", retries=4)
- # Expected sleep calls: 1s (after attempt 1), 2s (after attempt 2).
- # No sleep after the final attempt (the loop exits to raise).
sleep_delays = [call.args[0] for call in mock_sleep.call_args_list]
- self.assertEqual(sleep_delays, [1, 2])
+ self.assertEqual(sleep_delays, [1, 2, 4])
+ self.assertEqual(mock_urlopen.call_count, 4)
@patch("lib.http.urllib.request.urlopen")
@patch("lib.http.time.sleep")
diff --git a/tests/test_internals_v3.py b/tests/test_internals_v3.py
index 49b5d765..d39803b7 100644
--- a/tests/test_internals_v3.py
+++ b/tests/test_internals_v3.py
@@ -464,11 +464,7 @@ class TestGenericEngagementFormatter(unittest.TestCase):
def test_xiaohongshu_engagement_not_garbled(self):
item = _item(source="xiaohongshu", engagement={"likes": 500, "views": 10000})
result = render._format_engagement(item)
- if result is not None:
- self.assertNotIn("likes500", result, "Key used as value prefix")
- self.assertNotIn("views10000", result, "Key used as value prefix")
- # Should contain numeric values, not dict keys as numbers
- self.assertIn("500", result)
+ self.assertEqual(result, "[500likes, 10,000views]")
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_jobs_source.py b/tests/test_jobs_source.py
index 26492459..a88ca658 100644
--- a/tests/test_jobs_source.py
+++ b/tests/test_jobs_source.py
@@ -96,7 +96,7 @@ class JobsSourceTests(unittest.TestCase):
self.assertEqual(1, len(parsed))
self.assertEqual("Staff Engineer", parsed[0]["title"])
self.assertEqual("Engineering", parsed[0]["department"])
- self.assertRegex(parsed[0]["date"], r"\d{4}-\d{2}-\d{2}")
+ self.assertEqual("2025-06-10", parsed[0]["date"])
self.assertEqual("lever", parsed[0]["provider"])
def test_parse_smartrecruiters_response_core_fields(self):
diff --git a/tests/test_pipeline_v3.py b/tests/test_pipeline_v3.py
index 6dffffab..967125e1 100644
--- a/tests/test_pipeline_v3.py
+++ b/tests/test_pipeline_v3.py
@@ -1620,91 +1620,55 @@ class TestSupplementalSearches(unittest.TestCase):
self.assertEqual(pipeline.FROM_LANE_COUNT_PER, from_call.kwargs.get("count_per"))
self.assertEqual(pipeline.MENTION_LANE_COUNT_PER, mock_mentions.call_args.kwargs.get("count_per"))
- def test_phase2_skipped_in_quick_mode(self):
- """_run_supplemental_searches should return immediately when depth='quick'."""
- bundle = schema.RetrievalBundle()
- bundle.items_by_source["x"] = [
- _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
- ]
+ def _assert_phase2_guard(self, **blocked):
+ for allowed in (False, True):
+ with self.subTest(allowed=allowed):
+ bundle = schema.RetrievalBundle()
+ bundle.items_by_source["x"] = [
+ _make_source_item("x", "X1", "https://x.com/subject1/status/1", author="subject1"),
+ ]
+ options = dict(depth="default", mock=False, rate_limited_sources=set(), backend="bird")
+ if not allowed:
+ options.update(blocked)
+ backend = options.pop("backend")
+ with patch("lib.env.x_backend_chain", return_value=[]), patch(
+ "lib.entity_extract.extract_entities",
+ return_value={"x_handles": [], "x_hashtags": [], "reddit_subreddits": []},
+ ), patch("lib.bird_x.search_handles", return_value=[{
+ "id": "guard-control", "text": "subject1 test update",
+ "url": "https://x.com/subject1/status/2", "author_handle": "subject1",
+ "date": "2026-03-15", "engagement": {"likes": 12}, "relevance": 1.0,
+ }]) as handles, patch("lib.bird_x.search_mentions", return_value=[]) as mentions:
+ pipeline._run_supplemental_searches(
+ topic="subject1", bundle=bundle, plan=_make_plan("subject1"),
+ config={}, date_range=("2026-02-15", "2026-03-17"),
+ runtime=_make_runtime(backend), rate_limit_lock=threading.Lock(),
+ x_handle="subject1", **options,
+ )
+ if allowed:
+ self.assertEqual(handles.call_count, 1)
+ self.assertEqual(handles.call_args.args, (["subject1"], "subject1", "2026-02-15"))
+ self.assertEqual(mentions.call_count, 1)
+ self.assertEqual(
+ [i.url for i in bundle.items_by_source["x"]],
+ ["https://x.com/subject1/status/1", "https://x.com/subject1/status/2"],
+ )
+ else:
+ handles.assert_not_called()
+ mentions.assert_not_called()
+ self.assertEqual([i.item_id for i in bundle.items_by_source["x"]], ["X1"])
- # If it tries to import entity_extract, that's fine -- it should return before calling it
- pipeline._run_supplemental_searches(
- topic="test",
- bundle=bundle,
- plan=_make_plan(),
- config={},
- depth="quick",
- date_range=("2026-02-15", "2026-03-17"),
- runtime=_make_runtime("bird"),
- mock=False,
- rate_limited_sources=set(),
- rate_limit_lock=threading.Lock(),
- )
- # Bundle should be unchanged (only original item)
- self.assertEqual(len(bundle.items_by_source["x"]), 1)
+ def test_phase2_skipped_in_quick_mode(self):
+ self._assert_phase2_guard(depth="quick")
def test_phase2_skipped_in_mock_mode(self):
- """_run_supplemental_searches should return immediately when mock=True."""
- bundle = schema.RetrievalBundle()
- bundle.items_by_source["x"] = [
- _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
- ]
-
- pipeline._run_supplemental_searches(
- topic="test",
- bundle=bundle,
- plan=_make_plan(),
- config={},
- depth="default",
- date_range=("2026-02-15", "2026-03-17"),
- runtime=_make_runtime("bird"),
- mock=True,
- rate_limited_sources=set(),
- rate_limit_lock=threading.Lock(),
- )
- self.assertEqual(len(bundle.items_by_source["x"]), 1)
+ self._assert_phase2_guard(mock=True)
def test_phase2_skipped_when_x_rate_limited(self):
- """_run_supplemental_searches should skip when X is rate-limited."""
- bundle = schema.RetrievalBundle()
- bundle.items_by_source["x"] = [
- _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
- ]
-
- pipeline._run_supplemental_searches(
- topic="test",
- bundle=bundle,
- plan=_make_plan(),
- config={},
- depth="default",
- date_range=("2026-02-15", "2026-03-17"),
- runtime=_make_runtime("bird"),
- mock=False,
- rate_limited_sources={"x"},
- rate_limit_lock=threading.Lock(),
- )
- self.assertEqual(len(bundle.items_by_source["x"]), 1)
+ self._assert_phase2_guard(rate_limited_sources={"x"})
- def test_phase2_skipped_when_backend_not_bird(self):
- """_run_supplemental_searches should skip when X backend is not bird."""
- bundle = schema.RetrievalBundle()
- bundle.items_by_source["x"] = [
- _make_source_item("x", "X1", "https://x.com/a/1", author="someone"),
- ]
-
- pipeline._run_supplemental_searches(
- topic="test",
- bundle=bundle,
- plan=_make_plan(),
- config={},
- depth="default",
- date_range=("2026-02-15", "2026-03-17"),
- runtime=_make_runtime("xai"),
- mock=False,
- rate_limited_sources=set(),
- rate_limit_lock=threading.Lock(),
- )
- self.assertEqual(len(bundle.items_by_source["x"]), 1)
+ def test_phase2_skipped_when_backend_has_no_handle_support(self):
+ self._assert_phase2_guard(backend="xai")
class TestThinSourceRetry(unittest.TestCase):
@@ -1771,6 +1735,18 @@ class TestThinSourceRetry(unittest.TestCase):
self.assertIn("reddit", call_sources)
# X should NOT have been retried
self.assertNotIn("x", call_sources)
+ mock_retrieve.assert_called_once()
+ retry = mock_retrieve.call_args.kwargs["subquery"]
+ self.assertEqual(retry.search_query, "advanced ai safety")
+ self.assertEqual(
+ retry.ranking_query,
+ "What recent evidence from the last 30 days matters for advanced ai safety?",
+ )
+ self.assertEqual(retry.sources, ["reddit"])
+ self.assertEqual(
+ [i.url for i in bundle.items_by_source["reddit"]],
+ ["https://reddit.com/r/test/1", "https://reddit.com/r/test/2"],
+ )
def test_sources_with_enough_items_not_retried(self):
"""Sources with >= 3 items should not be retried."""
@@ -1968,17 +1944,31 @@ class TestXHandleFlag(unittest.TestCase):
self.assertIn("x_handle", sig.parameters, "pipeline.run() must accept x_handle parameter")
def test_x_handle_passed_to_supplemental_searches(self):
- """When x_handle is provided, it should trigger targeted handle search."""
- # Run pipeline in mock mode with x_handle -- should not raise
- report = pipeline.run(
- topic="test topic",
- config={"LAST30DAYS_REASONING_PROVIDER": "gemini"},
- depth="quick",
- requested_sources=["reddit", "x", "grounding"],
- mock=True,
- x_handle="testuser",
- )
- self.assertEqual("test topic", report.topic)
+ with patch("lib.providers.resolve_runtime", return_value=(_make_runtime(), None)), patch(
+ "lib.pipeline.available_sources", return_value=["x"],
+ ), patch("lib.pipeline._fetch_x_backend", return_value=([], None)), patch(
+ "lib.env.x_backend_chain", return_value=["bird"],
+ ), patch("lib.entity_extract.extract_entities", return_value={
+ "x_handles": [], "x_hashtags": [], "reddit_subreddits": [],
+ }), patch("lib.bird_x.search_handles", return_value=[{
+ "id": "explicit-handle", "text": "test topic from testuser",
+ "url": "https://x.com/testuser/status/123", "author_handle": "testuser",
+ "date": "2026-03-15", "engagement": {"likes": 15}, "relevance": 1.0,
+ }]) as handles, patch("lib.bird_x.search_mentions", return_value=[]):
+ report = pipeline.run(
+ topic="test topic", config={}, depth="default", requested_sources=["x"],
+ mock=False, x_handle="testuser", web_backend="none", as_of_date="2026-03-17",
+ external_plan={
+ "intent": "exploration", "freshness_mode": "balanced_recent", "cluster_mode": "topic",
+ "subqueries": [{"label": "primary", "search_query": "test topic",
+ "ranking_query": "test topic", "sources": ["x"]}],
+ },
+ )
+ handles.assert_called_once()
+ self.assertEqual(handles.call_args.args, (["testuser"], "test topic", "2026-02-15"))
+ self.assertEqual(handles.call_args.kwargs["to_date"], "2026-03-17")
+ self.assertEqual(handles.call_args.kwargs["count_per"], pipeline.FROM_LANE_COUNT_PER)
+ self.assertEqual([i.url for i in report.items_by_source["x"]], ["https://x.com/testuser/status/123"])
class TestWarnings(unittest.TestCase):
diff --git a/tests/test_planner_quiet_mode.py b/tests/test_planner_quiet_mode.py
index 8068080a..6a331777 100644
--- a/tests/test_planner_quiet_mode.py
+++ b/tests/test_planner_quiet_mode.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import io
import unittest
from contextlib import redirect_stderr
+from unittest.mock import Mock
from lib import planner
@@ -39,10 +40,25 @@ class PlannerQuietModeTests(unittest.TestCase):
def test_internal_subrun_still_allows_other_warnings(self):
"""Quiet mode only silences the LAW 7 block, not all planner output."""
- plan, _stderr = self._call(internal_subrun=True)
- # The plan itself is deterministic fallback; verify note carries
- # no planner-error indication.
- self.assertGreater(len(plan.subqueries), 0)
+ provider = Mock()
+ provider.generate_json.side_effect = ValueError("invalid-plan-sentinel")
+ err = io.StringIO()
+ with redirect_stderr(err):
+ plan = planner.plan_query(
+ topic="Acme Corp",
+ available_sources=["grounding", "reddit"],
+ requested_sources=None,
+ depth="default",
+ provider=provider,
+ model="test-model",
+ internal_subrun=True,
+ )
+ provider.generate_json.assert_called_once()
+ self.assertTrue(plan.subqueries)
+ self.assertIn("LLM planning failed", err.getvalue())
+ self.assertIn("ValueError: invalid-plan-sentinel", err.getvalue())
+ self.assertNotIn("No --plan passed", err.getvalue())
+ self.assertNotIn("YOU ARE the planner", err.getvalue())
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_query.py b/tests/test_query.py
index b8048550..3b4e5eb4 100644
--- a/tests/test_query.py
+++ b/tests/test_query.py
@@ -41,9 +41,8 @@ class TestExtractCoreSubject(unittest.TestCase):
self.assertTrue(len(result) > 0)
def test_only_first_prefix_stripped(self):
- # "how to" should match, stripping once, not recursively
- result = extract_core_subject("how to use how to debug")
- self.assertIn("debug", result)
+ result = extract_core_subject("how to use how to debug", noise=frozenset())
+ self.assertEqual(result, "how to debug")
class TestMaxWords(unittest.TestCase):
@@ -51,15 +50,14 @@ class TestMaxWords(unittest.TestCase):
def test_max_words_caps_output(self):
result = extract_core_subject(
- "multi agent reinforcement learning framework",
+ "multi agent reinforcement learning framework distributed training",
max_words=5,
)
- self.assertLessEqual(len(result.split()), 5)
+ self.assertEqual(result, "multi agent reinforcement learning framework")
def test_max_words_none_no_cap(self):
result = extract_core_subject("cursor IDE react native components")
- # Without max_words, no cap applied
- self.assertGreaterEqual(len(result.split()), 3)
+ self.assertEqual(result, "cursor ide react native components")
def test_max_words_fallback_on_empty(self):
# All words filtered + max_words should fall back to original
@@ -74,21 +72,21 @@ class TestStripSuffixes(unittest.TestCase):
result = extract_core_subject(
"claude code best practices",
strip_suffixes=True,
+ noise=frozenset(),
)
- self.assertNotIn("practices", result)
+ self.assertEqual(result, "claude code")
def test_strips_use_cases(self):
result = extract_core_subject(
"react hooks use cases",
strip_suffixes=True,
+ noise=frozenset(),
)
- self.assertNotIn("cases", result)
+ self.assertEqual(result, "react hooks")
def test_no_strip_without_flag(self):
- result = extract_core_subject("claude code best practices")
- # "best" and "practices" are noise words so they get filtered anyway
- # but the suffix phase doesn't run
- self.assertIn("claude", result)
+ result = extract_core_subject("claude code best practices", noise=frozenset())
+ self.assertEqual(result, "claude code best practices")
class TestCustomNoise(unittest.TestCase):
diff --git a/tests/test_reddit_enrich.py b/tests/test_reddit_enrich.py
index 7dc1b740..e4ab54e3 100644
--- a/tests/test_reddit_enrich.py
+++ b/tests/test_reddit_enrich.py
@@ -109,13 +109,15 @@ class TestExtractCommentInsights(unittest.TestCase):
self.assertEqual(len(insights), 1)
def test_filters_low_value_patterns(self):
+ substantive = "Measure the query latency before choosing a caching strategy."
comments = [
- {"body": "This."},
- {"body": "lol that's hilarious"},
- {"body": "A" * 50 + " Here's a real insight about how to approach this problem."},
+ {"body": "lol " + "that is hilarious " * 3},
+ {"body": "[deleted] " + "unavailable content " * 3},
+ {"body": "[removed] " + "unavailable content " * 3},
+ {"body": substantive},
]
insights = reddit_enrich.extract_comment_insights(comments)
- self.assertEqual(len(insights), 1)
+ self.assertEqual(insights, [substantive])
def test_respects_limit(self):
comments = [{"body": f"Comment number {i} " + "x" * 50} for i in range(20)]
diff --git a/tests/test_reddit_keyless_backoff.py b/tests/test_reddit_keyless_backoff.py
index a41fc9cb..e49708e4 100644
--- a/tests/test_reddit_keyless_backoff.py
+++ b/tests/test_reddit_keyless_backoff.py
@@ -42,12 +42,20 @@ class TestRateLimiter:
class TestRedditKeylessGetText:
def test_acquires_limiter_then_delegates(self):
+ calls = mock.Mock()
with mock.patch.object(http.REDDIT_KEYLESS_LIMITER, "acquire") as acq, \
mock.patch.object(http, "get_text", return_value="body") as gt:
+ calls.attach_mock(acq, "acquire")
+ calls.attach_mock(gt, "get_text")
out = http.reddit_keyless_get_text("https://www.reddit.com/svc/shreddit/search/?q=x", accept="text/html")
assert out == "body"
- acq.assert_called_once()
- gt.assert_called_once()
+ assert calls.mock_calls == [
+ mock.call.acquire(),
+ mock.call.get_text(
+ "https://www.reddit.com/svc/shreddit/search/?q=x",
+ timeout=http.DEFAULT_TIMEOUT, retries=2, accept="text/html", headers=None,
+ ),
+ ]
class TestRedditKeylessRateKnob:
diff --git a/tests/test_reddit_sc.py b/tests/test_reddit_sc.py
index 80d7ed61..01cb2f0b 100644
--- a/tests/test_reddit_sc.py
+++ b/tests/test_reddit_sc.py
@@ -74,12 +74,12 @@ class TestDiscoverSubreddits(unittest.TestCase):
def test_ranks_by_frequency(self):
results = [
+ {"subreddit": "python", "score": 10},
+ {"subreddit": "programming", "score": 10},
{"subreddit": "programming", "score": 10},
- {"subreddit": "programming", "score": 20},
- {"subreddit": "python", "score": 5},
]
subs = reddit.discover_subreddits(results, max_subs=5)
- self.assertEqual(subs[0], "programming")
+ self.assertEqual(subs, ["programming", "python"])
def test_utility_sub_penalty(self):
results = [
@@ -87,24 +87,24 @@ class TestDiscoverSubreddits(unittest.TestCase):
{"subreddit": "tipofmytongue", "score": 100},
{"subreddit": "python", "score": 10},
]
- subs = reddit.discover_subreddits(results, topic="python", max_subs=5)
- self.assertEqual(subs[0], "python")
+ subs = reddit.discover_subreddits(results, max_subs=5)
+ self.assertEqual(subs, ["python", "tipofmytongue"])
def test_topic_name_bonus(self):
results = [
- {"subreddit": "reactjs", "score": 10},
{"subreddit": "webdev", "score": 10},
+ {"subreddit": "reactjs", "score": 10},
]
subs = reddit.discover_subreddits(results, topic="react hooks", max_subs=5)
- self.assertEqual(subs[0], "reactjs")
+ self.assertEqual(subs, ["reactjs", "webdev"])
def test_engagement_bonus(self):
results = [
- {"subreddit": "AIsub", "ups": 500},
{"subreddit": "OtherSub", "ups": 5},
+ {"subreddit": "AIsub", "ups": 500},
]
subs = reddit.discover_subreddits(results, max_subs=5)
- self.assertEqual(subs[0], "AIsub")
+ self.assertEqual(subs, ["AIsub", "OtherSub"])
def test_max_subs_limit(self):
results = [{"subreddit": f"sub{i}"} for i in range(20)]
diff --git a/tests/test_reddit_shreddit.py b/tests/test_reddit_shreddit.py
index fc01eefd..9e383972 100644
--- a/tests/test_reddit_shreddit.py
+++ b/tests/test_reddit_shreddit.py
@@ -59,9 +59,13 @@ class TestParseComments:
assert any("$750" in b or "pending" in b for b in bodies)
def test_comment_url_built(self):
- for c in rs.parse_comments(_html()):
- if c["url"]:
- assert c["url"].startswith("https://reddit.com/r/")
+ comments = rs.parse_comments(_html())
+ assert [c["url"] for c in comments] == [
+ f"https://reddit.com/r/Rakuten/comments/1taeiw0/comment/{comment_id}/"
+ for comment_id in (
+ "ol8tp8n", "olcy1iv", "olaqzjk", "omlbiqg", "ol8undb", "ol8w8w6", "synthneg"
+ )
+ ]
def test_empty_html_returns_empty(self):
assert rs.parse_comments("") == []
diff --git a/tests/test_rerank_fun.py b/tests/test_rerank_fun.py
index f7ef5cc3..28bb6c76 100644
--- a/tests/test_rerank_fun.py
+++ b/tests/test_rerank_fun.py
@@ -51,12 +51,18 @@ class TestFunFallbackCommentText:
candidate = _make_candidate(
title="Boring press conference recap",
snippet="Coach talked about the game plan.",
- top_comments=[{"body": "lmao this is gold"}],
+ top_comments=[{"body": "lmao"}],
+ )
+ control = _make_candidate(
+ title="Boring press conference recap",
+ snippet="Coach talked about the game plan.",
+ top_comments=[{"body": "nope"}],
)
_apply_single_fun_fallback(candidate)
- # marker_bonus = 10, should be reflected in fun_score
+ _apply_single_fun_fallback(control)
assert candidate.fun_score is not None
- assert candidate.fun_score >= 10.0
+ assert control.fun_score is not None
+ assert candidate.fun_score - control.fun_score == pytest.approx(10.0)
assert candidate.fun_explanation == "heuristic-fallback"
def test_short_punchy_comment_higher_shortness(self):
diff --git a/tests/test_save_dir_env_fallback.py b/tests/test_save_dir_env_fallback.py
index bc843e04..1f146301 100644
--- a/tests/test_save_dir_env_fallback.py
+++ b/tests/test_save_dir_env_fallback.py
@@ -32,6 +32,7 @@ def _run_engine(
topic: str,
extra_argv: list[str],
env_overrides: dict[str, str],
+ cwd: Path | None = None,
) -> subprocess.CompletedProcess:
cmd = [
sys.executable,
@@ -55,6 +56,7 @@ def _run_engine(
encoding="utf-8",
errors="replace",
check=False,
+ cwd=cwd,
)
@@ -67,6 +69,10 @@ class SaveDirEnvFallbackTests(unittest.TestCase):
self.config_dir.mkdir()
self.save_target = self.tmp / "Last30Days"
self.save_target.mkdir()
+ self.home = self.tmp / "home"
+ self.home.mkdir()
+ self.cwd = self.tmp / "work"
+ self.cwd.mkdir()
def tearDown(self) -> None:
shutil.rmtree(self.tmp, ignore_errors=True)
@@ -74,6 +80,14 @@ class SaveDirEnvFallbackTests(unittest.TestCase):
def _write_dotenv(self, contents: str) -> None:
(self.config_dir / ".env").write_text(contents, encoding="utf-8")
+ def _assert_no_report_saved(self, result: subprocess.CompletedProcess) -> None:
+ self.assertEqual(result.returncode, 0, msg=result.stderr)
+ self.assertRegex(result.stdout, r"(?m)^# last30days v[^\n]+: OpenAI$")
+ self.assertEqual(list(self.home.rglob("*.md")), [])
+ self.assertEqual(list(self.cwd.rglob("*.md")), [])
+ self.assertEqual(list(self.save_target.rglob("*.md")), [])
+ self.assertNotIn("Saved output to", result.stderr)
+
def test_env_var_in_dotenv_file_triggers_save(self) -> None:
"""Setting LAST30DAYS_MEMORY_DIR in .env makes --save-dir-less runs save."""
self._write_dotenv(f"LAST30DAYS_MEMORY_DIR={self.save_target}\n")
@@ -133,11 +147,10 @@ class SaveDirEnvFallbackTests(unittest.TestCase):
result = _run_engine(
topic="OpenAI",
extra_argv=[],
- env_overrides={"LAST30DAYS_CONFIG_DIR": ""},
+ env_overrides={"LAST30DAYS_CONFIG_DIR": "", "HOME": str(self.home), "USERPROFILE": str(self.home)},
+ cwd=self.cwd,
)
- self.assertEqual(result.returncode, 0, msg=result.stderr)
- files = sorted(self.save_target.glob("*.md"))
- self.assertEqual(len(files), 0)
+ self._assert_no_report_saved(result)
def test_empty_string_env_var_does_not_trigger_save(self) -> None:
"""LAST30DAYS_MEMORY_DIR='' is treated as 'no fallback', not as a path."""
@@ -147,11 +160,12 @@ class SaveDirEnvFallbackTests(unittest.TestCase):
env_overrides={
"LAST30DAYS_CONFIG_DIR": "",
"LAST30DAYS_MEMORY_DIR": "",
+ "HOME": str(self.home),
+ "USERPROFILE": str(self.home),
},
+ cwd=self.cwd,
)
- self.assertEqual(result.returncode, 0, msg=result.stderr)
- files = sorted(self.save_target.glob("*.md"))
- self.assertEqual(len(files), 0)
+ self._assert_no_report_saved(result)
def test_explicit_empty_save_dir_flag_does_not_trigger_fallback(self) -> None:
"""--save-dir '' (explicit empty) suppresses save even when env var is set."""
diff --git a/tests/test_setup_wizard.py b/tests/test_setup_wizard.py
index ecbb66fa..592b2635 100644
--- a/tests/test_setup_wizard.py
+++ b/tests/test_setup_wizard.py
@@ -3,6 +3,7 @@
import json
import os
import subprocess
+import sys
import tempfile
from pathlib import Path
from unittest.mock import patch, MagicMock
@@ -10,6 +11,7 @@ from unittest.mock import patch, MagicMock
import pytest
from lib import setup_wizard
+import last30days as cli
class _NtOs:
@@ -921,27 +923,34 @@ class TestGetSetupStatusText:
class TestSetupSubcommand:
- """Tests for setup subcommand detection in argument parsing."""
-
- def test_setup_detected_as_topic(self):
- """The word 'setup' is treated as the setup subcommand."""
- # Simulate what argparse produces
- import argparse
- parser = argparse.ArgumentParser()
- parser.add_argument("topic", nargs="*")
- args = parser.parse_args(["setup"])
- topic = " ".join(args.topic) if args.topic else None
- assert topic is not None
- assert topic.strip().lower() == "setup"
+ """Tests for setup dispatch through the production entrypoint."""
+
+ def test_setup_detected_as_topic(self, capsys):
+ with patch.object(sys, "argv", ["last30days", "setup"]), patch.object(
+ cli.env, "get_config", return_value={}
+ ), patch.object(setup_wizard, "run_auto_setup", return_value={}) as setup, patch.object(
+ setup_wizard, "write_setup_config", return_value=True
+ ) as write_config, patch.object(
+ setup_wizard, "get_setup_status_text", return_value="setup-completed-sentinel"
+ ), patch.object(cli.pipeline, "run") as research:
+ assert cli.main() == 0
+ setup.assert_called_once_with({}, allow_browser_cookies=False)
+ write_config.assert_called_once()
+ research.assert_not_called()
+ assert "setup-completed-sentinel" in capsys.readouterr().err
def test_normal_topic_not_setup(self):
- """A normal topic is not confused with setup."""
- import argparse
- parser = argparse.ArgumentParser()
- parser.add_argument("topic", nargs="*")
- args = parser.parse_args(["AI", "video", "tools"])
- topic = " ".join(args.topic) if args.topic else None
- assert topic.strip().lower() != "setup"
+ with patch.object(sys, "argv", ["last30days", "AI", "setup", "tools", "--mock"]), patch.object(
+ cli.env, "get_config", return_value={}
+ ), patch.object(setup_wizard, "run_auto_setup", return_value={}) as setup, patch.object(
+ setup_wizard, "write_setup_config", return_value=True
+ ), patch.object(setup_wizard, "get_setup_status_text", return_value="setup-sentinel"), patch.object(
+ cli.pipeline, "diagnose", side_effect=RuntimeError("research-path-sentinel")
+ ) as research:
+ with pytest.raises(RuntimeError, match="research-path-sentinel"):
+ cli.main()
+ research.assert_called_once()
+ setup.assert_not_called()
class TestBrightDataStatusHonesty:
diff --git a/tests/test_tiktok.py b/tests/test_tiktok.py
index a5c5c4f3..f871d37f 100644
--- a/tests/test_tiktok.py
+++ b/tests/test_tiktok.py
@@ -140,14 +140,16 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
fake_sc_response = {
"comments": [
- {"text": "loved it", "user": {"nickname": "Alice"},
- "digg_count": 420, "create_time": 1709251200},
{"text": "meh", "user": {"nickname": "Bob"},
"digg_count": 3, "create_time": 1709251300},
+ {"text": "loved it", "user": {"nickname": "Alice"},
+ "digg_count": 420, "create_time": 1709251200},
+ {"text": "useful", "user": {"nickname": "Carol"},
+ "digg_count": 80, "create_time": 1709251300},
{"text": "", "user": {"nickname": "Skip"},
"digg_count": 999, "create_time": 1709251400},
],
- "total": 3,
+ "total": 4,
}
with patch.object(tiktok.http, "get", return_value=fake_sc_response):
@@ -157,12 +159,18 @@ class TestTikTokEnrichWithComments(unittest.TestCase):
max_comments=5,
)
# Empty-text comment dropped; rest sorted desc by digg_count.
- self.assertEqual(2, len(out))
+ self.assertEqual([c["text"] for c in out], ["loved it", "useful", "meh"])
self.assertEqual("loved it", out[0]["text"])
self.assertEqual(420, out[0]["digg_count"])
self.assertEqual("Alice", out[0]["author"])
self.assertEqual("2024-03-01", out[0]["date"])
- self.assertEqual(3, out[1]["digg_count"])
+ self.assertEqual([c["digg_count"] for c in out], [420, 80, 3])
+ capped_response = {"comments": fake_sc_response["comments"][:3]}
+ with patch.object(tiktok.http, "get", return_value=capped_response):
+ capped = tiktok._fetch_post_comments(
+ "https://www.tiktok.com/@u/video/1", token="k", max_comments=1,
+ )
+ self.assertEqual([c["text"] for c in capped], ["loved it"])
def test_fetch_post_comments_prefers_unique_id_over_nickname(self):
"""Author prefers unique_id (@handle) over nickname (display name)."""
diff --git a/tests/test_verify_v3.py b/tests/test_verify_v3.py
index 36cbd4e4..4a992f75 100644
--- a/tests/test_verify_v3.py
+++ b/tests/test_verify_v3.py
@@ -25,15 +25,26 @@ class VerifyV3Tests(unittest.TestCase):
def test_smoke_and_latency_request_raw_json_profile(self):
module = load_verify_module()
- completed = mock.Mock(stdout='{"clusters": [], "ranked_candidates": []}')
+ completed = mock.Mock(stdout='{"clusters": [{"id":"c1"}], "ranked_candidates": [{"id":"r1"}], "provider_runtime":{"reasoning_provider":"fixture"}, "errors_by_source":{"reddit":"fixture-error"}}')
with mock.patch.object(module, "run_command", return_value=completed) as run:
module.SMOKE_CASES = [("auto", ["--quick"])]
module.LATENCY_PROFILES = [("quick", ["--quick"])]
module.LATENCY_TOPICS = ["topic"]
- module.verify_smoke()
- module.verify_latency()
+ smoke = module.verify_smoke()
+ latency = module.verify_latency()
commands = [call.args[0] for call in run.call_args_list]
+ self.assertEqual(len(commands), 2)
+ self.assertEqual(commands[0], [module.PYTHON, str(module.ENGINE), module.SMOKE_TOPIC, "--emit=json", "--json-profile=raw", "--quick"])
+ self.assertEqual(commands[1], [module.PYTHON, str(module.ENGINE), "topic", "--emit=json", "--json-profile=raw", "--quick"])
+ self.assertEqual(run.call_args_list[0].kwargs["env"]["LAST30DAYS_REASONING_PROVIDER"], "auto")
+ self.assertEqual(len(smoke), 1)
+ self.assertEqual({key: value for key, value in smoke[0].items() if key != "duration_seconds"}, {
+ "provider": "auto", "reasoning_provider": "fixture", "cluster_count": 1,
+ "candidate_count": 1, "error_sources": ["reddit"],
+ })
+ self.assertEqual(set(latency), {"quick"})
+ self.assertEqual(len(latency["quick"]["times"]), 1)
self.assertTrue(all("--json-profile=raw" in command for command in commands))
def test_unit_stage_invokes_pytest_runner(self):
diff --git a/tests/test_x_judge.py b/tests/test_x_judge.py
index d42c0694..d5a52732 100644
--- a/tests/test_x_judge.py
+++ b/tests/test_x_judge.py
@@ -122,19 +122,18 @@ class TestJudgeXCorpus:
assert len(result["on_topic_items"]) >= 1
def test_multi_token_us_query_with_pronoun(self):
- """'US economy' should NOT match text with pronoun 'us' and 'economy'."""
+ """The pronoun is excluded from overlap and the query denominator."""
items = [
- {"author_handle": "offtopic", "text": "Tell us about the economy"},
- {"author_handle": "offtopic", "text": "Let us discuss economy trends"},
+ {"author_handle": "pronoun", "text": "Tell us about the economy"},
+ {"author_handle": "country", "text": "Tell US about the economy"},
+ {"author_handle": "unrelated", "text": "Tell us about the weather"},
]
- result = x_judge.judge_x_corpus(items, "US economy")
- # These have lowercase "us" (pronoun) not "US" (country)
- # Should match on "economy" alone (50% coverage), which may or may not
- # pass the relevance floor depending on thresholds
- # The key test is that "us" doesn't contribute to the match
- for item in result["on_topic_items"]:
- # If on-topic, it should be due to "economy" match only
- assert "economy" in item["text"].lower()
+ assert x_judge._compute_relevance("US economy growth", items[0]["text"]) == 0.5
+ assert x_judge._compute_relevance("US economy growth", items[1]["text"]) == pytest.approx(2 / 3)
+ result = x_judge.judge_x_corpus(items, "US economy growth")
+ assert result["on_topic_items"] == items[:2]
+ assert result["off_topic_items"] == [items[2]]
+ assert result["on_topic_ratio"] == pytest.approx(2 / 3)
def test_handle_stats_computed(self):
"""Handle stats should track on-topic and total counts."""
diff --git a/tests/test_youtube_yt.py b/tests/test_youtube_yt.py
index 74c1e204..b9d97b8e 100644
--- a/tests/test_youtube_yt.py
+++ b/tests/test_youtube_yt.py
@@ -764,13 +764,13 @@ class TestYtdlpSSHRouting(unittest.TestCase):
self.assertEqual(wrapped[dash_idx + 1], "macmini")
def test_host_alias_with_dash_prefix_is_rejected(self):
- """A host value starting with `-` is rejected by the alias validator.
+ os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "-p22"
+ self.assertIsNone(youtube_yt._ytdlp_ssh_host())
+ cmd = ["yt-dlp", "--version"]
+ self.assertEqual(youtube_yt._wrap_ytdlp_cmd(cmd), cmd)
- Without validation, ssh could parse `-oProxyCommand=...` as a flag
- instead of a hostname. The `--` terminator in _wrap_ytdlp_cmd is
- defense-in-depth; this regex on _ytdlp_ssh_host() rejects the value
- before it ever reaches the ssh command line.
- """
+ def test_host_alias_with_option_payload_is_rejected(self):
+ """Reject an SSH option payload independently of command option termination."""
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = "-oProxyCommand=evil"
self.assertIsNone(youtube_yt._ytdlp_ssh_host())
# And the wrap function falls back to the local-execution path.
@@ -788,7 +788,7 @@ class TestYtdlpSSHRouting(unittest.TestCase):
def test_host_alias_validator_accepts_realistic_aliases(self):
"""Valid SSH config aliases are accepted: bare names, FQDNs, IPs."""
- for good in ("macmini", "home-server", "pi5.local", "192.168.1.10", "homelab_box"):
+ for good in ("p22", "macmini", "home-server", "pi5.local", "192.168.1.10", "homelab_box"):
os.environ["LAST30DAYS_YOUTUBE_SSH_HOST"] = good
self.assertEqual(youtube_yt._ytdlp_ssh_host(), good)