AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
"""Contract tests for the restored first-run NUX wizard in SKILL.md.
Step 0 has three branches: a **Claude Code Modal Flow** (AskUserQuestion-driven,
the restored v3.0.0 NUX), a **Non-Modal Prose Flow** for hosts without modals
(OpenClaw, Codex, Cursor, Gemini CLI), and a **Grok Bot Prose Flow** (the X
connector lane first, keys written only through the engine, no browser-session
reads). These tests assert the structural guarantees of the branches, plus the cross-cutting copy rules: the hard
"Step 0 before Step 1" gate, Digg threaded alongside yt-dlp, the 10,000-free-calls
credit count, and Threads/Pinterest kept out of the onboarding offers. They read
SKILL.md as text - the model's runtime contract - matching
tests/test_runtime_preflight_contract.py.
These lock the flow against silent re-erosion (the failure mode that orphaned the
wizard in PR #659 and flattened it before this restoration).
"""
import unittest
from pathlib import Path
from lib import setup_wizard
from tests.skill_contract import contract_documents, reference_text
ROOT = Path(__file__).resolve().parents[1]
SKILL_MD = ROOT / "skills" / "last30days" / "SKILL.md"
AGENTS_MD = ROOT / "AGENTS.md"
class TestOnboardingContract(unittest.TestCase):
def setUp(self):
self.text = SKILL_MD.read_text(encoding="utf-8")
# Scope assertions to Step 0 so generic substrings elsewhere in the file
# do not satisfy ordering/presence checks.
self.step0 = reference_text("setup-wizard")
self.assertIn("## Step 0: First-Run Setup Wizard", self.step0)
# Branch slices.
modal_start = self.step0.index("### Claude Code Modal Flow")
prose_start = self.step0.index("### Non-Modal Prose Flow")
grok_start = self.step0.index("### Grok Bot Prose Flow")
manual_start = self.step0.index("### Manual Setup Guide")
self.modal = self.step0[modal_start:prose_start]
self.prose = self.step0[prose_start:grok_start]
self.grok = self.step0[grok_start:manual_start]
self.manual = self.step0[manual_start:]
# --- Platform split + hard gate ---
def test_platform_split_present(self):
"""Step 0 routes modal-capable hosts, prose hosts, and Grok Bot to
three distinct flows, in that order."""
self.assertIn("Platform split", self.step0)
self.assertIn("### Claude Code Modal Flow", self.step0)
self.assertIn("### Non-Modal Prose Flow", self.step0)
self.assertIn("### Grok Bot Prose Flow", self.step0)
split = self.step0[self.step0.index("Platform split"):self.step0.index("### Claude Code Modal Flow")]
self.assertIn("Grok Bot Prose Flow", split)
self.assertEqual(3, len([h for h in ("### Claude Code Modal Flow", "### Non-Modal Prose Flow", "### Grok Bot Prose Flow") if h in self.step0]))
self.assertLess(self.step0.index("### Non-Modal Prose Flow"), self.step0.index("### Grok Bot Prose Flow"))
def test_agents_md_names_three_step0_branches(self):
"""AGENTS.md's onboarding rule and this contract move together."""
agents = AGENTS_MD.read_text(encoding="utf-8")
self.assertIn("Step 0 has THREE branches", agents)
self.assertNotIn("Step 0 has TWO branches", agents)
for name in ("Claude Code Modal Flow", "Non-Modal Prose Flow", "Grok Bot Prose Flow"):
self.assertIn(name, agents, name)
def test_grok_flow_is_prose_and_connector_first(self):
"""The third branch has no modals, leads with the X connector, and
never routes through a browser-session step."""
self.assertNotIn("AskUserQuestion", self.grok)
self.assertNotIn("cookie", self.grok.lower())
self.assertLess(self.grok.index("search_posts_all"), self.grok.index("X_BEARER_TOKEN"))
self.assertIn("setup --store-key", self.grok)
def test_first_run_flows_do_not_invoke_preflight(self):
"""Status and permission inspection are not a required first-run beat.
`--preflight` stays an opt-in inspector; Step 0 must not dump `.env`."""
self.assertNotIn("--preflight", self.modal)
self.assertNotIn("--preflight", self.prose)
self.assertNotIn("--preflight", self.grok)
self.assertIn("Do not run it as a required first-run step", self.step0)
self.assertIn("Do not print `.env` contents or credential values", self.step0)
def test_first_run_gate_defers_to_step0_credential_sources(self):
"""The cheap SETUP_COMPLETE grep is not itself a first-run verdict."""
start = self.text.index("**FIRST-RUN GATE")
gate = self.text[start:].split("\n## ", 1)[0].replace("**", "")
self.assertIn("FIRST_RUN_DETECTED", gate)
self.assertIn("A missing `.env` alone is not a first run", gate)
self.assertIn("That section decides first-run from every credential source", gate)
def test_root_first_run_gate_routes_to_setup_wizard(self):
"""The wizard's own Step 0 gate is only read after this root route fires."""
start = self.text.index("**FIRST-RUN GATE")
gate = self.text[start:].split("\n## ", 1)[0]
first_run = next(
line for line in gate.splitlines() if line.startswith("- `FIRST_RUN_DETECTED`")
)
self.assertIn("read the setup-wizard reference through its root gate immediately", first_run)
self.assertIn("complete the applicable flow before topic research", first_run)
self.assertLess(gate.index("FIRST_RUN_DETECTED"), gate.index("Topic supplied: load the topic runbook"))
def test_root_consent_rules_hold_without_setup_wizard(self):
"""Configured runs never load setup-wizard.md, so consent must stay in the root."""
start = self.text.index("**Onboarding consent is model-led and host-split.**")
consent = self.text[start:].split("\n\n", 1)[0]
self.assertIn("Ask before browser-cookie reads, and preserve refusal", consent)
self.assertIn("a skip or no answer is never consent", consent)
self.assertIn("All three host flows offer the ScrapeCreators signup on first run", consent)
def test_complete_does_not_treat_setup_stdout_as_source_list(self):
self.assertIn(
"Setup stdout is what this run installed, not the runtime source list",
self.prose,
)
self.assertIn(
"Setup stdout is what this run installed, not the runtime source list",
self.grok,
)
def test_hard_gate_step0_before_step1(self):
"""The erosion-resistant gate that orphaned the wizard in #659 is restored."""
self.assertIn("ALWAYS execute Step 0 BEFORE Step 1", self.step0)
def test_waiting_topic_continues_after_x_decline_or_setup_skip(self):
"""Declining optional X access must never strand the requested topic."""
self.assertIn("RESEARCH CONTINUATION OVERRIDE", self.step0)
self.assertIn("declining or skipping X must never stop", self.step0)
self.assertIn("immediately research it with `--no-browser-cookies`", self.modal)
self.assertIn("immediately research it with `--no-browser-cookies`", self.prose)
self.assertIn("a skip or no answer is never consent", self.step0)
def test_waiting_topic_defers_optional_prompts_and_x_retry(self):
self.assertIn("skip the ScrapeCreators offer", self.step0)
self.assertIn("Do not ask another X question in the same run", self.step0)
self.assertIn("Offer ONE retry only when no research topic is waiting", self.modal)
self.assertIn("Offer ONE retry only when no research topic is waiting", self.prose)
def test_deferred_onboarding_resumes_after_the_findings(self):
"""Deferral is same-run only: SETUP_COMPLETE=true means later runs skip
Step 0, so a skip-X-with-topic run must itself resume the ScrapeCreators
offer after the findings or the offer is dropped forever."""
self.assertIn("RESUME the deferred onboarding in the SAME run", self.step0)
self.assertIn(
"this run is the only chance to make the offer", self.step0
)
# Both flows: Skip-for-now, Skip-X modal option, and the prose no-path
# all resume the deferred offer in the same run after the findings.
self.assertEqual(
2,
self.modal.count(
"then resume Step 4 (and Step 5 if a key is saved) in the same run"
),
)
self.assertIn("then resume the deferred onboarding in the same run", self.prose)
# The resume never turns back into a second X consent ask.
self.assertIn("the resume never re-asks X/browser-cookie consent", self.step0)
self.assertIn("Do not re-ask cookie consent as part of the resume", self.prose)
def test_x_handle_resolution_and_plan_follow_active_sources(self):
research = reference_text("research-runbook")
self.assertIn("If `ACTIVE_SOURCES_LIST` contains `x`", research)
self.assertIn("every applicable source from `ACTIVE_SOURCES_LIST`", research)
self.assertIn("Preserve X whenever it is active", research)
def test_post_report_x_note_is_non_blocking(self):
synthesis = reference_text("synthesis")
for name, text in contract_documents().items():
self.assertNotIn("Just-in-time X unlock", text, name)
self.assertIn("Optional X omission", synthesis)
self.assertIn("finish the useful findings first", synthesis)
# --- Modal flow: the restored NUX, stages in order ---
def test_modal_flow_stage_order(self):
"""Welcome -> setup modal -> cookie consent -> SC offer -> opt-in -> picker."""
anchors = [
"Welcome to /last30days!", # welcome pitch, embedded in the setup modal
"How would you like to set up?",
"your browser's x.com cookies", # cookie-consent modal
"Want to add TikTok and Instagram?", # SC offer
"Which ScrapeCreators sources?", # source opt-in
"What do you want to research first?", # topic picker
]
idxs = [self.modal.find(a) for a in anchors]
for a, i in zip(anchors, idxs):
self.assertGreater(i, -1, f"modal flow missing stage anchor: {a!r}")
self.assertEqual(idxs, sorted(idxs), "modal flow stages are out of order")
def test_modal_uses_askuserquestion(self):
self.assertIn("AskUserQuestion", self.modal)
def test_cookie_consent_names_all_installed_clis(self):
"""The cookie-consent modal must not frame X cookies as instead-of the CLIs,
and must name arXiv + Techmeme (not just 'YouTube + Digg') since auto-setup
installs all four regardless of the cookie choice."""
consent = self.modal[self.modal.find("your browser's x.com cookies"):]
consent = consent[: consent.find("Full Disk Access")] # bound to the consent modal
for cli in ("yt-dlp", "Digg", "arXiv", "Techmeme"):
self.assertIn(cli, consent, cli)
# The "skip X" option still installs the CLIs (not framed as X-or-CLIs).
self.assertIn("Skip X - just the CLIs", consent)
def test_github_option_advertises_auto_clipboard(self):
"""The recommended GitHub option tells the user the code is auto-copied to
their clipboard, so they just paste it."""
self.assertIn("clipboard automatically", self.modal)
def test_modal_cookie_consent_before_setup(self):
consent = self.modal.find("your browser's x.com cookies")
setup = self.modal.find('last30days.py" setup')
self.assertGreater(consent, -1, "no cookie-consent modal in modal flow")
self.assertGreater(setup, -1, "no setup invocation in modal flow")
self.assertLess(consent, setup, "cookie consent must precede setup in modal flow")
def test_topic_picker_skips_when_topic_supplied(self):
"""The picker documents skipping when the user already gave a topic."""
self.assertIn("What do you want to research first?", self.modal)
self.assertIn("SKIP this picker", self.modal)
# --- Prose flow: same work, modal-free ---
def test_prose_flow_has_no_modals(self):
self.assertNotIn("AskUserQuestion", self.prose)
def test_prose_cookie_consent_before_setup(self):
consent = self.prose.find("Cookie consent")
setup = self.prose.find('last30days.py" setup')
self.assertGreater(consent, -1, "no cookie-consent step in prose flow")
self.assertGreater(setup, -1, "no setup invocation in prose flow")
self.assertLess(consent, setup, "cookie consent must precede setup in prose flow")
def test_prose_decline_uses_from_browser_off(self):
self.assertIn("FROM_BROWSER=off", self.prose)
# --- Full Disk Access remediation (both branches) ---
def test_full_disk_access_remediation_present(self):
self.assertIn("Permission denied reading Cookies.binarycookies", self.modal)
self.assertIn("Full Disk Access", self.modal)
self.assertIn("Permission denied reading Cookies.binarycookies", self.prose)
self.assertIn("Full Disk Access", self.prose)
def test_skip_path_writes_setup_complete(self):
"""The 'Skip for now' setup choice must write SETUP_COMPLETE or the wizard loops."""
skip_idx = self.modal.find("If the user picks Skip for now")
self.assertGreater(skip_idx, -1, "no Skip-for-now handling in modal flow")
# The skip branch must persist the completion flag in its own paragraph.
skip_para = self.modal[skip_idx:skip_idx + 400]
self.assertIn("SETUP_COMPLETE=true", skip_para)
# --- ScrapeCreators signup + persisted edge case ---
def test_scrapecreators_signup_present_both_branches(self):
self.assertIn("setup --github", self.modal)
self.assertIn("setup --github", self.prose)
def test_persisted_false_edge_case_documented(self):
self.assertIn('"persisted": false', self.step0)
# --- Digg threaded alongside yt-dlp everywhere it appears ---
def test_digg_threaded_with_ytdlp(self):
self.assertIn("Digg", self.modal)
self.assertIn("Digg", self.prose)
self.assertIn("Digg", self.manual)
# The Auto-setup modal option names every installed CLI, not just two.
self.assertIn("yt-dlp (YouTube), Digg, arXiv, Techmeme", self.modal)
# --- Credit count = 10,000, no conflicting numbers in onboarding ---
def test_credit_count_is_10000(self):
self.assertIn("10,000 free calls", self.step0)
self.assertNotIn("1,000 free", self.step0)
self.assertNotIn("1000 free credit", self.step0)
self.assertNotIn("1000 credits", self.step0)
self.assertNotIn("100 free call", self.step0)
# --- Threads/Pinterest live ONLY in the Step 5 "Everything" opt-in ---
def _modal_step5(self):
start = self.modal.index("**Step 5:")
end = self.modal.index("**Step 6:", start)
return self.modal[start:end]
def _modal_before_step5(self):
# Welcome (Step 1) through the Step 4 ScrapeCreators offer.
return self.modal[: self.modal.index("**Step 5:")]
def test_threads_pinterest_only_in_step5_everything(self):
"""Threads/Pinterest are offered in the Step 5 Everything tier, and
must NOT appear in the welcome or the Step 4 offer (where they would
read as default-on). They are opt-in via INCLUDE_SOURCES.
"""
step5 = self._modal_step5()
self.assertIn("Threads", step5)
self.assertIn("Pinterest", step5)
before = self._modal_before_step5()
self.assertNotIn("Threads", before)
self.assertNotIn("Pinterest", before)
def test_offer_copy_names_comments_and_auto_enrichment(self):
"""The Step 4 offer states comments are part of the default value and
describes the key's real Reddit/YouTube roles (Reddit search backfill
below the 5-item floor + yt-dlp transcript backstop), not rate-limit
escalation or SC Reddit comment enrichment on the free path."""
before = self._modal_before_step5()
self.assertIn("comments", before.lower())
self.assertIn("Reddit", before)
self.assertIn("YouTube", before)
self.assertIn("10,000 free calls", before)
# Floor-triggered search backfill with the 0 opt-out (not
# transport/rate-limit escalation, and no longer empty-only).
self.assertIn("fewer than 5 items", before)
self.assertIn("LAST30DAYS_REDDIT_SC_MIN_ITEMS=0", before)
self.assertIn("YouTube search below 3", before)
self.assertIn("LAST30DAYS_YT_SC_MIN_ITEMS=0", before)
self.assertIn("spends credits", before)
self.assertNotIn("returns no items", before)
self.assertNotIn("empty-path Reddit", before)
self.assertNotIn("when they hit rate limits", before)
# Free-path comments are shreddit; do not claim SC comment preference.
self.assertNotIn("prefers ScrapeCreators for Reddit", before)
self.assertNotIn("enriches Reddit comments", before)
def test_step5_does_not_claim_merged_reddit_auto_enrichment(self):
"""Step 5 must not contradict Step 4 with 'public + ScrapeCreators' merge."""
step5 = self._modal_step5()
self.assertNotIn("public + ScrapeCreators", step5)
self.assertNotIn("Reddit auto-enrichment", step5)
self.assertIn("fewer than 5 items", step5)
self.assertNotIn("empty-only", step5)
def test_recommended_tier_writes_comments_by_default(self):
"""Comments are the DEFAULT: the recommended option enables YouTube +
TikTok + Instagram comments (posts on -> comments on)."""
step5 = self._modal_step5()
self.assertIn(
"INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments",
step5,
)
# There is no posts-only tier.
self.assertIn("recommended", step5.lower())
self.assertIn("comments", step5.lower())
def test_everything_tier_writes_full_include_sources(self):
"""The Everything option persists the full list incl. Threads + Pinterest."""
step5 = self._modal_step5()
self.assertIn(
"INCLUDE_SOURCES=tiktok,instagram,youtube_comments,tiktok_comments,instagram_comments,threads,pinterest",
step5,
)
# --- Chrome-first cookie scan (U2/U3) ---
def test_cookie_consent_leads_with_chrome(self):
"""Both flows tell the user Chrome is checked first, with the Keychain cue."""
for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
self.assertIn("Chrome", slice_text, f"{slice_name} cookie copy omits Chrome")
self.assertIn("Always Allow", slice_text, f"{slice_name} omits the Keychain cue")
def test_cookie_consent_explains_future_reads_before_the_answer(self):
prompts = (
self.modal.split('Question: "Auto setup', 1)[1].split("Options (", 1)[0],
self.prose.split("Otherwise ask. Example:", 1)[1].split("**Wait for the answer.**", 1)[0],
)
for prompt in prompts:
with self.subTest(prompt=prompt):
self.assertIn("later research runs", prompt)
self.assertIn("browser names", prompt)
self.assertIn("may prompt again", prompt)
self.assertIn("Always Allow", prompt)
self.assertIn("BROWSER_CONSENT=false", prompt)
self.assertIn("FROM_BROWSER=off", prompt)
self.assertIn("On macOS", prompt)
self.assertIn("Windows and Linux", prompt)
self.assertIn("Windows Firefox", prompt)
def test_cookie_persistence_contract_matches_each_host_flow(self):
for flow in (self.modal, self.prose):
with self.subTest(flow=flow[:30]):
self.assertIn("complete cookies", flow)
self.assertIn("comma-separated", flow)
self.assertIn("never saves cookie values", flow)
self.assertNotIn("one-time macOS Keychain prompt", flow)
self.assertNotIn("only when it is Firefox or Safari", flow)
self.assertNotIn("only a Firefox/Safari winner", flow)
self.assertNotIn("Chrome never re-", flow)
def test_browser_permission_remediation_keeps_chromium_first(self):
"""Browser permission guidance does not change consented scan order."""
self.assertNotIn("scan your browser (Firefox/Safari)", self.modal)
self.assertIn("X cookie access was permission denied", self.modal)
self.assertIn("X cookie access was permission denied", self.prose)
self.assertIn("AUTH_TOKEN and CT0", self.modal)
def test_welcome_embedded_in_modal(self):
"""The welcome pitch lives INSIDE the setup modal (the only always-visible
surface), not as a separate message/command that Claude Code folds away.
The engine --welcome command is kept for the non-modal prose flow."""
# Pitch is in the modal question.
self.assertIn("Welcome to /last30days!", self.modal)
self.assertIn("How would you like to set up?", self.modal)
# The modal flow explicitly does NOT run a separate --welcome command.
self.assertIn("Do NOT run a separate `--welcome`", self.modal)
# The non-modal flow still uses the engine welcome command.
self.assertIn('last30days.py" --welcome', self.prose)
def test_stocktwits_surfaced_as_conditional(self):
"""StockTwits is advertised in the engine welcome as a ticker/crypto-gated
source (welcome text moved out of SKILL.md into the engine)."""
self.assertIn("StockTwits", setup_wizard.render_welcome())
# --- Honest GitHub device-code copy (U4/U7) ---
def test_no_false_instant_gh_promise(self):
"""The '~2 seconds - no browser' claim (a nonexistent code path) is gone."""
self.assertNotIn("~2 seconds - no browser", self.step0)
self.assertNotIn("Registers via GitHub CLI in ~2 seconds", self.step0)
def test_device_code_surfacing_orchestration_present(self):
"""Both flows use the deterministic two-command split (start returns the
code fast, then poll) instead of a background-and-surface spinner."""
self.assertIn("setup --github-start", self.modal)
self.assertIn("setup --github-poll", self.modal)
self.assertIn("setup --github-start", self.prose)
self.assertIn("setup --github-poll", self.prose)
def test_already_registered_status_handled(self):
self.assertIn("already_registered", self.modal)
self.assertIn("already_registered", self.prose)
# --- Welcome must render before the modal (U1) ---
def test_welcome_pitch_is_in_the_modal_question(self):
"""The welcome pitch names the core sources inside the modal question, so
the user sees it without expanding folded tool output. The old skip-prone
'IMMEDIATELY call AskUserQuestion' wording stays gone."""
# Pitch names the core sources right in the modal.
for source in ("Reddit", "X,", "YouTube", "TikTok"):
self.assertIn(source, self.modal, source)
self.assertNotIn("Then IMMEDIATELY call AskUserQuestion", self.modal)
# --- Device code surfaced with a clipboard-paste hint (U3) ---
def test_device_code_clipboard_paste_instruction(self):
"""The GitHub flow tells the user the code is on their clipboard to paste,
and makes surfacing the code a required step (the bug the user hit).
"""
self.assertIn("on your clipboard", self.modal)
# Surfacing the code is a required, explicit step in the new split flow.
self.assertIn("SHOW THE CODE", self.modal)
self.assertIn("just paste", self.modal)
# --- Honest 'authorized but no key' branch, distinct from auth-failed (U4) ---
def test_authorized_but_no_key_branch_present(self):
"""A key-fetch failure after successful auth is handled honestly (likely
an already-linked account), not lumped into 'auth didn't complete'.
"""
for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
self.assertIn("Authorized but failed to fetch API key", slice_text, slice_name)
self.assertIn("already linked", slice_text, slice_name)
def test_upstream_profile_error_branch_distinct_from_already_linked(self):
"""A ScrapeCreators /profile 5xx must not be diagnosed as already-linked (#882)."""
for slice_name, slice_text in (("modal", self.modal), ("prose", self.prose)):
self.assertIn("ScrapeCreators profile failed", slice_text, slice_name)
self.assertIn("upstream_error", slice_text, slice_name)
self.assertIn("server error", slice_text, slice_name)
# Guidance forbids the already-linked misdiagnosis on this path.
self.assertIn("do **NOT** say", slice_text, slice_name)
self.assertIn("already linked", slice_text, slice_name)
# --- Legacy guarantees retained ---
def test_old_silent_wizard_instruction_removed(self):
for document in contract_documents().values():
self.assertNotIn("Follow the wizard's prompts end-to-end", document)
def test_consent_is_conversational_contract_documented(self):
self.assertIn("You are the conversational driver", self.step0)
self.assertIn("consent happens HERE, in chat", self.step0)
self.assertIn("gate each subprocess call on the answer", self.step0)
self.assertIn("non-interactive subprocess", self.step0)
if __name__ == "__main__":
unittest.main()