AI agent skill that researches any topic across Reddit, X, YouTube, HN, Polymarket, and the web - then synthesizes a grounded summary
Python ai-promptsai-skillblueskyclaudeclaude-codeclawhubdeep-researchhackernewsinstagramopenclawpolymarketrecencyredditresearchsocial-mediatiktoktrendstwitterweb-searchyoutube
"""xapi: the direct X API v2 backend and the shared v2 parser (U4).
Every test mocks ``lib.http.get`` (or urlopen for the fixture seam); no test
makes a live X call. The dummy token below is obvious fake data and must
never appear in any error string, source_log line, or recorded fixture.
"""
from __future__ import annotations
import io
import json
import threading
from contextlib import redirect_stderr
from datetime import datetime, timezone
from unittest import mock
from unittest.mock import MagicMock
import pytest
from lib import env, health, http, pipeline, schema, x_api, xurl_x
DUMMY_TOKEN = "dummy-x-bearer-secret-000"
ACCOUNT_ID = "acct-1234567890"
FIXED_NOW = datetime(2026, 9, 8, 12, 0, 0, tzinfo=timezone.utc)
FROM = "2026-08-10"
TO = "2026-09-08"
def _tweet(id_, text, author_id="u1", created_at="2026-08-20T12:00:00Z", metrics=None, **extra):
t = {"id": id_, "text": text, "author_id": author_id}
if created_at:
t["created_at"] = created_at
if metrics is not None:
t["public_metrics"] = metrics
t.update(extra)
return t
def _v2(tweets, users=None, next_token=None):
resp = {"data": tweets, "meta": {"result_count": len(tweets)}}
if users:
resp["includes"] = {"users": users}
if next_token:
resp["meta"]["next_token"] = next_token
return resp
def _users(*pairs):
return [{"id": uid, "username": name} for uid, name in pairs]
def _http_error(status, body="", message=None):
return http.HTTPError(message or f"HTTP {status}: Error", status_code=status, body=body)
@pytest.fixture
def fixed_now(monkeypatch):
monkeypatch.setattr(x_api, "_utcnow", lambda: FIXED_NOW)
return FIXED_NOW
@pytest.fixture
def get_mock(monkeypatch):
m = MagicMock(name="http.get")
monkeypatch.setattr(x_api.http, "get", m)
return m
def _params(call):
return call.kwargs.get("params") or {}
def _url(call):
return call.args[0] if call.args else call.kwargs["url"]
# ---------------------------------------------------------------------------
# parse_v2_response
# ---------------------------------------------------------------------------
class TestParseV2Response:
def test_fixture_with_users_parses_all_fields(self):
metrics = {
"like_count": 42, "retweet_count": 10, "reply_count": 5,
"quote_count": 2, "bookmark_count": 7, "impression_count": 9000,
}
resp = _v2(
[_tweet("1956158892141441450", "Claude Code agents are great", metrics=metrics)],
users=_users(("u1", "alice")),
)
items = x_api.parse_v2_response(resp, "Claude Code", (FROM, TO))
assert len(items) == 1
item = items[0]
assert item["id"] == "XAPI1"
assert item["post_id"] == "1956158892141441450"
assert item["author_handle"] == "alice"
assert item["url"] == "https://x.com/alice/status/1956158892141441450"
assert item["date"] == "2026-08-20"
assert item["engagement"] == {
"likes": 42, "reposts": 10, "replies": 5, "quotes": 2,
"bookmarks": 7, "views": 9000,
}
assert item["why_relevant"] == ""
assert item["relevance"] > 0.5
assert item["mentioned_handles"] == []
def test_without_users_keeps_item_with_i_status_url(self):
resp = _v2([_tweet("999", "text here", author_id="unknown")])
items = x_api.parse_v2_response(resp, "text", (FROM, TO))
assert len(items) == 1
assert items[0]["url"] == "https://x.com/i/status/999"
assert items[0]["author_handle"] == ""
def test_note_tweet_replaces_truncated_text(self):
resp = _v2([_tweet("5", "truncated...", note_tweet={"text": "the full long-form post text"})])
items = x_api.parse_v2_response(resp, "", None)
assert items[0]["text"] == "the full long-form post text"
def test_url_is_built_from_id_never_from_response_url_fields(self):
resp = _v2(
[_tweet("7", "hi", url="https://evil.example/phish",
entities={"urls": [{"expanded_url": "https://evil.example/x"}]})],
users=_users(("u1", "bob")),
)
items = x_api.parse_v2_response(resp, "", None)
assert items[0]["url"] == "https://x.com/bob/status/7"
assert "evil.example" not in json.dumps(items)
def test_username_outside_handle_grammar_falls_back_to_i_status(self):
resp = _v2([_tweet("8", "hi")], users=[{"id": "u1", "username": "bad name/../x"}])
items = x_api.parse_v2_response(resp, "", None)
assert items[0]["author_handle"] == ""
assert items[0]["url"] == "https://x.com/i/status/8"
def test_leading_mentions_become_mentioned_handles(self):
resp = _v2([_tweet("9", "@steipete @alice thanks for this")])
items = x_api.parse_v2_response(resp, "", None)
assert items[0]["mentioned_handles"] == ["steipete", "alice"]
def test_engagement_none_without_metrics_and_bookmarks_views_only_when_present(self):
resp = _v2([
_tweet("1", "a"),
_tweet("2", "b", metrics={"like_count": 1, "retweet_count": 0, "reply_count": 0, "quote_count": 0}),
])
items = x_api.parse_v2_response(resp, "", None)
assert items[0]["engagement"] is None
assert items[1]["engagement"] == {"likes": 1, "reposts": 0, "replies": 0, "quotes": 0}
def test_non_numeric_ids_are_dropped(self):
resp = _v2([_tweet("abc", "x"), _tweet("12", "y")])
items = x_api.parse_v2_response(resp, "", None)
assert [i["post_id"] for i in items] == ["12"]
def test_window_drops_dated_items_outside_it(self):
resp = _v2([
_tweet("1", "old", created_at="2026-07-01T00:00:00Z"),
_tweet("2", "in", created_at="2026-08-20T00:00:00Z"),
])
items = x_api.parse_v2_response(resp, "", (FROM, TO))
assert [i["post_id"] for i in items] == ["2"]
def test_error_response_yields_nothing(self):
assert x_api.parse_v2_response({"error": "x"}, "", None) == []
assert x_api.parse_v2_response({}, "", None) == []
def test_xurl_delegates_to_shared_parser_and_depth_config(self):
resp = _v2([_tweet("3", "hello")], users=_users(("u1", "alice")))
items = xurl_x.parse_x_response(resp, topic="hello")
assert items[0]["id"] == "XURL1"
assert items[0]["url"] == "https://x.com/alice/status/3"
assert items[0]["post_id"] == "3"
assert xurl_x.DEPTH_CONFIG is x_api.DEPTH_CONFIG
assert x_api.DEPTH_CONFIG == {"quick": 10, "default": 30, "deep": 60}
# ---------------------------------------------------------------------------
# Query compilation (R8a)
# ---------------------------------------------------------------------------
class TestBuildQuery:
def test_wraps_core_in_one_quote_pair_and_appends_no_retweets(self):
q = x_api.build_query('Claude Code "agents"')
assert q == '"Claude Code agents" -is:retweet'
def test_operator_injection_is_stripped(self):
q = x_api.build_query('foo" OR from:attacker since:2015-01-01 "')
assert q == '"foo" -is:retweet'
assert "from:" not in q and "since:" not in q and " OR " not in q
assert q.count('"') == 2
def test_negation_and_grouping_characters_are_stripped(self):
q = x_api.build_query("(Peter Steinberger) -steipete “quoted” [x] {y}")
assert q == '"Peter Steinberger quoted x y" -is:retweet'
def test_trailing_colon_handling_stays_out_of_phrase_query(self):
assert x_api.build_query("Python: what's new") == '"what\'s new" -is:retweet'
def test_600_char_topic_compiles_under_512_with_balanced_quotes(self):
topic = " ".join(f"word{i}" for i in range(100))
assert len(topic) >= 600
q = x_api.build_query(topic)
assert len(q) <= x_api.MAX_QUERY_CHARS
assert q.count('"') == 2
assert q.endswith('" -is:retweet')
assert not q.startswith('"word') or q.startswith('"word0 ')
# Cut at a token boundary: the last kept token is intact.
core = q[1:q.index('" -is:retweet')]
assert all(tok.startswith("word") and tok[4:].isdigit() for tok in core.split())
def test_single_giant_token_is_truncated_not_dropped(self):
q = x_api.build_query("a" * 700)
assert 0 < len(q) <= x_api.MAX_QUERY_CHARS
assert q.count('"') == 2
def test_only_operators_yields_empty_query(self):
assert x_api.build_query("from:attacker OR -is:reply lang:en") == ""
class TestBuildKeywordQuery:
def test_space_joined_keywords_without_phrase_or_retweet_filter(self):
assert x_api.build_keyword_query("Claude Code agents") == "Claude Code agents"
def test_bare_and_or_are_dropped_in_any_case(self):
q = x_api.build_keyword_query("review and security Or tools AND codex or x")
assert q == "review security tools codex x"
def test_operators_negation_grouping_and_quotes_are_stripped(self):
q = x_api.build_keyword_query('(foo) "bar from:attacker -baz lang:en')
assert q == "foo bar"
def test_capped_at_token_boundary(self):
q = x_api.build_keyword_query(" ".join(f"word{i}" for i in range(200)))
assert len(q) <= x_api.MAX_QUERY_CHARS
assert all(tok.startswith("word") and tok[4:].isdigit() for tok in q.split())
def test_single_giant_token_is_truncated(self):
q = x_api.build_keyword_query("a" * 700)
assert len(q) == x_api.MAX_QUERY_CHARS
def test_only_operators_yields_empty_query(self):
assert x_api.build_keyword_query("and from:x -y ()") == ""
# ---------------------------------------------------------------------------
# Request shape (KTD4)
# ---------------------------------------------------------------------------
class TestRequestShape:
def test_full_archive_request_shape(self, get_mock, fixed_now):
get_mock.return_value = _v2([_tweet("1", "Claude Code rocks")], users=_users(("u1", "a")))
result = x_api.search_x(DUMMY_TOKEN, "Claude Code", FROM, TO, depth="quick")
assert "error" not in result
assert [i["id"] for i in result["items"]] == ["XAPI1"]
call = get_mock.call_args
assert _url(call) == "https://api.x.com/2/tweets/search/all"
p = _params(call)
assert p["query"] == '"Claude Code" -is:retweet'
assert p["start_time"] == "2026-08-10T00:00:00Z"
# to_date is "today" relative to the fixed clock: end_time keeps a
# 30 second safety margin before now.
assert p["end_time"] == "2026-09-08T11:59:30Z"
assert p["sort_order"] == "recency"
assert p["max_results"] == 10
assert p["expansions"] == "author_id"
assert p["tweet.fields"] == "created_at,public_metrics,note_tweet,entities"
assert p["user.fields"] == "username"
assert "lang:" not in p["query"]
assert call.kwargs["headers"]["Authorization"] == f"Bearer {DUMMY_TOKEN}"
assert call.kwargs["timeout"] == 30
assert call.kwargs["retries"] == 2
def test_past_end_date_uses_end_of_day(self, get_mock, fixed_now):
get_mock.return_value = _v2([])
x_api.search_x(DUMMY_TOKEN, "topic", "2026-08-01", "2026-08-31", depth="quick")
assert _params(get_mock.call_args)["end_time"] == "2026-08-31T23:59:59Z"
def test_max_results_clamped_to_10_and_100(self, get_mock, fixed_now, monkeypatch):
get_mock.return_value = _v2([])
x_api.search_handles(["steipete"], "t", FROM, TO, count_per=3, token=DUMMY_TOKEN)
assert _params(get_mock.call_args)["max_results"] == 10
monkeypatch.setitem(x_api.DEPTH_CONFIG, "deep", 150)
x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="deep")
assert _params(get_mock.call_args)["max_results"] == 100
def test_pagination_stops_at_depth_count(self, get_mock, fixed_now):
page = lambda start, n, nxt: _v2(
[_tweet(str(1000 + start + i), f"post {i}") for i in range(n)], next_token=nxt,
)
get_mock.side_effect = [page(0, 20, "p2"), page(20, 20, "p3"), page(40, 20, "p4")]
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="default")
assert get_mock.call_count == 2, "30 posts reached after two pages; the third is never fetched"
assert len(result["items"]) == 30
assert "next_token" not in _params(get_mock.call_args_list[0])
assert _params(get_mock.call_args_list[1])["next_token"] == "p2"
assert len({i["id"] for i in result["items"]}) == 30
def test_pagination_stops_at_the_wall_clock_deadline(self, get_mock, fixed_now, monkeypatch):
page = lambda start, n, nxt: _v2(
[_tweet(str(1000 + start + i), f"post {i}") for i in range(n)], next_token=nxt,
)
get_mock.side_effect = [page(0, 20, "p2"), page(20, 20, "p3"), page(40, 20, "p4")]
clock = iter([0.0, x_api.DEADLINE_SECONDS + 1.0, x_api.DEADLINE_SECONDS + 2.0, x_api.DEADLINE_SECONDS + 3.0])
monkeypatch.setattr(x_api.time, "monotonic", lambda: next(clock))
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="deep")
assert get_mock.call_count == 1, "the first page always runs; the second is skipped past the deadline"
assert len(result["items"]) == 20
assert "error" not in result
assert result["warning"] == x_api.DEADLINE_DETAIL
def test_deadline_reaches_the_transport_and_keeps_pages_collected_before_it(self, get_mock, fixed_now, monkeypatch):
"""The lane deadline is the transport's wall deadline (no full 30s
timeout plus retries past it), and a deadline hit mid-walk returns
the pages already collected."""
page = lambda start, n, nxt: _v2(
[_tweet(str(1000 + start + i), f"post {i}") for i in range(n)], next_token=nxt,
)
monkeypatch.setattr(x_api.time, "monotonic", lambda: 100.0)
get_mock.side_effect = [page(0, 20, "p2"), http.DeadlineExceeded()]
result = x_api.search_handles(["steipete"], "t", FROM, TO, count_per=60, token=DUMMY_TOKEN, deadline=130.0)
assert get_mock.call_count == 2
for call in get_mock.call_args_list:
assert call.kwargs["deadline_monotonic"] == 130.0
assert len(result) == 20 and "error" not in result[0]
get_mock.reset_mock()
# The same stop on the topic search is a warning receipt, never a
# healthy-looking complete result.
get_mock.side_effect = [page(0, 20, "p2"), http.DeadlineExceeded()]
topic = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="deep")
assert len(topic["items"]) == 20
assert topic["warning"] == x_api.DEADLINE_DETAIL
get_mock.reset_mock()
get_mock.side_effect = [http.DeadlineExceeded()]
assert x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")["error"] == x_api.ERR_TIMED_OUT
def test_pagination_stops_without_next_token(self, get_mock, fixed_now):
get_mock.return_value = _v2([_tweet("1", "only one")])
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="deep")
assert get_mock.call_count == 1
assert len(result["items"]) == 1
def test_every_request_targets_api_x_com_and_carries_no_token_outside_the_header(self, get_mock, fixed_now):
get_mock.side_effect = [
_http_error(403, body='{"reason":"client-not-enrolled"}'),
_v2([_tweet("1", "a")], next_token="n"),
_v2([_tweet("2", "b")]),
]
x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
x_api.search_mentions(["steipete"], FROM, TO, topic="t", token=DUMMY_TOKEN)
assert get_mock.call_count >= 3
for call in get_mock.call_args_list:
url = _url(call)
assert url.startswith("https://api.x.com/2/tweets/search/")
assert DUMMY_TOKEN not in url
assert DUMMY_TOKEN not in json.dumps(_params(call))
headers = dict(call.kwargs["headers"])
headers.pop("Authorization")
assert DUMMY_TOKEN not in json.dumps(headers)
def test_operator_injection_still_carries_engine_window(self, get_mock, fixed_now):
get_mock.return_value = _v2([])
x_api.search_x(DUMMY_TOKEN, 'foo" OR from:attacker since:2015-01-01 "', FROM, TO)
p = _params(get_mock.call_args)
assert p["query"] == '"foo" -is:retweet'
assert p["start_time"] == "2026-08-10T00:00:00Z"
assert p["end_time"] == "2026-09-08T11:59:30Z"
def test_empty_query_after_sanitizing_makes_no_request(self, get_mock, fixed_now):
result = x_api.search_x(DUMMY_TOKEN, "from:attacker OR", FROM, TO)
assert result["items"] == []
assert result["error"] == x_api.ERR_EMPTY_QUERY
get_mock.assert_not_called()
def test_missing_token_makes_no_request(self, get_mock):
result = x_api.search_x("", "topic", FROM, TO)
assert result["items"] == [] and result["error"]
get_mock.assert_not_called()
# ---------------------------------------------------------------------------
# 403 enrollment fallback (KTD4)
# ---------------------------------------------------------------------------
class TestEnrollmentFallback:
def test_enrollment_403_retries_recent_once_with_clamped_window(self, get_mock, fixed_now):
get_mock.side_effect = [
_http_error(403, body='{"reason":"client-not-enrolled","detail":"...","client_id":"' + ACCOUNT_ID + '"}'),
_v2([_tweet("1", "recent post", created_at="2026-09-05T00:00:00Z")], users=_users(("u1", "a"))),
]
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
assert get_mock.call_count == 2
assert _url(get_mock.call_args_list[0]) == "https://api.x.com/2/tweets/search/all"
assert _url(get_mock.call_args_list[1]) == "https://api.x.com/2/tweets/search/recent"
p2 = _params(get_mock.call_args_list[1])
assert p2["start_time"] == "2026-09-01T12:00:30Z", "start clamped to now minus 7 days"
assert p2["end_time"] == "2026-09-08T11:59:30Z"
assert "error" not in result
assert result["warning"] == x_api.TRUNCATION_DETAIL == "window truncated to 7 days"
assert len(result["items"]) == 1
def test_enrollment_fallback_keeps_start_when_window_already_recent(self, get_mock, fixed_now):
get_mock.side_effect = [
_http_error(403, body="not enrolled"),
_v2([]),
]
x_api.search_x(DUMMY_TOKEN, "topic", "2026-09-06", TO, depth="quick")
assert _params(get_mock.call_args_list[1])["start_time"] == "2026-09-06T00:00:00Z"
def test_plain_403_does_not_retry(self, get_mock, fixed_now):
get_mock.side_effect = [_http_error(403, body="Forbidden")]
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
assert get_mock.call_count == 1
assert result["items"] == []
assert result["error"] == x_api.ERR_FORBIDDEN
assert http.classify_failure(message=result["error"]) == health.AUTH_FAILED
def test_enrollment_fallback_with_window_before_the_floor_sends_no_request(self, get_mock, fixed_now):
"""A window that ends before now-7d cannot be served by recent search:
no second request with start_time after end_time."""
get_mock.side_effect = [_http_error(403, body="client-not-enrolled")]
result = x_api.search_x(DUMMY_TOKEN, "topic", "2026-08-01", "2026-08-20", depth="quick")
assert get_mock.call_count == 1
assert result == {"items": [], "warning": x_api.TRUNCATION_DETAIL}
def test_second_403_on_recent_is_a_fixed_forbidden_error(self, get_mock, fixed_now):
get_mock.side_effect = [
_http_error(403, body="client-not-enrolled"),
_http_error(403, body="client-not-enrolled " + DUMMY_TOKEN),
]
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
assert get_mock.call_count == 2
assert result["error"] == x_api.ERR_FORBIDDEN
# ---------------------------------------------------------------------------
# Fixed-string errors (R8, KTD6)
# ---------------------------------------------------------------------------
# A body that echoes the token and an account id but carries no status
# marker of its own, so the status code alone decides the fixed string.
_LEAKY_BODY = json.dumps({
"title": "Client Error",
"detail": f"request for account {ACCOUNT_ID} rejected; token {DUMMY_TOKEN}",
"client_id": ACCOUNT_ID,
})
def _run_capturing_stderr(fn):
buf = io.StringIO()
with redirect_stderr(buf):
with mock.patch("sys.stderr.isatty", return_value=False, create=True):
out = fn()
return out, buf.getvalue()
class TestFixedErrors:
@pytest.mark.parametrize("status,expected,state", [
(402, x_api.ERR_PAYMENT_REQUIRED, health.PAYMENT_REQUIRED),
(403, x_api.ERR_FORBIDDEN, health.AUTH_FAILED),
(401, x_api.ERR_UNAUTHORIZED, health.AUTH_FAILED),
(429, x_api.ERR_RATE_LIMITED, health.RATE_LIMITED),
])
def test_status_maps_to_fixed_string_without_body(self, get_mock, fixed_now, status, expected, state):
get_mock.side_effect = _http_error(status, body=_LEAKY_BODY, message=f"HTTP {status}: {ACCOUNT_ID}")
result, stderr = _run_capturing_stderr(
lambda: x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
)
assert result["items"] == []
assert result["error"] == expected
assert http.classify_failure(message=result["error"]) == state
for leak in (DUMMY_TOKEN, ACCOUNT_ID, "Client Error", "request for account"):
assert leak not in result["error"]
assert leak not in stderr
assert "[xapi]" in stderr or "xapi" in stderr
def test_fixed_strings_are_the_planned_literals(self):
assert x_api.ERR_PAYMENT_REQUIRED == "xapi: payment required (X API credits exhausted)"
assert x_api.ERR_UNAUTHORIZED == "xapi: unauthorized (bearer token rejected)"
assert x_api.ERR_FORBIDDEN == "xapi: forbidden (bearer token lacks access)"
assert x_api.ERR_RATE_LIMITED == "xapi: rate limit exceeded (X API)"
assert x_api.ERR_TIMED_OUT == "xapi: timed out"
def test_credit_marker_in_body_maps_to_payment_required_on_any_status(self, get_mock, fixed_now):
get_mock.side_effect = _http_error(403, body='{"detail":"insufficient credits"}')
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
assert result["error"] == x_api.ERR_PAYMENT_REQUIRED
def test_other_http_status_is_fixed_http_n(self, get_mock, fixed_now):
get_mock.side_effect = _http_error(503, body=_LEAKY_BODY)
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
assert result["error"] == "xapi: http 503"
def test_timeout_is_fixed_string(self, get_mock, fixed_now):
get_mock.side_effect = http.HTTPError(
f"URL Error: timed out reading {ACCOUNT_ID}", outcome_state=health.TIMEOUT,
)
result = x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
assert result["error"] == x_api.ERR_TIMED_OUT
assert http.classify_failure(message=result["error"]) == health.TIMEOUT
def test_unexpected_exception_never_carries_its_message(self, get_mock, fixed_now):
get_mock.side_effect = ValueError(f"boom {DUMMY_TOKEN} {ACCOUNT_ID}")
result, stderr = _run_capturing_stderr(
lambda: x_api.search_x(DUMMY_TOKEN, "topic", FROM, TO, depth="quick")
)
assert result["error"] == "xapi: request failed (ValueError)"
assert DUMMY_TOKEN not in stderr and ACCOUNT_ID not in stderr
@pytest.mark.parametrize("status,state", [(402, health.PAYMENT_REQUIRED), (403, health.AUTH_FAILED)])
def test_pipeline_outcome_carries_only_the_fixed_string(self, get_mock, fixed_now, status, state):
get_mock.side_effect = _http_error(status, body=_LEAKY_BODY, message=f"HTTP {status}: {ACCOUNT_ID}")
sq = schema.SubQuery(label="primary", search_query="q", ranking_query="q?", sources=["x"])
runtime = schema.ProviderRuntime(
reasoning_provider="mock", planner_model="mock", rerank_model="mock",
x_search_backend=None,
)
def run():
with mock.patch("lib.env.x_backend_chain", return_value=["xapi"]):
with pytest.raises(pipeline.SourceRunError) as ctx:
pipeline._retrieve_stream(
topic="q", subquery=sq, source="x",
config={"X_BEARER_TOKEN": DUMMY_TOKEN},
depth="quick", date_range=(FROM, TO),
runtime=runtime, mock=False,
)
return ctx.value
exc, stderr = _run_capturing_stderr(run)
assert exc.outcome_state == state
rendered = str(exc)
assert DUMMY_TOKEN not in rendered and ACCOUNT_ID not in rendered
assert DUMMY_TOKEN not in stderr and ACCOUNT_ID not in stderr
detail_state, attempted = pipeline._classify_source_failure(exc)
assert (detail_state, attempted) == (state, True)
# ---------------------------------------------------------------------------
# Handle lanes (R7)
# ---------------------------------------------------------------------------
class TestHandleLanes:
def test_from_lane_query_excludes_topic(self, get_mock, fixed_now):
get_mock.return_value = _v2(
[_tweet("1", "anything", created_at="2026-08-20T00:00:00Z")], users=_users(("u1", "steipete")),
)
items = x_api.search_handles(["@steipete"], "Grok 4", FROM, TO, count_per=8, token=DUMMY_TOKEN)
p = _params(get_mock.call_args)
assert p["query"] == "from:steipete -is:retweet"
assert "Grok" not in p["query"]
assert p["start_time"] == "2026-08-10T00:00:00Z"
assert [i["id"] for i in items] == ["XF1"]
assert items[0]["relevance"] >= 0
def test_mention_lane_excludes_subjects_own_posts(self, get_mock, fixed_now):
get_mock.return_value = _v2(
[
_tweet("1", "@steipete nice", author_id="fan"),
_tweet("2", "my own post", author_id="me"),
],
users=_users(("fan", "fanuser"), ("me", "steipete")),
)
items = x_api.search_mentions(["steipete"], FROM, TO, topic="Grok 4", count_per=5, token=DUMMY_TOKEN)
p = _params(get_mock.call_args)
assert p["query"] == "@steipete -from:steipete -is:retweet"
assert {i["author_handle"] for i in items} == {"fanuser"}
assert items[0]["id"].startswith("XA")
def test_invalid_handle_skips_lane_with_receipt(self, get_mock, fixed_now):
items, stderr = _run_capturing_stderr(
lambda: x_api.search_handles(["x OR from:elonmusk"], "t", FROM, TO, token=DUMMY_TOKEN)
)
assert items == []
get_mock.assert_not_called()
assert "skip" in stderr.lower()
mentions, _ = _run_capturing_stderr(
lambda: x_api.search_mentions(["bad'; drop", "a" * 16], FROM, TO, token=DUMMY_TOKEN)
)
assert mentions == []
get_mock.assert_not_called()
def test_item_ids_unique_across_handles(self, get_mock, fixed_now):
get_mock.side_effect = [
_v2([_tweet("1", "a")], users=_users(("u1", "h1"))),
_v2([_tweet("2", "b")], users=_users(("u1", "h2"))),
]
items = x_api.search_handles(["h1", "h2"], "topic", FROM, TO, token=DUMMY_TOKEN)
ids = [i["id"] for i in items]
assert len(ids) == len(set(ids)) == 2
def test_fatal_auth_failure_stops_remaining_handles(self, get_mock, fixed_now):
# Handles fan out on a bounded pool (five in flight), so the handles
# already scheduled beside the failing one may still be called; the
# ones beyond the pool window are not, and nothing leaks.
handles = [f"h{i}" for i in range(1, 8)]
get_mock.side_effect = _http_error(401, body=_LEAKY_BODY)
items, stderr = _run_capturing_stderr(
lambda: x_api.search_handles(handles, "topic", FROM, TO, token=DUMMY_TOKEN)
)
assert items == []
assert 1 <= get_mock.call_count <= x_api._MAX_LANE_WORKERS
called = {_params(c)["query"] for c in get_mock.call_args_list}
assert called <= {f"from:{h} -is:retweet" for h in handles}
assert DUMMY_TOKEN not in stderr and ACCOUNT_ID not in stderr
def test_handle_results_merge_in_handle_order(self, get_mock, fixed_now):
def _by_query(url, headers=None, params=None, **kwargs):
handle = params["query"].split(":")[1].split(" ")[0]
return _v2([_tweet(str(ord(handle[-1])), handle)], users=_users(("u1", handle)))
get_mock.side_effect = _by_query
handles = ["h3", "h1", "h2"]
items = x_api.search_handles(handles, "topic", FROM, TO, token=DUMMY_TOKEN)
assert [i["author_handle"] for i in items] == handles
assert [i["id"] for i in items] == ["XF1", "XF2", "XF3"]
def test_transient_failure_continues_to_next_handle(self, get_mock, fixed_now):
get_mock.side_effect = [_http_error(500), _v2([_tweet("2", "b")], users=_users(("u1", "h2")))]
items = x_api.search_handles(["h1", "h2"], "topic", FROM, TO, token=DUMMY_TOKEN)
assert [i["author_handle"] for i in items] == ["h2"]
def test_no_token_or_handles_returns_empty(self, get_mock):
assert x_api.search_handles(["a"], "t", FROM, TO, token="") == []
assert x_api.search_handles([], "t", FROM, TO, token=DUMMY_TOKEN) == []
assert x_api.search_mentions([], FROM, TO, token=DUMMY_TOKEN) == []
get_mock.assert_not_called()
# ---------------------------------------------------------------------------
# Moved helpers (KTD3)
# ---------------------------------------------------------------------------
class TestSharedHelpers:
def test_grok_x_imports_helpers_from_x_api(self):
from lib import grok_x
assert grok_x._decode_snowflake is x_api._decode_snowflake
assert grok_x._clean_handle is x_api._clean_handle
assert grok_x._looks_generated is x_api._looks_generated
assert grok_x._HANDLE_RE is x_api._HANDLE_RE
def test_snowflake_decodes_to_utc_datetime(self):
when = x_api._decode_snowflake("1956158892141441450")
assert when is not None and when.tzinfo is not None
assert when.year == 2025
assert x_api._decode_snowflake("nope") is None
assert x_api._decode_snowflake("0") is None
def test_clean_handle_grammar(self):
assert x_api._clean_handle("@steipete") == "steipete"
assert x_api._clean_handle("Peter Steinberger") == ""
assert x_api._clean_handle("a" * 16) == ""
assert x_api._clean_handle("x OR from:elonmusk") == ""
def test_looks_generated_flags_uniform_runs(self):
assert x_api._looks_generated(["100", "200", "300", "400"]) is True
assert x_api._looks_generated(["100", "250", "300", "900"]) is False
assert x_api._looks_generated(["1", "2"]) is False
# ---------------------------------------------------------------------------
# Pipeline wiring (R7): lane selection and failover line
# ---------------------------------------------------------------------------
def _make_source_item(source, item_id, url, author=None, body=""):
return schema.SourceItem(
item_id=item_id, source=source, title=f"Item {item_id}", body=body, url=url, author=author,
)
def _make_plan(topic):
return schema.QueryPlan(
intent="exploration", freshness_mode="balanced_recent", cluster_mode="topic",
raw_topic=topic,
subqueries=[schema.SubQuery(
label="primary", search_query=topic,
ranking_query=f"What recent evidence matters for {topic}?", sources=["x"],
)],
source_weights={"x": 1.0},
)
class TestPipelineWiring:
def test_chain_xapi_yields_primary_xapi_and_runs_both_lanes(self):
bundle = schema.RetrievalBundle()
bundle.items_by_source["x"] = [
_make_source_item("x", "X1", "https://x.com/analyst1/status/1", author="analyst1", body="AI safety analysis"),
_make_source_item("x", "X2", "https://x.com/analyst1/status/2", author="analyst1", body="AI safety research"),
]
from_items = [{
"id": "XF1", "text": "from analyst1", "url": "https://x.com/analyst1/status/777",
"author_handle": "analyst1", "date": "2026-03-15",
"engagement": {"likes": 30}, "relevance": 0.8, "why_relevant": "",
}]
runtime = schema.ProviderRuntime(
reasoning_provider="mock", planner_model="mock", rerank_model="mock", x_search_backend=None,
)
with mock.patch("lib.env.x_backend_chain", return_value=["xapi"]), \
mock.patch("lib.entity_extract.extract_entities",
return_value={"x_handles": ["analyst1"], "x_hashtags": [], "reddit_subreddits": []}), \
mock.patch("lib.x_api.search_handles", return_value=from_items) as from_lane, \
mock.patch("lib.x_api.search_mentions", return_value=[]) as about_lane, \
mock.patch("lib.bird_x.search_handles") as bird_lane, \
mock.patch("lib.xquik.search_handles") as xquik_lane:
pipeline._run_supplemental_searches(
topic="AI safety", bundle=bundle, plan=_make_plan("AI safety"),
config={"X_BEARER_TOKEN": DUMMY_TOKEN}, depth="default",
date_range=("2026-02-15", "2026-03-17"), runtime=runtime, mock=False,
rate_limited_sources=set(), rate_limit_lock=threading.Lock(),
)
from_lane.assert_called_once()
about_lane.assert_called_once()
assert from_lane.call_args.kwargs["token"] == DUMMY_TOKEN
assert about_lane.call_args.kwargs["token"] == DUMMY_TOKEN
bird_lane.assert_not_called()
xquik_lane.assert_not_called()
x_urls = {item.url for item in bundle.items_by_source.get("x", [])}
assert "https://x.com/analyst1/status/777" in x_urls
def test_xapi_lanes_share_one_deadline(self):
"""Explicit-from, extracted-from, about, and related lanes must draw
on one wall-clock budget, never a fresh 90s each (review finding)."""
bundle = schema.RetrievalBundle()
bundle.items_by_source["x"] = [
_make_source_item("x", "X1", "https://x.com/analyst1/status/1", author="analyst1", body="AI safety analysis"),
]
runtime = schema.ProviderRuntime(
reasoning_provider="mock", planner_model="mock", rerank_model="mock", x_search_backend=None,
)
with mock.patch("lib.env.x_backend_chain", return_value=["xapi"]), \
mock.patch("lib.entity_extract.extract_entities",
return_value={"x_handles": ["analyst1"], "x_hashtags": [], "reddit_subreddits": []}), \
mock.patch("lib.x_api.search_handles", return_value=[]) as from_lane, \
mock.patch("lib.x_api.search_mentions", return_value=[]) as about_lane:
pipeline._run_supplemental_searches(
topic="AI safety", bundle=bundle, plan=_make_plan("AI safety"),
config={"X_BEARER_TOKEN": DUMMY_TOKEN}, depth="default",
date_range=("2026-02-15", "2026-03-17"), runtime=runtime, mock=False,
rate_limited_sources=set(), rate_limit_lock=threading.Lock(),
x_handle="steipete", x_related=["peer1"],
)
calls = from_lane.call_args_list + about_lane.call_args_list
assert len(calls) >= 3, "explicit from, extracted from, about, related"
deadlines = {c.kwargs.get("deadline") for c in calls}
assert len(deadlines) == 1 and None not in deadlines, deadlines
def test_xapi_lane_deadline_stop_reaches_report_warnings(self):
"""A lane cut short by the deadline is reported as partial coverage,
not presented as complete (review finding)."""
bundle = schema.RetrievalBundle()
bundle.items_by_source["x"] = [
_make_source_item("x", "X1", "https://x.com/analyst1/status/1", author="analyst1", body="AI safety analysis"),
]
runtime = schema.ProviderRuntime(
reasoning_provider="mock", planner_model="mock", rerank_model="mock", x_search_backend=None,
)
def cut_short(*args, **kwargs):
kwargs["warnings"].append(x_api.DEADLINE_DETAIL)
return []
with mock.patch("lib.env.x_backend_chain", return_value=["xapi"]), \
mock.patch("lib.entity_extract.extract_entities",
return_value={"x_handles": [], "x_hashtags": [], "reddit_subreddits": []}), \
mock.patch("lib.x_api.search_handles", side_effect=cut_short), \
mock.patch("lib.x_api.search_mentions", side_effect=cut_short):
pipeline._run_supplemental_searches(
topic="AI safety", bundle=bundle, plan=_make_plan("AI safety"),
config={"X_BEARER_TOKEN": DUMMY_TOKEN}, depth="default",
date_range=("2026-02-15", "2026-03-17"), runtime=runtime, mock=False,
rate_limited_sources=set(), rate_limit_lock=threading.Lock(),
x_handle="steipete",
)
receipts = [w for w in bundle.artifacts.get("x_partial_coverage", []) if x_api.DEADLINE_DETAIL in w]
assert receipts == [f"X handle lanes: {x_api.DEADLINE_DETAIL}"], bundle.artifacts.get("x_partial_coverage")
def test_lane_search_past_a_shared_deadline_sends_no_request(self, get_mock, fixed_now, monkeypatch):
monkeypatch.setattr(x_api.time, "monotonic", lambda: 1000.0)
notes: list[str] = []
assert x_api.search_handles(["steipete", "peer1"], "t", FROM, TO, token=DUMMY_TOKEN, deadline=999.0, warnings=notes) == []
assert x_api.search_mentions(["steipete"], FROM, TO, token=DUMMY_TOKEN, deadline=999.0, warnings=notes) == []
get_mock.assert_not_called()
assert notes == [x_api.DEADLINE_DETAIL], "one receipt per lane call, deduped across handles"
get_mock.return_value = _v2([_tweet("1", "hi")], users=_users(("u1", "steipete")))
assert len(x_api.search_handles(["steipete"], "t", FROM, TO, token=DUMMY_TOKEN, deadline=1001.0)) == 1
def test_all_backends_failed_keeps_the_payment_required_state(self):
"""xapi's 402 must not be masked by a later backend's generic failure:
the outcome that reaches doctor says top up, not re-authenticate."""
plan = {
"intent": "general", "freshness_mode": "balanced_recent", "cluster_mode": "story",
"subqueries": [{
"label": "primary", "search_query": "topic",
"ranking_query": "What are people saying about topic?", "sources": ["x"],
}],
"source_weights": {"x": 1.0},
}
answers = iter([
([], x_api.ERR_PAYMENT_REQUIRED),
([], "request failed (HTTPError)"),
])
with mock.patch("lib.env.x_backend_chain", return_value=["xapi", "xai"]), \
mock.patch("lib.pipeline._fetch_x_backend", side_effect=lambda *a, **k: next(answers)):
report = pipeline.run(
topic="topic", config={"X_BEARER_TOKEN": DUMMY_TOKEN, "XAI_API_KEY": "dummy-xai"},
depth="quick", requested_sources=["x"], mock=False,
external_plan=plan, web_backend="none", save_dir="",
)
assert report.source_status["x"].state == health.PAYMENT_REQUIRED
assert "payment required" in report.source_status["x"].detail
def test_fetch_x_backend_registers_xapi(self, get_mock, fixed_now):
get_mock.return_value = _v2([_tweet("1", "hello topic")], users=_users(("u1", "a")))
items, err = pipeline._fetch_x_backend(
"xapi", "topic", FROM, TO, "quick", {"X_BEARER_TOKEN": DUMMY_TOKEN},
)
assert err == ""
assert [i["id"] for i in items] == ["XAPI1"]
assert get_mock.call_args.kwargs["headers"]["Authorization"] == f"Bearer {DUMMY_TOKEN}"
def test_fetch_x_backend_surfaces_truncation_warning(self, get_mock, fixed_now):
get_mock.side_effect = [
_http_error(403, body="client-not-enrolled"),
_v2([_tweet("1", "hello topic", created_at="2026-09-05T00:00:00Z")]),
]
warnings: list[str] = []
items, err = pipeline._fetch_x_backend(
"xapi", "topic", FROM, TO, "quick", {"X_BEARER_TOKEN": DUMMY_TOKEN}, warnings=warnings,
)
assert err == "" and len(items) == 1
assert warnings and "window truncated to 7 days" in warnings[0]
def test_truncation_receipt_reaches_the_stream_artifact(self, get_mock, fixed_now):
get_mock.side_effect = [
_http_error(403, body="client-not-enrolled"),
_v2([_tweet("1", "hello topic", created_at="2026-09-05T00:00:00Z")]),
]
sq = schema.SubQuery(label="primary", search_query="q", ranking_query="q?", sources=["x"])
runtime = schema.ProviderRuntime(
reasoning_provider="mock", planner_model="mock", rerank_model="mock", x_search_backend=None,
)
def run():
with mock.patch("lib.env.x_backend_chain", return_value=["xapi"]):
return pipeline._retrieve_stream(
topic="q", subquery=sq, source="x", config={"X_BEARER_TOKEN": DUMMY_TOKEN},
depth="quick", date_range=(FROM, TO), runtime=runtime, mock=False,
)
(items, artifact), stderr = _run_capturing_stderr(run)
assert len(items) == 1
assert artifact["x_receipts"] == ["X: xapi window truncated to 7 days"]
assert "window truncated to 7 days" in stderr
def test_failover_stderr_line_names_xapi_and_credits(self):
sq = schema.SubQuery(label="primary", search_query="q", ranking_query="q?", sources=["x"])
runtime = schema.ProviderRuntime(
reasoning_provider="mock", planner_model="mock", rerank_model="mock", x_search_backend=None,
)
good = [{"id": "XAPI1", "text": "t", "url": "https://x.com/a/status/1", "author_handle": "a",
"date": "2026-08-20", "engagement": None, "relevance": 0.5, "why_relevant": ""}]
def fetch(backend, *_args, **_kwargs):
return (good, "") if backend == "xapi" else ([], "")
def run():
with mock.patch("lib.env.x_backend_chain", return_value=["xai", "xapi"]), \
mock.patch("lib.pipeline._fetch_x_backend", side_effect=fetch):
return pipeline._retrieve_stream(
topic="q", subquery=sq, source="x", config={"X_BEARER_TOKEN": DUMMY_TOKEN},
depth="quick", date_range=(FROM, TO), runtime=runtime, mock=False,
)
(items, _artifact), stderr = _run_capturing_stderr(run)
assert len(items) == 1
assert "used fallback 'xapi'" in stderr
assert "X API credits" in stderr
# ---------------------------------------------------------------------------
# Fixture redaction at both seams (R8)
# ---------------------------------------------------------------------------
def _urlopen_response(body: str):
response = MagicMock()
response.__enter__.return_value = response
response.__exit__.return_value = False
response.read.return_value = body.encode("utf-8")
response.status = 200
return response
def test_fixture_redacts_bearer_loaded_only_from_env_file(tmp_path, monkeypatch):
env_file = tmp_path / ".env"
env_file.write_text(f"X_BEARER_TOKEN={DUMMY_TOKEN}\n", encoding="utf-8")
env_file.chmod(0o600)
monkeypatch.setattr(env, "CONFIG_FILE", env_file)
monkeypatch.delenv("X_BEARER_TOKEN", raising=False)
monkeypatch.setenv("LAST30DAYS_SKIP_KEYCHAIN", "1")
monkeypatch.setattr(env, "_load_pass", lambda *a, **k: {})
monkeypatch.setattr(env, "_find_project_env", lambda: None)
monkeypatch.setattr(
http.urllib.request, "urlopen",
lambda *_a, **_k: _urlopen_response(json.dumps({"detail": f"token {DUMMY_TOKEN} rejected"})),
)
fixture_dir = tmp_path / "fixture"
with http.recording_requests(fixture_dir):
config = env.get_config()
assert config["X_BEARER_TOKEN"] == DUMMY_TOKEN
http.get(
"https://api.x.com/2/tweets/search/all",
headers={"Authorization": f"Bearer {DUMMY_TOKEN}"},
params={"query": "topic"},
)
http.fixture_source_record(
{"source": "x", "topic": "t", "search_query": "t", "date_range": [FROM, TO], "depth": "quick"},
[[{"text": f"echo {DUMMY_TOKEN}"}], {}],
)
payload = json.loads((fixture_dir / "http.json").read_text(encoding="utf-8"))
text = json.dumps(payload)
assert DUMMY_TOKEN not in text
assert "<redacted>" in json.dumps(payload["exchanges"])
assert "<redacted>" in json.dumps(payload["source_exchanges"])
def test_http_seam_redacts_bare_bearer_without_any_config(tmp_path, monkeypatch):
monkeypatch.delenv("X_BEARER_TOKEN", raising=False)
monkeypatch.setattr(
http.urllib.request, "urlopen",
lambda *_a, **_k: _urlopen_response(json.dumps({"echo": DUMMY_TOKEN})),
)
fixture_dir = tmp_path / "fixture"
with http.recording_requests(fixture_dir):
http.get("https://api.x.com/2/tweets/search/all", headers={"Authorization": f"Bearer {DUMMY_TOKEN}"})
text = (fixture_dir / "http.json").read_text(encoding="utf-8")
assert DUMMY_TOKEN not in text
assert '"echo": "<redacted>"' in text
# ---------------------------------------------------------------------------
# xurl fixed strings (same rule as xapi)
# ---------------------------------------------------------------------------
class TestXurlFixedStrings:
def _run(self, returncode, stderr="", stdout=""):
completed = mock.Mock(returncode=returncode, stdout=stdout, stderr=stderr)
with mock.patch("subprocess.run", return_value=completed):
return xurl_x.search_x("test")
def test_rate_limited_stderr_with_token_maps_to_fixed_string(self):
result = self._run(1, stderr=f"429 rate limit exceeded for {DUMMY_TOKEN} ({ACCOUNT_ID})")
assert result["error"] == xurl_x.ERR_RATE_LIMITED
assert DUMMY_TOKEN not in result["error"] and ACCOUNT_ID not in result["error"]
assert http.classify_failure(message=result["error"]) == health.RATE_LIMITED
@pytest.mark.parametrize("stderr,expected,state", [
("401 Unauthorized", "ERR_UNAUTHORIZED", health.AUTH_FAILED),
("403 Forbidden: client-not-enrolled", "ERR_FORBIDDEN", health.AUTH_FAILED),
("402 Payment Required", "ERR_PAYMENT_REQUIRED", health.PAYMENT_REQUIRED),
("400 Invalid Request: Ambiguous use of and", "ERR_INVALID_REQUEST", health.ERROR),
("something else entirely", "ERR_FAILED", health.ERROR),
])
def test_status_words_map_to_fixed_strings(self, stderr, expected, state):
result = self._run(1, stderr=f"{stderr} {DUMMY_TOKEN}")
assert result["error"] == getattr(xurl_x, expected)
assert DUMMY_TOKEN not in result["error"]
assert http.classify_failure(message=result["error"]) == state
def test_invalid_json_is_fixed_string(self):
result = self._run(0, stdout=f"NOT JSON {DUMMY_TOKEN}")
assert result["error"] == xurl_x.ERR_INVALID_JSON
assert DUMMY_TOKEN not in result["error"]