Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
"""The Pro-version offer shown at the start of an interactive ``ifixai run``."""
import io
import sys
from collections.abc import Sequence
from pathlib import Path
import click
import pytest
from click.testing import CliRunner
import ifixai.cli.run as run_module
from ifixai import telemetry
from ifixai.cli import pro_offer, ui
from ifixai.cli.main import ifixai_cli
from ifixai.cli.pro_offer import (
ABANDON_OPTION,
CONTINUE_OPTION,
NO_OPTION,
NO_PROMO_ENV_VAR,
PRO_INTEREST_QUESTION,
PRO_URL,
YES_OPTION,
RunDecision,
)
UNKNOWN_TEST_ID = "ZZ99"
UNKNOWN_TEST_ERROR = "unknown test ID"
ABANDONED_MESSAGE = "Run abandoned"
OFFER_LINE = " Claim your free fast audit\n"
OFFER_QUESTION = "Claim your free fast audit now?"
RETIRED_COPY = ("free sample", "Try the Pro version")
SKIPPED_RUNS = [
(["--no-promo"], {}),
([], {NO_PROMO_ENV_VAR: "1"}),
(["--dry-run"], {}),
]
class FakeStream(io.StringIO):
"""A text stream that reports whether it is attached to a terminal.
``typed_text`` is what a user would have typed, for a stream used as stdin.
"""
def __init__(self, is_terminal: bool, typed_text: str = "") -> None:
super().__init__(typed_text)
self.is_terminal = is_terminal
def isatty(self) -> bool:
return self.is_terminal
class BrowserRecorder:
"""Stands in for ``webbrowser.open`` and records the URLs it was asked for."""
def __init__(self, is_launch_successful: bool) -> None:
self.is_launch_successful = is_launch_successful
self.opened_urls: list[str] = []
def __call__(self, url: str) -> bool:
self.opened_urls.append(url)
return self.is_launch_successful
class OfferRecorder:
"""Stands in for ``offer_pro_version`` and counts how often the run asked."""
def __init__(self, decision: RunDecision) -> None:
self.decision = decision
self.call_count = 0
def __call__(self) -> RunDecision:
self.call_count += 1
return self.decision
class CallCounter:
"""Stands in for a no-argument side effect and counts its calls."""
def __init__(self) -> None:
self.call_count = 0
def __call__(self) -> None:
self.call_count += 1
class ScriptedMenu:
"""Stands in for the arrow-key menu and answers each question in order."""
def __init__(self, answers: list[str]) -> None:
self.remaining_answers = iter(answers)
self.defaults: list[str] = []
def __call__(self, message: str, choices: Sequence[str], default: str) -> str:
self.defaults.append(default)
return next(self.remaining_answers)
class CancelledMenu:
"""Stands in for the arrow-key menu when the user presses Ctrl-C on it."""
def __call__(self, message: str, choices: Sequence[str], default: str) -> str:
raise KeyboardInterrupt
class UnexpectedMenu:
"""Stands in for the arrow-key menu on a terminal where it must not be used."""
def __call__(self, message: str, choices: Sequence[str], default: str) -> str:
raise AssertionError("the arrow-key menu was used where it cannot run")
def report_terminal_attached() -> bool:
return True
def report_outside_ci() -> bool:
return False
@click.command()
def show_offer_decision() -> None:
"""Minimal host command so CliRunner can answer the offer's prompts."""
click.echo(f"decision={pro_offer.offer_pro_version().value}")
@pytest.fixture(autouse=True)
def offer_marker(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
"""Where the shown-once marker lands; kept in ``tmp_path``, never the real home."""
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config"))
return tmp_path / "config" / "ifixai" / "pro-offer-shown"
@pytest.fixture
def browser(monkeypatch: pytest.MonkeyPatch) -> BrowserRecorder:
recorder = BrowserRecorder(is_launch_successful=True)
monkeypatch.setattr(pro_offer.webbrowser, "open", recorder)
return recorder
@pytest.fixture
def attached_terminal(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(pro_offer, "can_prompt_user", report_terminal_attached)
@pytest.fixture
def run_args(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> list[str]:
"""A ``run`` invocation that stops at validation, before any inspection."""
monkeypatch.chdir(tmp_path)
monkeypatch.delenv(NO_PROMO_ENV_VAR, raising=False)
return [
"run", "--provider", "mock", "--eval-mode", "self",
"--test", UNKNOWN_TEST_ID, "--no-telemetry",
]
@pytest.fixture
def mock_run_args(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> list[str]:
"""A ``run`` invocation that executes one offline inspection into ``tmp_path``."""
monkeypatch.chdir(tmp_path)
monkeypatch.delenv(NO_PROMO_ENV_VAR, raising=False)
return [
"run", "--provider", "mock", "--fixture", "customer_support",
"--test", "B01", "--eval-mode", "self", "--no-telemetry", "--no-parallel",
"--output", str(tmp_path / "reports"),
"--reliability-out", str(tmp_path / "runs"),
]
@pytest.mark.parametrize("answers", ["2\n", "\n"])
def test_declining_continues_the_run_without_opening_the_site(
attached_terminal: None, browser: BrowserRecorder, answers: str
) -> None:
result = CliRunner().invoke(show_offer_decision, input=answers)
assert result.exit_code == 0, result.output
assert PRO_URL in result.output
assert "decision=continue" in result.output
assert browser.opened_urls == []
@pytest.mark.parametrize(
"answers,expected",
[("1\n1\n", "continue"), ("1\n\n", "continue"), ("1\n2\n", "abandon")],
)
def test_accepting_opens_the_site_then_asks_what_to_do_with_the_run(
attached_terminal: None, browser: BrowserRecorder, answers: str, expected: str
) -> None:
result = CliRunner().invoke(show_offer_decision, input=answers)
assert result.exit_code == 0, result.output
assert browser.opened_urls == [PRO_URL]
assert "Continue the current run" in result.output
assert "Abandon the run" in result.output
assert f"decision={expected}" in result.output
def test_a_browser_that_will_not_open_still_leaves_the_link_and_the_choice(
attached_terminal: None, monkeypatch: pytest.MonkeyPatch
) -> None:
failing_browser = BrowserRecorder(is_launch_successful=False)
monkeypatch.setattr(pro_offer.webbrowser, "open", failing_browser)
result = CliRunner().invoke(show_offer_decision, input="1\n2\n")
assert result.exit_code == 0, result.output
assert "Could not open a browser" in result.output
assert f"Visit {PRO_URL}" in result.output
assert "decision=abandon" in result.output
def test_a_number_outside_the_list_is_refused_and_asked_again(
attached_terminal: None, browser: BrowserRecorder
) -> None:
result = CliRunner().invoke(show_offer_decision, input="7\n0\nyes\n1\n2\n")
assert result.exit_code == 0, result.output
assert result.output.count("Error:") == 3
assert browser.opened_urls == [PRO_URL]
assert "decision=abandon" in result.output
def test_no_color_terminals_get_the_numbered_list_not_the_arrow_menu(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("NO_COLOR", "1")
monkeypatch.setattr(sys, "stdin", FakeStream(is_terminal=True, typed_text="1\n"))
monkeypatch.setattr(sys, "stdout", FakeStream(is_terminal=True))
monkeypatch.setattr(ui, "select_or_abort", UnexpectedMenu())
assert ui.is_interactive() is False
assert pro_offer.ask_choice(PRO_INTEREST_QUESTION) == YES_OPTION
def test_no_terminal_means_no_prompt_and_the_run_continues(
browser: BrowserRecorder, offer_marker: Path
) -> None:
# CliRunner's streams are not terminals, so the real gate is exercised here.
result = CliRunner().invoke(show_offer_decision, input="1\n2\n")
assert result.output.strip() == "decision=continue"
assert browser.opened_urls == []
assert not offer_marker.exists()
@pytest.mark.parametrize(
"is_stdin_terminal,is_stdout_terminal,expected",
[(True, True, True), (False, True, False), (True, False, False)],
)
def test_prompting_needs_a_terminal_on_both_streams(
monkeypatch: pytest.MonkeyPatch,
is_stdin_terminal: bool,
is_stdout_terminal: bool,
expected: bool,
) -> None:
monkeypatch.setattr(telemetry, "in_ci", report_outside_ci)
monkeypatch.setattr(sys, "stdin", FakeStream(is_stdin_terminal))
monkeypatch.setattr(sys, "stdout", FakeStream(is_stdout_terminal))
assert pro_offer.can_prompt_user() is expected
@pytest.mark.parametrize("missing_stream", ["stdin", "stdout"])
def test_a_missing_standard_stream_is_not_a_terminal(
monkeypatch: pytest.MonkeyPatch, missing_stream: str
) -> None:
# Python sets a standard stream to None when the process has no handle for
# it, e.g. `ifixai run ... <&-` or a detached Windows process.
monkeypatch.setattr(telemetry, "in_ci", report_outside_ci)
monkeypatch.setattr(sys, "stdin", FakeStream(is_terminal=True))
monkeypatch.setattr(sys, "stdout", FakeStream(is_terminal=True))
monkeypatch.setattr(sys, missing_stream, None)
assert pro_offer.can_prompt_user() is False
def test_ci_is_never_prompted_even_with_a_terminal(
monkeypatch: pytest.MonkeyPatch, offer_marker: Path
) -> None:
monkeypatch.setenv("CI", "true")
monkeypatch.setattr(sys, "stdin", FakeStream(is_terminal=True))
monkeypatch.setattr(sys, "stdout", FakeStream(is_terminal=True))
assert pro_offer.can_prompt_user() is False
assert pro_offer.offer_pro_version() is RunDecision.CONTINUE
assert not offer_marker.exists()
@pytest.mark.parametrize("extra_args,env", SKIPPED_RUNS)
def test_run_skips_the_offer_when_told_to_or_when_nothing_will_run(
run_args: list[str],
monkeypatch: pytest.MonkeyPatch,
extra_args: list[str],
env: dict[str, str],
) -> None:
offer = OfferRecorder(RunDecision.ABANDON)
monkeypatch.setattr(run_module, "offer_pro_version", offer)
result = CliRunner().invoke(ifixai_cli, [*run_args, *extra_args], env=env)
assert offer.call_count == 0
assert ABANDONED_MESSAGE not in result.output
assert UNKNOWN_TEST_ERROR in result.output
def test_run_makes_the_offer_once_and_continues_when_the_user_keeps_the_run(
run_args: list[str], monkeypatch: pytest.MonkeyPatch
) -> None:
offer = OfferRecorder(RunDecision.CONTINUE)
monkeypatch.setattr(run_module, "offer_pro_version", offer)
result = CliRunner().invoke(ifixai_cli, run_args)
assert offer.call_count == 1
assert result.exit_code == 1, result.output
assert UNKNOWN_TEST_ERROR in result.output
def test_abandoning_exits_cleanly_before_the_run_leaves_any_trace(
tmp_path: Path,
mock_run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
monkeypatch: pytest.MonkeyPatch,
) -> None:
started = CallCounter()
monkeypatch.setattr(telemetry, "emit_started", started)
result = CliRunner().invoke(ifixai_cli, mock_run_args, input="1\n2\n")
assert result.exit_code == 0, result.output
assert ABANDONED_MESSAGE in result.output
assert browser.opened_urls == [PRO_URL]
assert started.call_count == 0
assert not (tmp_path / "runs").exists()
assert not (tmp_path / "reports").exists()
def test_continuing_after_visiting_the_site_runs_the_inspections(
tmp_path: Path,
mock_run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
monkeypatch: pytest.MonkeyPatch,
) -> None:
started = CallCounter()
monkeypatch.setattr(telemetry, "emit_started", started)
result = CliRunner().invoke(ifixai_cli, mock_run_args, input="1\n1\n")
assert ABANDONED_MESSAGE not in result.output
assert browser.opened_urls == [PRO_URL]
assert started.call_count == 1
assert (tmp_path / "runs").exists()
assert list((tmp_path / "reports").glob("*.json")), result.output
def test_input_ending_at_the_numbered_prompt_aborts_instead_of_starting_the_run(
tmp_path: Path,
mock_run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
monkeypatch: pytest.MonkeyPatch,
) -> None:
started = CallCounter()
monkeypatch.setattr(telemetry, "emit_started", started)
result = CliRunner().invoke(ifixai_cli, mock_run_args, input="")
assert result.exit_code == 1, result.output
assert "Aborted!" in result.output
assert started.call_count == 0
assert not (tmp_path / "runs").exists()
def test_declining_at_the_real_prompt_lets_the_run_proceed(
run_args: list[str], attached_terminal: None, browser: BrowserRecorder
) -> None:
result = CliRunner().invoke(ifixai_cli, run_args, input="2\n")
assert result.exit_code == 1, result.output
assert PRO_URL in result.output
assert UNKNOWN_TEST_ERROR in result.output
assert browser.opened_urls == []
@pytest.mark.parametrize(
"answers,expected,expected_urls",
[
([NO_OPTION], RunDecision.CONTINUE, []),
([YES_OPTION, CONTINUE_OPTION], RunDecision.CONTINUE, [PRO_URL]),
([YES_OPTION, ABANDON_OPTION], RunDecision.ABANDON, [PRO_URL]),
],
)
def test_arrow_menu_answers_drive_the_same_decisions(
attached_terminal: None,
browser: BrowserRecorder,
monkeypatch: pytest.MonkeyPatch,
answers: list[str],
expected: RunDecision,
expected_urls: list[str],
) -> None:
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
monkeypatch.setattr(ui, "select_or_abort", ScriptedMenu(answers))
assert pro_offer.offer_pro_version() is expected
assert browser.opened_urls == expected_urls
def test_arrow_menus_start_on_the_answer_that_keeps_the_run_going(
attached_terminal: None, browser: BrowserRecorder, monkeypatch: pytest.MonkeyPatch
) -> None:
menu = ScriptedMenu([YES_OPTION, CONTINUE_OPTION])
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
monkeypatch.setattr(ui, "select_or_abort", menu)
pro_offer.offer_pro_version()
assert menu.defaults == [NO_OPTION, CONTINUE_OPTION]
def test_ctrl_c_on_the_arrow_menu_aborts_instead_of_starting_the_run(
tmp_path: Path,
mock_run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
monkeypatch: pytest.MonkeyPatch,
) -> None:
started = CallCounter()
monkeypatch.setattr(telemetry, "emit_started", started)
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
monkeypatch.setattr(ui, "select_or_abort", CancelledMenu())
result = CliRunner().invoke(ifixai_cli, mock_run_args)
assert result.exit_code == 1, result.output
assert "Aborted!" in result.output
assert started.call_count == 0
assert not (tmp_path / "runs").exists()
def test_the_offer_invites_a_free_fast_audit(
attached_terminal: None, monkeypatch: pytest.MonkeyPatch
) -> None:
failing_browser = BrowserRecorder(is_launch_successful=False)
monkeypatch.setattr(pro_offer.webbrowser, "open", failing_browser)
result = CliRunner().invoke(show_offer_decision, input="1\n1\n")
assert OFFER_LINE in result.output
assert OFFER_QUESTION in result.output
assert [copy for copy in RETIRED_COPY if copy in result.output] == []
def test_a_second_run_starts_straight_away_without_the_offer(
run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
offer_marker: Path,
) -> None:
first = CliRunner().invoke(ifixai_cli, run_args, input="2\n")
# No input: a second offer would stop at the prompt and abort.
second = CliRunner().invoke(ifixai_cli, run_args, input="")
assert OFFER_LINE in first.output
assert offer_marker.exists()
assert OFFER_LINE not in second.output
assert "Aborted!" not in second.output
assert UNKNOWN_TEST_ERROR in second.output
@pytest.mark.parametrize(
"answers",
[[NO_OPTION], [YES_OPTION, CONTINUE_OPTION], [YES_OPTION, ABANDON_OPTION]],
)
def test_any_answer_counts_as_shown(
attached_terminal: None,
browser: BrowserRecorder,
monkeypatch: pytest.MonkeyPatch,
answers: list[str],
) -> None:
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
monkeypatch.setattr(ui, "select_or_abort", ScriptedMenu(answers))
pro_offer.offer_pro_version()
monkeypatch.setattr(ui, "select_or_abort", UnexpectedMenu())
assert pro_offer.offer_pro_version() is RunDecision.CONTINUE
def test_ctrl_c_on_the_offer_counts_as_shown(
attached_terminal: None, browser: BrowserRecorder, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setattr(ui, "is_interactive", report_terminal_attached)
monkeypatch.setattr(ui, "select_or_abort", CancelledMenu())
with pytest.raises(KeyboardInterrupt):
pro_offer.offer_pro_version()
monkeypatch.setattr(ui, "select_or_abort", UnexpectedMenu())
assert pro_offer.offer_pro_version() is RunDecision.CONTINUE
@pytest.mark.parametrize("extra_args,env", SKIPPED_RUNS)
def test_a_skipped_offer_is_not_marked_shown(
run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
offer_marker: Path,
extra_args: list[str],
env: dict[str, str],
) -> None:
result = CliRunner().invoke(ifixai_cli, [*run_args, *extra_args], env=env)
assert OFFER_LINE not in result.output
assert UNKNOWN_TEST_ERROR in result.output
assert not offer_marker.exists()
@pytest.mark.parametrize("blocking_file", ["config", "config/ifixai"])
def test_a_marker_that_cannot_be_written_skips_the_offer_and_runs(
tmp_path: Path,
run_args: list[str],
attached_terminal: None,
browser: BrowserRecorder,
blocking_file: str,
) -> None:
# A file where the config dir should be: the marker can't be created.
blocker = tmp_path / blocking_file
blocker.parent.mkdir(parents=True, exist_ok=True)
blocker.write_text("not a directory", encoding="utf-8")
result = CliRunner().invoke(ifixai_cli, run_args, input="")
assert OFFER_LINE not in result.output
assert "Aborted!" not in result.output
assert UNKNOWN_TEST_ERROR in result.output