Skip to content

Turn any AI agent into an AI Scientist. The #1 Agent Skills library for science, used by 250,000+ scientists worldwide. 177 ready-to-use validated skills plus 100+ scientific databases covering biology, chemistry, medicine, and drug discovery. Compatible with Cursor, Claude Code, Codex, Pi, Antigravity, and the open Agent Skills standard.

8.3 KB
Raw Blame
"""`--help` contract for a skill's bundled command-line scripts.

Unlike `structure`, this half actually runs the scripts, so what it proves
depends on the environment. In the bare project environment most scientific
packages are absent and the affected scripts skip; under
`python tests/run_all.py --isolated`, where each skill gets the packages it
documents, the same tests run for real. That is the point of the split: one
definition, honest in both places.

Each script runs in its own subprocess -- `--help` must not need this
interpreter's `sys.path`, and a script that hangs must not hang the suite.
"""

from __future__ import annotations

import ast
import os
import re
import subprocess
import sys
import unittest
from pathlib import Path

DEFAULT_TIMEOUT = 120

#: Never invoked directly: shared helpers and package markers, not CLIs.
NON_CLI_NAMES = frozenset({"__init__.py", "__main__.py", "_common.py", "conftest.py"})


def cli_scripts(skill_root: Path) -> list[Path]:
    """Top-level scripts under `scripts/` that build an argparse parser.

    Only the top level: nested directories under `scripts/` are importable
    libraries, not entry points.
    """
    scripts = []
    for path in sorted((skill_root / "scripts").glob("*.py")):
        if path.name in NON_CLI_NAMES:
            continue
        try:
            tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
        except SyntaxError:
            continue  # structure.compile_problems reports it
        imports_argparse = any(
            (isinstance(node, ast.Import) and any(a.name == "argparse" for a in node.names))
            or (isinstance(node, ast.ImportFrom) and node.module == "argparse")
            for node in ast.walk(tree)
        )
        if imports_argparse:
            scripts.append(path)
    return scripts


def run_help(script: Path, timeout: int = DEFAULT_TIMEOUT) -> subprocess.CompletedProcess:
    environment = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}
    return subprocess.run(
        [sys.executable, str(script), "--help"],
        capture_output=True,
        text=True,
        timeout=timeout,
        env=environment,
        cwd=script.parent,
    )


#: A script that guards its own imports and prints a friendly install hint --
#: better behaviour than a traceback -- must still be recognised as "package
#: absent" rather than reported as a broken CLI. `exa-search` does exactly this.
_GRACEFUL_IMPORT_GUARD = re.compile(
    r"(?:^|\n)\s*([\w.\-]+) (?:is )?not installed\b"
    r"|No module named ['\"]?([\w.]+)"
    r"|pip install ([\w.\-\[\]]+)",
    re.IGNORECASE,
)


def _missing_dependency(output: str) -> str | None:
    """The package name, when a failed `--help` was caused by an absent import.

    Deliberately narrow. `ImportError: cannot import name 'PSICQUIC' from
    'bioservices'` means the package IS installed and the script is calling an
    API that no longer exists -- upstream drift, the most valuable thing this
    contract can catch. Treating that as "package absent" would skip the test
    and hide the breakage, so only a genuinely absent module counts:

    * `ModuleNotFoundError`, or an `ImportError` naming no such module;
    * a script that catches the ImportError itself and prints an install hint.
    """
    for line in reversed(output.strip().splitlines()):
        if line.startswith("ModuleNotFoundError:"):
            return line.split(":", 1)[1].strip()
        if line.startswith("ImportError:"):
            message = line.split(":", 1)[1].strip()
            return message if "No module named" in message else None

    match = _GRACEFUL_IMPORT_GUARD.search(output)
    if match:
        return next(group for group in match.groups() if group)
    return None


def help_test_case(
    skill_root: Path,
    *,
    skip: frozenset[str] | set[str] | tuple[str, ...] = (),
    timeout: int = DEFAULT_TIMEOUT,
) -> type[unittest.TestCase]:
    """Build the `--help` TestCase for one skill.

    `skip` names scripts (by filename) that legitimately have no working
    `--help` -- pass it sparingly and say why at the call site.
    """
    skipped = frozenset(skip)

    class CliHelpContractTests(unittest.TestCase):
        maxDiff = None

        def test_every_cli_answers_help(self) -> None:
            scripts = [
                path for path in cli_scripts(skill_root) if path.name not in skipped
            ]
            self.assertTrue(
                scripts,
                f"{skill_root.name} ships no argparse CLI -- drop this test case "
                "or fix the discovery",
            )
            for script in scripts:
                with self.subTest(script=script.name):
                    try:
                        result = run_help(script, timeout=timeout)
                    except subprocess.TimeoutExpired:
                        self.fail(
                            f"{script.name} --help did not return within {timeout}s; "
                            "it is doing work before parsing arguments"
                        )
                    if result.returncode != 0:
                        missing = _missing_dependency(result.stderr + result.stdout)
                        if missing:
                            self.skipTest(
                                f"{script.name} needs an uninstalled package ({missing}); "
                                "run under `tests/run_all.py --isolated` to exercise it"
                            )
                        self.fail(
                            f"{script.name} --help exited {result.returncode}:\n"
                            f"{result.stderr.strip()}"
                        )
                    self.assertIn(
                        "usage:",
                        result.stdout.lower(),
                        f"{script.name} --help printed no usage line",
                    )
                    self.assertNotIn("Traceback", result.stderr)

    CliHelpContractTests.__qualname__ = f"CliHelpContractTests[{skill_root.name}]"
    return CliHelpContractTests


def demo_test_case(
    skill_root: Path,
    scripts: tuple[str, ...],
    *,
    timeout: int = DEFAULT_TIMEOUT,
) -> type[unittest.TestCase]:
    """Build a TestCase that runs a library script's `__main__` demo block.

    Some skills ship importable modules rather than CLIs, with a worked example
    under `if __name__ == "__main__":`. That example is documentation, and
    documentation rots -- so run it and require a clean exit.

    Opt-in and explicit: pass the filenames, because a demo block that writes
    files or takes minutes should not be run by accident.
    """

    class DemoBlockTests(unittest.TestCase):
        maxDiff = None

        def test_every_demo_block_runs_clean(self) -> None:
            for name in scripts:
                script = skill_root / "scripts" / name
                with self.subTest(script=name):
                    self.assertTrue(script.is_file(), f"{name} is not shipped")
                    environment = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}
                    # Headless: a demo that plots must not open a window.
                    environment.setdefault("MPLBACKEND", "Agg")
                    try:
                        result = subprocess.run(
                            [sys.executable, str(script)],
                            capture_output=True,
                            text=True,
                            timeout=timeout,
                            env=environment,
                            cwd=script.parent,
                        )
                    except subprocess.TimeoutExpired:
                        self.fail(f"{name} did not finish within {timeout}s")
                    if result.returncode != 0:
                        missing = _missing_dependency(result.stderr + result.stdout)
                        if missing:
                            self.skipTest(
                                f"{name} needs an uninstalled package ({missing}); "
                                "run under `tests/run_all.py --isolated`"
                            )
                        self.fail(
                            f"{name} exited {result.returncode}:\n{result.stderr.strip()}"
                        )
                    self.assertTrue(
                        result.stdout.strip(), f"{name} printed nothing"
                    )

    DemoBlockTests.__qualname__ = f"DemoBlockTests[{skill_root.name}]"
    return DemoBlockTests