Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

15.1 KB
Raw Blame
"""Measure what coding agents learn from the AGENTS.md that `crewai create` ships.

Each question in the item bank is asked in a fresh, tool-less coding-agent session
(Claude Code or Codex, per model) inside a directory holding only the instruction
files, once with this branch's AGENTS.md and once with the base branch's. Answers
are graded by the item's answer key. The command fails when any model's score drops
more than ``--max-drop`` below the base, or when too many sessions give no answer
for the comparison to mean anything.
"""

from __future__ import annotations

from collections import Counter
from collections.abc import Callable, Sequence
from concurrent.futures import ThreadPoolExecutor
import json
import os
from pathlib import Path
import re
import subprocess
import tempfile
from typing import Final, Literal, NotRequired, TypedDict

import click
from rich.console import Console
from rich.markup import escape
from rich.table import Table


console = Console()

Grade = Literal["right", "partly", "wrong"]
Agent = Literal["claude", "codex"]
# One session: the agent's answer; raises NoAnswer (or TimeoutExpired) when there is none.
Ask = Callable[[Path, str], str]

ITEMS_FILE: Final = Path(__file__).parent / "agents_md_items.json"
TEMPLATES: Final = "lib/cli/src/crewai_cli/templates"
POINTS: Final[dict[Grade, float]] = {"right": 1.0, "partly": 0.5, "wrong": 0.0}
MODELS: Final = (
    "claude:claude-haiku-5-5",
    "claude:claude-sonnet-5-5",
    "codex:gpt-5.6-luna",
    "codex:gpt-5.6-sol",
)
SESSION_TIMEOUT: Final = 300
# Past this share of unanswered sessions, a score says more about the agent's
# availability than about the file.
MAX_UNANSWERED: Final = 0.02
# Every tool disabled, no MCP servers, only the project's settings: answers come
# from the instruction files and the model.
CLAUDE_CMD: Final = [
    "claude", "-p", "--tools", "", "--setting-sources", "project",
    "--strict-mcp-config", "--mcp-config", '{"mcpServers":{}}', "--no-session-persistence",
]  # fmt: skip
CODEX_EXEC: Final = [
    "codex",
    "exec",
    "--skip-git-repo-check",
    "--ephemeral",
    "-s",
    "read-only",
    "--json",
]
# Neither agent needs the other's key; Codex reads its own from CODEX_HOME.
API_KEYS: Final = ("ANTHROPIC_API_KEY", "OPENAI_API_KEY")
# Codex can't run without tools, so the question says not to use them and the
# sandbox is read-only.
NO_COMMANDS: Final = (
    "\n\nAnswer from what you know and this project's instructions; don't run commands."
)
EVALUATION_START: Final = "<!-- crewai-eval:start -->"
EVALUATION_END: Final = "<!-- crewai-eval:end -->"
# A trap counts only when the answer recommends it, not when it warns against it,
# before ("don't run `crewai update`") or after ("`crewai update` won't do this").
# Answers use straight or typographic (U+2019) apostrophes.
NEGATION: Final = re.compile(
    r"\b(don['\u2019]?t|doesn['\u2019]?t|didn['\u2019]?t|aren['\u2019]?t|can['\u2019]?t|do not|never|not"
    r"|won['\u2019]?t|isn['\u2019]?t|avoid|instead of|rather than|no need to"
    r"|replac(es|ed|ing)|deprecated|legacy)\b",
    re.I,
)
AFTER_NEGATION: Final = re.compile(
    r"[`'\"]?\s*(won['\u2019]?t|isn['\u2019]?t|doesn['\u2019]?t|does not|is not|is something else|is a different"
    r"|does something different|migrates)\b",
    re.I,
)


class Item(TypedDict):
    """One fact the file should teach, asked as a question and graded by its key."""

    id: str
    item: str
    q: str
    key: list[str]
    partial: NotRequired[list[str]]
    trap: NotRequired[list[str]]


def load_items(path: Path = ITEMS_FILE) -> list[Item]:
    """Read the item bank."""
    items: list[Item] = json.loads(path.read_text(encoding="utf-8"))
    return items


def grade(item: Item, answer: str) -> Grade:
    """Grade an answer: a recommended trap is wrong, every key right, any partial partly."""

    def hit(pattern: str) -> bool:
        return re.search(pattern, answer, re.I | re.S) is not None

    def recommends(pattern: str) -> bool:
        return any(
            not NEGATION.search(answer[max(0, m.start() - 60) : m.start()])
            and not AFTER_NEGATION.match(answer[m.end() : m.end() + 50])
            for m in re.finditer(pattern, answer, re.I | re.S)
        )

    if any(recommends(p) for p in item.get("trap", [])):
        return "wrong"
    if all(hit(p) for p in item["key"]):
        return "right"
    if any(hit(p) for p in item.get("partial", [])):
        return "partly"
    return "wrong"


def shipped_text(template: str) -> str:
    """The AGENTS.md a crew or flow project receives: evaluation blocks kept, markers dropped.

    Mirrors ``crewai_cli.utils._assistant_instructions(text, evaluation=True)``; a test
    holds the two to the same output.
    """
    return "".join(
        line
        for line in template.splitlines(keepends=True)
        if line.strip() not in (EVALUATION_START, EVALUATION_END)
    )


def read_at(ref: str | None, path: str) -> str:
    """A repo file's text at a git ref, or in the working tree when ``ref`` is None."""
    if ref is None:
        root = subprocess.run(
            ["git", "rev-parse", "--show-toplevel"],  # noqa: S607 - git from PATH, fixed argv
            capture_output=True,
            text=True,
            check=True,
        ).stdout.strip()
        return (Path(root) / path).read_text(encoding="utf-8")
    return subprocess.run(  # noqa: S603 - fixed argv; ref is a git ref the caller passed
        ["git", "show", f"{ref}:{path}"],  # noqa: S607 - git from PATH
        capture_output=True,
        text=True,
        check=True,
    ).stdout


def write_project(dest: Path, ref: str | None) -> None:
    """Write the instruction files a new project gets at ``ref`` into ``dest``."""
    dest.mkdir(parents=True, exist_ok=True)
    (dest / "AGENTS.md").write_text(
        shipped_text(read_at(ref, f"{TEMPLATES}/AGENTS.md")), encoding="utf-8"
    )
    (dest / "CLAUDE.md").write_text(
        read_at(ref, f"{TEMPLATES}/CLAUDE.md"), encoding="utf-8"
    )


class NoAnswer(Exception):  # noqa: N818 - a reason, not an error the caller handles as one
    """A session that ended without an answer; the message says why."""


SECRET: Final = re.compile(r"sk-[A-Za-z0-9_-]+")
# Claude Code can end its output with a JSON log line ({"model": ..., "query_source": ...})
# after the message that says what went wrong.
LOG_LINE: Final = re.compile(r"^\{.*\}$")


def _lines(output: str) -> list[str]:
    return [line.strip() for line in output.splitlines() if line.strip()]


def session_answer(proc: subprocess.CompletedProcess[str], text: str) -> str:
    """The session's answer, or NoAnswer naming its exit code and last error line."""
    if proc.returncode != 0:
        # Claude Code prints some failures ("Not logged in", "Invalid API key") on stdout.
        stderr, stdout = _lines(proc.stderr), _lines(proc.stdout)
        messages = [line for line in stderr if not LOG_LINE.match(line)] or [
            line for line in stdout if not LOG_LINE.match(line)
        ]
        line = (messages or stderr or stdout or ["no error output"])[-1]
        raise NoAnswer(SECRET.sub("sk-***", f"exit {proc.returncode}: {line[:160]}"))
    if not text:
        raise NoAnswer("empty answer")
    return text


def claude_asker(model: str) -> Ask:
    """Claude Code sessions with every tool disabled, so answers come from the files."""
    env = {
        k: v
        for k, v in os.environ.items()
        if not k.startswith("CLAUDE_CODE") and k not in ("CLAUDECODE", "OPENAI_API_KEY")
    }

    def ask(project: Path, question: str) -> str:
        proc = subprocess.run(  # noqa: S603 - fixed argv; the question goes on stdin
            [*CLAUDE_CMD, "--model", model],
            input=question,
            capture_output=True,
            text=True,
            cwd=project,
            env=env,
            timeout=SESSION_TIMEOUT,
            check=False,
        )
        return session_answer(proc, proc.stdout.strip())

    return ask


def codex_answer(events: str) -> str:
    """The agent's messages from ``codex exec --json`` output."""
    messages = []
    for line in events.splitlines():
        if not line.startswith("{"):
            continue
        event = json.loads(line)
        item = event.get("item") or {}
        if (
            event.get("type") == "item.completed"
            and item.get("type") == "agent_message"
        ):
            messages.append(str(item.get("text") or ""))
    return "\n".join(messages)


def codex_asker(codex_home: Path, model: str) -> Ask:
    """Codex sessions logged in with ``OPENAI_API_KEY``, in a home of their own."""
    codex_home.mkdir(parents=True, exist_ok=True)  # codex refuses a missing CODEX_HOME
    subprocess.run(
        ["codex", "login", "--with-api-key"],  # noqa: S607 - CLI from PATH
        input=os.environ["OPENAI_API_KEY"],
        capture_output=True,
        text=True,
        check=True,
        env={**os.environ, "CODEX_HOME": str(codex_home)},
    )
    env = {k: v for k, v in os.environ.items() if k not in API_KEYS}
    env["CODEX_HOME"] = str(codex_home)

    def ask(project: Path, question: str) -> str:
        proc = subprocess.run(  # noqa: S603 - the question is one argv entry, never a shell
            [*CODEX_EXEC, "-m", model, question + NO_COMMANDS],
            stdin=subprocess.DEVNULL,
            capture_output=True,
            text=True,
            cwd=project,
            timeout=SESSION_TIMEOUT,
            check=False,
            env=env,
        )
        return session_answer(proc, codex_answer(proc.stdout))

    return ask


def attempt(ask: Ask, project: Path, question: str) -> tuple[str | None, str]:
    """One session: its answer, or None and why there was none."""
    try:
        return ask(project, question), ""
    except subprocess.TimeoutExpired:
        return None, f"timed out after {SESSION_TIMEOUT}s"
    except NoAnswer as e:
        return None, str(e)


def answer(ask: Ask, project: Path, question: str) -> tuple[str | None, str]:
    """The session's answer with one retry, or None and why the last attempt gave none."""
    text, reason = attempt(ask, project, question)
    return (text, reason) if text is not None else attempt(ask, project, question)


def score(
    ask: Ask, project: Path, items: Sequence[Item], reps: int, workers: int
) -> tuple[dict[str, float], Counter[str]]:
    """Average points per item over ``reps`` sessions, and why sessions gave no answer.

    A session with no answer scores zero and is counted by reason, so the caller can
    tell a file that teaches less from an agent that wasn't answering.
    """
    jobs = [item for item in items for _ in range(reps)]
    with ThreadPoolExecutor(max_workers=workers) as pool:
        answers = list(pool.map(lambda item: answer(ask, project, item["q"]), jobs))
    totals: dict[str, float] = {}
    missing: Counter[str] = Counter()
    for item, (text, reason) in zip(jobs, answers, strict=True):
        if text is None:
            missing[reason] += 1
        points = POINTS[grade(item, text)] if text is not None else 0.0
        totals[item["id"]] = totals.get(item["id"], 0.0) + points / reps
    return totals, missing


def dropped(base: float, head: float, max_drop: float) -> bool:
    """Whether ``head`` is more than ``max_drop`` (a fraction) below ``base``."""
    return head < base * (1 - max_drop)


def parse_model(spec: str) -> tuple[Agent, str]:
    """Split ``claude:<model>`` or ``codex:<model>``."""
    cli, _, model = spec.partition(":")
    if cli == "claude" and model:
        return "claude", model
    if cli == "codex" and model:
        return "codex", model
    raise click.BadParameter(f"{spec!r}: use claude:<model> or codex:<model>")


@click.command("agents-md-eval")
@click.option(
    "--base",
    default="origin/main",
    show_default=True,
    help="Git ref whose AGENTS.md is the baseline.",
)
@click.option(
    "--head",
    default=None,
    help="Git ref whose AGENTS.md is evaluated. Default: the working tree.",
)
@click.option(
    "--model",
    "models",
    multiple=True,
    default=MODELS,
    show_default=True,
    help="Agent and model to ask, as claude:<model> or codex:<model>. Repeatable.",
)
@click.option("--reps", default=1, show_default=True, help="Sessions per question.")
@click.option("--workers", default=8, show_default=True, help="Sessions at once.")
@click.option(
    "--max-drop",
    default=0.10,
    show_default=True,
    help="Largest allowed score drop, as a fraction.",
)
def agents_md_eval(
    base: str,
    head: str | None,
    models: tuple[str, ...],
    reps: int,
    workers: int,
    max_drop: float,
) -> None:
    """Fail when this branch's AGENTS.md teaches coding agents less than BASE's."""
    specs = [parse_model(spec) for spec in models]
    items = load_items()
    sessions = len(items) * reps
    problems: list[str] = []
    table = Table("Model", "Base", "This branch", "Change", "No answer (base / branch)")
    with tempfile.TemporaryDirectory() as tmp:
        root = Path(tmp)
        write_project(root / "base", base)
        write_project(root / "head", head)
        for cli, model in specs:
            label = f"{cli}:{model}"
            ask = (
                claude_asker(model)
                if cli == "claude"
                else codex_asker(root / "codex-home", model)
            )
            console.print(f"Asking {label}: {sessions} sessions per file")
            before, base_missing = score(ask, root / "base", items, reps, workers)
            after, head_missing = score(ask, root / "head", items, reps, workers)
            old, new = sum(before.values()), sum(after.values())
            change = (new - old) / old if old else 0.0
            base_n, head_n = base_missing.total(), head_missing.total()
            table.add_row(
                label,
                f"{old:.1f}",
                f"{new:.1f}",
                f"{change:+.1%}",
                f"{base_n} / {head_n}",
            )
            for file, reasons in (("base", base_missing), ("branch", head_missing)):
                if reasons:
                    why = "; ".join(
                        f"{reason} ({n})" for reason, n in reasons.most_common(3)
                    )
                    # Reasons and items are data: `[Errno 8]` and `[tool.crewai]` aren't markup.
                    console.print(
                        f"[yellow]{label} gave no answer with the {file} file: {escape(why)}[/yellow]"
                    )
            if max(base_n, head_n) > sessions * MAX_UNANSWERED:
                problems.append(
                    f"{label}: too many sessions gave no answer ({base_n} / {head_n} of {sessions})"
                    ", so the comparison isn't reliable."
                )
            elif dropped(old, new, max_drop):
                problems.append(
                    f"{label}: scored {change:.1%} against {base}, past the {max_drop:.0%} limit."
                )
            for item in items:
                if after[item["id"]] < before[item["id"]] - 0.5:
                    console.print(
                        f"[yellow]{label} lost {item['id']}:[/yellow] {escape(item['item'])}"
                    )
    console.print(table)
    for problem in problems:
        console.print(f"[red]{escape(problem)}[/red]")
    if problems:
        raise SystemExit(1)