Skip to content

A complete AI agency at your fingertips - From frontend wizards to Reddit community ninjas, from whimsy injectors to reality checkers. Each agent is a specialized expert with personality, processes, and proven deliverables.

Shell
29.5 KB
Raw Blame
#!/usr/bin/env bash
#
# test-convert-outputs.sh — regression eval for the GENERATED product.
#
# Why: every converter bug so far passed lint and the existing tests while the
# product users actually install was broken. #778 shipped a double-wrapped
# description ('"..."') that parsed as valid YAML, so a wrapper check passed;
# #817 dropped a whole tool from --parallel and every remaining tool still
# looked fine. Both are invariant violations, not syntax errors. This script
# encodes what "correct output" means and checks all of it, for every agent,
# for every converted tool.
#
# Layer A — invariants (need no history):
#   round-trip   parsed(generated).description == source description
#   strict-parse every generated frontmatter/TOML/YAML parses with a real parser
#   count        every tool emits exactly one output per roster agent
#   source       every SOURCE agent's frontmatter strict-parses (the desktop app
#                reads sources with js-yaml — #473 was exactly this) and its
#                description carries no leaked quote character
#
# Layer B — drift (needs the committed manifest):
#   scripts/convert-outputs.sha256 (v2) holds
#     agent  <slug>  <hash>   one line per roster agent: that agent's generated output
#                             across every tool (its files, its section of the
#                             accumulated aider/windsurf files, its hermes JSON entry)
#     tool   <tool>  <hash>   the tool's NON-agent files (README, plugin code, manifests):
#                             moves only when a generator/template changes
#     contract <file> <hash>  divisions.json, tools.json, runbooks.json
#   Adding or editing one agent flips exactly its own line, so two agent PRs
#   never collide on this file. Hashes are platform-neutral: forward-slash paths
#   and LF line endings, so a Windows checkout produces the same manifest.
#
#   Contributors adding/editing agents do NOT need to touch the manifest: CI runs
#   this with --drift=advisory on pull requests (drift is printed, not failed) and
#   maintainers regenerate it when the PR lands. A generator change should ship
#   with --update so the tool line moves in the same commit.
#
# Usage:
#   ./scripts/test-convert-outputs.sh                   # generate into a temp dir, check everything
#   ./scripts/test-convert-outputs.sh --update          # ...and rewrite the manifest
#   ./scripts/test-convert-outputs.sh --drift=advisory  # drift is reported but does not fail (CI on PRs)
#   ./scripts/test-convert-outputs.sh --out=DIR         # check an already-generated DIR (no generation)
#
# Exit 0 only when every invariant passes AND the manifest matches (or --update,
# or --drift=advisory).
# Runs on bash 3.2 (macOS) and 5 (Linux); parsing is done by python3.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
MANIFEST="$REPO_ROOT/scripts/convert-outputs.sha256"

UPDATE=false; DIFF=false; OUT=""; DRIFT=strict
for a in "$@"; do
  case "$a" in
    --update) UPDATE=true ;;
    --diff)   DIFF=true ;;
    --drift=advisory) DRIFT=advisory ;;
    --drift=strict)   DRIFT=strict ;;
    --out=*)  OUT="${a#--out=}" ;;
    -h|--help) sed -n '2,44p' "$0"; exit 0 ;;
    *) printf 'unknown flag: %s\n' "$a" >&2; exit 2 ;;
  esac
done

command -v python3 >/dev/null 2>&1 || { echo "ERROR: python3 is required." >&2; exit 2; }
python3 -c 'import yaml, tomllib' 2>/dev/null \
  || { echo "ERROR: python3 needs PyYAML and tomllib (3.11+)." >&2; exit 2; }

# get_field (quote-aware) and agent_slug — the same helpers convert.sh uses,
# so the "expected" side is derived exactly the way the generator derives it.
# shellcheck source=lib.sh
source "$SCRIPT_DIR/lib.sh"

TMP="$(mktemp -d "${TMPDIR:-/tmp}/agency-convert-outputs.XXXXXX")"
trap 'rm -rf "$TMP"' EXIT

# --- roster: every agent file under a registered division --------------------
divisions_from_json() {
  awk '/"divisions"[[:space:]]*:[[:space:]]*\{/{f=1; next} f' "$REPO_ROOT/divisions.json" \
    | grep -oE '^[[:space:]]*"[a-z0-9-]+"[[:space:]]*:' \
    | sed -E 's/[[:space:]]*"([a-z0-9-]+)"[[:space:]]*:/\1/'
}

SOURCES="$TMP/sources.tsv"; : > "$SOURCES"
while IFS= read -r div; do
  [[ -n "$div" && -d "$REPO_ROOT/$div" ]] || continue
  while IFS= read -r f; do
    [[ "$(head -1 "$f")" == "---" ]] || continue
    printf '%s\t%s\t%s\t%s\n' \
      "$(agent_slug "$f")" "$(get_field description "$f")" "$(get_field name "$f")" "${f#"$REPO_ROOT"/}" \
      >> "$SOURCES"
  done < <(find "$REPO_ROOT/$div" -name '*.md' -type f | sort)
done < <(divisions_from_json)
N="$(wc -l < "$SOURCES" | tr -d ' ')"
[[ "$N" -gt 0 ]] || { echo "ERROR: no source agents found." >&2; exit 2; }

# --- generate: every converted tool, sequentially, into a scratch dir ---------
TOOLS="antigravity gemini-cli opencode cursor aider windsurf openclaw qwen zcode kimi codex osaurus hermes vibe dsh"
if [[ -z "$OUT" ]]; then
  OUT="$TMP/out"; mkdir -p "$OUT"
  for t in $TOOLS; do
    "$SCRIPT_DIR/convert.sh" --tool "$t" --out "$OUT" >/dev/null 2>&1 \
      || { echo "ERROR: convert.sh --tool $t failed." >&2; exit 1; }
  done
fi

# --- check: invariants + manifest (python does the parsing) -------------------
REPO_ROOT="$REPO_ROOT" OUT="$OUT" SOURCES="$SOURCES" N="$N" MANIFEST="$MANIFEST" \
UPDATE="$UPDATE" DIFF="$DIFF" DRIFT="$DRIFT" TOOLS="$TOOLS" python3 - <<'PY'
import os, re, sys, glob, json, hashlib, yaml, tomllib

R, OUT, N = os.environ["REPO_ROOT"], os.environ["OUT"], int(os.environ["N"])
MANIFEST, UPDATE, DIFF = os.environ["MANIFEST"], os.environ["UPDATE"] == "true", os.environ["DIFF"] == "true"
ADVISORY = os.environ.get("DRIFT") == "advisory"
TOOLS = os.environ["TOOLS"].split()

# slug -> (description, name, source path)
src = {}
for line in open(os.environ["SOURCES"], encoding="utf-8"):
    slug, desc, name, path = line.rstrip("\n").split("\t", 3)
    src[slug] = (desc, name, path)

fails, passes = [], 0
def ok(msg):   global passes; passes += 1
def bad(msg):  fails.append(msg)
def check(cond, msg): (ok if cond else bad)(msg)

# Per-tool output spec: (glob under OUT/<tool>, format). Formats were read off
# real generated output, not assumed:
#   yaml-fm   markdown with --- YAML frontmatter    round-trip description
#   toml      TOML with a description key           round-trip description
#   toml-id   TOML carrying only an identifier      id == slug + companion prompt file
#             (vibe: system_prompt_id -> prompts/<slug>.md)
#   yaml-id   YAML carrying only an identifier      id == slug + companion file
#             (kimi: agent.name -> <slug>/system.md)
#   accum     one file for all agents: "## Name" then the description line
#             (windsurf: bare line; aider: "> " blockquote)  round-trip both
#   plain     no structured metadata                count only
#   json      hermes agents.json                    count
SPEC = {
    "antigravity": ("agency-*/SKILL.md", "yaml-fm"),
    "osaurus":     ("agency-*/SKILL.md", "yaml-fm"),
    "dsh":         ("agency-*/SKILL.md", "yaml-fm"),
    "gemini-cli":  ("agents/*.md",       "yaml-fm"),
    "opencode":    ("agents/*.md",       "yaml-fm"),
    "qwen":        ("agents/*.md",       "yaml-fm"),
    "zcode":       ("agents/*.md",       "yaml-fm"),
    "cursor":      ("rules/*.mdc",       "yaml-fm"),
    "codex":       ("agents/*.toml",     "toml"),
    "vibe":        ("agents/*.toml",     "toml-id"),
    "kimi":        ("*/agent.yaml",      "yaml-id"),
    "openclaw":    ("*/SOUL.md",         "plain"),
    "aider":       ("CONVENTIONS.md",    "accum"),
    "windsurf":    (".windsurfrules",    "accum"),
    "hermes":      ("agency-agents-router/data/agents.json", "json"),
}

def slug_of(path):
    base = os.path.basename(path)
    if base in ("SKILL.md", "agent.yaml", "SOUL.md", "system.md", "AGENTS.md", "IDENTITY.md"):
        d = os.path.basename(os.path.dirname(path))
        return d[len("agency-"):] if d.startswith("agency-") else d
    return os.path.splitext(base)[0]

def find_desc(obj):
    """First 'description' string anywhere in a parsed mapping (TOML/YAML nest freely)."""
    if isinstance(obj, dict):
        if isinstance(obj.get("description"), str): return obj["description"]
        for v in obj.values():
            r = find_desc(v)
            if r is not None: return r
    return None

def frontmatter(text):
    lines = text.splitlines()
    if not lines or lines[0] != "---": raise ValueError("no frontmatter")
    for end in range(1, len(lines)):
        if lines[end] == "---":
            return yaml.safe_load("\n".join(lines[1:end]))
    raise ValueError("missing frontmatter closing ---")

def parsed_desc(path, fmt):
    text = open(path, encoding="utf-8").read()
    if fmt == "yaml-fm": data = frontmatter(text)
    elif fmt == "toml":  data = tomllib.loads(text)
    elif fmt == "yaml":  data = yaml.safe_load(text)
    else: return None
    if not isinstance(data, dict): raise ValueError("top level is not a mapping")
    return find_desc(data)

# --- expected values: an INDEPENDENT strict parse of every source ---------------
# The generator reads sources through lib.sh's get_field. If the expected side
# were derived the same way, a get_field bug would move both sides together and
# hide itself — that is exactly how #778's double-wrap stayed invisible. So the
# expected name/description come from PyYAML parsing the source frontmatter
# (the desktop app's js-yaml contract), and get_field's values are discarded.
# A source that does not strict-parse is an app-contract failure in itself; it
# is reported below and excluded from round-trips (desc=None).
src_bad = []
for slug, (_gf_desc, _gf_name, path) in list(src.items()):
    try:
        data = frontmatter(open(os.path.join(R, path), encoding="utf-8").read())
        assert isinstance(data, dict) and all(
            isinstance(data.get(field), str) and data[field].strip()
            for field in ("name", "description", "color")
        ), "missing or empty name/description/color"
        assert data["description"][:1] not in ('"', "'"), "description starts with a quote character"
        src[slug] = (data["description"], data["name"], path)
    except Exception as e:
        src_bad.append(f"source {path}: {str(e).splitlines()[0]}")
        src[slug] = (None, _gf_name, path)

# --- Layer A: per-tool count + strict-parse + round-trip -----------------------
def report(tool, bad_parse, bad_trip, label):
    if bad_trip > 3: bad(f"{tool}: ...and {bad_trip-3} more mismatches")
    if not bad_parse and not bad_trip: ok(f"{tool}: all {N} {label}")

for tool in TOOLS:
    pat, fmt = SPEC[tool]
    files = sorted(glob.glob(os.path.join(OUT, tool, pat)))

    if fmt == "accum":
        # One file for every agent. For each roster agent: "## <name>" exactly
        # once, and the description on the next non-blank line (aider quotes it
        # with "> "). Counting "## " lines would count body sections too.
        text = open(files[0], encoding="utf-8").read().split("\n") if files else []
        miss = 0
        for slug, (desc, name, path) in src.items():
            if desc is None: continue   # source failed strict parse; reported below
            idx = [i for i, l in enumerate(text) if l.rstrip() == f"## {name}"]
            if len(idx) != 1:
                miss += 1
                if miss <= 3: bad(f"{tool}: '## {name}' appears {len(idx)}x (want exactly 1)")
                continue
            nxt = next((l for l in text[idx[0]+1:idx[0]+4] if l.strip()), "")
            got = (nxt[2:] if nxt.startswith("> ") else nxt).strip()
            if got != desc:
                miss += 1
                if miss <= 3:
                    bad(f"{tool}: {slug} description mismatch\n"
                        f"        source:    {desc[:70]!r}\n        generated: {got[:70]!r}")
        report(tool, 0, miss, "present with descriptions round-tripped")
        continue

    if fmt == "json":
        try:
            data = json.load(open(files[0], encoding="utf-8")) if files else []
            items = data if isinstance(data, list) else data.get("agents", [])
            check(len(items) == N, f"{tool}: agents.json lists {len(items)} agents, roster has {N}")
        except Exception as e:
            bad(f"{tool}: agents.json unreadable ({e})")
        continue

    check(len(files) == N, f"{tool}: {len(files)} outputs, roster has {N}")
    if fmt == "plain": continue

    bad_parse = bad_trip = 0
    for f in files:
        slug = slug_of(f)
        if slug not in src:
            bad(f"{tool}: {os.path.relpath(f, OUT)} has no roster source for slug '{slug}'"); continue
        try:
            text = open(f, encoding="utf-8").read()
            if fmt == "yaml-fm":               data = frontmatter(text)
            elif fmt in ("toml", "toml-id"):   data = tomllib.loads(text)
            else:                              data = yaml.safe_load(text)   # yaml-id
            if not isinstance(data, dict): raise ValueError("top level is not a mapping")
        except Exception as e:
            bad_parse += 1; bad(f"{tool}: {os.path.relpath(f, OUT)} does not parse ({type(e).__name__}: {e})"); continue

        if fmt in ("yaml-fm", "toml"):
            got, want = find_desc(data), src[slug][0]
            if want is None: continue   # source failed strict parse; reported below
            if got != want:
                bad_trip += 1
                if bad_trip <= 3:
                    bad(f"{tool}: {slug} description round-trip mismatch\n"
                        f"        source:    {want[:70]!r}\n        generated: {str(got)[:70]!r}")
        else:
            # Identifier formats carry no description; the id must be the slug
            # and the prose file it points at must exist.
            if fmt == "toml-id":
                ident, companion = data.get("system_prompt_id"), os.path.join(OUT, tool, "prompts", f"{slug}.md")
            else:
                ident, companion = (data.get("agent") or {}).get("name"), os.path.join(os.path.dirname(f), "system.md")
            if ident != slug:
                bad_trip += 1
                if bad_trip <= 3: bad(f"{tool}: {os.path.relpath(f, OUT)} identifier {ident!r} != slug {slug!r}")
            elif not os.path.isfile(companion):
                bad_trip += 1
                if bad_trip <= 3: bad(f"{tool}: {slug} companion file missing: {os.path.relpath(companion, OUT)}")
    report(tool, bad_parse, bad_trip,
           "parse and round-trip" if fmt in ("yaml-fm", "toml") else "parse, carry their slug, and have their prose file")

# --- Layer A (color): a grey agent should be a grey agent ---------------------
# resolve_opencode_color() maps a name it does not know to #6B7280 and says
# nothing, so a typo or an unlisted name (`slate`, `navy`) reaches users as a
# deliberate-looking grey. Grey is a real choice for the agents that ask for it,
# so the check is not "never grey" — it is "grey only when the source said so".
GREY = "#6B7280"
grey_names = {"gray", "grey", "#6b7280", "6b7280"}
colour_bad = 0
for f in sorted(glob.glob(os.path.join(OUT, "opencode", "agents", "*.md"))):
    slug = os.path.splitext(os.path.basename(f))[0]
    entry = src.get(slug)
    if entry is None:
        continue
    try:
        emitted = str(frontmatter(open(f, encoding="utf-8").read()).get("color", "")).strip()
        source_color = str(frontmatter(open(os.path.join(R, entry[2]), encoding="utf-8").read())
                           .get("color", "")).strip().lower()
    except Exception:
        continue   # the strict-parse pass above already reported this
    if emitted.upper() == GREY and source_color not in grey_names:
        colour_bad += 1
        if colour_bad <= 3:
            bad(f"opencode: {slug} asked for color {source_color!r} and got {GREY} — "
                f"resolve_opencode_color() does not know that name")
if colour_bad > 3:
    bad(f"opencode: ...and {colour_bad-3} more colors silently replaced with grey")
elif not colour_bad:
    ok(f"opencode: every agent color resolves; none fell through to {GREY} by accident")

# --- Layer A (split integrity): a source fenced block must survive whole -------
# openclaw is the one tool that splits a single agent body across two files, at
# `## ` headings: SOUL.md (persona) / AGENTS.md (operations). A heading inside a
# fenced code block is block content, not a boundary. Splitting on one cuts the
# block in half and leaves each file holding a dangling fence, which renders as
# broken markdown for every user of that integration (#849). So every fenced
# block in a source must land intact in exactly one of the two files.
#
# This model must read fences exactly as lib.sh's fence_open_p / fence_closes_p
# do (and as GitHub renders): up to three spaces of indentation, same character,
# a closer at least as long as the opener with nothing but whitespace after it.
# Reading fences at column 0 only missed an indented opener and paired its
# column-0 closer with the following lines, reporting a tear the converter had
# not made.
OPEN_FENCE = re.compile(r"^( {0,3})(`{3,}|~{3,})")

def body_lines(text):
    """Mirror lib.sh's get_body, including `$(...)`'s trailing-newline strip."""
    out, fm = [], 0
    for line in text.split("\n"):
        if fm < 2 and line == "---":
            fm += 1
            continue
        if fm >= 2:
            out.append(line)
    while out and out[-1] == "":
        out.pop()
    return out

def fence_blocks(lines):
    """Inclusive (opener, closer) index pairs; closer = last line if unterminated.

    Mirrors lib.sh fence_open_p / fence_closes_p so the eval sees the same
    blocks the converter does.
    """
    res, marker, mlen, start = [], "", 0, None
    for i, line in enumerate(lines):
        m = OPEN_FENCE.match(line)
        if not m:
            continue
        tok, rest = m.group(2), line[m.end():]
        if not marker:
            marker, mlen, start = tok[0], len(tok), i
        elif tok[0] == marker and len(tok) >= mlen and not rest.strip():
            # only a bare run closes: "```bash" inside a block is content (lib.sh fence_closes_p)
            res.append((start, i)); marker, mlen, start = "", 0, None
    if marker and start is not None:
        res.append((start, len(lines) - 1))
    return res

def has_run(hay, needle):
    n = len(needle)
    return n > 0 and n <= len(hay) and any(hay[i:i+n] == needle for i in range(len(hay) - n + 1))

split_bad = 0
for slug, (_d, _n, path) in sorted(src.items()):
    sfile = os.path.join(OUT, "openclaw", slug, "SOUL.md")
    afile = os.path.join(OUT, "openclaw", slug, "AGENTS.md")
    if not (os.path.isfile(sfile) and os.path.isfile(afile)):
        continue
    body = body_lines(open(os.path.join(R, path), encoding="utf-8").read())
    bl = fence_blocks(body)
    if not bl:
        continue
    outs = [open(sfile, encoding="utf-8").read().split("\n"),
            open(afile, encoding="utf-8").read().split("\n")]
    for a, b in bl:
        if not any(has_run(o, body[a:b + 1]) for o in outs):
            split_bad += 1
            if split_bad <= 3:
                bad(f"openclaw: {slug} source fenced block at lines {a+1}-{b+1} is torn across "
                    f"SOUL.md/AGENTS.md — a `## ` heading inside the fence was treated as a section boundary")
if split_bad:
    if split_bad > 3: bad(f"openclaw: ...and {split_bad-3} more torn fenced blocks")
else:
    ok(f"openclaw: all {N} agents keep every source fenced block whole in one output file")

# --- Layer A (context budget): the Aider index has to stay an index ----------
# Aider keeps a conventions file in context for the whole session. Inlining the
# agent bodies made CONVENTIONS.md 3.8 million characters, which no model will
# take, so it carries one index entry per agent instead: description plus the
# path to the real file. Two things have to hold for that to be worth anything —
# the file stays small enough to load, and every path it prints resolves.
AIDER_INDEX_CEILING = 250_000
aider_index = os.path.join(OUT, "aider", "CONVENTIONS.md")
if os.path.isfile(aider_index):
    text = open(aider_index, encoding="utf-8").read()
    if len(text) > AIDER_INDEX_CEILING:
        bad(f"aider: CONVENTIONS.md is {len(text):,} characters — it is loaded into "
            f"every request, so it has to stay an index, not the agents themselves")
    paths = re.findall(r"^Full instructions: (.+)$", text, re.M)
    dangling = sorted({p for p in paths if not os.path.isfile(os.path.join(R, p))})
    if len(paths) != N:
        bad(f"aider: CONVENTIONS.md points at {len(paths)} agent files, roster has {N}")
    elif dangling:
        for d in dangling[:3]:
            bad(f"aider: CONVENTIONS.md points at a file that does not exist: {d}")
        if len(dangling) > 3:
            bad(f"aider: ...and {len(dangling)-3} more dangling paths")
    elif len(text) <= AIDER_INDEX_CEILING:
        ok(f"aider: index is {len(text):,} characters and all {N} agent paths resolve")

# --- Layer A (tool names): Qwen only grants tools it can name ---------------
# A Qwen subagent's `tools:` is an allow-list resolved against Qwen's registry
# by tool name or display name; an entry matching neither is kept as-is and
# grants nothing. Sources list Claude Code names, and Read/Write/Bash are not
# Qwen names, so they have to be translated (convert.sh qwen_tools). Every
# entry must be a Qwen built-in by a spelling Qwen resolves (tool name, display
# name or legacy alias; packages/core/src/tools/tool-names.ts) or an MCP tool.
QWEN_TOOLS = {
    # tool names
    "read_file", "write_file", "edit", "run_shell_command", "grep_search", "glob",
    "list_directory", "web_fetch", "web_search", "todo_write", "notebook_edit", "agent",
    # display names
    "ReadFile", "WriteFile", "Edit", "Shell", "Grep", "Glob", "ListFiles", "WebFetch",
    "WebSearch", "TodoList", "NotebookEdit", "Agent",
    # legacy aliases
    "SearchFiles", "FindFiles", "ReadFolder", "Task", "TodoWrite",
    "search_file_content", "replace", "task",
}
qwen_bad = []
for f in sorted(glob.glob(os.path.join(OUT, "qwen", "agents", "*.md"))):
    try:
        tools = (frontmatter(open(f, encoding="utf-8").read()) or {}).get("tools")
    except Exception:
        continue   # the strict-parse pass already reported this
    if tools is None:
        continue
    entries = tools if isinstance(tools, list) else [t.strip() for t in str(tools).split(",")]
    unknown = [t for t in entries if t and t not in QWEN_TOOLS and not t.startswith("mcp__")]
    if unknown:
        qwen_bad.append((os.path.basename(f)[:-3], unknown))
for slug, unknown in qwen_bad[:3]:
    bad(f"qwen: {slug} lists {', '.join(unknown)} — Qwen Code has no tool by that "
        f"name, so the allow-list grants nothing for it")
if len(qwen_bad) > 3:
    bad(f"qwen: ...and {len(qwen_bad)-3} more agents with unknown tool names")
if not qwen_bad:
    ok("qwen: every tools: entry names a Qwen Code tool")

# --- Layer A (app-facing): every SOURCE frontmatter strict-parsed above -------
for m in src_bad[:5]: bad(m)
if len(src_bad) > 5: bad(f"...and {len(src_bad)-5} more source frontmatter problems")
if not src_bad: ok(f"all {N} source agents strict-parse (app contract)")

# --- Layer B: manifest (v2: per-agent lines, platform-neutral hashes) ----------
def norm_bytes(b): return b.replace(b"\r\n", b"\n")
def sha(b): return hashlib.sha256(b).hexdigest()
def rel(f): return os.path.relpath(f, OUT).replace(os.sep, "/")

slugs = set(src)
names = {name: slug for slug, (_d, name, _p) in src.items()}
per_agent = {slug: [] for slug in slugs}     # slug -> [(label, bytes)]
per_tool  = {t: [] for t in TOOLS}           # tool -> [(label, bytes)] for non-agent files

def owner_of(path):
    """Which roster agent a generated file belongs to, by exact path component or stem."""
    parts = rel(path).split("/")[1:]         # drop the tool dir
    for comp in parts[:-1]:
        d = comp[len("agency-"):] if comp.startswith("agency-") else comp
        if d in slugs: return d
    stem = os.path.splitext(parts[-1])[0]
    return stem if stem in slugs else None

for tool in TOOLS:
    pat, fmt = SPEC[tool]
    for f in sorted(glob.glob(os.path.join(OUT, tool, "**", "*"), recursive=True)):
        if not os.path.isfile(f): continue
        data = norm_bytes(open(f, "rb").read())
        if fmt == "accum" and rel(f) == f"{tool}/{pat}":
            # One file for all agents: attribute each "## <name>" section to its agent;
            # anything outside a known section (preamble) is the tool's contract.
            lines_ = data.decode("utf-8", "replace").split("\n")
            cur, buf, pre = None, [], []
            def flush():
                if cur: per_agent[cur].append((f"{tool}:section", "\n".join(buf).encode()))
            for l in lines_:
                m = names.get(l.rstrip()[3:]) if l.startswith("## ") else None
                if m: flush(); cur, buf = m, [l]; continue
                (buf if cur else pre).append(l)
            flush()
            per_tool[tool].append((rel(f) + ":preamble", "\n".join(pre).encode()))
            continue
        if fmt == "json" and rel(f) == f"{tool}/{pat}":
            try:
                items = json.loads(data.decode("utf-8"))
                items = items if isinstance(items, list) else items.get("agents", [])
                for it in items:
                    sl = it.get("slug") if isinstance(it, dict) else None
                    if sl in slugs: per_agent[sl].append((f"{tool}:entry", json.dumps(it, sort_keys=True, ensure_ascii=False).encode()))
                    else: per_tool[tool].append((rel(f) + ":stray-entry", json.dumps(it, sort_keys=True, ensure_ascii=False).encode()))
            except Exception:
                per_tool[tool].append((rel(f), data))
            continue
        o = owner_of(f)
        if o is None and os.path.basename(f).lower() == "readme.md":
            # Roster-derived text in generated docs ("Generated agent count: 273") must not move
            # the tool line — only template changes should.
            data = re.sub(rb"(?m)^(Generated agent count: )\d+$", rb"\1N", data)
        (per_agent[o] if o else per_tool[tool]).append((rel(f), data))

def digest(entries):
    h = hashlib.sha256()
    for label, b in sorted(entries, key=lambda e: e[0]):
        h.update(label.encode()); h.update(b"\0"); h.update(sha(b).encode()); h.update(b"\n")
    return h.hexdigest()

rows = [("agent", slug, digest(per_agent[slug])) for slug in sorted(slugs)]
rows += [("tool", t, digest(per_tool[t])) for t in TOOLS]
for c in ("divisions.json", "tools.json", "strategy/runbooks.json"):
    p = os.path.join(R, c)
    rows.append(("contract", c, sha(norm_bytes(open(p, "rb").read())) if os.path.exists(p) else "MISSING"))
new = ("# convert-outputs manifest v2 — one line per agent (its output across every tool), one per tool\n"
       "# (non-agent files), one per contract. Platform-neutral hashes. Regenerate: scripts/test-convert-outputs.sh --update\n"
       + "".join(f"{k}\t{key}\t{h}\n" for k, key, h in rows))

def drift_report(old_text):
    old = {}
    for l in old_text.splitlines():
        if l.startswith("#") or "\t" not in l: continue
        f = l.split("\t")
        if len(f) == 3: old[(f[0], f[1])] = f[2]
        elif len(f) == 2: return None                    # v1 manifest
    cur = {(k, key): h for k, key, h in rows}
    changed = sorted(key for (k, key), h in cur.items() if (k, key) in old and old[(k, key)] != h and k == "agent")
    added   = sorted(key for (k, key) in cur if k == "agent" and (k, key) not in old)
    removed = sorted(key for (k, key) in old if k == "agent" and (k, key) not in cur)
    tools   = sorted(key for (k, key), h in cur.items() if k != "agent" and old.get((k, key)) != h)
    return changed, added, removed, tools

# Never make broken or incomplete generated output the new baseline.
if UPDATE and not fails:
    open(MANIFEST, "w", newline="\n").write(new); ok(f"manifest written: {os.path.relpath(MANIFEST, R)}")
elif UPDATE:
    print("  SKIP manifest update: generated outputs failed validation")
elif not os.path.exists(MANIFEST):
    bad(f"manifest missing: run with --update to create {os.path.relpath(MANIFEST, R)}")
else:
    d = drift_report(open(MANIFEST, encoding="utf-8").read())
    if d is None:
        bad("manifest is the old v1 format (per-tool aggregate hashes) — run --update once to migrate")
    else:
        changed, added, removed, tools = d
        if not (changed or added or removed or tools):
            ok("manifest matches (no output or contract drift)")
        else:
            def few(xs, n=8): return ", ".join(xs[:n]) + (f", … ({len(xs)} total)" if len(xs) > n else "")
            parts = []
            if added:   parts.append(f"new agents: {few(added)}")
            if changed: parts.append(f"changed agents: {few(changed)}")
            if removed: parts.append(f"removed agents: {few(removed)}")
            if tools:   parts.append(f"tool/contract lines: {', '.join(tools)}")
            msg = "manifest drift — " + "; ".join(parts)
            if len(changed) >= max(20, N // 4):
                msg += f"\n        {len(changed)} of {N} agents changed at once — that is a converter/template change, not an agent edit; review the generator diff"

            if ADVISORY:
                print(f"  ADVISORY {msg}\n           (expected for agent additions/edits; maintainers regenerate the manifest when this lands)")
                ok("manifest drift reported (advisory mode)")
            else:
                bad(msg + "\n        Agent lines move when agents are added/edited — regenerate with --update when landing."
                          "\n        A tool/contract line moving means a generator or contract changed — review it.")

# --- report --------------------------------------------------------------------
for m in fails: print(f"  FAIL {m}")
print(f"\nResults: {passes} passed, {len(fails)} failed  ({N} roster agents x {len(TOOLS)} tools)")
print("FAILED" if fails else "PASSED")
sys.exit(1 if fails else 0)
PY