Skip to content

An agent organization structured as a company — 15+ departments, 125+ skills, each independently installable, citing the standards and regulators that settle the question. Runs in Claude Code and ChatGPT.

10.4 KB
Raw Blame
#!/usr/bin/env python3
"""Emit each skill's sources into its own `references/sources.md`.

A catalog nothing reads is a bibliography. Claude Code and ChatGPT both load a skill's
`references/` files on demand, so putting each skill's sources there is what turns the catalog into
something an agent consults at the moment it is answering — without spending context on sources for
the other 171 skills.

Generated, never hand-edited: `--check` fails the build when a file drifts from the catalog.
"""
import argparse
import collections
import glob
import os
import sys

BANNER = "<!-- Generated by scripts/build-sources.py from sources/*.toml. Do not edit. -->"

# What an agent is allowed to do with each class, stated where it will be read rather than only in
# sources/README.md. The wording is deliberately blunt: the failure this prevents is an agent
# reproducing a standard it was only ever allowed to cite.
USE = {
    "public-domain-usgov": "public domain (US government) — quote freely",
    "public-domain": "public domain — quote freely",
    "cc0": "CC0 — quote freely",
    "cc-by": "CC BY — quote with attribution",
    "cc-by-sa": "CC BY-SA — quote with attribution; share-alike, so do not fold into this repository",
    "attribution-required": "free to use with attribution — credit the publisher",
    "open-data": "open data — check the terms before redistributing",
    "free-to-read": "**read and cite only — copyrighted, do not reproduce**",
    "registration-required": "**account required — cite it; the user fetches it**",
    "paywalled": "**sold — cite the identifier only, never the text**",
}


def load_catalog():
    """Reuse the checker's loader so the two cannot read the catalog differently."""
    import importlib.util
    path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "check-sources.py")
    spec = importlib.util.spec_from_file_location("check_sources", path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module.load()


def render(skill_ref, entries):
    department, skill = skill_ref.split(":")
    lines = [
        f"# Sources — `{skill_ref}`",
        "",
        BANNER,
        "",
        "Check these before answering on anything they cover, and cite what you used. The use note "
        "on each one is binding: most of what a professional cites is free to read and not free to "
        "reproduce.",
        "",
    ]
    for entry in sorted(entries, key=lambda e: e["title"]):
        lines += [
            f"## {entry['title']}",
            "",
            f"{entry['publisher']} · {entry['jurisdiction']} · {USE[entry['license']]}",
            "",
            f"<{entry['url']}>",
        ]
        if entry.get("data"):
            lines += ["", f"Machine-readable: <{entry['data']}>"]
        lines += ["", f"**Authoritative for:** {entry['authoritative_for']}", ""]
    lines += [
        "---",
        "",
        "Sources are maintained in `sources/` upstream, not here. If one is wrong, out of date, or "
        "missing, fix it there — this file is regenerated and an edit to it is lost.",
        "",
    ]
    return "\n".join(lines)


# Ordered from most permissive to most restrictive, so the index reads as a gradient rather than as
# an alphabetical list — which is the shape of the decision a reader is making.
LICENSE_ORDER = [
    "public-domain-usgov", "public-domain", "cc0", "cc-by", "open-data",
    "attribution-required", "cc-by-sa", "free-to-read", "registration-required", "paywalled",
]
# The index table names the license in its own column, so repeating it in the action column reads
# as stutter. Stated rather than sliced out of USE, because slicing a string that carries emphasis
# markers produces broken markdown at the seam.
ACTION = {
    "public-domain-usgov": "Quote freely",
    "public-domain": "Quote freely",
    "cc0": "Quote freely",
    "cc-by": "Quote with attribution",
    "open-data": "Use the data; read the terms before redistributing",
    "attribution-required": "Quote with the publisher's required credit",
    "cc-by-sa": "Quote with attribution; do not fold into this repository",
    "free-to-read": "**Read and cite only. Never reproduce.**",
    "registration-required": "**Cite it; the user fetches it.**",
    "paywalled": "**Cite the identifier only, never the text.**",
}
LABEL = {
    "public-domain-usgov": "Public domain (US government)",
    "public-domain": "Public domain",
    "cc0": "CC0",
    "cc-by": "CC BY",
    "open-data": "Open data",
    "attribution-required": "Free to use with attribution",
    "cc-by-sa": "CC BY-SA (share-alike)",
    "free-to-read": "Free to read, all rights reserved",
    "registration-required": "Free behind an account",
    "paywalled": "Sold",
}


assert set(ACTION) == set(USE), "ACTION and USE must cover the same license vocabulary"


def index(entries):
    """The human-facing index at docs/SOURCES.md.

    The per-skill files answer "what should I check for this question". This answers the other
    one — "what does this catalog actually cover, and how much of it may I quote" — which is the
    question a reader has before they trust any of it.
    """
    by_license = collections.Counter(e["license"] for _, e in entries)
    quotable = sum(n for k, n in by_license.items()
                   if k in {"public-domain-usgov", "public-domain", "cc0", "cc-by", "open-data",
                            "attribution-required"})
    skills = {ref for _, e in entries for ref in e.get("skills", [])}
    lines = [
        "# Source catalog",
        "",
        BANNER,
        "",
        f"{len(entries)} authoritative sources, mapped to {len(skills)} skills. Each skill that has "
        "any carries its own list at `references/sources.md` inside the skill, which is where an "
        "agent reads it while answering.",
        "",
        "**References, never copies.** The catalog holds a URL and a judgment about it, never "
        "source material. That is what keeps it live — a pointer to a regulator's site is right "
        "the moment the rule changes, and a snapshot is wrong the day after it is taken.",
        "",
        "## What you may do with it",
        "",
        f"**{quotable} of {len(entries)} are quotable**, with attribution where the license asks "
        "for it. The rest are free to read and not free to reproduce, which is the single most "
        "useful thing this catalog records — most of what a professional must cite is not open.",
        "",
        "| License | Sources | What an agent may do |",
        "|---|---|---|",
    ]
    for key in LICENSE_ORDER:
        if by_license.get(key):
            lines.append(f"| {LABEL[key]} | {by_license[key]} | {ACTION[key]} |")
    lines += ["", "## By subject", ""]
    by_file = collections.defaultdict(list)
    for path, entry in entries:
        by_file[path].append(entry)
    for path in sorted(by_file):
        subject = os.path.basename(path)[: -len(".toml")].replace("-", " ").capitalize()
        lines += [f"### {subject}", "", "| Source | Publisher | License | Skills |", "|---|---|---|---|"]
        for entry in sorted(by_file[path], key=lambda e: e["title"]):
            refs = ", ".join(f"`{r}`" for r in sorted(entry.get("skills", [])))
            lines.append(f"| [{entry['title']}]({entry['url']}) | {entry['publisher']} "
                         f"| {LABEL[entry['license']]} | {refs} |")
        lines.append("")
    lines += [
        "## Keeping it honest",
        "",
        "`scripts/check-sources.py` runs on every push and verifies structure, the license "
        "vocabulary, that every skill named resolves, and that a skill with sources actually points "
        "at them. Reachability is checked weekly by its own workflow instead, because a publisher "
        "being briefly down is not a reason to fail an unrelated pull request.",
        "",
        "Sources are maintained in `sources/*.toml`. See `sources/README.md` for the format and the "
        "full license vocabulary.",
        "",
    ]
    return "\n".join(lines)


def targets():
    """The file each skill's sources belong in, and what goes in it."""
    by_skill = collections.defaultdict(list)
    for _, entry in load_catalog():
        for ref in entry.get("skills", []):
            by_skill[ref].append(entry)
    import importlib.util
    spec = importlib.util.spec_from_file_location(
        "check_sources", os.path.join(os.path.dirname(os.path.abspath(__file__)), "check-sources.py"))
    checker = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(checker)

    out = {"docs/SOURCES.md": index(load_catalog())}
    for ref, entries in by_skill.items():
        department, skill = ref.split(":")
        path = checker.skill_path(department, skill)
        if not path:
            continue  # check-sources.py reports this; emitting is not the place to fail on it
        out[os.path.join(os.path.dirname(path), "references", "sources.md")] = render(ref, entries)
    return out


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--check", action="store_true", help="report drift instead of writing")
    args = parser.parse_args()

    wanted = targets()
    # A skill that loses its last source must lose the file too, or the catalog and the tree
    # disagree in the one direction a content comparison cannot see.
    existing = (set(glob.glob("plugins/*/skills/*/references/sources.md"))
                | set(glob.glob("verticals/*/skills/*/*/references/sources.md")))
    stale = sorted(existing - set(wanted))

    if args.check:
        drifted = [p for p, text in sorted(wanted.items())
                   if not os.path.exists(p) or open(p, encoding="utf-8").read() != text]
        for path in drifted:
            print(f"  stale or missing: {path}")
        for path in stale:
            print(f"  no longer in the catalog: {path}")
        problems = len(drifted) + len(stale)
        if problems:
            print("  run: python3 scripts/build-sources.py")
        print(f"skill source files: {len(wanted)} expected, {problems} out of date")
        return 1 if problems else 0

    for path, text in sorted(wanted.items()):
        os.makedirs(os.path.dirname(path), exist_ok=True)
        with open(path, "w", encoding="utf-8") as handle:
            handle.write(text)
    for path in stale:
        os.remove(path)
    print(f"sources: {len(wanted)} skill reference file(s) written"
          + (f", {len(stale)} removed" if stale else ""))
    return 0


if __name__ == "__main__":
    sys.exit(main())