Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python
13.7 KB
Raw Blame
"""Write LICENSE-THIRD-PARTY: what ManimGX's wheels hold that others hold the
copyright in, with the licenses and notices it comes with, each text once, after what it covers:

- the package's and the engine's files that name others (REUSE: by their SPDX lines, or
  REUSE.toml), with their copyright lines and their licenses' texts from LICENSES/: the modules
  ported from Manim CE, the engine's 3D lighting;
- the crates compiled into the engine, in every build of it a wheel holds, with the license and
  notice files each ships: the workspace's crates, those of the sources they fetch (x264's,
  FFmpeg's and libopus's, mitex's); a crate from crates.io that ships none, its license's text
  from LICENSES/, with its authors on the line the text keeps for them.

And check that ManimGX's `license` (pyproject.toml) covers all of it: that each
part's license is satisfied by the licenses it names. The Linux wheels' lavapipe has a notice of
its own, LICENSE-LAVAPIPE.

`python scripts/release/licenses.py` (`just licenses`) writes it; `--check` fails if it is stale or
something is not covered (tests/test_licenses.py).
"""

import json
import re
import subprocess
import sys
import textwrap
import tomllib
from pathlib import Path, PurePosixPath

ROOT = Path(__file__).resolve().parents[2]
# the repository's files a wheel holds: the package's, installed as they are, and the engine's,
# compiled into its extension
SOURCES = PurePosixPath("src")
ENGINE_SOURCES = PurePosixPath("rust/engine/src")
OUT = ROOT / "LICENSE-THIRD-PARTY"
ENGINE = ROOT / "rust" / "engine"
# the engine's builds a wheel holds, as (target, features): the extension on each platform a
# wheel is built for (its window too); Pyodide's ([tool.cibuildwheel.pyodide]); and the
# player for a page, which Pyodide's extension carries (rust/engine/build.rs)
BUILDS = (
    ("aarch64-apple-darwin", []),
    ("x86_64-apple-darwin", []),
    ("aarch64-unknown-linux-gnu", []),
    ("x86_64-unknown-linux-gnu", []),
    ("x86_64-pc-windows-msvc", []),
    (
        "wasm32-unknown-emscripten",
        ["--no-default-features", "--features", "python,typeset"],
    ),
    ("wasm32-unknown-unknown", ["--no-default-features", "--features", "web"]),
)
LICENSE_FILE = re.compile(
    r"^(licen[cs]e|copying|notice|unlicense|copyright)", re.IGNORECASE
)
# the line a license's text keeps for the copyright notice: "Copyright (c) <year> <owner>"
TEMPLATE = re.compile(r"^Copyright\b.*<")
# licenses that ask nothing of a binary: no notice, no text
UNASKING = {"0BSD", "BSL-1.0", "CC0-1.0", "MIT-0", "Unlicense"}


def run(*command: str, cwd: Path = ROOT) -> str:
    return subprocess.run(
        command, cwd=cwd, capture_output=True, check=True, text=True, encoding="utf-8"
    ).stdout


def crates(
    directory: Path = ENGINE, builds: tuple[tuple[str, list[str]], ...] = BUILDS
) -> dict[str, dict]:
    """The packages compiled into the selected builds, as `cargo tree` shows them: their
    normal dependencies, all the way down, with the features each build turns on (a build
    script's and a test's are not in it), the workspace's own among them."""
    metadata = json.loads(
        run(
            *("cargo", "metadata", "--format-version", "1", "--locked"),
            "--all-features",
            cwd=directory,
        )
    )
    packages = {(p["name"], p["version"]): p for p in metadata["packages"]}
    linked: dict[str, dict] = {}
    for target, features in builds:
        tree = run(
            *("cargo", "tree", "--locked", "--edges", "normal", "--target", target),
            *(*features, "--prefix", "none", "--format", "{p}"),
            cwd=directory,
        )
        for name, version in re.findall(r"^(\S+) v(\S+)", tree, re.MULTILINE):
            linked[f"{name} {version}"] = packages[name, version]
    return linked


def notice(line: str) -> str:
    """A copyright line as a notice reads it, "Copyright ...", however the file wrote it."""
    return re.sub(r"^SPDX-FileCopyrightText:\s*", "Copyright ", line.strip())


def holder(line: str) -> str:
    """Who a copyright line names, without its years."""
    years = r"(\d{4}(\s*[-,]\s*\d{4})*,?\s+)?"
    return re.sub(
        rf"^(Copyright\s*(\(c\)|©)?\s*)?{years}", "", notice(line), flags=re.I
    )


def others() -> dict[tuple[str, tuple[str, ...]], list[str]]:
    """The package's and the engine's files that others hold copyright in, as REUSE finds
    them (`reuse lint`, which `just check` runs), named as the wheel holds them (the
    engine's, compiled into it, by their path in the repository): by license and copyright
    lines, ManimGX's left out (those of the holder REUSE.toml gives its own files)."""
    reuse = tomllib.loads((ROOT / "REUSE.toml").read_text(encoding="utf-8"))
    own = holder(reuse["annotations"][0]["SPDX-FileCopyrightText"])
    project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
    name = f"manimgx {project['project']['version']}"
    report = subprocess.run(
        [sys.executable, "-m", "reuse", "lint", "--json"],
        cwd=ROOT,
        capture_output=True,
        text=True,
        encoding="utf-8",
    )
    # by license: the files under it, and the copyright lines of all of them
    groups: dict[str, tuple[list[str], set[str]]] = {}
    for file in json.loads(report.stdout)["files"]:
        path = PurePosixPath(file["path"])
        lines = {notice(c["value"]) for c in file["copyrights"]}
        if path.is_relative_to(SOURCES):
            shown = path.relative_to(SOURCES).as_posix()
        elif path.is_relative_to(ENGINE_SOURCES):
            shown = path.as_posix()
        else:
            continue
        if not (theirs := {line for line in lines if holder(line) != own}):
            continue
        expression = " AND ".join(e["value"] for e in file["spdx_expressions"])
        paths, holders = groups.setdefault(expression, ([], set()))
        paths.append(f"{name}: {shown}")
        holders |= theirs
    return {
        (expression, tuple(sorted(holders))): sorted(paths)
        for expression, (paths, holders) in sorted(groups.items())
    }


def options(expression: str) -> list[list[str]]:
    """The sets of licenses that satisfy an SPDX expression, in its order: OR is a choice, AND
    needs both (a license WITH an exception is one; "A/B" is an old way to say A OR B).
    """
    tokens = re.findall(
        r"\(|\)|[\w.+-]+(?: WITH [\w.+-]+)?", expression.replace("/", " OR ")
    )

    def either(at: int) -> tuple[list[list[str]], int]:
        sets, at = both(at)
        while at < len(tokens) and tokens[at] == "OR":
            more, at = both(at + 1)
            sets += more
        return sets, at

    def both(at: int) -> tuple[list[list[str]], int]:
        sets, at = one(at)
        while at < len(tokens) and tokens[at] == "AND":
            more, at = one(at + 1)
            sets = [a + [m for m in b if m not in a] for a in sets for b in more]
        return sets, at

    def one(at: int) -> tuple[list[list[str]], int]:
        if tokens[at] == "(":
            sets, at = either(at + 1)
            return sets, at + 1
        return [[tokens[at]]], at + 1

    return either(0)[0]


def texts(expression: str, holders: list[str], above: bool) -> str | None:
    """The texts, from LICENSES/, of the first licenses that satisfy an expression whose texts
    it holds all of (None if it holds none's), with the copyright lines of what they cover on
    the line a text keeps for them, or else, if `above`, above it."""
    for option in options(expression):
        parts = [
            ROOT / "LICENSES" / f"{p}.txt" for o in option for p in o.split(" WITH ")
        ]
        if not all(part.exists() for part in parts):
            continue
        written = []
        for part in parts:
            lines = clean(part.read_text(encoding="utf-8")).splitlines()
            at = next(
                (i for i, line in enumerate(lines[:5]) if TEMPLATE.match(line)), None
            )
            if at is not None:
                lines[at : at + 1] = holders
            elif above:
                lines[:0] = [*holders, ""]
            written.append("\n".join(lines))
        return "\n\n".join(written)
    return None


def clean(text: str) -> str:
    # as the repository's hooks keep text: no trailing whitespace
    return "\n".join(line.rstrip() for line in text.strip().splitlines())


def uncovered(licenses: dict[str, str | None]) -> list[str]:
    """What of the parts given pyproject.toml's `license` does not satisfy the license of."""
    project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))
    named = set(project["project"]["license"].split(" AND "))
    return [
        f"{part} ({expression})"
        for part, expression in sorted(licenses.items())
        if not any(set(option) <= named for option in options(expression or "?"))
    ]


def write(
    linked: dict[str, dict],
    theirs: dict[tuple[str, tuple[str, ...]], list[str]],
    *,
    header: str | None = None,
    include_local: bool = False,
) -> tuple[str, list[str]]:
    """The notice, and what LICENSES/ has no text for."""
    found: dict[str, list[str]] = {}
    missing = []
    for (expression, holders), users in theirs.items():
        if (written := texts(expression, list(holders), above=True)) is None:
            missing += users
            continue
        found.setdefault(written, []).extend(f"{u} ({expression})" for u in users)
    for key, package in sorted(linked.items()):
        folder = Path(package["manifest_path"]).parent
        shipped = sorted(
            f for f in folder.iterdir() if f.is_file() and LICENSE_FILE.match(f.name)
        )
        declared = f"{key} ({package.get('license') or 'no license declared'})"
        for file in shipped:
            written = clean(file.read_text(encoding="utf-8", errors="replace"))
            found.setdefault(written, []).append(f"{declared}: {file.name}")
        # the workspace's own code is ManimGX's (LICENSE)
        if not shipped and (include_local or package["source"] is not None):
            authors = [re.sub(r"\s*<.*?>", "", a) for a in package["authors"]]
            by = ", ".join(authors) or f"the {package['name']} authors"
            expression = package.get("license") or "?"
            written = texts(expression, [f"Copyright (c) {by}"], above=False)
            if written is None and any(
                set(option) <= UNASKING for option in options(expression)
            ):
                written = "(its license asks nothing of a binary)"
            if written is None:
                missing.append(declared)
                continue
            found.setdefault(written, []).append(f"{declared}: no license file")
    header = header or (
        "What ManimGX's wheels hold that others hold the copyright in: modules of its"
        " Python package, files of its engine (its 3D lighting, adapted from Google's"
        " Filament, https://github.com/google/filament), and the crates compiled into"
        " the engine (the extension, on each platform, Pyodide's, and the player for a"
        " page, which the extension carries), with the sources the workspace's crates"
        " fetch; with the licenses"
        " and notices they come with, each text once, after what it covers. FFmpeg is"
        " under the LGPL-2.1-or-later, and x264, in every wheel but Pyodide's, under"
        " the GPL-2.0-or-later, which makes such a wheel, as a whole, GPL-3.0-or-later."
        " Its complete source and build recipes are on the GitHub release of its"
        " version, https://github.com/academa-labs/manimgx/releases/tag/vX.Y.Z for"
        " ManimGX X.Y.Z: manimgx-X.Y.Z-source.tar.xz, ManimGX's committed repository"
        " with every crate rust/Cargo.lock names and each archive a workspace's crate"
        " fetches, with the Linux builds' Mesa/glslang archives and exact distribution"
        " source RPMs. System build tools remain prerequisites. The Linux wheels' lavapipe"
        " is in LICENSE-LAVAPIPE. Written by"
        " scripts/release/licenses.py. The Windows wheels' DXC is in LICENSE-DXC."
    )
    parts = [textwrap.fill(header, 84, break_on_hyphens=False) + "\n"]
    for written, users in sorted(found.items(), key=lambda item: item[1][0]):
        parts.append(
            "=" * 79 + "\n" + "\n".join(users) + "\n" + "-" * 79 + "\n" + written + "\n"
        )
    return "\n".join(parts), missing


def main() -> None:
    if sys.argv[1:2] == ["--crate"]:
        directory, target, output = sys.argv[2:]
        linked = crates(Path(directory), ((target, []),))
        text, missing = write(
            linked,
            {},
            header=f"The crates compiled into the executable's launcher for {target}, "
            "as its own Cargo.lock and default features select them, with the license "
            "and notice files they supply. Its sources and lockfile accompany the release.",
            include_local=True,
        )
        if missing:
            sys.exit(f"LICENSES/ has no text for {', '.join(missing)}")
        Path(output).write_text(text, encoding="utf-8")
        return
    linked = crates()
    theirs = others()
    text, missing = write(linked, theirs)
    problems = [f"LICENSES/ has no text for {', '.join(missing)}"] if missing else []
    licenses = {key: p.get("license") for key, p in linked.items()}
    licenses |= {
        user: expression for (expression, _), us in theirs.items() for user in us
    }
    if not_covered := uncovered(licenses):
        problems.append(
            f"pyproject.toml's license does not cover {', '.join(not_covered)}"
        )
    if "--check" not in sys.argv:
        OUT.write_text(text, encoding="utf-8")
    elif not (OUT.exists() and OUT.read_text(encoding="utf-8") == text):
        problems.append(f"{OUT.relative_to(ROOT)} is stale: just licenses")
    sys.exit("\n".join(problems) or None)


if __name__ == "__main__":
    main()