Skip to content

One self-hosted app for your shows, movies and music.

19.1 KB
Raw Blame
#!/usr/bin/env python3
"""Third-party notices: every part HolyArrs ships, with its licence.

Writes docs/third-party-notices.md (each part with its licence files) and
the "Open-source parts" table at the end of THIRD_PARTY_NOTICES.md from:

- the Go modules `go list -deps ./app/backend/cmd/holyarrs` links, for every release
  target in deploy/release/package.py (they differ per system), with the licence
  files from each linked package's folder up to its module's root;
- the npm packages the built app holds (app/frontend/bundled-packages.json,
  written by the app's build), with their licence files, plus hls.js,
  which the build copies beside the app;
- the third-party files in app/frontend/static (app/frontend/static-parts.json),
  which the build copies as they are;
- the few other parts below (Go itself and the Docker image's CA list).

    python deploy/release/notices.py          write both files
    python deploy/release/notices.py --check  fail when a shipped part is missing,
                                       stale or of another version

The check needs Go and its module cache, not app/frontend/node_modules.
"""
from __future__ import annotations

import argparse
import importlib.util
import json
import os
import re
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
DOCS = ROOT / "docs" / "third-party-notices.md"
SUMMARY = ROOT / "THIRD_PARTY_NOTICES.md"
WEB = ROOT / "app" / "frontend"
DIST = ROOT / "app" / "frontend" / "dist"
TABLE_START = "<!-- notices: generated by deploy/release/notices.py; do not edit by hand -->"
FONT = re.compile(r"(?i)\.(woff2?|ttf|otf|eot)$")
LICENCE_FILE = re.compile(r"(?i)^(licen[cs]e|copying|notice|patents|unlicense)([.\-_].*)?$")
# The video tool ships as one release program with one row; its bundled parts are never listed.
SKIPPED = re.compile(r"(?i)yt-dlp.*(bundled|third[-_ ]?party|credit)|third_party_licenses")
# A file that gathers the licences of several parts: every licence in it counts.
GATHERED = re.compile(r"(?i)(3rd|third)[-_ ]?party")

# What a part is for, where that helps a reader; the rest speak for themselves.
USED_FOR = {
    "github.com/coreos/go-oidc/v3": "Sign-in with an outside account (OpenID Connect).",
    "github.com/go-jose/go-jose/v4": "Checking sign-in tokens (OpenID Connect).",
    "golang.org/x/oauth2": "Sign-in with an outside account.",
    "golang.org/x/crypto": "Password hashing and other cryptography.",
    "golang.org/x/image": "Reading and resizing pictures.",
    "golang.org/x/sys": "System calls.",
    "golang.org/x/text": "Text handling (Unicode).",
    "modernc.org/sqlite": "The database (SQLite, in Go).",
    "modernc.org/libc": "Used by the database.",
    "modernc.org/mathutil": "Used by the database.",
    "modernc.org/memory": "Used by the database.",
    "github.com/dustin/go-humanize": "Used by the database.",
    "github.com/google/uuid": "Used by the database.",
    "github.com/mattn/go-isatty": "Used by the database.",
    "github.com/ncruces/go-strftime": "Used by the database.",
    "github.com/remyoudompheng/bigfft": "Used by the database.",
    "github.com/nwaples/rardecode/v2": "Unpacking RAR files.",
    "github.com/bodgit/sevenzip": "Unpacking 7-Zip archives.",
    "github.com/bodgit/plumbing": "Used by the 7-Zip reader.",
    "github.com/bodgit/windows": "Used by the 7-Zip reader.",
    "github.com/hashicorp/golang-lru/v2": "A small cache in the 7-Zip reader (encrypted archives). Used unchanged; source at https://github.com/hashicorp/golang-lru.",
    "github.com/therootcompany/xz": "Unpacking xz archives.",
    "github.com/ulikunitz/xz": "LZMA in 7-Zip archives.",
    "github.com/klauspost/compress": "Compression in 7-Zip archives.",
    "github.com/pierrec/lz4/v4": "LZ4 in 7-Zip archives.",
    "github.com/andybalholm/brotli": "Brotli in 7-Zip archives.",
    "github.com/stangelandcl/ppmd": "PPMd in 7-Zip archives.",
    "github.com/spf13/afero": "File access for the 7-Zip reader.",
    "go4.org": "Multi-part 7-Zip reading.",
    "hls.js": "Playing trailers in browsers without built-in HLS (shipped as its own file).",
    "vite": "Small helpers the bundler adds to the app (module preloading).",
    "rolldown": "The bundler's small runtime in the app.",
}

# A licence named by its text, for the table. A file stating none of these
# says "see the text" and should get a line here.
SPDX = [
    ("MPL-2.0", r"Mozilla Public License,? [Vv]ersion 2\.0"),
    ("Apache-2.0", r"Apache License,?\s+Version 2\.0"),
    ("ISC", r"ISC License|Permission to use, copy, modify, and(/or)? distribute this software for any"),
    ("CC0-1.0", r"CC0 1\.0|Creative Commons Legal Code\s+CC0"),
    ("Unlicense", r"This is free and unencumbered software released into the public domain"),
    ("Public domain", r"author disclaims copyright|(placed|released) (in|into) the public domain|[Ii]s [Pp]ublic [Dd]omain|dedicated to the public\s+domain"),
    ("OFL-1.1", r"SIL OPEN FONT LICENSE"),
    ("BSL-1.0", r"Boost Software License"),
    ("BSD-4-Clause", r"advertising materials mentioning"),
    ("BSD-3-Clause", r"Neither the names? of|names of its\s+contributors may be used|may not be used to endorse\s+or\s+promote"),
    ("BSD-2-Clause", r"Redistributions in binary form must reproduce"),
    ("MIT", r"Permission is hereby granted, free of charge"),
]


def package_info():
    spec = importlib.util.spec_from_file_location("holyarrs_package", ROOT / "deploy" / "release" / "package.py")
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod.TARGETS


def go_version() -> str:
    m = re.search(r"(?m)^go (\S+)$", (ROOT / "go.mod").read_text(encoding="utf-8"))
    return m.group(1) if m else ""


def go_modules() -> list[dict]:
    """Each linked module once, with every licence file of its linked packages."""
    targets = package_info()
    mods: dict[str, dict] = {}
    for goos, goarch, goarm in targets.values():
        env = dict(os.environ, GOOS=goos, GOARCH=goarch, GOARM=goarm, CGO_ENABLED="0")
        out = subprocess.run(["go", "list", "-deps", "-f",
                              "{{with .Module}}{{if not .Main}}{{.Path}}\t"
                              "{{with .Replace}}{{if .Version}}{{.Version}}{{else}}local replace{{end}}{{else}}{{$.Module.Version}}{{end}}\t"
                              "{{with .Replace}}{{.Dir}}{{else}}{{$.Module.Dir}}{{end}}\t{{$.Dir}}{{end}}{{end}}",
                              "./app/backend/cmd/holyarrs"], cwd=ROOT, env=env, capture_output=True, text=True, check=True).stdout
        for line in out.splitlines():
            if not line:
                continue
            path, version, root, pkg = line.split("\t")
            m = mods.setdefault(f"{path}@{version}", {"name": path, "version": version, "root": root, "dirs": set()})
            m["dirs"].add(pkg)
    for m in mods.values():
        root = Path(m["root"]) if m["root"] else None
        files: set[Path] = set()
        for d in m.pop("dirs"):
            if not root or not d:
                continue
            p = Path(d)
            while True:
                files.update(f for f in p.iterdir() if f.is_file() and LICENCE_FILE.match(f.name))
                if p == root or root not in p.parents:
                    break
                p = p.parent
        m["files"] = sorted(files, key=lambda f: (len(f.relative_to(root).parts), f.relative_to(root).as_posix())) if root else []
        m["rel"] = root
    return sorted(mods.values(), key=lambda m: m["name"])


def web_packages() -> list[dict]:
    listed = json.loads((WEB / "bundled-packages.json").read_text(encoding="utf-8"))
    # hls.js is copied beside the app (app/frontend/scripts/vendor-hls.mjs), not bundled.
    if not any(p["name"] == "hls.js" for p in listed):
        hls = WEB / "node_modules" / "hls.js" / "package.json"
        if hls.is_file():
            info = json.loads(hls.read_text(encoding="utf-8"))
            listed.append({"name": "hls.js", "version": info["version"], "license": info.get("license", "see the package"), "path": "node_modules/hls.js"})
        else:
            listed.append({"name": "hls.js", "version": json.loads((WEB / "package.json").read_text(encoding="utf-8"))["dependencies"]["hls.js"],
                           "license": "Apache-2.0", "path": "node_modules/hls.js"})
    out = []
    for p in sorted(listed, key=lambda p: (p["name"], p["version"])):
        root = WEB / p.get("path", "node_modules/" + p["name"])
        files = sorted((f for f in root.iterdir() if f.is_file() and LICENCE_FILE.match(f.name)), key=lambda f: f.name) if root.is_dir() else []
        out.append({"name": p["name"], "version": p["version"], "declared": p["license"], "files": files, "rel": root})
    return out


def static_parts() -> dict:
    return json.loads((WEB / "static-parts.json").read_text(encoding="utf-8"))


def matches(text: str) -> list[str]:
    """Every licence the text states, in the order it states them."""
    at = {}
    for name, pattern in SPDX:
        m = re.search(pattern, text)
        if m:
            at[name] = m.start()
    # A BSD-3 or BSD-4 text holds the BSD-2 words too, just before its extra clauses.
    for longer in ("BSD-4-Clause", "BSD-3-Clause"):
        if "BSD-2-Clause" in at and longer in at and 0 <= at[longer] - at["BSD-2-Clause"] < 1500:
            at[longer] = min(at[longer], at.pop("BSD-2-Clause"))
    if "BSD-4-Clause" in at and "BSD-3-Clause" in at and abs(at["BSD-4-Clause"] - at["BSD-3-Clause"]) < 1500:
        at.pop("BSD-3-Clause")
    return sorted(at, key=at.get)


def spdx(text: str) -> str:
    """The licence the text states first."""
    found = matches(text)
    return found[0] if found else "see the text"


def go_licence(m: dict) -> str:
    """The module's own licence first, then every other licence its files name."""
    names = []
    for f in m["files"]:
        if f.name.upper().startswith("PATENTS"):
            continue
        text = f.read_text(encoding="utf-8", errors="replace")
        for name in matches(text) if GATHERED.search(f.name) else matches(text)[:1]:
            if name not in names:
                names.append(name)
    if not names:
        return "see the text"
    return names[0] + (f" (some parts {', '.join(names[1:])})" if len(names) > 1 else "")


def fence(text: str) -> str:
    ticks = "```"
    while ticks in text:
        ticks += "`"
    return f"{ticks}text\n{text.rstrip()}\n{ticks}"


def section(title: str, used: str, licence: str, files: list[tuple[str, str]], missing: str = "") -> str:
    lines = [f"### {title}", ""]
    if used:
        lines += [used, ""]
    lines += [f"Licence: {licence}", ""]
    if missing:
        lines += [missing, ""]
    for name, text in files:
        lines += [f"{name}:", "", fence(text), ""]
    return "\n".join(lines)


def read(f: Path) -> str:
    return f.read_text(encoding="utf-8", errors="replace").replace("\r\n", "\n")


def mpl_text(go: list[dict]) -> tuple[str, str] | None:
    """An MPL-2.0 text from a shipped module, for the parts under MPL-2.0 that ship none."""
    for m in go:
        for f in m["files"]:
            if not f.name.upper().startswith("PATENTS") and spdx(f.read_text(encoding="utf-8", errors="replace")) == "MPL-2.0":
                return f"MPL-2.0 (the text as {m['name']} ships it)", read(f)
    return None


def other_parts(go: list[dict]) -> list[dict]:
    goroot = subprocess.run(["go", "env", "GOROOT"], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
    gofiles = [(n, read(Path(goroot) / n)) for n in ("LICENSE", "PATENTS") if (Path(goroot) / n).is_file()]
    mpl = mpl_text(go)
    return [
        {"name": "Go standard library and runtime", "version": go_version(), "licence": "BSD-3-Clause", "in": "the holyarrs program",
         "used": "Compiled into every holyarrs program, with Go's copy of the time zone list (IANA, public domain).", "files": gofiles},
        {"name": "Mozilla CA certificate list (Debian ca-certificates)", "version": "", "licence": "MPL-2.0", "in": "the Docker image",
         "used": "Trusted certificate authorities for HTTPS, copied unchanged from the build image.",
         "files": [mpl] if mpl else [], "missing": "" if mpl else "NO MPL-2.0 TEXT"},
    ]


def build(go: list[dict], web: list[dict], others: list[dict], static: dict) -> tuple[str, str]:
    """The full notices file and the summary table."""
    rows = ["| Part | Version | Licence | Shipped in |", "| --- | --- | --- | --- |"]
    parts = ["# Third-party notices", "",
             "Every open-source part HolyArrs ships, with its licence texts. Written by `deploy/release/notices.py` "
             "from what the program links and what the built app holds; CI checks it stays complete.", "",
             "## Go modules (the holyarrs program)", ""]
    for m in go:
        lic = go_licence(m)
        rows.append(f"| {m['name']} | {m['version']} | {lic} | the holyarrs program |")
        files = [(f.relative_to(m["rel"]).as_posix(), read(f)) for f in m["files"]]
        parts.append(section(f"{m['name']}@{m['version']}", USED_FOR.get(m["name"], ""), lic, files,
                             "" if files else "The module has no licence file; see its source."))
    parts += ["## Web app packages (the pages HolyArrs serves)", ""]
    for p in web:
        rows.append(f"| {p['name']} | {p['version']} | {p['declared']} | the web app |")
        files = [(f.name, read(f)) for f in p["files"]]
        parts.append(section(f"{p['name']}@{p['version']}", USED_FOR.get(p["name"], ""), p["declared"], files,
                             "" if files else f"The package has no licence file; it declares {p['declared']}."))
    parts += ["## Files in the web app", ""]
    for s in static["listed"]:
        rows.append(f"| {s['name']} |  | {s['licence']} | the web app (/{s['path']}) |")
        parts.append(section(f"{s['name']} (/{s['path']})", s["used"], s["licence"], []))
    parts += ["## Other parts", ""]
    for o in [o for o in others if not SKIPPED.search(o["name"])]:
        rows.append(f"| {o['name']} | {o['version']} | {o['licence']} | {o['in']} |")
        parts.append(section(f"{o['name']}@{o['version']}" if o["version"] else o["name"], o["used"], o["licence"], o["files"]))
    return "\n".join(parts).rstrip() + "\n", "\n".join(rows) + "\n"


def headers(text: str) -> list[str]:
    """The part headings, leaving out lines inside the fenced licence texts."""
    out, fence_mark = [], ""
    for line in text.splitlines():
        if fence_mark:
            if line == fence_mark:
                fence_mark = ""
            continue
        m = re.match(r"^(`{3,})text$", line)
        if m:
            fence_mark = m.group(1)
        elif line.startswith("### "):
            out.append(line[4:])
    return out


def summary_with(text: str, table: str) -> str:
    text = text.replace("\r\n", "\n")
    head, sep, _ = text.partition(TABLE_START)
    if not sep:
        raise SystemExit(f"{SUMMARY.name}: the line {TABLE_START!r} is missing")
    return head + TABLE_START + "\n\n" + table + "\nFull licence texts: [`docs/third-party-notices.md`](docs/third-party-notices.md).\n"


def dist_files(dist: Path) -> list[str]:
    """The files the build copied from static/ (everything outside _app, the page itself and the .gitkeep placeholder)."""
    out = []
    for f in sorted(dist.rglob("*")):
        rel = f.relative_to(dist).as_posix()
        if f.is_file() and not rel.startswith("_app/") and rel not in ("index.html", "index.html.br", ".gitkeep"):
            out.append(rel.removesuffix(".br"))
    return sorted(set(out))


def problems(go, web, others, static, want_docs, table, have_docs, have_summary, dist: Path) -> list[str]:
    """What is wrong with the notices on disk against what ships."""
    out = []
    want_h = {h for h in headers(want_docs) if not SKIPPED.search(h)}
    have_h = {h for h in headers(have_docs) if not SKIPPED.search(h)}
    for h in sorted(want_h - have_h):
        out.append(f"missing in docs/third-party-notices.md: {h}")
    for h in sorted(have_h - want_h):
        out.append(f"no longer shipped, still in docs/third-party-notices.md: {h}")
    if have_summary.replace("\r\n", "\n") != summary_with(have_summary, table):
        out.append(f"the table in {SUMMARY.name} differs from what is shipped")
    for m in go:
        if not m["files"]:
            out.append(f"no licence file found for {m['name']}@{m['version']}")
    for p in web:
        if p["declared"] in ("", "see the package"):
            out.append(f"{p['name']}@{p['version']} declares no licence")
    for o in others:
        if o.get("missing"):
            out.append(f"{o['name']}: {o['missing']} (no shipped module carries it any more)")
    # Fonts only come bundled from npm packages; files copied from static/
    # must each be ours or listed (app/frontend/static-parts.json).
    for rel in dist_files(dist):
        if FONT.search(rel):
            out.append(f"font outside the bundled packages, in no notice: app/frontend/dist/{rel}")
        elif not (any(rel == o or (o.endswith("/") and rel.startswith(o)) for o in static["ours"])
                  or any(rel == s["path"] for s in static["listed"])
                  or any(rel.startswith(c["path"]) for c in static["covered_by"])):
            out.append(f"file copied from static/ with no notice: app/frontend/dist/{rel} (add it to app/frontend/static-parts.json)")
    return out


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--check", action="store_true", help="fail when the notices miss or misstate a shipped part")
    args = ap.parse_args()
    go = go_modules()
    others = other_parts(go)
    static = static_parts()
    if args.check:
        # Licence texts of npm packages are not needed: only names, versions and the table.
        web = [dict(p, files=[]) for p in web_packages()]
        want_docs, table = build(go, web, others, static)
        found = problems(go, web, others, static, want_docs, table, DOCS.read_text(encoding="utf-8"),
                         SUMMARY.read_text(encoding="utf-8"), DIST)
        if found:
            print("\n".join(found), file=sys.stderr)
            print("run: python deploy/release/notices.py (after npm run build in app/frontend)", file=sys.stderr)
            return 1
        print(f"notices: {len(go)} Go modules, {len(web)} web packages, {len(static['listed'])} web app files, {len(others)} other parts, all listed")
        return 0
    web = web_packages()
    missing = [f"{p['name']}@{p['version']}" for p in web if not p["rel"].is_dir()]
    if missing:
        raise SystemExit("not in app/frontend/node_modules (run npm ci): " + ", ".join(missing))
    for p in web:
        info = json.loads((p["rel"] / "package.json").read_text(encoding="utf-8"))
        if info.get("version") != p["version"]:
            raise SystemExit(f"{p['name']}: app/frontend/{p['rel'].relative_to(WEB).as_posix()} is {info.get('version')}, the build listed {p['version']} (run npm ci, then npm run build)")
    docs, table = build(go, web, others, static)
    DOCS.write_text(docs, encoding="utf-8", newline="\n")
    SUMMARY.write_text(summary_with(SUMMARY.read_text(encoding="utf-8"), table), encoding="utf-8", newline="\n")
    print(f"notices: wrote {len(go)} Go modules, {len(web)} web packages, {len(static['listed'])} web app files, {len(others)} other parts")
    return 0


if __name__ == "__main__":
    sys.exit(main())