One self-hosted app for your shows, movies and music.
#!/usr/bin/env python3
"""Third-party notices: every part HolyArrs ships, with its licence.
Writes docs/third-party-notices.md (each part with its licence files) and
the "Open-source parts" table at the end of THIRD_PARTY_NOTICES.md from:
- the Go modules `go list -deps ./app/backend/cmd/holyarrs` links, for every release
target in deploy/release/package.py (they differ per system), with the licence
files from each linked package's folder up to its module's root;
- the npm packages the built app holds (app/frontend/bundled-packages.json,
written by the app's build), with their licence files, plus hls.js,
which the build copies beside the app;
- the third-party files in app/frontend/static (app/frontend/static-parts.json),
which the build copies as they are;
- the few other parts below (Go itself and the Docker image's CA list).
python deploy/release/notices.py write both files
python deploy/release/notices.py --check fail when a shipped part is missing,
stale or of another version
The check needs Go and its module cache, not app/frontend/node_modules.
"""
from __future__ import annotations
import argparse
import importlib.util
import json
import os
import re
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
DOCS = ROOT / "docs" / "third-party-notices.md"
SUMMARY = ROOT / "THIRD_PARTY_NOTICES.md"
WEB = ROOT / "app" / "frontend"
DIST = ROOT / "app" / "frontend" / "dist"
TABLE_START = "<!-- notices: generated by deploy/release/notices.py; do not edit by hand -->"
FONT = re.compile(r"(?i)\.(woff2?|ttf|otf|eot)$")
LICENCE_FILE = re.compile(r"(?i)^(licen[cs]e|copying|notice|patents|unlicense)([.\-_].*)?$")
# The video tool ships as one release program with one row; its bundled parts are never listed.
SKIPPED = re.compile(r"(?i)yt-dlp.*(bundled|third[-_ ]?party|credit)|third_party_licenses")
# A file that gathers the licences of several parts: every licence in it counts.
GATHERED = re.compile(r"(?i)(3rd|third)[-_ ]?party")
# What a part is for, where that helps a reader; the rest speak for themselves.
USED_FOR = {
"github.com/coreos/go-oidc/v3": "Sign-in with an outside account (OpenID Connect).",
"github.com/go-jose/go-jose/v4": "Checking sign-in tokens (OpenID Connect).",
"golang.org/x/oauth2": "Sign-in with an outside account.",
"golang.org/x/crypto": "Password hashing and other cryptography.",
"golang.org/x/image": "Reading and resizing pictures.",
"golang.org/x/sys": "System calls.",
"golang.org/x/text": "Text handling (Unicode).",
"modernc.org/sqlite": "The database (SQLite, in Go).",
"modernc.org/libc": "Used by the database.",
"modernc.org/mathutil": "Used by the database.",
"modernc.org/memory": "Used by the database.",
"github.com/dustin/go-humanize": "Used by the database.",
"github.com/google/uuid": "Used by the database.",
"github.com/mattn/go-isatty": "Used by the database.",
"github.com/ncruces/go-strftime": "Used by the database.",
"github.com/remyoudompheng/bigfft": "Used by the database.",
"github.com/nwaples/rardecode/v2": "Unpacking RAR files.",
"github.com/bodgit/sevenzip": "Unpacking 7-Zip archives.",
"github.com/bodgit/plumbing": "Used by the 7-Zip reader.",
"github.com/bodgit/windows": "Used by the 7-Zip reader.",
"github.com/hashicorp/golang-lru/v2": "A small cache in the 7-Zip reader (encrypted archives). Used unchanged; source at https://github.com/hashicorp/golang-lru.",
"github.com/therootcompany/xz": "Unpacking xz archives.",
"github.com/ulikunitz/xz": "LZMA in 7-Zip archives.",
"github.com/klauspost/compress": "Compression in 7-Zip archives.",
"github.com/pierrec/lz4/v4": "LZ4 in 7-Zip archives.",
"github.com/andybalholm/brotli": "Brotli in 7-Zip archives.",
"github.com/stangelandcl/ppmd": "PPMd in 7-Zip archives.",
"github.com/spf13/afero": "File access for the 7-Zip reader.",
"go4.org": "Multi-part 7-Zip reading.",
"hls.js": "Playing trailers in browsers without built-in HLS (shipped as its own file).",
"vite": "Small helpers the bundler adds to the app (module preloading).",
"rolldown": "The bundler's small runtime in the app.",
}
# A licence named by its text, for the table. A file stating none of these
# says "see the text" and should get a line here.
SPDX = [
("MPL-2.0", r"Mozilla Public License,? [Vv]ersion 2\.0"),
("Apache-2.0", r"Apache License,?\s+Version 2\.0"),
("ISC", r"ISC License|Permission to use, copy, modify, and(/or)? distribute this software for any"),
("CC0-1.0", r"CC0 1\.0|Creative Commons Legal Code\s+CC0"),
("Unlicense", r"This is free and unencumbered software released into the public domain"),
("Public domain", r"author disclaims copyright|(placed|released) (in|into) the public domain|[Ii]s [Pp]ublic [Dd]omain|dedicated to the public\s+domain"),
("OFL-1.1", r"SIL OPEN FONT LICENSE"),
("BSL-1.0", r"Boost Software License"),
("BSD-4-Clause", r"advertising materials mentioning"),
("BSD-3-Clause", r"Neither the names? of|names of its\s+contributors may be used|may not be used to endorse\s+or\s+promote"),
("BSD-2-Clause", r"Redistributions in binary form must reproduce"),
("MIT", r"Permission is hereby granted, free of charge"),
]
def package_info():
spec = importlib.util.spec_from_file_location("holyarrs_package", ROOT / "deploy" / "release" / "package.py")
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.TARGETS
def go_version() -> str:
m = re.search(r"(?m)^go (\S+)$", (ROOT / "go.mod").read_text(encoding="utf-8"))
return m.group(1) if m else ""
def go_modules() -> list[dict]:
"""Each linked module once, with every licence file of its linked packages."""
targets = package_info()
mods: dict[str, dict] = {}
for goos, goarch, goarm in targets.values():
env = dict(os.environ, GOOS=goos, GOARCH=goarch, GOARM=goarm, CGO_ENABLED="0")
out = subprocess.run(["go", "list", "-deps", "-f",
"{{with .Module}}{{if not .Main}}{{.Path}}\t"
"{{with .Replace}}{{if .Version}}{{.Version}}{{else}}local replace{{end}}{{else}}{{$.Module.Version}}{{end}}\t"
"{{with .Replace}}{{.Dir}}{{else}}{{$.Module.Dir}}{{end}}\t{{$.Dir}}{{end}}{{end}}",
"./app/backend/cmd/holyarrs"], cwd=ROOT, env=env, capture_output=True, text=True, check=True).stdout
for line in out.splitlines():
if not line:
continue
path, version, root, pkg = line.split("\t")
m = mods.setdefault(f"{path}@{version}", {"name": path, "version": version, "root": root, "dirs": set()})
m["dirs"].add(pkg)
for m in mods.values():
root = Path(m["root"]) if m["root"] else None
files: set[Path] = set()
for d in m.pop("dirs"):
if not root or not d:
continue
p = Path(d)
while True:
files.update(f for f in p.iterdir() if f.is_file() and LICENCE_FILE.match(f.name))
if p == root or root not in p.parents:
break
p = p.parent
m["files"] = sorted(files, key=lambda f: (len(f.relative_to(root).parts), f.relative_to(root).as_posix())) if root else []
m["rel"] = root
return sorted(mods.values(), key=lambda m: m["name"])
def web_packages() -> list[dict]:
listed = json.loads((WEB / "bundled-packages.json").read_text(encoding="utf-8"))
# hls.js is copied beside the app (app/frontend/scripts/vendor-hls.mjs), not bundled.
if not any(p["name"] == "hls.js" for p in listed):
hls = WEB / "node_modules" / "hls.js" / "package.json"
if hls.is_file():
info = json.loads(hls.read_text(encoding="utf-8"))
listed.append({"name": "hls.js", "version": info["version"], "license": info.get("license", "see the package"), "path": "node_modules/hls.js"})
else:
listed.append({"name": "hls.js", "version": json.loads((WEB / "package.json").read_text(encoding="utf-8"))["dependencies"]["hls.js"],
"license": "Apache-2.0", "path": "node_modules/hls.js"})
out = []
for p in sorted(listed, key=lambda p: (p["name"], p["version"])):
root = WEB / p.get("path", "node_modules/" + p["name"])
files = sorted((f for f in root.iterdir() if f.is_file() and LICENCE_FILE.match(f.name)), key=lambda f: f.name) if root.is_dir() else []
out.append({"name": p["name"], "version": p["version"], "declared": p["license"], "files": files, "rel": root})
return out
def static_parts() -> dict:
return json.loads((WEB / "static-parts.json").read_text(encoding="utf-8"))
def matches(text: str) -> list[str]:
"""Every licence the text states, in the order it states them."""
at = {}
for name, pattern in SPDX:
m = re.search(pattern, text)
if m:
at[name] = m.start()
# A BSD-3 or BSD-4 text holds the BSD-2 words too, just before its extra clauses.
for longer in ("BSD-4-Clause", "BSD-3-Clause"):
if "BSD-2-Clause" in at and longer in at and 0 <= at[longer] - at["BSD-2-Clause"] < 1500:
at[longer] = min(at[longer], at.pop("BSD-2-Clause"))
if "BSD-4-Clause" in at and "BSD-3-Clause" in at and abs(at["BSD-4-Clause"] - at["BSD-3-Clause"]) < 1500:
at.pop("BSD-3-Clause")
return sorted(at, key=at.get)
def spdx(text: str) -> str:
"""The licence the text states first."""
found = matches(text)
return found[0] if found else "see the text"
def go_licence(m: dict) -> str:
"""The module's own licence first, then every other licence its files name."""
names = []
for f in m["files"]:
if f.name.upper().startswith("PATENTS"):
continue
text = f.read_text(encoding="utf-8", errors="replace")
for name in matches(text) if GATHERED.search(f.name) else matches(text)[:1]:
if name not in names:
names.append(name)
if not names:
return "see the text"
return names[0] + (f" (some parts {', '.join(names[1:])})" if len(names) > 1 else "")
def fence(text: str) -> str:
ticks = "```"
while ticks in text:
ticks += "`"
return f"{ticks}text\n{text.rstrip()}\n{ticks}"
def section(title: str, used: str, licence: str, files: list[tuple[str, str]], missing: str = "") -> str:
lines = [f"### {title}", ""]
if used:
lines += [used, ""]
lines += [f"Licence: {licence}", ""]
if missing:
lines += [missing, ""]
for name, text in files:
lines += [f"{name}:", "", fence(text), ""]
return "\n".join(lines)
def read(f: Path) -> str:
return f.read_text(encoding="utf-8", errors="replace").replace("\r\n", "\n")
def mpl_text(go: list[dict]) -> tuple[str, str] | None:
"""An MPL-2.0 text from a shipped module, for the parts under MPL-2.0 that ship none."""
for m in go:
for f in m["files"]:
if not f.name.upper().startswith("PATENTS") and spdx(f.read_text(encoding="utf-8", errors="replace")) == "MPL-2.0":
return f"MPL-2.0 (the text as {m['name']} ships it)", read(f)
return None
def other_parts(go: list[dict]) -> list[dict]:
goroot = subprocess.run(["go", "env", "GOROOT"], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
gofiles = [(n, read(Path(goroot) / n)) for n in ("LICENSE", "PATENTS") if (Path(goroot) / n).is_file()]
mpl = mpl_text(go)
return [
{"name": "Go standard library and runtime", "version": go_version(), "licence": "BSD-3-Clause", "in": "the holyarrs program",
"used": "Compiled into every holyarrs program, with Go's copy of the time zone list (IANA, public domain).", "files": gofiles},
{"name": "Mozilla CA certificate list (Debian ca-certificates)", "version": "", "licence": "MPL-2.0", "in": "the Docker image",
"used": "Trusted certificate authorities for HTTPS, copied unchanged from the build image.",
"files": [mpl] if mpl else [], "missing": "" if mpl else "NO MPL-2.0 TEXT"},
]
def build(go: list[dict], web: list[dict], others: list[dict], static: dict) -> tuple[str, str]:
"""The full notices file and the summary table."""
rows = ["| Part | Version | Licence | Shipped in |", "| --- | --- | --- | --- |"]
parts = ["# Third-party notices", "",
"Every open-source part HolyArrs ships, with its licence texts. Written by `deploy/release/notices.py` "
"from what the program links and what the built app holds; CI checks it stays complete.", "",
"## Go modules (the holyarrs program)", ""]
for m in go:
lic = go_licence(m)
rows.append(f"| {m['name']} | {m['version']} | {lic} | the holyarrs program |")
files = [(f.relative_to(m["rel"]).as_posix(), read(f)) for f in m["files"]]
parts.append(section(f"{m['name']}@{m['version']}", USED_FOR.get(m["name"], ""), lic, files,
"" if files else "The module has no licence file; see its source."))
parts += ["## Web app packages (the pages HolyArrs serves)", ""]
for p in web:
rows.append(f"| {p['name']} | {p['version']} | {p['declared']} | the web app |")
files = [(f.name, read(f)) for f in p["files"]]
parts.append(section(f"{p['name']}@{p['version']}", USED_FOR.get(p["name"], ""), p["declared"], files,
"" if files else f"The package has no licence file; it declares {p['declared']}."))
parts += ["## Files in the web app", ""]
for s in static["listed"]:
rows.append(f"| {s['name']} | | {s['licence']} | the web app (/{s['path']}) |")
parts.append(section(f"{s['name']} (/{s['path']})", s["used"], s["licence"], []))
parts += ["## Other parts", ""]
for o in [o for o in others if not SKIPPED.search(o["name"])]:
rows.append(f"| {o['name']} | {o['version']} | {o['licence']} | {o['in']} |")
parts.append(section(f"{o['name']}@{o['version']}" if o["version"] else o["name"], o["used"], o["licence"], o["files"]))
return "\n".join(parts).rstrip() + "\n", "\n".join(rows) + "\n"
def headers(text: str) -> list[str]:
"""The part headings, leaving out lines inside the fenced licence texts."""
out, fence_mark = [], ""
for line in text.splitlines():
if fence_mark:
if line == fence_mark:
fence_mark = ""
continue
m = re.match(r"^(`{3,})text$", line)
if m:
fence_mark = m.group(1)
elif line.startswith("### "):
out.append(line[4:])
return out
def summary_with(text: str, table: str) -> str:
text = text.replace("\r\n", "\n")
head, sep, _ = text.partition(TABLE_START)
if not sep:
raise SystemExit(f"{SUMMARY.name}: the line {TABLE_START!r} is missing")
return head + TABLE_START + "\n\n" + table + "\nFull licence texts: [`docs/third-party-notices.md`](docs/third-party-notices.md).\n"
def dist_files(dist: Path) -> list[str]:
"""The files the build copied from static/ (everything outside _app, the page itself and the .gitkeep placeholder)."""
out = []
for f in sorted(dist.rglob("*")):
rel = f.relative_to(dist).as_posix()
if f.is_file() and not rel.startswith("_app/") and rel not in ("index.html", "index.html.br", ".gitkeep"):
out.append(rel.removesuffix(".br"))
return sorted(set(out))
def problems(go, web, others, static, want_docs, table, have_docs, have_summary, dist: Path) -> list[str]:
"""What is wrong with the notices on disk against what ships."""
out = []
want_h = {h for h in headers(want_docs) if not SKIPPED.search(h)}
have_h = {h for h in headers(have_docs) if not SKIPPED.search(h)}
for h in sorted(want_h - have_h):
out.append(f"missing in docs/third-party-notices.md: {h}")
for h in sorted(have_h - want_h):
out.append(f"no longer shipped, still in docs/third-party-notices.md: {h}")
if have_summary.replace("\r\n", "\n") != summary_with(have_summary, table):
out.append(f"the table in {SUMMARY.name} differs from what is shipped")
for m in go:
if not m["files"]:
out.append(f"no licence file found for {m['name']}@{m['version']}")
for p in web:
if p["declared"] in ("", "see the package"):
out.append(f"{p['name']}@{p['version']} declares no licence")
for o in others:
if o.get("missing"):
out.append(f"{o['name']}: {o['missing']} (no shipped module carries it any more)")
# Fonts only come bundled from npm packages; files copied from static/
# must each be ours or listed (app/frontend/static-parts.json).
for rel in dist_files(dist):
if FONT.search(rel):
out.append(f"font outside the bundled packages, in no notice: app/frontend/dist/{rel}")
elif not (any(rel == o or (o.endswith("/") and rel.startswith(o)) for o in static["ours"])
or any(rel == s["path"] for s in static["listed"])
or any(rel.startswith(c["path"]) for c in static["covered_by"])):
out.append(f"file copied from static/ with no notice: app/frontend/dist/{rel} (add it to app/frontend/static-parts.json)")
return out
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--check", action="store_true", help="fail when the notices miss or misstate a shipped part")
args = ap.parse_args()
go = go_modules()
others = other_parts(go)
static = static_parts()
if args.check:
# Licence texts of npm packages are not needed: only names, versions and the table.
web = [dict(p, files=[]) for p in web_packages()]
want_docs, table = build(go, web, others, static)
found = problems(go, web, others, static, want_docs, table, DOCS.read_text(encoding="utf-8"),
SUMMARY.read_text(encoding="utf-8"), DIST)
if found:
print("\n".join(found), file=sys.stderr)
print("run: python deploy/release/notices.py (after npm run build in app/frontend)", file=sys.stderr)
return 1
print(f"notices: {len(go)} Go modules, {len(web)} web packages, {len(static['listed'])} web app files, {len(others)} other parts, all listed")
return 0
web = web_packages()
missing = [f"{p['name']}@{p['version']}" for p in web if not p["rel"].is_dir()]
if missing:
raise SystemExit("not in app/frontend/node_modules (run npm ci): " + ", ".join(missing))
for p in web:
info = json.loads((p["rel"] / "package.json").read_text(encoding="utf-8"))
if info.get("version") != p["version"]:
raise SystemExit(f"{p['name']}: app/frontend/{p['rel'].relative_to(WEB).as_posix()} is {info.get('version')}, the build listed {p['version']} (run npm ci, then npm run build)")
docs, table = build(go, web, others, static)
DOCS.write_text(docs, encoding="utf-8", newline="\n")
SUMMARY.write_text(summary_with(SUMMARY.read_text(encoding="utf-8"), table), encoding="utf-8", newline="\n")
print(f"notices: wrote {len(go)} Go modules, {len(web)} web packages, {len(static['listed'])} web app files, {len(others)} other parts")
return 0
if __name__ == "__main__":
sys.exit(main())