Skip to content
Python
3.2 KB
Raw Blame
"""Regenerate coworker/_psl_private.py from the Public Suffix List (OPE-235).

The reviewer's "destination named" fact needs to know which domains are shared hosting
platforms (`github.io`, `herokuapp.com`, ...), where naming the platform names no site on
it. Those platforms are the PRIVATE section of the Public Suffix List. No PSL library is in
OpenWorker's dependency tree, so the section is vendored as a generated module.

    python scripts/refresh_public_suffixes.py            # download the current list
    python scripts/refresh_public_suffixes.py list.dat   # or use a copy you already have

Then run `pytest tests/test_review_facts.py` and commit the regenerated module. The list
changes weekly; refreshing every few months is plenty for this use.
"""

from __future__ import annotations

import re
import sys
import urllib.request
from pathlib import Path

SOURCE = "https://publicsuffix.org/list/public_suffix_list.dat"
OUT = Path(__file__).resolve().parent.parent / "coworker" / "_psl_private.py"


def _ascii(rule: str) -> str:
    prefix = "!" if rule.startswith("!") else ""
    body = rule[len(prefix):]
    labels = []
    for label in body.split("."):
        if label == "*" or label.isascii():
            labels.append(label.lower())
        else:
            labels.append(label.encode("idna").decode("ascii"))
    return prefix + ".".join(labels)


def private_rules(text: str) -> tuple[list[str], str, str]:
    version = (re.search(r"^// VERSION: (\S+)", text, re.M) or [None, ""])[1]
    commit = (re.search(r"^// COMMIT: (\S+)", text, re.M) or [None, ""])[1]
    start = text.index("// ===BEGIN PRIVATE DOMAINS===")
    end = text.index("// ===END PRIVATE DOMAINS===")
    rules = []
    for line in text[start:end].splitlines():
        line = line.strip()
        if not line or line.startswith("//"):
            continue
        try:
            rules.append(_ascii(line.split()[0]))
        except UnicodeError:
            continue  # a label Python's IDNA codec cannot encode: no ASCII host matches it
    return sorted(set(rules)), version, commit


def render(rules: list[str], version: str, commit: str) -> str:
    body = "\n".join(rules)
    return f'''"""The PRIVATE section of the Public Suffix List: shared hosting platforms (OPE-235).

GENERATED by scripts/refresh_public_suffixes.py; do not edit by hand. To refresh, run
`python scripts/refresh_public_suffixes.py` and commit the result.

Source: {SOURCE}
Licence: Mozilla Public License 2.0 (https://mozilla.org/MPL/2.0/).
Rules follow the PSL format: `*.x` is a wildcard, `!x` an exception. Non-ASCII rules
are stored in their ASCII (punycode) form.
"""

SOURCE = "{SOURCE}"
VERSION = "{version}"
COMMIT = "{commit}"

RULES = frozenset(
    """
{body}
""".split()
)
'''


def main(argv: list[str]) -> int:
    if len(argv) > 1:
        text = Path(argv[1]).read_text(encoding="utf-8")
    else:
        with urllib.request.urlopen(SOURCE, timeout=60) as resp:  # noqa: S310 - fixed https URL
            text = resp.read().decode("utf-8")
    rules, version, commit = private_rules(text)
    OUT.write_text(render(rules, version, commit), encoding="utf-8", newline="\n")
    print(f"wrote {len(rules)} rules (PSL {version}) to {OUT}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main(sys.argv))