Python
"""Regenerate coworker/_psl_private.py from the Public Suffix List (OPE-235).
The reviewer's "destination named" fact needs to know which domains are shared hosting
platforms (`github.io`, `herokuapp.com`, ...), where naming the platform names no site on
it. Those platforms are the PRIVATE section of the Public Suffix List. No PSL library is in
OpenWorker's dependency tree, so the section is vendored as a generated module.
python scripts/refresh_public_suffixes.py # download the current list
python scripts/refresh_public_suffixes.py list.dat # or use a copy you already have
Then run `pytest tests/test_review_facts.py` and commit the regenerated module. The list
changes weekly; refreshing every few months is plenty for this use.
"""
from __future__ import annotations
import re
import sys
import urllib.request
from pathlib import Path
SOURCE = "https://publicsuffix.org/list/public_suffix_list.dat"
OUT = Path(__file__).resolve().parent.parent / "coworker" / "_psl_private.py"
def _ascii(rule: str) -> str:
prefix = "!" if rule.startswith("!") else ""
body = rule[len(prefix):]
labels = []
for label in body.split("."):
if label == "*" or label.isascii():
labels.append(label.lower())
else:
labels.append(label.encode("idna").decode("ascii"))
return prefix + ".".join(labels)
def private_rules(text: str) -> tuple[list[str], str, str]:
version = (re.search(r"^// VERSION: (\S+)", text, re.M) or [None, ""])[1]
commit = (re.search(r"^// COMMIT: (\S+)", text, re.M) or [None, ""])[1]
start = text.index("// ===BEGIN PRIVATE DOMAINS===")
end = text.index("// ===END PRIVATE DOMAINS===")
rules = []
for line in text[start:end].splitlines():
line = line.strip()
if not line or line.startswith("//"):
continue
try:
rules.append(_ascii(line.split()[0]))
except UnicodeError:
continue # a label Python's IDNA codec cannot encode: no ASCII host matches it
return sorted(set(rules)), version, commit
def render(rules: list[str], version: str, commit: str) -> str:
body = "\n".join(rules)
return f'''"""The PRIVATE section of the Public Suffix List: shared hosting platforms (OPE-235).
GENERATED by scripts/refresh_public_suffixes.py; do not edit by hand. To refresh, run
`python scripts/refresh_public_suffixes.py` and commit the result.
Source: {SOURCE}
Licence: Mozilla Public License 2.0 (https://mozilla.org/MPL/2.0/).
Rules follow the PSL format: `*.x` is a wildcard, `!x` an exception. Non-ASCII rules
are stored in their ASCII (punycode) form.
"""
SOURCE = "{SOURCE}"
VERSION = "{version}"
COMMIT = "{commit}"
RULES = frozenset(
"""
{body}
""".split()
)
'''
def main(argv: list[str]) -> int:
if len(argv) > 1:
text = Path(argv[1]).read_text(encoding="utf-8")
else:
with urllib.request.urlopen(SOURCE, timeout=60) as resp: # noqa: S310 - fixed https URL
text = resp.read().decode("utf-8")
rules, version, commit = private_rules(text)
OUT.write_text(render(rules, version, commit), encoding="utf-8", newline="\n")
print(f"wrote {len(rules)} rules (PSL {version}) to {OUT}")
return 0
if __name__ == "__main__":
raise SystemExit(main(sys.argv))