Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C
10.3 KB
Raw Blame
#!/usr/bin/env python3
"""Every binary file, and every firmware blob hidden in a source file, must say where it came from.

usage: tools/provenance-check.py [--report] [--root DIR]

The rule of this repository is that it holds only what its authors may redistribute: the source of
the project, and files built from that source. Nothing of Apple's closed software (Darwin, its images,
fonts, pictures), nothing of a phone vendor's firmware. A rule like that holds only if a machine
checks it on every push, so this does. It reads the files git tracks and fails when it finds

  - a binary file (an executable, an image, a font, a firmware image, an archive ...) that no entry of
    tools/provenance.allow explains;
  - a source file that is mostly a table of bytes (firmware compiled into a header) with no entry;
  - a file of a kind the project never carries (screenshots, disk images, IPSW or IPA files,
    cryptexes, the dyld shared cache, fonts), with no entry;
  - a file with an entry whose status is `vendor` (proprietary, not cleared for redistribution);
  - a script that would write to the phone's flash (fastboot flash, erase, format, oem unlock, dd to a
    block device): the project boots from RAM and promises not to touch the phone's storage.

tools/provenance.allow has one line per rule:   STATUS  GLOB  reason
  open     redistributable: source-built from this tree, or under an open licence, with the licence named
  pending  known and not settled; the check passes with a warning, and the entry says who must decide
  vendor   proprietary; the check fails until the file is removed from the tree
  flash    a script that is allowed to contain a command that writes to the phone's storage, because a
           person runs it by hand for a one-time job; named here so that it is a decision, not an accident
The first line whose glob matches a path decides. A path no line matches is unexplained.

--report prints the explained files as a Markdown table (docs/provenance.md is made with it).
Exit status: 0 clean, 1 something to fix.
"""
import argparse
import fnmatch
import os
import re
import subprocess
import sys

# kinds the project never carries; each needs an entry to be allowed at all
NEVER = re.compile(r"\.(png|jpe?g|gif|heic|webp|bmp|tiff?|mov|mp4|m4v|m4a|mp3|aiff?|caf|ttf|ttc|otf|woff2?|"
                   r"ipsw|ipa|dmg|img4|im4p|car|nib|tbd|mobileconfig|strings|zip|tar|tgz|gz|xz|7z)$", re.I)
NEVER_PATH = re.compile(r"cryptex|dyld_shared_cache|\.ipsw|SpringBoard\.app|(^|/)System/Library/", re.I)
# a source file that is mostly 0x.., 0x.., ... lines: firmware compiled into a header
HEXLINE = re.compile(r"^\s*(0x[0-9a-fA-F]{2},\s*){8,}")
HEX_LINES_LIMIT = 200
# commands that write to a phone's storage; the project's scripts must not contain them
FLASHY = re.compile(r"fastboot\s+(flash|erase|format|flashing|oem\s+unlock|oem\s+lock|set_active)\b|"
                    r"\bdd\s+[^|;&\n]*of=/dev/(block|mmcblk|sd[a-z]|nvme)|\bmke2fs\b.*\b/dev/|"
                    r"\badb\s+(shell\s+)?(su\s+-c\s+)?['\"]?(flash_image|erase_image|blkdiscard)")
# ... and in C: the loader's write to a block device, and the kernel's UFS driver's. Both are limited in code to
# the partitions named in the `flash` lines of tools/provenance.allow, and both are declared there.
FLASHY_C = re.compile(r"\bwrite_blocks\s*\(|\bBLOCK_OP_WRITE\b")
C_WRITERS = re.compile(r"^(xnuloader/[^/]+|drivers/storage/hakios\.ufs[^/]*)\.c$")
SCRIPT_DIRS = ("tools/", "devices/", "qemu/", "sysimage/", "xnuloader/", "userland/build.sh")
SCRIPT_FILES = ("xl", "Makefile", "xnu-build.sh")
# this file, its test and the doctor mention these commands in order to look for them (the test has them as data)
SKIP_SCAN = {"tools/provenance-check.py", "tools/provenance-test.py", "tools/doctor.sh"}


def tracked(root):
    out = subprocess.run(["git", "ls-files", "-z"], cwd=root, capture_output=True, check=True).stdout
    return [p.decode() for p in out.split(b"\0") if p]


def load_rules(root):
    rules = []
    path = os.path.join(root, "tools", "provenance.allow")
    if not os.path.exists(path):
        return rules
    for n, line in enumerate(open(path, encoding="utf-8"), 1):
        line = line.rstrip("\n")
        if not line.strip() or line.lstrip().startswith("#"):
            continue
        parts = line.split(None, 2)
        if len(parts) < 3 or parts[0] not in ("open", "pending", "vendor", "flash"):
            sys.exit("provenance.allow:%d: want  open|pending|vendor|flash  GLOB  reason  (got: %s)" % (n, line))
        rules.append((parts[0], parts[1], parts[2], n))
    return rules


def match(rules, path, flash=False):
    """The first rule for `path`: among the `flash` rules when asked for those, among the others if not."""
    for status, glob, reason, _ in rules:
        if (status == "flash") == flash and fnmatch.fnmatchcase(path, glob):
            return status, reason
    return None


def kind_of(head):
    if head[:4] in (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe", b"\xca\xfe\xba\xbe"):
        return "Mach-O"
    if head[:2] == b"MZ":
        return "PE/EFI"
    if head[:4] == b"\x7fELF":
        return "ELF"
    if head[:4] == b"\xd0\x0d\xfe\xed":
        return "device tree"
    if head[:8] == b"\x89PNG\r\n\x1a\n":
        return "PNG"
    if head[:3] == b"\xff\xd8\xff":
        return "JPEG"
    if head[:4] == b"NXSB" or head[32:36] == b"NXSB":
        return "APFS"
    return "binary"


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--report", action="store_true")
    ap.add_argument("--root", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
    args = ap.parse_args()
    root = os.path.abspath(args.root)
    rules = load_rules(root)
    used = set()
    problems, warnings, explained = [], [], []

    files = tracked(root)
    for path in files:
        full = os.path.join(root, path)
        if not os.path.isfile(full) or os.path.islink(full):
            continue
        with open(full, "rb") as f:
            head = f.read(8192)
        binary = b"\0" in head
        why = None
        if binary:
            why = kind_of(head)
        elif NEVER.search(path) or NEVER_PATH.search(path):
            why = "a kind of file the project does not carry"
        elif path.endswith((".h", ".c", ".inc", ".S")):
            with open(full, "r", errors="replace") as f:
                hexlines = sum(1 for line in f if HEXLINE.match(line))
            if hexlines >= HEX_LINES_LIMIT:
                why = "firmware or data compiled into source (%d lines of bytes)" % hexlines
        if why is not None:
            hit = match(rules, path)
            if hit is None:
                problems.append("%s: %s, and no line of tools/provenance.allow explains it" % (path, why))
                continue
            status, reason = hit
            used.add(path)
            explained.append((path, why, status, reason, os.path.getsize(full)))
            if status == "vendor":
                problems.append("%s: %s -- marked vendor (proprietary): remove it from the tree" % (path, why))
            elif status == "pending":
                warnings.append("%s: %s -- pending: %s" % (path, why, reason))
        # scripts, and C code of the loader and the UFS driver, that would write to the phone's storage
        scan_c = bool(C_WRITERS.match(path))
        is_script = (path in SCRIPT_FILES or path.startswith(SCRIPT_DIRS)) and path.endswith((".sh", ".py", "xl", "Makefile"))
        if path not in SKIP_SCAN and (scan_c or is_script):
            try:
                text = open(full, "r", errors="replace").read()
            except OSError:
                continue
            for n, line in enumerate(text.splitlines(), 1):
                code = line.split("#", 1)[0] if path.endswith((".sh", "xl", "Makefile")) else line
                found = (FLASHY_C if scan_c else FLASHY).search(code)
                if not found:
                    continue
                hit = match(rules, path, flash=True)
                if hit is None:
                    problems.append("%s:%d: writes to the phone's storage (%s): the project boots from RAM; a "
                                    "deliberate exception needs a `flash` line in tools/provenance.allow"
                                    % (path, n, found.group(0).strip()))
                elif path not in used:
                    used.add(path)
                    explained.append((path, "writes to the phone's storage, by a person's hand", "flash", hit[1],
                                      os.path.getsize(full)))
                    warnings.append("%s: declared exception to \"nothing is written to the phone\": %s"
                                    % (path, hit[1]))
                break

    # a declared writer is a statement about a file, so it is listed even if no pattern fired in it today
    for status, glob, reason, _ in rules:
        if status == "flash":
            for p in files:
                if fnmatch.fnmatchcase(p, glob) and p not in used:
                    used.add(p)
                    explained.append((p, "declared writer to the phone's storage", "flash", reason,
                                      os.path.getsize(os.path.join(root, p))))
    # an entry that matches nothing has gone stale: a rule for a file that left the tree is clutter that
    # would explain a file that comes back
    # (a `vendor` line is a tripwire: it stays after the files are gone, so that putting them back fails)
    stale = [(s, g, n) for s, g, _, n in rules if s != "vendor" and not any(fnmatch.fnmatchcase(p, g) for p in files)]
    for s, g, n in stale:
        warnings.append("tools/provenance.allow:%d: '%s' matches no tracked file (stale)" % (n, g))

    if args.report:
        print("| File | What | Status | Why it may be here | Bytes |")
        print("|---|---|---|---|---:|")
        for path, why, status, reason, size in sorted(explained):
            print("| `%s` | %s | %s | %s | %d |" % (path, why, status, reason, size))
        return 0

    for w in warnings:
        print("warn: " + w)
    for p in problems:
        print("FAIL: " + p)
    print("provenance: %d tracked files, %d explained binaries or blobs, %d warnings, %d problems"
          % (len(files), len(explained), len(warnings), len(problems)))
    return 1 if problems else 0


if __name__ == "__main__":
    sys.exit(main())