Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C
10.3 KB
Raw Blame
#!/usr/bin/env python3
"""How far every compute kernel of the corpus gets, as a table that has to add up.

The corpus's 47 kernels are the part of it a draw cannot test, and air_run
dispatches them (air_run.c, dispatch_kernel). This reports where each one
stops, grouped by cause.

The grouping exists because the causes are not alike and lead to different
work: one of them is ours to fix, one is softpipe's and may not be the
Adreno's, and two are known limits with names. Telling them apart is the
whole point -- "6 of 47 dispatch" on its own invites the reader to guess why
the other 41 do not.

Every kernel lands in exactly one row and the rows are checked against the
total before anything is printed. That check is here because the first version
of this table was assembled by hand in a report, and it came to 45 of 47: two
kernels whose message matched no pattern fell into a prose aside instead of a
row, which is exactly the habit README's "Как врут собственные проверки" warns
about -- an exclusion without its own number. A table that cannot fail to add
up is worth more than a careful author.

usage: kernelcheck.py [--list] [FILE.metallib...]
       kernelcheck.py --log RUN.log [FILE.metallib...]
       (no files: the five libraries of the corpus, as check.sh has them)

--log reads a device run instead of dispatching here, so the table from the
phone is generated by the same code and checked against the same total. What it
can say is narrower than what the host table says, and the difference is not a
detail: softpipe will not run 38 of these kernels, so there is no executed
reference for them anywhere, and a number out of the Adreno has nothing to be
compared against. The rows mean dispatched or refused. They do not mean right.
"""

import os
import subprocess
import sys

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
AIR2NIR = os.environ.get(
    "AIR2NIR", os.path.join(ROOT, "build/mesa-host/build/src/compiler/air/air2nir"))
AIR_RUN = os.environ.get(
    "AIR_RUN",
    os.path.join(ROOT, "build/mesa-host/build/src/gallium/drivers/freedreno/air_run"))

LIBS = [
    "sysimage/rootfs/System/Library/VideoDecoders/AV1SW.metallib",
    "sysimage/rootfs/System/Library/Extensions/AppleMetalGLRenderer.bundle/default.metallib",
    "sysimage/rootfs/System/Library/CoreImage/PortraitFilters.cifilter/default.metallib",
    "sysimage/rootfs/System/Library/CoreImage/PortraitFilters.cifilter/portrait_filters.metallib",
]

# One row per cause, in the order a reader wants them: what works, then what is
# ours, then what is not ours, then the named limits. The matcher is a substring
# of air_run's own message, so a message this does not know falls to "unmatched"
# and is printed with the kernel's name rather than counted as something else.
CAUSES = [
    ("dispatched", None,
     "dispatched and read back"),
    ("harness", "this harness",
     "refused by this harness (ours to fix)"),
    ("softpipe", ("tgsi_ureg.h", "nir_to_tgsi.c", "Unknown intrinsic"),
     "softpipe's nir_to_tgsi: scratch and intrinsics it lacks (ir3 has these)"),
    ("library", "intrinsic: a call of",
     "calls an air.visible library function in the module"),
    ("shuffle", "a shuffle mask that is not constant",
     "a shuffle mask that is not constant (a frontend limit)"),
]


def kernels(libs):
    out = []
    for lib in libs:
        path = lib if os.path.isabs(lib) else os.path.join(ROOT, lib)
        if not os.path.exists(path):
            print("kernelcheck: missing %s" % lib, file=sys.stderr)
            sys.exit(1)
        q = subprocess.run([AIR2NIR, "--list", path], capture_output=True)
        for line in q.stdout.decode(errors="replace").splitlines():
            f = line.split()
            if len(f) >= 2 and f[1] == "kernel":
                out.append((path, f[0]))
    return out


def classify(text):
    for key, needles, _ in CAUSES:
        if needles is None:
            continue
        if isinstance(needles, str):
            needles = (needles,)
        if any(n in text for n in needles):
            return key
    return None


def from_log(path, ks):
    """Each kernel's outcome as a device run reported it.

    Keyed on the name air_run prints, not on the order of the lines: a console
    shared with launchd interleaves them, and the loader's log drops its own
    middle when it grows (README, "Как врут собственные проверки"). A kernel
    whose block never appears is "not in the log", which is its own row rather
    than being folded into a refusal -- the two have different causes and the
    log being truncated is one of them.
    """
    import re
    blocks, cur = {}, None
    for line in open(path, errors="replace"):
        m = re.match(r"^air_run: (\S+) on ", line)
        if m:
            cur = m.group(1)
            blocks.setdefault(cur, [])
            continue
        if cur:
            blocks[cur].append(line)
    out = {}
    for _, name in ks:
        b = blocks.get(name)
        if b is None:
            out[name] = ("missing", "")
            continue
        text = "".join(b)
        # A crash is not a refusal and must not be counted as one. air_run's
        # fault handler prints "signal N at ADDR" with a backtrace, and a
        # refusal prints a sentence saying what it would not do: the first is a
        # defect in this harness, the second is a limit it states. Folding them
        # together hid four segfaults inside a row called "refused", which read
        # as though the frontend had declined them.
        crash = next((l.strip() for l in b if "signal " in l and " at 0x" in l), None)
        if crash:
            out[name] = ("crashed", crash[:110])
        elif "came back as" in text or "after the dispatch" in text:
            out[name] = ("dispatched", "")
        else:
            # the LAST air_run line, not the first: the first is usually the
            # progress line it had already printed before giving up
            why = ""
            for l in b:
                if "air_run:" in l or "Assertion" in l:
                    why = l.strip()
            out[name] = ("refused", why[:110])
    return out


def main(argv):
    want_list = "--list" in argv
    log = None
    if "--log" in argv:
        i = argv.index("--log")
        log = argv[i + 1]
        argv = argv[:i] + argv[i + 2:]
    libs = [a for a in argv if not a.startswith("--")] or LIBS
    for prog in (AIR2NIR, AIR_RUN):
        if not os.path.exists(prog):
            print("kernelcheck: run tools/air/mk-host.sh first (%s)" % prog,
                  file=sys.stderr)
            return 1

    ks = kernels(libs)
    counts = {key: [] for key, _, _ in CAUSES}
    unmatched = []

    if log:
        res = from_log(log, ks)
        # A log with not one of these kernels in it is the wrong log, not a run
        # where every one went missing. Those two read identically -- 0, 0, 47 --
        # and only one of them is a result, so it is refused rather than
        # tabulated. (The same shape as intrincheck's refusal to judge a log
        # older than its manifest.)
        if all(v[0] == "missing" for v in res.values()):
            print("kernelcheck: %s holds no block for any of the %d kernels, so it is a log "
                  "of some other run rather than a run where all of them vanished"
                  % (os.path.basename(log), len(ks)), file=sys.stderr)
            return 1
        got = sum(1 for v in res.values() if v[0] == "dispatched")
        ref = {n: v[1] for n, v in res.items() if v[0] == "refused"}
        crashed = {n: v[1] for n, v in res.items() if v[0] == "crashed"}
        miss = [n for n, v in res.items() if v[0] == "missing"]
        print("kernelcheck: %d compute kernels, as %s reports them"
              % (len(ks), os.path.basename(log)))
        print("  %2d  dispatched and read back" % got)
        print("  %2d  refused, with a reason in the log" % len(ref))
        print("  %2d  CRASHED air_run (a defect here, not a limit)" % len(crashed))
        print("  %2d  not in the log at all (the loader drops its middle when it grows)"
              % len(miss))
        if want_list:
            for n in sorted(ref):
                print("        %-44s %s" % (n, ref[n]))
            for n in sorted(crashed):
                print("        %-44s %s" % (n, crashed[n]))
            for n in sorted(miss):
                print("        %-44s (absent)" % n)
        total = got + len(ref) + len(crashed) + len(miss)
        if total != len(ks):
            print("kernelcheck: the rows add to %d and there are %d kernels" % (total, len(ks)),
                  file=sys.stderr)
            return 1
        print("kernelcheck: the rows add to %d, which is every kernel. These rows say "
              "dispatched or refused, never right: softpipe runs too few of these for there "
              "to be a reference." % total)
        return 0

    for path, name in ks:
        q = subprocess.run([AIR_RUN, path, name], capture_output=True, timeout=300)
        if q.returncode == 0:
            counts["dispatched"].append(name)
            continue
        key = classify((q.stdout + q.stderr).decode(errors="replace"))
        if key is None:
            unmatched.append(name)
        else:
            counts[key].append(name)

    # before printing anything: the rows must account for every kernel. A table
    # that does not add up is worse than no table, because the reader checks the
    # sum in their head and believes it.
    total = sum(len(v) for v in counts.values()) + len(unmatched)
    print("kernelcheck: %d compute kernels in %d librar%s"
          % (len(ks), len(libs), "y" if len(libs) == 1 else "ies"))
    for key, _, label in CAUSES:
        print("  %2d  %s" % (len(counts[key]), label))
        if want_list and counts[key]:
            for n in sorted(counts[key]):
                print("        %s" % n)
    if unmatched:
        print("  %2d  stopped with a message this script does not know: %s"
              % (len(unmatched), ", ".join(sorted(unmatched))))
    if total != len(ks):
        print("kernelcheck: the rows add to %d and there are %d kernels, so this table "
              "is wrong and not merely incomplete" % (total, len(ks)), file=sys.stderr)
        return 1
    print("kernelcheck: the rows add to %d, which is every kernel" % total)
    return 1 if unmatched else 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))