Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
xnuloader, tools: the banner says hello hackios-kernel! and kernel.ver=; a version for the kernel
shimoku5c5808f56dbf
diff --git a/.gitignore b/.gitignoreindex 7138285..5480a8b 100644--- a/.gitignore+++ b/.gitignore@@ -4,6 +4,7 @@/*!/.gitignore!/.gitattributes+!/VERSION!/README.md!/CONTRIBUTING.md!/ci/diff --git a/VERSION b/VERSIONnew file mode 100644index 0000000..25df9b5--- /dev/null+++ b/VERSION@@ -0,0 +1,24 @@+# The version of the HakiOS kernel, given the way the Makefile of Linux and an Android kernel give theirs:+#+# VERSION.PATCHLEVEL.SUBLEVEL EXTRAVERSION+#+# tools/kernel-version.py makes the whole string from this file, the active configuration (.config) and+# git: 2.1.13-hakios-darwin22.5-sc7280-g1a2b3c4d (the version, EXTRAVERSION, the Darwin it is built on, the+# SoC family the configuration is for, and the commit; "-dirty" when the tracked files have changed).+# The loader prints it as kernel.ver= under its banner, passes it to the kernel as the boot-arg hakios_kver,+# and the kernel keeps it as sysctl kern.hakios.version. docs/versioning.md says when each number moves.+#+# VERSION major: a change that breaks what a user, a port or a boot image relies on (the payload format, the+# boot-arguments, the layout of the partition `darwin`)+# PATCHLEVEL a feature: a driver, a device, a tool+# SUBLEVEL a fix+# EXTRAVERSION is what the tree is called; a vendor's branch says so here (Android's "-android12-9")+VERSION = 2+PATCHLEVEL = 1+SUBLEVEL = 13+EXTRAVERSION = -hakios+NAME = Hello Darwin++# the Darwin this kernel is built on: Apple's open xnu tag (xnu/apply.sh fetches it)+DARWIN = 22.5.0+XNU = xnu-8796.121.2diff --git a/ci/github-check.yml b/ci/github-check.ymlindex 828ab37..d2611b6 100644--- a/ci/github-check.yml+++ b/ci/github-check.yml@@ -25,6 +25,8 @@ jobs:python3 tools/provenance-check.py- name: The provenance check itself: what it lets through and what it stopsrun: python3 tools/provenance-test.py+ - name: The kernel's version string is made as it should be+ run: python3 tools/kernel-version-test.py- name: The kernel patch applies to Apple's treerun: |git clone -q --depth 1 --branch xnu-8796.121.2 https://github.com/apple-oss-distributions/xnu.git /tmp/xnudiff --git a/tools/kernel-version-test.py b/tools/kernel-version-test.pynew file mode 100644index 0000000..f8d6f5d--- /dev/null+++ b/tools/kernel-version-test.py@@ -0,0 +1,112 @@+#!/usr/bin/env python3+"""tools/kernel-version.py: how the version string is made, and what it refuses.++ the real VERSION file -> 2.1.13-hakios-darwin22.5-g<8 hex>, in the form Android's kernels have+ a tracked file changed -> "-dirty"; a new untracked file is not a change (as in Linux's setlocalversion)+ CONFIG_SOC in a .config -> "-<soc>" between the Darwin and the commit+ HAKIOS_VERSION_SUFFIX -> replaces the git part+ not a git tree -> no git part, and no failure+ a malformed VERSION -> an error that says which line+"""+import os+import re+import shutil+import subprocess+import sys+import tempfile++HERE = os.path.dirname(os.path.abspath(__file__))+ROOT = os.path.dirname(HERE)+TOOL = os.path.join(HERE, "kernel-version.py")+failures = []+++def check(label, ok, detail=""):+ print("%s %s" % ("ok " if ok else "FAIL", label))+ if not ok:+ failures.append(label)+ if detail:+ print(" " + detail.strip().replace("\n", "\n ")[:400])+++def run(root, *args, env=None):+ e = dict(os.environ)+ e.pop("HAKIOS_VERSION_SUFFIX", None)+ e.update(env or {})+ return subprocess.run([sys.executable, TOOL, "--root", root, *args], capture_output=True, text=True, env=e)+++def git(d, *args):+ subprocess.run(["git", "-c", "user.name=t", "-c", "user.email=t@t", "-C", d, *args], check=True, capture_output=True)+++work = tempfile.mkdtemp(prefix="kernel-version-test-")+try:+ # a repository with the real VERSION file, committed+ repo = os.path.join(work, "repo")+ os.makedirs(repo)+ subprocess.run(["git", "init", "-q", repo], check=True)+ shutil.copyfile(os.path.join(ROOT, "VERSION"), os.path.join(repo, "VERSION"))+ open(os.path.join(repo, "tracked.txt"), "w").write("one\n")+ git(repo, "add", "-A")+ git(repo, "commit", "-q", "-m", "x")++ full = run(repo)+ check("the full string has the Android form", re.fullmatch(r"\d+\.\d+\.\d+-hakios-darwin22\.5-g[0-9a-f]{8}", full.stdout.strip()) is not None,+ full.stdout + full.stderr)+ check("--short is VERSION.PATCHLEVEL.SUBLEVEL", re.fullmatch(r"\d+\.\d+\.\d+", run(repo, "--short").stdout.strip()) is not None)+ check("--release adds EXTRAVERSION", run(repo, "--release").stdout.strip() == run(repo, "--short").stdout.strip() + "-hakios")+ check("--name and --darwin", run(repo, "--name").stdout.strip() == "Hello Darwin" and+ run(repo, "--darwin").stdout.split() == ["22.5.0", "xnu-8796.121.2"])+ check("the string starts with the release", full.stdout.startswith(run(repo, "--release").stdout.strip() + "-darwin"))++ open(os.path.join(repo, "untracked.txt"), "w").write("new\n")+ check("an untracked file is not a change", "-dirty" not in run(repo).stdout)+ open(os.path.join(repo, "tracked.txt"), "w").write("two\n")+ check("a changed tracked file makes it -dirty", run(repo).stdout.strip().endswith("-dirty"), run(repo).stdout)+ git(repo, "checkout", "--", "tracked.txt")+ check("restored: clean again", "-dirty" not in run(repo).stdout)++ open(os.path.join(repo, ".config"), "w").write('# a configuration\nCONFIG_SOC="sc7280"\nCONFIG_CPUS=8\n')+ check("CONFIG_SOC of .config goes between the Darwin and the commit",+ re.fullmatch(r"\d+\.\d+\.\d+-hakios-darwin22\.5-sc7280-g[0-9a-f]{8}", run(repo).stdout.strip()) is not None, run(repo).stdout)+ other = os.path.join(work, "other.config")+ open(other, "w").write('CONFIG_SOC="sm8550"\n')+ check("--config names another configuration", "-sm8550-" in run(repo, "--config", other).stdout, run(repo, "--config", other).stdout)++ check("HAKIOS_VERSION_SUFFIX replaces the git part",+ run(repo, env={"HAKIOS_VERSION_SUFFIX": "-gdeadbeef"}).stdout.strip().endswith("-sc7280-gdeadbeef"))+ check("an empty HAKIOS_VERSION_SUFFIX means no git part", run(repo, env={"HAKIOS_VERSION_SUFFIX": ""}).stdout.strip().endswith("-sc7280"))++ # not a git tree+ plain = os.path.join(work, "plain")+ os.makedirs(plain)+ shutil.copyfile(os.path.join(ROOT, "VERSION"), os.path.join(plain, "VERSION"))+ r = run(plain)+ check("not a git tree: no git part, no failure", r.returncode == 0 and r.stdout.strip() == "%s-darwin22.5" % run(plain, "--release").stdout.strip(),+ r.stdout + r.stderr)++ # malformed files+ def with_version(text):+ d = os.path.join(work, "bad%d" % len(os.listdir(work)))+ os.makedirs(d)+ open(os.path.join(d, "VERSION"), "w").write(text)+ return run(d)++ good = open(os.path.join(ROOT, "VERSION")).read()+ r = with_version(good.replace("SUBLEVEL = 13", "SUBLEVEL = thirteen"))+ check("a SUBLEVEL that is not a number: refused", r.returncode != 0 and "SUBLEVEL" in r.stderr, r.stderr)+ r = with_version(good.replace("EXTRAVERSION = -hakios", "EXTRAVERSION = hakios"))+ check("an EXTRAVERSION without its dash: refused", r.returncode != 0 and "EXTRAVERSION" in r.stderr, r.stderr)+ r = with_version(good.replace("NAME = Hello Darwin\n", ""))+ check("a missing key: refused, and named", r.returncode != 0 and "NAME" in r.stderr, r.stderr)+ r = with_version(good + "this is not a setting\n")+ check("a line that is no setting: refused, with its number", r.returncode != 0 and re.search(r"VERSION:\d+", r.stderr), r.stderr)+ r = run(os.path.join(work, "nothing-here"))+ check("no VERSION file: refused", r.returncode != 0 and "no VERSION file" in r.stderr, r.stderr)+finally:+ shutil.rmtree(work, ignore_errors=True)++if failures:+ sys.exit("kernel-version-test: %d FAILED" % len(failures))+print("kernel-version-test: all passed")diff --git a/tools/kernel-version.py b/tools/kernel-version.pynew file mode 100644index 0000000..f2e6280--- /dev/null+++ b/tools/kernel-version.py@@ -0,0 +1,126 @@+#!/usr/bin/env python3+"""The version string of the HakiOS kernel, made from VERSION, the active configuration and git.++usage: tools/kernel-version.py [--short | --name | --darwin | --release | --all]+ [--config FILE] [--root DIR]++ (none) 2.1.13-hakios-darwin22.5-sc7280-g1a2b3c4d[-dirty] what goes under the banner as kernel.ver=+ --short 2.1.13 VERSION.PATCHLEVEL.SUBLEVEL+ --release 2.1.13-hakios with EXTRAVERSION: what a tag is called (v2.1.13-hakios)+ --name Hello Darwin the release's name+ --darwin 22.5.0 xnu-8796.121.2 what it is built on+ --all every part, one per line, for the build log++The parts of the full string, in the order Android's kernels give theirs:+ VERSION.PATCHLEVEL.SUBLEVEL from the file VERSION+ EXTRAVERSION ... likewise+ -darwin<major>.<minor> the Darwin it is built on (DARWIN in VERSION)+ -<soc> CONFIG_SOC of the active configuration, if there is one (`make redwood_defconfig`)+ -g<commit>[-dirty] git: the commit, and "-dirty" if a tracked file differs from it++HAKIOS_VERSION_SUFFIX in the environment replaces the git part (a reproducible build, or a tree that is+not a git checkout): the string it holds is added as it is.+"""+import argparse+import os+import re+import subprocess+import sys++HERE = os.path.dirname(os.path.abspath(__file__))+++def read_version(root):+ """VERSION as {KEY: value}: `KEY = value` lines, `#` comments."""+ path = os.path.join(root, "VERSION")+ if not os.path.isfile(path):+ sys.exit("kernel-version: no VERSION file in %s" % root)+ values = {}+ for n, line in enumerate(open(path, encoding="utf-8"), 1):+ line = line.split("#", 1)[0].strip()+ if not line:+ continue+ m = re.fullmatch(r"([A-Z]+)\s*=\s*(.*)", line)+ if not m:+ sys.exit("kernel-version: VERSION:%d: want KEY = value, got %r" % (n, line))+ values[m.group(1)] = m.group(2).strip()+ for key in ("VERSION", "PATCHLEVEL", "SUBLEVEL", "EXTRAVERSION", "NAME", "DARWIN", "XNU"):+ if key not in values:+ sys.exit("kernel-version: VERSION lacks %s" % key)+ for key in ("VERSION", "PATCHLEVEL", "SUBLEVEL"):+ if not values[key].isdigit():+ sys.exit("kernel-version: %s must be a number, got %r" % (key, values[key]))+ if values["EXTRAVERSION"] and not re.fullmatch(r"-[A-Za-z0-9._]+", values["EXTRAVERSION"]):+ sys.exit("kernel-version: EXTRAVERSION is a dash and letters or digits (-hakios), got %r" % values["EXTRAVERSION"])+ return values+++def config_soc(path):+ """CONFIG_SOC of a .config, or None: the SoC family the configuration is for."""+ if not path or not os.path.isfile(path):+ return None+ for line in open(path, encoding="utf-8"):+ m = re.fullmatch(r'CONFIG_SOC="?([a-z0-9]+)"?\s*', line.strip())+ if m:+ return m.group(1)+ return None+++def git_part(root):+ """-g<commit>[-dirty], or "" when this is not a git tree (or git is not there)."""+ def git(*args):+ return subprocess.run(["git", "-C", root, *args], capture_output=True, text=True)++ try:+ head = git("rev-parse", "--short=8", "HEAD")+ except OSError:+ return ""+ if head.returncode != 0:+ return ""+ git("update-index", "-q", "--refresh") # stat data may be stale after a checkout: not a change+ dirty = git("diff-index", "--quiet", "HEAD", "--").returncode != 0+ return "-g" + head.stdout.strip() + ("-dirty" if dirty else "")+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ what = ap.add_mutually_exclusive_group()+ for flag in ("short", "release", "name", "darwin", "all"):+ what.add_argument("--" + flag, action="store_true")+ ap.add_argument("--config", help="the configuration to take CONFIG_SOC from (default: .config at the top)")+ ap.add_argument("--root", default=os.path.dirname(HERE))+ args = ap.parse_args()++ root = os.path.abspath(args.root)+ v = read_version(root)+ short = "%s.%s.%s" % (v["VERSION"], v["PATCHLEVEL"], v["SUBLEVEL"])+ release = short + v["EXTRAVERSION"]+ darwin = "-darwin" + ".".join(v["DARWIN"].split(".")[:2])+ soc = config_soc(args.config or os.path.join(root, ".config"))+ suffix = os.environ.get("HAKIOS_VERSION_SUFFIX")+ git = suffix if suffix is not None else git_part(root)+ full = release + darwin + ("-" + soc if soc else "") + git++ if args.short:+ print(short)+ elif args.release:+ print(release)+ elif args.name:+ print(v["NAME"])+ elif args.darwin:+ print(v["DARWIN"], v["XNU"])+ elif args.all:+ print("full=" + full)+ print("short=" + short)+ print("release=" + release)+ print("name=" + v["NAME"])+ print("darwin=" + v["DARWIN"])+ print("xnu=" + v["XNU"])+ print("soc=" + (soc or ""))+ print("git=" + git)+ else:+ print(full)+++if __name__ == "__main__":+ main()diff --git a/xnuloader/loader.c b/xnuloader/loader.cindex 99ee699..9358b31 100644--- a/xnuloader/loader.c+++ b/xnuloader/loader.c@@ -2491,6 +2491,7 @@ struct payload_info {uint64_t checkpoint;uint64_t forced_base;uint64_t resident_rd; /* size of a root image the host put at the resident address */+ const char *kernel_version; /* kernel.ver: the version of the kernel in this payload (tools/kernel-version.py) */int usb_mode;};@@ -2540,6 +2541,8 @@ parse_payload(struct payload_info *pi, int verbose)pi->forced_base = parse_number(text, 16); /* testing aid; must be free RAM */} else if (str_eq(e->name, "rdres")) {pi->resident_rd = parse_number(text, 16); /* e.g. loaded there by QEMU; see RESIDENT_RD_OFFSET */+ } else if (str_eq(e->name, "kernel.ver")) {+ pi->kernel_version = text; /* printed under the banner; the kernel gets it as hakios_kver */} else if (str_eq(e->name, "checkpoint")) {pi->checkpoint = parse_number(text, 10); /* where the kernel should stop and come back */} else if (str_eq(e->name, "usb")) {@@ -3008,9 +3011,38 @@ run_kernel(int have_checkpoint, uint64_t checkpoint)}if (n < BOOT_LINE_LENGTH - 1) ba->commandLine[n++] = ' ';}- for (const char *c = pi.cmdline; *c && n < BOOT_LINE_LENGTH - 1; c++) {- ba->commandLine[n++] = *c;+ /* the version of this kernel, for sysctl kern.hakios.version and the kernel's log (no spaces in it) */+ if (pi.kernel_version != NULL && pi.kernel_version[0] != 0) {+ static const char prefix[] = "hakios_kver=";+ uint32_t before = n, need = (uint32_t)(sizeof(prefix) - 1 + str_len(pi.kernel_version) + 1);++ if (n + need < BOOT_LINE_LENGTH - 1) {+ for (const char *c = prefix; *c; c++) {+ ba->commandLine[n++] = *c;+ }+ for (const char *c = pi.kernel_version; *c && *c != ' '; c++) {+ ba->commandLine[n++] = *c;+ }+ ba->commandLine[n++] = ' ';+ } else {+ print("WARNING: hakios_kver= does not fit in the kernel command line (%d of %d used)\n", (uint64_t)before,+ (uint64_t)(BOOT_LINE_LENGTH - 1));+ }+ }+ /*+ * The payload's own command line, last. The kernel's boot_args holds BOOT_LINE_LENGTH characters and the+ * copy used to stop at the end without a word: a long line lost its tail (the last argument arrived+ * shortened and the kernel took it as that). Now it is said, with how much was lost.+ */+ const char *rest = pi.cmdline;+ for (; *rest && n < BOOT_LINE_LENGTH - 1; rest++) {+ ba->commandLine[n++] = *rest;+ }+ if (*rest != 0) {+ print("WARNING: the kernel command line is TRUNCATED: %d characters of the payload's cmdline did not fit in the "+ "%d the kernel has; the last argument is cut\n", (uint64_t)str_len(rest), (uint64_t)(BOOT_LINE_LENGTH - 1));}+ print("cmdline: %d of %d characters\n", (uint64_t)n, (uint64_t)(BOOT_LINE_LENGTH - 1));/* --- device tree ---------------------------------------------------- */const struct platform *plat = PLAT;@@ -3907,13 +3939,45 @@ ramfb_phone_screen(void)}/*- * The greeting. Printed from efi_main, while the UEFI text console is still ours: the USB command- * loop that follows sets screen_quiet and leaves the screen alone, so what is on it now stays- * there. It also goes into the log, which "xl log" fetches.+ * The version of the kernel in the payload (the entry kernel.ver, made by xnuloader/mkpayload.py from+ * tools/kernel-version.py), or NULL if the payload has none. A look at the payload that changes+ * nothing and prints nothing, so that the banner can use it before the rest of the loader runs.+ */+static const char *+payload_kernel_version(void)+{+ const struct payload_header *ph = (const struct payload_header *)PAYLOAD_BASE;++ if (ph->magic != PAYLOAD_MAGIC) {+ return NULL;+ }+ const uint8_t *p = (const uint8_t *)(ph + 1), *pend = p + ph->size;+ while (p + sizeof(struct payload_entry) <= pend) {+ const struct payload_entry *e = (const struct payload_entry *)p;++ if (e->name[0] == 0) {+ break;+ }+ if (str_eq(e->name, "kernel.ver")) {+ return (const char *)(p + sizeof(*e));+ }+ p += sizeof(*e) + ((e->size + 15) & ~15u);+ }+ return NULL;+}++/*+ * The greeting. Printed from efi_main, while the UEFI text console is still ours (on the phone that is+ * Project Silicium's UEFI): the USB command loop that follows sets screen_quiet and leaves the screen+ * alone, so what is on it now stays there. It also goes into the log, which "xl log" fetches.+ * The two lines under the letters are the ones the project's banner has: a greeting from the kernel,+ * and its version, kernel.ver=, as tools/kernel-version.py makes it.*/static voidprint_banner(void){+ const char *version = payload_kernel_version();+print("\n");print(" # # ### # # ##### ### ####\n");print(" # # # # # # # # # #\n");@@ -3921,8 +3985,8 @@ print_banner(void)print(" # # # # # # # # # #\n");print(" # # # # # # ##### ### ####\n");print("\n");- print(" hello hakios!\n");- print(" XNU loader: iOS on Android hardware\n");+ print(" hello hackios-kernel!\n");+ print(" kernel.ver=%s\n", version ? version : "unknown");print("\n");}diff --git a/xnuloader/mkpayload.py b/xnuloader/mkpayload.pyindex 9afa17e..4eb8bfa 100755--- a/xnuloader/mkpayload.py+++ b/xnuloader/mkpayload.py@@ -17,6 +17,9 @@ ap.add_argument('--physbase', default='', help='hex: force XNU physical base (teap.add_argument('--usb', action='store_true', help='make the loader stay resident and take commands over USB')ap.add_argument('--rdres', default='', help='hex size of a root image already placed at the resident address (QEMU)')ap.add_argument('--checkpoint', default='', help='kernel checkpoint to stop at; 999 = loader self test')+ap.add_argument('--kernel-version', default='',+ help='the kernel.ver the loader prints under its banner (default: tools/kernel-version.py)')+ap.add_argument('--no-version', action='store_true', help='no kernel.ver entry')ap.add_argument('items', nargs='*')args = ap.parse_args()@@ -38,6 +41,19 @@ if args.usb:add('usb', b'1\0')if args.checkpoint:add('checkpoint', args.checkpoint.encode() + b'\0')+# The version of the kernel in this payload, which the loader prints as "kernel.ver=" under its banner and+# hands to the kernel as the boot-arg hakios_kver. A loader that does not know the entry skips it.+if not args.no_version:+ version = args.kernel_version+ if not version:+ import os, subprocess, sys+ tool = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'tools', 'kernel-version.py')+ try:+ version = subprocess.run([sys.executable, tool], capture_output=True, text=True, check=True).stdout.strip()+ except (OSError, subprocess.CalledProcessError) as e:+ print('mkpayload.py: no kernel.ver (tools/kernel-version.py failed: %s)' % e, file=sys.stderr)+ if version:+ add('kernel.ver', version.encode() + b'\0')body += b'\0' * 24open(args.output, 'wb').write(struct.pack('<II', 0x50554E58, len(body)) + body)print(f'{args.output}: {len(body) + 8} bytes')