Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
#!/usr/bin/env python3
# Read a UDIF disk image (.dmg): the block table in its XML property list, and the
# compressed chunks it points at. Enough of the format to get at what is inside:
# random reads anywhere in the decompressed image, and a whole-image extract.
#
# A UDIF file is laid out back to front: the compressed data comes first, then an XML
# property list, then a 512-byte trailer ("koly") that says where the plist is. The
# plist holds one "blkx" resource per partition -- each a "mish" block table whose
# entries map a run of output sectors to a run of bytes in the file.
#
# usage: dmg.py info IMAGE
# dmg.py extract IMAGE OUT [--part N | --apfs]
# dmg.py cat IMAGE OFFSET LENGTH (hex dump, for poking at structures)
import base64
import bz2
import lzma
import plistlib
import struct
import sys
import zlib
SECTOR = 512
# chunk types, as they appear in a block table entry
CHUNK_ZERO = 0x00000000 # a run of zeros, no data in the file
CHUNK_RAW = 0x00000001 # stored as is
CHUNK_IGNORE = 0x00000002 # unallocated, reads as zeros
CHUNK_COMMENT = 0x7FFFFFFE
CHUNK_ADC = 0x80000004
CHUNK_ZLIB = 0x80000005
CHUNK_BZIP2 = 0x80000006
CHUNK_LZFSE = 0x80000007
CHUNK_LZMA = 0x80000008
CHUNK_LAST = 0xFFFFFFFF
CHUNK_NAMES = {
CHUNK_ZERO: "zero", CHUNK_RAW: "raw", CHUNK_IGNORE: "ignore",
CHUNK_COMMENT: "comment", CHUNK_ADC: "adc", CHUNK_ZLIB: "zlib",
CHUNK_BZIP2: "bzip2", CHUNK_LZFSE: "lzfse", CHUNK_LZMA: "lzma",
CHUNK_LAST: "last",
}
class Chunk:
__slots__ = ("type", "sector", "count", "in_off", "in_len")
def __init__(self, type_, sector, count, in_off, in_len):
self.type = type_
self.sector = sector # first output sector, absolute
self.count = count # output sectors
self.in_off = in_off # byte offset in the file
self.in_len = in_len # bytes in the file
@property
def out_off(self):
return self.sector * SECTOR
@property
def out_len(self):
return self.count * SECTOR
def __repr__(self):
return "<chunk %s out=%d+%d in=%d+%d>" % (
CHUNK_NAMES.get(self.type, hex(self.type)),
self.out_off, self.out_len, self.in_off, self.in_len)
class Blkx:
"""One block table: the chunks of one partition (or of the whole image)."""
def __init__(self, name, id_, blob):
self.name = name
self.id = id_
magic, ver, sector, count, data_off, _bufs, _descs = struct.unpack_from(
">4sIQQQII", blob, 0)
if magic != b"mish":
raise ValueError("block table %r: bad magic %r" % (name, magic))
self.version = ver
self.sector = sector # where this partition starts in the image
self.count = count # its length in sectors
nchunks = struct.unpack_from(">I", blob, 0xC8)[0]
self.chunks = []
off = 0xCC
for _ in range(nchunks):
t, _comment, csec, ccnt, coff, clen = struct.unpack_from(">IIQQQQ", blob, off)
off += 40
if t in (CHUNK_LAST, CHUNK_COMMENT):
continue
# a chunk's sector number is relative to the table, its file offset to data_off
self.chunks.append(Chunk(t, sector + csec, ccnt, data_off + coff, clen))
@property
def out_off(self):
return self.sector * SECTOR
@property
def out_len(self):
return self.count * SECTOR
def _adc_decompress(src, out_len):
"""Apple Data Compression: a byte-oriented LZ with three kinds of opcode."""
out = bytearray()
i = 0
while i < len(src) and len(out) < out_len:
b = src[i]
if b & 0x80: # literal run, 1..128 bytes
n = (b & 0x7F) + 1
out += src[i + 1:i + 1 + n]
i += 1 + n
elif b & 0x40: # long match: 4..67 bytes, distance <= 65535
n = (b & 0x3F) + 4
dist = (src[i + 1] << 8) | src[i + 2]
i += 3
for _ in range(n):
out.append(out[-dist - 1])
else: # short match: 3..10 bytes, distance <= 1023
n = ((b & 0x3C) >> 2) + 3
dist = ((b & 0x03) << 8) | src[i + 1]
i += 2
for _ in range(n):
out.append(out[-dist - 1])
return bytes(out[:out_len])
def _lzfse_decompress(src, out_len):
try:
import lzfse
except ImportError:
raise SystemExit("dmg.py: this image uses LZFSE chunks; install the python "
"'lzfse' module (pip install lzfse)")
return lzfse.decompress(src)[:out_len]
class Dmg:
"""A UDIF image, readable at any offset of its decompressed contents."""
def __init__(self, path):
self.path = path
self.f = open(path, "rb")
self.f.seek(0, 2)
self.file_size = self.f.tell()
self._read_trailer()
self._read_plist()
# one cached decompressed chunk: reads walk forward, so this is nearly always a hit
self._cache_key = None
self._cache_buf = None
def _read_trailer(self):
self.f.seek(self.file_size - 512)
k = self.f.read(512)
if k[:4] != b"koly":
raise ValueError("%s: no UDIF trailer (koly) at the end of the file" % self.path)
self.version = struct.unpack_from(">I", k, 4)[0]
self.flags = struct.unpack_from(">I", k, 0xC)[0]
self.data_fork_off, self.data_fork_len = struct.unpack_from(">QQ", k, 0x18)
self.rsrc_fork_off, self.rsrc_fork_len = struct.unpack_from(">QQ", k, 0x28)
self.xml_off, self.xml_len = struct.unpack_from(">QQ", k, 0xD8)
self.image_variant = struct.unpack_from(">I", k, 0x1E8)[0]
self.sector_count = struct.unpack_from(">Q", k, 0x1EC)[0]
def _read_plist(self):
if not self.xml_len:
raise ValueError("%s: no XML property list; a resource fork table is not "
"supported" % self.path)
self.f.seek(self.xml_off)
pl = plistlib.loads(self.f.read(self.xml_len))
self.tables = []
for ent in pl["resource-fork"]["blkx"]:
data = ent["Data"]
if isinstance(data, str):
data = base64.b64decode(data)
self.tables.append(Blkx(ent.get("Name", ""), ent.get("ID", ""), data))
self.tables.sort(key=lambda b: b.sector)
# one flat chunk list for the whole image, so a read need not care about partitions
self._chunks = []
for b in self.tables:
self._chunks.extend(b.chunks)
self._chunks.sort(key=lambda c: c.sector)
self._starts = [c.out_off for c in self._chunks]
@property
def size(self):
return self.sector_count * SECTOR
def _chunk_data(self, c):
"""The decompressed bytes of one chunk."""
if self._cache_key == c.in_off and self._cache_buf is not None:
return self._cache_buf
if c.type in (CHUNK_ZERO, CHUNK_IGNORE):
buf = b"\0" * c.out_len
else:
self.f.seek(c.in_off)
raw = self.f.read(c.in_len)
if c.type == CHUNK_RAW:
buf = raw
elif c.type == CHUNK_ZLIB:
buf = zlib.decompress(raw)
elif c.type == CHUNK_BZIP2:
buf = bz2.decompress(raw)
elif c.type == CHUNK_LZMA:
buf = lzma.decompress(raw)
elif c.type == CHUNK_LZFSE:
buf = _lzfse_decompress(raw, c.out_len)
elif c.type == CHUNK_ADC:
buf = _adc_decompress(raw, c.out_len)
else:
raise ValueError("unknown chunk type 0x%08x at output offset %d"
% (c.type, c.out_off))
if len(buf) != c.out_len:
raise ValueError("chunk at output offset %d: %d bytes out of %s, "
"expected %d" % (c.out_off, len(buf),
CHUNK_NAMES.get(c.type, "?"), c.out_len))
self._cache_key, self._cache_buf = c.in_off, buf
return buf
def read(self, off, length):
"""Read the decompressed image. Holes between chunks read as zeros."""
import bisect
out = bytearray()
end = off + length
while off < end:
i = bisect.bisect_right(self._starts, off) - 1
c = self._chunks[i] if i >= 0 else None
if c is None or off >= c.out_off + c.out_len:
# before the first chunk, or in a hole: zeros up to the next chunk
nxt = self._starts[i + 1] if i + 1 < len(self._starts) else end
n = min(end, nxt) - off
out += b"\0" * n
off += n
continue
buf = self._chunk_data(c)
inner = off - c.out_off
n = min(end - off, c.out_len - inner)
out += buf[inner:inner + n]
off += n
return bytes(out)
def extract(self, out_path, start=0, length=None, progress=None):
"""Write a byte range of the decompressed image to a file."""
if length is None:
length = self.size - start
end = start + length
written = 0
with open(out_path, "wb") as g:
# walk the chunks that overlap the range, so nothing is decompressed twice
import bisect
i = max(0, bisect.bisect_right(self._starts, start) - 1)
pos = start
while pos < end:
c = self._chunks[i] if i < len(self._chunks) else None
if c is None or c.out_off >= end:
g.write(b"\0" * (end - pos))
written += end - pos
break
if pos < c.out_off: # hole before this chunk
n = c.out_off - pos
g.write(b"\0" * n)
written += n
pos = c.out_off
if pos >= c.out_off + c.out_len:
i += 1
continue
buf = self._chunk_data(c)
inner = pos - c.out_off
n = min(end - pos, c.out_len - inner)
g.write(buf[inner:inner + n])
written += n
pos += n
if progress and (i & 0x3FF) == 0:
progress(written, length)
i += 1
if progress:
progress(written, length)
return written
# ---- GPT, just enough to find the partitions in an extracted whole-disk image ----
APFS_PART_TYPE = "7C3457EF-0000-11AA-AA11-00306543ECAC"
def _guid(b):
"""A GPT type GUID: three little-endian fields then eight bytes as they lie."""
d1, d2, d3 = struct.unpack_from("<IHH", b, 0)
return "%08X-%04X-%04X-%s-%s" % (
d1, d2, d3, b[8:10].hex().upper(), b[10:16].hex().upper())
class Partition:
def __init__(self, index, type_guid, part_guid, first, last, name):
self.index = index
self.type_guid = type_guid
self.guid = part_guid
self.first_lba = first
self.last_lba = last
self.name = name
@property
def offset(self):
return self.first_lba * SECTOR
@property
def length(self):
return (self.last_lba - self.first_lba + 1) * SECTOR
def read_gpt(reader):
"""Partitions of a GPT, from anything with a read(off, len). [] if there is no GPT."""
hdr = reader.read(SECTOR, SECTOR)
if hdr[:8] != b"EFI PART":
return []
part_lba, nparts, entsz = struct.unpack_from("<QII", hdr, 72)
tbl = reader.read(part_lba * SECTOR, nparts * entsz)
out = []
for i in range(nparts):
e = tbl[i * entsz:(i + 1) * entsz]
t = _guid(e[0:16])
if t == "00000000-0000-0000-0000-000000000000":
continue
first, last = struct.unpack_from("<QQ", e, 32)
name = e[56:128].decode("utf-16-le", "replace").rstrip("\0")
out.append(Partition(i, t, _guid(e[16:32]), first, last, name))
return out
def _fmt(n):
return "%d (%.2f GB)" % (n, n / 1e9) if n >= 1 << 30 else "%d (%.1f MB)" % (n, n / 1e6)
def cmd_info(path):
d = Dmg(path)
print("%s: %d bytes on disk" % (path, d.file_size))
print("UDIF version %d, flags 0x%x, image variant %d" % (d.version, d.flags, d.image_variant))
print("data fork off=%d len=%d" % (d.data_fork_off, d.data_fork_len))
print("xml plist off=%d len=%d" % (d.xml_off, d.xml_len))
print("image size %s in %d sectors" % (_fmt(d.size), d.sector_count))
print()
kinds = {}
for c in d._chunks:
k = kinds.setdefault(CHUNK_NAMES.get(c.type, hex(c.type)), [0, 0, 0])
k[0] += 1
k[1] += c.in_len
k[2] += c.out_len
print("%d block tables, %d chunks:" % (len(d.tables), len(d._chunks)))
for name, (n, cin, cout) in sorted(kinds.items()):
print(" %-8s %7d chunks %14d -> %14d bytes" % (name, n, cin, cout))
print()
print("block tables:")
for b in d.tables:
print(" id=%-4s out=%d+%d (%s) %d chunks %s"
% (b.id, b.out_off, b.out_len, _fmt(b.out_len), len(b.chunks), b.name))
print()
parts = read_gpt(d)
if not parts:
print("no GPT at sector 1 -- the image is a bare file system, not a whole disk")
return
print("GPT, %d partitions:" % len(parts))
for p in parts:
tag = " <- APFS" if p.type_guid == APFS_PART_TYPE else ""
print(" %d lba %d..%d off=%d len=%s %-24s %s%s"
% (p.index, p.first_lba, p.last_lba, p.offset, _fmt(p.length),
p.name, p.type_guid, tag))
def cmd_extract(path, out, which):
d = Dmg(path)
start, length, what = 0, d.size, "the whole image"
if which is not None:
parts = read_gpt(d)
if not parts:
raise SystemExit("dmg.py: no GPT in this image, nothing to pick from")
if which == "apfs":
cand = [p for p in parts if p.type_guid == APFS_PART_TYPE]
if not cand:
raise SystemExit("dmg.py: no APFS partition in the GPT")
p = cand[0]
else:
sel = [p for p in parts if p.index == int(which)]
if not sel:
raise SystemExit("dmg.py: no partition %s" % which)
p = sel[0]
start, length = p.offset, p.length
what = "partition %d (%s)" % (p.index, p.name)
def prog(done, total):
pct = 100.0 * done / total if total else 100.0
sys.stderr.write("\r %s: %6.2f%% %d/%d bytes" % (what, pct, done, total))
sys.stderr.flush()
n = d.extract(out, start, length, prog)
sys.stderr.write("\n")
print("%s -> %s: %d bytes from offset %d" % (path, out, n, start))
head = d.read(start, 64)
print("first 32 bytes: %s" % head[:32].hex())
# an APFS container starts with an object header; its magic is 32 bytes in
if head[32:36] == b"NXSB":
print("magic at offset 32: NXSB -- an APFS container")
def cmd_cat(path, off, length):
d = Dmg(path)
buf = d.read(off, length)
for i in range(0, len(buf), 16):
row = buf[i:i + 16]
txt = "".join(chr(c) if 32 <= c < 127 else "." for c in row)
print("%012x %-47s %s" % (off + i, row.hex(" "), txt))
def main(argv):
if len(argv) < 3:
print(__doc__ or "", file=sys.stderr)
print("usage: dmg.py info IMAGE\n"
" dmg.py extract IMAGE OUT [--part N | --apfs]\n"
" dmg.py cat IMAGE OFFSET LENGTH", file=sys.stderr)
return 2
cmd = argv[1]
if cmd == "info":
cmd_info(argv[2])
elif cmd == "extract":
which = None
for a in argv[4:]:
if a == "--apfs":
which = "apfs"
elif a.startswith("--part"):
which = a.split("=", 1)[1] if "=" in a else argv[argv.index(a) + 1]
cmd_extract(argv[2], argv[3], which)
elif cmd == "cat":
cmd_cat(argv[2], int(argv[3], 0), int(argv[4], 0))
else:
raise SystemExit("dmg.py: unknown command %r" % cmd)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv))