Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

tools/apfs: mkapfs.py makes APFS images of the port's own trees, check-image.py proves them

vcvkkc8ade8168838
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.c
index cbe793a..74f19ea 100644
--- a/tools/apfs/apfs-tool.c
+++ b/tools/apfs/apfs-tool.c
@@ -184,8 +184,11 @@ open_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol)
host_locking(nx);
op_begin();
}
- /* the first volume that has a root directory, like apfs.py picks it */
+ /* the first volume that has a root directory, like apfs.py picks it; or APFS_TOOL_VOLUME=N */
for (i = 0; i < APFS_NX_MAX_FILE_SYSTEMS; i++) {
+ if (getenv("APFS_TOOL_VOLUME") != NULL && (long)i != strtol(getenv("APFS_TOOL_VOLUME"), NULL, 0)) {
+ continue;
+ }
struct apfs_inode ino;
if (nx->fs_oid[i] == 0) {
diff --git a/tools/apfs/check-image.py b/tools/apfs/check-image.py
new file mode 100755
index 0000000..66368c8
--- /dev/null
+++ b/tools/apfs/check-image.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Check an image made by mkapfs.py against what it was made of, by two readers that are not it.
+
+usage: check-image.py IMAGE [VOLUME=]DIR[,manifest=FILE] ... (volumes in the order mkapfs.py had them)
+
+1. apfsck (linux-apfs/apfsprogs, xnu-deps/build/apfsprogs; tools/apfs/check-image.sh builds it)
+ with -cw has to find the container clean: every object's checksum, the trees, the space
+ manager's bitmaps against every block in use, inode and link counts, extents and references.
+2. tools/apfs/apfs-tool (the kernel's own apfs_lib) lists every volume as census lines -- path,
+ kind, mode, owner, group, size, FNV-1a 64 of the contents (or of a link's target) -- and they
+ have to be the lines of DIR (with the manifest's owners and modes), every one, in both
+ directions.
+"""
+import os
+import stat
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, HERE)
+import mkapfs # noqa: E402 (the same reading of DIR and manifest)
+
+APFSCK = os.path.join(ROOT, 'xnu-deps/build/apfsprogs/apfsck/apfsck')
+TOOL = os.path.join(ROOT, 'apfs-work/apfs-tool')
+FNVSUM = os.path.join(ROOT, 'apfs-work/fnvsum')
+
+
+def fnv1a(data):
+ h = 0xcbf29ce484222325
+ for b in data:
+ h = ((h ^ b) * 0x100000001b3) & 0xffffffffffffffff
+ return h
+
+
+def expected(root):
+ lines, files = {}, []
+
+ def walk(node, path):
+ for name in sorted(node.children):
+ child = node.children[name]
+ p = path + '/' + name
+ if stat.S_ISDIR(child.mode):
+ lines[p] = [4, child.mode, child.uid, child.gid, 0, 0]
+ walk(child, p)
+ elif stat.S_ISLNK(child.mode):
+ lines[p] = [10, child.mode, child.uid, child.gid, len(child.target), fnv1a(child.target)]
+ else:
+ lines[p] = [8, child.mode, child.uid, child.gid, os.path.getsize(child.source), None]
+ files.append((p, child.source))
+ walk(root, '')
+ if files:
+ out = subprocess.run([FNVSUM], input='\n'.join(s for _, s in files) + '\n', capture_output=True,
+ text=True, check=True).stdout.splitlines()
+ for (p, _), line in zip(files, out):
+ lines[p][5] = int(line.split('\t')[0], 16)
+ return {p: '%s\t%u\t%06o\t%u\t%u\t%u\t%016x' % (p, *v) for p, v in lines.items()}
+
+
+def main():
+ if len(sys.argv) < 3:
+ sys.exit(__doc__)
+ image = sys.argv[1]
+ if not os.path.exists(FNVSUM) or os.path.getmtime(FNVSUM) < os.path.getmtime(os.path.join(HERE, 'fnvsum.c')):
+ subprocess.run(['cc', '-O2', '-o', FNVSUM, os.path.join(HERE, 'fnvsum.c')], check=True)
+ r = subprocess.run([APFSCK, '-cw', image], capture_output=True, text=True)
+ print('check-image.py: apfsck -cw: %s%s' % ('clean' if r.returncode == 0 else 'NOT CLEAN (exit %d)' % r.returncode,
+ (': ' + (r.stdout + r.stderr).strip()) if (r.stdout + r.stderr).strip() else ''))
+ bad = r.returncode != 0
+ for index, spec in enumerate(sys.argv[2:]):
+ if '=' in spec.split(',')[0]:
+ spec = spec.split('=', 1)[1]
+ parts = spec.split(',')
+ root = mkapfs.read_tree(parts[0])
+ for opt in parts[1:]:
+ k, _, val = opt.partition('=')
+ if k == 'manifest':
+ mkapfs.apply_manifest(root, val)
+ want = expected(root)
+ env = dict(os.environ, APFS_TOOL_VOLUME=str(index))
+ r = subprocess.run([TOOL, 'census', image], capture_output=True, text=True, env=env)
+ got = {line.split('\t')[0]: line for line in r.stdout.splitlines() if line}
+ missing = sorted(set(want) - set(got))
+ extra = sorted(set(got) - set(want))
+ differ = sorted(p for p in set(want) & set(got) if want[p] != got[p])
+ ok = r.returncode == 0 and not missing and not extra and not differ
+ bad |= not ok
+ print('check-image.py: volume %d: %d entries read back by apfs-tool, %s' % (
+ index, len(got), 'every one as in %s' % parts[0] if ok else
+ '%d missing, %d extra, %d different, apfs-tool exit %d' % (len(missing), len(extra), len(differ),
+ r.returncode)))
+ for p in (missing[:5] + extra[:5]):
+ print(' %s %s' % ('missing' if p in missing else 'extra', p))
+ for p in differ[:5]:
+ print(' want %s\n got %s' % (want[p], got[p]))
+ sys.exit(1 if bad else 0)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/apfs/fnvsum.c b/tools/apfs/fnvsum.c
new file mode 100644
index 0000000..78c9183
--- /dev/null
+++ b/tools/apfs/fnvsum.c
@@ -0,0 +1,36 @@
+/*
+ * fnvsum: the FNV-1a 64 of files, as tools/apfs/apfs-tool census and the kernel's apfs_check
+ * hash contents. Paths one per line on stdin, "HASH<TAB>PATH" out. (Python is a hundred
+ * times slower at it, and an image of the system is gigabytes.)
+ */
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+int
+main(void)
+{
+ static unsigned char buf[1 << 20];
+ char path[4096];
+
+ while (fgets(path, sizeof(path), stdin) != NULL) {
+ uint64_t h = 0xcbf29ce484222325ULL;
+ size_t n;
+ FILE *f;
+
+ path[strcspn(path, "\n")] = '\0';
+ f = fopen(path, "rb");
+ if (f == NULL) {
+ printf("-\t%s\n", path);
+ continue;
+ }
+ while ((n = fread(buf, 1, sizeof(buf), f)) > 0) {
+ for (size_t i = 0; i < n; i++) {
+ h = (h ^ buf[i]) * 0x100000001b3ULL;
+ }
+ }
+ fclose(f);
+ printf("%016llx\t%s\n", (unsigned long long)h, path);
+ }
+ return 0;
+}
diff --git a/tools/apfs/mkapfs.py b/tools/apfs/mkapfs.py
new file mode 100755
index 0000000..d729188
--- /dev/null
+++ b/tools/apfs/mkapfs.py
@@ -0,0 +1,911 @@
+#!/usr/bin/env python3
+"""Build an APFS container from directories: the port's own read-only volumes, as images.
+
+usage: mkapfs.py -o IMAGE [-z] [--time SECONDS] [--slack BLOCKS] NAME=DIR[,manifest=FILE] [NAME=DIR ...]
+
+Each NAME=DIR becomes one volume of the container (volume 0 first), case-sensitive and
+normalization-insensitive like the iOS volumes, unencrypted, read-only content laid out once.
+Permission bits and symbolic links are taken from DIR; everything is owned by root:wheel
+unless a manifest says otherwise (the same lines as mkpackfs.py -m: mode in octal with the
+type bits, uid, gid, path, [link target]; a directory or symbolic link that is only in the
+manifest is made). With -z a regular file is stored compressed when that makes it smaller:
+zlib in 64 KB pieces in its resource fork (decmpfs type 4), as Apple stores most of the iOS
+root volume. Times are all --time (default: 2023-05-12, the iOS 16.5 images'), so the same
+input gives the same image byte for byte.
+
+The image is what Apple's newfs_apfs and a single transaction would leave (one checkpoint,
+transaction 1): the container superblock, the checkpoint (its map, the space manager with
+its internal pool, the reaper, the two free queues), the container's object map, and per
+volume its superblock, object map, file-system tree, extent-reference tree and an empty
+snapshot tree; then the files' data. What the layout of the space manager has to be is
+taken from linux-apfs/apfsprogs' mkapfs (and checked with its apfsck, which has to find the
+image clean: tools/apfs/check-image.sh). Hard links are not made: a name is a file.
+"""
+import argparse
+import concurrent.futures
+import hashlib
+import os
+import stat
+import struct
+import sys
+import unicodedata
+import zlib
+
+BS = 4096
+XID = 1
+OID_NX = 1
+OID_RESERVED = 1024 # the first object id that is not reserved
+ROOT_DIR_PARENT, ROOT_DIR_INO, PRIV_DIR_INO = 1, 2, 3
+MIN_USER_INO = 16
+
+# object types
+T_NX, T_BTREE, T_BTNODE, T_SPACEMAN, T_CAB, T_CIB = 0x1, 0x2, 0x3, 0x5, 0x6, 0x7
+T_FREE_QUEUE, T_OMAP, T_CPMAP, T_FS, T_FSTREE, T_BLOCKREF, T_SNAPMETA, T_REAPER = 0x9, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, 0x11
+VIRTUAL, EPHEMERAL, PHYSICAL = 0x0, 0x80000000, 0x40000000
+
+# b-tree node flags, tree flags
+BTNODE_ROOT, BTNODE_LEAF, BTNODE_FIXED = 0x1, 0x2, 0x4
+BTREE_ALLOW_GHOSTS, BTREE_EPHEMERAL, BTREE_PHYSICAL, BTREE_KV_NONALIGNED = 0x4, 0x8, 0x10, 0x40
+BTOFF_INVALID = 0xffff
+
+# file-system records
+J_SNAP_METADATA, J_EXTENT, J_INODE, J_XATTR, J_SIBLING_LINK, J_DSTREAM_ID = 1, 2, 3, 4, 5, 6
+J_FILE_EXTENT, J_DIR_REC = 8, 9
+DT_DIR, DT_REG, DT_LNK = 4, 8, 10
+INODE_HAS_RSRC_FORK, INODE_NO_RSRC_FORK, INODE_HAS_UNCOMPRESSED_SIZE = 0x4000, 0x8000, 0x40000
+UF_COMPRESSED = 0x20
+INO_EXT_TYPE_NAME, INO_EXT_TYPE_DSTREAM = 4, 8
+XF_DO_NOT_COPY, XF_SYSTEM_FIELD = 0x2, 0x20
+XATTR_DATA_STREAM, XATTR_DATA_EMBEDDED, XATTR_FS_OWNED = 0x1, 0x2, 0x4
+KIND_NEW = 1
+
+DECMPFS_MAGIC = 0x636d7066 # 'cmpf'
+DECMPFS_ZLIB_RSRC = 4
+PIECE = 65536
+
+SM_BITMAP_XID_OFF = 0x150 # (apfsprogs' mkapfs: from a test image)
+IP_BM_TX_MULTIPLIER = 16
+IP_BM_INDEX_INVALID = 0xffff
+
+
+def fletcher64(block):
+ """The checksum of an object: Fletcher-64 over its 32-bit words after the first eight bytes."""
+ words = struct.unpack_from('<%dI' % ((len(block) - 8) // 4), block, 8)
+ s1 = s2 = 0
+ for w in words:
+ s1 = (s1 + w) % 0xffffffff
+ s2 = (s2 + s1) % 0xffffffff
+ c1 = 0xffffffff - ((s1 + s2) % 0xffffffff)
+ c2 = 0xffffffff - ((s1 + c1) % 0xffffffff)
+ return (c2 << 32) | c1
+
+
+def seal(obj, oid, otype, subtype=0):
+ """Fill in an object's header (oid, transaction, type) and its checksum, in place."""
+ struct.pack_into('<QQII', obj, 8, oid, XID, otype, subtype)
+ struct.pack_into('<Q', obj, 0, fletcher64(obj))
+
+
+_CRC32C = []
+for _i in range(256):
+ _c = _i
+ for _ in range(8):
+ _c = (_c >> 1) ^ 0x82F63B78 if _c & 1 else _c >> 1
+ _CRC32C.append(_c)
+
+
+def name_hash(name):
+ """A directory record's hash: CRC-32C (no final inversion) over the name decomposed, as UTF-32."""
+ h = 0xffffffff
+ for ch in unicodedata.normalize('NFD', name):
+ for b in struct.pack('<I', ord(ch)):
+ h = (h >> 8) ^ _CRC32C[(h ^ b) & 0xff]
+ return h & 0x3fffff
+
+
+def div_up(a, b):
+ return (a + b - 1) // b
+
+
+# ---------------------------------------------------------------------- the input tree
+
+class Node:
+ def __init__(self, name, mode, source=None, target=None):
+ self.name = name # str
+ self.mode = mode # st_mode
+ self.uid = self.gid = 0
+ self.source = source # a regular file's path on this machine
+ self.target = target # a symbolic link's target, bytes
+ self.children = {} # name -> Node, for a directory
+ self.oid = 0
+
+
+def read_tree(path, name=''):
+ st = os.lstat(path)
+ if stat.S_ISDIR(st.st_mode):
+ node = Node(name, st.st_mode)
+ for entry in sorted(os.listdir(path)):
+ child = read_tree(os.path.join(path, entry), entry)
+ if child is not None:
+ node.children[entry] = child
+ return node
+ if stat.S_ISREG(st.st_mode):
+ return Node(name, st.st_mode, source=path)
+ if stat.S_ISLNK(st.st_mode):
+ return Node(name, st.st_mode, target=os.fsencode(os.readlink(path)))
+ return None # device nodes, FIFOs, sockets: /dev is devfs
+
+
+def apply_manifest(root, manifest):
+ for raw in open(manifest, 'rb'):
+ fields = raw.rstrip(b'\n').split(b'\t')
+ if len(fields) < 4 or not fields[3]:
+ continue
+ mode, uid, gid = int(fields[0], 8), int(fields[1]), int(fields[2])
+ parts = fields[3].decode('utf-8', 'surrogateescape').split('/')
+ node = root
+ for part in parts[:-1]:
+ node = node.children.get(part)
+ if node is None or not stat.S_ISDIR(node.mode):
+ break
+ else:
+ leaf = node.children.get(parts[-1])
+ if leaf is None:
+ if stat.S_ISDIR(mode):
+ leaf = node.children[parts[-1]] = Node(parts[-1], mode)
+ elif stat.S_ISLNK(mode) and len(fields) > 4:
+ leaf = node.children[parts[-1]] = Node(parts[-1], mode, target=fields[4])
+ else:
+ continue
+ leaf.mode, leaf.uid, leaf.gid = mode, uid, gid
+
+
+# ---------------------------------------------------------------------- b-tree nodes
+
+class Tree:
+ """A b-tree laid out from records already in order, leaves first, full nodes."""
+
+ def __init__(self, subtype, storage, fixed=None, tree_flags=0):
+ self.subtype = subtype # T_FSTREE, T_OMAP, ...
+ self.storage = storage # VIRTUAL or PHYSICAL
+ self.fixed = fixed # (key_size, val_size) for a tree of fixed sizes
+ self.tree_flags = tree_flags
+ self.levels = [] # [[(records, ...)]] per level, leaves first
+
+ @staticmethod
+ def entry_size(fixed, klen, vlen):
+ return (4 if fixed else 8) + klen + vlen
+
+ def plan(self, records):
+ """records: [(key bytes, value bytes)] in order. Splits them into nodes (no addresses yet)."""
+ self.records = records
+ self.levels = []
+ level = self._pack(records, leaf=True)
+ self.levels.append(level)
+ while len(level) > 1:
+ index = [(node[0][0], b'\0' * 8) for node in level] # child pointers come later
+ level = self._pack(index, leaf=False)
+ self.levels.append(level)
+ return sum(len(lv) for lv in self.levels)
+
+ def toc_len(self, n, leaf):
+ """The table of contents for n entries: a node of a fixed-size tree has its whole table up
+ front (as newfs_apfs and mkapfs make it), any other a multiple of 8 entries."""
+ if self.fixed is not None:
+ # (index nodes too, their values being 8-byte child pointers -- apfsck insists)
+ ks, vs = self.fixed
+ return ((BS - 56) // (ks + (vs if leaf else 8) + 4)) * 4
+ entry = 4 if self.fixed is not None else 8
+ return div_up(max(n, 8), 8) * 8 * entry
+
+ def _pack(self, records, leaf):
+ nodes, cur, kv = [], [], 0
+ for k, v in records:
+ # every node is sized as though it were the root, with its 40-byte btree_info
+ if cur and 56 + self.toc_len(len(cur) + 1, leaf) + kv + len(k) + len(v) > BS - 40:
+ nodes.append(cur)
+ cur, kv = [], 0
+ cur.append((k, v))
+ kv += len(k) + len(v)
+ nodes.append(cur)
+ return nodes
+
+ def node_count(self):
+ return sum(len(lv) for lv in self.levels)
+
+ def build(self, addrs, oids):
+ """Make the nodes. addrs: block address of every node, root first (top level down);
+ oids: their object ids (= addresses for a physical tree). Returns {address: block}."""
+ out = {}
+ flat = []
+ for lvl in range(len(self.levels) - 1, -1, -1):
+ for i in range(len(self.levels[lvl])):
+ flat.append((lvl, i))
+ where = {key: (addrs[n], oids[n]) for n, key in enumerate(flat)}
+ longest_k = max((len(k) for k, v in self.records), default=0)
+ longest_v = max((len(v) for k, v in self.records), default=0)
+ for (lvl, i), (addr, oid) in where.items():
+ recs = self.levels[lvl][i]
+ if lvl > 0:
+ # the child pointers: the children's ids (virtual) or addresses (physical)
+ first_child = sum(len(n) for n in self.levels[lvl - 1][:0])
+ children_before = sum(len(self.levels[lvl][j]) for j in range(i))
+ recs = [(k, struct.pack('<Q', where[(lvl - 1, children_before + c)][1]))
+ for c, (k, _) in enumerate(recs)]
+ is_root = (lvl == len(self.levels) - 1)
+ out[addr] = self._node(recs, lvl, is_root, oid, longest_k, longest_v)
+ return out
+
+ def _node(self, recs, level, is_root, oid, longest_k, longest_v):
+ block = bytearray(BS)
+ fixed = self.fixed is not None
+ leaf = level == 0
+ flags = (BTNODE_ROOT if is_root else 0) | (BTNODE_LEAF if leaf else 0) | (BTNODE_FIXED if fixed else 0)
+ val_end = BS - (40 if is_root else 0)
+ toc_len = self.toc_len(len(recs), leaf)
+ key_base = 56 + toc_len
+ koff = 0
+ voff = 0
+ for n, (k, v) in enumerate(recs):
+ block[key_base + koff:key_base + koff + len(k)] = k
+ voff += len(v)
+ block[val_end - voff:val_end - voff + len(v)] = v
+ if fixed:
+ struct.pack_into('<HH', block, 56 + 4 * n, koff, voff)
+ else:
+ struct.pack_into('<HHHH', block, 56 + 8 * n, koff, len(k), voff, len(v))
+ koff += len(k)
+ free = val_end - voff - (key_base + koff)
+ if free < 0:
+ raise RuntimeError('a b-tree node overflows')
+ struct.pack_into('<HHI', block, 32, flags, level, len(recs))
+ struct.pack_into('<HHHHHHHH', block, 40, 0, toc_len, koff, free, BTOFF_INVALID, 0, BTOFF_INVALID, 0)
+ if is_root:
+ ks, vs = self.fixed if fixed else (0, 0)
+ lk, lv = (ks, vs) if fixed else (longest_k, longest_v)
+ struct.pack_into('<IIIIIIQQ', block, BS - 40, self.tree_flags, BS, ks, vs, lk, lv,
+ len(self.records), self.node_count())
+ otype = (T_BTREE if is_root else T_BTNODE) | self.storage
+ seal(block, oid, otype, self.subtype)
+ return bytes(block)
+
+
+# ---------------------------------------------------------------------- volumes
+
+def jkey(oid, jtype):
+ return struct.pack('<Q', oid | (jtype << 60))
+
+
+def xfields(fields):
+ """An inode's (or a record's) extended fields: [(type, flags, data bytes)] in that order."""
+ head = struct.pack('<HH', len(fields), sum(div_up(len(d), 8) * 8 for _, _, d in fields))
+ descs = b''.join(struct.pack('<BBH', t, f, len(d)) for t, f, d in fields)
+ data = b''.join(d + b'\0' * (div_up(len(d), 8) * 8 - len(d)) for _, _, d in fields)
+ return head + descs + data
+
+
+class Volume:
+ def __init__(self, index, name, root, compress, when, spool=None):
+ self.spool = spool # where compressed resource forks wait to be written (a file)
+ self.index = index
+ self.name = name
+ self.root = root
+ self.compress = compress
+ self.when = when
+ self.streams = [] # (owning private id, bytes, is_rsrc) -- data to place
+ self.counts = {'files': 0, 'dirs': 0, 'symlinks': 0}
+ self.next_oid = MIN_USER_INO
+
+ def new_oid(self):
+ oid = self.next_oid
+ self.next_oid += 1
+ return oid
+
+ def collect(self):
+ """Inode numbers, contents (compressed or not) and the records that do not depend on
+ where the data lands; the extents are added once it is known (place_data)."""
+ self.items = [] # (node, parent oid)
+ self.root.oid = ROOT_DIR_INO
+
+ def walk(node, parent):
+ for name in sorted(node.children):
+ child = node.children[name]
+ child.oid = self.new_oid()
+ self.items.append((child, node.oid))
+ if stat.S_ISDIR(child.mode):
+ walk(child, child.oid)
+ walk(self.root, ROOT_DIR_INO)
+ # private id -> where its bytes are: ('file', path, size) or ('spool', offset, size); never in memory
+ self.data = {}
+ self.decmpfs = {} # file oid -> (header, rsrc stream id)
+ files = [node for node, parent in self.items if stat.S_ISREG(node.mode)]
+ for node in files:
+ node.size = os.path.getsize(node.source)
+ forks = iter(())
+ if self.compress:
+ # compressed by as many processes as there are processors, taken in the files' order
+ pool = concurrent.futures.ProcessPoolExecutor()
+ forks = pool.map(compressed_fork, [n.source for n in files if n.size > 0], chunksize=16)
+ for node in files:
+ if node.size == 0:
+ continue
+ if self.compress:
+ rsrc = next(forks)
+ if rsrc is not None:
+ sid = self.new_oid()
+ self.spool.seek(0, 2)
+ self.data[sid] = ('spool', self.spool.tell(), len(rsrc))
+ self.spool.write(rsrc)
+ self.decmpfs[node.oid] = (struct.pack('<IIQ', DECMPFS_MAGIC, DECMPFS_ZLIB_RSRC, node.size), sid)
+ continue
+ self.data[node.oid] = ('file', node.source, node.size)
+ if self.compress:
+ pool.shutdown()
+
+ def place(self, first_block):
+ """Data in order of private id, each stream in one extent. Returns the next free block."""
+ self.extents = {}
+ at = first_block
+ for sid in sorted(self.data):
+ n = div_up(self.data[sid][2], BS)
+ self.extents[sid] = (at, n)
+ at += n
+ self.data_blocks = at - first_block
+ return at
+
+ def fs_records(self):
+ """Every record of the file-system tree, in key order."""
+ recs = []
+ self.counts = {'files': 0, 'dirs': 0, 'symlinks': 0}
+ t = self.when
+ children = {ROOT_DIR_INO: 0}
+ for node, parent in self.items:
+ children[parent] = children.get(parent, 0) + 1
+
+ def inode(oid, parent, node_mode, uid, gid, name, nchild_or_link, internal, bsd_flags=0,
+ dstream=None, uncompressed=0):
+ fields = [(INO_EXT_TYPE_NAME, XF_DO_NOT_COPY, name.encode('utf-8', 'surrogateescape') + b'\0')]
+ if dstream is not None:
+ fields.append((INO_EXT_TYPE_DSTREAM, XF_SYSTEM_FIELD, dstream))
+ val = struct.pack('<QQQQQQQiIIIIIHHQ', parent, oid, t, t, t, t, internal, nchild_or_link, 0, 0,
+ bsd_flags, uid, gid, node_mode, 0, uncompressed) + xfields(fields)
+ return (jkey(oid, J_INODE), val)
+
+ def dstream(size, blocks):
+ return struct.pack('<QQQQQ', size, blocks * BS, 0, 0, 0)
+
+ def drec(parent, name, oid, dtype):
+ enc = name.encode('utf-8', 'surrogateescape') + b'\0'
+ key = jkey(parent, J_DIR_REC) + struct.pack('<I', (name_hash(name) << 10) | len(enc)) + enc
+ return (key, struct.pack('<QQH', oid, t, dtype))
+
+ def xattr(oid, name, flags, data):
+ enc = name.encode() + b'\0'
+ return (jkey(oid, J_XATTR) + struct.pack('<H', len(enc)) + enc,
+ struct.pack('<HH', flags, len(data)) + data)
+
+ def extents(sid, own_stream=True):
+ """A stream's extent, and for a file's own stream its reference count; an attribute's
+ stream has none (apfsck: \"xattrs can't be cloned\")."""
+ start, n = self.extents[sid]
+ recs = [(jkey(sid, J_FILE_EXTENT) + struct.pack('<Q', 0), struct.pack('<QQQ', n * BS, start, 0))]
+ if own_stream:
+ recs.insert(0, (jkey(sid, J_DSTREAM_ID), struct.pack('<I', 1)))
+ return recs
+
+ by_oid = {}
+ # the two special directories, named by records of the root's parent
+ by_oid.setdefault(ROOT_DIR_PARENT, []).extend([
+ drec(ROOT_DIR_PARENT, 'private-dir', PRIV_DIR_INO, DT_DIR),
+ drec(ROOT_DIR_PARENT, 'root', ROOT_DIR_INO, DT_DIR)])
+ r = self.root
+ by_oid.setdefault(ROOT_DIR_INO, []).append(
+ inode(ROOT_DIR_INO, ROOT_DIR_PARENT, r.mode, r.uid, r.gid, 'root', children.get(ROOT_DIR_INO, 0),
+ INODE_NO_RSRC_FORK))
+ by_oid.setdefault(PRIV_DIR_INO, []).append(
+ inode(PRIV_DIR_INO, ROOT_DIR_PARENT, stat.S_IFDIR | 0o755, 0, 0, 'private-dir', 0, INODE_NO_RSRC_FORK))
+ for node, parent in self.items:
+ oid = node.oid
+ mine = by_oid.setdefault(oid, [])
+ if stat.S_ISDIR(node.mode):
+ self.counts['dirs'] += 1
+ mine.append(inode(oid, parent, node.mode, node.uid, node.gid, node.name, children.get(oid, 0),
+ INODE_NO_RSRC_FORK))
+ dtype = DT_DIR
+ elif stat.S_ISLNK(node.mode):
+ self.counts['symlinks'] += 1
+ mine.append(inode(oid, parent, node.mode, node.uid, node.gid, node.name, 1, INODE_NO_RSRC_FORK))
+ mine.append(xattr(oid, 'com.apple.fs.symlink', XATTR_DATA_EMBEDDED | XATTR_FS_OWNED,
+ node.target + b'\0'))
+ dtype = DT_LNK
+ else:
+ self.counts['files'] += 1
+ if oid in self.decmpfs:
+ header, sid = self.decmpfs[oid]
+ rsrc_len = self.data[sid][2]
+ mine.append(inode(oid, parent, node.mode, node.uid, node.gid, node.name, 1,
+ INODE_HAS_RSRC_FORK | INODE_HAS_UNCOMPRESSED_SIZE, UF_COMPRESSED,
+ uncompressed=node.size))
+ xd = struct.pack('<Q', sid) + dstream(rsrc_len, self.extents[sid][1])
+ mine.append(xattr(oid, 'com.apple.ResourceFork', XATTR_DATA_STREAM, xd))
+ mine.append(xattr(oid, 'com.apple.decmpfs', XATTR_DATA_EMBEDDED, header))
+ by_oid.setdefault(sid, []).extend(extents(sid, own_stream=False))
+ elif oid in self.data:
+ mine.append(inode(oid, parent, node.mode, node.uid, node.gid, node.name, 1, INODE_NO_RSRC_FORK,
+ dstream=dstream(node.size, self.extents[oid][1])))
+ mine.extend(extents(oid))
+ else:
+ mine.append(inode(oid, parent, node.mode, node.uid, node.gid, node.name, 1, INODE_NO_RSRC_FORK))
+ dtype = DT_REG
+ by_oid.setdefault(parent, []).append(drec(parent, node.name, oid, dtype))
+ for oid in sorted(by_oid):
+ recs.extend(sorted(by_oid[oid], key=self.sort_key))
+ return recs
+
+ @staticmethod
+ def sort_key(rec):
+ k = rec[0]
+ jtype = struct.unpack_from('<Q', k)[0] >> 60
+ if jtype == J_DIR_REC:
+ return (jtype, struct.unpack_from('<I', k, 8)[0] & ~0x3ff, k[12:])
+ if jtype == J_XATTR:
+ return (jtype, 0, k[10:])
+ if jtype == J_FILE_EXTENT:
+ return (jtype, struct.unpack_from('<Q', k, 8)[0], b'')
+ return (jtype, 0, b'')
+
+ def extentref_records(self):
+ recs = []
+ for sid in sorted(self.extents, key=lambda s: self.extents[s][0]):
+ start, n = self.extents[sid]
+ recs.append((jkey(start, J_EXTENT), struct.pack('<QQI', n | (KIND_NEW << 60), sid, 1)))
+ return recs
+
+
+def compressed_fork(path):
+ """A file's resource fork compressed, or None when that would not take fewer blocks."""
+ content = open(path, 'rb').read()
+ rsrc = zlib_rsrc(content)
+ return rsrc if div_up(len(rsrc), BS) < div_up(len(content), BS) else None
+
+
+def zlib_rsrc(content):
+ """A resource fork as decmpfs keeps zlib pieces in it (type 4): an HFS resource fork whose one
+ resource is a table of the 64 KB pieces, each zlib (or stored, after a 0xff)."""
+ pieces = []
+ for at in range(0, len(content), PIECE):
+ chunk = content[at:at + PIECE]
+ z = zlib.compress(chunk, 9)
+ pieces.append(z if len(z) < len(chunk) else b'\xff' + chunk)
+ count = len(pieces)
+ table_len = 4 + 8 * count
+ body = bytearray(struct.pack('<I', count))
+ off = table_len
+ for p in pieces:
+ body += struct.pack('<II', off, len(p))
+ off += len(p)
+ for p in pieces:
+ body += p
+ data_off = 0x100
+ data = struct.pack('>I', len(body)) + bytes(body)
+ # the resource map: Apple's own for these forks (the 'cmpf' resource, id 1, named nothing)
+ rmap = bytearray(50)
+ rmap_off = data_off + len(data)
+ struct.pack_into('>IIII', rmap, 0, data_off, rmap_off, len(data), 50)
+ struct.pack_into('>HH', rmap, 24, 28, 50) # type list at 28, name list at 50 (none)
+ struct.pack_into('>H', rmap, 28, 0) # one type (count - 1)
+ rmap[30:34] = b'cmpf'
+ struct.pack_into('>HH', rmap, 34, 0, 10) # one resource of it, its list at 10 from the type list
+ struct.pack_into('>hHI', rmap, 38, 1, 0xffff, 0) # id 1, no name, data at 0 (attrs 0)
+ head = struct.pack('>IIII', data_off, rmap_off, len(data), 50)
+ out = head + b'\0' * (data_off - len(head)) + data + bytes(rmap)
+ return out
+
+
+# ---------------------------------------------------------------------- the container
+
+def cpoint_desc_blocks(block_count):
+ if block_count < 512 * 1024 // 4:
+ return 8
+ if block_count < 1024 * 1024 // 4:
+ return 12
+ if block_count < 50 * 1024 * 1024 // 4:
+ return 20 + 60 * ((block_count - 1024 * 1024 // 4) // (512 * 1024 // 4)) // 23
+ return 280
+
+
+def cpoint_data_blocks(block_count):
+ b = block_count
+ if b < 4545:
+ return 52
+ if b < 13633:
+ return 124
+ if b < 36353:
+ return 160 + 36 * ((b - 13633) // 4544)
+ if b < 131777:
+ return 308 + 4 * ((b - 36353) // 4544)
+ if b < 262144:
+ return 648 + 4 * ((b - 131777) // 4544)
+ if b == 262144:
+ return 992
+ if b < 1048576:
+ off = (b - 262144) // 131072
+ return 1248 + 488 * off + 4 * ((b - (261280 + off * 131776)) // 2272)
+ if b < 4063232:
+ return 4112 + 256 * ((b - 1048576) // 131072)
+ if b < 13107200:
+ return 10000 + 256 * ((b - 4063232) // 131072)
+ return 27672
+
+
+def main_fq_node_limit(blocks):
+ if blocks < 0x40000:
+ r = 1 + (blocks - 1) // 4544
+ elif blocks < 0x100000:
+ r = 116 + (blocks - 261281) // 2272
+ else:
+ r = 512
+ return 3 if r == 2 else r
+
+
+def ip_fq_node_limit(chunks):
+ r = 3 * (chunks + 751) // 1127 - 1
+ return 3 if r == 2 else r
+
+
+class Container:
+ def __init__(self, volumes, slack, uuid_seed):
+ self.vols = volumes
+ self.slack = slack
+ self.seed = uuid_seed
+
+ def uuid(self, what):
+ return hashlib.sha256((self.seed + what).encode()).digest()[:16]
+
+ def layout(self):
+ """Decide every block: done twice, since the container's size decides the sizes of the
+ checkpoint areas and of the space manager, which come before everything else."""
+ for v in self.vols:
+ v.collect()
+ guess = 1024
+ while True:
+ total = self._layout(guess)
+ if total + self.slack <= guess:
+ break
+ guess = total + self.slack
+ self.block_count = guess
+
+ def _layout(self, block_count):
+ self.bc = block_count
+ bpc = BS * 8 # blocks per chunk: one bitmap block each
+ self.chunks = div_up(block_count, bpc)
+ self.chunks_per_cib = (BS - 40) // 32
+ self.cibs = div_up(self.chunks, self.chunks_per_cib)
+ self.cibs_per_cab = (BS - 40) // 8
+ cabs = div_up(self.cibs, self.cibs_per_cab)
+ self.cabs = 0 if cabs == 1 else cabs
+ self.ip_blocks = (self.chunks + self.cibs + self.cabs) * 3
+ self.ip_bm_size = div_up(self.ip_blocks, bpc)
+ self.ip_bmap_blocks = 16 * self.ip_bm_size
+ self.bm_addr_off = SM_BITMAP_XID_OFF + 8 * self.ip_bm_size
+ self.bm_free_next_off = self.bm_addr_off + div_up(2 * self.ip_bm_size, 8) * 8
+ self.cib_addr_off = self.bm_free_next_off + self.ip_bmap_blocks * 2
+ entries = self.cabs if self.cabs > 1 else self.cibs
+ self.sm_size = div_up(entries * 8 + self.cib_addr_off, BS) * BS
+ self.sm_blocks = self.sm_size // BS
+
+ self.desc_base = 1
+ self.desc_blocks = cpoint_desc_blocks(block_count)
+ self.data_base = self.desc_base + self.desc_blocks
+ self.data_blocks = cpoint_data_blocks(block_count)
+ self.reaper_bno = self.data_base
+ self.sm_bno = self.reaper_bno + 1
+ self.ipfq_bno = self.sm_bno + self.sm_blocks
+ self.mainfq_bno = self.ipfq_bno + 1
+ self.eph_blocks = self.mainfq_bno - self.reaper_bno + 1
+ at = self.data_base + self.data_blocks
+ self.nx_omap_bno, self.nx_omap_root = at, at + 1
+ at += 2
+ # object ids: the ephemeral objects, then each volume's superblock and tree nodes
+ oid = OID_RESERVED
+ self.sm_oid, self.reaper_oid, self.ipfq_oid, self.mainfq_oid = oid, oid + 1, oid + 2, oid + 3
+ oid += 4
+ for v in self.vols:
+ v.oid = oid
+ oid += 1
+ v.sb_bno = at
+ v.omap_bno = at + 1
+ at += 2
+ # the trees: their shapes do not depend on the data's addresses (fixed-size fields)
+ v.place(0)
+ v.fs_tree = Tree(T_FSTREE, VIRTUAL, tree_flags=BTREE_KV_NONALIGNED)
+ n_fs = v.fs_tree.plan(v.fs_records())
+ v.fs_oids = list(range(oid, oid + n_fs))
+ oid += n_fs
+ v.omap_tree = Tree(T_OMAP, PHYSICAL, fixed=(16, 16), tree_flags=BTREE_PHYSICAL)
+ n_om = v.omap_tree.plan([(struct.pack('<QQ', o, XID), b'\0' * 16) for o in v.fs_oids])
+ v.omap_addrs = list(range(at, at + n_om))
+ at += n_om
+ v.fs_addrs = list(range(at, at + n_fs))
+ at += n_fs
+ v.er_tree = Tree(T_BLOCKREF, PHYSICAL, tree_flags=BTREE_PHYSICAL | BTREE_KV_NONALIGNED)
+ n_er = v.er_tree.plan(v.extentref_records())
+ v.er_addrs = list(range(at, at + n_er))
+ at += n_er
+ v.snap_bno = at
+ at += 1
+ self.next_oid = oid
+ self.ip_bm_base = at
+ at += self.ip_bmap_blocks
+ self.ip_base = at
+ at += self.ip_blocks
+ for v in self.vols:
+ at = v.place(at)
+ # the extents are known now: the records again, with them (same sizes, same shape)
+ v.fs_tree.plan(v.fs_records())
+ v.er_tree.plan(v.extentref_records())
+ self.used_end = at
+ return at
+
+ def write(self, path):
+ bc = self.bc
+ blocks = {}
+ used = bytearray(div_up(bc, 8))
+
+ def mark(start, n=1):
+ b, end = start, start + n
+ while b < end and b & 7:
+ used[b >> 3] |= 1 << (b & 7)
+ b += 1
+ if end - b >= 8:
+ used[b >> 3:(end >> 3)] = b'\xff' * ((end >> 3) - (b >> 3))
+ b = end & ~7
+ while b < end:
+ used[b >> 3] |= 1 << (b & 7)
+ b += 1
+
+ def put(bno, data):
+ assert len(data) % BS == 0
+ blocks[bno] = data
+
+ bpc = BS * 8
+ used_chunks = div_up(self.used_end, bpc)
+ chunk_bmap_base = self.ip_base
+ cib_base = chunk_bmap_base + used_chunks
+ cab_base = cib_base + self.cibs
+
+ # --- per volume: trees, object map, superblock
+ for v in self.vols:
+ fs_blocks = v.fs_tree.build(v.fs_addrs, v.fs_oids)
+ for b, d in fs_blocks.items():
+ put(b, d)
+ omap_recs = [(struct.pack('<QQ', o, XID), struct.pack('<IIQ', 0, BS, a))
+ for o, a in zip(v.fs_oids, v.fs_addrs)]
+ v.omap_tree.plan(omap_recs)
+ for b, d in v.omap_tree.build(v.omap_addrs, v.omap_addrs).items():
+ put(b, d)
+ omap = bytearray(BS)
+ struct.pack_into('<IIIIQ', omap, 32, 0, 0, PHYSICAL | T_BTREE, PHYSICAL | T_BTREE, v.omap_addrs[0])
+ seal(omap, v.omap_bno, PHYSICAL | T_OMAP)
+ put(v.omap_bno, bytes(omap))
+ for b, d in v.er_tree.build(v.er_addrs, v.er_addrs).items():
+ put(b, d)
+ snap = Tree(T_SNAPMETA, PHYSICAL, tree_flags=BTREE_PHYSICAL | BTREE_KV_NONALIGNED)
+ snap.plan([])
+ for b, d in snap.build([v.snap_bno], [v.snap_bno]).items():
+ put(b, d)
+ alloc = 1 + len(v.omap_addrs) + len(v.fs_addrs) + len(v.er_addrs) + 1 + v.data_blocks
+ sb = bytearray(BS)
+ struct.pack_into('<4sI', sb, 32, b'APSB', v.index)
+ struct.pack_into('<QQQ', sb, 40, 0x2, 0, 0x8) # hard-link map records; normalization-insensitive
+ struct.pack_into('<Q', sb, 0x58, alloc)
+ struct.pack_into('<HHIIIHH', sb, 0x60, 5, 0, 0, 6, 0, 1, 0) # meta crypto: v5.0, class F, revision 1
+ struct.pack_into('<III', sb, 0x74, VIRTUAL | T_BTREE, PHYSICAL | T_BTREE, PHYSICAL | T_BTREE)
+ struct.pack_into('<QQQQ', sb, 0x80, v.omap_bno, v.fs_oids[0], v.er_addrs[0], v.snap_bno)
+ struct.pack_into('<Q', sb, 0xb0, v.next_oid)
+ struct.pack_into('<QQQQQ', sb, 0xb8, v.counts['files'], v.counts['dirs'], v.counts['symlinks'], 0, 0)
+ struct.pack_into('<QQ', sb, 0xe0, alloc, 0)
+ sb[0xf0:0x100] = self.uuid('volume %d' % v.index)
+ struct.pack_into('<QQ', sb, 0x100, v.when, 1) # last modified; unencrypted
+ fmt = b'mkapfs.py (darwin-redwood)'
+ sb[0x110:0x110 + len(fmt)] = fmt
+ struct.pack_into('<QQ', sb, 0x130, v.when, XID)
+ nm = v.name.encode()[:255]
+ sb[0x2c0:0x2c0 + len(nm)] = nm
+ struct.pack_into('<I', sb, 0x3c0, 3) # next document id
+ seal(sb, v.oid, VIRTUAL | T_FS)
+ put(v.sb_bno, bytes(sb))
+ for b in [v.sb_bno, v.omap_bno] + v.omap_addrs + v.fs_addrs + v.er_addrs + [v.snap_bno]:
+ mark(b)
+ for start, n in v.extents.values():
+ mark(start, n)
+
+ # --- the container's object map: the volumes' superblocks
+ nx_omap = Tree(T_OMAP, PHYSICAL, fixed=(16, 16), tree_flags=BTREE_PHYSICAL)
+ nx_omap.plan([(struct.pack('<QQ', v.oid, XID), struct.pack('<IIQ', 0, BS, v.sb_bno)) for v in self.vols])
+ assert nx_omap.node_count() == 1
+ for b, d in nx_omap.build([self.nx_omap_root], [self.nx_omap_root]).items():
+ put(b, d)
+ omap = bytearray(BS)
+ struct.pack_into('<IIIIQ', omap, 32, 1, 0, PHYSICAL | T_BTREE, PHYSICAL | T_BTREE, self.nx_omap_root)
+ seal(omap, self.nx_omap_bno, PHYSICAL | T_OMAP)
+ put(self.nx_omap_bno, bytes(omap))
+
+ # --- what else is in use: block 0, the checkpoint areas, the internal pool and its bitmaps
+ mark(0)
+ mark(self.desc_base, self.desc_blocks)
+ mark(self.data_base, self.data_blocks)
+ mark(self.nx_omap_bno, 2)
+ mark(self.ip_bm_base, self.ip_bmap_blocks)
+ mark(self.ip_base, self.ip_blocks)
+
+ # --- the space manager: chunk bitmaps, chunk-info blocks, the internal pool's own bitmap
+ free_total = 0
+ cib_blocks = []
+ chunk = 0
+ for c in range(self.cibs):
+ cib = bytearray(BS)
+ n = min(self.chunks_per_cib, self.chunks - chunk)
+ struct.pack_into('<II', cib, 32, c, n)
+ for i in range(n):
+ start = chunk * bpc
+ count = min(bpc, bc - start)
+ bm = bytes(used[start // 8:(start + count + 7) // 8])
+ in_use = sum(bin(x).count('1') for x in bm)
+ bmap_addr = 0
+ if chunk < used_chunks:
+ bmap_addr = chunk_bmap_base + chunk
+ put(bmap_addr, bm + b'\0' * (BS - len(bm)))
+ struct.pack_into('<QQIIQ', cib, 40 + 32 * i, XID, start, count, count - in_use, bmap_addr)
+ free_total += count - in_use
+ chunk += 1
+ seal(cib, cib_base + c, PHYSICAL | T_CIB)
+ cib_blocks.append(bytes(cib))
+ put(cib_base + c, bytes(cib))
+ for c in range(self.cabs):
+ cab = bytearray(BS)
+ n = min(self.cibs_per_cab, self.cibs - c * self.cibs_per_cab)
+ struct.pack_into('<II', cab, 32, c, n)
+ for i in range(n):
+ struct.pack_into('<Q', cab, 40 + 8 * i, cib_base + c * self.cibs_per_cab + i)
+ seal(cab, cab_base + c, PHYSICAL | T_CAB)
+ put(cab_base + c, bytes(cab))
+ ip_bm = bytearray(self.ip_bm_size * BS)
+ for b in list(range(chunk_bmap_base, chunk_bmap_base + used_chunks)) + \
+ list(range(cib_base, cib_base + self.cibs)) + list(range(cab_base, cab_base + self.cabs)):
+ r = b - self.ip_base
+ ip_bm[r >> 3] |= 1 << (r & 7)
+ put(self.ip_bm_base, bytes(ip_bm))
+ for i in range(self.ip_bm_size, self.ip_bmap_blocks):
+ put(self.ip_bm_base + i, bytes(BS))
+
+ sm = bytearray(self.sm_size)
+ struct.pack_into('<IIII', sm, 32, BS, bpc, self.chunks_per_cib, self.cibs_per_cab)
+ struct.pack_into('<QQIIQIIQ', sm, 48, bc, self.chunks, self.cibs, self.cabs, free_total,
+ self.cib_addr_off, 0, 0)
+ struct.pack_into('<IIQIIQQQQ', sm, 144, 0, IP_BM_TX_MULTIPLIER, self.ip_blocks, self.ip_bm_size,
+ self.ip_bmap_blocks, self.ip_bm_base, self.ip_base, 0, 0)
+ struct.pack_into('<QQQHHIQ', sm, 200, 0, self.ipfq_oid, 0, ip_fq_node_limit(self.chunks), 0, 0, 0)
+ struct.pack_into('<QQQHHIQ', sm, 240, 0, self.mainfq_oid, 0, main_fq_node_limit(bc), 0, 0, 0)
+ struct.pack_into('<HHIII', sm, 320, self.ip_bm_size, self.ip_bmap_blocks - 1, SM_BITMAP_XID_OFF,
+ self.bm_addr_off, self.bm_free_next_off)
+ for i in range(self.ip_bm_size):
+ struct.pack_into('<Q', sm, SM_BITMAP_XID_OFF + 8 * i, XID)
+ struct.pack_into('<H', sm, self.bm_addr_off + 2 * i, i)
+ for i in range(self.ip_bmap_blocks):
+ if i < self.ip_bm_size or i == self.ip_bmap_blocks - 1:
+ nxt = IP_BM_INDEX_INVALID
+ else:
+ nxt = i + 1
+ struct.pack_into('<H', sm, self.bm_free_next_off + 2 * i, nxt)
+ if self.cabs:
+ for i in range(self.cabs):
+ struct.pack_into('<Q', sm, self.cib_addr_off + 8 * i, cab_base + i)
+ else:
+ for i in range(self.cibs):
+ struct.pack_into('<Q', sm, self.cib_addr_off + 8 * i, cib_base + i)
+ seal(sm, self.sm_oid, EPHEMERAL | T_SPACEMAN)
+ put(self.sm_bno, bytes(sm))
+
+ # --- the reaper and the two free queues (empty)
+ reaper = bytearray(BS)
+ struct.pack_into('<Q', reaper, 32, 1)
+ struct.pack_into('<I', reaper, 64, 1) # nr_flags: BHM
+ struct.pack_into('<I', reaper, 108, BS - 112) # nr_state_buffer_size
+ seal(reaper, self.reaper_oid, EPHEMERAL | T_REAPER)
+ put(self.reaper_bno, bytes(reaper))
+ for bno, oid in ((self.ipfq_bno, self.ipfq_oid), (self.mainfq_bno, self.mainfq_oid)):
+ fq = bytearray(BS)
+ toc = ((BS - 56) // (16 + 8 + 4)) * 4
+ struct.pack_into('<HHI', fq, 32, BTNODE_ROOT | BTNODE_LEAF | BTNODE_FIXED, 0, 0)
+ struct.pack_into('<HHHHHHHH', fq, 40, 0, toc, 0, BS - 56 - toc - 40, BTOFF_INVALID, 0, BTOFF_INVALID, 0)
+ struct.pack_into('<IIIIIIQQ', fq, BS - 40, BTREE_EPHEMERAL | BTREE_ALLOW_GHOSTS, BS, 16, 8, 16, 8, 0, 1)
+ seal(fq, oid, EPHEMERAL | T_BTREE, T_FREE_QUEUE)
+ put(bno, bytes(fq))
+
+ # --- the container superblock, the checkpoint map, the superblock's copy
+ nx = bytearray(BS)
+ struct.pack_into('<4sIQQQQ', nx, 32, b'NXSB', BS, bc, 0, 0, 0x2) # incompatible: version 2
+ nx[72:88] = self.uuid('container')
+ struct.pack_into('<QQ', nx, 88, self.next_oid, XID + 1)
+ struct.pack_into('<IIQQIIIIII', nx, 104, self.desc_blocks, self.data_blocks, self.desc_base, self.data_base,
+ 2, self.eph_blocks, 0, 2, 0, self.eph_blocks)
+ struct.pack_into('<QQQII', nx, 152, self.sm_oid, self.nx_omap_bno, self.reaper_oid, 0,
+ min(100, div_up(bc * BS, 512 * 1024 * 1024)))
+ for i, v in enumerate(self.vols):
+ struct.pack_into('<Q', nx, 184 + 8 * i, v.oid)
+ min_blocks = main_fq_node_limit(bc) if bc * BS < 128 * 1024 * 1024 else 8
+ struct.pack_into('<Q', nx, 1312, (min_blocks << 32) | (4 << 16) | 1)
+ seal(nx, OID_NX, EPHEMERAL | T_NX)
+ put(0, bytes(nx))
+ cpm = bytearray(BS)
+ maps = [(EPHEMERAL | T_REAPER, 0, BS, self.reaper_oid, self.reaper_bno),
+ (EPHEMERAL | T_SPACEMAN, 0, self.sm_size, self.sm_oid, self.sm_bno),
+ (EPHEMERAL | T_BTREE, T_FREE_QUEUE, BS, self.ipfq_oid, self.ipfq_bno),
+ (EPHEMERAL | T_BTREE, T_FREE_QUEUE, BS, self.mainfq_oid, self.mainfq_bno)]
+ struct.pack_into('<II', cpm, 32, 1, len(maps)) # the last map of the checkpoint
+ for i, (t, st, size, oid, paddr) in enumerate(maps):
+ struct.pack_into('<IIIIQQQ', cpm, 40 + 40 * i, t, st, size, 0, 0, oid, paddr)
+ seal(cpm, self.desc_base, PHYSICAL | T_CPMAP)
+ put(self.desc_base, bytes(cpm))
+ put(self.desc_base + 1, bytes(nx))
+
+ # --- out, in order (holes stay holes in a sparse file); the files' data straight from where it is
+ with open(path, 'wb') as f:
+ f.truncate(bc * BS)
+ for bno in sorted(blocks):
+ f.seek(bno * BS)
+ f.write(blocks[bno])
+ for v in self.vols:
+ for sid, (start, n) in sorted(v.extents.items(), key=lambda e: e[1][0]):
+ kind, where, size = v.data[sid]
+ f.seek(start * BS)
+ if kind == 'file':
+ with open(where, 'rb') as src:
+ left = size
+ while left > 0:
+ chunk = src.read(min(left, 1 << 22))
+ if not chunk:
+ raise RuntimeError('%s is shorter than it was' % where)
+ f.write(chunk)
+ left -= len(chunk)
+ else:
+ v.spool.seek(where)
+ f.write(v.spool.read(size))
+ return free_total
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
+ ap.add_argument('-o', '--output', required=True)
+ ap.add_argument('-z', '--compress', action='store_true', help='zlib in the resource fork where it saves space')
+ ap.add_argument('--time', type=int, default=1683900000, help='every timestamp, in seconds')
+ ap.add_argument('--slack', type=int, default=256, help='free blocks to leave (at least)')
+ ap.add_argument('volumes', nargs='+', help='NAME=DIR[,manifest=FILE]')
+ args = ap.parse_args()
+ vols = []
+ spool = open(args.output + '.spool', 'w+b') if args.compress else None
+ for i, spec in enumerate(args.volumes):
+ name, _, rest = spec.partition('=')
+ parts = rest.split(',')
+ root = read_tree(parts[0])
+ for opt in parts[1:]:
+ k, _, val = opt.partition('=')
+ if k == 'manifest':
+ apply_manifest(root, val)
+ vols.append(Volume(i, name, root, args.compress, args.time * 1000000000, spool))
+ c = Container(vols, args.slack, os.path.basename(args.output))
+ c.layout()
+ free = c.write(args.output)
+ if spool is not None:
+ spool.close()
+ os.unlink(args.output + '.spool')
+ for v in vols:
+ print('mkapfs.py: volume %d "%s": %d files, %d directories, %d symbolic links, %d blocks of data%s' % (
+ v.index, v.name, v.counts['files'], v.counts['dirs'], v.counts['symlinks'], v.data_blocks,
+ ' (%d compressed)' % len(v.decmpfs) if v.decmpfs else ''))
+ print('mkapfs.py: %s: %d blocks of %d bytes, %d free' % (args.output, c.block_count, BS, free))
+
+
+if __name__ == '__main__':
+ main()