Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
ramfs.c comes from sileo2 whole; the half-fix written here is dropped
vcvkk20208f044991
diff --git a/README.md b/README.mdindex 5b335a5..5be7a00 100644--- a/README.md+++ b/README.md@@ -1185,14 +1185,14 @@ XL_CMDLINE="root_apfs=ufs_darwin ufs=2 ufs_native=1 ufs_write=1 var_store=1 bootпростоя не печёт ничего — в `/var/mobile/Library` нет ни `PaperBoard`, ни `PosterBoard`, ни `Wallpaper`, — такчто хранить ещё нечего (проверено в другой ветке; здесь это не считается проверкой этой работы).-**Та же беда с другой стороны.** В ветке `sileo2` её нашли независимо и глубже: все зоны ядра делят один-потолок, и несколько сотен мегабайт файлов в `data.kalloc.16384` в него упёрлись — сперва `ENOSPC` на записи, а-потом паника на выделении, которому нельзя не удаться. Там страницы содержимого взяты не из зоны, а прямо у VM-(`ramfs_page_alloc`, `kmem_alloc` в `kernel_map`), и массив страниц тоже выращивается без замка-(`ramfs_reserve_slots`: вырастить до взятия замка, под замком перепроверить). Это покрывает и то, что здесь, и-причину, которой здесь нет. При слиянии `ramfs.c` надо брать оттуда целиком, а здешнюю правку-(`ramfs_grow_slots` с `EAGAIN` и массивом от зовущего) — убирать: она делает ровно половину того же. Ничего-другого здешнего это не задевает — пояс в `var_store` остаётся полезным и так.+**Та же беда с другой стороны, и чей `ramfs.c` здесь теперь.** В ветке `sileo2` её нашли независимо и глубже:+все зоны ядра делят один потолок, и несколько сотен мегабайт файлов в `data.kalloc.16384` в него упёрлись —+сперва `ENOSPC` на записи, а потом паника на выделении, которому нельзя не удаться. Там страницы содержимого+берутся не из зоны, а прямо у VM (`ramfs_page_alloc`, `kmem_alloc` в `kernel_map`), и массив страниц тоже+выращивается без замка (`ramfs_reserve_slots`: вырастить до взятия замка, под замком перепроверить). Это+покрывает и то, что чинила здешняя правка, и причину, которой здешняя не знала, — поэтому `ramfs.c` здесь+теперь оттуда целиком, а здешнее (`ramfs_grow_slots` с `EAGAIN` и массивом от зовущего) убрано. Пояс в+`var_store` — выжидание короткой записи — остался: путь подкачки по-прежнему не может ждать.**Отказ не по месту, а по странице.** В прогоне 2 восемь файлов из 2401 не поместились (`No space left ondevice`, около 2,6 МБ), хотя место было: `vol_ls fs /private/var` показывает 68 608 блоков по 16 КБ — 1,07 ГБ,diff --git a/xnu/files/bsd/miscfs/ramfs/ramfs.c b/xnu/files/bsd/miscfs/ramfs/ramfs.cindex 8776736..e3a4949 100644--- a/xnu/files/bsd/miscfs/ramfs/ramfs.c+++ b/xnu/files/bsd/miscfs/ramfs/ramfs.c@@ -45,6 +45,7 @@#include <sys/xattr.h>#include <uuid/uuid.h>#include <vfs/vfs_support.h>+#include <vm/vm_kern.h>#define RAMFS_MAX_NAME 255#define RAMFS_MAX_XATTR (16u * 1024 * 1024)@@ -98,6 +99,9 @@ struct ramfs_node {/* symlink */char *rn_target;+ struct ramfs_node *rn_hnext; /* the next node in rm_hash's chain */+ bool rn_hashed;+TAILQ_HEAD(, ramfs_xattr) rn_xattrs;};#define RAMFS_N_CREATING 0x1 /* a thread is in vnode_create() for this node */@@ -111,8 +115,11 @@ struct ramfs_mount {uint64_t rm_nodes;uint64_t rm_pages; /* file pages in use */uint64_t rm_max_pages;+ struct ramfs_node **rm_hash; /* every node by its number, for vfs_vget */};+#define RAMFS_HASH_SIZE 4096u /* chains, a power of two */+/* file pages in use on all the mounts together (each mount has its own lock): for the kernel's memory report */_Atomic uint64_t ramfs_pages_in_use;@@ -178,13 +185,80 @@ ramfs_node_alloc(struct ramfs_mount *rm, enum vtype type, struct vnode_attr *vapreturn node;}+/*+ * A page of a file's contents, and giving it back. Not from a kalloc zone, as it once was: all the+ * kernel's zones share one ceiling, and a few hundred MB of files in data.kalloc.16384 met it --+ * first writes failing with ENOSPC ("there isn't enough space" while most of this filesystem's own+ * allowance was free: dpkg's database could not be written), then, on an allocation that may not+ * fail, the kernel panicked with the zone map exhausted. Straight from the VM instead, a wired page+ * of kernel_map each, as the contents of a RAM disk are. wait=false: never wait for memory (the+ * page-out path, which may be the pageout daemon looking for some).+ */+static void *+ramfs_page_alloc(bool wait)+{+ vm_offset_t addr = 0;+ kma_flags_t flags = KMA_KOBJECT | KMA_DATA | KMA_ZERO | (wait ? KMA_NONE : KMA_NOPAGEWAIT);++ return kmem_alloc(kernel_map, &addr, PAGE_SIZE, flags, VM_KERN_MEMORY_FILE) == KERN_SUCCESS ?+ (void *)addr : NULL;+}++static void+ramfs_page_free(void *page)+{+ kmem_free(kernel_map, (vm_offset_t)page, PAGE_SIZE);+}++/*+ * Make the page array big enough for page `index`, before the mount lock is taken: under it the+ * array can only grow by an allocation that does not wait (ramfs_grow_slots), and one that failed+ * under memory pressure failed the write with ENOSPC. Nothing is promised: ramfs_store looks again.+ */+static void+ramfs_reserve_slots(struct ramfs_mount *rm, struct ramfs_node *node, uint64_t index)+{+ for (int tries = 0; tries < 4; tries++) {+ uint64_t have, want;+ void **slots;++ lck_mtx_lock(&rm->rm_lock);+ have = node->rn_nslots;+ lck_mtx_unlock(&rm->rm_lock);+ if (index < have) {+ return;+ }+ for (want = have ? have : 4; want <= index; want *= 2) {+ }+ slots = kalloc_type(void *, want, Z_WAITOK | Z_ZERO);+ if (slots == NULL) {+ return;+ }+ lck_mtx_lock(&rm->rm_lock);+ if (node->rn_nslots == have) {+ if (node->rn_pages != NULL) {+ memcpy(slots, node->rn_pages, have * sizeof(void *));+ kfree_type(void *, have, node->rn_pages);+ }+ node->rn_pages = slots;+ node->rn_nslots = want;+ slots = NULL;+ }+ lck_mtx_unlock(&rm->rm_lock);+ if (slots == NULL) {+ return;+ }+ kfree_type(void *, want, slots); /* it grew meanwhile: look again */+ }+}+/* Drop file pages from slot `from` on. The mount lock is held. */static voidramfs_free_pages(struct ramfs_node *node, uint64_t from){for (uint64_t i = from; i < node->rn_nslots; i++) {if (node->rn_pages[i] != NULL) {- kfree_data(node->rn_pages[i], PAGE_SIZE);+ ramfs_page_free(node->rn_pages[i]);node->rn_pages[i] = NULL;node->rn_nalloc--;node->rn_mount->rm_pages--;@@ -203,12 +277,47 @@ ramfs_xattr_free(struct ramfs_xattr *xa)kfree_type(struct ramfs_xattr, xa);}+/*+ * Nodes by their number. What asks for a file by number rather than by name -- fsgetpath(), and through+ * it LaunchServices, which keeps an application as its volume and file number and finds the path again+ * from them -- comes to vfs_vget, which this filesystem did not have: an application in /private/var+ * (the package manager's prefix, /var/jb/Applications) could not be registered at all ("Unable to+ * register"), while one on an APFS volume could. The mount lock is held.+ */+static void+ramfs_hash_insert(struct ramfs_mount *rm, struct ramfs_node *node)+{+ struct ramfs_node **chain = &rm->rm_hash[node->rn_ino & (RAMFS_HASH_SIZE - 1)];++ node->rn_hnext = *chain;+ *chain = node;+ node->rn_hashed = true;+}++static void+ramfs_hash_remove(struct ramfs_mount *rm, struct ramfs_node *node)+{+ struct ramfs_node **link = &rm->rm_hash[node->rn_ino & (RAMFS_HASH_SIZE - 1)];++ while (*link != NULL && *link != node) {+ link = &(*link)->rn_hnext;+ }+ if (*link == node) {+ *link = node->rn_hnext;+ }+ node->rn_hashed = false;+}+/* The node is named by nothing and has no vnode. The mount lock is held. */static voidramfs_node_free(struct ramfs_node *node){struct ramfs_xattr *xa;+ if (node->rn_hashed) {+ ramfs_hash_remove(node->rn_mount, node);+ }+assert(TAILQ_EMPTY(&node->rn_entries));while ((xa = TAILQ_FIRST(&node->rn_xattrs)) != NULL) {TAILQ_REMOVE(&node->rn_xattrs, xa, rx_link);@@ -358,70 +467,43 @@ ramfs_vget_locked(struct ramfs_mount *rm, struct ramfs_node *node, vnode_t dvp,/* ------------------------------------------------------------------------ *//* File contents */-/* How many slots an array needs to hold page `index` of a file that has `have` of them. */-static uint64_t-ramfs_slots_wanted(uint64_t have, uint64_t index)-{- uint64_t want = have ? have : 4;-- while (want <= index) {- want *= 2;- }- return want;-}--/*- * Make sure the page array has a slot for page `index`. The mount lock is held, and nothing waits- * for memory under it -- a wait could end in the pageout daemon, which comes back here for the- * lock. So the array is the caller's to find room for, without the lock: *slots of *nslots- * entries, used and consumed when it is big enough. EAGAIN means only that: bring one of- * ramfs_slots_wanted() entries and come again; it is not ENOSPC, and a caller that turns it into- * one says a volume with room to spare is full. A caller that cannot wait either -- the pageout- * path -- brings none, and then this tries without waiting, as it always did.- */+/* Make sure the page array has a slot for page `index`. The mount lock is held. */static int-ramfs_grow_slots(struct ramfs_node *node, uint64_t index, void ***slots, uint64_t *nslots)+ramfs_grow_slots(struct ramfs_node *node, uint64_t index){- void **grown;- uint64_t grown_n;+ uint64_t want = node->rn_nslots ? node->rn_nslots : 4;+ void **slots;if (index < node->rn_nslots) {return 0;}- if (*slots != NULL && *nslots > index && *nslots >= node->rn_nslots) {- grown = *slots;- grown_n = *nslots;- *slots = NULL;- } else {- grown_n = ramfs_slots_wanted(node->rn_nslots, index);- grown = kalloc_type(void *, grown_n, Z_NOWAIT | Z_ZERO);- if (grown == NULL) {- return EAGAIN;- }+ while (want <= index) {+ want *= 2;+ }+ slots = kalloc_type(void *, want, Z_NOWAIT | Z_ZERO);+ if (slots == NULL) {+ return ENOSPC;}if (node->rn_pages != NULL) {- memcpy(grown, node->rn_pages, node->rn_nslots * sizeof(void *));+ memcpy(slots, node->rn_pages, node->rn_nslots * sizeof(void *));kfree_type(void *, node->rn_nslots, node->rn_pages);}- node->rn_pages = grown;- node->rn_nslots = grown_n;+ node->rn_pages = slots;+ node->rn_nslots = want;return 0;}/** Copy len bytes (within one page) into the file at offset off, extending* the file if needed. *spare is a zeroed page the caller allocated without- * the lock; it is consumed if the slot was empty. *slots is the page array- * the caller allocated the same way (see ramfs_grow_slots). The mount lock- * is held.+ * the lock; it is consumed if the slot was empty. The mount lock is held.*/static int-ramfs_store(struct ramfs_node *node, uint64_t off, const void *src, size_t len, void **spare,- void ***slots, uint64_t *nslots)+ramfs_store(struct ramfs_node *node, uint64_t off, const void *src, size_t len, void **spare){struct ramfs_mount *rm = node->rn_mount;uint64_t index = off / PAGE_SIZE;- int error = ramfs_grow_slots(node, index, slots, nslots);+ int error = ramfs_grow_slots(node, index);if (error) {return error;@@ -618,6 +700,7 @@ ramfs_make(vnode_t dvp, vnode_t *vpp, struct componentname *cnp, struct vnode_atreturn error;}node->rn_ino = rm->rm_next_ino++;+ ramfs_hash_insert(rm, node);if (type == VDIR) {node->rn_nlink = 2;node->rn_parent = dir;@@ -897,6 +980,21 @@ ramfs_getattr(struct vnop_getattr_args *ap)VATTR_RETURN(vap, va_gen, 0);if (node->rn_type == VDIR && node->rn_parent != NULL) {VATTR_RETURN(vap, va_parentid, node->rn_parent->rn_ino);+ /*+ * And its name, from its entry in the parent: what the kernel asks for to make a path out of a+ * number alone (fsgetpath() of a vnode vfs_vget made, which has no name of its own).+ */+ if (VATTR_IS_ACTIVE(vap, va_name) && vap->va_name != NULL && node != rm->rm_root) {+ struct ramfs_dirent *de;++ TAILQ_FOREACH(de, &node->rn_parent->rn_entries, rd_link) {+ if (de->rd_node == node) {+ strlcpy(vap->va_name, de->rd_name, MAXPATHLEN);+ VATTR_SET_SUPPORTED(vap, va_name);+ break;+ }+ }+ }}lck_mtx_unlock(&rm->rm_lock);return 0;@@ -1017,8 +1115,7 @@ ramfs_write(struct vnop_write_args *ap)struct uio *uio = ap->a_uio;struct ramfs_node *node = RAMFS_NODE(vp);struct ramfs_mount *rm = node->rn_mount;- void *spare = NULL, **slots = NULL;- uint64_t nslots = 0;+ void *spare = NULL;char *bounce;uint64_t start, old_size;int error = 0;@@ -1052,34 +1149,12 @@ ramfs_write(struct vnop_write_args *ap)break;}if (spare == NULL) {- spare = kalloc_data(PAGE_SIZE, Z_WAITOK | Z_ZERO);- }- /*- * EAGAIN says the page array has to grow and that room for it is this side's to find,- * which is done here, where waiting is allowed. rn_nslots is read without the lock as a- * hint only; another writer growing the same file at the same time makes the hint too- * small, and then this asks again with a fresh one.- */- for (int again = 0; again < 4; again++) {- lck_mtx_lock(&rm->rm_lock);- error = ramfs_store(node, off, bounce, len, &spare, &slots, &nslots);- lck_mtx_unlock(&rm->rm_lock);- if (error != EAGAIN) {- break;- }- if (slots != NULL) {- kfree_type(void *, nslots, slots);- }- nslots = ramfs_slots_wanted(node->rn_nslots, off / PAGE_SIZE);- slots = kalloc_type(void *, nslots, Z_WAITOK | Z_ZERO);- if (slots == NULL) {- nslots = 0;- break;- }- }- if (error == EAGAIN) {- error = ENOSPC; /* it never got the array: out of memory, not out of volume */+ spare = ramfs_page_alloc(true);}+ ramfs_reserve_slots(rm, node, off / PAGE_SIZE);+ lck_mtx_lock(&rm->rm_lock);+ error = ramfs_store(node, off, bounce, len, &spare);+ lck_mtx_unlock(&rm->rm_lock);if (error) {/* nothing of this piece was stored: hand it back to the caller's count */uio_setoffset(uio, (off_t)off);@@ -1089,10 +1164,7 @@ ramfs_write(struct vnop_write_args *ap)}kfree_data(bounce, PAGE_SIZE);if (spare != NULL) {- kfree_data(spare, PAGE_SIZE);- }- if (slots != NULL) {- kfree_type(void *, nslots, slots);+ ramfs_page_free(spare);}if ((uint64_t)uio_offset(uio) > start) {@@ -1371,16 +1443,13 @@ ramfs_pageout(struct vnop_pageout_args *ap)lck_mtx_lock(&rm->rm_lock);for (upl_size_t done = 0; done < size && off + done < node->rn_size && !error; done += PAGE_SIZE) {size_t len = (size_t)MIN((uint64_t)PAGE_SIZE, node->rn_size - (off + done));+ uint64_t index = (off + done) / PAGE_SIZE;/* this may be the pageout daemon looking for memory: never wait for more of it here */- void *spare = kalloc_data(PAGE_SIZE, Z_NOWAIT | Z_ZERO), **slots = NULL;- uint64_t nslots = 0;+ void *spare = index < node->rn_nslots && node->rn_pages[index] != NULL ? NULL : ramfs_page_alloc(false);- error = ramfs_store(node, off + done, (const void *)(addr + done), len, &spare, &slots, &nslots);- if (error == EAGAIN) {- error = ENOSPC; /* nothing here may wait for the array either: the pages stay dirty */- }+ error = ramfs_store(node, off + done, (const void *)(addr + done), len, &spare);if (spare != NULL) {- kfree_data(spare, PAGE_SIZE);+ ramfs_page_free(spare);}}if (!error) {@@ -1661,8 +1730,10 @@ ramfs_mount(mount_t mp, __unused vnode_t devvp, __unused user_addr_t data, __unu/* a runaway writer must not be able to take the whole machine: a quarter of memory at most */rm->rm_max_pages = (max_mem / 4) / PAGE_SIZE;+ rm->rm_hash = kalloc_type(struct ramfs_node *, RAMFS_HASH_SIZE, Z_WAITOK | Z_ZERO | Z_NOFAIL);rm->rm_root = ramfs_node_alloc(rm, VDIR, NULL);rm->rm_root->rn_ino = RAMFS_ROOT_INO;+ ramfs_hash_insert(rm, rm->rm_root);rm->rm_root->rn_nlink = 2;rm->rm_root->rn_parent = rm->rm_root;rm->rm_nodes = 1;@@ -1722,6 +1793,7 @@ ramfs_unmount(mount_t mp, int mntflags, __unused vfs_context_t ctx)vfs_setfsprivate(mp, NULL);lck_mtx_destroy(&rm->rm_lock, &ramfs_lck_grp);+ kfree_type(struct ramfs_node *, RAMFS_HASH_SIZE, rm->rm_hash);kfree_type(struct ramfs_mount, rm);return 0;}@@ -1739,6 +1811,29 @@ ramfs_root(mount_t mp, vnode_t *vpp, __unused vfs_context_t ctx)return error;}+/* A node by its number (fsgetpath, and whoever else asks by file ID): see ramfs_hash_insert(). */+static int+ramfs_vfs_vget(mount_t mp, ino64_t ino, vnode_t *vpp, __unused vfs_context_t ctx)+{+ struct ramfs_mount *rm = RAMFS_MP(mp);+ int error;++ do {+ struct ramfs_node *node;++ lck_mtx_lock(&rm->rm_lock);+ for (node = rm->rm_hash[ino & (RAMFS_HASH_SIZE - 1)]; node != NULL && node->rn_ino != ino;+ node = node->rn_hnext) {+ }+ if (node == NULL || node->rn_nlink == 0) {+ lck_mtx_unlock(&rm->rm_lock);+ return ENOENT;+ }+ error = ramfs_vget_locked(rm, node, NULLVP, NULL, vpp);+ } while (error == EAGAIN);+ return error;+}+static intramfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, __unused vfs_context_t ctx){@@ -1771,7 +1866,15 @@ ramfs_vfs_getattr(mount_t mp, struct vfs_attr *fsap, __unused vfs_context_t ctx)VOL_CAP_FMT_CASE_SENSITIVE |VOL_CAP_FMT_CASE_PRESERVING |VOL_CAP_FMT_FAST_STATFS |- VOL_CAP_FMT_2TB_FILESIZE;+ VOL_CAP_FMT_2TB_FILESIZE |+ /*+ * A file keeps its number for as long as it exists (the life of the mount, a boot), a node is+ * found by it (vfs_vget) and a path made from it (fsgetpath): LaunchServices keeps an+ * application by number, and takes none from a volume that does not say so -- everything a+ * package installed under the prefix was "Unable to register", with nothing tried on its files.+ */+ VOL_CAP_FMT_PERSISTENTOBJECTIDS |+ VOL_CAP_FMT_PATH_FROM_ID;cap->capabilities[VOL_CAPABILITIES_INTERFACES] =VOL_CAP_INT_ADVLOCK |VOL_CAP_INT_FLOCK |@@ -1888,6 +1991,7 @@ struct vfsops ramfs_vfsops = {.vfs_mount = ramfs_mount,.vfs_unmount = ramfs_unmount,.vfs_root = ramfs_root,+ .vfs_vget = ramfs_vfs_vget,.vfs_getattr = ramfs_vfs_getattr,.vfs_sync = ramfs_sync,.vfs_init = ramfs_init,