Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: metadata read without the volume's lock, so that 8 readers are faster than 4
vcvkke0ec919cf315
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.cindex 60cc31c..bc66d3f 100644--- a/tools/apfs/apfs-tool.c+++ b/tools/apfs/apfs-tool.c@@ -140,6 +140,11 @@ host_parallel(void *ctx, uint32_t items, uint32_t workers,return atomic_load(&hp.error);}+static bool want_lock; /* the library under a lock, as in the kernel (walkers, cfrace) */+static void host_locking(struct apfs_nx *nx);+static void op_begin(void);+static void op_end(void);+static intopen_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol){@@ -171,6 +176,14 @@ open_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol)nx->parallel_max = (uint32_t)(t < 16 ? t : 16);}}+ if (getenv("APFS_TOOL_WALKERS") != NULL && strtol(getenv("APFS_TOOL_WALKERS"), NULL, 0) > 1) {+ want_lock = true;+ }+ if (want_lock) {+ /* handed over now, as the kernel's mount does; from here on every call is with the lock */+ host_locking(nx);+ op_begin();+ }/* the first volume that has a root directory, like apfs.py picks it */for (i = 0; i < APFS_NX_MAX_FILE_SYSTEMS; i++) {struct apfs_inode ino;@@ -182,9 +195,15 @@ open_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol)continue;}if (apfs_inode_get(vol, APFS_ROOT_DIR_INO_NUM, &ino) == 0) {+ if (want_lock) {+ op_end();+ }return 0;}}+ if (want_lock) {+ op_end();+ }fprintf(stderr, "apfs-tool: no readable volume with a root directory\n");return 1;}@@ -413,6 +432,93 @@ cmd_cat(const char *path, const char *where)return 0;}+/*+ * ---- several threads on the library at once, as in the kernel ----+ *+ * The kernel holds one lock round apfs_lib for each operation and lets apfs_lib let go of it+ * in the middle (reading file contents, and a block of metadata not in the cache); a block+ * leaving the cache keeps its memory until no operation that began before can be reading it+ * (apfs_nx_reclaim). The same here, with a pthread mutex, so that it can be run under+ * ThreadSanitizer: op_begin/op_end are the kernel's apfs_lock/apfs_unlock.+ */+static pthread_mutex_t host_lock; /* error checking: unlocking one not held aborts, as the kernel panics */+static struct apfs_nx *host_nx;++static void+host_lock_check(int error, const char *what)+{+ if (error != 0) {+ fprintf(stderr, "apfs-tool: %s the library's lock: %s\n", what, strerror(error));+ abort();+ }+}+#define HOST_OPS 64+static struct {+ pthread_t th;+ uint64_t epoch;+ bool used;+} host_ops[HOST_OPS];++static void+host_unlock(void *ctx)+{+ (void)ctx;+ host_lock_check(pthread_mutex_unlock(&host_lock), "letting go of");+}++static void+host_relock(void *ctx)+{+ (void)ctx;+ host_lock_check(pthread_mutex_lock(&host_lock), "taking");+}++static void+op_begin(void)+{+ host_lock_check(pthread_mutex_lock(&host_lock), "taking");+ for (int i = 0; i < HOST_OPS; i++) {+ if (!host_ops[i].used) {+ host_ops[i].used = true;+ host_ops[i].th = pthread_self();+ host_ops[i].epoch = host_nx->epoch;+ return;+ }+ }+ fprintf(stderr, "apfs-tool: more operations at once than %d\n", HOST_OPS);+ abort();+}++static void+op_end(void)+{+ uint64_t oldest = host_nx->epoch;++ for (int i = 0; i < HOST_OPS; i++) {+ if (host_ops[i].used && pthread_equal(host_ops[i].th, pthread_self())) {+ host_ops[i].used = false;+ } else if (host_ops[i].used && host_ops[i].epoch < oldest) {+ oldest = host_ops[i].epoch;+ }+ }+ apfs_nx_reclaim(host_nx, oldest);+ host_lock_check(pthread_mutex_unlock(&host_lock), "letting go of");+}++static void+host_locking(struct apfs_nx *nx)+{+ pthread_mutexattr_t a;++ pthread_mutexattr_init(&a);+ pthread_mutexattr_settype(&a, PTHREAD_MUTEX_ERRORCHECK);+ pthread_mutex_init(&host_lock, &a);+ host_nx = nx;+ nx->unlock = host_unlock;+ nx->relock = host_relock;+ nx->lock_ctx = NULL;+}+/* ---- census: one line per entry, for diffing against apfs.py ---- */struct census {@@ -429,6 +535,8 @@ struct census {};static int census_dir(struct census *cs, uint64_t dir_id);+static int census_parallel(struct apfs_nx *nx, struct apfs_vol *vol, uint64_t start, const char *where,+ size_t chunk, bool meta_only, int n);static uint64_tfile_hash(struct census *cs, const struct apfs_inode *ino, uint64_t size)@@ -537,9 +645,20 @@ cmd_census(const char *path, const char *where, bool meta_only)if (open_image(path, &nx, &vol) != 0) {return 1;}- if (where != NULL && apfs_path_lookup(&vol, where, &start, &dt) != 0) {- fprintf(stderr, "apfs-tool: %s: not found\n", where);- return 1;+ if (where != NULL) {+ int e;++ if (want_lock) {+ op_begin();+ }+ e = apfs_path_lookup(&vol, where, &start, &dt);+ if (want_lock) {+ op_end();+ }+ if (e != 0) {+ fprintf(stderr, "apfs-tool: %s: not found\n", where);+ return 1;+ }}memset(&cs, 0, sizeof(cs));cs.vol = &vol;@@ -577,6 +696,11 @@ cmd_census(const char *path, const char *where, bool meta_only)cs.path_len = n;cs.path[n] = '\0';}+ if (getenv("APFS_TOOL_WALKERS") != NULL && strtol(getenv("APFS_TOOL_WALKERS"), NULL, 0) > 1) {+ long n = strtol(getenv("APFS_TOOL_WALKERS"), NULL, 0);++ return census_parallel(&nx, &vol, start, where, cs.chunk, meta_only, (int)(n > 16 ? 16 : n));+ }error = census_dir(&cs, start);if (error && error != EEXIST) {fprintf(stderr, "apfs-tool: census: %s\n", strerror(error));@@ -602,6 +726,361 @@ cmd_census(const char *path, const char *where, bool meta_only)return cs.error ? 1 : 0;}+/*+ * APFS_TOOL_WALKERS=N with census: N threads walk at once, taking directories off one shared+ * stack, each line hashed and the hashes added up (order does not matter): the same sum the+ * kernel's apfs_check prints. Lines are not printed.+ */+struct pjob {+ uint64_t dir_id;+ char *path;+};++static pthread_mutex_t pmtx = PTHREAD_MUTEX_INITIALIZER;+static pthread_cond_t pcond = PTHREAD_COND_INITIALIZER;+static struct pjob *pstack;+static size_t pcount, pcap;+static int pbusy;++struct pwalker {+ struct apfs_vol *vol;+ const char *dir; /* the directory being walked */+ uint8_t *buf;+ size_t chunk;+ bool meta_only;+ uint64_t sum, entries, bytes;+ int error;+};++static void+ppush(uint64_t dir_id, const char *path)+{+ pthread_mutex_lock(&pmtx);+ if (pcount == pcap) {+ pcap = pcap ? 2 * pcap : 1024;+ pstack = realloc(pstack, pcap * sizeof(*pstack));+ if (pstack == NULL) {+ abort();+ }+ }+ pstack[pcount].dir_id = dir_id;+ pstack[pcount].path = strdup(path);+ pcount++;+ pthread_cond_signal(&pcond);+ pthread_mutex_unlock(&pmtx);+}++static uint64_t+pfile_hash(struct pwalker *w, const struct apfs_inode *ino, uint64_t size, const char *path)+{+ uint64_t h = FNV_INIT, off = 0;+ struct apfs_cfile cf;++ memset(&cf, 0, sizeof(cf));+ cf.held = APFS_NO_PIECE;+ for (;;) {+ size_t got = 0;+ int error = apfs_read_file(w->vol, ino, &cf, off, w->buf, w->chunk, &got);++ if (error) {+ fprintf(stderr, "apfs-tool: %s: read at %" PRIu64 ": %s\n", path, off, strerror(error));+ w->error = error;+ break;+ }+ if (got == 0) {+ break;+ }+ h = fnv1a(w->buf, got, h);+ off += got;+ w->bytes += got;+ if (off >= size) {+ break;+ }+ }+ apfs_cfile_fini(&cf);+ return h;+}++static int+pcensus_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)+{+ struct pwalker *w = ctx;+ struct apfs_inode ino;+ uint64_t h = 0, size = 0;+ char path[4096], line[4096 + 128];+ int n;++ n = snprintf(path, sizeof(path), "%s/%.*s", w->dir, (int)name_len, name);+ if (n < 0 || (size_t)n >= sizeof(path)) {+ w->error = ENAMETOOLONG;+ return 0;+ }+ if (apfs_inode_get(w->vol, file_id, &ino) != 0) {+ fprintf(stderr, "apfs-tool: %s: no inode\n", path);+ w->error = ENOENT;+ return 0;+ }+ if (apfs_inode_size(w->vol, &ino, &size) != 0) {+ size = ino.size;+ }+ if (!w->meta_only && ((ino.mode & APFS_S_IFMT) == APFS_S_IFREG || (ino.mode & APFS_S_IFMT) == APFS_S_IFLNK)) {+ h = pfile_hash(w, &ino, size, path);+ }+ n = snprintf(line, sizeof(line), "%s\t%u\t%06o\t%u\t%u\t%" PRIu64 "\t%016" PRIx64 "\n", path, d_type,+ ino.mode, ino.owner, ino.group, size, h);+ w->sum += fnv1a((const uint8_t *)line, (size_t)n, FNV_INIT);+ w->entries++;+ if (d_type == APFS_DT_DIR) {+ ppush(file_id, path);+ }+ return 0;+}++static void *+pwalker_thread(void *p)+{+ struct pwalker *w = p;++ for (;;) {+ struct pjob j;+ int error;++ pthread_mutex_lock(&pmtx);+ while (pcount == 0 && pbusy > 0) {+ pthread_cond_wait(&pcond, &pmtx);+ }+ if (pcount == 0) {+ pthread_cond_broadcast(&pcond);+ pthread_mutex_unlock(&pmtx);+ return NULL;+ }+ j = pstack[--pcount];+ pbusy++;+ pthread_mutex_unlock(&pmtx);++ w->dir = j.path;+ op_begin();+ error = apfs_readdir(w->vol, j.dir_id, pcensus_cb, w);+ op_end();+ if (error) {+ fprintf(stderr, "apfs-tool: %s: readdir: %s\n", j.path, strerror(error));+ w->error = error;+ }+ free(j.path);++ pthread_mutex_lock(&pmtx);+ pbusy--;+ pthread_cond_broadcast(&pcond);+ pthread_mutex_unlock(&pmtx);+ }+}++static int+census_parallel(struct apfs_nx *nx, struct apfs_vol *vol, uint64_t start, const char *where, size_t chunk,+ bool meta_only, int n)+{+ struct pwalker w[16];+ pthread_t th[16];+ uint64_t sum = 0, entries = 0, bytes = 0;+ int error = 0;++ ppush(start, where != NULL ? where : "");+ for (int i = 0; i < n; i++) {+ memset(&w[i], 0, sizeof(w[i]));+ w[i].vol = vol;+ w[i].chunk = chunk;+ w[i].meta_only = meta_only;+ w[i].buf = malloc(chunk);+ if (w[i].buf == NULL || pthread_create(&th[i], NULL, pwalker_thread, &w[i]) != 0) {+ abort();+ }+ }+ for (int i = 0; i < n; i++) {+ pthread_join(th[i], NULL);+ sum += w[i].sum;+ entries += w[i].entries;+ bytes += w[i].bytes;+ if (w[i].error) {+ error = w[i].error;+ }+ free(w[i].buf);+ }+ fprintf(stderr, "apfs-tool: %d walkers: %" PRIu64 " entries, %" PRIu64 " content bytes, line-hash sum %016" PRIx64+ ", %s\n", n, entries, bytes, sum, error ? "ERRORS" : "no errors");+ fprintf(stderr, "apfs-tool: metadata read without the lock %" PRIu64 " times, %" PRIu64 " of them already in; "+ "retired blocks %" PRIu64 " now, %" PRIu64 " at most\n", nx->unlocked_fills, nx->fill_races, nx->retired_now,+ nx->retired_most);+ free(pstack);+ {+ int fd = (int)(intptr_t)nx->dev.ctx;++ apfs_nx_close(nx);+ close(fd);+ }+ return error ? 1 : 0;+}++/*+ * cfrace IMAGE N ROUNDS: the compressed files of the volume (up to 400, taken from a walk),+ * each read whole by N threads at once through one shared apfs_cfile -- as two readers of one+ * vnode share it in the kernel -- with the cache emptied before every round, so that setting+ * up the file and its read-ahead let go of the lock in the middle; every read must hash the+ * same as a read on one thread.+ */+struct cf_file {+ uint64_t id;+ struct apfs_inode ino;+ uint64_t size;+ uint64_t want; /* the hash on one thread */+ struct apfs_cfile cf; /* shared by the threads, touched with the lock held */+};++static struct cf_file *cf_files;+static size_t cf_nfiles;+static struct apfs_vol *cf_vol;+static atomic_uint cf_bad, cf_reads;++static int+cf_collect_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)+{+ struct apfs_inode ino;+ struct apfs_cfile cf;+ uint64_t size;++ (void)ctx;+ (void)name;+ (void)name_len;+ if (cf_nfiles >= 400) {+ return EEXIST;+ }+ if (d_type == APFS_DT_DIR) {+ return apfs_readdir(cf_vol, file_id, cf_collect_cb, NULL) == EEXIST ? EEXIST : 0;+ }+ if (d_type != APFS_DT_REG || apfs_inode_get(cf_vol, file_id, &ino) != 0 ||+ apfs_inode_size(cf_vol, &ino, &size) != 0 || size < 3 * 65536) {+ return 0;+ }+ memset(&cf, 0, sizeof(cf));+ cf.held = APFS_NO_PIECE;+ if (apfs_cfile_init(cf_vol, &ino, &cf) == 0 && cf.compressed && cf.in_rsrc) {+ cf_files[cf_nfiles].id = file_id;+ cf_files[cf_nfiles].ino = ino;+ cf_files[cf_nfiles].size = size;+ cf_nfiles++;+ }+ apfs_cfile_fini(&cf);+ return 0;+}++static uint64_t+cf_read_whole(struct cf_file *f, struct apfs_cfile *cf, uint8_t *buf, size_t chunk, bool locked)+{+ uint64_t h = FNV_INIT, off = 0;++ while (off < f->size) {+ size_t got = 0;+ int error;++ if (locked) {+ op_begin();+ }+ error = apfs_read_file(cf_vol, &f->ino, cf, off, buf, chunk, &got);+ if (locked) {+ op_end();+ }+ if (error || got == 0) {+ return 0;+ }+ h = fnv1a(buf, got, h);+ off += got;+ }+ return h;+}++static void *+cf_thread(void *p)+{+ uint8_t *buf = malloc(65536);++ (void)p;+ for (size_t i = 0; i < cf_nfiles; i++) {+ uint64_t h = cf_read_whole(&cf_files[i], &cf_files[i].cf, buf, 65536, true);++ atomic_fetch_add(&cf_reads, 1);+ if (h != cf_files[i].want) {+ atomic_fetch_add(&cf_bad, 1);+ fprintf(stderr, "apfs-tool: cfrace: inode %" PRIu64 " read DIFFERENTLY\n", cf_files[i].id);+ }+ }+ free(buf);+ return NULL;+}++static int+cmd_cfrace(const char *path, int n, int rounds)+{+ struct apfs_nx nx;+ struct apfs_vol vol;+ pthread_t th[16];+ uint8_t *buf = malloc(65536);++ want_lock = true;+ if (open_image(path, &nx, &vol) != 0 || buf == NULL || n < 2 || n > 16) {+ return 1;+ }+ cf_vol = &vol;+ cf_files = calloc(400, sizeof(*cf_files));+ op_begin();+ (void)apfs_readdir(&vol, APFS_ROOT_DIR_INO_NUM, cf_collect_cb, NULL);+ op_end();+ for (size_t i = 0; i < cf_nfiles; i++) {+ struct apfs_cfile cf;++ memset(&cf, 0, sizeof(cf));+ cf.held = APFS_NO_PIECE;+ cf_files[i].want = cf_read_whole(&cf_files[i], &cf, buf, 65536, true);+ op_begin();+ apfs_cfile_fini(&cf);+ op_end();+ }+ fprintf(stderr, "apfs-tool: cfrace: %zu compressed files of 192 KB or more, %d threads, %d rounds\n", cf_nfiles,+ n, rounds);+ if (nx.parallel == NULL) {+ nx.parallel = host_parallel; /* read-ahead needs helpers to spare */+ nx.parallel_max = 4;+ }+ for (int r = 0; r < rounds; r++) {+ pthread_mutex_lock(&host_lock);+ for (size_t i = 0; i < cf_nfiles; i++) {+ apfs_cfile_fini(&cf_files[i].cf);+ }+ apfs_nx_cache_flush(&nx);+ pthread_mutex_unlock(&host_lock);+ for (int t = 0; t < n; t++) {+ pthread_create(&th[t], NULL, cf_thread, NULL);+ }+ for (int t = 0; t < n; t++) {+ pthread_join(th[t], NULL);+ }+ }+ fprintf(stderr, "apfs-tool: cfrace: %u reads, %u different; metadata read without the lock %" PRIu64+ " times, %" PRIu64 " of them already in; read ahead %" PRIu64 " times\n", atomic_load(&cf_reads),+ atomic_load(&cf_bad), nx.unlocked_fills, nx.fill_races, nx.ra_batches);+ /* all given back, so that a leak checker sees only what the library lost */+ for (size_t i = 0; i < cf_nfiles; i++) {+ apfs_cfile_fini(&cf_files[i].cf);+ }+ free(cf_files);+ free(buf);+ {+ int fd = (int)(intptr_t)nx.dev.ctx;++ apfs_nx_close(&nx);+ close(fd);+ }+ return atomic_load(&cf_bad) != 0;+}+/** Every name in the volume looked up by name, the way the kernel's vnop_lookup does it,* against what reading its directory gave; and each name with "#" after it, which must@@ -768,7 +1247,8 @@ main(int argc, char **argv)" apfs-tool stat IMAGE PATH\n"" apfs-tool cat IMAGE PATH\n"" apfs-tool census IMAGE [PATH] [--meta]\n"- " apfs-tool lookups IMAGE\n");+ " apfs-tool lookups IMAGE\n"+ " apfs-tool cfrace IMAGE [THREADS [ROUNDS]]\n");return 2;}if (strcmp(argv[1], "info") == 0) {@@ -790,6 +1270,9 @@ main(int argc, char **argv)if (strcmp(argv[1], "lookups") == 0) {return cmd_lookups(argv[2]);}+ if (strcmp(argv[1], "cfrace") == 0) {+ return cmd_cfrace(argv[2], argc > 3 ? atoi(argv[3]) : 4, argc > 4 ? atoi(argv[4]) : 3);+ }fprintf(stderr, "apfs-tool: unknown command %s\n", argv[1]);return 2;}diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.cindex 04f254c..4d27974 100644--- a/xnu/files/bsd/dev/redwood_apfs_check.c+++ b/xnu/files/bsd/dev/redwood_apfs_check.c@@ -190,6 +190,9 @@ struct census {bool no_contents;/* apfs_walkers=N: one of N walkers; a subdirectory is left on the shared stack (par_push) */bool par;+ /* where a walker's time goes (mach ticks): lookups, getattr, reading files and links+ * (hashing included), readdir, waiting for a directory to walk (par); on a processor */+ uint64_t t_lookup, t_attr, t_read, t_readdir, t_idle, t_cpu, t_wall;};/* The boot-args both kinds of walk take the same way. */@@ -295,7 +298,12 @@ census_entry(struct census *cs, uint8_t d_type)* path leads from the ramfs that holds the mount point into the volume itself.*/PHASE("lookup");- error = vnode_lookup(cs->path, cs->lookup_flags, &vp, cs->ctx);+ {+ uint64_t t0 = mach_absolute_time();++ error = vnode_lookup(cs->path, cs->lookup_flags, &vp, cs->ctx);+ cs->t_lookup += mach_absolute_time() - t0;+ }if (error) {printf("apfs-check: %s: lookup failed: %d\n", cs->path + cs->rel, error);cs->errors++;@@ -311,7 +319,12 @@ census_entry(struct census *cs, uint8_t d_type)VATTR_WANTED(&va, va_gid);VATTR_WANTED(&va, va_data_size);PHASE("getattr");- error = vnode_getattr(vp, &va, cs->ctx);+ {+ uint64_t t0 = mach_absolute_time();++ error = vnode_getattr(vp, &va, cs->ctx);+ cs->t_attr += mach_absolute_time() - t0;+ }if (error) {printf("apfs-check: %s: getattr failed: %d\n", cs->path + cs->rel, error);cs->errors++;@@ -325,13 +338,19 @@ census_entry(struct census *cs, uint8_t d_type)cs->files++;PHASE("read");if (!cs->no_contents) {+ uint64_t t0 = mach_absolute_time();+error = hash_file(cs, vp, size, &h);+ cs->t_read += mach_absolute_time() - t0;}} else if (vnode_vtype(vp) == VLNK) {cs->links++;PHASE("readlink");if (!cs->no_contents) {+ uint64_t t0 = mach_absolute_time();+error = hash_link(cs, vp, &h);+ cs->t_read += mach_absolute_time() - t0;}} else if (vnode_vtype(vp) == VDIR) {cs->dirs++;@@ -520,8 +539,13 @@ census_dir(struct census *cs)off_t offset = 0;int error;- error = vnode_lookup(cs->len != 0 ? cs->path : "/", cs->lookup_flags, &dvp,- cs->ctx);+ {+ uint64_t t0 = mach_absolute_time();++ error = vnode_lookup(cs->len != 0 ? cs->path : "/", cs->lookup_flags, &dvp,+ cs->ctx);+ cs->t_lookup += mach_absolute_time() - t0;+ }if (error == 0 && cs->mp == NULL) {cs->mp = vnode_mount(dvp);}@@ -552,7 +576,12 @@ census_dir(struct census *cs)}uio_addiov(uio, CAST_USER_ADDR_T(dbuf), DIR_CHUNK);PHASE("readdir");- error = VNOP_READDIR(dvp, uio, 0, &eof, &items, cs->ctx);+ {+ uint64_t t0 = mach_absolute_time();++ error = VNOP_READDIR(dvp, uio, 0, &eof, &items, cs->ctx);+ cs->t_readdir += mach_absolute_time() - t0;+ }if (error) {printf("apfs-check: %s: readdir failed: %d\n", cs->path + cs->rel, error);cs->errors++;@@ -651,16 +680,21 @@ static voidpar_walker(void *arg, __unused wait_result_t wr){struct census *cs = arg;+ uint64_t cpu0 = thread_get_runtime_self(), wall0 = mach_absolute_time();for (;;) {struct par_dir d;+ uint64_t t0 = mach_absolute_time();lck_mtx_lock(&par_lock);while (par_count == 0 && par_busy > 0) {msleep(&par_stack, &par_lock, PRIBIO, "apfs-walkers", NULL);}+ cs->t_idle += mach_absolute_time() - t0;if (par_count == 0) {/* nothing left, and no walker that could leave more */+ cs->t_cpu = thread_get_runtime_self() - cpu0;+ cs->t_wall = mach_absolute_time() - wall0;par_finished++;wakeup(&par_stack);lck_mtx_unlock(&par_lock);@@ -685,6 +719,29 @@ par_walker(void *arg, __unused wait_result_t wr)}}+/* where a walker's (or, summed, all walkers') time went */+static void+census_times(const struct census *cs, int who)+{+ uint64_t v[8] = { cs->t_wall, cs->t_cpu, cs->t_lookup, cs->t_attr, cs->t_read, cs->hash_ticks, cs->t_readdir,+ cs->t_idle };++ for (int i = 0; i < 8; i++) {+ absolutetime_to_nanoseconds(v[i], &v[i]);+ v[i] /= 1000000;+ }+ char name[24];++ if (who < 0) {+ strlcpy(name, "all walkers", sizeof(name));+ } else {+ snprintf(name, sizeof(name), "walker %d", who);+ }+ printf("apfs-check: %s: %llu entries, %llu bytes: %llu ms, on a processor %llu; lookups %llu, getattr %llu, "+ "reading %llu (of it hashing %llu), readdir %llu, waiting for a directory %llu\n",+ name, cs->entries, cs->bytes, v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]);+}+/* The whole walk on n threads, its totals into cs */static voidpar_walk(struct census *cs, uint32_t n)@@ -743,7 +800,14 @@ par_walk(struct census *cs, uint32_t n)cs->bytes += w[i]->bytes;cs->errors += w[i]->errors;cs->hash_ticks += w[i]->hash_ticks;- printf("apfs-check: walker %u: %llu entries, %llu bytes\n", i, w[i]->entries, w[i]->bytes);+ census_times(w[i], i);+ cs->t_lookup += w[i]->t_lookup;+ cs->t_attr += w[i]->t_attr;+ cs->t_read += w[i]->t_read;+ cs->t_readdir += w[i]->t_readdir;+ cs->t_idle += w[i]->t_idle;+ cs->t_cpu += w[i]->t_cpu;+ cs->t_wall += w[i]->t_wall;kfree_data(w[i]->path, PATH_CAP);kfree_data(w[i]->buf, READ_CHUNK);kfree_type(struct census, w[i]);@@ -752,6 +816,7 @@ par_walk(struct census *cs, uint32_t n)printf("apfs-check: the walkers ran out of memory for their stack: the walk is NOT whole\n");cs->errors++;}+ census_times(cs, -1);printf("apfs-check: at most %u directories waited on the walkers' stack\n", par_most);if (par_stack != NULL) {kfree_data(par_stack, par_cap * sizeof(*par_stack));@@ -1101,7 +1166,12 @@ walk:if (mode == 6 && walkers > 1) {par_walk(&cs, walkers > PAR_MAX ? PAR_MAX : walkers);} else {+ uint64_t cpu0 = thread_get_runtime_self(), wall0 = mach_absolute_time();+census_dir(&cs);+ cs.t_cpu = thread_get_runtime_self() - cpu0;+ cs.t_wall = mach_absolute_time() - wall0;+ census_times(&cs, 0);}}watch_stop();diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.hindex d102904..28dbb10 100644--- a/xnu/files/bsd/miscfs/apfs/apfs.h+++ b/xnu/files/bsd/miscfs/apfs/apfs.h@@ -83,6 +83,24 @@ struct apfs_mount {uint64_t am_lock_wait_ticks;uint64_t am_lock_held_ticks;uint64_t am_lock_t0; /* when the holder took it; only the holder writes */+ /* how long takes waited: up to 2 us, 10 us, 100 us, 1 ms, more; and the longest (the holder writes) */+ uint64_t am_lock_wait_hist[5];+ uint64_t am_lock_wait_max;+ /*+ * The operations going on (taken apfs_lock, not yet apfs_unlock; apfs_lib may let go of+ * the lock in between), and the container's epoch each began at: retired cache memory+ * older than the oldest of them is freed (apfs_nx_reclaim). Under am_lock.+ */+#define APFS_OPS_MAX 128+ struct {+ thread_t th;+ uint64_t epoch;+ } am_ops[APFS_OPS_MAX];+ uint32_t am_nops;+ uint32_t am_untracked; /* operations the table had no room for: nothing is freed meanwhile */+ /* how long it was held: up to 10 us, 100 us, 1 ms, 10 ms, more; and the longest */+ uint64_t am_lock_hold_hist[5];+ uint64_t am_lock_hold_max;/** vnop_read's buffers, kept for the next read rather than allocated and freed for* every one (at that size the kernel goes to the VM). Under am_lock.@@ -145,8 +163,10 @@ void apfs_node_detach(struct apfs_node *an);/* the breakdown of where the time went, straight to the log (apfs_vfsops.c) */void apfs_stats_print(void);-void apfs_lock(struct apfs_mount *am);-void apfs_unlock(struct apfs_mount *am);+void apfs_lock(struct apfs_mount *am); /* an operation begins */+void apfs_unlock(struct apfs_mount *am); /* and ends */+void apfs_lock_inner(struct apfs_mount *am); /* the lock taken back in the middle of one (apfs_lib) */+void apfs_unlock_inner(struct apfs_mount *am); /* let go of in the middle of one *//* apfs_vnops.c */extern const struct vnodeopv_desc apfs_vnodeop_opv_desc;diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.cindex 8b86354..4c652d0 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c@@ -116,6 +116,67 @@ cache_bucket(uint64_t addr)return (uint32_t)((addr * 0x9E3779B97F4A7C15ULL) >> 40) & (APFS_CACHE_HASH - 1);}+/* A cached block's memory leaves the cache; it is freed by apfs_nx_reclaim once nobody can be reading it. */+static void+cache_retire(struct apfs_nx *nx, uint8_t *data)+{+ struct apfs_retired *r = apfs_alloc(sizeof(*r));++ if (r == NULL) {+ return; /* never freed then: that is safe, merely a block lost */+ }+ r->data = data;+ r->epoch = nx->epoch++;+ r->next = nx->retired;+ nx->retired = r;+ if (++nx->retired_now > nx->retired_most) {+ nx->retired_most = nx->retired_now;+ }+}++void+apfs_nx_reclaim(struct apfs_nx *nx, uint64_t oldest)+{+ struct apfs_retired **pp = &nx->retired, *r;++ /* newest first: past the first one old enough, all are */+ while (*pp != NULL && (*pp)->epoch >= oldest) {+ pp = &(*pp)->next;+ }+ while ((r = *pp) != NULL) {+ *pp = r->next;+ apfs_free(r->data, nx->block_size);+ apfs_free(r, sizeof(*r));+ nx->retired_now--;+ }+}++#ifndef KERNEL+void+apfs_nx_cache_flush(struct apfs_nx *nx)+{+ uint32_t i;++ for (i = 0; i < APFS_CACHE_BLOCKS; i++) {+ struct apfs_buf *b = &nx->cache[i];++ if (b->data != NULL && nx->unlock != NULL) {+ cache_retire(nx, b->data);+ b->data = NULL;+ }+ b->addr = APFS_BUF_EMPTY;+ b->next = -1;+ b->ref = false;+ }+ for (i = 0; i < APFS_CACHE_HASH; i++) {+ nx->cache_hash[i] = -1;+ }+ if (nx->omap_cache != NULL) {+ memset(nx->omap_cache, 0, APFS_OMAP_CACHE * sizeof(*nx->omap_cache));+ }+}+#endif+/* A free slot, or the first the clock hand finds neither used since it last passed nor recent. */static struct apfs_buf *cache_victim(struct apfs_nx *nx)@@ -151,6 +212,16 @@ cache_victim(struct apfs_nx *nx)break;}}+ if (nx->unlock != NULL && victim->data != NULL && victim->addr != APFS_BUF_EMPTY) {+ /* somebody may still be reading the block that had it: new memory for the new one */+ uint8_t *fresh = apfs_alloc(nx->block_size);++ if (fresh == NULL) {+ return NULL;+ }+ cache_retire(nx, victim->data);+ victim->data = fresh;+ }if (victim->addr != APFS_BUF_EMPTY) {/* out of its old chain */for (pp = &nx->cache_hash[cache_bucket(victim->addr)]; *pp >= 0;@@ -189,6 +260,96 @@ cache_find(struct apfs_nx *nx, uint64_t addr)return NULL;}+static inline void nx_unlock(struct apfs_nx *nx);+static inline void nx_relock(struct apfs_nx *nx);++/*+ * A block of metadata not in the cache, read without the lock: the block and its aligned+ * neighbours into memory of this call's own, the lock taken back, and put in -- unless+ * somebody put the block in meanwhile, whose copy is used then (and only the neighbours not+ * there yet go in). Cache slots that get a block get new memory (cache_victim), so nothing+ * anyone still reads is overwritten.+ */+static int+nx_block_unlocked(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)+{+ uint32_t bs = nx->block_size;+ uint32_t k = nx->meta_cluster != 0 ?+ (nx->meta_cluster > APFS_META_CLUSTER ? APFS_META_CLUSTER : nx->meta_cluster) : APFS_META_CLUSTER_DEFAULT;+ uint64_t first = addr - addr % k, count = nx->block_count - first < k ? nx->block_count - first : k, i, t0, dt;+ struct apfs_buf *b;+ uint8_t *mine = apfs_alloc((size_t)count * bs);+ int error;++ if (mine == NULL) {+ return ENOMEM;+ }+ nx->unlocked_fills++;+ nx_unlock(nx);+ t0 = apfs_ticks();+ error = nx->dev.read_data(nx->dev.ctx, first * (uint64_t)bs, mine, (size_t)count * bs);+ dt = apfs_ticks() - t0;+ nx_relock(nx);+ nx->dev_ticks += dt;+ nx->dev_calls++;+ nx->dev_bytes += count * bs;+ if (error) {+ apfs_free(mine, (size_t)count * bs);+ return error;+ }+ nx->fetches++;+ b = cache_find(nx, addr);+ if (b != NULL) {+ nx->fill_races++; /* in already, by somebody else's read */+ if (verify && !b->checked) {+ if (!apfs_checksum_ok(b->data, bs)) {+ apfs_log("block %llu: bad checksum\n", (unsigned long long)addr);+ apfs_free(mine, (size_t)count * bs);+ return EIO;+ }+ b->checked = true;+ }+ } else {+ b = cache_victim(nx);+ if (b == NULL) {+ apfs_free(mine, (size_t)count * bs);+ return ENOMEM;+ }+ memcpy(b->data, mine + (addr - first) * bs, bs);+ nx->reads++;+ if (verify && !apfs_checksum_ok(b->data, bs)) {+ apfs_log("block %llu: bad checksum\n", (unsigned long long)addr);+ apfs_free(mine, (size_t)count * bs);+ return EIO; /* (the slot stays empty: cache_victim took it out of its chain) */+ }+ b->checked = verify;+ cache_link(nx, b, addr);+ }+ b->ref = true;+ b->used = nx->fetches;+ /* the neighbours not there yet: kept as if long unused, checked when asked for */+ for (i = first; i < first + count; i++) {+ struct apfs_buf *nb;++ if (i == addr || i == 0 || cache_find(nx, i) != NULL) {+ continue;+ }+ nb = cache_victim(nx);+ if (nb == NULL) {+ break;+ }+ memcpy(nb->data, mine + (i - first) * bs, bs);+ nb->checked = false;+ nb->ref = false;+ nb->used = nx->fetches - APFS_CACHE_RECENT;+ cache_link(nx, nb, i);+ nx->prefetched++;+ }+ apfs_free(mine, (size_t)count * bs);+ *out = b->data;+ return 0;+}+static intnx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify){@@ -221,6 +382,9 @@ nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)*out = b->data;return 0;}+ if (nx->unlock != NULL && nx->dev.read_data != NULL && !nx->cheap_requests) {+ return nx_block_unlocked(nx, addr, out, verify);+ }victim = cache_victim(nx);if (victim == NULL) {return ENOMEM;@@ -872,6 +1036,7 @@ apfs_nx_close(struct apfs_nx *nx)apfs_free(nx->cluster, (size_t)APFS_META_CLUSTER * nx->block_size);nx->cluster = NULL;}+ apfs_nx_reclaim(nx, ~0ULL);}int@@ -2169,16 +2334,43 @@ out:return error;}+static int cfile_setup(struct apfs_vol *vol, const struct apfs_inode *ino, struct apfs_cfile *cf);++/*+ * Worked out into a copy of its own and only then put in place: working it out reads the+ * tree, and a read can let go of the lock (nx_block_unlocked), so another reader of the same+ * file may be doing the same meanwhile -- whoever finishes second throws its copy away.+ */intapfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,struct apfs_cfile *cf){+ struct apfs_cfile mine;int error;if (cf->ready && cf->obj_id == ino->obj_id) {return 0; /* already worked out, with its table and piece */}+ memset(&mine, 0, sizeof(mine));+ mine.held = APFS_NO_PIECE;+ error = cfile_setup(vol, ino, &mine);+ if (error) {+ return error; /* (cfile_setup gave back what it had) */+ }+ if (cf->ready && cf->obj_id == ino->obj_id) {+ apfs_cfile_fini(&mine);+ return 0;+ }apfs_cfile_fini(cf);+ *cf = mine;+ return 0;+}++static int+cfile_setup(struct apfs_vol *vol, const struct apfs_inode *ino, struct apfs_cfile *cf)+{+ int error;+cf->obj_id = ino->obj_id;cf->held = APFS_NO_PIECE;@@ -3000,6 +3192,11 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,vol->nx->requests : 1) >= 2) {uint64_t k;+ /*+ * Claimed before the pieces are planned: planning reads the tree,+ * which can let go of the lock, and nobody else may fill ra_data.+ */+ cf->ra_filling = true;if (cf->ra_data == NULL) {cf->ra_data = apfs_alloc((size_t)APFS_RA_PIECES *APFS_DECMPFS_BLOCK_SIZE);@@ -3037,8 +3234,9 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,nra++;}if (nra > 0) {- cf->ra_filling = true;cf->ra_first = i + 1;+ } else {+ cf->ra_filling = false;}}continue;diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.hindex fb8bb0c..6b149fd 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h@@ -102,6 +102,13 @@ struct apfs_dev {int (*read_data)(void *ctx, uint64_t off, void *buf, size_t len);};+/* A cached block's memory, given up while somebody may still be reading it (nx->retired) */+struct apfs_retired {+ struct apfs_retired *next;+ uint64_t epoch;+ uint8_t *data;+};+/* One cached block of metadata. */struct apfs_buf {uint64_t addr; /* block address, APFS_BUF_EMPTY if unused */@@ -190,6 +197,18 @@ struct apfs_nx {uint32_t hand;uint32_t fetches; /* blocks read into the cache, ever (wraps) */uint8_t *cluster; /* APFS_META_CLUSTER blocks, made at the first miss */+ /*+ * With a caller's lock (unlock/relock set), a block of metadata missing from the cache is+ * read with the lock let go of -- other threads go on meanwhile -- into memory of its own,+ * and put in afterwards (unless somebody else put it in first). So that nobody ever reads+ * a cached block's memory while it is given to another block, a block that leaves the+ * cache leaves its memory behind on `retired`, tagged with `epoch`, and the cache slot+ * gets new memory; the caller frees what no operation still running can be looking at+ * (apfs_nx_reclaim, with the oldest epoch any of its operations began at).+ */+ uint64_t epoch;+ struct apfs_retired *retired; /* newest first */+ uint64_t retired_now, retired_most, unlocked_fills, fill_races;uint32_t meta_cluster; /* blocks a metadata read brings in, 1..APFS_META_CLUSTER (0: the default) *//** A request to the device costs next to nothing (a disk in memory): no reading pieces@@ -401,6 +420,12 @@ struct apfs_cfile {int apfs_nx_open(struct apfs_nx *nx, const struct apfs_dev *dev);void apfs_nx_close(struct apfs_nx *nx);+/* Free the retired blocks' memory no operation that began at `oldest` or later can see (lock held) */+void apfs_nx_reclaim(struct apfs_nx *nx, uint64_t oldest);+#ifndef KERNEL+/* For tests on the build machine: every cached block out (retired, with a caller's lock), so all is read again */+void apfs_nx_cache_flush(struct apfs_nx *nx);+#endifint apfs_vol_open(struct apfs_nx *nx, uint32_t index, struct apfs_vol *vol);/* ---- records ---- */diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.cindex 0a60fa7..32cf54d 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c@@ -207,6 +207,10 @@ sysctl_apfs_stats SYSCTL_HANDLER_ARGSam->am_nx.inflate_in = am->am_nx.inflate_out = am->am_nx.inflate_fallbacks = 0;am->am_nx.reads = am->am_nx.hits = 0;am->am_lock_waits = am->am_lock_wait_ticks = am->am_lock_held_ticks = 0;+ memset(am->am_lock_wait_hist, 0, sizeof(am->am_lock_wait_hist));+ am->am_lock_wait_max = 0;+ memset(am->am_lock_hold_hist, 0, sizeof(am->am_lock_hold_hist));+ am->am_lock_hold_max = 0;apfs_unlock(am);}return 0;@@ -264,6 +268,22 @@ apfs_stats_print(void)(unsigned long long)am->am_lock_waits,(unsigned long long)apfs_ticks_ms(am->am_lock_wait_ticks),(unsigned long long)apfs_ticks_ms(am->am_lock_held_ticks));+ {+ uint64_t max_ns;++ absolutetime_to_nanoseconds(am->am_lock_wait_max, &max_ns);+ printf("apfs: lock waits: %llu under 2 us, %llu to 10 us, %llu to 100 us, %llu to 1 ms, %llu longer; "+ "longest %llu us; metadata read from the device %llu ms\n",+ am->am_lock_wait_hist[0], am->am_lock_wait_hist[1], am->am_lock_wait_hist[2], am->am_lock_wait_hist[3],+ am->am_lock_wait_hist[4], max_ns / 1000, (unsigned long long)apfs_ticks_ms(nx->dev_ticks));+ absolutetime_to_nanoseconds(am->am_lock_hold_max, &max_ns);+ printf("apfs: lock holds: %llu under 10 us, %llu to 100 us, %llu to 1 ms, %llu to 10 ms, %llu longer; "+ "longest %llu us\n", am->am_lock_hold_hist[0], am->am_lock_hold_hist[1], am->am_lock_hold_hist[2],+ am->am_lock_hold_hist[3], am->am_lock_hold_hist[4], max_ns / 1000);+ printf("apfs: metadata read without the lock: %llu times, %llu of them already in by another's read; "+ "retired blocks %llu now, %llu at most; operations going on now %u\n", nx->unlocked_fills, nx->fill_races,+ nx->retired_now, nx->retired_most, am->am_nops);+ }}SYSCTL_PROC(_kern, OID_AUTO, apfs_stats, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,@@ -292,19 +312,19 @@ apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,int error;/* already have one? */- lck_mtx_lock(&am->am_lock);+ apfs_lock(am);for (an = am->am_hash[b]; an != NULL; an = an->an_next) {if (an->an_id == obj_id && an->an_attached) {vnode_t vp = an->an_vp;uint32_t vid = vnode_vid(vp);- lck_mtx_unlock(&am->am_lock);+ apfs_unlock(am);if (vnode_getwithvid(vp, vid) == 0) {*vpp = vp;return 0;}/* it was going away under us: fall through and make a new one */- lck_mtx_lock(&am->am_lock);+ apfs_lock(am);break;}}@@ -346,7 +366,7 @@ apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,}}}- lck_mtx_unlock(&am->am_lock);+ apfs_unlock(am);if (error) {return error;}@@ -396,11 +416,11 @@ apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,}vnode_settag(an->an_vp, VT_OTHER);- lck_mtx_lock(&am->am_lock);+ apfs_lock(am);an->an_next = am->am_hash[b];am->am_hash[b] = an;an->an_attached = true;- lck_mtx_unlock(&am->am_lock);+ apfs_unlock(am);*vpp = an->an_vp;return 0;@@ -505,7 +525,7 @@ apfs_sysctl_synthetic SYSCTL_HANDLER_ARGS}dn = APFS_VP(dvp);am = dn->an_mount;- lck_mtx_lock(&am->am_lock);+ apfs_lock(am);if (am->am_synth == NULL) {am->am_synth = kalloc_data(APFS_SYNTH_MAX * sizeof(*am->am_synth), Z_WAITOK | Z_ZERO);}@@ -516,22 +536,27 @@ apfs_sysctl_synthetic SYSCTL_HANDLER_ARGS} else if (am->am_nsynth >= APFS_SYNTH_MAX) {error = ENOSPC;} else {- struct apfs_synth *sy = &am->am_synth[am->am_nsynth];+ uint32_t n0 = am->am_nsynth;+ uint64_t id = (tvp != NULLVP) ? target_id : APFS_SYNTH_BASE + n0;struct apfs_inode ino;- sy->parent = dn->an_id;- sy->alias = (tvp != NULLVP);- sy->id = sy->alias ? target_id : APFS_SYNTH_BASE + am->am_nsynth;- strlcpy(sy->name, name, sizeof(sy->name));- /* an id the volume does not use itself, or nothing */- if (!sy->alias && apfs_inode_get(&am->am_vol, sy->id, &ino) != ENOENT) {+ /* an id the volume does not use itself, or nothing (reading the tree may let go of the lock) */+ if (tvp == NULLVP && apfs_inode_get(&am->am_vol, id, &ino) != ENOENT) {error = EEXIST;+ } else if (am->am_nsynth != n0 || apfs_synth_find(am, dn->an_id, name, len) != NULL) {+ error = EBUSY; /* another was made meanwhile: try again */} else {+ struct apfs_synth *sy = &am->am_synth[am->am_nsynth];++ sy->parent = dn->an_id;+ sy->alias = (tvp != NULLVP);+ sy->id = id;+ strlcpy(sy->name, name, sizeof(sy->name));am->am_nsynth++;error = 0;}}- lck_mtx_unlock(&am->am_lock);+ apfs_unlock(am);/* the parent's entries changed: what the name cache has of it is stale */cache_purge(dvp);vnode_put(dvp);@@ -554,7 +579,7 @@ apfs_node_detach(struct apfs_node *an)uint32_t b = apfs_hash_index(an->an_id);struct apfs_node **pp;- lck_mtx_lock(&am->am_lock);+ apfs_lock(am);for (pp = &am->am_hash[b]; *pp != NULL; pp = &(*pp)->an_next) {if (*pp == an) {*pp = an->an_next;@@ -562,20 +587,20 @@ apfs_node_detach(struct apfs_node *an)}}an->an_attached = false;- lck_mtx_unlock(&am->am_lock);+ apfs_unlock(am);}/* apfs_lib lets go of the mount's lock round decompression, through these */static voidapfs_lib_unlock(void *ctx){- apfs_unlock((struct apfs_mount *)ctx);+ apfs_unlock_inner((struct apfs_mount *)ctx); /* the operation goes on */}static voidapfs_lib_relock(void *ctx){- apfs_lock((struct apfs_mount *)ctx);+ apfs_lock_inner((struct apfs_mount *)ctx);}/* ------------------------------------------------------- threads for one request */@@ -847,6 +872,8 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)uint32_t i;bool got = false;+ /* with the lock, as every call into apfs_lib from now on (it may let go of it) */+ apfs_lock(am);for (i = 0; i < APFS_NX_MAX_FILE_SYSTEMS && !got; i++) {struct apfs_inode ino;@@ -860,6 +887,7 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)got = true;}}+ apfs_unlock(am);if (!got) {printf("apfs: no readable volume with a root directory\n");kfree_type(struct apfs_devctx, dc);diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c b/xnu/files/bsd/miscfs/apfs/apfs_vnops.cindex 53509fb..b4bf69f 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c@@ -26,23 +26,99 @@/* How much is read out of a file at a time when copying into a uio. */void-apfs_lock(struct apfs_mount *am)+apfs_lock_inner(struct apfs_mount *am){- uint64_t t0 = apfs_ticks();+ static uint64_t limits[4];+ uint64_t t0 = apfs_ticks(), waited;+ unsigned k;+ if (limits[3] == 0) {+ nanoseconds_to_absolutetime(2000, &limits[0]);+ nanoseconds_to_absolutetime(10000, &limits[1]);+ nanoseconds_to_absolutetime(100000, &limits[2]);+ nanoseconds_to_absolutetime(1000000, &limits[3]);+ }lck_mtx_lock(&am->am_lock);- OSAddAtomic64((int64_t)(apfs_ticks() - t0), (volatile int64_t *)&am->am_lock_wait_ticks);- OSAddAtomic64(1, (volatile int64_t *)&am->am_lock_waits);+ waited = apfs_ticks() - t0;+ /* (the holder's own: no atomics needed) */+ am->am_lock_wait_ticks += waited;+ am->am_lock_waits++;+ for (k = 0; k < 4 && waited >= limits[k]; k++) {+ }+ am->am_lock_wait_hist[k]++;+ if (waited > am->am_lock_wait_max) {+ am->am_lock_wait_max = waited;+ }am->am_lock_t0 = apfs_ticks();}void-apfs_unlock(struct apfs_mount *am)+apfs_unlock_inner(struct apfs_mount *am){- am->am_lock_held_ticks += apfs_ticks() - am->am_lock_t0;+ static uint64_t limits[4];+ uint64_t held = apfs_ticks() - am->am_lock_t0;+ unsigned k;++ if (limits[3] == 0) {+ nanoseconds_to_absolutetime(10000, &limits[0]);+ nanoseconds_to_absolutetime(100000, &limits[1]);+ nanoseconds_to_absolutetime(1000000, &limits[2]);+ nanoseconds_to_absolutetime(10000000, &limits[3]);+ }+ am->am_lock_held_ticks += held;+ for (k = 0; k < 4 && held >= limits[k]; k++) {+ }+ am->am_lock_hold_hist[k]++;+ if (held > am->am_lock_hold_max) {+ am->am_lock_hold_max = held;+ }lck_mtx_unlock(&am->am_lock);}+/* An operation begins: the lock, and the epoch it began at noted */+void+apfs_lock(struct apfs_mount *am)+{+ apfs_lock_inner(am);+ if (am->am_nops < APFS_OPS_MAX) {+ am->am_ops[am->am_nops].th = current_thread();+ am->am_ops[am->am_nops].epoch = am->am_nx.epoch;+ am->am_nops++;+ } else {+ am->am_untracked++;+ }+}++/* An operation ends: what nobody still going can be reading is freed, and the lock let go of */+void+apfs_unlock(struct apfs_mount *am)+{+ thread_t self = current_thread();+ uint64_t oldest = am->am_nx.epoch;+ uint32_t i;+ bool found = false;++ for (i = 0; i < am->am_nops; i++) {+ if (!found && am->am_ops[i].th == self) {+ am->am_ops[i] = am->am_ops[--am->am_nops];+ found = true;+ if (i == am->am_nops) {+ break;+ }+ }+ if (am->am_ops[i].epoch < oldest) {+ oldest = am->am_ops[i].epoch;+ }+ }+ if (!found && am->am_untracked > 0) {+ am->am_untracked--;+ }+ if (am->am_untracked == 0 && am->am_nx.retired != NULL) {+ apfs_nx_reclaim(&am->am_nx, oldest);+ }+ apfs_unlock_inner(am);+}+/* The inode record of the object a vnode stands for. */static intapfs_vp_inode(vnode_t vp, struct apfs_inode *ino)