Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: metadata read without the volume's lock, so that 8 readers are faster than 4

vcvkke0ec919cf315
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.c
index 60cc31c..bc66d3f 100644
--- a/tools/apfs/apfs-tool.c
+++ b/tools/apfs/apfs-tool.c
@@ -140,6 +140,11 @@ host_parallel(void *ctx, uint32_t items, uint32_t workers,
return atomic_load(&hp.error);
}
+static bool want_lock; /* the library under a lock, as in the kernel (walkers, cfrace) */
+static void host_locking(struct apfs_nx *nx);
+static void op_begin(void);
+static void op_end(void);
+
static int
open_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol)
{
@@ -171,6 +176,14 @@ open_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol)
nx->parallel_max = (uint32_t)(t < 16 ? t : 16);
}
}
+ if (getenv("APFS_TOOL_WALKERS") != NULL && strtol(getenv("APFS_TOOL_WALKERS"), NULL, 0) > 1) {
+ want_lock = true;
+ }
+ if (want_lock) {
+ /* handed over now, as the kernel's mount does; from here on every call is with the lock */
+ host_locking(nx);
+ op_begin();
+ }
/* the first volume that has a root directory, like apfs.py picks it */
for (i = 0; i < APFS_NX_MAX_FILE_SYSTEMS; i++) {
struct apfs_inode ino;
@@ -182,9 +195,15 @@ open_image(const char *path, struct apfs_nx *nx, struct apfs_vol *vol)
continue;
}
if (apfs_inode_get(vol, APFS_ROOT_DIR_INO_NUM, &ino) == 0) {
+ if (want_lock) {
+ op_end();
+ }
return 0;
}
}
+ if (want_lock) {
+ op_end();
+ }
fprintf(stderr, "apfs-tool: no readable volume with a root directory\n");
return 1;
}
@@ -413,6 +432,93 @@ cmd_cat(const char *path, const char *where)
return 0;
}
+/*
+ * ---- several threads on the library at once, as in the kernel ----
+ *
+ * The kernel holds one lock round apfs_lib for each operation and lets apfs_lib let go of it
+ * in the middle (reading file contents, and a block of metadata not in the cache); a block
+ * leaving the cache keeps its memory until no operation that began before can be reading it
+ * (apfs_nx_reclaim). The same here, with a pthread mutex, so that it can be run under
+ * ThreadSanitizer: op_begin/op_end are the kernel's apfs_lock/apfs_unlock.
+ */
+static pthread_mutex_t host_lock; /* error checking: unlocking one not held aborts, as the kernel panics */
+static struct apfs_nx *host_nx;
+
+static void
+host_lock_check(int error, const char *what)
+{
+ if (error != 0) {
+ fprintf(stderr, "apfs-tool: %s the library's lock: %s\n", what, strerror(error));
+ abort();
+ }
+}
+#define HOST_OPS 64
+static struct {
+ pthread_t th;
+ uint64_t epoch;
+ bool used;
+} host_ops[HOST_OPS];
+
+static void
+host_unlock(void *ctx)
+{
+ (void)ctx;
+ host_lock_check(pthread_mutex_unlock(&host_lock), "letting go of");
+}
+
+static void
+host_relock(void *ctx)
+{
+ (void)ctx;
+ host_lock_check(pthread_mutex_lock(&host_lock), "taking");
+}
+
+static void
+op_begin(void)
+{
+ host_lock_check(pthread_mutex_lock(&host_lock), "taking");
+ for (int i = 0; i < HOST_OPS; i++) {
+ if (!host_ops[i].used) {
+ host_ops[i].used = true;
+ host_ops[i].th = pthread_self();
+ host_ops[i].epoch = host_nx->epoch;
+ return;
+ }
+ }
+ fprintf(stderr, "apfs-tool: more operations at once than %d\n", HOST_OPS);
+ abort();
+}
+
+static void
+op_end(void)
+{
+ uint64_t oldest = host_nx->epoch;
+
+ for (int i = 0; i < HOST_OPS; i++) {
+ if (host_ops[i].used && pthread_equal(host_ops[i].th, pthread_self())) {
+ host_ops[i].used = false;
+ } else if (host_ops[i].used && host_ops[i].epoch < oldest) {
+ oldest = host_ops[i].epoch;
+ }
+ }
+ apfs_nx_reclaim(host_nx, oldest);
+ host_lock_check(pthread_mutex_unlock(&host_lock), "letting go of");
+}
+
+static void
+host_locking(struct apfs_nx *nx)
+{
+ pthread_mutexattr_t a;
+
+ pthread_mutexattr_init(&a);
+ pthread_mutexattr_settype(&a, PTHREAD_MUTEX_ERRORCHECK);
+ pthread_mutex_init(&host_lock, &a);
+ host_nx = nx;
+ nx->unlock = host_unlock;
+ nx->relock = host_relock;
+ nx->lock_ctx = NULL;
+}
+
/* ---- census: one line per entry, for diffing against apfs.py ---- */
struct census {
@@ -429,6 +535,8 @@ struct census {
};
static int census_dir(struct census *cs, uint64_t dir_id);
+static int census_parallel(struct apfs_nx *nx, struct apfs_vol *vol, uint64_t start, const char *where,
+ size_t chunk, bool meta_only, int n);
static uint64_t
file_hash(struct census *cs, const struct apfs_inode *ino, uint64_t size)
@@ -537,9 +645,20 @@ cmd_census(const char *path, const char *where, bool meta_only)
if (open_image(path, &nx, &vol) != 0) {
return 1;
}
- if (where != NULL && apfs_path_lookup(&vol, where, &start, &dt) != 0) {
- fprintf(stderr, "apfs-tool: %s: not found\n", where);
- return 1;
+ if (where != NULL) {
+ int e;
+
+ if (want_lock) {
+ op_begin();
+ }
+ e = apfs_path_lookup(&vol, where, &start, &dt);
+ if (want_lock) {
+ op_end();
+ }
+ if (e != 0) {
+ fprintf(stderr, "apfs-tool: %s: not found\n", where);
+ return 1;
+ }
}
memset(&cs, 0, sizeof(cs));
cs.vol = &vol;
@@ -577,6 +696,11 @@ cmd_census(const char *path, const char *where, bool meta_only)
cs.path_len = n;
cs.path[n] = '\0';
}
+ if (getenv("APFS_TOOL_WALKERS") != NULL && strtol(getenv("APFS_TOOL_WALKERS"), NULL, 0) > 1) {
+ long n = strtol(getenv("APFS_TOOL_WALKERS"), NULL, 0);
+
+ return census_parallel(&nx, &vol, start, where, cs.chunk, meta_only, (int)(n > 16 ? 16 : n));
+ }
error = census_dir(&cs, start);
if (error && error != EEXIST) {
fprintf(stderr, "apfs-tool: census: %s\n", strerror(error));
@@ -602,6 +726,361 @@ cmd_census(const char *path, const char *where, bool meta_only)
return cs.error ? 1 : 0;
}
+/*
+ * APFS_TOOL_WALKERS=N with census: N threads walk at once, taking directories off one shared
+ * stack, each line hashed and the hashes added up (order does not matter): the same sum the
+ * kernel's apfs_check prints. Lines are not printed.
+ */
+struct pjob {
+ uint64_t dir_id;
+ char *path;
+};
+
+static pthread_mutex_t pmtx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t pcond = PTHREAD_COND_INITIALIZER;
+static struct pjob *pstack;
+static size_t pcount, pcap;
+static int pbusy;
+
+struct pwalker {
+ struct apfs_vol *vol;
+ const char *dir; /* the directory being walked */
+ uint8_t *buf;
+ size_t chunk;
+ bool meta_only;
+ uint64_t sum, entries, bytes;
+ int error;
+};
+
+static void
+ppush(uint64_t dir_id, const char *path)
+{
+ pthread_mutex_lock(&pmtx);
+ if (pcount == pcap) {
+ pcap = pcap ? 2 * pcap : 1024;
+ pstack = realloc(pstack, pcap * sizeof(*pstack));
+ if (pstack == NULL) {
+ abort();
+ }
+ }
+ pstack[pcount].dir_id = dir_id;
+ pstack[pcount].path = strdup(path);
+ pcount++;
+ pthread_cond_signal(&pcond);
+ pthread_mutex_unlock(&pmtx);
+}
+
+static uint64_t
+pfile_hash(struct pwalker *w, const struct apfs_inode *ino, uint64_t size, const char *path)
+{
+ uint64_t h = FNV_INIT, off = 0;
+ struct apfs_cfile cf;
+
+ memset(&cf, 0, sizeof(cf));
+ cf.held = APFS_NO_PIECE;
+ for (;;) {
+ size_t got = 0;
+ int error = apfs_read_file(w->vol, ino, &cf, off, w->buf, w->chunk, &got);
+
+ if (error) {
+ fprintf(stderr, "apfs-tool: %s: read at %" PRIu64 ": %s\n", path, off, strerror(error));
+ w->error = error;
+ break;
+ }
+ if (got == 0) {
+ break;
+ }
+ h = fnv1a(w->buf, got, h);
+ off += got;
+ w->bytes += got;
+ if (off >= size) {
+ break;
+ }
+ }
+ apfs_cfile_fini(&cf);
+ return h;
+}
+
+static int
+pcensus_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)
+{
+ struct pwalker *w = ctx;
+ struct apfs_inode ino;
+ uint64_t h = 0, size = 0;
+ char path[4096], line[4096 + 128];
+ int n;
+
+ n = snprintf(path, sizeof(path), "%s/%.*s", w->dir, (int)name_len, name);
+ if (n < 0 || (size_t)n >= sizeof(path)) {
+ w->error = ENAMETOOLONG;
+ return 0;
+ }
+ if (apfs_inode_get(w->vol, file_id, &ino) != 0) {
+ fprintf(stderr, "apfs-tool: %s: no inode\n", path);
+ w->error = ENOENT;
+ return 0;
+ }
+ if (apfs_inode_size(w->vol, &ino, &size) != 0) {
+ size = ino.size;
+ }
+ if (!w->meta_only && ((ino.mode & APFS_S_IFMT) == APFS_S_IFREG || (ino.mode & APFS_S_IFMT) == APFS_S_IFLNK)) {
+ h = pfile_hash(w, &ino, size, path);
+ }
+ n = snprintf(line, sizeof(line), "%s\t%u\t%06o\t%u\t%u\t%" PRIu64 "\t%016" PRIx64 "\n", path, d_type,
+ ino.mode, ino.owner, ino.group, size, h);
+ w->sum += fnv1a((const uint8_t *)line, (size_t)n, FNV_INIT);
+ w->entries++;
+ if (d_type == APFS_DT_DIR) {
+ ppush(file_id, path);
+ }
+ return 0;
+}
+
+static void *
+pwalker_thread(void *p)
+{
+ struct pwalker *w = p;
+
+ for (;;) {
+ struct pjob j;
+ int error;
+
+ pthread_mutex_lock(&pmtx);
+ while (pcount == 0 && pbusy > 0) {
+ pthread_cond_wait(&pcond, &pmtx);
+ }
+ if (pcount == 0) {
+ pthread_cond_broadcast(&pcond);
+ pthread_mutex_unlock(&pmtx);
+ return NULL;
+ }
+ j = pstack[--pcount];
+ pbusy++;
+ pthread_mutex_unlock(&pmtx);
+
+ w->dir = j.path;
+ op_begin();
+ error = apfs_readdir(w->vol, j.dir_id, pcensus_cb, w);
+ op_end();
+ if (error) {
+ fprintf(stderr, "apfs-tool: %s: readdir: %s\n", j.path, strerror(error));
+ w->error = error;
+ }
+ free(j.path);
+
+ pthread_mutex_lock(&pmtx);
+ pbusy--;
+ pthread_cond_broadcast(&pcond);
+ pthread_mutex_unlock(&pmtx);
+ }
+}
+
+static int
+census_parallel(struct apfs_nx *nx, struct apfs_vol *vol, uint64_t start, const char *where, size_t chunk,
+ bool meta_only, int n)
+{
+ struct pwalker w[16];
+ pthread_t th[16];
+ uint64_t sum = 0, entries = 0, bytes = 0;
+ int error = 0;
+
+ ppush(start, where != NULL ? where : "");
+ for (int i = 0; i < n; i++) {
+ memset(&w[i], 0, sizeof(w[i]));
+ w[i].vol = vol;
+ w[i].chunk = chunk;
+ w[i].meta_only = meta_only;
+ w[i].buf = malloc(chunk);
+ if (w[i].buf == NULL || pthread_create(&th[i], NULL, pwalker_thread, &w[i]) != 0) {
+ abort();
+ }
+ }
+ for (int i = 0; i < n; i++) {
+ pthread_join(th[i], NULL);
+ sum += w[i].sum;
+ entries += w[i].entries;
+ bytes += w[i].bytes;
+ if (w[i].error) {
+ error = w[i].error;
+ }
+ free(w[i].buf);
+ }
+ fprintf(stderr, "apfs-tool: %d walkers: %" PRIu64 " entries, %" PRIu64 " content bytes, line-hash sum %016" PRIx64
+ ", %s\n", n, entries, bytes, sum, error ? "ERRORS" : "no errors");
+ fprintf(stderr, "apfs-tool: metadata read without the lock %" PRIu64 " times, %" PRIu64 " of them already in; "
+ "retired blocks %" PRIu64 " now, %" PRIu64 " at most\n", nx->unlocked_fills, nx->fill_races, nx->retired_now,
+ nx->retired_most);
+ free(pstack);
+ {
+ int fd = (int)(intptr_t)nx->dev.ctx;
+
+ apfs_nx_close(nx);
+ close(fd);
+ }
+ return error ? 1 : 0;
+}
+
+/*
+ * cfrace IMAGE N ROUNDS: the compressed files of the volume (up to 400, taken from a walk),
+ * each read whole by N threads at once through one shared apfs_cfile -- as two readers of one
+ * vnode share it in the kernel -- with the cache emptied before every round, so that setting
+ * up the file and its read-ahead let go of the lock in the middle; every read must hash the
+ * same as a read on one thread.
+ */
+struct cf_file {
+ uint64_t id;
+ struct apfs_inode ino;
+ uint64_t size;
+ uint64_t want; /* the hash on one thread */
+ struct apfs_cfile cf; /* shared by the threads, touched with the lock held */
+};
+
+static struct cf_file *cf_files;
+static size_t cf_nfiles;
+static struct apfs_vol *cf_vol;
+static atomic_uint cf_bad, cf_reads;
+
+static int
+cf_collect_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)
+{
+ struct apfs_inode ino;
+ struct apfs_cfile cf;
+ uint64_t size;
+
+ (void)ctx;
+ (void)name;
+ (void)name_len;
+ if (cf_nfiles >= 400) {
+ return EEXIST;
+ }
+ if (d_type == APFS_DT_DIR) {
+ return apfs_readdir(cf_vol, file_id, cf_collect_cb, NULL) == EEXIST ? EEXIST : 0;
+ }
+ if (d_type != APFS_DT_REG || apfs_inode_get(cf_vol, file_id, &ino) != 0 ||
+ apfs_inode_size(cf_vol, &ino, &size) != 0 || size < 3 * 65536) {
+ return 0;
+ }
+ memset(&cf, 0, sizeof(cf));
+ cf.held = APFS_NO_PIECE;
+ if (apfs_cfile_init(cf_vol, &ino, &cf) == 0 && cf.compressed && cf.in_rsrc) {
+ cf_files[cf_nfiles].id = file_id;
+ cf_files[cf_nfiles].ino = ino;
+ cf_files[cf_nfiles].size = size;
+ cf_nfiles++;
+ }
+ apfs_cfile_fini(&cf);
+ return 0;
+}
+
+static uint64_t
+cf_read_whole(struct cf_file *f, struct apfs_cfile *cf, uint8_t *buf, size_t chunk, bool locked)
+{
+ uint64_t h = FNV_INIT, off = 0;
+
+ while (off < f->size) {
+ size_t got = 0;
+ int error;
+
+ if (locked) {
+ op_begin();
+ }
+ error = apfs_read_file(cf_vol, &f->ino, cf, off, buf, chunk, &got);
+ if (locked) {
+ op_end();
+ }
+ if (error || got == 0) {
+ return 0;
+ }
+ h = fnv1a(buf, got, h);
+ off += got;
+ }
+ return h;
+}
+
+static void *
+cf_thread(void *p)
+{
+ uint8_t *buf = malloc(65536);
+
+ (void)p;
+ for (size_t i = 0; i < cf_nfiles; i++) {
+ uint64_t h = cf_read_whole(&cf_files[i], &cf_files[i].cf, buf, 65536, true);
+
+ atomic_fetch_add(&cf_reads, 1);
+ if (h != cf_files[i].want) {
+ atomic_fetch_add(&cf_bad, 1);
+ fprintf(stderr, "apfs-tool: cfrace: inode %" PRIu64 " read DIFFERENTLY\n", cf_files[i].id);
+ }
+ }
+ free(buf);
+ return NULL;
+}
+
+static int
+cmd_cfrace(const char *path, int n, int rounds)
+{
+ struct apfs_nx nx;
+ struct apfs_vol vol;
+ pthread_t th[16];
+ uint8_t *buf = malloc(65536);
+
+ want_lock = true;
+ if (open_image(path, &nx, &vol) != 0 || buf == NULL || n < 2 || n > 16) {
+ return 1;
+ }
+ cf_vol = &vol;
+ cf_files = calloc(400, sizeof(*cf_files));
+ op_begin();
+ (void)apfs_readdir(&vol, APFS_ROOT_DIR_INO_NUM, cf_collect_cb, NULL);
+ op_end();
+ for (size_t i = 0; i < cf_nfiles; i++) {
+ struct apfs_cfile cf;
+
+ memset(&cf, 0, sizeof(cf));
+ cf.held = APFS_NO_PIECE;
+ cf_files[i].want = cf_read_whole(&cf_files[i], &cf, buf, 65536, true);
+ op_begin();
+ apfs_cfile_fini(&cf);
+ op_end();
+ }
+ fprintf(stderr, "apfs-tool: cfrace: %zu compressed files of 192 KB or more, %d threads, %d rounds\n", cf_nfiles,
+ n, rounds);
+ if (nx.parallel == NULL) {
+ nx.parallel = host_parallel; /* read-ahead needs helpers to spare */
+ nx.parallel_max = 4;
+ }
+ for (int r = 0; r < rounds; r++) {
+ pthread_mutex_lock(&host_lock);
+ for (size_t i = 0; i < cf_nfiles; i++) {
+ apfs_cfile_fini(&cf_files[i].cf);
+ }
+ apfs_nx_cache_flush(&nx);
+ pthread_mutex_unlock(&host_lock);
+ for (int t = 0; t < n; t++) {
+ pthread_create(&th[t], NULL, cf_thread, NULL);
+ }
+ for (int t = 0; t < n; t++) {
+ pthread_join(th[t], NULL);
+ }
+ }
+ fprintf(stderr, "apfs-tool: cfrace: %u reads, %u different; metadata read without the lock %" PRIu64
+ " times, %" PRIu64 " of them already in; read ahead %" PRIu64 " times\n", atomic_load(&cf_reads),
+ atomic_load(&cf_bad), nx.unlocked_fills, nx.fill_races, nx.ra_batches);
+ /* all given back, so that a leak checker sees only what the library lost */
+ for (size_t i = 0; i < cf_nfiles; i++) {
+ apfs_cfile_fini(&cf_files[i].cf);
+ }
+ free(cf_files);
+ free(buf);
+ {
+ int fd = (int)(intptr_t)nx.dev.ctx;
+
+ apfs_nx_close(&nx);
+ close(fd);
+ }
+ return atomic_load(&cf_bad) != 0;
+}
+
/*
* Every name in the volume looked up by name, the way the kernel's vnop_lookup does it,
* against what reading its directory gave; and each name with "#" after it, which must
@@ -768,7 +1247,8 @@ main(int argc, char **argv)
" apfs-tool stat IMAGE PATH\n"
" apfs-tool cat IMAGE PATH\n"
" apfs-tool census IMAGE [PATH] [--meta]\n"
- " apfs-tool lookups IMAGE\n");
+ " apfs-tool lookups IMAGE\n"
+ " apfs-tool cfrace IMAGE [THREADS [ROUNDS]]\n");
return 2;
}
if (strcmp(argv[1], "info") == 0) {
@@ -790,6 +1270,9 @@ main(int argc, char **argv)
if (strcmp(argv[1], "lookups") == 0) {
return cmd_lookups(argv[2]);
}
+ if (strcmp(argv[1], "cfrace") == 0) {
+ return cmd_cfrace(argv[2], argc > 3 ? atoi(argv[3]) : 4, argc > 4 ? atoi(argv[4]) : 3);
+ }
fprintf(stderr, "apfs-tool: unknown command %s\n", argv[1]);
return 2;
}
diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.c
index 04f254c..4d27974 100644
--- a/xnu/files/bsd/dev/redwood_apfs_check.c
+++ b/xnu/files/bsd/dev/redwood_apfs_check.c
@@ -190,6 +190,9 @@ struct census {
bool no_contents;
/* apfs_walkers=N: one of N walkers; a subdirectory is left on the shared stack (par_push) */
bool par;
+ /* where a walker's time goes (mach ticks): lookups, getattr, reading files and links
+ * (hashing included), readdir, waiting for a directory to walk (par); on a processor */
+ uint64_t t_lookup, t_attr, t_read, t_readdir, t_idle, t_cpu, t_wall;
};
/* The boot-args both kinds of walk take the same way. */
@@ -295,7 +298,12 @@ census_entry(struct census *cs, uint8_t d_type)
* path leads from the ramfs that holds the mount point into the volume itself.
*/
PHASE("lookup");
- error = vnode_lookup(cs->path, cs->lookup_flags, &vp, cs->ctx);
+ {
+ uint64_t t0 = mach_absolute_time();
+
+ error = vnode_lookup(cs->path, cs->lookup_flags, &vp, cs->ctx);
+ cs->t_lookup += mach_absolute_time() - t0;
+ }
if (error) {
printf("apfs-check: %s: lookup failed: %d\n", cs->path + cs->rel, error);
cs->errors++;
@@ -311,7 +319,12 @@ census_entry(struct census *cs, uint8_t d_type)
VATTR_WANTED(&va, va_gid);
VATTR_WANTED(&va, va_data_size);
PHASE("getattr");
- error = vnode_getattr(vp, &va, cs->ctx);
+ {
+ uint64_t t0 = mach_absolute_time();
+
+ error = vnode_getattr(vp, &va, cs->ctx);
+ cs->t_attr += mach_absolute_time() - t0;
+ }
if (error) {
printf("apfs-check: %s: getattr failed: %d\n", cs->path + cs->rel, error);
cs->errors++;
@@ -325,13 +338,19 @@ census_entry(struct census *cs, uint8_t d_type)
cs->files++;
PHASE("read");
if (!cs->no_contents) {
+ uint64_t t0 = mach_absolute_time();
+
error = hash_file(cs, vp, size, &h);
+ cs->t_read += mach_absolute_time() - t0;
}
} else if (vnode_vtype(vp) == VLNK) {
cs->links++;
PHASE("readlink");
if (!cs->no_contents) {
+ uint64_t t0 = mach_absolute_time();
+
error = hash_link(cs, vp, &h);
+ cs->t_read += mach_absolute_time() - t0;
}
} else if (vnode_vtype(vp) == VDIR) {
cs->dirs++;
@@ -520,8 +539,13 @@ census_dir(struct census *cs)
off_t offset = 0;
int error;
- error = vnode_lookup(cs->len != 0 ? cs->path : "/", cs->lookup_flags, &dvp,
- cs->ctx);
+ {
+ uint64_t t0 = mach_absolute_time();
+
+ error = vnode_lookup(cs->len != 0 ? cs->path : "/", cs->lookup_flags, &dvp,
+ cs->ctx);
+ cs->t_lookup += mach_absolute_time() - t0;
+ }
if (error == 0 && cs->mp == NULL) {
cs->mp = vnode_mount(dvp);
}
@@ -552,7 +576,12 @@ census_dir(struct census *cs)
}
uio_addiov(uio, CAST_USER_ADDR_T(dbuf), DIR_CHUNK);
PHASE("readdir");
- error = VNOP_READDIR(dvp, uio, 0, &eof, &items, cs->ctx);
+ {
+ uint64_t t0 = mach_absolute_time();
+
+ error = VNOP_READDIR(dvp, uio, 0, &eof, &items, cs->ctx);
+ cs->t_readdir += mach_absolute_time() - t0;
+ }
if (error) {
printf("apfs-check: %s: readdir failed: %d\n", cs->path + cs->rel, error);
cs->errors++;
@@ -651,16 +680,21 @@ static void
par_walker(void *arg, __unused wait_result_t wr)
{
struct census *cs = arg;
+ uint64_t cpu0 = thread_get_runtime_self(), wall0 = mach_absolute_time();
for (;;) {
struct par_dir d;
+ uint64_t t0 = mach_absolute_time();
lck_mtx_lock(&par_lock);
while (par_count == 0 && par_busy > 0) {
msleep(&par_stack, &par_lock, PRIBIO, "apfs-walkers", NULL);
}
+ cs->t_idle += mach_absolute_time() - t0;
if (par_count == 0) {
/* nothing left, and no walker that could leave more */
+ cs->t_cpu = thread_get_runtime_self() - cpu0;
+ cs->t_wall = mach_absolute_time() - wall0;
par_finished++;
wakeup(&par_stack);
lck_mtx_unlock(&par_lock);
@@ -685,6 +719,29 @@ par_walker(void *arg, __unused wait_result_t wr)
}
}
+/* where a walker's (or, summed, all walkers') time went */
+static void
+census_times(const struct census *cs, int who)
+{
+ uint64_t v[8] = { cs->t_wall, cs->t_cpu, cs->t_lookup, cs->t_attr, cs->t_read, cs->hash_ticks, cs->t_readdir,
+ cs->t_idle };
+
+ for (int i = 0; i < 8; i++) {
+ absolutetime_to_nanoseconds(v[i], &v[i]);
+ v[i] /= 1000000;
+ }
+ char name[24];
+
+ if (who < 0) {
+ strlcpy(name, "all walkers", sizeof(name));
+ } else {
+ snprintf(name, sizeof(name), "walker %d", who);
+ }
+ printf("apfs-check: %s: %llu entries, %llu bytes: %llu ms, on a processor %llu; lookups %llu, getattr %llu, "
+ "reading %llu (of it hashing %llu), readdir %llu, waiting for a directory %llu\n",
+ name, cs->entries, cs->bytes, v[0], v[1], v[2], v[3], v[4], v[5], v[6], v[7]);
+}
+
/* The whole walk on n threads, its totals into cs */
static void
par_walk(struct census *cs, uint32_t n)
@@ -743,7 +800,14 @@ par_walk(struct census *cs, uint32_t n)
cs->bytes += w[i]->bytes;
cs->errors += w[i]->errors;
cs->hash_ticks += w[i]->hash_ticks;
- printf("apfs-check: walker %u: %llu entries, %llu bytes\n", i, w[i]->entries, w[i]->bytes);
+ census_times(w[i], i);
+ cs->t_lookup += w[i]->t_lookup;
+ cs->t_attr += w[i]->t_attr;
+ cs->t_read += w[i]->t_read;
+ cs->t_readdir += w[i]->t_readdir;
+ cs->t_idle += w[i]->t_idle;
+ cs->t_cpu += w[i]->t_cpu;
+ cs->t_wall += w[i]->t_wall;
kfree_data(w[i]->path, PATH_CAP);
kfree_data(w[i]->buf, READ_CHUNK);
kfree_type(struct census, w[i]);
@@ -752,6 +816,7 @@ par_walk(struct census *cs, uint32_t n)
printf("apfs-check: the walkers ran out of memory for their stack: the walk is NOT whole\n");
cs->errors++;
}
+ census_times(cs, -1);
printf("apfs-check: at most %u directories waited on the walkers' stack\n", par_most);
if (par_stack != NULL) {
kfree_data(par_stack, par_cap * sizeof(*par_stack));
@@ -1101,7 +1166,12 @@ walk:
if (mode == 6 && walkers > 1) {
par_walk(&cs, walkers > PAR_MAX ? PAR_MAX : walkers);
} else {
+ uint64_t cpu0 = thread_get_runtime_self(), wall0 = mach_absolute_time();
+
census_dir(&cs);
+ cs.t_cpu = thread_get_runtime_self() - cpu0;
+ cs.t_wall = mach_absolute_time() - wall0;
+ census_times(&cs, 0);
}
}
watch_stop();
diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.h
index d102904..28dbb10 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs.h
@@ -83,6 +83,24 @@ struct apfs_mount {
uint64_t am_lock_wait_ticks;
uint64_t am_lock_held_ticks;
uint64_t am_lock_t0; /* when the holder took it; only the holder writes */
+ /* how long takes waited: up to 2 us, 10 us, 100 us, 1 ms, more; and the longest (the holder writes) */
+ uint64_t am_lock_wait_hist[5];
+ uint64_t am_lock_wait_max;
+ /*
+ * The operations going on (taken apfs_lock, not yet apfs_unlock; apfs_lib may let go of
+ * the lock in between), and the container's epoch each began at: retired cache memory
+ * older than the oldest of them is freed (apfs_nx_reclaim). Under am_lock.
+ */
+#define APFS_OPS_MAX 128
+ struct {
+ thread_t th;
+ uint64_t epoch;
+ } am_ops[APFS_OPS_MAX];
+ uint32_t am_nops;
+ uint32_t am_untracked; /* operations the table had no room for: nothing is freed meanwhile */
+ /* how long it was held: up to 10 us, 100 us, 1 ms, 10 ms, more; and the longest */
+ uint64_t am_lock_hold_hist[5];
+ uint64_t am_lock_hold_max;
/*
* vnop_read's buffers, kept for the next read rather than allocated and freed for
* every one (at that size the kernel goes to the VM). Under am_lock.
@@ -145,8 +163,10 @@ void apfs_node_detach(struct apfs_node *an);
/* the breakdown of where the time went, straight to the log (apfs_vfsops.c) */
void apfs_stats_print(void);
-void apfs_lock(struct apfs_mount *am);
-void apfs_unlock(struct apfs_mount *am);
+void apfs_lock(struct apfs_mount *am); /* an operation begins */
+void apfs_unlock(struct apfs_mount *am); /* and ends */
+void apfs_lock_inner(struct apfs_mount *am); /* the lock taken back in the middle of one (apfs_lib) */
+void apfs_unlock_inner(struct apfs_mount *am); /* let go of in the middle of one */
/* apfs_vnops.c */
extern const struct vnodeopv_desc apfs_vnodeop_opv_desc;
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index 8b86354..4c652d0 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -116,6 +116,67 @@ cache_bucket(uint64_t addr)
return (uint32_t)((addr * 0x9E3779B97F4A7C15ULL) >> 40) & (APFS_CACHE_HASH - 1);
}
+/* A cached block's memory leaves the cache; it is freed by apfs_nx_reclaim once nobody can be reading it. */
+static void
+cache_retire(struct apfs_nx *nx, uint8_t *data)
+{
+ struct apfs_retired *r = apfs_alloc(sizeof(*r));
+
+ if (r == NULL) {
+ return; /* never freed then: that is safe, merely a block lost */
+ }
+ r->data = data;
+ r->epoch = nx->epoch++;
+ r->next = nx->retired;
+ nx->retired = r;
+ if (++nx->retired_now > nx->retired_most) {
+ nx->retired_most = nx->retired_now;
+ }
+}
+
+void
+apfs_nx_reclaim(struct apfs_nx *nx, uint64_t oldest)
+{
+ struct apfs_retired **pp = &nx->retired, *r;
+
+ /* newest first: past the first one old enough, all are */
+ while (*pp != NULL && (*pp)->epoch >= oldest) {
+ pp = &(*pp)->next;
+ }
+ while ((r = *pp) != NULL) {
+ *pp = r->next;
+ apfs_free(r->data, nx->block_size);
+ apfs_free(r, sizeof(*r));
+ nx->retired_now--;
+ }
+}
+
+#ifndef KERNEL
+void
+apfs_nx_cache_flush(struct apfs_nx *nx)
+{
+ uint32_t i;
+
+ for (i = 0; i < APFS_CACHE_BLOCKS; i++) {
+ struct apfs_buf *b = &nx->cache[i];
+
+ if (b->data != NULL && nx->unlock != NULL) {
+ cache_retire(nx, b->data);
+ b->data = NULL;
+ }
+ b->addr = APFS_BUF_EMPTY;
+ b->next = -1;
+ b->ref = false;
+ }
+ for (i = 0; i < APFS_CACHE_HASH; i++) {
+ nx->cache_hash[i] = -1;
+ }
+ if (nx->omap_cache != NULL) {
+ memset(nx->omap_cache, 0, APFS_OMAP_CACHE * sizeof(*nx->omap_cache));
+ }
+}
+#endif
+
/* A free slot, or the first the clock hand finds neither used since it last passed nor recent. */
static struct apfs_buf *
cache_victim(struct apfs_nx *nx)
@@ -151,6 +212,16 @@ cache_victim(struct apfs_nx *nx)
break;
}
}
+ if (nx->unlock != NULL && victim->data != NULL && victim->addr != APFS_BUF_EMPTY) {
+ /* somebody may still be reading the block that had it: new memory for the new one */
+ uint8_t *fresh = apfs_alloc(nx->block_size);
+
+ if (fresh == NULL) {
+ return NULL;
+ }
+ cache_retire(nx, victim->data);
+ victim->data = fresh;
+ }
if (victim->addr != APFS_BUF_EMPTY) {
/* out of its old chain */
for (pp = &nx->cache_hash[cache_bucket(victim->addr)]; *pp >= 0;
@@ -189,6 +260,96 @@ cache_find(struct apfs_nx *nx, uint64_t addr)
return NULL;
}
+static inline void nx_unlock(struct apfs_nx *nx);
+static inline void nx_relock(struct apfs_nx *nx);
+
+/*
+ * A block of metadata not in the cache, read without the lock: the block and its aligned
+ * neighbours into memory of this call's own, the lock taken back, and put in -- unless
+ * somebody put the block in meanwhile, whose copy is used then (and only the neighbours not
+ * there yet go in). Cache slots that get a block get new memory (cache_victim), so nothing
+ * anyone still reads is overwritten.
+ */
+static int
+nx_block_unlocked(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)
+{
+ uint32_t bs = nx->block_size;
+ uint32_t k = nx->meta_cluster != 0 ?
+ (nx->meta_cluster > APFS_META_CLUSTER ? APFS_META_CLUSTER : nx->meta_cluster) : APFS_META_CLUSTER_DEFAULT;
+ uint64_t first = addr - addr % k, count = nx->block_count - first < k ? nx->block_count - first : k, i, t0, dt;
+ struct apfs_buf *b;
+ uint8_t *mine = apfs_alloc((size_t)count * bs);
+ int error;
+
+ if (mine == NULL) {
+ return ENOMEM;
+ }
+ nx->unlocked_fills++;
+ nx_unlock(nx);
+ t0 = apfs_ticks();
+ error = nx->dev.read_data(nx->dev.ctx, first * (uint64_t)bs, mine, (size_t)count * bs);
+ dt = apfs_ticks() - t0;
+ nx_relock(nx);
+ nx->dev_ticks += dt;
+ nx->dev_calls++;
+ nx->dev_bytes += count * bs;
+ if (error) {
+ apfs_free(mine, (size_t)count * bs);
+ return error;
+ }
+ nx->fetches++;
+ b = cache_find(nx, addr);
+ if (b != NULL) {
+ nx->fill_races++; /* in already, by somebody else's read */
+ if (verify && !b->checked) {
+ if (!apfs_checksum_ok(b->data, bs)) {
+ apfs_log("block %llu: bad checksum\n", (unsigned long long)addr);
+ apfs_free(mine, (size_t)count * bs);
+ return EIO;
+ }
+ b->checked = true;
+ }
+ } else {
+ b = cache_victim(nx);
+ if (b == NULL) {
+ apfs_free(mine, (size_t)count * bs);
+ return ENOMEM;
+ }
+ memcpy(b->data, mine + (addr - first) * bs, bs);
+ nx->reads++;
+ if (verify && !apfs_checksum_ok(b->data, bs)) {
+ apfs_log("block %llu: bad checksum\n", (unsigned long long)addr);
+ apfs_free(mine, (size_t)count * bs);
+ return EIO; /* (the slot stays empty: cache_victim took it out of its chain) */
+ }
+ b->checked = verify;
+ cache_link(nx, b, addr);
+ }
+ b->ref = true;
+ b->used = nx->fetches;
+ /* the neighbours not there yet: kept as if long unused, checked when asked for */
+ for (i = first; i < first + count; i++) {
+ struct apfs_buf *nb;
+
+ if (i == addr || i == 0 || cache_find(nx, i) != NULL) {
+ continue;
+ }
+ nb = cache_victim(nx);
+ if (nb == NULL) {
+ break;
+ }
+ memcpy(nb->data, mine + (i - first) * bs, bs);
+ nb->checked = false;
+ nb->ref = false;
+ nb->used = nx->fetches - APFS_CACHE_RECENT;
+ cache_link(nx, nb, i);
+ nx->prefetched++;
+ }
+ apfs_free(mine, (size_t)count * bs);
+ *out = b->data;
+ return 0;
+}
+
static int
nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)
{
@@ -221,6 +382,9 @@ nx_block(struct apfs_nx *nx, uint64_t addr, const uint8_t **out, bool verify)
*out = b->data;
return 0;
}
+ if (nx->unlock != NULL && nx->dev.read_data != NULL && !nx->cheap_requests) {
+ return nx_block_unlocked(nx, addr, out, verify);
+ }
victim = cache_victim(nx);
if (victim == NULL) {
return ENOMEM;
@@ -872,6 +1036,7 @@ apfs_nx_close(struct apfs_nx *nx)
apfs_free(nx->cluster, (size_t)APFS_META_CLUSTER * nx->block_size);
nx->cluster = NULL;
}
+ apfs_nx_reclaim(nx, ~0ULL);
}
int
@@ -2169,16 +2334,43 @@ out:
return error;
}
+static int cfile_setup(struct apfs_vol *vol, const struct apfs_inode *ino, struct apfs_cfile *cf);
+
+/*
+ * Worked out into a copy of its own and only then put in place: working it out reads the
+ * tree, and a read can let go of the lock (nx_block_unlocked), so another reader of the same
+ * file may be doing the same meanwhile -- whoever finishes second throws its copy away.
+ */
int
apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,
struct apfs_cfile *cf)
{
+ struct apfs_cfile mine;
int error;
if (cf->ready && cf->obj_id == ino->obj_id) {
return 0; /* already worked out, with its table and piece */
}
+ memset(&mine, 0, sizeof(mine));
+ mine.held = APFS_NO_PIECE;
+ error = cfile_setup(vol, ino, &mine);
+ if (error) {
+ return error; /* (cfile_setup gave back what it had) */
+ }
+ if (cf->ready && cf->obj_id == ino->obj_id) {
+ apfs_cfile_fini(&mine);
+ return 0;
+ }
apfs_cfile_fini(cf);
+ *cf = mine;
+ return 0;
+}
+
+static int
+cfile_setup(struct apfs_vol *vol, const struct apfs_inode *ino, struct apfs_cfile *cf)
+{
+ int error;
+
cf->obj_id = ino->obj_id;
cf->held = APFS_NO_PIECE;
@@ -3000,6 +3192,11 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
vol->nx->requests : 1) >= 2) {
uint64_t k;
+ /*
+ * Claimed before the pieces are planned: planning reads the tree,
+ * which can let go of the lock, and nobody else may fill ra_data.
+ */
+ cf->ra_filling = true;
if (cf->ra_data == NULL) {
cf->ra_data = apfs_alloc((size_t)APFS_RA_PIECES *
APFS_DECMPFS_BLOCK_SIZE);
@@ -3037,8 +3234,9 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
nra++;
}
if (nra > 0) {
- cf->ra_filling = true;
cf->ra_first = i + 1;
+ } else {
+ cf->ra_filling = false;
}
}
continue;
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
index fb8bb0c..6b149fd 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
@@ -102,6 +102,13 @@ struct apfs_dev {
int (*read_data)(void *ctx, uint64_t off, void *buf, size_t len);
};
+/* A cached block's memory, given up while somebody may still be reading it (nx->retired) */
+struct apfs_retired {
+ struct apfs_retired *next;
+ uint64_t epoch;
+ uint8_t *data;
+};
+
/* One cached block of metadata. */
struct apfs_buf {
uint64_t addr; /* block address, APFS_BUF_EMPTY if unused */
@@ -190,6 +197,18 @@ struct apfs_nx {
uint32_t hand;
uint32_t fetches; /* blocks read into the cache, ever (wraps) */
uint8_t *cluster; /* APFS_META_CLUSTER blocks, made at the first miss */
+ /*
+ * With a caller's lock (unlock/relock set), a block of metadata missing from the cache is
+ * read with the lock let go of -- other threads go on meanwhile -- into memory of its own,
+ * and put in afterwards (unless somebody else put it in first). So that nobody ever reads
+ * a cached block's memory while it is given to another block, a block that leaves the
+ * cache leaves its memory behind on `retired`, tagged with `epoch`, and the cache slot
+ * gets new memory; the caller frees what no operation still running can be looking at
+ * (apfs_nx_reclaim, with the oldest epoch any of its operations began at).
+ */
+ uint64_t epoch;
+ struct apfs_retired *retired; /* newest first */
+ uint64_t retired_now, retired_most, unlocked_fills, fill_races;
uint32_t meta_cluster; /* blocks a metadata read brings in, 1..APFS_META_CLUSTER (0: the default) */
/*
* A request to the device costs next to nothing (a disk in memory): no reading pieces
@@ -401,6 +420,12 @@ struct apfs_cfile {
int apfs_nx_open(struct apfs_nx *nx, const struct apfs_dev *dev);
void apfs_nx_close(struct apfs_nx *nx);
+/* Free the retired blocks' memory no operation that began at `oldest` or later can see (lock held) */
+void apfs_nx_reclaim(struct apfs_nx *nx, uint64_t oldest);
+#ifndef KERNEL
+/* For tests on the build machine: every cached block out (retired, with a caller's lock), so all is read again */
+void apfs_nx_cache_flush(struct apfs_nx *nx);
+#endif
int apfs_vol_open(struct apfs_nx *nx, uint32_t index, struct apfs_vol *vol);
/* ---- records ---- */
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
index 0a60fa7..32cf54d 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
@@ -207,6 +207,10 @@ sysctl_apfs_stats SYSCTL_HANDLER_ARGS
am->am_nx.inflate_in = am->am_nx.inflate_out = am->am_nx.inflate_fallbacks = 0;
am->am_nx.reads = am->am_nx.hits = 0;
am->am_lock_waits = am->am_lock_wait_ticks = am->am_lock_held_ticks = 0;
+ memset(am->am_lock_wait_hist, 0, sizeof(am->am_lock_wait_hist));
+ am->am_lock_wait_max = 0;
+ memset(am->am_lock_hold_hist, 0, sizeof(am->am_lock_hold_hist));
+ am->am_lock_hold_max = 0;
apfs_unlock(am);
}
return 0;
@@ -264,6 +268,22 @@ apfs_stats_print(void)
(unsigned long long)am->am_lock_waits,
(unsigned long long)apfs_ticks_ms(am->am_lock_wait_ticks),
(unsigned long long)apfs_ticks_ms(am->am_lock_held_ticks));
+ {
+ uint64_t max_ns;
+
+ absolutetime_to_nanoseconds(am->am_lock_wait_max, &max_ns);
+ printf("apfs: lock waits: %llu under 2 us, %llu to 10 us, %llu to 100 us, %llu to 1 ms, %llu longer; "
+ "longest %llu us; metadata read from the device %llu ms\n",
+ am->am_lock_wait_hist[0], am->am_lock_wait_hist[1], am->am_lock_wait_hist[2], am->am_lock_wait_hist[3],
+ am->am_lock_wait_hist[4], max_ns / 1000, (unsigned long long)apfs_ticks_ms(nx->dev_ticks));
+ absolutetime_to_nanoseconds(am->am_lock_hold_max, &max_ns);
+ printf("apfs: lock holds: %llu under 10 us, %llu to 100 us, %llu to 1 ms, %llu to 10 ms, %llu longer; "
+ "longest %llu us\n", am->am_lock_hold_hist[0], am->am_lock_hold_hist[1], am->am_lock_hold_hist[2],
+ am->am_lock_hold_hist[3], am->am_lock_hold_hist[4], max_ns / 1000);
+ printf("apfs: metadata read without the lock: %llu times, %llu of them already in by another's read; "
+ "retired blocks %llu now, %llu at most; operations going on now %u\n", nx->unlocked_fills, nx->fill_races,
+ nx->retired_now, nx->retired_most, am->am_nops);
+ }
}
SYSCTL_PROC(_kern, OID_AUTO, apfs_stats, CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_LOCKED,
@@ -292,19 +312,19 @@ apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,
int error;
/* already have one? */
- lck_mtx_lock(&am->am_lock);
+ apfs_lock(am);
for (an = am->am_hash[b]; an != NULL; an = an->an_next) {
if (an->an_id == obj_id && an->an_attached) {
vnode_t vp = an->an_vp;
uint32_t vid = vnode_vid(vp);
- lck_mtx_unlock(&am->am_lock);
+ apfs_unlock(am);
if (vnode_getwithvid(vp, vid) == 0) {
*vpp = vp;
return 0;
}
/* it was going away under us: fall through and make a new one */
- lck_mtx_lock(&am->am_lock);
+ apfs_lock(am);
break;
}
}
@@ -346,7 +366,7 @@ apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,
}
}
}
- lck_mtx_unlock(&am->am_lock);
+ apfs_unlock(am);
if (error) {
return error;
}
@@ -396,11 +416,11 @@ apfs_node_vnode(struct apfs_mount *am, uint64_t obj_id, vnode_t dvp,
}
vnode_settag(an->an_vp, VT_OTHER);
- lck_mtx_lock(&am->am_lock);
+ apfs_lock(am);
an->an_next = am->am_hash[b];
am->am_hash[b] = an;
an->an_attached = true;
- lck_mtx_unlock(&am->am_lock);
+ apfs_unlock(am);
*vpp = an->an_vp;
return 0;
@@ -505,7 +525,7 @@ apfs_sysctl_synthetic SYSCTL_HANDLER_ARGS
}
dn = APFS_VP(dvp);
am = dn->an_mount;
- lck_mtx_lock(&am->am_lock);
+ apfs_lock(am);
if (am->am_synth == NULL) {
am->am_synth = kalloc_data(APFS_SYNTH_MAX * sizeof(*am->am_synth), Z_WAITOK | Z_ZERO);
}
@@ -516,22 +536,27 @@ apfs_sysctl_synthetic SYSCTL_HANDLER_ARGS
} else if (am->am_nsynth >= APFS_SYNTH_MAX) {
error = ENOSPC;
} else {
- struct apfs_synth *sy = &am->am_synth[am->am_nsynth];
+ uint32_t n0 = am->am_nsynth;
+ uint64_t id = (tvp != NULLVP) ? target_id : APFS_SYNTH_BASE + n0;
struct apfs_inode ino;
- sy->parent = dn->an_id;
- sy->alias = (tvp != NULLVP);
- sy->id = sy->alias ? target_id : APFS_SYNTH_BASE + am->am_nsynth;
- strlcpy(sy->name, name, sizeof(sy->name));
- /* an id the volume does not use itself, or nothing */
- if (!sy->alias && apfs_inode_get(&am->am_vol, sy->id, &ino) != ENOENT) {
+ /* an id the volume does not use itself, or nothing (reading the tree may let go of the lock) */
+ if (tvp == NULLVP && apfs_inode_get(&am->am_vol, id, &ino) != ENOENT) {
error = EEXIST;
+ } else if (am->am_nsynth != n0 || apfs_synth_find(am, dn->an_id, name, len) != NULL) {
+ error = EBUSY; /* another was made meanwhile: try again */
} else {
+ struct apfs_synth *sy = &am->am_synth[am->am_nsynth];
+
+ sy->parent = dn->an_id;
+ sy->alias = (tvp != NULLVP);
+ sy->id = id;
+ strlcpy(sy->name, name, sizeof(sy->name));
am->am_nsynth++;
error = 0;
}
}
- lck_mtx_unlock(&am->am_lock);
+ apfs_unlock(am);
/* the parent's entries changed: what the name cache has of it is stale */
cache_purge(dvp);
vnode_put(dvp);
@@ -554,7 +579,7 @@ apfs_node_detach(struct apfs_node *an)
uint32_t b = apfs_hash_index(an->an_id);
struct apfs_node **pp;
- lck_mtx_lock(&am->am_lock);
+ apfs_lock(am);
for (pp = &am->am_hash[b]; *pp != NULL; pp = &(*pp)->an_next) {
if (*pp == an) {
*pp = an->an_next;
@@ -562,20 +587,20 @@ apfs_node_detach(struct apfs_node *an)
}
}
an->an_attached = false;
- lck_mtx_unlock(&am->am_lock);
+ apfs_unlock(am);
}
/* apfs_lib lets go of the mount's lock round decompression, through these */
static void
apfs_lib_unlock(void *ctx)
{
- apfs_unlock((struct apfs_mount *)ctx);
+ apfs_unlock_inner((struct apfs_mount *)ctx); /* the operation goes on */
}
static void
apfs_lib_relock(void *ctx)
{
- apfs_lock((struct apfs_mount *)ctx);
+ apfs_lock_inner((struct apfs_mount *)ctx);
}
/* ------------------------------------------------------- threads for one request */
@@ -847,6 +872,8 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)
uint32_t i;
bool got = false;
+ /* with the lock, as every call into apfs_lib from now on (it may let go of it) */
+ apfs_lock(am);
for (i = 0; i < APFS_NX_MAX_FILE_SYSTEMS && !got; i++) {
struct apfs_inode ino;
@@ -860,6 +887,7 @@ apfs_mount_common(mount_t mp, vnode_t devvp, vfs_context_t ctx)
got = true;
}
}
+ apfs_unlock(am);
if (!got) {
printf("apfs: no readable volume with a root directory\n");
kfree_type(struct apfs_devctx, dc);
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
index 53509fb..b4bf69f 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
@@ -26,23 +26,99 @@
/* How much is read out of a file at a time when copying into a uio. */
void
-apfs_lock(struct apfs_mount *am)
+apfs_lock_inner(struct apfs_mount *am)
{
- uint64_t t0 = apfs_ticks();
+ static uint64_t limits[4];
+ uint64_t t0 = apfs_ticks(), waited;
+ unsigned k;
+ if (limits[3] == 0) {
+ nanoseconds_to_absolutetime(2000, &limits[0]);
+ nanoseconds_to_absolutetime(10000, &limits[1]);
+ nanoseconds_to_absolutetime(100000, &limits[2]);
+ nanoseconds_to_absolutetime(1000000, &limits[3]);
+ }
lck_mtx_lock(&am->am_lock);
- OSAddAtomic64((int64_t)(apfs_ticks() - t0), (volatile int64_t *)&am->am_lock_wait_ticks);
- OSAddAtomic64(1, (volatile int64_t *)&am->am_lock_waits);
+ waited = apfs_ticks() - t0;
+ /* (the holder's own: no atomics needed) */
+ am->am_lock_wait_ticks += waited;
+ am->am_lock_waits++;
+ for (k = 0; k < 4 && waited >= limits[k]; k++) {
+ }
+ am->am_lock_wait_hist[k]++;
+ if (waited > am->am_lock_wait_max) {
+ am->am_lock_wait_max = waited;
+ }
am->am_lock_t0 = apfs_ticks();
}
void
-apfs_unlock(struct apfs_mount *am)
+apfs_unlock_inner(struct apfs_mount *am)
{
- am->am_lock_held_ticks += apfs_ticks() - am->am_lock_t0;
+ static uint64_t limits[4];
+ uint64_t held = apfs_ticks() - am->am_lock_t0;
+ unsigned k;
+
+ if (limits[3] == 0) {
+ nanoseconds_to_absolutetime(10000, &limits[0]);
+ nanoseconds_to_absolutetime(100000, &limits[1]);
+ nanoseconds_to_absolutetime(1000000, &limits[2]);
+ nanoseconds_to_absolutetime(10000000, &limits[3]);
+ }
+ am->am_lock_held_ticks += held;
+ for (k = 0; k < 4 && held >= limits[k]; k++) {
+ }
+ am->am_lock_hold_hist[k]++;
+ if (held > am->am_lock_hold_max) {
+ am->am_lock_hold_max = held;
+ }
lck_mtx_unlock(&am->am_lock);
}
+/* An operation begins: the lock, and the epoch it began at noted */
+void
+apfs_lock(struct apfs_mount *am)
+{
+ apfs_lock_inner(am);
+ if (am->am_nops < APFS_OPS_MAX) {
+ am->am_ops[am->am_nops].th = current_thread();
+ am->am_ops[am->am_nops].epoch = am->am_nx.epoch;
+ am->am_nops++;
+ } else {
+ am->am_untracked++;
+ }
+}
+
+/* An operation ends: what nobody still going can be reading is freed, and the lock let go of */
+void
+apfs_unlock(struct apfs_mount *am)
+{
+ thread_t self = current_thread();
+ uint64_t oldest = am->am_nx.epoch;
+ uint32_t i;
+ bool found = false;
+
+ for (i = 0; i < am->am_nops; i++) {
+ if (!found && am->am_ops[i].th == self) {
+ am->am_ops[i] = am->am_ops[--am->am_nops];
+ found = true;
+ if (i == am->am_nops) {
+ break;
+ }
+ }
+ if (am->am_ops[i].epoch < oldest) {
+ oldest = am->am_ops[i].epoch;
+ }
+ }
+ if (!found && am->am_untracked > 0) {
+ am->am_untracked--;
+ }
+ if (am->am_untracked == 0 && am->am_nx.retired != NULL) {
+ apfs_nx_reclaim(&am->am_nx, oldest);
+ }
+ apfs_unlock_inner(am);
+}
+
/* The inode record of the object a vnode stands for. */
static int
apfs_vp_inode(vnode_t vp, struct apfs_inode *ino)