Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
diff --git a/Makefile b/Makefile
index e78d8476c346..03fc3ff646b8 100644
--- a/Makefile
+++ b/Makefile
@@ -143,8 +143,8 @@ installsrc:
else ifneq ($(findstring xnu_tests,$(RC_ProjectName)),)
-export SYSCTL_HW_PHYSICALCPU := $(shell /usr/sbin/sysctl -n hw.physicalcpu)
-export SYSCTL_HW_LOGICALCPU := $(shell /usr/sbin/sysctl -n hw.logicalcpu)
+export SYSCTL_HW_PHYSICALCPU := $(shell nproc)
+export SYSCTL_HW_LOGICALCPU := $(shell nproc)
MAKEJOBS := --jobs=$(shell expr $(SYSCTL_HW_LOGICALCPU) + 1)
default: install
@@ -160,8 +160,8 @@ installsrc:
else ifeq ($(RC_ProjectName),xnu_tests_driverkit)
-export SYSCTL_HW_PHYSICALCPU := $(shell /usr/sbin/sysctl -n hw.physicalcpu)
-export SYSCTL_HW_LOGICALCPU := $(shell /usr/sbin/sysctl -n hw.logicalcpu)
+export SYSCTL_HW_PHYSICALCPU := $(shell nproc)
+export SYSCTL_HW_LOGICALCPU := $(shell nproc)
MAKEJOBS := --jobs=$(shell expr $(SYSCTL_HW_LOGICALCPU) + 1)
default: install
@@ -193,8 +193,8 @@ endif
# CPUs. If the system does not support SMT, use N+1.
# If MAKEJOBS or -jN is passed on the make line, that takes precedence.
#
-export SYSCTL_HW_PHYSICALCPU := $(shell /usr/sbin/sysctl -n hw.physicalcpu)
-export SYSCTL_HW_LOGICALCPU := $(shell /usr/sbin/sysctl -n hw.logicalcpu)
+export SYSCTL_HW_PHYSICALCPU := $(shell nproc)
+export SYSCTL_HW_LOGICALCPU := $(shell nproc)
MAKEJOBS := --jobs=$(shell expr $(SYSCTL_HW_LOGICALCPU) + 1)
TOP_TARGETS = \
diff --git a/SETUP/config/doconf b/SETUP/config/doconf
index 222158a93918..030e55b82ac1 100755
--- a/SETUP/config/doconf
+++ b/SETUP/config/doconf
@@ -1,4 +1,4 @@
-#!/bin/csh -f
+#!/usr/bin/env -S csh -f
set path = ($path .)
######################################################################
# HISTORY
@@ -184,6 +184,13 @@ if ($?beverbose) then
echo MASTER_CPU=$MASTER_CPU
endif
+# HakiOS: the drivers a configuration builds (CONFIG_HAKIOS_*), as tools/hakios-config.py kernel writes them: a list
+# HAKIOS_DRIVERS of attributes, which the configurations of MASTER.arm64.bcm2837 include, and an `options` line for each
+set MASTER_HAKIOS = /dev/null
+if (-f ${MASTER_DIR}/MASTER.hakios) then
+ set MASTER_HAKIOS = ${MASTER_DIR}/MASTER.hakios
+endif
+
foreach SYS ($argv)
set SYSID=${SYS}_${ID}
set SYSCONF=$OBJDIR/config.$SYSID
@@ -193,8 +200,8 @@ foreach SYS ($argv)
endif
echo +$SYS \
| \
- cat $MASTER $MASTER_CPU - \
- $MASTER $MASTER_CPU \
+ cat $MASTER $MASTER_CPU $MASTER_HAKIOS - \
+ $MASTER $MASTER_CPU $MASTER_HAKIOS \
| \
unifdef -t -DPLATFORM_${PLATFORM} -DCPU_$cpu -DSOC_CONFIG_${SOC_CONFIG} -DSYS_${SYS} $DEF_AVP $DEF_SEED - \
| \
diff --git a/bsd/conf/Makefile b/bsd/conf/Makefile
index 050f34a53c49..c58ff62cd419 100644
--- a/bsd/conf/Makefile
+++ b/bsd/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/bsd/conf/Makefile.arm64 b/bsd/conf/Makefile.arm64
index a4d25040f38f..aad72993d139 100644
--- a/bsd/conf/Makefile.arm64
+++ b/bsd/conf/Makefile.arm64
@@ -5,6 +5,11 @@
# Inline assembly doesn't interact well with LTO
fbt_arm.o_CFLAGS_ADD += $(CFLAGS_NOLTO_FLAG)
+# libdeflate's arm headers (bsd/miscfs/apfs/libdeflate) include the compiler's own <arm_acle.h>
+# and <arm_neon.h>, which the kernel's include paths do not have: they come after those paths,
+# so the kernel's <stdint.h> and the rest are still the ones found.
+apfs_deflate.o_CFLAGS_ADD += -idirafter $(shell $(CC) -print-resource-dir)/include
+
kern_csr.o_INCFLAGS += -I$(SRCROOT)/osfmk
kern_debug.o_INCFLAGS += -I$(SRCROOT)/osfmk
diff --git a/bsd/conf/Makefile.template b/bsd/conf/Makefile.template
index 4b83160780c3..8997af1f0e1b 100644
--- a/bsd/conf/Makefile.template
+++ b/bsd/conf/Makefile.template
@@ -588,5 +588,13 @@ do_build_all:: do_all
%RULES
+# HakiOS: HFS Plus built in (bsd/hfs, Apple's hfs-627.100.6), compiled as its kext was: against the kernel's KPI,
+# not its insides (no XNU/BSD/MACH_KERNEL_PRIVATE), kext-config.h first, its own headers beside it, and its warnings
+# its own (it is Apple's code as it was published, not this kernel's)
+HFS_OBJS = $(patsubst %.c,%.o,$(notdir $(wildcard $(SRCROOT)/bsd/hfs/*.c)))
+$(foreach o,$(HFS_OBJS),$(eval $(o)_CFLAGS_ADD += -UXNU_KERNEL_PRIVATE -UBSD_KERNEL_PRIVATE -UMACH_KERNEL_PRIVATE \
+ -include $(SRCROOT)/bsd/hfs/kext-config.h -I$(SRCROOT)/bsd/hfs -idirafter $(SDKROOT)/usr/include))
+$(foreach o,$(HFS_OBJS),$(eval $(o)_CWARNFLAGS_ADD += -w))
+
include $(MakeInc_rule)
include $(MakeInc_dir)
diff --git a/bsd/conf/files b/bsd/conf/files
index 44274016a4cf..27cccab98c8e 100644
--- a/bsd/conf/files
+++ b/bsd/conf/files
@@ -109,6 +109,51 @@ bsd/dev/munge.c optional config_requires_u32_munging
bsd/dev/unix_startup.c standard
bsd/dev/monotonic.c optional monotonic
+# HakiOS drivers: made by tools/hakios-config.py kernel from drivers/*/conf. Edit those, not these lines.
+bsd/dev/hakios.audio.c standard
+bsd/dev/hakios.version.c standard
+bsd/dev/hakios.platform.c standard
+bsd/dev/hakios.stubs.c standard
+bsd/dev/hakios.input.c standard
+bsd/dev/hakios.net.c standard
+bsd/dev/hakios.image.c standard
+bsd/dev/hakios.sparse.c standard
+bsd/dev/hakios.apfs_check.c optional apfs
+bsd/dev/hakios.adsp.c optional hakios_adsp
+bsd/dev/hakios.speaker.c optional hakios_speaker
+bsd/dev/hakios.sound.c optional hakios_sound
+bsd/dev/hakios.guard.c optional hakios_sound
+bsd/dev/hakios.mic.c optional hakios_mic
+bsd/dev/hakios.virtio_sound.c optional hakios_virtio_sound
+bsd/dev/hakios.bt.c optional hakios_bt
+bsd/dev/hakios.camera.c optional hakios_camera
+bsd/dev/hakios.torch.c optional hakios_torch
+bsd/dev/hakios.socinfo.c optional hakios_socinfo
+bsd/dev/hakios.adreno.c optional hakios_adreno
+bsd/dev/hakios.haptic.c optional hakios_haptic
+bsd/dev/hakios.virtio_input.c optional hakios_virtio_input
+bsd/dev/hakios.goodix.c optional hakios_goodix
+bsd/dev/hakios.buttons.c optional hakios_buttons
+bsd/dev/hakios.qmi.c optional hakios_qmi
+bsd/dev/hakios.mpss.c optional hakios_modem
+bsd/dev/hakios.cell.c optional hakios_modem
+bsd/dev/hakios.nas.c optional hakios_modem
+bsd/dev/hakios.ipa.c optional hakios_modem
+bsd/dev/hakios.virtio_net.c optional hakios_virtio_net
+bsd/dev/hakios.battery.c optional hakios_battery
+bsd/dev/hakios.cpufreq.c optional hakios_cpufreq
+bsd/dev/hakios.thermal.c optional hakios_thermal
+bsd/dev/hakios.pmic.c optional hakios_pmic
+bsd/dev/hakios.battguard.c optional hakios_battguard
+bsd/dev/hakios.ufs.c optional hakios_ufs
+bsd/dev/hakios.ufshc.c optional hakios_ufs
+bsd/dev/hakios.virtio_blk.c optional hakios_virtio_blk
+bsd/dev/hakios.wifi.c optional hakios_wifi
+bsd/dev/hakios.wlfw.c optional hakios_wifi
+bsd/dev/hakios.lx.c optional hakios_wifi
+bsd/dev/hakios.sta.c optional hakios_wifi
+bsd/dev/hakios.apple80211.c optional hakios_wifi
+# end of the HakiOS drivers
bsd/dev/dev_perfmon.c optional config_perfmon
bsd/libkern/crc32.c standard
@@ -155,6 +200,52 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfs
bsd/miscfs/mockfs/mockfs_fsnode.c optional mockfs
bsd/miscfs/mockfs/mockfs_vfsops.c optional mockfs
bsd/miscfs/mockfs/mockfs_vnops.c optional mockfs
+bsd/miscfs/apfs/apfs_deflate.c optional apfs
+bsd/miscfs/apfs/apfs_zbitmap.c optional apfs
+bsd/miscfs/apfs/apfs_lib.c optional apfs
+bsd/miscfs/apfs/apfs_write.c optional apfs
+bsd/miscfs/apfs/apfs_store.c optional apfs
+bsd/miscfs/apfs/apfs_vfsops.c optional apfs
+bsd/miscfs/apfs/apfs_vnops.c optional apfs
+bsd/miscfs/ramfs/ramfs.c optional ramfs
+bsd/hfs/BTree.c optional hfs
+bsd/hfs/BTreeAllocate.c optional hfs
+bsd/hfs/BTreeMiscOps.c optional hfs
+bsd/hfs/BTreeNodeOps.c optional hfs
+bsd/hfs/BTreeNodeReserve.c optional hfs
+bsd/hfs/BTreeScanner.c optional hfs
+bsd/hfs/BTreeTreeOps.c optional hfs
+bsd/hfs/BTreeWrapper.c optional hfs
+bsd/hfs/CatalogUtilities.c optional hfs
+bsd/hfs/FileExtentMapping.c optional hfs
+bsd/hfs/FileIDsServices.c optional hfs
+bsd/hfs/MacOSStubs.c optional hfs
+bsd/hfs/UnicodeWrappers.c optional hfs
+bsd/hfs/VolumeAllocation.c optional hfs
+bsd/hfs/hfs_attrlist.c optional hfs
+bsd/hfs/hfs_btreeio.c optional hfs
+bsd/hfs/hfs_builtin.c optional hfs
+bsd/hfs/hfs_catalog.c optional hfs
+bsd/hfs/hfs_chash.c optional hfs
+bsd/hfs/hfs_cnode.c optional hfs
+bsd/hfs/hfs_cprotect.c optional hfs
+bsd/hfs/hfs_endian.c optional hfs
+bsd/hfs/hfs_extents.c optional hfs
+bsd/hfs/hfs_fsinfo.c optional hfs
+bsd/hfs/hfs_hotfiles.c optional hfs
+bsd/hfs/hfs_journal.c optional hfs
+bsd/hfs/hfs_link.c optional hfs
+bsd/hfs/hfs_lookup.c optional hfs
+bsd/hfs/hfs_notification.c optional hfs
+bsd/hfs/hfs_quota.c optional hfs
+bsd/hfs/hfs_readwrite.c optional hfs
+bsd/hfs/hfs_resize.c optional hfs
+bsd/hfs/hfs_search.c optional hfs
+bsd/hfs/hfs_vfsops.c optional hfs
+bsd/hfs/hfs_vfsutils.c optional hfs
+bsd/hfs/hfs_vnops.c optional hfs
+bsd/hfs/hfs_xattr.c optional hfs
+bsd/hfs/rangelist.c optional hfs
bsd/miscfs/routefs/routefs_ops.c optional routefs
@@ -361,6 +452,7 @@ bsd/security/audit/audit_syscalls.c standard
bsd/security/audit/audit_worker.c optional config_audit
bsd/pthread/pthread_shims.c standard
+bsd/pthread/pthread_kext_static.c standard
bsd/pthread/pthread_priority.c standard
bsd/pthread/pthread_workqueue.c standard
@@ -383,6 +475,7 @@ bsd/kern/kern_debug.c standard
bsd/kern/kern_symfile.c standard
bsd/kern/kern_descrip.c standard
bsd/kern/kern_guarded.c standard
+bsd/kern/kern_minimac.c optional config_macf
bsd/kern/kern_event.c standard
bsd/kern/kern_control.c optional networking
bsd/kern/kern_exec.c standard
diff --git a/bsd/dev/arm/km.c b/bsd/dev/arm/km.c
index fd7a94b20794..69eaceedd248 100644
--- a/bsd/dev/arm/km.c
+++ b/bsd/dev/arm/km.c
@@ -205,6 +205,15 @@ kmioctl(dev_t dev, u_long cmd, caddr_t data, int flag, proc_t p)
error = EINVAL;
break;
+ case TIOCSCTTY:
+ /*
+ * The console is every daemon's standard error in this system. If
+ * the first session leader to open it took it for its controlling
+ * terminal, its exit would revoke the console for all the others.
+ */
+ error = ENOTTY;
+ break;
+
/* Bodge in the CLOCAL flag as the km device is always local */
case TIOCSETA_32:
case TIOCSETAW_32:
@@ -252,6 +261,9 @@ fallthrough:
* assumptions here, this routine should be static (and
* inlined, given there is only one call site).
*/
+
+extern void loader_log_putc(char c);
+
int
kmputc(__unused dev_t dev, char c)
{
@@ -363,6 +375,8 @@ kmoutput(struct tty * tp)
for (cp = buf; cp < &buf[cc]; cp++) {
/* output the buffer one charatcer at a time */
*cp = *cp & 0x7f;
+ /* What user space writes to /dev/console also goes to the loader's log. */
+ loader_log_putc((char)*cp);
}
if (cc > 1) {
console_write((char *)buf, cc);
diff --git a/bsd/dev/arm/systemcalls.c b/bsd/dev/arm/systemcalls.c
index 7fd9be1a5f38..0d035255b030 100644
--- a/bsd/dev/arm/systemcalls.c
+++ b/bsd/dev/arm/systemcalls.c
@@ -36,6 +36,92 @@
#include <security/mac_framework.h>
#endif
+/*
+ * Bring-up aid: boot-arg systrace=1 logs every BSD system call that fails,
+ * systrace=2 logs all of them (to the kernel message buffer). For calls that
+ * take a path, the path is shown. systrace_name=NAME keeps it to the processes
+ * of that name: with a whole system running, everybody's failed calls are
+ * megabytes a minute.
+ */
+static TUNABLE(uint32_t, systrace_level, "systrace", 0);
+static TUNABLE_STR(systrace_name, MAXCOMLEN + 1, "systrace_name", "");
+extern const char *syscallnames[];
+
+static void
+systrace_report(struct proc *proc, unsigned short syscode, struct uthread *uthread, int error)
+{
+ /* system calls whose first (or, for the second list, second) argument is a path */
+ static const unsigned short path_arg0[] = {
+ SYS_open, SYS_link, SYS_unlink, SYS_chdir, SYS_mknod, SYS_chmod, SYS_chown, SYS_access, SYS_readlink,
+ SYS_execve, SYS_chroot, SYS_rename, SYS_mkdir, SYS_rmdir, SYS_pathconf, SYS_getattrlist, SYS_setattrlist,
+ SYS_getxattr, SYS_listxattr, SYS_fsctl, SYS_stat64, SYS_lstat64, SYS_statfs64, SYS_open_nocancel,
+ SYS_open_dprotected_np, SYS_symlink, SYS_unmount, SYS_stat64_extended, SYS_lstat64_extended,
+ };
+ static const unsigned short path_arg1[] = {
+ SYS_mount, SYS_posix_spawn, SYS_openat, SYS_openat_nocancel, SYS_fstatat64, SYS_faccessat, SYS_readlinkat,
+ SYS_getattrlistat, SYS_mkdirat, SYS_unlinkat,
+ };
+ char path[160] = "";
+ /*
+ * Two calls are worth more than their arguments' addresses, because what they ASK is the whole
+ * of what they do. __mac_syscall carries the policy's name and the request number: this
+ * kernel's AMFI stand-in answers only 0x5a and 0x60 and returns ENOPOLICY to anything else
+ * (bsd/kern/kern_minimac.c), so a daemon refused there is refused in a way nothing in user
+ * space can see -- and a library of ours cannot watch it either, since the call is made inside
+ * libsystem and dyld interposing only stands between images. The name matters as much as the
+ * number: the same call goes to Sandbox as well. And sysctl carries its MIB, which is the only
+ * thing that says what was asked about.
+ */
+ char detail[112] = "";
+ size_t done;
+
+ if (systrace_level < 2 && (error == 0 || error == EJUSTRETURN || error == ERESTART)) {
+ return;
+ }
+ if (systrace_name[0] != 0 && strncmp(proc->p_comm, systrace_name, MAXCOMLEN) != 0) {
+ return;
+ }
+ for (unsigned i = 0; i < sizeof(path_arg0) / sizeof(path_arg0[0]); i++) {
+ if (path_arg0[i] == syscode) {
+ (void)copyinstr((user_addr_t)uthread->uu_arg[0], path, sizeof(path), &done);
+ }
+ }
+ for (unsigned i = 0; i < sizeof(path_arg1) / sizeof(path_arg1[0]); i++) {
+ if (path_arg1[i] == syscode) {
+ (void)copyinstr((user_addr_t)uthread->uu_arg[1], path, sizeof(path), &done);
+ }
+ }
+ path[sizeof(path) - 1] = 0;
+ if (syscode == SYS___mac_syscall) {
+ char policy[32] = "";
+
+ (void)copyinstr((user_addr_t)uthread->uu_arg[0], policy, sizeof(policy), &done);
+ policy[sizeof(policy) - 1] = 0;
+ snprintf(detail, sizeof(detail), " policy \"%s\" request %lld", policy,
+ (long long)uthread->uu_arg[1]);
+ } else if (syscode == SYS_sysctl) {
+ int mib[6];
+ unsigned count = (unsigned)uthread->uu_arg[1];
+
+ if (count > 6) {
+ count = 6;
+ }
+ if (count > 0 && copyin((user_addr_t)uthread->uu_arg[0], mib, count * sizeof(mib[0])) == 0) {
+ size_t used = (size_t)snprintf(detail, sizeof(detail), " mib");
+
+ for (unsigned i = 0; i < count && used + 12 < sizeof(detail); i++) {
+ used += (size_t)snprintf(detail + used, sizeof(detail) - used, " %d", mib[i]);
+ }
+ }
+ }
+ detail[sizeof(detail) - 1] = 0;
+ printf("systrace: %s[%d] %s(0x%llx, 0x%llx, 0x%llx, 0x%llx)%s%s%s%s = %d\n",
+ proc->p_comm, proc_getpid(proc), syscallnames[syscode],
+ (unsigned long long)uthread->uu_arg[0], (unsigned long long)uthread->uu_arg[1],
+ (unsigned long long)uthread->uu_arg[2], (unsigned long long)uthread->uu_arg[3],
+ path[0] ? " \"" : "", path, path[0] ? "\"" : "", detail, error);
+}
+
#if CONFIG_DTRACE
extern int32_t dtrace_systrace_syscall(struct proc *, void *, int *);
extern void dtrace_systrace_syscall_return(unsigned short, int, int *);
@@ -181,6 +267,10 @@ unix_syscall(
error = (*(callp->sy_call))(proc, &uthread->uu_arg[0], &(uthread->uu_rval[0]));
AUDIT_SYSCALL_EXIT(code, proc, uthread, error);
+ if (__improbable(systrace_level != 0)) {
+ systrace_report(proc, syscode, uthread, error);
+ }
+
#if CONFIG_MACF
skip_syscall:
#endif /* CONFIG_MACF */
diff --git a/bsd/dev/memdev.c b/bsd/dev/memdev.c
index 3e6bffa8c48e..43cffb98bdcc 100644
--- a/bsd/dev/memdev.c
+++ b/bsd/dev/memdev.c
@@ -159,7 +159,8 @@ static const struct cdevsw mdevcdevsw = {
struct mdev {
uint64_t mdBase; /* base page number (pages are assumed to be 4K). Multiply by 4096 to find actual address */
- uint32_t mdSize; /* size in pages (pages are assumed to be 4K). Multiply by 4096 to find actual size. */
+ uint64_t mdSize; /* size in pages (pages are assumed to be 4K). Multiply by 4096 to find actual size.
+ * 64 bits wide so that "mdSize << 12" does not wrap for a disk of 4 GB or more. */
int mdFlags; /* flags */
int mdSecsize; /* sector size */
int mdBDev; /* Block device number */
@@ -179,6 +180,7 @@ int mdevCMajor = -1;
static int mdevioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p, int is_char);
dev_t mdevadd(int devid, uint64_t base, unsigned int size, int phys);
dev_t mdevlookup(int devid);
+int mdevgetkernelrange(dev_t dev, vm_offset_t *base, uint64_t *size);
void mdevremoveall(void);
int mdevgetrange(int devid, uint64_t *base, uint64_t *size);
@@ -250,7 +252,9 @@ mdevrw(dev_t dev, struct uio *uio, __unused int ioflag)
if (uio->uio_offset > (mdev[devid].mdSize << 12)) {
count = 0;
} else {
- count = imin(uio_resid(uio), (mdev[devid].mdSize << 12) - uio->uio_offset);
+ uint64_t room = (mdev[devid].mdSize << 12) - (uint64_t)uio->uio_offset;
+
+ count = (uint64_t)uio_resid(uio) < room ? (int)uio_resid(uio) : (int)(room > INT_MAX ? INT_MAX : room);
}
status = uiomove64(mdata, count, uio); /* Move the data */
@@ -455,8 +459,8 @@ mdevioctl(dev_t dev, u_long cmd, caddr_t data, __unused int flag,
}
memdev_info->mi_mdev = TRUE;
memdev_info->mi_phys = (mdev[devid].mdFlags & mdPhys) ? TRUE : FALSE;
- memdev_info->mi_base = (uint32_t)mdev[devid].mdBase;
- memdev_info->mi_size = mdev[devid].mdSize;
+ memdev_info->mi_base = mdev[devid].mdBase;
+ memdev_info->mi_size = (typeof(memdev_info->mi_size))mdev[devid].mdSize;
break;
default:
@@ -725,6 +729,26 @@ mdevlookup(int devid)
return mdev[devid].mdBDev; /* Return the device number */
}
+/*
+ * For a filesystem that reads a memory disk in place: where the disk is in the kernel's
+ * address space. Not for disks given by physical address (those are not mapped).
+ */
+int
+mdevgetkernelrange(dev_t dev, vm_offset_t *base, uint64_t *size)
+{
+ for (int i = 0; i < NB_MAX_MDEVICES; i++) {
+ if ((mdev[i].mdFlags & mdInited) && mdev[i].mdBDev == dev) {
+ if (mdev[i].mdFlags & mdPhys) {
+ return ENOTSUP;
+ }
+ *base = (vm_offset_t)(mdev[i].mdBase << 12);
+ *size = mdev[i].mdSize << 12;
+ return 0;
+ }
+ }
+ return ENXIO;
+}
+
void
mdevremoveall(void)
{
diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c
index 4df41ba1f61b..b5a79a02a3f5 100644
--- a/bsd/kern/bsd_init.c
+++ b/bsd/kern/bsd_init.c
@@ -147,6 +147,14 @@
#include <net/if_ether.h> /* for ether_family_init() */
#include <net/if_gif.h> /* for gif_init() */
#include <miscfs/devfs/devfsdefs.h> /* for devfs_kernel_mount() */
+#include <sys/hakios.input.h>
+#include <sys/hakios.audio.h>
+#include <sys/hakios.virtio_blk.h>
+#include <sys/hakios.sparse.h>
+#include <sys/hakios.version.h>
+#if APFS
+#include <sys/hakios.apfs_check.h>
+#endif
#include <vm/vm_kern.h> /* for kmem_suballoc() */
#include <sys/proc_uuid_policy.h> /* proc_uuid_policy_init() */
#include <netinet/flow_divert.h> /* flow_divert_init() */
@@ -490,7 +498,7 @@ bsd_init(void)
platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_BSD_INIT);
#endif
-#define DEBUG_BSDINIT 0
+#define DEBUG_BSDINIT 1 /* bring-up: trace bsd_init steps (only visible with -enable_kprintf_spam) */
#if DEBUG_BSDINIT
#define bsd_init_kprintf(x, ...) kprintf("bsd_init: " x, ## __VA_ARGS__)
@@ -976,6 +984,15 @@ bsd_init(void)
bsd_init_kprintf("calling devfs_kernel_mount\n");
devfs_kernel_mount(mounthere);
+ hakios_version_init(); /* kern.hakios.version, and the version in the boot log */
+ hakios_input_init(); /* /dev/hakios-touch */
+ hakios_audio_init(); /* /dev/hakios-audio */
+ hakios_virtio_blk_init(); /* /dev/rvblk0, a disk image of the host (QEMU) */
+ hakios_sparse_init(); /* /dev/rsp0, part of an image too big to hold */
+#if APFS
+ /* mounts and walks /dev/rvblk0 once userland has /private/var up */
+ hakios_apfs_check_init();
+#endif
}
#endif /* DEVFS */
diff --git a/bsd/kern/decmpfs.c b/bsd/kern/decmpfs.c
index 95192dd62449..b00da803b5b4 100644
--- a/bsd/kern/decmpfs.c
+++ b/bsd/kern/decmpfs.c
@@ -515,7 +515,7 @@ decmpfs_fetch_compressed_header(vnode_t vp, decmpfs_cnode *cp, decmpfs_header **
const bool no_additional_data = ((cp != NULL)
&& (cp->cmp_type != 0)
&& (cp->cmp_minimal_xattr != 0));
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
decmpfs_header *hdr = NULL;
/*
diff --git a/bsd/kern/kern_exec.c b/bsd/kern/kern_exec.c
index 1d71c66629fb..a22095ccf5f0 100644
--- a/bsd/kern/kern_exec.c
+++ b/bsd/kern/kern_exec.c
@@ -1157,6 +1157,9 @@ SYSCTL_PROC(_vm, OID_AUTO, shared_region_control,
* activators should not be given the opportunity to attempt
* to activate the image.
*/
+/* Bring-up aid: with boot-arg systrace set, say which stage of an exec refused the image. */
+static TUNABLE(uint32_t, exec_trace, "systrace", 0);
+
static int
exec_mach_imgact(struct image_params *imgp)
{
@@ -1315,6 +1318,9 @@ grade:
* Actually load the image file we previously decided to load.
*/
lret = load_machfile(imgp, mach_header, thread, &map, &load_result);
+ if (lret != LOAD_SUCCESS && exec_trace) {
+ printf("exectrace: %s: load_machfile failed: %d\n", imgp->ip_ndp->ni_cnd.cn_pnbuf, lret);
+ }
if (lret != LOAD_SUCCESS) {
error = load_return_to_errno(lret);
@@ -2197,6 +2203,9 @@ again:
}
error = exec_check_permissions(imgp);
+ if (error && exec_trace) {
+ printf("exectrace: %s: exec_check_permissions failed: %d\n", imgp->ip_ndp->ni_cnd.cn_pnbuf, error);
+ }
if (error) {
goto bad;
}
@@ -2640,6 +2649,57 @@ done:
return ret;
}
+/*
+ * Bring-up aid, boot-arg applog=1: what applications and other programs started without a
+ * terminal say is worth reading here. Their standard output and error, which launchd opens on
+ * /dev/null, go to the console instead (that shows NSLog and the "Terminating app due to
+ * uncaught exception" kind of message). With applog=2 every program also gets
+ * OS_ACTIVITY_DT_MODE in its environment, which makes os_log write to standard error as well:
+ * everything any program logs, and a much slower system. applog_name=NAME[,NAME...] with
+ * applog=1 does that for the programs whose file names begin with one of the NAMEs only,
+ * which is how to read what one application extension has to say (launchd starts those for
+ * whoever asks, so there is no job description of ours to put the variable in).
+ */
+static TUNABLE(uint32_t, exec_applog, "applog", 0);
+static TUNABLE_STR(exec_applog_name, 128, "applog_name", "");
+
+static bool
+exec_applog_oslog(struct image_params *imgp)
+{
+ const char *name, *p = exec_applog_name;
+
+ if (exec_applog >= 2) {
+ return true;
+ }
+ if (exec_applog == 0 || *p == 0 || imgp->ip_ndp == NULL) {
+ return false;
+ }
+ name = imgp->ip_ndp->ni_cnd.cn_nameptr;
+ while (*p != 0) {
+ const char *word = p;
+
+ while (*p != 0 && *p != ',') {
+ p++;
+ }
+ if (p != word && (size_t)(p - word) <= imgp->ip_ndp->ni_cnd.cn_namelen && strncmp(word, name, (size_t)(p - word)) == 0) {
+ return true;
+ }
+ if (*p == ',') {
+ p++;
+ }
+ }
+ return false;
+}
+
+static const char *
+exec_applog_path(int fd, const char *path)
+{
+ if (exec_applog && (fd == 1 || fd == 2) && strcmp(path, "/dev/null") == 0) {
+ return "/dev/console";
+ }
+ return path;
+}
+
/*
* exec_handle_file_actions
*
@@ -2709,7 +2769,7 @@ exec_handle_file_actions(struct image_params *imgp, short psa_flags)
AUDIT_SUBCALL_ENTER(OPEN, p, uthread);
NDINIT(ndp, LOOKUP, OP_OPEN, FOLLOW | AUDITVNPATH1, UIO_SYSSPACE,
- CAST_USER_ADDR_T(psfa->psfaa_openargs.psfao_path),
+ CAST_USER_ADDR_T(exec_applog_path(psfa->psfaa_filedes, psfa->psfaa_openargs.psfao_path)),
imgp->ip_vfs_context);
error = open1(imgp->ip_vfs_context, ndp,
@@ -3094,10 +3154,16 @@ spawn_validate_persona(struct _posix_spawn_persona_info *px_persona)
int error = 0;
struct persona *persona = NULL;
- if (!IOCurrentTaskHasEntitlement( PERSONA_MGMT_ENTITLEMENT)) {
- return EPERM;
- }
-
+ /*
+ * hakios: two of the three checks below cannot be passed in this port, and both stand in
+ * the way of something ordinary. The entitlement is one nobody carries here (nothing is
+ * signed, AMFI is a stub), and the persona is one nobody makes -- yet a package manager
+ * becomes root by asking for exactly this: Sileo spawns apt and dpkg into persona 99 with
+ * the uid it wants (its APTWrapper). Refused here, the spawn failed before the program was
+ * ever started, and the application waited for an answer that could not come. The check
+ * that still means something -- the size of the group list -- stays; see
+ * spawn_persona_adopt() below for what happens then.
+ */
if (px_persona->pspi_flags & POSIX_SPAWN_PERSONA_GROUPS) {
if (px_persona->pspi_ngroups > NGROUPS_MAX) {
return EINVAL;
@@ -3105,12 +3171,6 @@ spawn_validate_persona(struct _posix_spawn_persona_info *px_persona)
}
persona = persona_lookup(px_persona->pspi_id);
- if (!persona) {
- error = ESRCH;
- goto out;
- }
-
-out:
if (persona) {
persona_put(persona);
}
@@ -3130,9 +3190,6 @@ spawn_persona_adopt(proc_t p, struct _posix_spawn_persona_info *px_persona)
* the kauth with a different UID/GID combo
*/
persona = persona_lookup(px_persona->pspi_id);
- if (!persona) {
- return ESRCH;
- }
cred = kauth_cred_proc_ref(p);
@@ -3158,6 +3215,48 @@ spawn_persona_adopt(proc_t p, struct _posix_spawn_persona_info *px_persona)
px_persona->pspi_gmuid);
}
+ if (persona == NULL) {
+ /*
+ * hakios: there are no personas in this port -- nothing makes them, and
+ * kpersona_find_by_type() answers "no such process" to everyone who asks. Darwin's own
+ * programs get along without them, but a package manager becomes root by way of one:
+ * Sileo asks posix_spawn for persona 99 with OVERRIDE and uid 0 (its APTWrapper), and
+ * that is how it runs apt and dpkg. With the persona missing the lookup failed, the
+ * whole spawn failed with ESRCH, and the application sat for ever waiting for an
+ * answer from a program that was never started. What the caller is after is the
+ * credential, not the persona, so the credential is what it gets: the uid, gid and
+ * groups it asked for, set on the process being spawned.
+ *
+ * The thread's own credential is not touched here: the thread running this is the
+ * parent's, and the new process gets its credential when its image is activated.
+ *
+ * And the count of processes per real uid follows, as persona_proc_adopt() keeps it:
+ * fork1() counted the new process under the parent's uid (mobile's), its exit takes
+ * it off under the one it has then (root's). Without that, every program Sileo ran as
+ * root took one away from root's count that had never been given -- and once Sileo's
+ * own mv, cp and rm ran at all, a few dozen of them brought root's count to nothing,
+ * and the next process of root's to exit panicked the kernel ("chgproccnt: lost user").
+ */
+ __block kauth_cred_t new_cred = cred;
+ __block uid_t old_ruid = KAUTH_UID_NONE;
+ uid_t new_ruid = kauth_cred_getruid(cred);
+
+ (void)proc_update_label(p, true, ^kauth_cred_t (kauth_cred_t cur_cred) {
+ old_ruid = kauth_cred_getruid(cur_cred);
+ if (new_cred != cur_cred) {
+ kauth_cred_ref(new_cred);
+ kauth_cred_unref(&cur_cred);
+ }
+ return new_cred;
+ });
+ kauth_cred_unref(&cred);
+ if (old_ruid != KAUTH_UID_NONE && old_ruid != new_ruid) {
+ (void)chgproccnt(new_ruid, 1);
+ (void)chgproccnt(old_ruid, -1);
+ }
+ return 0;
+ }
+
ret = persona_proc_adopt(p, persona, cred);
kauth_cred_unref(&cred);
@@ -5586,10 +5685,75 @@ bad:
* Note: The argument and environment vectors are user space pointers
* to arrays of user space pointers.
*/
+/*
+ * Bring-up aid: the sysctl kern.exec_inject holds "LIBRARY NAME NAME ..."; every program
+ * whose file name is one of the NAMEs is started with DYLD_INSERT_LIBRARIES=LIBRARY in its
+ * environment, so that dyld loads our library into it. Several such groups may follow each
+ * other, separated by ";". A program named in more than one group gets every one of those
+ * libraries, joined with ":" in the one DYLD_INSERT_LIBRARIES, so SpringBoard can carry both
+ * our window-server library and the tweak loader. launchd refuses to pass DYLD_ variables
+ * from a job description, which is why the kernel adds this one. Set once by our init (from
+ * etc/inject.conf) before it hands over to launchd.
+ */
+static char exec_inject_conf[512];
+SYSCTL_STRING(_kern, OID_AUTO, exec_inject, CTLFLAG_RW | CTLFLAG_LOCKED, exec_inject_conf, sizeof(exec_inject_conf),
+ "LIBRARY NAME... [; LIBRARY NAME...]: programs started with the library inserted");
+
+static bool
+exec_inject_env(struct image_params *imgp, char *env, size_t size)
+{
+ const char *name, *p = exec_inject_conf;
+ size_t namelen, used;
+ bool any = false;
+
+ if (*p == 0 || imgp->ip_ndp == NULL) {
+ return false;
+ }
+ name = imgp->ip_ndp->ni_cnd.cn_nameptr;
+ namelen = imgp->ip_ndp->ni_cnd.cn_namelen;
+ used = strlcpy(env, "DYLD_INSERT_LIBRARIES=", size);
+ while (*p != 0) {
+ const char *library;
+ size_t liblen;
+ bool matched = false;
+
+ while (*p == ' ' || *p == ';') {
+ p++;
+ }
+ library = p;
+ while (*p != 0 && *p != ' ' && *p != ';') {
+ p++;
+ }
+ liblen = (size_t)(p - library);
+ while (*p == ' ') {
+ const char *word = ++p;
+
+ while (*p != 0 && *p != ' ' && *p != ';') {
+ p++;
+ }
+ if (!matched && (size_t)(p - word) == namelen && namelen != 0 && liblen != 0 &&
+ strncmp(word, name, namelen) == 0) {
+ matched = true; /* this group names this program; its library is added once */
+ }
+ }
+ if (matched && liblen != 0 && used + 1 + liblen < size) {
+ if (any) {
+ env[used++] = ':'; /* dyld takes several libraries separated by ":" */
+ }
+ memcpy(env + used, library, liblen);
+ used += liblen;
+ env[used] = 0;
+ any = true;
+ }
+ }
+ return any;
+}
+
__attribute__((noinline))
static int
exec_extract_strings(struct image_params *imgp)
{
+ char inject_env[512];
int error = 0;
int ptr_size = (imgp->ip_flags & IMGPF_WAS_64BIT_ADDR) ? 8 : 4;
int new_ptr_size = (imgp->ip_flags & IMGPF_IS_64BIT_ADDR) ? 8 : 4;
@@ -5759,6 +5923,33 @@ exec_extract_strings(struct image_params *imgp)
imgp->ip_envc++;
}
+ /* one more variable for the programs that get our library inside them (see exec_inject_env) */
+ if (exec_inject_env(imgp, inject_env, sizeof(inject_env))) {
+ error = exec_add_user_string(imgp, CAST_USER_ADDR_T(inject_env), UIO_SYSSPACE, TRUE);
+ if (error) {
+ goto bad;
+ }
+ if (imgp->ip_argspace < new_ptr_size) {
+ error = E2BIG;
+ goto bad;
+ }
+ imgp->ip_argspace -= new_ptr_size;
+ imgp->ip_envc++;
+ }
+ /* ... and one for everybody when the logs are wanted on the console (see exec_applog_path) */
+ if (exec_applog_oslog(imgp)) {
+ error = exec_add_user_string(imgp, CAST_USER_ADDR_T("OS_ACTIVITY_DT_MODE=YES"), UIO_SYSSPACE, TRUE);
+ if (error) {
+ goto bad;
+ }
+ if (imgp->ip_argspace < new_ptr_size) {
+ error = E2BIG;
+ goto bad;
+ }
+ imgp->ip_argspace -= new_ptr_size;
+ imgp->ip_envc++;
+ }
+
/* Save space for envv[] NULL terminator */
if (imgp->ip_argspace < new_ptr_size) {
error = E2BIG;
@@ -6636,7 +6827,23 @@ handle_mac_transition:
}
/* Update the process' identity version and set the security token */
- proc_setpidversion(p, OSIncrementAtomic(&nextpidversion));
+ {
+ /*
+ * (redwood) Except when xpcproxy becomes the service it was spawned for: then the process keeps
+ * xpcproxy's identity version, as if launchd had spawned the service itself, which is what launchd
+ * means by eliding xpcproxy ("service will attempt to elide xpcproxy"; here it spawns xpcproxy
+ * all the same). RunningBoard keeps the audit token of such a process from before the exec and
+ * never takes the newer one: every entitlement check it made for it failed (SecTaskLoadEntitlements
+ * error 3, a thousand times a minute with Safari open), and Safari's web processes got no
+ * assertions ("target is not running or doesn't have entitlement
+ * com.apple.runningboard.assertions.webkit"). Boot-arg xpcproxy_idversion=1: a new version, as XNU.
+ */
+ static TUNABLE(uint32_t, xpcproxy_idversion, "xpcproxy_idversion", 0);
+
+ if (xpcproxy_idversion || strncmp(p->p_comm, "xpcproxy", sizeof(p->p_comm)) != 0) {
+ proc_setpidversion(p, OSIncrementAtomic(&nextpidversion));
+ }
+ }
task_set_uniqueid(task);
set_security_token_task_internal(p, task);
@@ -7538,6 +7745,11 @@ process_signature(proc_t p, struct image_params *imgp)
}
done:
+ if (error && exec_trace) {
+ printf("exectrace: %s[%d]: process_signature failed: %d (csflags 0x%x, mac %d, cs_error %d)\n",
+ p->p_comm, proc_getpid(p), error, imgp->ip_csflags, imgp->ip_mac_return,
+ signature_failure_reason != OS_REASON_NULL);
+ }
if (0 == error) {
/*
* Update the new process's signature-dependent process state.
diff --git a/bsd/kern/kern_memorystatus.c b/bsd/kern/kern_memorystatus.c
index 1750d3a52d3c..0373d0dd160f 100644
--- a/bsd/kern/kern_memorystatus.c
+++ b/bsd/kern/kern_memorystatus.c
@@ -222,6 +222,15 @@ MACRO_END
/* General tunables */
+/*
+ * hakios: the memory there is to manage. max_mem is the whole of the kernel's memory window, and
+ * here more than half of that window is the system image -- a RAM disk the kernel boots from, which
+ * is nobody's to use or to give back. Thresholds that are per cent of max_mem had idle daemons
+ * killed with a quarter of the usable memory still available, and the window server not long after.
+ * sane_size leaves the image (and the kernel itself) out.
+ */
+#define memstat_memory_size sane_size
+
unsigned long delta_percentage = 5;
unsigned long critical_threshold_percentage = 5;
// On embedded devices with more than 3GB of memory we lower the critical percentage.
@@ -1365,7 +1374,7 @@ memorystatus_init(void)
if (delta_percentage == 0) {
delta_percentage = 5;
}
- if (max_mem > config_jetsam_large_memory_cutoff) {
+ if (memstat_memory_size > config_jetsam_large_memory_cutoff) {
critical_threshold_percentage = critical_threshold_percentage_larger_devices;
delta_percentage = delta_percentage_larger_devices;
}
@@ -1400,11 +1409,11 @@ memorystatus_init(void)
PE_get_default("kern.jetsam_idle_snapshot", &memorystatus_idle_snapshot, sizeof(memorystatus_idle_snapshot));
}
- memorystatus_delta = (unsigned int) (delta_percentage * atop_64(max_mem) / 100);
- memorystatus_available_pages_critical_idle_offset = (unsigned int) (idle_offset_percentage * atop_64(max_mem) / 100);
+ memorystatus_delta = (unsigned int) (delta_percentage * atop_64(memstat_memory_size) / 100);
+ memorystatus_available_pages_critical_idle_offset = (unsigned int) (idle_offset_percentage * atop_64(memstat_memory_size) / 100);
memorystatus_available_pages_critical_base = (unsigned int) ((critical_threshold_percentage / delta_percentage) * memorystatus_delta);
memorystatus_policy_more_free_offset_pages = (unsigned int) ((policy_more_free_offset_percentage / delta_percentage) * memorystatus_delta);
- memorystatus_sysproc_aging_aggr_pages = (unsigned int) (sysproc_aging_aggr_threshold_percentage * atop_64(max_mem) / 100);
+ memorystatus_sysproc_aging_aggr_pages = (unsigned int) (sysproc_aging_aggr_threshold_percentage * atop_64(memstat_memory_size) / 100);
/* Set the swapin trigger in pages based on the maximum size allocated for each c_seg */
memorystatus_swapin_trigger_pages = (unsigned int) atop_64(memorystatus_swapin_trigger_segments * c_seg_allocsize);
@@ -5103,7 +5112,7 @@ memorystatus_pages_update(unsigned int pages_avail)
if (critical || delta) {
unsigned int total_pages;
- total_pages = (unsigned int) atop_64(max_mem);
+ total_pages = (unsigned int) atop_64(memstat_memory_size);
#if CONFIG_SECLUDED_MEMORY
total_pages -= vm_page_secluded_count;
#endif /* CONFIG_SECLUDED_MEMORY */
@@ -6609,7 +6618,7 @@ memorystatus_update_levels_locked(boolean_t critical_only)
}
#if VM_PRESSURE_EVENTS
- memorystatus_available_pages_pressure = (int32_t)(pressure_threshold_percentage * (atop_64(max_mem) / 100));
+ memorystatus_available_pages_pressure = (int32_t)(pressure_threshold_percentage * (atop_64(memstat_memory_size) / 100));
#endif
}
diff --git a/bsd/kern/kern_priv.c b/bsd/kern/kern_priv.c
index 88adf2dda936..168384889bc1 100644
--- a/bsd/kern/kern_priv.c
+++ b/bsd/kern/kern_priv.c
@@ -82,6 +82,19 @@ priv_check_cred(kauth_cred_t cred, int priv, int flags)
#endif
int error;
+ /*
+ * hakios: nothing is signed here and no process carries an entitlement (AMFI is a stub), so
+ * every privilege that an entitlement would answer for is refused -- and Darwin leans on one of
+ * them for ordinary work. A daemon that goes to the network on behalf of an application
+ * (trustd checking the certificate for whoever opened the connection, nsurlsessiond carrying
+ * out its downloads) describes the connection to NECP as delegated, and without this
+ * privilege the kernel refuses it with EPERM: the download then never starts and never fails.
+ * These daemons run as mobile, so the "root gets everything" rule below does not reach them.
+ */
+ if (priv == PRIV_NET_PRIVILEGED_SOCKET_DELEGATE) {
+ return 0;
+ }
+
/*
* We first evaluate policies that may deny the granting of
* privilege unilaterally.
diff --git a/bsd/kern/kern_sig.c b/bsd/kern/kern_sig.c
index ae4a5f33df1e..986b517dfd18 100644
--- a/bsd/kern/kern_sig.c
+++ b/bsd/kern/kern_sig.c
@@ -1099,6 +1099,14 @@ __pthread_kill(__unused proc_t p, struct __pthread_kill_args *uap,
thread_t target_act;
int error = 0;
int signum = uap->sig;
+
+#if defined(__arm64__)
+ if (signum == SIGABRT) {
+ extern void user_abort_trace(void);
+
+ user_abort_trace(); /* no crash reporter here: this is where an abort() gets recorded */
+ }
+#endif
struct uthread *uth;
target_act = (thread_t)port_name_to_thread(uap->thread_port,
diff --git a/bsd/kern/kpi_socket.c b/bsd/kern/kpi_socket.c
index 003cfb074a99..c597065f24f1 100644
--- a/bsd/kern/kpi_socket.c
+++ b/bsd/kern/kpi_socket.c
@@ -727,7 +727,7 @@ sock_receive_internal(socket_t sock, struct msghdr *msg, mbuf_t *data,
int error = 0;
user_ssize_t length = 0;
struct sockaddr *fromsa = NULL;
- uio_stackbuf_t uio_buf[UIO_SIZEOF((msg != NULL) ? msg->msg_iovlen : 0)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF((msg != NULL) ? msg->msg_iovlen : 0) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (sock == NULL) {
return EINVAL;
diff --git a/bsd/kern/sys_generic.c b/bsd/kern/sys_generic.c
index 4ca22ee4371a..e95e618b4cb6 100644
--- a/bsd/kern/sys_generic.c
+++ b/bsd/kern/sys_generic.c
@@ -303,7 +303,7 @@ dofileread(vfs_context_t ctx, struct fileproc *fp,
user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
user_ssize_t *retval)
{
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
uio_t uio;
int spacetype;
@@ -350,7 +350,7 @@ static int
read_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
off_t offset, int flags, user_ssize_t *retval)
{
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
uio_t uio;
int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
@@ -631,7 +631,7 @@ dofilewrite(vfs_context_t ctx, struct fileproc *fp,
user_addr_t bufp, user_size_t nbyte, off_t offset, int flags,
user_ssize_t *retval)
{
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
uio_t uio;
int spacetype;
@@ -678,7 +678,7 @@ int
write_internal(struct proc *p, int fd, user_addr_t buf, user_size_t nbyte,
off_t offset, int flags, guardid_t *puguard, user_ssize_t *retval)
{
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
uio_t uio;
int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
diff --git a/bsd/kern/sys_work_interval.c b/bsd/kern/sys_work_interval.c
index b544bf307dfe..a06cbc68ffdb 100644
--- a/bsd/kern/sys_work_interval.c
+++ b/bsd/kern/sys_work_interval.c
@@ -265,6 +265,22 @@ work_interval_ctl(__unused proc_t p, struct work_interval_ctl_args *uap,
break;
+ case 6:
+ case 7:
+ case 8:
+ /*
+ * (redwood) Not in the open source: what os_workgroup_interval_start(), _update() and
+ * _finish() of Darwin send, for the performance controller, which is not here either.
+ * CoreAudio does it in every I/O cycle and logs an error each time it is refused: fifty
+ * lines a second from the audio server's real-time thread. For its threads, and only
+ * for them, there is nothing to do and nothing wrong. (The window server asks for its
+ * frames too, and is used to the refusal.)
+ */
+ if (kern_work_interval_type_of_thread(current_thread()) == WORK_INTERVAL_TYPE_COREAUDIO) {
+ return 0;
+ }
+ return ENOTSUP;
+
default:
return ENOTSUP;
}
diff --git a/bsd/kern/uipc_syscalls.c b/bsd/kern/uipc_syscalls.c
index dce802fc36f1..e2e6b3a3f3db 100644
--- a/bsd/kern/uipc_syscalls.c
+++ b/bsd/kern/uipc_syscalls.c
@@ -3561,7 +3561,7 @@ sendfile(proc_ref_t p, struct sendfile_args *uap, __unused int *retval)
mbuf_t m;
unsigned int nbufs = SFUIOBUFS, i;
uio_t auio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(SFUIOBUFS)]; /* 1 KB !!! */
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(SFUIOBUFS) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)]; /* 1 KB !!! */
size_t uiolen;
user_ssize_t rlen;
off_t pgoff;
diff --git a/bsd/miscfs/bindfs/bind_vfsops.c b/bsd/miscfs/bindfs/bind_vfsops.c
index 412f9ecfc22c..c88c240d4262 100644
--- a/bsd/miscfs/bindfs/bind_vfsops.c
+++ b/bsd/miscfs/bindfs/bind_vfsops.c
@@ -139,8 +139,12 @@ bindfs_mount(struct mount * mp, __unused vnode_t devvp, user_addr_t user_data, v
return ENOTSUP;
}
- /* check entitlement */
- if (!IOCurrentTaskHasEntitlement(BINDFS_ENTITLEMENT)) {
+ /*
+ * check entitlement -- hakios: or process 1, the port's own init, unsigned, which binds
+ * its choice of launchd jobs and the like onto the read-only Darwin root volume before it
+ * hands over to launchd (userland/src/init.c, root_apfs=)
+ */
+ if (!IOCurrentTaskHasEntitlement(BINDFS_ENTITLEMENT) && proc_getpid(current_proc()) != 1) {
return EPERM;
}
diff --git a/bsd/miscfs/mockfs/mockfs.h b/bsd/miscfs/mockfs/mockfs.h
index 07c91ffaf686..5ce41d043901 100644
--- a/bsd/miscfs/mockfs/mockfs.h
+++ b/bsd/miscfs/mockfs/mockfs.h
@@ -59,7 +59,7 @@ struct mockfs_mount {
mockfs_fsnode_t mockfs_root; /* Root of the node tree */
boolean_t mockfs_memory_backed; /* Does the backing store reside in memory */
boolean_t mockfs_physical_memory; /* (valid if memory backed) */
- uint32_t mockfs_memdev_base; /* Base page of the backing store (valid if memory backed) */
+ uint64_t mockfs_memdev_base; /* Base page of the backing store (valid if memory backed) */
uint64_t mockfs_memdev_size; /* Size of the backing store (valid if memory backed) */
};
diff --git a/bsd/miscfs/mockfs/mockfs_fsnode.c b/bsd/miscfs/mockfs/mockfs_fsnode.c
index e00012622e77..4f993b3cd47f 100644
--- a/bsd/miscfs/mockfs/mockfs_fsnode.c
+++ b/bsd/miscfs/mockfs/mockfs_fsnode.c
@@ -85,7 +85,7 @@ mockfs_fsnode_create(mount_t mp, uint8_t type, mockfs_fsnode_t * fsnpp)
goto done;
}
- *fsnpp = kalloc_type(mockfs_fsnode_t, Z_WAITOK | Z_ZERO | Z_NOFAIL);
+ *fsnpp = kalloc_type(struct mockfs_fsnode, Z_WAITOK | Z_ZERO | Z_NOFAIL);
(*fsnpp)->size = new_size;
(*fsnpp)->type = type;
(*fsnpp)->mnt = mp;
@@ -154,7 +154,7 @@ mockfs_fsnode_destroy(mockfs_fsnode_t fsnp)
}
}
- kfree_type(mockfs_fsnode_t, fsnp);
+ kfree_type(struct mockfs_fsnode, fsnp);
done:
return rvalue;
}
@@ -345,10 +345,12 @@ mockfs_fsnode_vnode(mockfs_fsnode_t fsnp, vnode_t * vpp)
panic("mockfs_fsvnode failed to get ubc_mem_object for a new vnode");
}
- rvalue = pager_map_to_phys_contiguous(ubc_mem_object, 0, (mockfs_mnt->mockfs_memdev_base << PAGE_SHIFT), fsnp->size);
+ /* memdev counts its base in 4K pages whatever the kernel page size is (see bsd/dev/memdev.c) */
+ rvalue = pager_map_to_phys_contiguous(ubc_mem_object, 0, (mockfs_mnt->mockfs_memdev_base << 12), fsnp->size);
if (rvalue) {
- panic("mockfs_fsnode_vnode failed to create fictitious pages for a memory-backed device; rvalue = %d", rvalue);
+ panic("mockfs_fsnode_vnode failed to create fictitious pages for a memory-backed device; rvalue = %d, base page 0x%llx, size 0x%llx",
+ rvalue, (unsigned long long)mockfs_mnt->mockfs_memdev_base, (unsigned long long)fsnp->size);
}
}
diff --git a/bsd/nfs/nfs_serv.c b/bsd/nfs/nfs_serv.c
index 6ae9b6bc3fde..58556c324b67 100644
--- a/bsd/nfs/nfs_serv.c
+++ b/bsd/nfs/nfs_serv.c
@@ -769,8 +769,8 @@ nfsrv_readlink(
struct nfsm_chain *nmreq, nmrep;
mbuf_t mpath, mp;
uio_t auio = NULL;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(4)];
- char *uio_bufp = &uio_buf[0];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(4) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
+ char *uio_bufp = (char *)&uio_buf[0];
int uio_buflen = UIO_SIZEOF(4);
error = 0;
@@ -917,7 +917,7 @@ nfsrv_read(
struct vnode_attr vattr, *vap = &vattr;
off_t off;
uid_t saved_uid;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(0)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(0) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
struct nfsm_chain *nmreq, nmrep;
error = 0;
@@ -3559,7 +3559,7 @@ nfsrv_symlink(
struct nfs_export *nx = NULL;
struct nfs_export_options *nxo = NULL;
uio_t auio = NULL;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
struct nfsm_chain *nmreq, nmrep;
__unused const int nfs_vers = nd->nd_vers;
assert(nd->nd_vers == NFS_VER2 || nd->nd_vers == NFS_VER3);
@@ -4215,7 +4215,7 @@ nfsrv_readdir(
struct nfs_export *nx;
struct nfs_export_options *nxo;
uio_t auio = NULL;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
int len, nlen, rem, xfer, error, attrerr;
int siz, count, fullsiz, eofflag, nentries;
u_quad_t off, toff, verf = 0;
@@ -4462,7 +4462,7 @@ nfsrv_readdirplus(
struct nfs_export *nx;
struct nfs_export_options *nxo;
uio_t auio = NULL;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
struct vnode_attr attr, va, *vap = &va;
int len, nlen, rem, xfer, error, attrerr, gotfh, gotattr;
int siz, dircount, maxcount, fullsiz, eofflag, dirlen, nentries, isdotdot;
diff --git a/bsd/sys/disk.h b/bsd/sys/disk.h
index 82efb4fcc000..47a4cd98e59f 100644
--- a/bsd/sys/disk.h
+++ b/bsd/sys/disk.h
@@ -284,7 +284,7 @@ typedef struct{
typedef struct{
boolean_t mi_mdev; /* Is this a memdev device? */
boolean_t mi_phys; /* Physical memory? */
- uint32_t mi_base; /* Base page number of the device? */
+ uint64_t mi_base; /* Base page number of the device (a kernel VA page number does not fit in 32 bits) */
uint64_t mi_size; /* Size of the device (in ) */
} dk_memdev_info_t;
diff --git a/bsd/sys/uio_internal.h b/bsd/sys/uio_internal.h
index 74a2036cf69e..a68b79ce48fa 100644
--- a/bsd/sys/uio_internal.h
+++ b/bsd/sys/uio_internal.h
@@ -170,7 +170,7 @@ __END_DECLS
#define UIO_SIZEOF( a_iovcount ) \
( sizeof(struct uio) + UIO_SIZEOF_IOVS(a_iovcount) )
-typedef char __attribute__((aligned(_Alignof(union iovecs)))) uio_stackbuf_t;
+typedef uint64_t uio_stackbuf_t; /* was over-aligned char; array of those is rejected by clang >= 16 */
#define UIO_IS_USER_SPACE32( a_uio_t ) \
( (a_uio_t)->uio_segflg == UIO_USERSPACE32 || (a_uio_t)->uio_segflg == UIO_PHYS_USERSPACE32 || \
diff --git a/bsd/vfs/vfs_attrlist.c b/bsd/vfs/vfs_attrlist.c
index cbaa0b221894..78315a2d54cf 100644
--- a/bsd/vfs/vfs_attrlist.c
+++ b/bsd/vfs/vfs_attrlist.c
@@ -878,7 +878,7 @@ static int
setattrlist_setfinderinfo(vnode_t vp, char *fndrinfo, struct vfs_context *ctx)
{
uio_t auio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
int error;
if ((auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_WRITE, uio_buf, sizeof(uio_buf))) == NULL) {
@@ -1847,7 +1847,7 @@ attr_pack_common(vfs_context_t ctx, mount_t mp, vnode_t vp, struct attrlist *alp
error = 0;
if (vp && !is_bulk) {
uio_t auio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if ((auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE,
UIO_READ, uio_buf, sizeof(uio_buf))) == NULL) {
@@ -3120,7 +3120,7 @@ getattrlist_internal(vfs_context_t ctx, vnode_t vp, struct attrlist *alp,
int pack_invalid;
int vtype = 0;
uio_t auio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
// must be true for fork attributes to be used as new common attributes
const int use_fork = (options & FSOPT_ATTR_CMN_EXTENDED) != 0;
@@ -3474,7 +3474,7 @@ refill_fd_direntries(vfs_context_t ctx, vnode_t dvp, struct fd_vn_data *fvd,
int *eofflagp)
{
uio_t rdir_uio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
size_t rdirbufsiz;
size_t rdirbufused;
int eofflag;
@@ -4048,7 +4048,7 @@ getattrlistbulk(proc_t p, struct getattrlistbulk_args *uap, int32_t *retval)
enum uio_seg segflg;
int count;
uio_t auio = NULL;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
kauth_action_t action;
int eofflag;
uint64_t options;
diff --git a/bsd/vfs/vfs_conf.c b/bsd/vfs/vfs_conf.c
index bae70cda3a96..63814100c68f 100644
--- a/bsd/vfs/vfs_conf.c
+++ b/bsd/vfs/vfs_conf.c
@@ -103,6 +103,19 @@ extern struct vfsops mockfs_vfsops;
extern int mockfs_mountroot(mount_t, vnode_t, vfs_context_t);
#endif /* MOCKFS */
+#if RAMFS
+extern struct vfsops ramfs_vfsops;
+#endif /* RAMFS */
+
+#if HFS
+extern struct vfsops hfs_vfsops;
+#endif /* HFS */
+
+#if APFS
+extern struct vfsops apfs_vfsops;
+extern int apfs_mountroot(mount_t, vnode_t, vfs_context_t);
+#endif /* APFS */
+
enum fs_type_num {
FT_NFS = 2,
FT_DEVFS = 19,
@@ -110,7 +123,10 @@ enum fs_type_num {
FT_ROUTEFS = 21,
FT_NULLFS = 22,
FT_BINDFS = 23,
- FT_MOCKFS = 0x6D6F636B
+ FT_MOCKFS = 0x6D6F636B,
+ FT_RAMFS = 0x72616D66,
+ FT_HFS = 0x11,
+ FT_APFS = 0x61706673
};
int fstypenumstart = FT_BINDFS + 1;
@@ -194,6 +210,79 @@ static struct vfstable vfstbllist[] = {
},
#endif /* BINDFS */
+#if RAMFS
+ /* Read-write filesystem in memory (no device, no mount arguments) */
+ {
+ .vfc_vfsops = &ramfs_vfsops,
+ .vfc_name = "ramfs",
+ .vfc_typenum = FT_RAMFS,
+ .vfc_refcount = 0,
+ .vfc_flags = MNT_LOCAL,
+ .vfc_mountroot = NULL,
+ .vfc_next = NULL,
+ .vfc_reserved1 = 0,
+ .vfc_reserved2 = 0,
+ .vfc_vfsflags = VFC_VFSGENERICARGS | VFC_VFS64BITREADY | VFC_VFSNATIVEXATTR,
+ .vfc_descptr = NULL,
+ .vfc_descsize = 0,
+ .vfc_sysctl = NULL
+ },
+#endif /* RAMFS */
+
+#if HFS
+ /*
+ * HFS Plus, read-write, built in rather than a kext (bsd/hfs, Apple's hfs-627.100.6): the data volume on the
+ * phone's flash, /private/var, where ramfs keeps it in memory. Its own arguments (struct hfs_mount_args) and
+ * local ones, as the kext registered itself (no VFS_TBLGENERICMNTARGS): the kernel looks up the device named
+ * by their first field and hands its vnode to hfs_mount(). Mounted by init, from user space.
+ */
+ {
+ .vfc_vfsops = &hfs_vfsops,
+ .vfc_name = "hfs",
+ .vfc_typenum = FT_HFS,
+ .vfc_refcount = 0,
+ .vfc_flags = MNT_LOCAL | MNT_DOVOLFS,
+ .vfc_mountroot = NULL,
+ .vfc_next = NULL,
+ .vfc_reserved1 = 0,
+ .vfc_reserved2 = 0,
+ .vfc_vfsflags = VFC_VFSLOCALARGS | VFC_VFSREADDIR_EXTENDED | VFC_VFS64BITREADY | VFC_VFSVNOP_PAGEOUTV2 |
+ VFC_VFSVNOP_PAGEINV2 | VFC_VFSNATIVEXATTR,
+ .vfc_descptr = NULL,
+ .vfc_descsize = 0,
+ .vfc_sysctl = NULL
+ },
+#endif /* HFS */
+
+#if APFS
+ /*
+ * Apple File System, read only: the root and every image of this port's (the RAM
+ * disks are APFS containers). Before mockfs, which accepts any root device: a device
+ * that is not APFS comes back EINVAL and vfs_mountroot() goes on to the next.
+ */
+ {
+ .vfc_vfsops = &apfs_vfsops,
+ .vfc_name = "apfs",
+ .vfc_typenum = FT_APFS,
+ .vfc_refcount = 0,
+ .vfc_flags = MNT_LOCAL | MNT_RDONLY,
+ .vfc_mountroot = apfs_mountroot,
+ .vfc_next = NULL,
+ .vfc_reserved1 = 0,
+ .vfc_reserved2 = 0,
+ /*
+ * Generic args, not local: a kernel mount of a filesystem marked
+ * VFC_VFSLOCALARGS is refused outright (mount_common), and this one has to be
+ * mountable from the kernel. So apfs_mount() takes the device's path as its
+ * mount data and opens it itself.
+ */
+ .vfc_vfsflags = VFC_VFSGENERICARGS | VFC_VFS64BITREADY | VFC_VFSNATIVEXATTR,
+ .vfc_descptr = NULL,
+ .vfc_descsize = 0,
+ .vfc_sysctl = NULL
+ },
+#endif /* APFS */
+
#if MOCKFS
/* If we are configured for it, mockfs should always be the last standard entry (and thus the last FS we attempt mountroot with) */
{
@@ -298,6 +387,21 @@ extern const struct vnodeopv_desc devfs_fdesc_vnodeop_opv_desc;
extern const struct vnodeopv_desc mockfs_vnodeop_opv_desc;
#endif /* MOCKFS */
+
+#if RAMFS
+extern const struct vnodeopv_desc ramfs_vnodeop_opv_desc;
+#endif /* RAMFS */
+
+#if HFS
+extern const struct vnodeopv_desc hfs_vnodeop_opv_desc;
+extern const struct vnodeopv_desc hfs_specop_opv_desc;
+extern const struct vnodeopv_desc hfs_fifoop_opv_desc;
+#endif /* HFS */
+
+#if APFS
+extern const struct vnodeopv_desc apfs_vnodeop_opv_desc;
+#endif /* APFS */
+
extern const struct vnodeopv_desc nullfs_vnodeop_opv_desc;
extern const struct vnodeopv_desc bindfs_vnodeop_opv_desc;
@@ -327,5 +431,16 @@ const struct vnodeopv_desc *vfs_opv_descs[] = {
#if MOCKFS
&mockfs_vnodeop_opv_desc,
#endif /* MOCKFS */
+#if RAMFS
+ &ramfs_vnodeop_opv_desc,
+#endif /* RAMFS */
+#if HFS
+ &hfs_vnodeop_opv_desc,
+ &hfs_specop_opv_desc,
+ &hfs_fifoop_opv_desc,
+#endif /* HFS */
+#if APFS
+ &apfs_vnodeop_opv_desc,
+#endif /* APFS */
NULL
};
diff --git a/bsd/vfs/vfs_lookup.c b/bsd/vfs/vfs_lookup.c
index a05c8e767b33..5f2846cb8893 100644
--- a/bsd/vfs/vfs_lookup.c
+++ b/bsd/vfs/vfs_lookup.c
@@ -1670,7 +1670,7 @@ lookup_handle_symlink(struct nameidata *ndp, vnode_t *new_dp, bool *new_dp_has_i
int error;
char *cp; /* pointer into pathname argument */
uio_t auio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
int need_newpathbuf;
u_int linklen = 0;
struct componentname *cnp = &ndp->ni_cnd;
diff --git a/bsd/vfs/vfs_quota.c b/bsd/vfs/vfs_quota.c
index d0476214e5f0..c81cadfab1d4 100644
--- a/bsd/vfs/vfs_quota.c
+++ b/bsd/vfs/vfs_quota.c
@@ -382,7 +382,7 @@ dqfileopen(struct quotafile *qfp, int type)
off_t file_size;
uio_t auio;
int error = 0;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
context.vc_thread = current_thread();
context.vc_ucred = qfp->qf_cred;
@@ -440,7 +440,7 @@ dqfileclose(struct quotafile *qfp, __unused int type)
struct dqfilehdr header;
struct vfs_context context;
uio_t auio;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
auio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ,
&uio_buf[0], sizeof(uio_buf));
@@ -767,7 +767,7 @@ dqlookup(struct quotafile *qfp, u_int32_t id, struct dqblk *dqb, uint32_t *index
int i, skip, last;
u_int32_t mask;
int error = 0;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
qf_lock(qfp);
@@ -986,7 +986,7 @@ dqsync_locked(struct dquot *dq)
struct dqblk dqb, *dqblkp;
uio_t auio;
int error;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (dq->dq_id == 0) {
dq->dq_flags &= ~DQ_MOD;
diff --git a/bsd/vfs/vfs_subr.c b/bsd/vfs/vfs_subr.c
index f64d8c647365..f2e13c1aa7f3 100644
--- a/bsd/vfs/vfs_subr.c
+++ b/bsd/vfs/vfs_subr.c
@@ -10995,7 +10995,7 @@ rmdir_remove_orphaned_appleDouble(vnode_t vp, vfs_context_t ctx, int * restart_f
uio_t auio = NULL;
int eofflag, siz = UIO_BUFF_SIZE, alloc_size = 0, nentries = 0;
int open_flag = 0, full_erase_flag = 0;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
char *rbuf = NULL;
void *dir_pos;
void *dir_end;
diff --git a/bsd/vfs/vfs_syscalls.c b/bsd/vfs/vfs_syscalls.c
index 2bbba037c2dd..ee65d5943196 100644
--- a/bsd/vfs/vfs_syscalls.c
+++ b/bsd/vfs/vfs_syscalls.c
@@ -7261,7 +7261,7 @@ readlinkat_internal(vfs_context_t ctx, int fd, vnode_t lnk_vp, user_addr_t path,
uio_t auio;
int error;
struct nameidata nd;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
bool put_vnode;
if (bufsize > INT32_MAX) {
@@ -10314,7 +10314,7 @@ getdirentries_common(int fd, user_addr_t bufp, user_size_t bufsize, ssize_t *byt
int spacetype = proc_is64bit(vfs_context_proc(&context)) ? UIO_USERSPACE64 : UIO_USERSPACE32;
off_t loff;
int error, numdirent;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
get_from_fd:
error = fp_getfvp(vfs_context_proc(&context), fd, &fp, &vp);
@@ -10613,7 +10613,7 @@ getdirentriesattr(proc_t p, struct getdirentriesattr_args *uap, int32_t *retval)
struct attrlist attributelist;
vfs_context_t ctx = vfs_context_current();
int fd = uap->fd;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
kauth_action_t action;
AUDIT_ARG(fd, fd);
@@ -10954,7 +10954,7 @@ searchfs(proc_t p, struct searchfs_args *uap, __unused int32_t *retval)
size_t mallocsize;
uint32_t nameiflags;
vfs_context_t ctx = vfs_context_current();
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
/* Start by copying in fsearchblock parameter list */
if (IS_64BIT_PROCESS(p)) {
@@ -12906,7 +12906,7 @@ getxattr(proc_t p, struct getxattr_args *uap, user_ssize_t *retval)
size_t namelen;
u_int32_t nameiflags;
int error;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (uap->options & (XATTR_NOSECURITY | XATTR_NODEFAULT)) {
return EINVAL;
@@ -12990,7 +12990,7 @@ fgetxattr(proc_t p, struct fgetxattr_args *uap, user_ssize_t *retval)
size_t attrsize = 0;
size_t namelen;
int error;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT)) {
return EINVAL;
@@ -13038,7 +13038,7 @@ out:
struct setxattr_ctx {
struct nameidata nd;
char attrname[XATTR_MAXNAMELEN + 1];
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
};
/*
@@ -13134,7 +13134,7 @@ fsetxattr(proc_t p, struct fsetxattr_args *uap, int *retval)
int spacetype = IS_64BIT_PROCESS(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
size_t namelen;
int error;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT)) {
return EINVAL;
@@ -13311,7 +13311,7 @@ listxattr(proc_t p, struct listxattr_args *uap, user_ssize_t *retval)
size_t attrsize = 0;
u_int32_t nameiflags;
int error;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (uap->options & (XATTR_NOSECURITY | XATTR_NODEFAULT)) {
return EINVAL;
@@ -13353,7 +13353,7 @@ flistxattr(proc_t p, struct flistxattr_args *uap, user_ssize_t *retval)
int spacetype = proc_is64bit(p) ? UIO_USERSPACE64 : UIO_USERSPACE32;
size_t attrsize = 0;
int error;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
if (uap->options & (XATTR_NOFOLLOW | XATTR_NOSECURITY | XATTR_NODEFAULT)) {
return EINVAL;
diff --git a/bsd/vfs/vfs_vnops.c b/bsd/vfs/vfs_vnops.c
index 5b51e9c46a8d..6872b94631c9 100644
--- a/bsd/vfs/vfs_vnops.c
+++ b/bsd/vfs/vfs_vnops.c
@@ -964,7 +964,7 @@ vn_rdwr_64(
int spacetype;
struct vfs_context context;
int error = 0;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
context.vc_thread = current_thread();
context.vc_ucred = cred;
diff --git a/bsd/vm/vm_unix.c b/bsd/vm/vm_unix.c
index 60c8b9eedd50..4384b6fec634 100644
--- a/bsd/vm/vm_unix.c
+++ b/bsd/vm/vm_unix.c
@@ -2538,7 +2538,19 @@ after_root_check:
#define PREBOOT_CRYPTEX_PATH "/private/preboot/Cryptexes"
#endif
error = vnode_lookup(PREBOOT_CRYPTEX_PATH, 0, &preboot_vp, vfs_context_current());
- if (error || srfmp->vp->v_mount != preboot_vp->v_mount) {
+ /*
+ * (redwood) Or a volume of its own mounted on a directory right inside it: Darwin
+ * grafts each cryptex image into the Preboot volume (graftdmg), so its files are
+ * that volume's; this port mounts the image there as it is (bsd/miscfs/apfs), a
+ * mount of its own whose covered directory is Preboot's Cryptexes/<name>.
+ */
+ if (error == 0 && srfmp->vp->v_mount != preboot_vp->v_mount &&
+ srfmp->vp->v_mount != NULL && srfmp->vp->v_mount->mnt_vnodecovered != NULLVP &&
+ srfmp->vp->v_mount->mnt_vnodecovered->v_mount == preboot_vp->v_mount &&
+ srfmp->vp->v_mount->mnt_vnodecovered->v_parent == preboot_vp) {
+ (void)vnode_put(preboot_vp);
+ preboot_vp = NULL;
+ } else if (error || srfmp->vp->v_mount != preboot_vp->v_mount) {
SHARED_REGION_TRACE_ERROR(
("shared_region: %p [%d(%s)] map(%p:'%s'): "
"not on process' root volume nor preboot volume\n",
diff --git a/config/MASTER b/config/MASTER
index 07147cb3188b..86e9eefb5ec7 100644
--- a/config/MASTER
+++ b/config/MASTER
@@ -113,6 +113,9 @@ options CONFIG_WORKLOOP_DEBUG # <config_workloop_debug>
# 4.4 filesystems
#
options MOCKFS # Boot from an executable # <mockfs>
+options APFS # Apple File System, read only # <apfs>
+options RAMFS # Read-write filesystem in memory # <ramfs>
+options HFS # HFS Plus, read-write: the data volume on the flash # <hfs>
options FIFO # fifo support # <fifo>
options FDESC # fdesc_fs support # <fdesc>
options DEVFS # devfs support # <devfs>
diff --git a/config/MASTER.arm64.bcm2837 b/config/MASTER.arm64.bcm2837
index 1ed75eea2052..4c67d211fb60 100644
--- a/config/MASTER.arm64.bcm2837
+++ b/config/MASTER.arm64.bcm2837
@@ -16,15 +16,15 @@
# Standard Apple OS Configurations:
# -------- ----- -- ---------------
#
-# KERNEL_BASE = [ arm64 config_pmap_ppl xsmall msgb_small config_embedded config_requires_u32_munging ARM_EXTRAS_BASE ]
+# KERNEL_BASE = [ arm64 config_pmap_ppl xsmall msgb_small config_embedded config_requires_u32_munging nos_arm_pmap nos_arm_asm ]
# KERNEL_RELEASE = [ KERNEL_BASE ]
# KERNEL_DEV = [ KERNEL_BASE development mach_assert config_xnupost proc_ref_debug ]
# KERNEL_DEBUG = [ KERNEL_BASE debug mach_assert config_xnupost config_waitq_stats config_workloop_debug ]
# BSD_BASE = [ mach_bsd psynch config_proc_uuid_policy config_personas ]
# BSD_RELEASE = [ BSD_BASE no_printf_str no_kprintf_str secure_kernel ]
-# BSD_DEV = [ BSD_BASE config_netboot config_imageboot config_coredump pgo config_vnguard ]
-# BSD_DEBUG = [ BSD_BASE config_netboot config_imageboot config_coredump pgo config_vnguard ]
-# FILESYS_BASE = [ devfs fifo fs_compression config_protect config_mnt_rootsnap config_fse routefs namedstreams bindfs]
+# BSD_DEV = [ BSD_BASE config_netboot config_imageboot config_coredump config_vnguard ]
+# BSD_DEBUG = [ BSD_BASE config_netboot config_imageboot config_coredump config_vnguard ]
+# FILESYS_BASE = [ devfs fifo fs_compression config_protect config_mnt_rootsnap config_fse routefs namedstreams bindfs mockfs ramfs apfs hfs ]
# FILESYS_RELEASE= [ FILESYS_BASE ]
# FILESYS_DEV = [ FILESYS_BASE config_union_mounts fdesc ]
# FILESYS_DEBUG = [ FILESYS_BASE config_union_mounts fdesc ]
@@ -50,8 +50,8 @@
# LIBKERN_DEBUG = [ LIBKERN_BASE iotracking ]
# PERF_DBG_BASE = [ mach_kdp config_serial_kdp kperf kpc ]
# PERF_DBG_RELEASE=[ PERF_DBG_BASE ist_kdebug ]
-# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger sched_hygiene_debug config_iotrace]
-# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger sched_hygiene_debug config_iotrace]
+# PERF_DBG_DEV = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger config_iotrace]
+# PERF_DBG_DEBUG = [ PERF_DBG_BASE config_dtrace zleaks kdp_interactive_debugging alternate_debugger config_iotrace]
# MACH_BASE = [ mach config_ecc_logging vc_progress_white importance_inheritance config_atm config_coalitions config_iosched config_library_validation config_sysdiagnose config_telemetry config_mach_bridge_recv_time config_quiesce_counter config_service_port_info ]
# MACH_RELEASE = [ MACH_BASE config_skip_precise_user_kernel_time debugger_for_zone_info ]
# MACH_DEV = [ MACH_BASE config_io_accounting importance_trace ]
@@ -70,10 +70,10 @@
# SECURITY_DEV = [ SECURITY_BASE config_setuid config_kas_info ]
# SECURITY_DEBUG = [ SECURITY_BASE config_setuid config_kas_info ]
# BASE = [ MULTIPATH PF VPN ]
-# RELEASE = [ BASE KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE SKYWALK_RELEASE NETWORKING_RELEASE IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG_RELEASE MACH_RELEASE SCHED_RELEASE VM_RELEASE SECURITY_RELEASE ]
-# DEVELOPMENT = [ BASE KERNEL_DEV BSD_DEV FILESYS_DEV NFS_DEV SKYWALK_DEV NETWORKING_DEV IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_DEV SECURITY_DEV ]
-# DEBUG = [ BASE KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG SKYWALK_DEBUG NETWORKING_DEBUG IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG_DEBUG MACH_DEBUG SCHED_DEBUG VM_DEBUG SECURITY_DEBUG ]
-# KASAN = [ BASE KERNEL_DEV BSD_DEV FILESYS_DEV NFS_DEV SKYWALK_DEV NETWORKING_DEV IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_KASAN SECURITY_DEV ]
+# RELEASE = [ BASE KERNEL_RELEASE BSD_RELEASE FILESYS_RELEASE SKYWALK_RELEASE NETWORKING_RELEASE IOKIT_RELEASE LIBKERN_RELEASE PERF_DBG_RELEASE MACH_RELEASE SCHED_RELEASE VM_RELEASE SECURITY_RELEASE HAKIOS_DRIVERS ]
+# DEVELOPMENT = [ BASE KERNEL_DEV BSD_DEV FILESYS_DEV NFS_DEV SKYWALK_DEV NETWORKING_DEV IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_DEV SECURITY_DEV HAKIOS_DRIVERS ]
+# DEBUG = [ BASE KERNEL_DEBUG BSD_DEBUG FILESYS_DEBUG SKYWALK_DEBUG NETWORKING_DEBUG IOKIT_DEBUG LIBKERN_DEBUG PERF_DBG_DEBUG MACH_DEBUG SCHED_DEBUG VM_DEBUG SECURITY_DEBUG HAKIOS_DRIVERS ]
+# KASAN = [ BASE KERNEL_DEV BSD_DEV FILESYS_DEV NFS_DEV SKYWALK_DEV NETWORKING_DEV IOKIT_DEV LIBKERN_DEV PERF_DBG_DEV MACH_DEV SCHED_DEV VM_KASAN SECURITY_DEV HAKIOS_DRIVERS ]
#
######################################################################
#
diff --git a/iokit/Kernel/IOPMrootDomain.cpp b/iokit/Kernel/IOPMrootDomain.cpp
index 4a06407aea5f..2d2383953f7f 100644
--- a/iokit/Kernel/IOPMrootDomain.cpp
+++ b/iokit/Kernel/IOPMrootDomain.cpp
@@ -555,7 +555,16 @@ static uint32_t gDarkWakeFlags = kDarkWakeFlagPromotionEarly;
#endif /* !defined(XNU_TARGET_OS_OSX) */
static uint32_t gNoIdleFlag = 0;
-static uint32_t gSleepDisabledFlag = 0;
+/*
+ * hakios: the system does not sleep. There is nothing here to sleep into and nothing that would
+ * wake it: with the screen locked, powerd asks for system sleep within seconds, and that request
+ * ended in IOCPUSleepKernel() -- with one processor it came straight back out, again and again
+ * (every time a round of sleep and wake notifications to all the system); with more than one it
+ * stopped there for good, waiting for a processor to go that never goes. Refused instead, the
+ * system stays awake with its display off, and the side button or a tap wakes the screen.
+ * Boot-arg syssleep=1 gives the old behaviour back.
+ */
+static uint32_t gSleepDisabledFlag = 1;
static uint32_t gSwdPanic = 1;
static uint32_t gSwdSleepTimeout = 0;
static uint32_t gSwdWakeTimeout = 0;
@@ -1587,6 +1596,13 @@ IOPMrootDomain::start( IOService * nub )
#endif /* DEVELOPMENT || DEBUG */
PE_parse_boot_argn("noidle", &gNoIdleFlag, sizeof(gNoIdleFlag));
+ {
+ uint32_t syssleep = 0;
+
+ if (PE_parse_boot_argn("syssleep", &syssleep, sizeof(syssleep)) && syssleep) {
+ gSleepDisabledFlag = 0;
+ }
+ }
PE_parse_boot_argn("swd_sleeptimeout", &gSwdSleepTimeout, sizeof(gSwdSleepTimeout));
PE_parse_boot_argn("swd_waketimeout", &gSwdWakeTimeout, sizeof(gSwdWakeTimeout));
PE_parse_boot_argn("swd_timeout", &gSwdSleepWakeTimeout, sizeof(gSwdSleepWakeTimeout));
diff --git a/iokit/Kernel/IOPlatformExpert.cpp b/iokit/Kernel/IOPlatformExpert.cpp
index 7a32a23e4bb1..35dc7b503846 100644
--- a/iokit/Kernel/IOPlatformExpert.cpp
+++ b/iokit/Kernel/IOPlatformExpert.cpp
@@ -51,6 +51,9 @@
#include <libkern/c++/OSContainers.h>
#include <libkern/c++/OSSharedPtr.h>
#include <libkern/crypto/sha1.h>
+#include <libkern/crypto/register_crypto.h>
+
+extern "C" crypto_functions_t g_crypto_funcs; /* NULL until a crypto provider has registered */
#include <libkern/OSAtomic.h>
#if defined(__arm64__)
@@ -2176,6 +2179,14 @@ IOPlatformExpertDevice::generatePlatformUUID( void )
uuid_string_t uuid;
#if !defined(__x86_64__)
+ /*
+ * The first call (configureDefaults, from InitIOKit) comes before the crypto provider
+ * has registered when that provider is not Apple's corecrypto; the identifier is then
+ * made by the second call, when the platform expert starts.
+ */
+ if (g_crypto_funcs == NULL) {
+ return;
+ }
entry = IORegistryEntry::fromPath( "/chosen", gIODTPlane );
if (entry) {
OSData * data1;
diff --git a/iokit/Kernel/IOService.cpp b/iokit/Kernel/IOService.cpp
index d2442944e5f0..e90ba6738516 100644
--- a/iokit/Kernel/IOService.cpp
+++ b/iokit/Kernel/IOService.cpp
@@ -5179,7 +5179,10 @@ IOService::setRootMedia(IOService * root)
bool unhide;
ioblockstoragedriverClass = OSMetaClass::getMetaClassWithName(gIOBlockStorageDriverKey);
- assert(ioblockstoragedriverClass);
+ if (!ioblockstoragedriverClass) {
+ // No IOStorageFamily in this kernel: the root cannot be a block storage device
+ root = NULL;
+ }
while (root) {
if (root->metaCast(ioblockstoragedriverClass)) {
diff --git a/iokit/Kernel/IOUserClient.cpp b/iokit/Kernel/IOUserClient.cpp
index 07aa0be4a0ca..3832eaff2184 100644
--- a/iokit/Kernel/IOUserClient.cpp
+++ b/iokit/Kernel/IOUserClient.cpp
@@ -3536,6 +3536,25 @@ IOCopyPropertyCompatible(IORegistryEntry * regEntry, const char * name)
return obj;
}
+/*
+ * Boot-arg regtrace=1: report the registry properties user space asks for and does not
+ * find (regtrace=2: every property asked for). Shows what a daemon expects of a driver.
+ */
+static TUNABLE(uint32_t, regtrace_level, "regtrace", 0);
+
+static void
+regtrace_report(IORegistryEntry * entry, const char * property, bool found)
+{
+ char procname[32] = "?";
+
+ if (regtrace_level == 0 || (found && regtrace_level < 2)) {
+ return;
+ }
+ proc_selfname(procname, sizeof(procname));
+ IOLog("regtrace: %s[%d] %s(%s).%s %s\n", procname, proc_selfpid(), entry->getName(),
+ entry->getMetaClass()->getClassName(), property, found ? "found" : "MISSING");
+}
+
/* Routine io_registry_entry_get_property */
kern_return_t
is_io_registry_entry_get_property_bytes(
@@ -3563,6 +3582,7 @@ is_io_registry_entry_get_property_bytes(
#endif
obj = IOCopyPropertyCompatible(entry, property_name);
+ regtrace_report(entry, property_name, obj != NULL);
if (!obj) {
return kIOReturnNoResources;
}
@@ -3631,6 +3651,7 @@ is_io_registry_entry_get_property(
#endif
obj = IOCopyPropertyCompatible(entry, property_name);
+ regtrace_report(entry, property_name, obj != NULL);
if (!obj) {
return kIOReturnNotFound;
}
@@ -3679,6 +3700,7 @@ is_io_registry_entry_get_property_recursively(
obj = entry->copyProperty( property_name,
IORegistryEntry::getPlane( plane ), options );
+ regtrace_report(entry, property_name, obj != NULL);
if (!obj) {
return kIOReturnNotFound;
}
@@ -3873,6 +3895,7 @@ is_io_registry_entry_get_property_bin_buf(
}
err = kIOReturnNotFound;
+ IORegistryEntry * asked = entry;
if (gIORegistryEntryPropertyKeysKey == sym) {
obj = entry->copyPropertyKeys();
} else {
@@ -3905,6 +3928,7 @@ is_io_registry_entry_get_property_bin_buf(
}
sym->release();
+ regtrace_report(asked, property_name, obj != NULL);
if (!obj) {
return err;
}
diff --git a/iokit/KernelConfigTables.cpp b/iokit/KernelConfigTables.cpp
index 2b14eda8f68f..09337bad5b01 100644
--- a/iokit/KernelConfigTables.cpp
+++ b/iokit/KernelConfigTables.cpp
@@ -32,6 +32,10 @@
* Version 2.0.
*/
+#if defined(__arm64__)
+#include <pexpert/arm64/board_config.h>
+#endif
+
const char * gIOKernelConfigTables =
"("
" {"
@@ -39,4 +43,11 @@ const char * gIOKernelConfigTables =
" 'IOProviderClass' = IOPlatformExpertDevice;"
" 'IOProbeScore' = 0:32;"
" }"
+#if defined(__arm64__) && !defined(APPLE_ARM64_ARCH_FAMILY)
+ " ,{"
+ " 'IOClass' = GenericARMPE;"
+ " 'IOProviderClass' = IOPlatformExpertDevice;"
+ " 'IOProbeScore' = 1000:32;"
+ " }"
+#endif
")";
diff --git a/iokit/bsddev/IOKitBSDInit.cpp b/iokit/bsddev/IOKitBSDInit.cpp
index 5219fb0d9ed6..29978e9cff43 100644
--- a/iokit/bsddev/IOKitBSDInit.cpp
+++ b/iokit/bsddev/IOKitBSDInit.cpp
@@ -683,7 +683,14 @@ IOFindBSDRoot( char * rootName, unsigned int rootNameSize,
assert(matching);
matching->setObject(gIOResourceMatchedKey, gIOBSDKey);
- if ((service = IOService::waitForMatchingService(matching, 30ULL * kSecondScale))) {
+ // A memory-device root needs no filesystem kext, so do not sit out the timeout for one
+ char rdEarly[kMaxBootVar];
+ uint64_t bsdWait = 30ULL * kSecondScale;
+ if (PE_parse_boot_argn("rd", rdEarly, sizeof(rdEarly)) && rdEarly[0] == 'm' && rdEarly[1] == 'd') {
+ bsdWait = kSecondScale / 10;
+ }
+
+ if ((service = IOService::waitForMatchingService(matching, bsdWait))) {
OSSafeReleaseNULL(service);
} else {
IOLog("!BSD\n");
@@ -762,6 +769,17 @@ IOFindBSDRoot( char * rootName, unsigned int rootNameSize,
#endif /* __LP64__ */
(void)mdevadd(-1, ml_static_ptovirt(ramdParms[0]) >> 12, (unsigned int) (ramdParms[1] >> 12), 0); /* Initialize it and pass back the device number */
}
+ /*
+ * A second image the loader may have left in memory (it becomes
+ * /dev/md1): a volume for user space to mount, never the root.
+ */
+ data = (OSData *)regEntry->getProperty("RAMDisk1");
+ if (data && data->getLength() >= 2 * sizeof(uintptr_t)) {
+ uintptr_t *ramdParms = (uintptr_t *)data->getBytesNoCopy();
+ if (ramdParms[1] <= (((uint64_t)UINT_MAX) << 12)) {
+ (void)mdevadd(-1, ml_static_ptovirt(ramdParms[0]) >> 12, (unsigned int) (ramdParms[1] >> 12), 0);
+ }
+ }
regEntry->release(); /* Toss the entry */
}
}
diff --git a/iokit/conf/Makefile b/iokit/conf/Makefile
index 83259ebf9835..8864e5c7cc41 100644
--- a/iokit/conf/Makefile
+++ b/iokit/conf/Makefile
@@ -16,6 +16,9 @@ endif
MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/iokit/conf/files b/iokit/conf/files
index 5fb0a52062e0..7cfee3142b5b 100644
--- a/iokit/conf/files
+++ b/iokit/conf/files
@@ -79,6 +79,17 @@ iokit/Kernel/IOSubMemoryDescriptor.cpp optional iokitcpp
iokit/Kernel/IOGuardPageMemoryDescriptor.cpp optional iokitcpp
iokit/Kernel/IOPlatformExpert.cpp optional iokitcpp
+iokit/Kernel/GenericARMPE.cpp optional iokitcpp
+iokit/Kernel/MiniIOSurface.cpp optional iokitcpp
+iokit/Kernel/MiniIOMobileFramebuffer.cpp optional iokitcpp
+# HakiOS drivers: made by tools/hakios-config.py kernel from drivers/*/conf. Edit those, not these lines.
+iokit/Kernel/hakios.stubs.cpp optional iokitcpp
+iokit/Kernel/hakios.gpu.cpp optional hakios_adreno iokitcpp
+iokit/Kernel/hakios.battery.cpp optional hakios_battery iokitcpp
+iokit/Kernel/hakios.wlan.cpp optional hakios_wifi iokitcpp
+# end of the HakiOS drivers
+iokit/Kernel/MiniAppleKeyStore.cpp optional iokitcpp
+iokit/Kernel/MiniIONetworkStack.cpp optional iokitcpp
iokit/Kernel/IOPlatformIO.cpp optional iokitcpp
iokit/Kernel/IOCPU.cpp optional iokitcpp
diff --git a/libkern/conf/Makefile b/libkern/conf/Makefile
index 050f34a53c49..c58ff62cd419 100644
--- a/libkern/conf/Makefile
+++ b/libkern/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/libkern/conf/files b/libkern/conf/files
index ece14c7d41d3..4a903f73a9ef 100644
--- a/libkern/conf/files
+++ b/libkern/conf/files
@@ -109,6 +109,8 @@ libkern/apple_encrypted_archive/apple_encrypted_archive.c standard
libkern/img4/interface.c standard
libkern/amfi/amfi.c standard
+libkern/amfi/miniamfi.c standard
+libkern/amfi/miniamfi_entitlements.cpp standard
libkern/stack_protector.c standard
libkern/ptrauth_utils.c standard
diff --git a/libkern/libkern/c++/OSData.h b/libkern/libkern/c++/OSData.h
index 28ea44c2ae9d..9c7a1fe9e23e 100644
--- a/libkern/libkern/c++/OSData.h
+++ b/libkern/libkern/c++/OSData.h
@@ -965,7 +965,7 @@ private:
"only trivially copyable types can be contained");
static_assert(!__is_pointer(T) || (acquisition == kValueNoCopy),
"pointers cannot be contained");
- static_assert(KALLOC_TYPE_IS_DATA_ONLY(T) || (acquisition == kValueNoCopy),
+ static_assert(XNU_NO_TYPE_BUILTINS || KALLOC_TYPE_IS_DATA_ONLY(T) || (acquisition == kValueNoCopy),
"only plain data types can be contained (consider using OSValueObject)");
return 0; // C++11 does not support void-returning constexpr functions
}
diff --git a/libsa/conf/Makefile b/libsa/conf/Makefile
index 050f34a53c49..c58ff62cd419 100644
--- a/libsa/conf/Makefile
+++ b/libsa/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/makedefs/MakeInc.cmd b/makedefs/MakeInc.cmd
index bc39e19b3b8e..0a72a2d43908 100644
--- a/makedefs/MakeInc.cmd
+++ b/makedefs/MakeInc.cmd
@@ -100,12 +100,12 @@ ifeq ($(VERBOSE),YES)
_v =
_vstdout =
_vstderr =
- XCRUN = /usr/bin/xcrun -verbose
+ XCRUN = xcrun -verbose
else
_v = @
_vstdout = > /dev/null
_vstderr = 2> /dev/null
- XCRUN = /usr/bin/xcrun
+ XCRUN = xcrun
endif
VERBOSE_GENERATED_MAKE_FRAGMENTS = NO
@@ -275,27 +275,27 @@ REPLACECONTENTS = $(OBJROOT)/SETUP/replacecontents/replacecontents
JSONCOMPILATIONDB = $(OBJROOT)/SETUP/json_compilation_db/json_compilation_db
# Standard BSD tools
-RM = /bin/rm -f
-RMDIR = /bin/rmdir
-CP = /bin/cp
-MV = /bin/mv
-LN = /bin/ln -fs
-CAT = /bin/cat
-MKDIR = /bin/mkdir -p
-CHMOD = /bin/chmod
-FIND = /usr/bin/find
-XARGS = /usr/bin/xargs
-PAX = /bin/pax
-BASENAME = /usr/bin/basename
-DIRNAME = /usr/bin/dirname
-TR = /usr/bin/tr
-TOUCH = /usr/bin/touch
-SLEEP = /bin/sleep
-AWK = /usr/bin/awk
-SED = /usr/bin/sed
-PLUTIL = /usr/bin/plutil
-PATCH = /usr/bin/patch
-GREP = /usr/bin/grep
+RM = rm -f
+RMDIR = rmdir
+CP = cp
+MV = mv
+LN = ln -fs
+CAT = cat
+MKDIR = mkdir -p
+CHMOD = chmod
+FIND = find
+XARGS = xargs
+PAX = pax
+BASENAME = basename
+DIRNAME = dirname
+TR = tr
+TOUCH = touch
+SLEEP = sleep
+AWK = awk
+SED = sed
+PLUTIL = plutil
+PATCH = patch
+GREP = grep
#
# Command to generate host binaries. Intentionally not
@@ -317,7 +317,7 @@ ifeq ($(origin HOST_GM4),undefined)
export HOST_GM4 := $(shell $(XCRUN) -sdk $(HOST_SDKROOT) -find gm4)
endif
ifeq ($(origin HOST_CODESIGN),undefined)
- export HOST_CODESIGN := /usr/bin/codesign
+ export HOST_CODESIGN := codesign
endif
ifeq ($(origin HOST_CODESIGN_ALLOCATE),undefined)
export HOST_CODESIGN_ALLOCATE := $(shell $(XCRUN) -sdk $(HOST_SDKROOT) -find codesign_allocate)
diff --git a/makedefs/MakeInc.def b/makedefs/MakeInc.def
index 5bd6f23dd10c..252d50062da4 100644
--- a/makedefs/MakeInc.def
+++ b/makedefs/MakeInc.def
@@ -60,7 +60,7 @@ MACHINE_FLAGS_ARM64_T8101 = -DARM64_BOARD_CONFIG_T8101 -mcpu=apple-a14
MACHINE_FLAGS_ARM64_T8103 = -DARM64_BOARD_CONFIG_T8103 -mcpu=apple-a14
MACHINE_FLAGS_ARM64_T6000 = -DARM64_BOARD_CONFIG_T6000 -mcpu=apple-a14
MACHINE_FLAGS_ARM64_VMAPPLE = -DARM64_BOARD_CONFIG_VMAPPLE -march=armv8.5a
-MACHINE_FLAGS_ARM64_BCM2837 = -DARM64_BOARD_CONFIG_BCM2837
+MACHINE_FLAGS_ARM64_BCM2837 = -DARM64_BOARD_CONFIG_BCM2837 -Xclang -target-feature -Xclang +pan
#
@@ -857,7 +857,10 @@ LDFILES_KERNEL_ONLY = $(TARGET)/all-kpi.exp $(TARGET)/all-alias.exp $(TARGET)/sy
# Default runtime libraries to be linked with the kernel
#
LD_KERNEL_LIBS = -lcc_kext
-LD_KERNEL_ARCHIVES = $(LDFLAGS_KERNEL_SDK) -lfirehose_kernel
+LD_KERNEL_ARCHIVES = $(LDFLAGS_KERNEL_SDK) -lfirehose_kernel -lpthread_kext
+# Darwin-on-hakios: objects made outside this tree, linked when the build is given any (the
+# Wi-Fi driver, see bsd/dev/hakios.lx.c)
+LD_KERNEL_ARCHIVES += $(HAKIOS_OBJECTS)
# Link opensource binary library
ifneq ($(filter T6000 T8101 VMAPPLE T6000 T8101 VMAPPLE,$(CURRENT_MACHINE_CONFIG)),)
LDFLAGS_KERNEL_ONLY += -rdynamic -Wl,-force_load,$(KDKROOT)/System/Library/KernelSupport/lib$(CURRENT_MACHINE_CONFIG).os.$(CURRENT_KERNEL_CONFIG).a
diff --git a/makedefs/MakeInc.top b/makedefs/MakeInc.top
index 0316792a3f42..b9c195b09cd1 100644
--- a/makedefs/MakeInc.top
+++ b/makedefs/MakeInc.top
@@ -285,7 +285,7 @@ endif
# $(warning PRIMARY_KERNEL_CONFIGS is $(PRIMARY_KERNEL_CONFIGS))
# $(warning VARIANT_CONFIGS is $(VARIANT_CONFIGS))
-MEMORY_SIZE := $(shell /usr/sbin/sysctl -n hw.memsize)
+MEMORY_SIZE := $(shell if [ -r /proc/meminfo ]; then awk '/MemTotal/{printf "%d", $$2*1024}' /proc/meminfo; else sysctl -n hw.memsize; fi)
# Assume LTO scaling by default, unless it is being explicitly passed on the command-line
LARGE_BUILD_FOOTPRINT := $(if $(BUILD_LTO),$(BUILD_LTO),1)
diff --git a/osfmk/arm/arm_init.c b/osfmk/arm/arm_init.c
index e26a1d70f470..f91095c98d26 100644
--- a/osfmk/arm/arm_init.c
+++ b/osfmk/arm/arm_init.c
@@ -73,6 +73,9 @@
#include <sys/kdebug.h>
#include <pexpert/pexpert.h>
+#if defined(__arm64__)
+#include <arm64/psci_smp.h>
+#endif
#include <console/serial_protos.h>
@@ -335,7 +338,11 @@ arm_setup_pre_sign(void)
*/
__startup_func
+/* Bring-up checkpoints, see osfmk/arm64/loader_return.c */
+extern void loader_checkpoint(uint64_t n);
+
void
+
arm_init(
boot_args *args)
{
@@ -354,6 +361,7 @@ arm_init(
const_boot_args = *args;
BootArgs = args = &const_boot_args;
+ loader_checkpoint(10);
cpu_data_init(&BootCpuData);
#if defined(HAS_APPLE_PAC)
/* bootstrap cpu process dependent key for kernel has been loaded by start.s */
@@ -362,14 +370,19 @@ arm_init(
#endif /* defined(HAS_APPLE_PAC) */
PE_init_platform(FALSE, args); /* Get platform expert set up */
+ loader_checkpoint(11);
#if __arm64__
configure_timer_apple_regs();
+ loader_checkpoint(30);
wfe_timeout_configure();
+ loader_checkpoint(31);
wfe_timeout_init();
+ loader_checkpoint(32);
configure_misc_apple_boot_args();
configure_misc_apple_regs();
+ loader_checkpoint(33);
#if (DEVELOPMENT || DEBUG)
unsigned long const *platform_stall_ptr = NULL;
@@ -385,6 +398,7 @@ arm_init(
}
platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_ARM_INIT);
+ loader_checkpoint(34);
chosen = NULL; // Force a re-lookup later on since VM addresses are not final at this point
dt_entry_size = 0;
@@ -422,10 +436,13 @@ arm_init(
}
#endif
+ loader_checkpoint(35);
ml_parse_cpu_topology();
+ loader_checkpoint(12);
master_cpu = ml_get_boot_cpu_number();
+ loader_checkpoint(40);
assert(master_cpu >= 0 && master_cpu <= ml_get_max_cpu_number());
BootCpuData.cpu_number = (unsigned short)master_cpu;
@@ -439,11 +456,14 @@ arm_init(
CpuDataEntries[master_cpu].cpu_data_paddr = (void *)((uintptr_t)(args->physBase)
+ ((uintptr_t)&BootCpuData
- (uintptr_t)(args->virtBase)));
+ loader_checkpoint(41);
thread = thread_bootstrap();
+ loader_checkpoint(42);
thread->machine.CpuDatap = &BootCpuData;
thread->machine.pcpu_data_base = (vm_offset_t)0;
machine_set_current_thread(thread);
+ loader_checkpoint(43);
/*
* Preemption is enabled for this thread so that it can lock mutexes without
@@ -452,12 +472,16 @@ arm_init(
* preemption level is not really meaningful for the bootstrap thread.
*/
thread->machine.preemption_count = 0;
+ loader_checkpoint(44);
cpu_bootstrap();
+ loader_checkpoint(13);
rtclock_early_init();
+ loader_checkpoint(14);
kernel_debug_string_early("kernel_startup_bootstrap");
kernel_startup_bootstrap();
+ loader_checkpoint(15);
/*
* Initialize the timer callout world
@@ -465,6 +489,7 @@ arm_init(
timer_call_init();
cpu_init();
+ loader_checkpoint(16);
processor_bootstrap();
@@ -514,6 +539,7 @@ arm_init(
#endif /* __ARM_PAN_AVAILABLE__ */
arm_vm_init(xmaxmem, args);
+ loader_checkpoint(17);
if (debug_boot_arg) {
patch_low_glo();
@@ -527,7 +553,9 @@ arm_init(
/* Setup debugging output. */
const unsigned int serial_exists = serial_init();
+ loader_checkpoint(18);
kernel_startup_initialize_upto(STARTUP_SUB_KPRINTF);
+ loader_checkpoint(19);
kprintf("kprintf initialized\n");
serialmode = 0;
@@ -569,6 +597,7 @@ arm_init(
disableConsoleOutput = FALSE; /* Allow printfs to happen */
}
PE_create_console();
+ loader_checkpoint(20);
/* setup console output */
PE_init_printf(FALSE);
@@ -647,6 +676,7 @@ arm_init(
* against string vulnerabilities
*/
__stack_chk_guard &= ~(0xFFULL << 8);
+ loader_checkpoint(21);
machine_startup(args);
}
@@ -673,6 +703,9 @@ arm_init_cpu(
machine_set_current_thread(cpu_data_ptr->cpu_active_thread);
+#if defined(GENERIC_PSCI_SMP)
+ psci_smp_cpu_reached(PSCI_SMP_CPU_IN_KERNEL);
+#endif
#if APPLE_ARM64_ARCH_FAMILY
configure_late_apple_regs();
diff --git a/osfmk/arm/model_dep.c b/osfmk/arm/model_dep.c
index 39daaf8db949..64ceb1a5f201 100644
--- a/osfmk/arm/model_dep.c
+++ b/osfmk/arm/model_dep.c
@@ -79,6 +79,9 @@
#include <arm/cpu_data_internal.h>
#include <arm/cpu_internal.h>
#include <arm/misc_protos.h>
+#if defined(__arm64__)
+#include <arm64/psci_smp.h>
+#endif
#include <libkern/OSKextLibPrivate.h>
#include <vm/vm_kern.h>
#include <kern/kern_cdata.h>
@@ -1156,6 +1159,13 @@ wait_while_mp_kdp_trap(bool check_SIGPdebug)
found_SIGPdebug = true;
break;
}
+#if defined(GENERIC_PSCI_SMP)
+ /*
+ * The panic this processor is waiting out ends in a return to the boot loader:
+ * its interrupts are off here, so this is where it is told (psci_smp.c).
+ */
+ psci_smp_stop_here();
+#endif
__builtin_arm_wfe();
}
os_atomic_clear_exclusive();
diff --git a/osfmk/arm/pmap/pmap.c b/osfmk/arm/pmap/pmap.c
index fc2096fa0010..8f198ed1b820 100644
--- a/osfmk/arm/pmap/pmap.c
+++ b/osfmk/arm/pmap/pmap.c
@@ -93,6 +93,8 @@
#include <pexpert/device_tree.h>
+MARK_AS_PMAP_TEXT static void pmap_phys_write_disable(vm_address_t va);
+
#include <san/kasan.h>
#include <sys/cdefs.h>
@@ -4676,8 +4678,8 @@ pmap_page_protect_options_with_flush_range(
}
#endif
if (remove && (options & PMAP_OPTIONS_COMPRESSOR)) {
- panic("pmap_page_protect: attempt to compress ppnum 0x%x owned by iommu %p, pve_p=%p",
- ppnum, ptep_get_iommu(pte_p), pve_p);
+ panic("pmap_page_protect: attempt to compress ppnum 0x%x owned by iommu, pte_p=%p, pve_p=%p",
+ ppnum, pte_p, pve_p);
}
goto protect_skip_pve_pass1;
}
@@ -6877,7 +6879,8 @@ void
pmap_sync_page_attributes_phys(
ppnum_t pp)
{
- flush_dcache((vm_offset_t) (pp << PAGE_SHIFT), PAGE_SIZE, TRUE);
+ /* (hakios: the page number is 32 bits wide and memory goes on above 4 GB: widen before shifting) */
+ flush_dcache((vm_offset_t) pp << PAGE_SHIFT, PAGE_SIZE, TRUE);
}
#if CONFIG_COREDUMP
@@ -11623,7 +11626,8 @@ pmap_cs_lockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable)
#if XNU_MONITOR
pmap_ppl_lockdown_pages(kva, size, PVH_FLAG_LOCKDOWN_CS, ppl_writable);
#else
- pmap_ppl_lockdown_pages(kva, size, 0, ppl_writable);
+ /* No PPL on this platform: nothing to lock down. */
+ (void)kva; (void)size; (void)ppl_writable;
#endif
}
@@ -11633,7 +11637,8 @@ pmap_cs_unlockdown_pages(vm_address_t kva, vm_size_t size, bool ppl_writable)
#if XNU_MONITOR
pmap_ppl_unlockdown_pages(kva, size, PVH_FLAG_LOCKDOWN_CS, ppl_writable);
#else
- pmap_ppl_unlockdown_pages(kva, size, 0, ppl_writable);
+ /* No PPL on this platform: nothing to lock down. */
+ (void)kva; (void)size; (void)ppl_writable;
#endif
}
diff --git a/osfmk/arm/pmap/pmap.h b/osfmk/arm/pmap/pmap.h
index df672c65b144..d7e1da7c5db2 100644
--- a/osfmk/arm/pmap/pmap.h
+++ b/osfmk/arm/pmap/pmap.h
@@ -112,7 +112,13 @@
*/
#define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 34)
#else /* defined(ARM_LARGE_MEMORY) */
-#define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 8)
+/*
+ * With 4K pages every GB of DRAM costs one L2 page in each of the two
+ * bootstrap mappings (start.s maps all of memSize). Apple sized this for a
+ * 2 GB device (8 pages); 20 pages cover 8 GB, enough for a boot image of
+ * several GB next to the kernel.
+ */
+#define BOOTSTRAP_TABLE_SIZE (ARM_PGBYTES * 20)
#endif /* defined(ARM_LARGE_MEMORY) */
typedef uint64_t tt_entry_t; /* translation table entry type */
@@ -442,7 +448,7 @@ struct pmap {
#if HAS_APPLE_PAC
bool disable_jop;
#else
- bool reserved10;
+ bool reserved12;
#endif /* HAS_APPLE_PAC */
bool reserved11;
diff --git a/osfmk/arm64/corecrypto/arm64_isa_compatibility.h b/osfmk/arm64/corecrypto/arm64_isa_compatibility.h
index b9549da357b4..cca31ea32502 100644
--- a/osfmk/arm64/corecrypto/arm64_isa_compatibility.h
+++ b/osfmk/arm64/corecrypto/arm64_isa_compatibility.h
@@ -28,7 +28,7 @@
// #include <Availability.h>
#include <sys/cdefs.h>
-#if defined(__clang__) && ((defined(__apple_build_version__) && __apple_build_version__ > 5010000))
+#if defined(__clang__) && (!defined(__apple_build_version__) || __apple_build_version__ > 5010000)
#define __USES_V_CRYPTO_INTRINSICS 1
#else
#define __USES_V_CRYPTO_INTRINSICS 0
diff --git a/osfmk/arm64/cpu.c b/osfmk/arm64/cpu.c
index 94e91306b143..0ec1323086b7 100644
--- a/osfmk/arm64/cpu.c
+++ b/osfmk/arm64/cpu.c
@@ -568,7 +568,12 @@ cpu_idle(void)
#endif /* MONOTONIC */
if (wfi) {
-#if !defined(APPLE_ARM64_ARCH_FAMILY)
+#if !defined(APPLE_ARM64_ARCH_FAMILY) && !defined(GENERIC_PSCI_SMP)
+ /*
+ * (Not on the generic boards either. There a core keeps its cache while it waits,
+ * and this would empty it by set and way every time: slow, wrong among several
+ * processors, and under a hypervisor every one of those instructions is a trap.)
+ */
platform_cache_idle_enter();
#endif
diff --git a/osfmk/arm64/machine_routines.c b/osfmk/arm64/machine_routines.c
index e76e7e70ab9c..828a51171c7e 100644
--- a/osfmk/arm64/machine_routines.c
+++ b/osfmk/arm64/machine_routines.c
@@ -734,6 +734,9 @@ ml_get_machine_mem(void)
return machine_info.memory_size;
}
+extern void loader_return(void);
+extern void hakios_adreno_stop(void);
+
__attribute__((noreturn))
void
halt_all_cpus(boolean_t reboot)
@@ -745,6 +748,9 @@ halt_all_cpus(boolean_t reboot)
printf("CPU halted\n");
PEHaltRestart(kPEHaltCPU);
}
+ /* No platform driver powers the machine off: hand control (and the log) back to the loader. */
+ hakios_adreno_stop();
+ loader_return();
while (1) {
;
}
@@ -932,6 +938,99 @@ ml_read_chip_revision(unsigned int *rev __unused)
#endif
}
+/*
+ * The slower cores (an L2 access penalty, or the loader's "small-core"): each is made the SMT
+ * sibling ("secondary") of a faster core of its own, and the scheduler then treats it as XNU
+ * treats the second thread of a hyperthreaded core: a thread is put on it only when no faster
+ * core is idle or running something of lower priority; it looks for work only when none of
+ * them is idle; and what runs on it goes back to the run queue at every rescheduling, to be
+ * put on a faster core that has come free meanwhile. (XNU itself pairs such a core with the
+ * boot processor, which allows one of them; the phone has three: with cpus=8, cores 3, 2 and
+ * 1 go with 4, 5 and 6, so that 7, the boot core and the fastest, where real-time threads go
+ * first, stays alone: the scheduler idles a secondary while its primary runs a real-time
+ * thread.)
+ */
+static uint32_t small_cores_apart = 2; /* boot-arg small_cores, read in ml_parse_cpu_topology() (before tunables are) */
+static int ml_slow_core_partner[MAX_CPUS]; /* by cpu_id: the faster core it is paired with, -1 if none */
+
+static void
+ml_choose_slow_core_partners(void)
+{
+ bool taken[MAX_CPUS] = { false };
+
+ for (int i = 0; i < MAX_CPUS; i++) {
+ ml_slow_core_partner[i] = -1;
+ }
+ for (int i = 0; i < (int)topology_info.num_cpus; i++) {
+ int partner = -1;
+
+ if (topology_info.cpus[i].l2_access_penalty == 0) {
+ continue;
+ }
+ /* the last fast core not yet taken; the boot core only if there is no other */
+ for (int j = (int)topology_info.num_cpus - 1; j >= 0; j--) {
+ if (topology_info.cpus[j].l2_access_penalty != 0 || taken[j]) {
+ continue;
+ }
+ if (&topology_info.cpus[j] == topology_info.boot_cpu) {
+ if (partner < 0) {
+ partner = j;
+ }
+ continue;
+ }
+ partner = j;
+ break;
+ }
+ if (partner >= 0) {
+ taken[partner] = true;
+ ml_slow_core_partner[i] = partner;
+ }
+ }
+}
+
+extern void loader_log_line(const char *line);
+
+/* At the registration of each processor: pairs it with its partner, if that one is registered too. */
+static void
+ml_pair_slow_processors(int cpu_id)
+{
+ for (int i = 0; i < (int)topology_info.num_cpus; i++) {
+ int partner = ml_slow_core_partner[i];
+ processor_t slow, fast;
+ char line[128];
+
+ if (partner < 0 || (i != cpu_id && partner != cpu_id)) {
+ continue;
+ }
+ slow = processor_array[i];
+ fast = processor_array[partner];
+ if (slow == PROCESSOR_NULL || fast == PROCESSOR_NULL || slow->processor_primary == fast) {
+ continue;
+ }
+ sched_slow_secondaries = small_cores_apart >= 2;
+ processor_set_primary(slow, fast);
+ snprintf(line, sizeof(line), "cpu %d (phys id 0x%x) is a slower core: the scheduler uses it after cpu %d (phys id 0x%x)\n",
+ i, topology_info.cpus[i].phys_id, partner, topology_info.cpus[partner].phys_id);
+ printf("%s", line);
+ loader_log_line(line); /* (the kernel's own log of a run keeps only its end) */
+ }
+}
+
+/* How long processor cpu_id has been idle since it started, in mach_absolute_time() units (hakios.cpufreq.c's cpuload). */
+bool ml_cpu_idle_time(int cpu_id, uint64_t *idle);
+
+bool
+ml_cpu_idle_time(int cpu_id, uint64_t *idle)
+{
+ struct recount_usage usage = { 0 };
+
+ if (cpu_id < 0 || cpu_id >= MAX_SCHED_CPUS || processor_array[cpu_id] == PROCESSOR_NULL) {
+ return false;
+ }
+ recount_processor_usage(&processor_array[cpu_id]->pr_recount, &usage, idle);
+ return true;
+}
+
void
ml_parse_cpu_topology(void)
{
@@ -945,6 +1044,7 @@ ml_parse_cpu_topology(void)
int64_t cluster_max_cpu_phys_id[MAX_CPU_CLUSTER_PHY_ID + 1];
const boolean_t cpus_boot_arg_present = PE_parse_boot_argn("cpus", &cpu_boot_arg, sizeof(cpu_boot_arg));
const boolean_t cpumask_boot_arg_present = PE_parse_boot_argn("cpumask", &cpumask_boot_arg, sizeof(cpumask_boot_arg));
+ PE_parse_boot_argn("small_cores", &small_cores_apart, sizeof(small_cores_apart));
// The cpus=N and cpumask=N boot args cannot be used simultaneously. Flag this
// so that we trigger a panic later in the boot process, once serial is enabled.
@@ -1005,6 +1105,20 @@ ml_parse_cpu_topology(void)
cpu->phys_id = (uint32_t)ml_readprop(child, "reg", ML_READPROP_MANDATORY);
cpu->l2_access_penalty = (uint32_t)ml_readprop(child, "l2-access-penalty", 0);
+ /*
+ * A core the boot loader calls small (on the phone the Cortex-A55s; the Cortex-A78 the
+ * kernel boots on is about 2.5 times as fast) is to be used only when the faster ones are all
+ * busy: what XNU does with a core that has an L2 access penalty (ml_pair_slow_processors()).
+ * The loader gives it a name of its own because a kernel without that function would make
+ * every such core the sibling of the boot processor, and stop at the assertion that a
+ * processor has only one. Boot-arg small_cores=0: all cores alike, as before; 1: paired as
+ * XNU pairs the two threads of a hyperthreaded core (a secondary kept idle while its primary
+ * runs a real-time thread, and so on); 2 (the default): as cores of their own (see
+ * sched_slow_secondaries in osfmk/kern/sched.h).
+ */
+ if (cpu->l2_access_penalty == 0 && ml_readprop(child, "small-core", 0) != 0 && small_cores_apart) {
+ cpu->l2_access_penalty = 1;
+ }
cpu->l2_cache_size = (uint32_t)ml_readprop(child, "l2-cache-size", 0);
cpu->l2_cache_id = (uint32_t)ml_readprop(child, "l2-cache-id", 0);
cpu->l3_cache_size = (uint32_t)ml_readprop(child, "l3-cache-size", 0);
@@ -1107,6 +1221,7 @@ ml_parse_cpu_topology(void)
cluster_offsets[0] = 0;
#endif
assert(topology_info.boot_cpu != NULL);
+ ml_choose_slow_core_partners();
ml_read_chip_revision(&topology_info.chip_revision);
/*
@@ -1427,17 +1542,15 @@ ml_processor_register(ml_processor_info_t *in_processor_info,
processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, this_cpu_datap);
if (!is_boot_cpu) {
processor_init(processor, this_cpu_datap->cpu_number, pset);
-
- if (this_cpu_datap->cpu_l2_access_penalty) {
- /*
- * Cores that have a non-zero L2 access penalty compared
- * to the boot processor should be de-prioritized by the
- * scheduler, so that threads use the cores with better L2
- * preferentially.
- */
- processor_set_primary(processor, master_processor);
- }
}
+ /*
+ * Cores that have a non-zero L2 access penalty compared
+ * to the boot processor should be de-prioritized by the
+ * scheduler, so that threads use the cores with better L2
+ * preferentially. (Each with a faster core of its own, not all with the boot
+ * processor: see ml_choose_slow_core_partners().)
+ */
+ ml_pair_slow_processors(this_cpu_datap->cpu_number);
*processor_out = processor;
*ipi_handler_out = cpu_signal_handler;
@@ -2237,8 +2350,16 @@ _enable_virtual_timer(void)
uint64_t cntvctl = CNTV_CTL_EL0_ENABLE; /* One wants to use 32 bits, but "mrs" prefers it this way */
__builtin_arm_wsr64("CNTV_CTL_EL0", cntvctl);
+#if defined(APPLE_ARM64_ARCH_FAMILY)
/* disable the physical timer as a precaution, as its registers reset to architecturally unknown values */
__builtin_arm_wsr64("CNTP_CTL_EL0", CNTP_CTL_EL0_IMASKED);
+#else
+ /*
+ * When running at EL1 under a third-party hypervisor (e.g. Qualcomm's),
+ * the EL1 physical timer registers are trapped and the access faults.
+ * The guest only owns the virtual timer.
+ */
+#endif
}
void
diff --git a/osfmk/arm64/start.s b/osfmk/arm64/start.s
index e5336e5a1cce..a54e35035cde 100644
--- a/osfmk/arm64/start.s
+++ b/osfmk/arm64/start.s
@@ -126,6 +126,27 @@ LEXT(LowResetVectorBase)
.globl EXT(reset_vector)
LEXT(reset_vector)
// Preserve x0 for start_first_cpu, if called
+#if defined(GENERIC_PSCI_SMP)
+ /*
+ * A core that the firmware has just started for us (PSCI CPU_ON, see
+ * psci_smp.c) has no vectors at all, and under a third-party hypervisor
+ * even the OS lock write can trap: vectors first. Not the boot
+ * processor's early ones, which hand the exception to the boot loader:
+ * that is not for this core to do.
+ */
+ msr DAIFSet, #(DAIFSC_ALL) // Disable all interrupts
+ adrp x0, EXT(LowSecondaryExceptionVectorBase)@page
+ add x0, x0, EXT(LowSecondaryExceptionVectorBase)@pageoff
+ msr VBAR_EL1, x0
+ isb
+ // The MMU and the caches are off, says PSCI; of the rest of SCTLR it
+ // says nothing. Only the bits that must be set, until common_start.
+ MOV64 x0, 0x30D00800
+ msr SCTLR_EL1, x0
+ isb
+ // Unlock the core for debugging
+ msr OSLAR_EL1, xzr
+#else /* defined(GENERIC_PSCI_SMP) */
// Unlock the core for debugging
msr OSLAR_EL1, xzr
msr DAIFSet, #(DAIFSC_ALL) // Disable all interrupts
@@ -136,6 +157,7 @@ LEXT(reset_vector)
add x0, x0, EXT(LowExceptionVectorBase)@pageoff
msr VBAR_EL1, x0
#endif
+#endif /* defined(GENERIC_PSCI_SMP) */
@@ -143,7 +165,9 @@ LEXT(reset_vector)
adrp x19, EXT(ResetHandlerData)@page // Get address of the reset handler data
add x19, x19, EXT(ResetHandlerData)@pageoff
mrs x15, MPIDR_EL1 // Load MPIDR to get CPU number
-#if HAS_CLUSTER
+#if HAS_CLUSTER || defined(GENERIC_PSCI_SMP)
+ // (generic boards: some number their cores in Affinity1 and leave Affinity0 at zero;
+ // a cpu data's phys id is both fields, as the device tree's "reg" has them)
and x0, x15, #0xFFFF // CPU number in Affinity0, cluster ID in Affinity1
#else
and x0, x15, #0xFF // CPU number is in MPIDR Affinity Level 0
@@ -233,40 +257,175 @@ LEXT(_start)
.global EXT(LowExceptionVectorBase)
LEXT(LowExceptionVectorBase)
/* EL1 SP 0 */
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
/* EL1 SP1 */
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
/* EL0 64 */
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
/* EL0 32 */
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
.align 7
- b .
+ b Lloader_early_exception
+
+/*
+ * Early-boot exceptions used to spin here. When the XNU loader has patched
+ * the address of its return stub into the slot below, report the exception
+ * to it instead (see osfmk/arm64/loader_return.c):
+ * x0/x1 = PA/length of the captured kernel log, x2 = 1 (early exception),
+ * x3 = ESR, x4 = ELR, x5 = FAR, x6 = LR, x7 = SP
+ * This vector is only installed while VBAR holds a physical address, so
+ * PC-relative addressing yields physical addresses, MMU on (V=P) or off.
+ */
+ .align 3
+Lloader_early_exception:
+ mov x6, lr
+ mov x7, sp
+ mrs x3, ESR_EL1
+ mrs x4, ELR_EL1
+ mrs x5, FAR_EL1
+ adrp x0, EXT(loader_log)@page
+ add x0, x0, EXT(loader_log)@pageoff
+ adrp x1, EXT(loader_log_len)@page
+ add x1, x1, EXT(loader_log_len)@pageoff
+ ldr w1, [x1]
+ // Write the log back to RAM: cached copies are lost once the MMU is off
+ mov x9, x0
+ add x10, x0, x1
+1:
+ dc cvac, x9
+ add x9, x9, #64
+ cmp x9, x10
+ b.lo 1b
+ dsb sy
+ mov x2, #1
+ mrs x9, SCTLR_EL1
+ bic x9, x9, #1
+ msr SCTLR_EL1, x9
+ isb
+ adr x9, Lloader_stub_slot
+ ldr x9, [x9, #8]
+ cbz x9, .
+ br x9
+ .align 3
+ .globl EXT(loader_stub_slot)
+LEXT(loader_stub_slot)
+Lloader_stub_slot:
+ .ascii "XNULDRST" // magic the loader searches for
+ .quad 0 // PA of the loader's return stub
+ .quad 0 // checkpoint number to stop at (0 = none)
+ .quad 0 // (physical - virtual) offset of the static kernel image
+
+#if defined(GENERIC_PSCI_SMP)
+/*
+ * The same for a core other than the boot processor, from its first
+ * instruction (reset_vector) until common_start gives it the kernel's
+ * vectors. The boot loader is not this core's to return to; it leaves a note
+ * of what happened where the processor that is waiting for it looks
+ * (psci_smp_start_cpu()) and switches itself off again.
+ *
+ * VBAR holds a physical address here too, so the note is addressed
+ * physically; the store goes straight to memory when the MMU is still off,
+ * and is sent there when it is not.
+ */
+ .align 11
+ .globl EXT(LowSecondaryExceptionVectorBase)
+LEXT(LowSecondaryExceptionVectorBase)
+ /* EL1 SP 0 */
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ /* EL1 SP1 */
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ /* EL0 64 */
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ /* EL0 32 */
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+ .align 7
+ b Lsecondary_early_exception
+
+ .align 3
+Lsecondary_early_exception:
+ // struct psci_smp_start_fault: { valid, MPIDR, ESR, ELR, FAR, LR, SP }
+ adrp x9, EXT(psci_smp_start_fault)@page
+ add x9, x9, EXT(psci_smp_start_fault)@pageoff
+ mrs x10, MPIDR_EL1
+ str x10, [x9, #8]
+ mrs x10, ESR_EL1
+ str x10, [x9, #16]
+ mrs x10, ELR_EL1
+ str x10, [x9, #24]
+ mrs x10, FAR_EL1
+ str x10, [x9, #32]
+ str lr, [x9, #40]
+ mov x10, sp
+ str x10, [x9, #48]
+ mov x10, #1
+ str x10, [x9]
+ dc cvac, x9
+ dsb sy
+ // PSCI CPU_OFF, by the firmware's calling convention (psci_smp_hvc)
+ adrp x9, EXT(psci_smp_hvc)@page
+ add x9, x9, EXT(psci_smp_hvc)@pageoff
+ ldr w9, [x9]
+ mov x0, #0x0002
+ movk x0, #0x8400, lsl #16
+ cbnz w9, 1f
+ smc #0
+ b 2f
+1:
+ hvc #0
+2:
+ wfi // it does not return; if it does, stay here
+ b 2b
+#endif /* defined(GENERIC_PSCI_SMP) */
.align 12, 0
#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR)
@@ -452,22 +611,51 @@ start_cpu:
* Cold boot init routine. Called from __start
* x0 - Boot args
*/
+/*
+ * LOADER_CHECKPOINT n
+ * Bring-up aid: if the loader asked to stop at checkpoint n, return to it
+ * right here (reason 2, code n). Usable while executing at physical
+ * addresses, MMU off or on (V=P). Clobbers x0-x3, x9-x11.
+ */
+.macro LOADER_CHECKPOINT
+ adrp x9, EXT(loader_stub_slot)@page
+ add x9, x9, EXT(loader_stub_slot)@pageoff
+ ldr x10, [x9, #16]
+ cmp x10, #$0
+ b.ne 1f
+ mov x0, #0
+ mov x1, #0
+ mov x2, #2
+ mov x3, #$0
+ mrs x11, SCTLR_EL1
+ bic x11, x11, #1
+ msr SCTLR_EL1, x11
+ isb
+ ldr x9, [x9, #8]
+ br x9
+1:
+.endmacro
+
.align 2
.globl EXT(start_first_cpu)
LEXT(start_first_cpu)
- // Unlock the core for debugging
- msr OSLAR_EL1, xzr
msr DAIFSet, #(DAIFSC_ALL) // Disable all interrupts
mov x20, x0
mov x21, #0
+ LOADER_CHECKPOINT 1
- // Set low reset vector before attempting any loads
+ // Set low reset vector before attempting anything that might trap
+ // (under a third-party hypervisor even the OS lock write below can).
adrp x0, EXT(LowExceptionVectorBase)@page
add x0, x0, EXT(LowExceptionVectorBase)@pageoff
MSR_VBAR_EL1_X0
+ // Unlock the core for debugging
+ msr OSLAR_EL1, xzr
+ LOADER_CHECKPOINT 2
+
// Get the kernel memory parameters from the boot args
ldr x22, [x20, BA_VIRT_BASE] // Get the kernel virt base
@@ -612,6 +800,7 @@ Lkernelcache_base_found:
/* Ensure TTEs are visible */
dsb ish
+ LOADER_CHECKPOINT 3
b common_start
@@ -686,7 +875,10 @@ common_start:
dsb ish
-#if defined(BCM2837)
+#if defined(BCM2837) && defined(BCM2837_CORE_TIMER_ROUTING)
+ // Raspberry Pi 3 only: pokes the SoC's local-interrupt block at a fixed
+ // physical address (0x4000004X). Fatal on any other board, so it is
+ // disabled unless explicitly requested.
// Setup timer interrupt routing; must be done before MMU is enabled
mrs x15, MPIDR_EL1 // Load MPIDR to get CPU number
and x15, x15, #0xFF // CPU number is in MPIDR Affinity Level 0
@@ -740,10 +932,17 @@ common_start:
PAC_INIT_KEY_STATE tmp=x0, tmp2=x1
#endif /* HAS_APPLE_PAC */
+ cbnz x21, 1f // boot CPU cold start only
+ LOADER_CHECKPOINT 4
+1:
// Enable caches, MMU, ROP and JOP
MOV64 x0, SCTLR_EL1_DEFAULT
MSR_SCTLR_EL1_X0
isb sy
+ cbnz x21, 1f
+ LOADER_CHECKPOINT 5
+ MOV64 x0, SCTLR_EL1_DEFAULT // restore what the checkpoint clobbered
+1:
#if !VMAPPLE
MOV64 x1, SCTLR_EL1_DEFAULT
diff --git a/osfmk/conf/Makefile b/osfmk/conf/Makefile
index 050f34a53c49..c58ff62cd419 100644
--- a/osfmk/conf/Makefile
+++ b/osfmk/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/osfmk/conf/files b/osfmk/conf/files
index 9d98c50094a0..9e3476124e54 100644
--- a/osfmk/conf/files
+++ b/osfmk/conf/files
@@ -361,3 +361,6 @@ osfmk/corecrypto/ccsha256_ltc_di.c standard
osfmk/prng/prng_random.c standard
osfmk/prng/entropy.c standard
+osfmk/minicrypto/mc_digests.c standard
+osfmk/minicrypto/mc_provider.c standard
+osfmk/minicrypto/mc_aes.c standard
diff --git a/osfmk/conf/files.arm64 b/osfmk/conf/files.arm64
index c40d57bcc677..1b7a1fcdfcdf 100644
--- a/osfmk/conf/files.arm64
+++ b/osfmk/conf/files.arm64
@@ -25,8 +25,8 @@ osfmk/arm64/WKdmCompress_4k.s standard
osfmk/arm64/WKdmData.s standard
osfmk/arm64/lz4_decode_arm64.s standard
osfmk/arm64/lz4_encode_arm64.s standard
-osfmk/arm64/amcc_rorgn_ppl.c optional nos_arm_asm config_pmap_ppl
-osfmk/arm64/amcc_rorgn_common.c optional nos_arm_asm
+#osfmk/arm64/amcc_rorgn_ppl.c optional nos_arm_asm config_pmap_ppl
+#osfmk/arm64/amcc_rorgn_common.c optional nos_arm_asm
osfmk/arm64/bcopy.s standard
osfmk/arm64/bzero.s standard
osfmk/arm/caches.c standard
@@ -104,3 +104,8 @@ osfmk/arm64/machine_remote_time.c optional config_mach_bridge_recv_time
osfmk/arm64/corecrypto/sha256_compress_arm64.s standard
osfmk/arm/counter.c standard
+osfmk/arm64/generic_arm64_support.c optional nos_arm_asm
+osfmk/arm64/loader_return.c optional nos_arm_asm
+osfmk/arm64/user_exception_trace.c standard
+osfmk/arm64/gic_v3_irq.c optional nos_arm_asm
+osfmk/arm64/psci_smp.c optional nos_arm_asm
diff --git a/osfmk/console/video_console.c b/osfmk/console/video_console.c
index 384118e8d7a9..2573d2ebd4e6 100644
--- a/osfmk/console/video_console.c
+++ b/osfmk/console/video_console.c
@@ -2855,8 +2855,12 @@ initialize_screen(PE_Video * boot_vinfo, unsigned int op)
gc_acquired = TRUE;
} else {
if (makeMapping) {
-#if HAS_UCNORMAL_MEM
+#if HAS_UCNORMAL_MEM || defined(BCM2837)
/*
+ * (Generic ARM boards too: scrolling reads the framebuffer
+ * back, and as device memory that is one bus access per
+ * word - seconds per line on a phone-sized screen.)
+ *
* Framebuffers would normally use VM_WIMG_RT, which
* io_map doesn't support. However this buffer is set up
* by the bootloader and doesn't require D$ cleaning, so
diff --git a/osfmk/kern/debug.c b/osfmk/kern/debug.c
index 0a00656a0fdd..bfad3cbca5bc 100644
--- a/osfmk/kern/debug.c
+++ b/osfmk/kern/debug.c
@@ -1147,9 +1147,12 @@ panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsign
__builtin_unreachable();
}
+extern void loader_return(void);
+
void
panic_spin_forever(void)
{
+ loader_return();
for (;;) {
#if defined(__arm__) || defined(__arm64__)
/* On arm32, which doesn't have a WFE timeout, this may not return. But that should be OK on this path. */
@@ -1163,6 +1166,7 @@ panic_spin_forever(void)
static void
kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags)
{
+ loader_return();
if ((type == kPEPanicRestartCPU) && (debugger_flags & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS)) {
PEHaltRestart(kPEPanicRestartCPUNoCallouts);
} else {
diff --git a/osfmk/kern/exception.c b/osfmk/kern/exception.c
index 809dafaaa2a7..71c6580f44e2 100644
--- a/osfmk/kern/exception.c
+++ b/osfmk/kern/exception.c
@@ -636,6 +636,11 @@ check_exc_receiver_dependency(
* Returns:
* KERN_SUCCESS if exception is handled by any of the handlers.
*/
+#if defined(__arm64__)
+extern void user_exception_trace(exception_type_t exception, mach_exception_data_t code,
+ mach_msg_type_number_t count, thread_t thread);
+#endif
+
kern_return_t
exception_triage_thread(
exception_type_t exception,
@@ -650,6 +655,11 @@ exception_triage_thread(
struct exception_action *actions;
kern_return_t kr = KERN_FAILURE;
+#if defined(__arm64__)
+ /* bring-up aid, see osfmk/arm64/user_exception_trace.c */
+ user_exception_trace(exception, code, codeCnt, thread);
+#endif
+
assert(exception != EXC_RPC_ALERT);
/*
diff --git a/osfmk/kern/kalloc.h b/osfmk/kern/kalloc.h
index dfa89c7b0d29..07e0ba5bf44c 100644
--- a/osfmk/kern/kalloc.h
+++ b/osfmk/kern/kalloc.h
@@ -821,12 +821,16 @@ __options_decl(kt_granule_t, uint32_t, {
* @param ptr the pointer whose type needs to be checked.
* @param type the type which the pointer will be checked against.
*/
+#if XNU_NO_TYPE_BUILTINS
+#define KALLOC_TYPE_IS_COMPATIBLE_PTR(ptr, type) (1)
+#else
#define KALLOC_TYPE_IS_COMPATIBLE_PTR(ptr, type) \
_Pragma("clang diagnostic push") \
_Pragma("clang diagnostic ignored \"-Wvoid-ptr-dereference\"") \
(__builtin_xnu_types_compatible(__typeof__(*(ptr)), type) || \
__builtin_xnu_types_compatible(__typeof__(*(ptr)), void)) \
_Pragma("clang diagnostic pop")
+#endif /* XNU_NO_TYPE_BUILTINS */
#define KALLOC_TYPE_ASSERT_COMPATIBLE_POINTER(ptr, type) \
_Static_assert(KALLOC_TYPE_IS_COMPATIBLE_PTR(ptr, type), \
diff --git a/osfmk/kern/percpu.h b/osfmk/kern/percpu.h
index 4fb223ee9788..df563bc04910 100644
--- a/osfmk/kern/percpu.h
+++ b/osfmk/kern/percpu.h
@@ -236,15 +236,22 @@ extern vm_offset_t other_percpu_base(int cpu_number);
* @param it the name of the iterator
* @param name the per-CPU variable name
*/
+/*
+ * (redwood) All of the arithmetic is on integers here. As Apple wrote it, the end was a pointer
+ * computed from the start, and a compiler may then take "start <= start + n" for "n >= 0":
+ * with one processor start is VM_MAX_KERNEL_ADDRESS and end is 0, n looks positive, and the
+ * loop that must not run at all runs once on memory that does not exist (clang 18+ does this;
+ * the panic was in mbuf_mtypes_sync(), the first time a daemon asked for the mbuf statistics).
+ */
#define percpu_foreach_secondary(it, name) \
for (__PERCPU_TYPE(name) it, \
- __base_ ## it = (typeof(it))__unsafe_forge_bidi_indexable(void *, percpu_base.start, percpu_base.end - percpu_base.start), \
- __end_ ## it = (typeof(it))((caddr_t)__base_ ## it + percpu_base.end - percpu_base.start); \
+ __base_ ## it = (typeof(it))percpu_base.start, \
+ __end_ ## it = (typeof(it))percpu_base.end; \
\
- (it = (typeof(it))((caddr_t)__base_ ## it + __PERCPU_ADDR(name)), \
- __base_ ## it <= __end_ ## it); \
+ (it = (typeof(it))((vm_address_t)__base_ ## it + __PERCPU_ADDR(name)), \
+ (vm_address_t)__base_ ## it <= (vm_address_t)__end_ ## it); \
\
- __base_ ## it = (typeof(it))((caddr_t)__base_ ## it + percpu_section_size()))
+ __base_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
#pragma mark - implementation details
diff --git a/osfmk/kern/sched.h b/osfmk/kern/sched.h
index a5f68ade084b..9741eb1b3ecd 100644
--- a/osfmk/kern/sched.h
+++ b/osfmk/kern/sched.h
@@ -464,4 +464,13 @@ extern int sched_rt_n_backup_processors;
extern bool system_is_SMT;
+/*
+ * The "secondaries" are not the second threads of hyperthreaded cores but slower cores of their
+ * own, each paired with a faster one (ml_pair_slow_processors() in osfmk/arm64/machine_routines.c):
+ * the scheduler still prefers the primaries, but what runs on one core does not slow the other,
+ * so a secondary is not kept idle for its primary's sake, and when a primary goes idle a thread
+ * running on a secondary is moved to it.
+ */
+extern bool sched_slow_secondaries;
+
#endif /* _KERN_SCHED_H_ */
diff --git a/osfmk/kern/sched_clutch.c b/osfmk/kern/sched_clutch.c
index 3657d83c5b6f..ded2ff427d68 100644
--- a/osfmk/kern/sched_clutch.c
+++ b/osfmk/kern/sched_clutch.c
@@ -3204,7 +3204,7 @@ extern int sched_allow_rt_smt;
static bool
sched_clutch_thread_avoid_processor(processor_t processor, thread_t thread)
{
- if (processor->processor_primary != processor) {
+ if (processor->processor_primary != processor && !sched_slow_secondaries) {
/*
* This is a secondary SMT processor. If the primary is running
* a realtime thread, only allow realtime threads on the secondary.
diff --git a/osfmk/kern/sched_prim.c b/osfmk/kern/sched_prim.c
index 8e29586464de..0a7de6fc4be7 100644
--- a/osfmk/kern/sched_prim.c
+++ b/osfmk/kern/sched_prim.c
@@ -2042,6 +2042,7 @@ sched_vm_group_maintenance(void)
#endif
int sched_allow_rt_smt = 1;
+bool sched_slow_secondaries = false;
int sched_avoid_cpu0 = SCHED_AVOID_CPU0;
int sched_allow_rt_steal = 1;
int sched_backup_cpu_timeout_count = 5; /* The maximum number of 10us delays to wait before using a backup cpu */
@@ -2097,6 +2098,7 @@ void
sched_SMT_balance(processor_t cprocessor, processor_set_t cpset)
{
processor_t ast_processor = NULL;
+ sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
#if (DEVELOPMENT || DEBUG)
if (__improbable(sched_smt_balance == 0)) {
@@ -2105,7 +2107,39 @@ sched_SMT_balance(processor_t cprocessor, processor_set_t cpset)
#endif
assert(cprocessor == current_processor());
- if (cprocessor->is_SMT == FALSE) {
+ if (cprocessor->is_SMT == FALSE && !sched_slow_secondaries) {
+ goto smt_balance_exit;
+ }
+
+ if (sched_slow_secondaries) {
+ /*
+ * A faster core is going idle: the thread of highest priority that runs on a slower one
+ * (not a real-time one: those stay where they are) is sent here. Its core is made to
+ * reschedule; a thread on a secondary goes back to the run queue then, and an idle
+ * primary is where it is put.
+ */
+ integer_t highest = -1;
+
+ if (cprocessor->processor_primary != cprocessor) {
+ goto smt_balance_exit;
+ }
+ uint64_t running_secondaries = (cpset->cpu_state_map[PROCESSOR_RUNNING] & ~cpset->primary_map &
+ ~cpset->pending_AST_URGENT_cpu_mask);
+ for (int cpuid = lsb_first(running_secondaries); cpuid >= 0; cpuid = lsb_next(running_secondaries, cpuid)) {
+ processor_t sprocessor = processor_array[cpuid];
+
+ if (sprocessor->current_pri < BASEPRI_RTQUEUES && sprocessor->current_pri > highest &&
+ sprocessor->current_pri > MINPRI) {
+ highest = sprocessor->current_pri;
+ ast_processor = sprocessor;
+ }
+ }
+ if (ast_processor != PROCESSOR_NULL) {
+ ipi_type = sched_ipi_action(ast_processor, NULL, SCHED_IPI_EVENT_SMT_REBAL);
+ if (ipi_type == SCHED_IPI_NONE) {
+ ast_processor = PROCESSOR_NULL;
+ }
+ }
goto smt_balance_exit;
}
@@ -2120,7 +2154,6 @@ sched_SMT_balance(processor_t cprocessor, processor_set_t cpset)
processor_t sprocessor;
- sched_ipi_type_t ipi_type = SCHED_IPI_NONE;
uint64_t running_secondary_map = (cpset->cpu_state_map[PROCESSOR_RUNNING] &
~cpset->primary_map);
for (int cpuid = lsb_first(running_secondary_map); cpuid >= 0; cpuid = lsb_next(running_secondary_map, cpuid)) {
@@ -2581,6 +2614,10 @@ restart:
bool is_yielding = (*reason & AST_YIELD) == AST_YIELD;
bool needs_smt_rebalance = !is_yielding && thread->sched_pri < BASEPRI_RTQUEUES && processor->processor_primary != processor;
+ if (needs_smt_rebalance && sched_slow_secondaries) {
+ /* (a slower core of its own: worth leaving only for a faster one that is idle) */
+ needs_smt_rebalance = (pset->recommended_bitmask & pset->primary_map & pset->cpu_state_map[PROCESSOR_IDLE]) != 0;
+ }
bool affinity_mismatch = thread->affinity_set != AFFINITY_SET_NULL && thread->affinity_set->aset_pset != pset;
@@ -2705,7 +2742,7 @@ pick_new_rt_thread:
clear_pending_AST_bits(pset, processor, 3);
- if (processor->processor_secondary != NULL) {
+ if (processor->processor_secondary != NULL && !sched_slow_secondaries) {
processor_t sprocessor = processor->processor_secondary;
if ((sprocessor->state == PROCESSOR_RUNNING) || (sprocessor->state == PROCESSOR_DISPATCHING)) {
ipi_type = sched_ipi_action(sprocessor, NULL, SCHED_IPI_EVENT_SMT_REBAL);
@@ -2774,7 +2811,8 @@ send_followup_ipi_before_idle:
goto restart;
}
- if ((processor->processor_primary != processor) && (processor->processor_primary->current_pri >= BASEPRI_RTQUEUES)) {
+ if ((processor->processor_primary != processor) && (processor->processor_primary->current_pri >= BASEPRI_RTQUEUES) &&
+ !sched_slow_secondaries) {
/* secondary can only run realtime thread */
if (idle_reason == 0) {
idle_reason = 4;
diff --git a/osfmk/kern/startup.c b/osfmk/kern/startup.c
index bec97fbf7243..203e1f48783f 100644
--- a/osfmk/kern/startup.c
+++ b/osfmk/kern/startup.c
@@ -156,6 +156,7 @@
#include <security/mac_mach_internal.h>
#if CONFIG_VNGUARD
extern void vnguard_policy_init(void);
+extern void minimac_policy_init(void);
#endif
#endif
@@ -721,6 +722,9 @@ kernel_bootstrap_thread(void)
kernel_bootstrap_log("vnguard_policy_init");
vnguard_policy_init();
#endif
+ /* stand-ins for the AMFI and Sandbox policies (bsd/kern/kern_minimac.c) */
+ kernel_bootstrap_log("minimac_policy_init");
+ minimac_policy_init();
#endif
#if CONFIG_DTRACE
diff --git a/osfmk/kern/thread.c b/osfmk/kern/thread.c
index 384ef23fe0e1..d5ab6db0edbc 100644
--- a/osfmk/kern/thread.c
+++ b/osfmk/kern/thread.c
@@ -3065,6 +3065,26 @@ thread_get_tag(thread_t th)
return thread_get_tag_internal(th);
}
+/*
+ * For apfs's lock statistics (bsd/miscfs/apfs/apfs_vnops.c): how long the current thread has
+ * been runnable (running included) and how long it has run, in mach_absolute_time() units.
+ * Taken before and after a wait for a lock, the two tell the wait apart: blocked, queued for
+ * a processor, or spinning on one.
+ */
+void thread_runnable_and_run_self(uint64_t *runnable, uint64_t *run);
+
+void
+thread_runnable_and_run_self(uint64_t *runnable, uint64_t *run)
+{
+ thread_t thread = current_thread();
+ boolean_t istate = ml_set_interrupts_enabled(FALSE);
+ uint64_t now = mach_absolute_time();
+
+ *runnable = thread->runnable_timer.all_bits + (now - thread->runnable_timer.tstamp);
+ *run = recount_current_thread_time_mach();
+ ml_set_interrupts_enabled(istate);
+}
+
uint64_t
thread_last_run_time(thread_t th)
{
diff --git a/osfmk/kern/work_interval.c b/osfmk/kern/work_interval.c
index f2b9ce24bd8d..3aed167fe32e 100644
--- a/osfmk/kern/work_interval.c
+++ b/osfmk/kern/work_interval.c
@@ -683,6 +683,18 @@ work_interval_get_priority(thread_t thread)
return priority;
}
+/*
+ * (redwood) What kind of work the thread's work interval is for: one of WORK_INTERVAL_TYPE_*,
+ * or WORK_INTERVAL_TYPE_LAST for a thread that has joined none.
+ */
+uint32_t
+kern_work_interval_type_of_thread(thread_t thread)
+{
+ const struct work_interval *work_interval = thread->th_work_interval;
+
+ return work_interval ? work_interval->wi_create_flags & WORK_INTERVAL_TYPE_MASK : WORK_INTERVAL_TYPE_LAST;
+}
+
/*
* Switch to a policy driven by the work interval (if applicable).
*/
diff --git a/osfmk/kern/work_interval.h b/osfmk/kern/work_interval.h
index 8b46efb05c86..e161ca426b1a 100644
--- a/osfmk/kern/work_interval.h
+++ b/osfmk/kern/work_interval.h
@@ -83,6 +83,8 @@ extern kern_return_t
kern_work_interval_notify(thread_t thread, struct kern_work_interval_args* kwi_args);
extern kern_return_t
kern_work_interval_set_name(mach_port_name_t port_name, char *name, size_t len);
+extern uint32_t
+kern_work_interval_type_of_thread(thread_t thread);
extern kern_return_t
kern_work_interval_set_workload_id(mach_port_name_t port_name,
struct kern_work_interval_workload_id_args *workload_id_args);
diff --git a/pexpert/arm/pe_fiq.c b/pexpert/arm/pe_fiq.c
index 3ad743fc3efb..ce036c5a2c26 100644
--- a/pexpert/arm/pe_fiq.c
+++ b/pexpert/arm/pe_fiq.c
@@ -169,6 +169,15 @@ pe_init_fiq()
__builtin_arm_wsr64("ICC_IGRPEN0_EL1", 1);
__builtin_arm_isb(ISB_SY);
}
+#elif defined(GENERIC_GIC_V3_IRQ)
+/* Generic boards: Group 1 IRQs through a GICv3, see osfmk/arm64/gic_v3_irq.c */
+extern void gic_v3_irq_cpu_init(void);
+
+void
+pe_init_fiq()
+{
+ gic_v3_irq_cpu_init();
+}
#else
void
pe_init_fiq()
diff --git a/pexpert/arm/pe_kprintf.c b/pexpert/arm/pe_kprintf.c
index 65144c74239c..ab6260d52d84 100644
--- a/pexpert/arm/pe_kprintf.c
+++ b/pexpert/arm/pe_kprintf.c
@@ -29,6 +29,7 @@ SECURITY_READ_ONLY_LATE(bool) enable_dklog_serial_output = false;
static SIMPLE_LOCK_DECLARE(kprintf_lock, 0);
static void serial_putc_crlf(char c);
+extern void loader_log_putc(char c);
__startup_func
static void
@@ -159,6 +160,7 @@ serial_putc_crlf(char c)
if (c == '\n') {
uart_putc('\r');
}
+ loader_log_putc(c);
uart_putc(c);
}
diff --git a/pexpert/conf/Makefile b/pexpert/conf/Makefile
index 050f34a53c49..c58ff62cd419 100644
--- a/pexpert/conf/Makefile
+++ b/pexpert/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/pexpert/pexpert/arm64/BCM2837.h b/pexpert/pexpert/arm64/BCM2837.h
index d9f0723244e5..53fe4f371d7f 100644
--- a/pexpert/pexpert/arm64/BCM2837.h
+++ b/pexpert/pexpert/arm64/BCM2837.h
@@ -11,6 +11,15 @@
#define BCM2837
#define BCM2837_BRINGUP
#define ARM_ARCH_TIMER
+#define GENERIC_GIC_V3_IRQ 1 /* interrupts via GICv3 Group 1, see osfmk/arm64/gic_v3_irq.c */
+#define GENERIC_PSCI_SMP 1 /* more cores are started through PSCI, see osfmk/arm64/psci_smp.c */
+
+/*
+ * The cores this kernel really runs on (Cortex-A55/A78, QEMU cortex-a76) are
+ * ARMv8.2: they set PSTATE.PAN on every exception entry, so the kernel must
+ * know to lift it around user memory accesses.
+ */
+#define __ARM_PAN_AVAILABLE__ 1
#define __ARM_ARCH__ 8
#define __ARM_VMSA__ 8
@@ -22,6 +31,7 @@
#ifndef ASSEMBLER
#define PI3_UART
+#define VMAPPLE_UART /* PL011, as found on QEMU virt */
#define PI3_BREAK asm volatile("brk #0");
@@ -60,8 +70,11 @@
#define BCM2837_PUT32(addr, value) do { *((volatile uint32_t *) addr) = value; } while(0)
#define BCM2837_GET32(addr) *((volatile uint32_t *) addr)
-#define PLATFORM_PANIC_LOG_PADDR 0x3c0fc000
-#define PLATFORM_PANIC_LOG_SIZE 16384 // 16kb
+/*
+ * No fixed PLATFORM_PANIC_LOG_PADDR: the Raspberry Pi address (0x3c0fc000) is
+ * not RAM on other boards. The panic log region comes from the device tree
+ * instead (/pram and /chosen/embedded-panic-log-size), like on Apple devices.
+ */
#endif /* ! ASSEMBLER */
#endif /* ! _PEXPERT_ARM_BCM2837_H */
diff --git a/pexpert/pexpert/arm64/board_config.h b/pexpert/pexpert/arm64/board_config.h
index c84b2100bc19..aaad5cd2e8c6 100644
--- a/pexpert/pexpert/arm64/board_config.h
+++ b/pexpert/pexpert/arm64/board_config.h
@@ -91,7 +91,7 @@
#include <pexpert/arm64/BCM2837.h>
#define MAX_L2_CLINE 6
-#define MAX_CPUS 4
+#define MAX_CPUS 8 /* the Raspberry Pi has four; the boards this runs on now have up to eight */
#define MAX_CPU_CLUSTERS 1
#define CORE_NCTRS 8 /* Placeholder; KPC is not enabled for this target */
diff --git a/san/conf/Makefile b/san/conf/Makefile
index 050f34a53c49..c58ff62cd419 100644
--- a/san/conf/Makefile
+++ b/san/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/security/conf/Makefile b/security/conf/Makefile
index 2efc18a7a9d6..04fc48e02ccb 100644
--- a/security/conf/Makefile
+++ b/security/conf/Makefile
@@ -17,6 +17,9 @@ MASTERCONFDIR = $(SRCROOT)/config
DOCONFDEPS = $(addprefix $(MASTERCONFDIR)/, MASTER MASTER.$(DOCONF_ARCH_CONFIG_LC)) \
$(addprefix $(SOURCE)/, Makefile.template Makefile.$(DOCONF_ARCH_CONFIG_LC) files files.$(DOCONF_ARCH_CONFIG_LC))
+# HakiOS: the drivers' choice (tools/hakios-config.py kernel); a new choice makes the configuration again
+DOCONFDEPS += $(wildcard $(MASTERCONFDIR)/MASTER.hakios)
+
ifneq (,$(wildcard $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)))
DOCONFDEPS += $(MASTERCONFDIR)/MASTER.$(DOCONF_ARCH_CONFIG_LC).$(CURRENT_MACHINE_CONFIG_LC)
endif
diff --git a/security/mac_vfs_subr.c b/security/mac_vfs_subr.c
index 97028c06bb92..0daad830d0c0 100644
--- a/security/mac_vfs_subr.c
+++ b/security/mac_vfs_subr.c
@@ -167,7 +167,7 @@ mac_vnop_setxattr(struct vnode *vp, const char *name, char *buf, size_t len)
{
vfs_context_t ctx;
int options = XATTR_NOSECURITY;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
uio_t auio;
int error;
@@ -198,7 +198,7 @@ mac_vnop_getxattr(struct vnode *vp, const char *name, char *buf, size_t len,
{
vfs_context_t ctx = vfs_context_current();
int options = XATTR_NOSECURITY;
- uio_stackbuf_t uio_buf[UIO_SIZEOF(1)];
+ uio_stackbuf_t uio_buf[(UIO_SIZEOF(1) + sizeof(uio_stackbuf_t) - 1) / sizeof(uio_stackbuf_t)];
uio_t auio;
int error;
diff --git a/tools/lldbmacros/core/syntax_checker.py b/tools/lldbmacros/core/syntax_checker.py
index 3ebc295117a3..de101eafc83b 100755
--- a/tools/lldbmacros/core/syntax_checker.py
+++ b/tools/lldbmacros/core/syntax_checker.py
@@ -12,10 +12,13 @@ Usage:
"""
import sys
import os
+# When this file is executed directly, its directory is first on sys.path.
+# That directory contains collections.py, which shadows Python's standard
+# library collections module while importing re on Python 3.14 and newer.
+_script_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path = [p for p in sys.path if os.path.abspath(p or os.curdir) != _script_dir]
import re
-
tabs_search_rex = re.compile("^\s*\t+",re.MULTILINE|re.DOTALL)
-
def find_non_ascii(s):
for c in s:
if ord(c) >= 0x80: return True