Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

HakiosGPU: the client's own pages as a buffer (userptr); Turnip imports any host pointer

yehorkaa8e3346721a6d
diff --git a/drivers/gpu/hakios.gpu.cpp b/drivers/gpu/hakios.gpu.cpp
index 8ed7772..0751dc7 100644
--- a/drivers/gpu/hakios.gpu.cpp
+++ b/drivers/gpu/hakios.gpu.cpp
@@ -82,6 +82,7 @@ enum {
kMsmSubmitQueueNew = 0x4a,
kMsmSubmitQueueClose = 0x4b,
kMsmSubmitQueueQuery = 0x4c,
+ kHakiosUserMemory = 0x7d, /* ours: pages of the caller's own as a buffer (Linux's userptr) */
kHakiosSurface = 0x7e, /* ours: an IOSurface's memory as a buffer */
kHakiosShow = 0x7f, /* ours: pixels of the caller's onto the panel, to see a test's result */
};
@@ -141,6 +142,11 @@ struct msm_submitqueue_query {
struct drm_gem_close {
uint32_t handle, pad;
};
+struct hakios_user_memory {
+ uint64_t address, size; /* in the caller, in whole pages */
+ uint32_t handle; /* back: the buffer */
+ uint32_t flags;
+};
struct hakios_surface {
uint32_t surface; /* the IOSurface's ID */
uint32_t handle; /* kHakiosDescribe: no buffer is wanted; else, back: the buffer, */
@@ -352,6 +358,7 @@ struct gpu_buffer {
IOMemoryDescriptor * memory; /* NULL: this handle is free */
uint8_t * kernel; /* where it is in the kernel */
MiniIOSurface * surface; /* whose memory it is, when not its own */
+ IOMemoryMap * user; /* the kernel's view of it, when it is the client's own pages */
uint64_t address; /* the GPU's for it */
uint64_t size;
uint32_t flags;
@@ -371,6 +378,7 @@ struct gpu_buffer {
struct gpu_left {
IOMemoryDescriptor * memory;
MiniIOSurface * surface;
+ IOMemoryMap * user;
uint64_t address, size;
uint32_t fence, flags;
};
@@ -425,6 +433,7 @@ class HakiosGPUUserClient : public IOUserClient
void reap(void);
IOReturn gemNew(struct msm_gem_new * request);
IOReturn surfaceBuffer(struct hakios_surface * request);
+ IOReturn userBuffer(struct hakios_user_memory * request);
IOReturn gemInfo(struct msm_gem_info * request);
IOReturn submit(struct msm_gem_submit * request);
IOReturn show(const struct hakios_show * request);
@@ -478,6 +487,7 @@ HakiosGPUUserClient::freeBuffer(struct gpu_buffer * slot)
slot->memory = NULL;
slot->surface = NULL;
+ slot->user = NULL;
fBufferCount--;
fBufferBytes -= b->size;
@@ -491,6 +501,7 @@ HakiosGPUUserClient::freeBuffer(struct gpu_buffer * slot)
left->memory = b->memory;
left->surface = b->surface;
+ left->user = b->user;
left->address = b->address;
left->size = b->size;
left->fence = b->fence;
@@ -507,12 +518,16 @@ HakiosGPUUserClient::freeBuffer(struct gpu_buffer * slot)
if (said++ < 8) {
IOLog("HakiosGPU: the GPU is not done with what it was given: a buffer of %llu bytes stays its own\n", b->size);
}
- } else if (b->surface == NULL && hakios_gpu_ready() && keep(b->memory, b->kernel, b->address, b->size, b->flags)) {
+ } else if (b->surface == NULL && b->user == NULL && hakios_gpu_ready() &&
+ keep(b->memory, b->kernel, b->address, b->size, b->flags)) {
b->memory = NULL; /* (now the kept one's) */
} else if (hakios_gpu_ready()) {
hakios_gpu_unmap(b->address, b->size);
hakios_gpu_address_free(b->address, b->size);
}
+ if (b->user != NULL && !busy) {
+ b->user->release(); /* (the pages are unwired as the descriptor goes) */
+ }
if (b->memory != NULL) {
b->memory->release();
}
@@ -533,13 +548,16 @@ HakiosGPUUserClient::reap(void)
i++;
continue;
}
- if (left->surface == NULL && hakios_gpu_ready() &&
+ if (left->surface == NULL && left->user == NULL && hakios_gpu_ready() &&
keep(left->memory, NULL, left->address, left->size, left->flags)) {
left->memory = NULL;
} else if (hakios_gpu_ready()) {
hakios_gpu_unmap(left->address, left->size);
hakios_gpu_address_free(left->address, left->size);
}
+ if (left->user != NULL) {
+ left->user->release();
+ }
if (left->memory != NULL) {
left->memory->release();
}
@@ -646,6 +664,7 @@ HakiosGPUUserClient::newBuffer(IOMemoryDescriptor * memory, uint64_t from, uint6
b->seen = false;
b->fence = 0;
b->surface = NULL;
+ b->user = NULL;
fBufferCount++;
fBufferBytes += size;
return b;
@@ -855,6 +874,59 @@ HakiosGPUUserClient::surfaceBuffer(struct hakios_surface * request)
return kIOReturnSuccess;
}
+/*
+ * Pages of the client's own as a buffer, as Linux's userptr: wired for as long as the buffer is there,
+ * entered into the GPU's address space, and seen by the kernel through a mapping of its own (for the
+ * cache: they are ordinary memory, pushed out of this processor's cache at every submission that
+ * names them). Vulkan's VK_EXT_external_memory_host over it (Turnip, tools/mesa/hakios-drm): Metal's
+ * no-copy buffers, CoreAnimation's backing stores.
+ */
+IOReturn
+HakiosGPUUserClient::userBuffer(struct hakios_user_memory * request)
+{
+ IOMemoryDescriptor * memory;
+ IOMemoryMap * map;
+ struct gpu_buffer * b;
+ uint64_t began = mach_absolute_time();
+
+ if ((request->address & PAGE_MASK) || (request->size & PAGE_MASK) || request->size == 0 ||
+ request->size > GPU_BUFFER_MAX) {
+ return kIOReturnBadArgument;
+ }
+ memory = IOMemoryDescriptor::withAddressRange(request->address, request->size, kIODirectionInOut, fTask);
+ if (memory == NULL) {
+ return kIOReturnNoMemory;
+ }
+ if (memory->prepare() != kIOReturnSuccess) {
+ memory->release();
+ return kIOReturnVMError;
+ }
+ map = memory->createMappingInTask(kernel_task, 0, kIOMapAnywhere);
+ if (map == NULL) {
+ memory->complete();
+ memory->release();
+ return kIOReturnNoMemory;
+ }
+ IOLockLock(gGPULock);
+ b = newBuffer(memory, 0, request->size, request->flags, &request->handle);
+ if (b != NULL) {
+ b->memory = memory;
+ b->kernel = (uint8_t *) map->getVirtualAddress();
+ b->user = map;
+ b->seen = true;
+ gGPUStat.surfaces++;
+ }
+ gGPUStat.taking += mach_absolute_time() - began;
+ IOLockUnlock(gGPULock);
+ if (b == NULL) {
+ map->release();
+ memory->complete();
+ memory->release();
+ return kIOReturnNoMemory;
+ }
+ return kIOReturnSuccess;
+}
+
IOReturn
HakiosGPUUserClient::gemInfo(struct msm_gem_info * request)
{
@@ -931,8 +1003,8 @@ HakiosGPUUserClient::submit(struct msm_gem_submit * request)
}
b->seed = seed;
b->pushed = began;
- } else if (!b->seen || !gGPUCached) {
- continue;
+ } else if (b->user == NULL && (!b->seen || !gGPUCached)) {
+ continue; /* (the client's own pages are ordinary memory: every time) */
}
hakios_gpu_to_memory(b->kernel, b->size);
gGPUStat.flushed += b->size;
@@ -1110,6 +1182,7 @@ HakiosGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArguments
struct msm_submitqueue_query query;
struct drm_gem_close close;
struct hakios_surface surface;
+ struct hakios_user_memory user;
struct hakios_show show;
} r;
size_t size = args->structureInputSize;
@@ -1200,6 +1273,9 @@ HakiosGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArguments
case kHakiosSurface:
ret = surfaceBuffer(&r.surface);
break;
+ case kHakiosUserMemory:
+ ret = userBuffer(&r.user);
+ break;
case kHakiosShow:
ret = show(&r.show);
break;
diff --git a/tools/mesa/hakios-drm/hakios.drm.c b/tools/mesa/hakios-drm/hakios.drm.c
index 9bfb780..b729fba 100644
--- a/tools/mesa/hakios-drm/hakios.drm.c
+++ b/tools/mesa/hakios-drm/hakios.drm.c
@@ -22,6 +22,7 @@
#include "drm-uapi/msm_drm.h"
#define HAKIOS_GPU_SERVICE "HakiosGPU"
+#define HAKIOS_GPU_USER_MEMORY 0x7d /* ours too: pages of this process as a buffer */
#define HAKIOS_GPU_SURFACE 0x7e /* the request that is not Linux's */
#define HAKIOS_GPU_DESCRIBE 0xffffffffu /* ... asked only what the buffer would be like */
@@ -627,3 +628,18 @@ hakios_drm_host_surface_at(const void *base, uint64_t size)
pthread_mutex_unlock(&host_surfaces_lock);
return surface;
}
+
+int
+hakios_drm_user_memory(int fd, const void *address, uint64_t size, uint32_t *handle)
+{
+ struct { uint64_t address, size; uint32_t handle, flags; } request = { (uintptr_t)address, size, 0, 0 };
+ int error = call(HAKIOS_GPU_USER_MEMORY, &request, sizeof(request));
+
+ (void)fd;
+ if (error != 0) {
+ errno = error;
+ return -1;
+ }
+ *handle = request.handle;
+ return 0;
+}
diff --git a/tools/mesa/hakios-drm/xf86drm.h b/tools/mesa/hakios-drm/xf86drm.h
index 09b3735..5449a1c 100644
--- a/tools/mesa/hakios-drm/xf86drm.h
+++ b/tools/mesa/hakios-drm/xf86drm.h
@@ -108,6 +108,8 @@ int hakios_drm_surface(int fd, struct hakios_drm_surface *surface);
* tools/m2v), for Turnip's VK_EXT_external_memory_host, which is given only the address: its ID, 0 if nobody said. */
void hakios_drm_host_surface(const void *base, uint64_t size, uint32_t surface);
uint32_t hakios_drm_host_surface_at(const void *base, uint64_t size);
+/* Not libdrm's: pages of this process (whole ones) as a buffer, Linux's userptr (the kernel's request 0x7d) */
+int hakios_drm_user_memory(int fd, const void *address, uint64_t size, uint32_t *handle);
#ifdef __cplusplus
}
diff --git a/tools/mesa/turnip-darwin.patch b/tools/mesa/turnip-darwin.patch
index e2a59e7..66f8b41 100644
--- a/tools/mesa/turnip-darwin.patch
+++ b/tools/mesa/turnip-darwin.patch
@@ -39,20 +39,21 @@ index d0a38c9..4f68424 100644
install : true,
)
diff --git a/src/freedreno/vulkan/tu_device.cc b/src/freedreno/vulkan/tu_device.cc
-index 62ebfd0..d273d0e 100644
+index 62ebfd0..951a198 100644
--- a/src/freedreno/vulkan/tu_device.cc
+++ b/src/freedreno/vulkan/tu_device.cc
-@@ -8,6 +8,9 @@
+@@ -8,6 +8,10 @@
*/
#include "tu_device.h"
+#if defined(__APPLE__) && defined(TU_HAS_MSM)
+#include <xf86drm.h> /* hakios_drm_host_surface_at (src/hakios-drm) */
++extern thread_local const void *tu_darwin_host_pointer;
+#endif
#include "drm-uapi/drm_fourcc.h"
#include "fdl/freedreno_layout.h"
-@@ -269,6 +272,10 @@ get_device_extensions(const struct tu_physical_device *device,
+@@ -269,6 +273,10 @@ get_device_extensions(const struct tu_physical_device *device,
.EXT_extended_dynamic_state2 = true,
.EXT_extended_dynamic_state3 = true,
.EXT_external_memory_dma_buf = true,
@@ -63,7 +64,7 @@ index 62ebfd0..d273d0e 100644
.EXT_filter_cubic = device->info->a6xx.has_tex_filter_cubic,
.EXT_fragment_density_map = true,
.EXT_global_priority = true,
-@@ -1160,6 +1167,9 @@ tu_get_properties(struct tu_physical_device *pdevice,
+@@ -1160,6 +1168,9 @@ tu_get_properties(struct tu_physical_device *pdevice,
/* VK_EXT_custom_border_color */
props->maxCustomBorderColorSamplers = TU_BORDER_COLOR_COUNT;
@@ -73,7 +74,7 @@ index 62ebfd0..d273d0e 100644
/* VK_KHR_performance_query */
props->allowCommandBufferQueryCopies = false;
-@@ -2709,6 +2719,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,
+@@ -2709,6 +2720,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,
"pthread condattr init");
goto fail_timeline_cond;
}
@@ -81,7 +82,7 @@ index 62ebfd0..d273d0e 100644
if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC) != 0) {
pthread_condattr_destroy(&condattr);
result = vk_startup_errorf(physical_device->instance,
-@@ -2716,6 +2727,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,
+@@ -2716,6 +2728,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,
"pthread condattr clock setup");
goto fail_timeline_cond;
}
@@ -89,7 +90,7 @@ index 62ebfd0..d273d0e 100644
if (pthread_cond_init(&device->timeline_cond, &condattr) != 0) {
pthread_condattr_destroy(&condattr);
result = vk_startup_errorf(physical_device->instance,
-@@ -3015,6 +3027,16 @@ vk_icdGetInstanceProcAddr(VkInstance instance, const char *pName)
+@@ -3015,6 +3028,16 @@ vk_icdGetInstanceProcAddr(VkInstance instance, const char *pName)
return tu_GetInstanceProcAddr(instance, pName);
}
@@ -106,7 +107,7 @@ index 62ebfd0..d273d0e 100644
VKAPI_ATTR VkResult VKAPI_CALL
tu_AllocateMemory(VkDevice _device,
const VkMemoryAllocateInfo *pAllocateInfo,
-@@ -3064,6 +3086,15 @@ tu_AllocateMemory(VkDevice _device,
+@@ -3064,6 +3087,17 @@ tu_AllocateMemory(VkDevice _device,
/* take ownership and close the fd */
close(fd_info->fd);
}
@@ -114,15 +115,17 @@ index 62ebfd0..d273d0e 100644
+ } else if (const VkImportMemoryHostPointerInfoEXT *host_info =
+ vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT)) {
+ /* an IOSurface's memory: the kernel makes a buffer of the surface (as of a dma-buf, whose fd is its ID here) */
++ /* any other pages: the kernel wires them (its userptr; "fd" -1 with the pointer beside it) */
+ uint32_t surface = hakios_drm_host_surface_at(host_info->pHostPointer, pAllocateInfo->allocationSize);
-+ result = surface != 0 ? tu_bo_init_dmabuf(device, &mem->bo, pAllocateInfo->allocationSize, (int) surface)
-+ : vk_errorf(device, VK_ERROR_INVALID_EXTERNAL_HANDLE,
-+ "host pointer %p is not an IOSurface's", host_info->pHostPointer);
++ if (surface == 0)
++ tu_darwin_host_pointer = host_info->pHostPointer;
++ result = tu_bo_init_dmabuf(device, &mem->bo, pAllocateInfo->allocationSize, surface != 0 ? (int) surface : -1);
++ tu_darwin_host_pointer = NULL;
+#endif
} else if (mem->vk.ahardware_buffer) {
#if DETECT_OS_ANDROID
const native_handle_t *handle = AHardwareBuffer_getNativeHandle(mem->vk.ahardware_buffer);
-@@ -3402,6 +3433,21 @@ tu_GetMemoryFdPropertiesKHR(VkDevice _device,
+@@ -3402,6 +3436,21 @@ tu_GetMemoryFdPropertiesKHR(VkDevice _device,
return VK_SUCCESS;
}
@@ -134,9 +137,9 @@ index 62ebfd0..d273d0e 100644
+ VkMemoryHostPointerPropertiesEXT *pMemoryHostPointerProperties)
+{
+ VK_FROM_HANDLE(tu_device, device, _device);
-+ /* only an IOSurface's memory can be imported: any other pointer, no type (its user places the memory as ever) */
++ /* an IOSurface's memory, or pages of this process's own (whole ones): any type */
+ pMemoryHostPointerProperties->memoryTypeBits =
-+ hakios_drm_host_surface_at(pHostPointer, 1) != 0 ? (1 << device->physical_device->memory.type_count) - 1 : 0;
++ ((uintptr_t) pHostPointer & 16383) == 0 ? (1 << device->physical_device->memory.type_count) - 1 : 0;
+ return VK_SUCCESS;
+}
+#endif
@@ -245,7 +248,7 @@ index 27916d0..569b1d6 100644
TU_RMV(bo_destroy, dev, bo);
diff --git a/src/freedreno/vulkan/tu_knl_drm_msm.cc b/src/freedreno/vulkan/tu_knl_drm_msm.cc
-index 08831e7..0a75392 100644
+index 08831e7..8da1a57 100644
--- a/src/freedreno/vulkan/tu_knl_drm_msm.cc
+++ b/src/freedreno/vulkan/tu_knl_drm_msm.cc
@@ -191,7 +191,12 @@ tu_drm_is_memory_type_supported(int fd, uint32_t flags)
@@ -261,14 +264,27 @@ index 08831e7..0a75392 100644
if (fd < 0) {
return vk_startup_errorf(
dev->physical_device->instance, VK_ERROR_INITIALIZATION_FAILED,
-@@ -680,9 +685,15 @@ msm_bo_init_dmabuf(struct tu_device *dev,
+@@ -674,15 +679,28 @@ msm_bo_init(struct tu_device *dev,
+ return result;
+ }
+
++#ifdef __APPLE__
++/* the host pointer an import of "fd" -1 is of (tu_AllocateMemory, VK_EXT_external_memory_host) */
++thread_local const void *tu_darwin_host_pointer;
++#endif
++
+ static VkResult
+ msm_bo_init_dmabuf(struct tu_device *dev,
+ struct tu_bo **out_bo,
uint64_t size,
int prime_fd)
{
+#ifdef __APPLE__
-+ /* the "fd" is an IOSurface's ID (src/hakios-drm): its size is the kernel's answer */
++ /* the "fd" is an IOSurface's ID (src/hakios-drm): its size is the kernel's answer; -1, the pages at
++ * tu_darwin_host_pointer (VK_EXT_external_memory_host) */
+ struct hakios_drm_surface described = { .surface = (uint32_t) prime_fd };
-+ off_t real_size = hakios_drm_surface(dev->fd, &described) == 0 ? (off_t) described.size : -1;
++ off_t real_size = prime_fd < 0 ? (off_t) size
++ : hakios_drm_surface(dev->fd, &described) == 0 ? (off_t) described.size : -1;
+#else
/* lseek() to get the real size */
off_t real_size = lseek(prime_fd, 0, SEEK_END);
@@ -277,7 +293,21 @@ index 08831e7..0a75392 100644
if (real_size < 0 || (uint64_t) real_size < size)
return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE);
-@@ -735,6 +746,12 @@ msm_bo_init_dmabuf(struct tu_device *dev,
+@@ -699,8 +717,13 @@ msm_bo_init_dmabuf(struct tu_device *dev,
+ msm_vma_lock(dev);
+
+ uint32_t gem_handle;
++#ifdef __APPLE__
++ int ret = prime_fd < 0 ? hakios_drm_user_memory(dev->fd, tu_darwin_host_pointer, size, &gem_handle)
++ : drmPrimeFDToHandle(dev->fd, prime_fd, &gem_handle);
++#else
+ int ret = drmPrimeFDToHandle(dev->fd, prime_fd,
+ &gem_handle);
++#endif
+ if (ret) {
+ msm_vma_unlock(dev);
+ u_rwlock_wrunlock(&dev->dma_bo_lock);
+@@ -735,6 +758,12 @@ msm_bo_init_dmabuf(struct tu_device *dev,
static VkResult
msm_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr)
{
@@ -290,7 +320,7 @@ index 08831e7..0a75392 100644
uint64_t offset = tu_gem_info(dev, bo->gem_handle, MSM_INFO_GET_OFFSET);
if (!offset)
return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);
-@@ -743,6 +760,7 @@ msm_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr)
+@@ -743,6 +772,7 @@ msm_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr)
void *map = mmap(placed_addr, bo->size, PROT_READ | PROT_WRITE,
MAP_SHARED | (placed_addr != NULL ? MAP_FIXED : 0),
dev->fd, offset);