Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
HakiosGPU: the client's own pages as a buffer (userptr); Turnip imports any host pointer
yehorkaa8e3346721a6d
diff --git a/drivers/gpu/hakios.gpu.cpp b/drivers/gpu/hakios.gpu.cppindex 8ed7772..0751dc7 100644--- a/drivers/gpu/hakios.gpu.cpp+++ b/drivers/gpu/hakios.gpu.cpp@@ -82,6 +82,7 @@ enum {kMsmSubmitQueueNew = 0x4a,kMsmSubmitQueueClose = 0x4b,kMsmSubmitQueueQuery = 0x4c,+ kHakiosUserMemory = 0x7d, /* ours: pages of the caller's own as a buffer (Linux's userptr) */kHakiosSurface = 0x7e, /* ours: an IOSurface's memory as a buffer */kHakiosShow = 0x7f, /* ours: pixels of the caller's onto the panel, to see a test's result */};@@ -141,6 +142,11 @@ struct msm_submitqueue_query {struct drm_gem_close {uint32_t handle, pad;};+struct hakios_user_memory {+ uint64_t address, size; /* in the caller, in whole pages */+ uint32_t handle; /* back: the buffer */+ uint32_t flags;+};struct hakios_surface {uint32_t surface; /* the IOSurface's ID */uint32_t handle; /* kHakiosDescribe: no buffer is wanted; else, back: the buffer, */@@ -352,6 +358,7 @@ struct gpu_buffer {IOMemoryDescriptor * memory; /* NULL: this handle is free */uint8_t * kernel; /* where it is in the kernel */MiniIOSurface * surface; /* whose memory it is, when not its own */+ IOMemoryMap * user; /* the kernel's view of it, when it is the client's own pages */uint64_t address; /* the GPU's for it */uint64_t size;uint32_t flags;@@ -371,6 +378,7 @@ struct gpu_buffer {struct gpu_left {IOMemoryDescriptor * memory;MiniIOSurface * surface;+ IOMemoryMap * user;uint64_t address, size;uint32_t fence, flags;};@@ -425,6 +433,7 @@ class HakiosGPUUserClient : public IOUserClientvoid reap(void);IOReturn gemNew(struct msm_gem_new * request);IOReturn surfaceBuffer(struct hakios_surface * request);+ IOReturn userBuffer(struct hakios_user_memory * request);IOReturn gemInfo(struct msm_gem_info * request);IOReturn submit(struct msm_gem_submit * request);IOReturn show(const struct hakios_show * request);@@ -478,6 +487,7 @@ HakiosGPUUserClient::freeBuffer(struct gpu_buffer * slot)slot->memory = NULL;slot->surface = NULL;+ slot->user = NULL;fBufferCount--;fBufferBytes -= b->size;@@ -491,6 +501,7 @@ HakiosGPUUserClient::freeBuffer(struct gpu_buffer * slot)left->memory = b->memory;left->surface = b->surface;+ left->user = b->user;left->address = b->address;left->size = b->size;left->fence = b->fence;@@ -507,12 +518,16 @@ HakiosGPUUserClient::freeBuffer(struct gpu_buffer * slot)if (said++ < 8) {IOLog("HakiosGPU: the GPU is not done with what it was given: a buffer of %llu bytes stays its own\n", b->size);}- } else if (b->surface == NULL && hakios_gpu_ready() && keep(b->memory, b->kernel, b->address, b->size, b->flags)) {+ } else if (b->surface == NULL && b->user == NULL && hakios_gpu_ready() &&+ keep(b->memory, b->kernel, b->address, b->size, b->flags)) {b->memory = NULL; /* (now the kept one's) */} else if (hakios_gpu_ready()) {hakios_gpu_unmap(b->address, b->size);hakios_gpu_address_free(b->address, b->size);}+ if (b->user != NULL && !busy) {+ b->user->release(); /* (the pages are unwired as the descriptor goes) */+ }if (b->memory != NULL) {b->memory->release();}@@ -533,13 +548,16 @@ HakiosGPUUserClient::reap(void)i++;continue;}- if (left->surface == NULL && hakios_gpu_ready() &&+ if (left->surface == NULL && left->user == NULL && hakios_gpu_ready() &&keep(left->memory, NULL, left->address, left->size, left->flags)) {left->memory = NULL;} else if (hakios_gpu_ready()) {hakios_gpu_unmap(left->address, left->size);hakios_gpu_address_free(left->address, left->size);}+ if (left->user != NULL) {+ left->user->release();+ }if (left->memory != NULL) {left->memory->release();}@@ -646,6 +664,7 @@ HakiosGPUUserClient::newBuffer(IOMemoryDescriptor * memory, uint64_t from, uint6b->seen = false;b->fence = 0;b->surface = NULL;+ b->user = NULL;fBufferCount++;fBufferBytes += size;return b;@@ -855,6 +874,59 @@ HakiosGPUUserClient::surfaceBuffer(struct hakios_surface * request)return kIOReturnSuccess;}+/*+ * Pages of the client's own as a buffer, as Linux's userptr: wired for as long as the buffer is there,+ * entered into the GPU's address space, and seen by the kernel through a mapping of its own (for the+ * cache: they are ordinary memory, pushed out of this processor's cache at every submission that+ * names them). Vulkan's VK_EXT_external_memory_host over it (Turnip, tools/mesa/hakios-drm): Metal's+ * no-copy buffers, CoreAnimation's backing stores.+ */+IOReturn+HakiosGPUUserClient::userBuffer(struct hakios_user_memory * request)+{+ IOMemoryDescriptor * memory;+ IOMemoryMap * map;+ struct gpu_buffer * b;+ uint64_t began = mach_absolute_time();++ if ((request->address & PAGE_MASK) || (request->size & PAGE_MASK) || request->size == 0 ||+ request->size > GPU_BUFFER_MAX) {+ return kIOReturnBadArgument;+ }+ memory = IOMemoryDescriptor::withAddressRange(request->address, request->size, kIODirectionInOut, fTask);+ if (memory == NULL) {+ return kIOReturnNoMemory;+ }+ if (memory->prepare() != kIOReturnSuccess) {+ memory->release();+ return kIOReturnVMError;+ }+ map = memory->createMappingInTask(kernel_task, 0, kIOMapAnywhere);+ if (map == NULL) {+ memory->complete();+ memory->release();+ return kIOReturnNoMemory;+ }+ IOLockLock(gGPULock);+ b = newBuffer(memory, 0, request->size, request->flags, &request->handle);+ if (b != NULL) {+ b->memory = memory;+ b->kernel = (uint8_t *) map->getVirtualAddress();+ b->user = map;+ b->seen = true;+ gGPUStat.surfaces++;+ }+ gGPUStat.taking += mach_absolute_time() - began;+ IOLockUnlock(gGPULock);+ if (b == NULL) {+ map->release();+ memory->complete();+ memory->release();+ return kIOReturnNoMemory;+ }+ return kIOReturnSuccess;+}+IOReturnHakiosGPUUserClient::gemInfo(struct msm_gem_info * request){@@ -931,8 +1003,8 @@ HakiosGPUUserClient::submit(struct msm_gem_submit * request)}b->seed = seed;b->pushed = began;- } else if (!b->seen || !gGPUCached) {- continue;+ } else if (b->user == NULL && (!b->seen || !gGPUCached)) {+ continue; /* (the client's own pages are ordinary memory: every time) */}hakios_gpu_to_memory(b->kernel, b->size);gGPUStat.flushed += b->size;@@ -1110,6 +1182,7 @@ HakiosGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentsstruct msm_submitqueue_query query;struct drm_gem_close close;struct hakios_surface surface;+ struct hakios_user_memory user;struct hakios_show show;} r;size_t size = args->structureInputSize;@@ -1200,6 +1273,9 @@ HakiosGPUUserClient::externalMethod(uint32_t selector, IOExternalMethodArgumentscase kHakiosSurface:ret = surfaceBuffer(&r.surface);break;+ case kHakiosUserMemory:+ ret = userBuffer(&r.user);+ break;case kHakiosShow:ret = show(&r.show);break;diff --git a/tools/mesa/hakios-drm/hakios.drm.c b/tools/mesa/hakios-drm/hakios.drm.cindex 9bfb780..b729fba 100644--- a/tools/mesa/hakios-drm/hakios.drm.c+++ b/tools/mesa/hakios-drm/hakios.drm.c@@ -22,6 +22,7 @@#include "drm-uapi/msm_drm.h"#define HAKIOS_GPU_SERVICE "HakiosGPU"+#define HAKIOS_GPU_USER_MEMORY 0x7d /* ours too: pages of this process as a buffer */#define HAKIOS_GPU_SURFACE 0x7e /* the request that is not Linux's */#define HAKIOS_GPU_DESCRIBE 0xffffffffu /* ... asked only what the buffer would be like */@@ -627,3 +628,18 @@ hakios_drm_host_surface_at(const void *base, uint64_t size)pthread_mutex_unlock(&host_surfaces_lock);return surface;}++int+hakios_drm_user_memory(int fd, const void *address, uint64_t size, uint32_t *handle)+{+ struct { uint64_t address, size; uint32_t handle, flags; } request = { (uintptr_t)address, size, 0, 0 };+ int error = call(HAKIOS_GPU_USER_MEMORY, &request, sizeof(request));++ (void)fd;+ if (error != 0) {+ errno = error;+ return -1;+ }+ *handle = request.handle;+ return 0;+}diff --git a/tools/mesa/hakios-drm/xf86drm.h b/tools/mesa/hakios-drm/xf86drm.hindex 09b3735..5449a1c 100644--- a/tools/mesa/hakios-drm/xf86drm.h+++ b/tools/mesa/hakios-drm/xf86drm.h@@ -108,6 +108,8 @@ int hakios_drm_surface(int fd, struct hakios_drm_surface *surface);* tools/m2v), for Turnip's VK_EXT_external_memory_host, which is given only the address: its ID, 0 if nobody said. */void hakios_drm_host_surface(const void *base, uint64_t size, uint32_t surface);uint32_t hakios_drm_host_surface_at(const void *base, uint64_t size);+/* Not libdrm's: pages of this process (whole ones) as a buffer, Linux's userptr (the kernel's request 0x7d) */+int hakios_drm_user_memory(int fd, const void *address, uint64_t size, uint32_t *handle);#ifdef __cplusplus}diff --git a/tools/mesa/turnip-darwin.patch b/tools/mesa/turnip-darwin.patchindex e2a59e7..66f8b41 100644--- a/tools/mesa/turnip-darwin.patch+++ b/tools/mesa/turnip-darwin.patch@@ -39,20 +39,21 @@ index d0a38c9..4f68424 100644install : true,)diff --git a/src/freedreno/vulkan/tu_device.cc b/src/freedreno/vulkan/tu_device.cc-index 62ebfd0..d273d0e 100644+index 62ebfd0..951a198 100644--- a/src/freedreno/vulkan/tu_device.cc+++ b/src/freedreno/vulkan/tu_device.cc-@@ -8,6 +8,9 @@+@@ -8,6 +8,10 @@*/#include "tu_device.h"+#if defined(__APPLE__) && defined(TU_HAS_MSM)+#include <xf86drm.h> /* hakios_drm_host_surface_at (src/hakios-drm) */++extern thread_local const void *tu_darwin_host_pointer;+#endif#include "drm-uapi/drm_fourcc.h"#include "fdl/freedreno_layout.h"-@@ -269,6 +272,10 @@ get_device_extensions(const struct tu_physical_device *device,+@@ -269,6 +273,10 @@ get_device_extensions(const struct tu_physical_device *device,.EXT_extended_dynamic_state2 = true,.EXT_extended_dynamic_state3 = true,.EXT_external_memory_dma_buf = true,@@ -63,7 +64,7 @@ index 62ebfd0..d273d0e 100644.EXT_filter_cubic = device->info->a6xx.has_tex_filter_cubic,.EXT_fragment_density_map = true,.EXT_global_priority = true,-@@ -1160,6 +1167,9 @@ tu_get_properties(struct tu_physical_device *pdevice,+@@ -1160,6 +1168,9 @@ tu_get_properties(struct tu_physical_device *pdevice,/* VK_EXT_custom_border_color */props->maxCustomBorderColorSamplers = TU_BORDER_COLOR_COUNT;@@ -73,7 +74,7 @@ index 62ebfd0..d273d0e 100644/* VK_KHR_performance_query */props->allowCommandBufferQueryCopies = false;-@@ -2709,6 +2719,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,+@@ -2709,6 +2720,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,"pthread condattr init");goto fail_timeline_cond;}@@ -81,7 +82,7 @@ index 62ebfd0..d273d0e 100644if (pthread_condattr_setclock(&condattr, CLOCK_MONOTONIC) != 0) {pthread_condattr_destroy(&condattr);result = vk_startup_errorf(physical_device->instance,-@@ -2716,6 +2727,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,+@@ -2716,6 +2728,7 @@ tu_CreateDevice(VkPhysicalDevice physicalDevice,"pthread condattr clock setup");goto fail_timeline_cond;}@@ -89,7 +90,7 @@ index 62ebfd0..d273d0e 100644if (pthread_cond_init(&device->timeline_cond, &condattr) != 0) {pthread_condattr_destroy(&condattr);result = vk_startup_errorf(physical_device->instance,-@@ -3015,6 +3027,16 @@ vk_icdGetInstanceProcAddr(VkInstance instance, const char *pName)+@@ -3015,6 +3028,16 @@ vk_icdGetInstanceProcAddr(VkInstance instance, const char *pName)return tu_GetInstanceProcAddr(instance, pName);}@@ -106,7 +107,7 @@ index 62ebfd0..d273d0e 100644VKAPI_ATTR VkResult VKAPI_CALLtu_AllocateMemory(VkDevice _device,const VkMemoryAllocateInfo *pAllocateInfo,-@@ -3064,6 +3086,15 @@ tu_AllocateMemory(VkDevice _device,+@@ -3064,6 +3087,17 @@ tu_AllocateMemory(VkDevice _device,/* take ownership and close the fd */close(fd_info->fd);}@@ -114,15 +115,17 @@ index 62ebfd0..d273d0e 100644+ } else if (const VkImportMemoryHostPointerInfoEXT *host_info =+ vk_find_struct_const(pAllocateInfo->pNext, IMPORT_MEMORY_HOST_POINTER_INFO_EXT)) {+ /* an IOSurface's memory: the kernel makes a buffer of the surface (as of a dma-buf, whose fd is its ID here) */++ /* any other pages: the kernel wires them (its userptr; "fd" -1 with the pointer beside it) */+ uint32_t surface = hakios_drm_host_surface_at(host_info->pHostPointer, pAllocateInfo->allocationSize);-+ result = surface != 0 ? tu_bo_init_dmabuf(device, &mem->bo, pAllocateInfo->allocationSize, (int) surface)-+ : vk_errorf(device, VK_ERROR_INVALID_EXTERNAL_HANDLE,-+ "host pointer %p is not an IOSurface's", host_info->pHostPointer);++ if (surface == 0)++ tu_darwin_host_pointer = host_info->pHostPointer;++ result = tu_bo_init_dmabuf(device, &mem->bo, pAllocateInfo->allocationSize, surface != 0 ? (int) surface : -1);++ tu_darwin_host_pointer = NULL;+#endif} else if (mem->vk.ahardware_buffer) {#if DETECT_OS_ANDROIDconst native_handle_t *handle = AHardwareBuffer_getNativeHandle(mem->vk.ahardware_buffer);-@@ -3402,6 +3433,21 @@ tu_GetMemoryFdPropertiesKHR(VkDevice _device,+@@ -3402,6 +3436,21 @@ tu_GetMemoryFdPropertiesKHR(VkDevice _device,return VK_SUCCESS;}@@ -134,9 +137,9 @@ index 62ebfd0..d273d0e 100644+ VkMemoryHostPointerPropertiesEXT *pMemoryHostPointerProperties)+{+ VK_FROM_HANDLE(tu_device, device, _device);-+ /* only an IOSurface's memory can be imported: any other pointer, no type (its user places the memory as ever) */++ /* an IOSurface's memory, or pages of this process's own (whole ones): any type */+ pMemoryHostPointerProperties->memoryTypeBits =-+ hakios_drm_host_surface_at(pHostPointer, 1) != 0 ? (1 << device->physical_device->memory.type_count) - 1 : 0;++ ((uintptr_t) pHostPointer & 16383) == 0 ? (1 << device->physical_device->memory.type_count) - 1 : 0;+ return VK_SUCCESS;+}+#endif@@ -245,7 +248,7 @@ index 27916d0..569b1d6 100644TU_RMV(bo_destroy, dev, bo);diff --git a/src/freedreno/vulkan/tu_knl_drm_msm.cc b/src/freedreno/vulkan/tu_knl_drm_msm.cc-index 08831e7..0a75392 100644+index 08831e7..8da1a57 100644--- a/src/freedreno/vulkan/tu_knl_drm_msm.cc+++ b/src/freedreno/vulkan/tu_knl_drm_msm.cc@@ -191,7 +191,12 @@ tu_drm_is_memory_type_supported(int fd, uint32_t flags)@@ -261,14 +264,27 @@ index 08831e7..0a75392 100644if (fd < 0) {return vk_startup_errorf(dev->physical_device->instance, VK_ERROR_INITIALIZATION_FAILED,-@@ -680,9 +685,15 @@ msm_bo_init_dmabuf(struct tu_device *dev,+@@ -674,15 +679,28 @@ msm_bo_init(struct tu_device *dev,+ return result;+ }+++#ifdef __APPLE__++/* the host pointer an import of "fd" -1 is of (tu_AllocateMemory, VK_EXT_external_memory_host) */++thread_local const void *tu_darwin_host_pointer;++#endif+++ static VkResult+ msm_bo_init_dmabuf(struct tu_device *dev,+ struct tu_bo **out_bo,uint64_t size,int prime_fd){+#ifdef __APPLE__-+ /* the "fd" is an IOSurface's ID (src/hakios-drm): its size is the kernel's answer */++ /* the "fd" is an IOSurface's ID (src/hakios-drm): its size is the kernel's answer; -1, the pages at++ * tu_darwin_host_pointer (VK_EXT_external_memory_host) */+ struct hakios_drm_surface described = { .surface = (uint32_t) prime_fd };-+ off_t real_size = hakios_drm_surface(dev->fd, &described) == 0 ? (off_t) described.size : -1;++ off_t real_size = prime_fd < 0 ? (off_t) size++ : hakios_drm_surface(dev->fd, &described) == 0 ? (off_t) described.size : -1;+#else/* lseek() to get the real size */off_t real_size = lseek(prime_fd, 0, SEEK_END);@@ -277,7 +293,21 @@ index 08831e7..0a75392 100644if (real_size < 0 || (uint64_t) real_size < size)return vk_error(dev, VK_ERROR_INVALID_EXTERNAL_HANDLE);-@@ -735,6 +746,12 @@ msm_bo_init_dmabuf(struct tu_device *dev,+@@ -699,8 +717,13 @@ msm_bo_init_dmabuf(struct tu_device *dev,+ msm_vma_lock(dev);++ uint32_t gem_handle;++#ifdef __APPLE__++ int ret = prime_fd < 0 ? hakios_drm_user_memory(dev->fd, tu_darwin_host_pointer, size, &gem_handle)++ : drmPrimeFDToHandle(dev->fd, prime_fd, &gem_handle);++#else+ int ret = drmPrimeFDToHandle(dev->fd, prime_fd,+ &gem_handle);++#endif+ if (ret) {+ msm_vma_unlock(dev);+ u_rwlock_wrunlock(&dev->dma_bo_lock);+@@ -735,6 +758,12 @@ msm_bo_init_dmabuf(struct tu_device *dev,static VkResultmsm_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr){@@ -290,7 +320,7 @@ index 08831e7..0a75392 100644uint64_t offset = tu_gem_info(dev, bo->gem_handle, MSM_INFO_GET_OFFSET);if (!offset)return vk_error(dev, VK_ERROR_OUT_OF_DEVICE_MEMORY);-@@ -743,6 +760,7 @@ msm_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr)+@@ -743,6 +772,7 @@ msm_bo_map(struct tu_device *dev, struct tu_bo *bo, void *placed_addr)void *map = mmap(placed_addr, bo->size, PROT_READ | PROT_WRITE,MAP_SHARED | (placed_addr != NULL ? MAP_FIXED : 0),dev->fd, offset);