Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
/*
* MiniIOSurface: a stand-in for Apple's closed IOSurface.kext (see MiniIOSurface.h).
*
* A surface is a piece of wired kernel memory: one page of state that clients read directly,
* followed by the pixels. Every client that knows a surface has the whole of it mapped once.
* Surfaces travel between processes as Mach send rights (IOSurfaceSendRight) or by number.
*
* What is not here: child surfaces, wrapping of client memory in pieces, purgeability, compression,
* notifications. Calls that are not implemented are logged once and refused.
*/
#include <IOKit/IOLib.h>
#include <pexpert/pexpert.h>
#include <IOKit/IOLocks.h>
#include <IOKit/IOKitKeys.h>
#include <IOKit/IOMemoryDescriptor.h>
#include <libkern/c++/OSSerialize.h>
#include <libkern/c++/OSUnserialize.h>
#include <libkern/c++/OSDictionary.h>
#include <libkern/c++/OSCollectionIterator.h>
#include <libkern/c++/OSSymbol.h>
#include <libkern/c++/OSArray.h>
#include <libkern/c++/OSNumber.h>
#include <libkern/c++/OSBoolean.h>
#include <libkern/c++/OSString.h>
#include <libkern/c++/OSSymbol.h>
#include <kern/thread_call.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include "MiniIOSurface.h"
static_assert(sizeof(struct minisurf_plane) == 0x80, "plane layout");
static_assert(sizeof(struct minisurf_result) == 0x68, "result layout");
static_assert(offsetof(struct minisurf_shared, attachments) == 0x30, "shared layout");
static_assert(sizeof(struct minisurf_result) + 20 * sizeof(struct minisurf_plane) == MINISURF_RESULT_MAX, "result size");
/* IOConnectCallMethod selectors of IOSurfaceRootUserClient (Darwin userland 16.5) */
enum {
kSurfaceMethodCreate = 0,
kSurfaceMethodLookup = 4,
kSurfaceMethodSetYCbCrMatrix = 5,
kSurfaceMethodWrapClientImage = 6,
kSurfaceMethodWrapClientMemory = 7,
kSurfaceMethodSetValue = 9,
kSurfaceMethodCopyValue = 10,
kSurfaceMethodRemoveValue = 11,
kSurfaceMethodBindAccel = 12,
kSurfaceMethodInit = 13,
kSurfaceMethodLog = 19,
kSurfaceMethodSetPurgeable = 20,
kSurfaceMethodSetOwnership = 21,
kSurfaceMethodSetTiled = 22,
kSurfaceMethodSetTimestamp = 24,
kSurfaceMethodSetBulkAttachments = 27,
kSurfaceMethodCommPageAddress = 32,
kSurfaceMethodSetIndexedTimestamp = 33,
kSurfaceMethodLookupFromPort = 34,
kSurfaceMethodCreatePort = 35,
kSurfaceMethodDetachModeCode = 43,
kSurfaceMethodSetOwnershipIdentity = 44,
};
/* IOConnectTrap indices */
enum {
kSurfaceTrapIncrementUseCount = 0,
kSurfaceTrapDecrementUseCount = 1,
kSurfaceTrapLock = 2,
kSurfaceTrapUnlock = 3,
kSurfaceTrapDetach = 4, /* the client is about to release: forget it has the surface */
kSurfaceTrapRelease = 5,
kSurfaceTrapCount
};
#define kSurfaceLockReadOnly 1u
#define MINISURF_TABLE_SIZE 16384
#define MINISURF_MAX_DIMENSION 16384
#define MINISURF_MAX_ALLOC (512u << 20)
#define MINISURF_ROW_ALIGN 64
#define FOURCC(a, b, c, d) ((uint32_t)(a) << 24 | (uint32_t)(b) << 16 | (uint32_t)(c) << 8 | (uint32_t)(d))
static IOLock * gSurfaceLock;
static MiniIOSurface * gSurfaceTable[MINISURF_TABLE_SIZE]; /* index = surface number; [0] unused */
static uint32_t gSurfaceNext = 1;
static uint32_t gSurfaceCount;
static uint64_t gSurfaceBytes;
static uint32_t gSurfaceUnheld; /* surfaces around client memory that could not be made (said a few times) */
/* what making surfaces and taking them away has cost since it was last asked (MiniIOSurfaceCost()) */
static uint64_t gSurfaceMaking, gSurfaceFreeing;
static uint32_t gSurfacesMade, gSurfacesFreed;
static uint64_t gSurfaceLogged[2]; /* selectors already complained about */
/* Boot-arg surftrace=N: log the first N surfaces created (default 24). */
static TUNABLE(uint32_t, gSurfaceTrace, "surftrace", 24);
static uint32_t gSurfaceTraced; /* (a tunable is read-only after boot) */
/*
* Boot-arg surftrace_of=NAME: only the surfaces made by programs whose name begins with NAME count for
* surftrace (then up to N of theirs), and what becomes of them is said as well: the ports made for them
* and who looks them up from a port (another program, which shows it: the window server for a layer).
*/
static char gSurfaceTraceOf[32];
extern "C" void loader_log_line(const char *line);
/* A line about a traced surface: to the loader's log too, where it outlives the kernel's own, short log. */
static void
traceLine(const char * format, ...)
{
char line[256];
va_list list;
va_start(list, format);
vsnprintf(line, sizeof(line), format, list);
va_end(list);
if (gSurfaceTraceOf[0] != 0) {
loader_log_line(line);
}
IOLog("%s", line);
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
OSDefineMetaClassAndStructors(MiniIOSurface, OSObject);
/*
* The memory of the big surfaces is kept when they go, a few pieces of it, and given to the
* next surface of the same size. CoreAnimation makes a surface of the whole display for every
* frame and frees one: 14 MB got, cleared, entered into the kernel's map and taken out of it
* again, sixty times a second -- 2.8 ms to make and 2 ms to free each, a quarter of the time
* there is for a frame, on the thread that draws it. A piece that is kept is cleared again
* before it is given out: a new surface is all zeroes, whoever makes it. The clearing is done
* by the display's driver between two frames (MiniIOSurfaceClearKept(): while it copies a
* frame to the panel the memory is busy enough, and the copy took half as long again when
* this ran beside it), or a moment later by a thread of this file's if no frames are going
* out, or by whoever asks for the memory if that comes first.
*/
#define SURFACE_KEEP_FROM (4u << 20) /* bytes: smaller ones are not worth it */
#define SURFACE_KEPT 4
#define SURFACE_CLEAR_MS 40 /* kept memory is cleared this long after, if the display's driver has not done it */
static struct {
IOBufferMemoryDescriptor * memory;
bool clear; /* cleared since it was last a surface's */
bool busy; /* being cleared right now */
uint64_t since; /* when it was kept */
} gSurfaceKept[SURFACE_KEPT];
static uint32_t gSurfacesNotKept[2]; /* memory not kept: somebody else had it too; it put out a piece kept longer */
static IOLock * gSurfaceKeptLock;
static thread_call_t gSurfaceClearing;
static uint32_t gSurfacesGivenAgain; /* for MiniIOSurfaceCost() */
/* boot-arg surf_clear=0: kept memory is given out as it is (to see what the clearing costs; a surface is then not all zeroes) */
static TUNABLE(uint32_t, gSurfaceClear, "surf_clear", 1);
/* boot-arg surf_wrap=0: surfaces around the client's pictures refused, as before methodWrapClientImage (for comparing) */
static TUNABLE(uint32_t, gSurfaceWrap, "surf_wrap", 1);
void
MiniIOSurfaceClearKept(void)
{
if (gSurfaceKeptLock == NULL || !gSurfaceClear) {
return;
}
IOLockLock(gSurfaceKeptLock);
for (unsigned i = 0; i < SURFACE_KEPT; i++) {
IOBufferMemoryDescriptor * memory = gSurfaceKept[i].memory;
if (memory == NULL || gSurfaceKept[i].clear || gSurfaceKept[i].busy) {
continue;
}
gSurfaceKept[i].busy = true;
IOLockUnlock(gSurfaceKeptLock);
bzero(memory->getBytesNoCopy(), memory->getCapacity());
IOLockLock(gSurfaceKeptLock);
gSurfaceKept[i].busy = false;
gSurfaceKept[i].clear = true;
}
IOLockUnlock(gSurfaceKeptLock);
}
static void
surfaceMemoryClear(thread_call_param_t unused0 __unused, thread_call_param_t unused1 __unused)
{
MiniIOSurfaceClearKept();
}
/* Memory for a surface: the state page and `bytes` after it, all zeroes. */
static IOBufferMemoryDescriptor *
surfaceMemory(vm_size_t bytes)
{
IOBufferMemoryDescriptor * memory = NULL;
bool clear = false;
if (bytes >= SURFACE_KEEP_FROM && gSurfaceKeptLock != NULL) {
IOLockLock(gSurfaceKeptLock);
for (int pass = 0; pass < 2 && memory == NULL; pass++) {
/* one that is clear already, if there is one */
for (unsigned i = 0; i < SURFACE_KEPT && memory == NULL; i++) {
if (gSurfaceKept[i].memory != NULL && !gSurfaceKept[i].busy && (pass == 1 || gSurfaceKept[i].clear) &&
gSurfaceKept[i].memory->getCapacity() == bytes) {
memory = gSurfaceKept[i].memory;
clear = gSurfaceKept[i].clear;
gSurfaceKept[i].memory = NULL;
}
}
}
gSurfacesGivenAgain += memory != NULL;
IOLockUnlock(gSurfaceKeptLock);
if (memory != NULL && !clear) {
bzero(memory->getBytesNoCopy(), gSurfaceClear ? bytes : PAGE_SIZE);
}
}
if (memory == NULL) {
memory = IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, kIODirectionInOut | kIOMemoryKernelUserShared, bytes,
PAGE_SIZE);
}
return memory;
}
/*
* A surface is gone: its memory is kept for the next, if it is big and nobody else has it. When
* all the places are taken, it takes the place of the piece kept longest: pieces of a size nobody
* asks for again (a big picture made once) once stayed there for good, and then no frame's memory
* was kept any more -- each made afresh and freed (2 to 4 ms, and 1 to 2) instead of 0.3 and 0.05.
*/
static void
surfaceMemoryBack(IOBufferMemoryDescriptor * memory)
{
IOBufferMemoryDescriptor * evicted = NULL;
if (memory->getCapacity() >= SURFACE_KEEP_FROM && memory->getRetainCount() != 1) {
OSIncrementAtomic(&gSurfacesNotKept[0]);
}
if (memory->getCapacity() >= SURFACE_KEEP_FROM && memory->getRetainCount() == 1 && gSurfaceKeptLock != NULL) {
int oldest = -1;
IOLockLock(gSurfaceKeptLock);
for (unsigned i = 0; i < SURFACE_KEPT && memory != NULL; i++) {
if (gSurfaceKept[i].memory == NULL) {
gSurfaceKept[i].memory = memory;
gSurfaceKept[i].clear = gSurfaceKept[i].busy = false;
gSurfaceKept[i].since = mach_absolute_time();
memory = NULL;
} else if (!gSurfaceKept[i].busy && (oldest < 0 || gSurfaceKept[i].since < gSurfaceKept[oldest].since)) {
oldest = (int) i;
}
}
if (memory != NULL && oldest >= 0) {
evicted = gSurfaceKept[oldest].memory;
gSurfaceKept[oldest].memory = memory;
gSurfaceKept[oldest].clear = gSurfaceKept[oldest].busy = false;
gSurfaceKept[oldest].since = mach_absolute_time();
memory = NULL;
gSurfacesNotKept[1]++;
}
IOLockUnlock(gSurfaceKeptLock);
if (evicted != NULL) {
evicted->release();
}
if (memory == NULL) {
if (gSurfaceClear) {
AbsoluteTime deadline;
clock_interval_to_deadline(SURFACE_CLEAR_MS, kMillisecondScale, &deadline);
thread_call_enter_delayed(gSurfaceClearing, deadline);
}
return;
}
}
memory->release();
}
void
MiniIOSurface::free(void)
{
uint64_t began = mach_absolute_time();
if (traced && kpixels != NULL) {
uint32_t nonzero = 0, middle = 0;
for (uint32_t at = 0; at + 4 <= info.allocSize; at += 64) {
nonzero += kpixels[at] != 0 || kpixels[at + 1] != 0 || kpixels[at + 2] != 0 || kpixels[at + 3] != 0;
}
memcpy(&middle, kpixels + (info.allocSize / 2 & ~3u), 4);
traceLine("MiniIOSurface: surface %u (%ux%u) goes: of every 64 bytes %u of %u begin with a pixel that is not 0, one in "
"the middle %#010x\n", sid, info.width, info.height, nonzero, info.allocSize / 64, middle);
}
OSSafeReleaseNULL(values);
OSSafeReleaseNULL(clientMap);
if (client != NULL) {
client->complete();
client->release();
client = NULL;
}
if (memory != NULL) {
surfaceMemoryBack(memory);
memory = NULL;
}
OSObject::free();
OSAddAtomic64((SInt64) (mach_absolute_time() - began), (SInt64 *) &gSurfaceFreeing);
OSIncrementAtomic(&gSurfacesFreed);
}
/* How many surfaces were made and freed since this was last asked, and the microseconds each took on average. */
void
MiniIOSurfaceCost(uint32_t * made, uint64_t * making, uint32_t * freed, uint64_t * freeing, uint32_t * again)
{
uint64_t nanos;
*again = OSBitAndAtomic(0, &gSurfacesGivenAgain);
*made = OSBitAndAtomic(0, &gSurfacesMade);
*freed = OSBitAndAtomic(0, &gSurfacesFreed);
absolutetime_to_nanoseconds(gSurfaceMaking, &nanos);
*making = *made ? nanos / NSEC_PER_USEC / *made : 0;
absolutetime_to_nanoseconds(gSurfaceFreeing, &nanos);
*freeing = *freed ? nanos / NSEC_PER_USEC / *freed : 0;
gSurfaceMaking = gSurfaceFreeing = 0;
}
/* The values changed: a new seed, and how much room a client needs to fetch them. */
void
MiniIOSurface::updateValuesState(void)
{
struct minisurf_shared * sh = shared();
uint32_t size = 0;
if (values != NULL && values->getCount() != 0) {
OSSerialize * s = OSSerialize::binaryWithCapacity(4096);
if (s != NULL) {
if (values->serialize(s)) {
size = s->getLength() + 8;
}
s->release();
}
}
sh->valuesSeed = (sh->valuesSeed + 2) | 1;
sh->valuesSize = size;
}
static uint32_t
dictNumber(OSDictionary * dict, const char * key, uint32_t missing)
{
OSObject * obj = dict ? dict->getObject(key) : NULL;
OSNumber * num = OSDynamicCast(OSNumber, obj);
if (num != NULL) {
return num->unsigned32BitValue();
}
if (obj == kOSBooleanTrue) {
return 1;
}
if (obj == kOSBooleanFalse) {
return 0;
}
return missing;
}
static uint32_t
alignUp(uint64_t value, uint32_t to)
{
return (uint32_t) ((value + to - 1) / to * to);
}
/* Bytes per element when the client did not say. */
static uint32_t
defaultBytesPerElement(uint32_t format)
{
switch (format) {
case FOURCC('L', '0', '0', '8'):
case FOURCC('A', '0', '0', '8'):
case 0x00000008:
return 1;
case FOURCC('L', '5', '6', '5'):
case FOURCC('B', '5', '6', '5'):
case FOURCC('L', '5', '5', '5'):
case FOURCC('2', 'v', 'u', 'y'):
case FOURCC('y', 'u', 'v', 's'):
case FOURCC('y', 'u', 'v', 'f'):
case FOURCC('L', '0', '1', '6'):
case FOURCC('L', '0', '0', 'h'):
return 2;
case FOURCC('R', 'G', 'h', 'A'):
return 8;
case FOURCC('R', 'G', 'f', 'A'):
return 16;
default:
return 4;
}
}
/*
* Two-plane YCbCr formats: luma, then interleaved chroma.
* Returns false for anything else. *wide: 10-bit samples in 16 bits.
*/
static bool
biplanarLayout(uint32_t format, uint32_t * hsub, uint32_t * vsub, bool * wide)
{
switch (format) {
case FOURCC('4', '2', '0', 'v'): case FOURCC('4', '2', '0', 'f'):
*hsub = 2; *vsub = 2; *wide = false; return true;
case FOURCC('4', '2', '2', 'v'): case FOURCC('4', '2', '2', 'f'):
*hsub = 2; *vsub = 1; *wide = false; return true;
case FOURCC('4', '4', '4', 'v'): case FOURCC('4', '4', '4', 'f'):
*hsub = 1; *vsub = 1; *wide = false; return true;
case FOURCC('x', '4', '2', '0'): case FOURCC('x', 'f', '2', '0'):
*hsub = 2; *vsub = 2; *wide = true; return true;
case FOURCC('x', '4', '2', '2'): case FOURCC('x', 'f', '2', '2'):
*hsub = 2; *vsub = 1; *wide = true; return true;
case FOURCC('x', '4', '4', '4'): case FOURCC('x', 'f', '4', '4'):
*hsub = 1; *vsub = 1; *wide = true; return true;
default:
return false;
}
}
static void
describeComponents(struct minisurf_plane * plane, uint32_t format)
{
/* in memory order; names: 1 alpha, 2 red, 3 green, 4 blue */
static const uint8_t bgra[4] = { 4, 3, 2, 1 };
static const uint8_t rgba[4] = { 2, 3, 4, 1 };
const uint8_t * names;
if (format == FOURCC('B', 'G', 'R', 'A')) {
names = bgra;
} else if (format == FOURCC('R', 'G', 'B', 'A')) {
names = rgba;
} else {
return;
}
plane->componentCount = 4;
for (unsigned i = 0; i < 4; i++) {
plane->componentBitDepth[i] = 8;
plane->componentBitOffset[i] = (uint8_t) (8 * i);
plane->componentName[i] = names[i];
plane->componentType[i] = 1; /* unsigned integer */
plane->componentRange[i] = 1; /* full range */
}
}
/*
* Make a surface from the properties a client gave to IOSurfaceCreate().
* Returned with one holder, and entered in the table.
*/
static MiniIOSurface *
surfaceCreate(OSDictionary * props, IOReturn * error)
{
MiniIOSurface * s;
struct minisurf_result * info;
uint32_t width, height, format, bpe, ew, eh, bpr, alloc, offset, planeCount = 0;
uint32_t hsub, vsub;
bool wide, onePlane = false;
OSArray * planeInfo;
OSNumber * wrapped;
uint64_t needed;
*error = kIOReturnBadArgument;
width = dictNumber(props, "IOSurfaceWidth", 0);
height = dictNumber(props, "IOSurfaceHeight", 0);
format = dictNumber(props, "IOSurfacePixelFormat", 0);
/*
* Ten bits a colour, packed into 32-bit pixels: what a device with a wide-gamut display
* (the iPhone X is one) asks for when it wants a picture of the screen, a snapshot for a
* transition for instance. CoreAnimation's software renderer does not draw into those
* ("SW renderer invalid destination format w30r": the transition comes out black), and
* there is no wide gamut on this panel anyway. The surface is made as ordinary BGRA of
* the same geometry, and says so: whoever uses it reads the format back from the surface.
*/
if (format == 'w30r' || format == 'l10r' || format == 'w30R') {
format = 'BGRA';
}
/*
* The same with alpha in a plane of its own: what CoreAnimation makes the screen's own
* surfaces of once it draws with a GPU's renderer (OpenGL ES, userland/src/eagl.c) instead
* of its software one. One plane of BGRA here too; the framebuffer shows nothing else.
*/
if (format == 'b3a8') {
format = 'BGRA';
onePlane = true;
}
ew = dictNumber(props, "IOSurfaceElementWidth", 1);
eh = dictNumber(props, "IOSurfaceElementHeight", 1);
bpe = dictNumber(props, "IOSurfaceBytesPerElement", 0);
offset = dictNumber(props, "IOSurfaceOffset", 0);
if (width > MINISURF_MAX_DIMENSION || height > MINISURF_MAX_DIMENSION || ew == 0 || eh == 0 || ew > 255 || eh > 255) {
return NULL;
}
if (props->getObject("IOSurfaceAddressRanges") != NULL) {
IOLog("MiniIOSurface: wrapping client memory in pieces is not supported\n");
*error = kIOReturnUnsupported;
return NULL;
}
/*
* A surface around memory the client has already: CoreAnimation's OpenGL ES renderer makes
* them around pictures it has in memory (an application's icon, a layer's contents as the
* application has drawn them), to hand them to the GPU as they are. The surface's pixels
* are that memory itself, not a copy: the picture goes on being drawn into after the
* surface is made, and whoever looks at the surface is to see what is there now.
*/
wrapped = OSDynamicCast(OSNumber, props->getObject("IOSurfaceAddress"));
s = OSTypeAlloc(MiniIOSurface);
if (s == NULL || !s->init()) {
OSSafeReleaseNULL(s);
*error = kIOReturnNoMemory;
return NULL;
}
info = &s->info;
planeInfo = onePlane ? NULL : OSDynamicCast(OSArray, props->getObject("IOSurfacePlaneInfo"));
needed = 0;
if (planeInfo != NULL && planeInfo->getCount() != 0) {
planeCount = planeInfo->getCount();
if (planeCount > MINISURF_MAX_PLANES) {
IOLog("MiniIOSurface: %u planes asked for, %u supported\n", planeCount, MINISURF_MAX_PLANES);
s->release();
return NULL;
}
for (uint32_t i = 0; i < planeCount; i++) {
OSDictionary * pd = OSDynamicCast(OSDictionary, planeInfo->getObject(i));
struct minisurf_plane * p = &s->planes[i];
uint32_t pew = dictNumber(pd, "IOSurfacePlaneElementWidth", 1);
uint32_t peh = dictNumber(pd, "IOSurfacePlaneElementHeight", 1);
if (pew == 0 || peh == 0 || pew > 255 || peh > 255) {
s->release();
return NULL;
}
p->width = dictNumber(pd, "IOSurfacePlaneWidth", width);
p->height = dictNumber(pd, "IOSurfacePlaneHeight", height);
/* a plane is no bigger than a surface may be: the sizes below are not allowed to wrap */
if (p->width > MINISURF_MAX_DIMENSION || p->height > MINISURF_MAX_DIMENSION) {
s->release();
return NULL;
}
p->bytesPerElement = (uint16_t) dictNumber(pd, "IOSurfacePlaneBytesPerElement", bpe ? bpe : 1);
p->elementWidth = (uint8_t) pew;
p->elementHeight = (uint8_t) peh;
p->bytesPerRow = dictNumber(pd, "IOSurfacePlaneBytesPerRow", 0);
if (p->bytesPerRow == 0) {
p->bytesPerRow = alignUp((uint64_t) ((p->width + pew - 1) / pew) * p->bytesPerElement, MINISURF_ROW_ALIGN);
}
p->size = dictNumber(pd, "IOSurfacePlaneSize", 0);
if (p->size == 0) {
/* (in 64 bits: a stride of the caller's times the rows can pass 32) */
uint64_t planeBytes = (uint64_t) p->bytesPerRow * ((p->height + peh - 1) / peh);
if (planeBytes > MINISURF_MAX_ALLOC) {
s->release();
return NULL;
}
p->size = (uint32_t) planeBytes;
}
p->offset = dictNumber(pd, "IOSurfacePlaneOffset", dictNumber(pd, "IOSurfacePlaneBase", (uint32_t) needed));
p->extendedPixels[0] = dictNumber(pd, "IOSurfacePlaneExtendedPixelsOnLeft", 0);
p->extendedPixels[1] = dictNumber(pd, "IOSurfacePlaneExtendedPixelsOnRight", 0);
p->extendedPixels[2] = dictNumber(pd, "IOSurfacePlaneExtendedPixelsOnTop", 0);
p->extendedPixels[3] = dictNumber(pd, "IOSurfacePlaneExtendedPixelsOnBottom", 0);
if ((uint64_t) p->offset + p->size > needed) {
needed = (uint64_t) p->offset + p->size;
}
}
} else if (biplanarLayout(format, &hsub, &vsub, &wide) && width != 0 && height != 0) {
struct minisurf_plane * y = &s->planes[0], * c = &s->planes[1];
planeCount = 2;
y->width = width;
y->height = height;
y->bytesPerElement = wide ? 2 : 1;
y->elementWidth = y->elementHeight = 1;
y->bytesPerRow = alignUp((uint64_t) width * y->bytesPerElement, MINISURF_ROW_ALIGN);
y->size = y->bytesPerRow * height;
y->offset = 0;
c->width = (width + hsub - 1) / hsub;
c->height = (height + vsub - 1) / vsub;
c->bytesPerElement = wide ? 4 : 2;
c->elementWidth = c->elementHeight = 1;
c->bytesPerRow = alignUp((uint64_t) c->width * c->bytesPerElement, MINISURF_ROW_ALIGN);
c->size = c->bytesPerRow * c->height;
c->offset = y->size;
needed = (uint64_t) c->offset + c->size;
}
if (bpe == 0) {
bpe = planeCount ? s->planes[0].bytesPerElement : defaultBytesPerElement(format);
}
bpr = dictNumber(props, "IOSurfaceBytesPerRow", 0);
if (bpr == 0) {
bpr = planeCount ? s->planes[0].bytesPerRow
: alignUp((uint64_t) ((width + ew - 1) / ew) * bpe, MINISURF_ROW_ALIGN);
}
if (planeCount == 0) {
needed = (uint64_t) bpr * ((height + eh - 1) / eh);
/* the one implicit plane, for the per-plane getters */
s->planes[0].width = width;
s->planes[0].height = height;
s->planes[0].bytesPerRow = bpr;
s->planes[0].bytesPerElement = (uint16_t) bpe;
s->planes[0].elementWidth = (uint8_t) ew;
s->planes[0].elementHeight = (uint8_t) eh;
s->planes[0].size = (uint32_t) needed;
describeComponents(&s->planes[0], format);
}
needed += offset;
alloc = dictNumber(props, "IOSurfaceAllocSize", 0);
if (alloc < needed) {
/* (a surface around the owner's memory has the owner's size: a mask's last row is only as long as it is wide) */
if (alloc != 0 && wrapped == NULL) {
IOLog("MiniIOSurface: alloc size %u too small for %ux%u '%.4s', using %llu\n", alloc, width, height,
(const char *) &format, needed);
}
if (needed > MINISURF_MAX_ALLOC) {
s->release();
return NULL;
}
alloc = (uint32_t) needed;
}
if (alloc == 0 || alloc > MINISURF_MAX_ALLOC) {
s->release();
return NULL;
}
if (wrapped == NULL) {
alloc = alignUp(alloc, (uint32_t) PAGE_SIZE);
} else {
/* (the memory is as big as its owner says, and not a byte of ours is added to it) */
alloc = dictNumber(props, "IOSurfaceAllocSize", (uint32_t) needed);
/* the owner's number replaced the checked one above: it gets the same limits, so that a size of 0 or of gigabytes is not taken */
if (alloc == 0 || alloc > MINISURF_MAX_ALLOC) {
s->release();
return NULL;
}
}
s->memory = surfaceMemory((vm_size_t) PAGE_SIZE + (wrapped ? 0 : alloc));
if (s->memory == NULL) {
IOLog("MiniIOSurface: no memory for %u bytes (%u surfaces, %llu MB in use)\n", alloc, gSurfaceCount,
gSurfaceBytes >> 20);
s->release();
*error = kIOReturnNoMemory;
return NULL;
}
s->kbase = (uint8_t *) s->memory->getBytesNoCopy();
s->values = OSDictionary::withCapacity(4);
s->kpixels = s->kbase + PAGE_SIZE;
if (wrapped != NULL) {
s->clientTask = current_task();
s->clientAddress = wrapped->unsigned64BitValue();
IOReturn held = kIOReturnNoMemory;
/* (held for writing if it can be written, which a picture that is only shown need not be) */
for (unsigned attempt = 0; attempt < 2 && held != kIOReturnSuccess; attempt++) {
OSSafeReleaseNULL(s->client);
s->client = IOMemoryDescriptor::withAddressRange(s->clientAddress, alloc,
attempt == 0 ? kIODirectionInOut : kIODirectionOut, s->clientTask);
held = s->client != NULL ? s->client->prepare() : kIOReturnNoMemory;
}
if (held == kIOReturnSuccess) {
s->clientMap = s->client->createMappingInTask(kernel_task, 0, kIOMapAnywhere | kIOMapReadOnly);
} else {
OSSafeReleaseNULL(s->client);
}
if (s->clientMap == NULL) {
if (gSurfaceUnheld++ < 8) {
IOLog("MiniIOSurface: the memory to make a surface around (%u bytes at %#llx) cannot be %s (%#x)\n", alloc,
s->clientAddress, held == kIOReturnSuccess ? "seen from here" : "held", held);
}
s->release();
*error = kIOReturnVMError;
return NULL;
}
s->kpixels = (uint8_t *) s->clientMap->getVirtualAddress();
}
info->allocSize = alloc;
info->width = width;
info->height = height;
info->bytesPerRow = bpr;
info->offset = offset;
info->pixelFormat = format;
info->planeCount = planeCount;
info->bytesPerElement = (uint16_t) bpe;
info->elementWidth = (uint8_t) ew;
info->elementHeight = (uint8_t) eh;
info->cacheMode = dictNumber(props, "IOSurfaceCacheMode", 0);
info->subsampling = (uint8_t) dictNumber(props, "IOSurfaceSubsampling", 0);
info->prohibitUseCount = (uint8_t) dictNumber(props, "IOSurfaceProhibitUseCount", 0);
info->sliceCount = dictNumber(props, "IOSurfaceSliceCount", 0);
info->protectionOptions = dictNumber(props, "IOSurfaceProtectionOptions", 0);
IOLockLock(gSurfaceLock);
uint32_t sid = 0;
for (uint32_t tries = 0; tries < MINISURF_TABLE_SIZE - 1; tries++) {
uint32_t candidate = gSurfaceNext;
gSurfaceNext = (gSurfaceNext + 1 < MINISURF_TABLE_SIZE) ? gSurfaceNext + 1 : 1;
if (gSurfaceTable[candidate] == NULL) {
sid = candidate;
break;
}
}
if (sid == 0) {
IOLockUnlock(gSurfaceLock);
IOLog("MiniIOSurface: the surface table is full\n");
s->release();
*error = kIOReturnNoResources;
return NULL;
}
s->sid = sid;
s->holders = 1;
info->sid = sid;
info->traceID = sid;
gSurfaceTable[sid] = s; /* the table owns the object until the last holder goes */
gSurfaceCount++;
gSurfaceBytes += alloc;
struct minisurf_shared * sh = s->shared();
sh->registryID = 0x100000000ULL + sid;
sh->seed = 1;
sh->valuesSeed = 1;
sh->pixelSizeCasting = (uint8_t) dictNumber(props, "IOSurfacePixelSizeCastingAllowed", 1);
sh->displayable = 1;
sh->sysMemOnly = (uint8_t) dictNumber(props, "IOSurfaceSysMemOnly", 0);
IOLockUnlock(gSurfaceLock);
*error = kIOReturnSuccess;
return s;
}
/* With the lock held. */
static void
surfaceHold(MiniIOSurface * s)
{
s->holders++;
}
/* With the lock held. The caller must not touch the surface afterwards. */
static void
surfaceUnhold(MiniIOSurface * s)
{
if (--s->holders != 0) {
return;
}
gSurfaceTable[s->sid] = NULL;
gSurfaceCount--;
gSurfaceBytes -= s->info.allocSize;
s->release();
}
MiniIOSurface *
MiniIOSurfaceCopy(uint32_t sid)
{
MiniIOSurface * s = NULL;
if (gSurfaceLock == NULL || sid == 0 || sid >= MINISURF_TABLE_SIZE) {
return NULL;
}
IOLockLock(gSurfaceLock);
s = gSurfaceTable[sid];
if (s != NULL) {
surfaceHold(s);
}
IOLockUnlock(gSurfaceLock);
return s;
}
/* How many surfaces there are and how much (wired) memory they hold: for the kernel's memory report. */
extern "C" void
MiniIOSurfaceUsage(uint32_t * count, uint64_t * bytes)
{
*count = gSurfaceCount;
*bytes = gSurfaceBytes;
}
void
MiniIOSurfaceDrop(MiniIOSurface * s)
{
if (s == NULL) {
return;
}
IOLockLock(gSurfaceLock);
surfaceUnhold(s);
IOLockUnlock(gSurfaceLock);
}
MiniIOSurface *
MiniIOSurfaceCreateSimple(uint32_t width, uint32_t height, uint32_t pixelFormat)
{
OSDictionary * props = OSDictionary::withCapacity(4);
MiniIOSurface * s = NULL;
IOReturn error;
if (props == NULL || gSurfaceLock == NULL) {
OSSafeReleaseNULL(props);
return NULL;
}
OSNumber * n;
n = OSNumber::withNumber(width, 32); props->setObject("IOSurfaceWidth", n); n->release();
n = OSNumber::withNumber(height, 32); props->setObject("IOSurfaceHeight", n); n->release();
n = OSNumber::withNumber(pixelFormat, 32); props->setObject("IOSurfacePixelFormat", n); n->release();
s = surfaceCreate(props, &error);
props->release();
return s;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* What a Mach send right for a surface points at. Holds the surface while any right exists. */
class IOSurfaceSendRight : public OSObject
{
OSDeclareDefaultStructors(IOSurfaceSendRight);
public:
MiniIOSurface * surface;
virtual bool
init(void) APPLE_KEXT_OVERRIDE
{
return OSObject::init();
}
virtual void free(void) APPLE_KEXT_OVERRIDE;
};
OSDefineMetaClassAndStructors(IOSurfaceSendRight, OSObject);
void
IOSurfaceSendRight::free(void)
{
MiniIOSurfaceDrop(surface);
surface = NULL;
OSObject::free();
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
class IOCoreSurfaceRoot : public IOService
{
OSDeclareDefaultStructors(IOCoreSurfaceRoot);
public:
virtual bool start(IOService * provider) APPLE_KEXT_OVERRIDE;
};
OSDefineMetaClassAndStructors(IOCoreSurfaceRoot, IOService);
bool
IOCoreSurfaceRoot::start(IOService * provider)
{
if (!IOService::start(provider)) {
return false;
}
setProperty(kIOUserClientClassKey, "IOSurfaceRootUserClient");
setName("IOCoreSurfaceRoot");
registerService();
return true;
}
void
MiniIOSurfaceStart(IOService * provider)
{
IOCoreSurfaceRoot * root;
gSurfaceLock = IOLockAlloc();
gSurfaceKeptLock = IOLockAlloc();
if (!PE_parse_boot_arg_str("surftrace_of", gSurfaceTraceOf, sizeof(gSurfaceTraceOf))) {
gSurfaceTraceOf[0] = 0;
}
gSurfaceClearing = thread_call_allocate(surfaceMemoryClear, NULL);
root = OSTypeAlloc(IOCoreSurfaceRoot);
if (root == NULL || !root->init() || !root->attach(provider)) {
OSSafeReleaseNULL(root);
IOLog("MiniIOSurface: cannot create the root service\n");
return;
}
if (!root->start(provider)) {
root->detach(provider);
IOLog("MiniIOSurface: the root service did not start\n");
}
root->release();
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/* One surface as one client has it. */
struct minisurf_client_ref {
MiniIOSurface * surface; /* held */
IOMemoryMap * map; /* the whole of it in the client's address space */
IOMemoryMap * pixelMap; /* ... the pixels, where those are another client's memory */
uint32_t use[2]; /* what this client added to the use counts */
bool detached; /* the client let go of it and will release shortly */
};
class IOSurfaceRootUserClient : public IOUserClient
{
OSDeclareDefaultStructors(IOSurfaceRootUserClient);
task_t fTask;
struct minisurf_client_ref * fRefs;
uint32_t fRefCount;
uint32_t fRefCapacity;
IOExternalTrap fTraps[kSurfaceTrapCount];
struct minisurf_client_ref * findRef(uint32_t sid, bool detached);
struct minisurf_client_ref * addRef(MiniIOSurface * surface);
void dropRef(struct minisurf_client_ref * ref);
IOReturn returnSurface(struct minisurf_client_ref * ref, bool known, IOExternalMethodArguments * args);
IOReturn methodInit(IOExternalMethodArguments * args);
IOReturn methodCreate(IOExternalMethodArguments * args);
IOReturn methodWrapClientImage(IOExternalMethodArguments * args);
IOReturn methodLookup(IOExternalMethodArguments * args);
IOReturn methodLookupFromPort(IOExternalMethodArguments * args);
IOReturn methodCreatePort(IOExternalMethodArguments * args);
IOReturn methodSetValue(IOExternalMethodArguments * args, const uint8_t * in, size_t inSize);
IOReturn methodCopyValue(IOExternalMethodArguments * args, const uint8_t * in, size_t inSize);
IOReturn methodRemoveValue(IOExternalMethodArguments * args, const uint8_t * in, size_t inSize);
IOReturn methodSetBulkAttachments(const uint8_t * in, size_t inSize);
IOReturn trapIncrementUseCount(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
IOReturn trapDecrementUseCount(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
IOReturn trapLock(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
IOReturn trapUnlock(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
IOReturn trapDetach(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
IOReturn trapRelease(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6);
IOReturn lockCommon(uint32_t sid, uint32_t options, user_addr_t seedOut);
IOReturn useCountCommon(uint32_t sid, uint32_t category, int delta);
public:
virtual bool initWithTask(task_t owningTask, void * securityID, UInt32 type, OSDictionary * properties) APPLE_KEXT_OVERRIDE;
virtual bool start(IOService * provider) APPLE_KEXT_OVERRIDE;
virtual IOReturn clientClose(void) APPLE_KEXT_OVERRIDE;
virtual void free(void) APPLE_KEXT_OVERRIDE;
virtual IOReturn externalMethod(uint32_t selector, IOExternalMethodArguments * args,
IOExternalMethodDispatch * dispatch, OSObject * target, void * reference) APPLE_KEXT_OVERRIDE;
virtual IOExternalTrap * getTargetAndTrapForIndex(IOService ** targetP, UInt32 index) APPLE_KEXT_OVERRIDE;
static mach_port_name_t makeSendRight(task_t task, MiniIOSurface * surface);
};
OSDefineMetaClassAndStructors(IOSurfaceRootUserClient, IOUserClient);
bool
IOSurfaceRootUserClient::initWithTask(task_t owningTask, void * securityID, UInt32 type, OSDictionary * properties)
{
if (!IOUserClient::initWithTask(owningTask, securityID, type, properties)) {
return false;
}
fTask = owningTask;
return true;
}
bool
IOSurfaceRootUserClient::start(IOService * provider)
{
if (!IOUserClient::start(provider)) {
return false;
}
fTraps[kSurfaceTrapIncrementUseCount].func = (IOTrap) & IOSurfaceRootUserClient::trapIncrementUseCount;
fTraps[kSurfaceTrapDecrementUseCount].func = (IOTrap) & IOSurfaceRootUserClient::trapDecrementUseCount;
fTraps[kSurfaceTrapLock].func = (IOTrap) & IOSurfaceRootUserClient::trapLock;
fTraps[kSurfaceTrapUnlock].func = (IOTrap) & IOSurfaceRootUserClient::trapUnlock;
fTraps[kSurfaceTrapDetach].func = (IOTrap) & IOSurfaceRootUserClient::trapDetach;
fTraps[kSurfaceTrapRelease].func = (IOTrap) & IOSurfaceRootUserClient::trapRelease;
for (unsigned i = 0; i < kSurfaceTrapCount; i++) {
fTraps[i].object = this;
}
return true;
}
IOReturn
IOSurfaceRootUserClient::clientClose(void)
{
IOLockLock(gSurfaceLock);
while (fRefCount != 0) {
dropRef(&fRefs[fRefCount - 1]);
}
IOLockUnlock(gSurfaceLock);
if (!isInactive()) {
terminate();
}
return kIOReturnSuccess;
}
void
IOSurfaceRootUserClient::free(void)
{
if (fRefs != NULL) {
if (gSurfaceLock != NULL) {
IOLockLock(gSurfaceLock);
while (fRefCount != 0) {
dropRef(&fRefs[fRefCount - 1]);
}
IOLockUnlock(gSurfaceLock);
}
IODelete(fRefs, struct minisurf_client_ref, fRefCapacity);
fRefs = NULL;
}
IOUserClient::free();
}
/* All three with the lock held. */
struct minisurf_client_ref *
IOSurfaceRootUserClient::findRef(uint32_t sid, bool detached)
{
for (uint32_t i = 0; i < fRefCount; i++) {
if (fRefs[i].surface->sid == sid && fRefs[i].detached == detached) {
return &fRefs[i];
}
}
return NULL;
}
struct minisurf_client_ref *
IOSurfaceRootUserClient::addRef(MiniIOSurface * surface)
{
struct minisurf_client_ref * ref;
IOMemoryMap * map;
if (fRefCount == fRefCapacity) {
uint32_t capacity = fRefCapacity ? fRefCapacity * 2 : 16;
struct minisurf_client_ref * refs = IONew(struct minisurf_client_ref, capacity);
if (refs == NULL) {
return NULL;
}
if (fRefs != NULL) {
memcpy(refs, fRefs, fRefCount * sizeof(*refs));
IODelete(fRefs, struct minisurf_client_ref, fRefCapacity);
}
fRefs = refs;
fRefCapacity = capacity;
}
map = surface->memory->createMappingInTask(fTask, 0, kIOMapAnywhere);
if (map == NULL) {
IOLog("MiniIOSurface: cannot map surface %u (%u bytes) into the client\n", surface->sid, surface->info.allocSize);
return NULL;
}
ref = &fRefs[fRefCount++];
bzero(ref, sizeof(*ref));
ref->surface = surface;
ref->map = map;
if (surface->client != NULL && surface->clientTask != fTask) {
ref->pixelMap = surface->client->createMappingInTask(fTask, 0, kIOMapAnywhere);
}
surfaceHold(surface);
return ref;
}
void
IOSurfaceRootUserClient::dropRef(struct minisurf_client_ref * ref)
{
MiniIOSurface * s = ref->surface;
for (unsigned c = 0; c < 2; c++) {
s->useCount[c] -= ref->use[c];
}
s->shared()->useCount = (uint64_t) s->useCount[1] << 32 | s->useCount[0];
OSSafeReleaseNULL(ref->map);
OSSafeReleaseNULL(ref->pixelMap);
*ref = fRefs[--fRefCount];
surfaceUnhold(s);
}
static uint32_t
resultSize(const MiniIOSurface * s)
{
uint32_t planes = s->info.planeCount ? s->info.planeCount : 1;
return (uint32_t) (sizeof(struct minisurf_result) + planes * sizeof(struct minisurf_plane));
}
/* Describe the surface to the client. known: it has the surface already, do not build a second view. */
IOReturn
IOSurfaceRootUserClient::returnSurface(struct minisurf_client_ref * ref, bool known, IOExternalMethodArguments * args)
{
MiniIOSurface * s = ref->surface;
uint32_t size = resultSize(s);
struct minisurf_result * r = (struct minisurf_result *) args->structureOutput;
mach_vm_address_t address = ref->map->getAddress();
if (r == NULL || args->structureOutputSize < size) {
return kIOReturnBadArgument;
}
*r = s->info;
memcpy(r + 1, s->planes, size - sizeof(*r));
r->shared = address;
r->aux = address + MINISURF_AUX_OFFSET;
r->base = known ? ~0ULL : s->client == NULL ? address + PAGE_SIZE :
ref->pixelMap != NULL ? ref->pixelMap->getAddress() : s->clientAddress;
args->structureOutputSize = size;
return kIOReturnSuccess;
}
IOReturn
IOSurfaceRootUserClient::methodInit(IOExternalMethodArguments * args)
{
struct {
uint32_t allocAlignMask;
uint32_t rowAlignMask;
uint32_t maxBytesPerRow;
uint32_t maxWidth;
uint32_t maxHeight;
uint16_t reserved;
uint16_t version;
} reply;
static_assert(sizeof(reply) == 0x18, "init reply");
if (args->structureOutput == NULL || args->structureOutputSize < sizeof(reply)) {
return kIOReturnBadArgument;
}
reply.allocAlignMask = (uint32_t) PAGE_MASK;
reply.rowAlignMask = MINISURF_ROW_ALIGN - 1;
reply.maxBytesPerRow = MINISURF_MAX_DIMENSION * 16;
reply.maxWidth = MINISURF_MAX_DIMENSION;
reply.maxHeight = MINISURF_MAX_DIMENSION;
reply.reserved = 0;
reply.version = 1;
memcpy(args->structureOutput, &reply, sizeof(reply));
args->structureOutputSize = sizeof(reply);
return kIOReturnSuccess;
}
/* The serialized object in a request, or NULL. */
static OSObject *
unserialize(const uint8_t * data, size_t size)
{
if (data == NULL || size < 4) {
return NULL;
}
return OSUnserializeXML((const char *) data, size, NULL);
}
IOReturn
IOSurfaceRootUserClient::methodCreate(IOExternalMethodArguments * args)
{
IOReturn ret;
IOMemoryMap * inMap = NULL;
const uint8_t * in = (const uint8_t *) args->structureInput;
size_t inSize = args->structureInputSize;
OSObject * obj;
OSDictionary * props;
MiniIOSurface * s;
struct minisurf_client_ref * ref;
uint64_t began = mach_absolute_time();
if (args->scalarInputCount >= 1 && args->scalarInput[0] != 0) {
IOLog("MiniIOSurface: child surfaces are not supported (parent %llu)\n", args->scalarInput[0]);
return kIOReturnUnsupported;
}
if (args->structureInputDescriptor != NULL) {
inMap = args->structureInputDescriptor->map(kIOMapReadOnly);
if (inMap == NULL) {
return kIOReturnNoMemory;
}
in = (const uint8_t *) inMap->getVirtualAddress();
inSize = inMap->getLength();
}
obj = unserialize(in, inSize);
OSSafeReleaseNULL(inMap);
props = OSDynamicCast(OSDictionary, obj);
if (props == NULL) {
OSSafeReleaseNULL(obj);
return kIOReturnBadArgument;
}
uint64_t made_at = mach_absolute_time();
s = surfaceCreate(props, &ret);
made_at = mach_absolute_time() - made_at;
if (s == NULL) {
IOLog("MiniIOSurface: create failed 0x%x: %ux%u format 0x%x alloc %u\n", ret,
dictNumber(props, "IOSurfaceWidth", 0), dictNumber(props, "IOSurfaceHeight", 0),
dictNumber(props, "IOSurfacePixelFormat", 0), dictNumber(props, "IOSurfaceAllocSize", 0));
props->release();
return ret;
}
{
char procname[32] = "?";
static unsigned keysSaid;
proc_selfname(procname, sizeof(procname));
/* what a traced program asks for, key by key (the first few surfaces) */
if (gSurfaceTraceOf[0] != 0 && strncmp(procname, gSurfaceTraceOf, strlen(gSurfaceTraceOf)) == 0 && keysSaid++ < 6) {
OSCollectionIterator * keys = OSCollectionIterator::withCollection(props);
char line[256];
size_t used = 0;
OSSymbol * key;
line[0] = 0;
while (keys != NULL && (key = OSDynamicCast(OSSymbol, keys->getNextObject())) != NULL && used < sizeof(line)) {
OSNumber * n = OSDynamicCast(OSNumber, props->getObject(key));
used += (size_t) snprintf(line + used, sizeof(line) - used, " %s=%lld", key->getCStringNoCopy(),
n ? (long long) n->unsigned64BitValue() : -1LL);
}
OSSafeReleaseNULL(keys);
traceLine("MiniIOSurface: %s asks for a surface with%s\n", procname, line);
}
}
props->release();
{
char procname[32] = "?";
proc_selfname(procname, sizeof(procname));
if (gSurfaceTraceOf[0] != 0 && strncmp(procname, gSurfaceTraceOf, strlen(gSurfaceTraceOf)) != 0) {
procname[0] = 0; /* (not one of those asked about: not counted) */
}
if (procname[0] != 0 && gSurfaceTraced < gSurfaceTrace) {
s->traced = gSurfaceTraceOf[0] != 0;
}
}
if (s->traced || (gSurfaceTraceOf[0] == 0 && gSurfaceTraced < gSurfaceTrace)) {
char procname[32] = "?";
gSurfaceTraced++;
proc_selfname(procname, sizeof(procname));
traceLine("MiniIOSurface: %s[%d] created surface %u: %ux%u '%c%c%c%c', %u bytes per row, %u planes, %u bytes\n",
procname, proc_selfpid(), s->sid, s->info.width, s->info.height, (int) (s->info.pixelFormat >> 24 & 0xff),
(int) (s->info.pixelFormat >> 16 & 0xff), (int) (s->info.pixelFormat >> 8 & 0xff),
(int) (s->info.pixelFormat & 0xff), s->info.bytesPerRow, s->info.planeCount, s->info.allocSize);
}
uint64_t mapped_at = mach_absolute_time();
/* the surface can be gone once the creator's hold is dropped (when the reply did not fit): its size is read now, not after */
uint32_t createdBytes = s->info.allocSize;
IOLockLock(gSurfaceLock);
ref = addRef(s);
ret = ref ? returnSurface(ref, false, args) : kIOReturnNoMemory;
if (ref != NULL && ret != kIOReturnSuccess) {
dropRef(ref);
}
surfaceUnhold(s); /* the creator's hold; the client's own is in the ref */
IOLockUnlock(gSurfaceLock);
mapped_at = mach_absolute_time() - mapped_at;
OSAddAtomic64((SInt64) (mach_absolute_time() - began), (SInt64 *) &gSurfaceMaking);
OSIncrementAtomic(&gSurfacesMade);
if (createdBytes >= SURFACE_KEEP_FROM) {
/* the big ones (the window server's of the whole display): where the time to make one goes, every 5 s */
static uint64_t since, all, making, mapping;
static uint32_t count;
uint64_t now = mach_absolute_time(), ns;
count++;
all += now - began;
making += made_at;
mapping += mapped_at;
absolutetime_to_nanoseconds(now - since, &ns);
if (since == 0) {
since = now;
} else if (ns >= 5 * NSEC_PER_SEC) {
uint64_t a, m, p;
absolutetime_to_nanoseconds(all, &a);
absolutetime_to_nanoseconds(making, &m);
absolutetime_to_nanoseconds(mapping, &p);
IOLog("MiniIOSurface: %u big surfaces made, %llu us each: %llu getting the memory and the surface, %llu mapping it "
"for its maker; memory not kept as somebody else had it %u times, a piece kept longer put out for it %u times\n",
count, a / 1000 / count, m / 1000 / count, p / 1000 / count, OSBitAndAtomic(0, &gSurfacesNotKept[0]),
OSBitAndAtomic(0, &gSurfacesNotKept[1]));
since = now;
count = 0;
all = making = mapping = 0;
}
}
return ret;
}
/*
* A surface around a picture of the client's own (IOSurfaceWrapClientImage): what CoreAnimation's OpenGL ES renderer asks
* for each image it is to draw from -- a picture, a glyph mask, a gradient. While it was refused the window server asked
* about 4000 times a second, drew those images another way every frame, and 9% of its drawing thread's time went to the
* refusals alone (eagl.c's glprof). The request is 32 bytes (as seen on the phone, see externalMethod's log of it):
*
* address (8), width, height, pixel format, bytes per element, bytes per row, size (4 each)
*
* with the size the owner's (a mask's last row only as long as it is wide). Made as a create with IOSurfaceAddress is:
* the surface's pixels are that memory itself, and the reply is a create's.
*/
IOReturn
IOSurfaceRootUserClient::methodWrapClientImage(IOExternalMethodArguments * args)
{
struct {
uint64_t address;
uint32_t width, height, format, bytesPerElement, bytesPerRow, size;
} request;
static const char * const keys[] = { "IOSurfaceWidth", "IOSurfaceHeight", "IOSurfacePixelFormat", "IOSurfaceBytesPerElement",
"IOSurfaceBytesPerRow", "IOSurfaceAllocSize" };
struct minisurf_client_ref * ref;
OSDictionary * props;
MiniIOSurface * s;
IOReturn ret = kIOReturnSuccess;
uint64_t began = mach_absolute_time();
if (args->structureInput == NULL || args->structureInputSize < sizeof(request)) {
return kIOReturnBadArgument;
}
memcpy(&request, args->structureInput, sizeof(request));
if (request.address == 0 || request.size == 0) {
return kIOReturnBadArgument;
}
props = OSDictionary::withCapacity(8);
OSNumber * address = OSNumber::withNumber(request.address, 64);
if (props == NULL || address == NULL) {
OSSafeReleaseNULL(address);
OSSafeReleaseNULL(props);
return kIOReturnNoMemory;
}
props->setObject("IOSurfaceAddress", address);
address->release();
const uint32_t values[] = { request.width, request.height, request.format, request.bytesPerElement, request.bytesPerRow,
request.size };
for (unsigned i = 0; i < sizeof(keys) / sizeof(keys[0]) && ret == kIOReturnSuccess; i++) {
OSNumber * n = OSNumber::withNumber(values[i], 32);
ret = n != NULL && props->setObject(keys[i], n) ? kIOReturnSuccess : kIOReturnNoMemory;
OSSafeReleaseNULL(n);
}
s = ret == kIOReturnSuccess ? surfaceCreate(props, &ret) : NULL;
props->release();
if (s == NULL) {
return ret;
}
IOLockLock(gSurfaceLock);
ref = addRef(s);
ret = ref ? returnSurface(ref, false, args) : kIOReturnNoMemory;
if (ref != NULL && ret != kIOReturnSuccess) {
dropRef(ref);
}
surfaceUnhold(s); /* the maker's hold; the client's own is in the ref */
IOLockUnlock(gSurfaceLock);
OSAddAtomic64((SInt64) (mach_absolute_time() - began), (SInt64 *) &gSurfaceMaking);
OSIncrementAtomic(&gSurfacesMade);
return ret;
}
IOReturn
IOSurfaceRootUserClient::methodLookup(IOExternalMethodArguments * args)
{
IOReturn ret = kIOReturnNotFound;
uint32_t sid;
struct minisurf_client_ref * ref;
if (args->scalarInputCount < 1) {
return kIOReturnBadArgument;
}
sid = (uint32_t) args->scalarInput[0];
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref == NULL && sid != 0 && sid < MINISURF_TABLE_SIZE && gSurfaceTable[sid] != NULL) {
ref = addRef(gSurfaceTable[sid]);
if (ref == NULL) {
ret = kIOReturnNoMemory;
}
}
if (ref != NULL) {
ret = returnSurface(ref, false, args);
}
IOLockUnlock(gSurfaceLock);
return ret;
}
IOReturn
IOSurfaceRootUserClient::methodLookupFromPort(IOExternalMethodArguments * args)
{
IOReturn ret;
OSObject * obj = NULL;
IOSurfaceSendRight * right;
struct minisurf_client_ref * ref;
bool known;
if (args->scalarInputCount < 1) {
return kIOReturnBadArgument;
}
ret = copyObjectForPortNameInTask(fTask, (mach_port_name_t) args->scalarInput[0], &obj);
right = OSDynamicCast(IOSurfaceSendRight, obj);
if (ret != kIOReturnSuccess || right == NULL || right->surface == NULL) {
static unsigned said;
if (said++ < 16) {
char procname[32] = "?";
proc_selfname(procname, sizeof(procname));
traceLine("MiniIOSurface: %s[%d] looked up a surface from port %#llx: %s\n", procname, proc_selfpid(),
args->scalarInput[0], ret != kIOReturnSuccess ? "no object behind the port" : obj ? "not a surface's port" :
"no surface");
}
OSSafeReleaseNULL(obj);
return kIOReturnBadArgument;
}
IOLockLock(gSurfaceLock);
ref = findRef(right->surface->sid, false);
known = ref != NULL;
if (ref == NULL) {
ref = addRef(right->surface);
}
ret = ref ? returnSurface(ref, known, args) : kIOReturnNoMemory;
IOLockUnlock(gSurfaceLock);
if (right->surface->traced) {
char procname[32] = "?";
uint32_t format = right->surface->info.pixelFormat, nonzero = 0, first = 0, middle = 0;
const uint8_t * pixels = right->surface->kpixels;
uint32_t size = right->surface->info.allocSize;
/* what is in it by now: how many of its bytes are not zero, its first pixel and one in the middle */
for (uint32_t at = 0; pixels != NULL && at < size; at += 64) {
nonzero += pixels[at] != 0 || pixels[at + 1] != 0 || pixels[at + 2] != 0 || pixels[at + 3] != 0;
}
if (pixels != NULL && size >= 8) {
memcpy(&first, pixels, 4);
memcpy(&middle, pixels + (size / 2 & ~3u), 4);
}
proc_selfname(procname, sizeof(procname));
traceLine("MiniIOSurface: %s[%d] looked up surface %u (%ux%u '%c%c%c%c') from a port: %#x; of every 64 bytes %u of %u "
"begin with a pixel that is not 0, the first %#010x, one in the middle %#010x\n", procname, proc_selfpid(),
right->surface->sid, right->surface->info.width, right->surface->info.height, (int) (format >> 24 & 0xff),
(int) (format >> 16 & 0xff), (int) (format >> 8 & 0xff), (int) (format & 0xff), ret, nonzero, size / 64, first, middle);
}
obj->release();
return ret;
}
mach_port_name_t
IOSurfaceRootUserClient::makeSendRight(task_t task, MiniIOSurface * s)
{
IOSurfaceSendRight * right;
mach_port_name_t name = 0;
right = OSTypeAlloc(IOSurfaceSendRight);
if (right == NULL || !right->init()) {
OSSafeReleaseNULL(right);
return 0;
}
IOLockLock(gSurfaceLock);
surfaceHold(s); /* for as long as the send right exists */
IOLockUnlock(gSurfaceLock);
right->surface = s;
copyPortNameForObjectInTask(task, right, &name);
right->release();
return name;
}
mach_port_name_t
MiniIOSurfaceMakeSendRight(task_t task, MiniIOSurface * surface)
{
return IOSurfaceRootUserClient::makeSendRight(task, surface);
}
IOReturn
IOSurfaceRootUserClient::methodCreatePort(IOExternalMethodArguments * args)
{
MiniIOSurface * s;
mach_port_name_t name;
if (args->scalarInputCount < 1 || args->scalarOutputCount < 1) {
return kIOReturnBadArgument;
}
s = MiniIOSurfaceCopy((uint32_t) args->scalarInput[0]);
if (s == NULL) {
return kIOReturnNotFound;
}
name = makeSendRight(fTask, s);
if (s->traced) {
char procname[32] = "?";
proc_selfname(procname, sizeof(procname));
traceLine("MiniIOSurface: %s[%d] made a port for surface %u: %#x\n", procname, proc_selfpid(), s->sid, name);
}
MiniIOSurfaceDrop(s);
args->scalarOutput[0] = name;
args->scalarOutputCount = 1;
return name != 0 ? kIOReturnSuccess : kIOReturnNoMemory;
}
/* Requests about values start with the surface number in 8 bytes; the reply to a change is the new seed. */
static IOReturn
returnSeed(IOExternalMethodArguments * args, uint32_t seed)
{
if (args->structureOutput != NULL && args->structureOutputSize >= sizeof(seed)) {
memcpy(args->structureOutput, &seed, sizeof(seed));
args->structureOutputSize = sizeof(seed);
}
return kIOReturnSuccess;
}
IOReturn
IOSurfaceRootUserClient::methodSetValue(IOExternalMethodArguments * args, const uint8_t * in, size_t inSize)
{
uint32_t sid, seed = 0;
OSObject * obj;
OSArray * array;
struct minisurf_client_ref * ref;
IOReturn ret = kIOReturnBadArgument;
if (inSize < 12) {
return kIOReturnBadArgument;
}
memcpy(&sid, in, sizeof(sid));
obj = unserialize(in + 8, inSize - 8);
array = OSDynamicCast(OSArray, obj);
if (array == NULL) {
OSSafeReleaseNULL(obj);
return kIOReturnBadArgument;
}
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref == NULL) {
ret = kIOReturnNotFound;
} else if (ref->surface->values != NULL) {
MiniIOSurface * s = ref->surface;
OSString * key = OSDynamicCast(OSString, array->getObject(1));
OSDictionary * all = OSDynamicCast(OSDictionary, array->getObject(0));
if (array->getCount() == 2 && key != NULL) {
s->values->setObject(key, array->getObject(0)); /* [value, key] */
ret = kIOReturnSuccess;
} else if (array->getCount() == 1 && all != NULL) {
s->values->merge(all); /* [all the values] */
ret = kIOReturnSuccess;
}
if (ret == kIOReturnSuccess) {
s->updateValuesState();
seed = s->shared()->valuesSeed;
}
}
IOLockUnlock(gSurfaceLock);
obj->release();
return ret == kIOReturnSuccess ? returnSeed(args, seed) : ret;
}
IOReturn
IOSurfaceRootUserClient::methodRemoveValue(IOExternalMethodArguments * args, const uint8_t * in, size_t inSize)
{
uint32_t sid, seed = 0;
struct minisurf_client_ref * ref;
IOReturn ret = kIOReturnNotFound;
if (inSize < 9 || in[inSize - 1] != 0) {
return kIOReturnBadArgument;
}
memcpy(&sid, in, sizeof(sid));
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref != NULL && ref->surface->values != NULL) {
MiniIOSurface * s = ref->surface;
const char * key = (const char *) in + 8;
if (key[0] == 0) {
s->values->flushCollection();
} else {
s->values->removeObject(key);
}
s->updateValuesState();
seed = s->shared()->valuesSeed;
ret = kIOReturnSuccess;
}
IOLockUnlock(gSurfaceLock);
return ret == kIOReturnSuccess ? returnSeed(args, seed) : ret;
}
/* The reply: 4 bytes unused, the seed, then the serialized value (a dictionary when all were asked for). */
IOReturn
IOSurfaceRootUserClient::methodCopyValue(IOExternalMethodArguments * args, const uint8_t * in, size_t inSize)
{
uint32_t sid, header[2] = { 0, 0 };
struct minisurf_client_ref * ref;
OSSerialize * s = NULL;
IOReturn ret = kIOReturnNotFound;
uint32_t length = 0, total;
if (inSize < 9 || in[inSize - 1] != 0) {
return kIOReturnBadArgument;
}
memcpy(&sid, in, sizeof(sid));
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref != NULL && ref->surface->values != NULL) {
const char * key = (const char *) in + 8;
OSObject * value = key[0] ? ref->surface->values->getObject(key) : ref->surface->values;
header[1] = ref->surface->shared()->valuesSeed;
ret = kIOReturnSuccess;
if (value != NULL && (key[0] != 0 || ref->surface->values->getCount() != 0)) {
s = OSSerialize::binaryWithCapacity(4096);
if (s == NULL || !value->serialize(s)) {
ret = kIOReturnNoMemory;
} else {
length = s->getLength();
}
}
}
IOLockUnlock(gSurfaceLock);
total = (uint32_t) sizeof(header) + length;
if (ret == kIOReturnSuccess) {
if (args->structureOutputDescriptor != NULL) {
IOMemoryDescriptor * out = args->structureOutputDescriptor;
if (out->getLength() < total) {
ret = kIOReturnNoSpace;
} else if ((ret = out->prepare()) == kIOReturnSuccess) {
out->writeBytes(0, header, sizeof(header));
if (length != 0) {
out->writeBytes(sizeof(header), s->text(), length);
}
out->complete();
args->structureOutputDescriptorSize = total;
}
} else if (args->structureOutput == NULL || args->structureOutputSize < total) {
ret = kIOReturnNoSpace;
} else {
memcpy(args->structureOutput, header, sizeof(header));
if (length != 0) {
memcpy((uint8_t *) args->structureOutput + sizeof(header), s->text(), length);
}
args->structureOutputSize = total;
}
}
OSSafeReleaseNULL(s);
return ret;
}
/*
* Colour space, transfer function and the like: a fixed structure kept in the shared page.
* The request: the structure, its size, a mask of the fields that changed, the surface number.
*/
IOReturn
IOSurfaceRootUserClient::methodSetBulkAttachments(const uint8_t * in, size_t inSize)
{
static const struct {
uint8_t from, to;
} fields[15] = {
{ 0x00, 0x20 }, { 0x20, 0x30 }, { 0x30, 0x38 }, { 0x38, 0x39 }, { 0x39, 0x3a }, { 0x3a, 0x3b }, { 0x3b, 0x3c },
{ 0x3c, 0x3d }, { 0x3d, 0x3e }, { 0x3e, 0x3f }, { 0x3f, 0x40 }, { 0x40, 0x58 }, { 0x58, 0x5c }, { 0x5c, 0x64 },
{ 0x64, 0x65 },
};
uint64_t size, mask;
uint32_t sid;
struct minisurf_client_ref * ref;
IOReturn ret = kIOReturnNotFound;
if (inSize < 0x88) {
return kIOReturnBadArgument;
}
memcpy(&size, in + 0x70, sizeof(size));
memcpy(&mask, in + 0x78, sizeof(mask));
memcpy(&sid, in + 0x80, sizeof(sid));
if (size > 0x70) {
return kIOReturnBadArgument;
}
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref != NULL) {
uint8_t * attachments = ref->surface->shared()->attachments;
for (unsigned i = 0; i < 15; i++) {
if ((mask & (1ULL << i)) && fields[i].to <= size) {
memcpy(attachments + fields[i].from, in + fields[i].from, fields[i].to - fields[i].from);
}
}
ret = kIOReturnSuccess;
}
IOLockUnlock(gSurfaceLock);
return ret;
}
IOReturn
IOSurfaceRootUserClient::externalMethod(uint32_t selector, IOExternalMethodArguments * args,
IOExternalMethodDispatch * dispatch, OSObject * target, void * reference)
{
const uint8_t * in = (const uint8_t *) args->structureInput;
size_t inSize = args->structureInputSize;
IOMemoryMap * inMap = NULL;
struct minisurf_client_ref * ref;
IOReturn ret;
switch (selector) {
case kSurfaceMethodInit:
return methodInit(args);
case kSurfaceMethodCreate:
return methodCreate(args);
case kSurfaceMethodWrapClientImage:
if (gSurfaceWrap) {
return methodWrapClientImage(args);
}
break;
case kSurfaceMethodLookup:
return methodLookup(args);
case kSurfaceMethodLookupFromPort:
return methodLookupFromPort(args);
case kSurfaceMethodCreatePort:
return methodCreatePort(args);
case kSurfaceMethodSetValue:
case kSurfaceMethodCopyValue:
case kSurfaceMethodRemoveValue:
case kSurfaceMethodSetBulkAttachments:
if (args->structureInputDescriptor != NULL) {
inMap = args->structureInputDescriptor->map(kIOMapReadOnly);
if (inMap == NULL) {
return kIOReturnNoMemory;
}
in = (const uint8_t *) inMap->getVirtualAddress();
inSize = inMap->getLength();
}
if (in == NULL) {
ret = kIOReturnBadArgument;
} else if (selector == kSurfaceMethodSetValue) {
ret = methodSetValue(args, in, inSize);
} else if (selector == kSurfaceMethodCopyValue) {
ret = methodCopyValue(args, in, inSize);
} else if (selector == kSurfaceMethodRemoveValue) {
ret = methodRemoveValue(args, in, inSize);
} else {
ret = methodSetBulkAttachments(in, inSize);
}
OSSafeReleaseNULL(inMap);
return ret;
case kSurfaceMethodSetPurgeable: /* in: surface, new state; out: old state */
if (args->scalarInputCount < 2) {
return kIOReturnBadArgument;
}
ret = kIOReturnNotFound;
IOLockLock(gSurfaceLock);
ref = findRef((uint32_t) args->scalarInput[0], false);
if (ref != NULL) {
if (args->scalarOutputCount >= 1) {
args->scalarOutput[0] = ref->surface->purgeable;
args->scalarOutputCount = 1;
}
if (args->scalarInput[1] != 3) { /* 3 = keep current */
ref->surface->purgeable = (uint32_t) args->scalarInput[1];
}
ret = kIOReturnSuccess;
}
IOLockUnlock(gSurfaceLock);
return ret;
case kSurfaceMethodBindAccel: /* in: surface, accelerator, sub-id */
if (args->scalarInputCount < 3) {
return kIOReturnBadArgument;
}
IOLockLock(gSurfaceLock);
ref = findRef((uint32_t) args->scalarInput[0], false);
if (ref != NULL) {
ref->surface->shared()->accelID = (uint16_t) args->scalarInput[1];
ref->surface->shared()->accelSubID = (uint16_t) args->scalarInput[2];
}
IOLockUnlock(gSurfaceLock);
return ref ? kIOReturnSuccess : kIOReturnNotFound;
case kSurfaceMethodSetYCbCrMatrix: /* in: surface, matrix */
if (args->scalarInputCount < 2) {
break;
}
IOLockLock(gSurfaceLock);
ref = findRef((uint32_t) args->scalarInput[0], false);
if (ref != NULL) {
ref->surface->shared()->ycbcrMatrix = (uint32_t) args->scalarInput[1];
}
IOLockUnlock(gSurfaceLock);
return ref ? kIOReturnSuccess : kIOReturnNotFound;
case kSurfaceMethodSetOwnership:
case kSurfaceMethodSetOwnershipIdentity:
case kSurfaceMethodSetTimestamp:
case kSurfaceMethodSetIndexedTimestamp:
case kSurfaceMethodDetachModeCode:
case kSurfaceMethodLog:
/* accounting and diagnostics: nothing depends on them here */
if (args->scalarOutputCount != 0) {
bzero(args->scalarOutput, args->scalarOutputCount * sizeof(args->scalarOutput[0]));
}
return kIOReturnSuccess;
default:
break;
}
/*
* Surfaces around memory of the client's own (IOSurfaceWrapClientMemory; the image's are made: methodWrapClientImage):
* what is asked, said for the first few and every thousandth, is the layout of the request for when they are made --
* that is how the image's was learnt, at about 4000 refused requests a second from the window server.
*/
if (selector == kSurfaceMethodWrapClientImage || selector == kSurfaceMethodWrapClientMemory) {
static uint32_t wraps[2];
uint32_t n = ++wraps[selector - kSurfaceMethodWrapClientImage];
if (n <= 6 || n % 1000 == 0) {
char hex[3 * 64 + 1] = "", procname[32] = "?";
const uint8_t * in = (const uint8_t *)args->structureInput;
uint32_t size = args->structureInputSize < 64 ? args->structureInputSize : 64;
for (uint32_t i = 0; in != NULL && i < size; i++) {
snprintf(hex + 3 * i, sizeof(hex) - 3 * i, "%02x ", in[i]);
}
proc_selfname(procname, sizeof(procname));
IOLog("MiniIOSurface: wrap client %s #%u from %s[%d]: scalars %u (%#llx %#llx %#llx), structure %u in, %u out: %s\n",
selector == kSurfaceMethodWrapClientImage ? "image" : "memory", n, procname, proc_selfpid(),
args->scalarInputCount, args->scalarInputCount > 0 ? args->scalarInput[0] : 0ULL,
args->scalarInputCount > 1 ? args->scalarInput[1] : 0ULL, args->scalarInputCount > 2 ? args->scalarInput[2] : 0ULL,
args->structureInputSize, args->structureOutputSize, hex);
}
}
if (selector < 128 && !(gSurfaceLogged[selector / 64] & (1ULL << (selector % 64)))) {
gSurfaceLogged[selector / 64] |= 1ULL << (selector % 64);
IOLog("MiniIOSurface: selector %u not implemented (scalars %u in, %u out; structure %u in, %u out)\n", selector,
args->scalarInputCount, args->scalarOutputCount, args->structureInputSize, args->structureOutputSize);
}
return kIOReturnUnsupported;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
IOExternalTrap *
IOSurfaceRootUserClient::getTargetAndTrapForIndex(IOService ** targetP, UInt32 index)
{
if (index >= kSurfaceTrapCount) {
return NULL;
}
*targetP = this;
return &fTraps[index];
}
IOReturn
IOSurfaceRootUserClient::useCountCommon(uint32_t sid, uint32_t category, int delta)
{
struct minisurf_client_ref * ref;
IOReturn ret = kIOReturnNotFound;
if (category > 1) {
return kIOReturnBadArgument;
}
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref != NULL) {
MiniIOSurface * s = ref->surface;
if (delta > 0 || ref->use[category] != 0) {
ref->use[category] += delta;
s->useCount[category] += delta;
s->shared()->useCount = (uint64_t) s->useCount[1] << 32 | s->useCount[0];
}
ret = kIOReturnSuccess;
}
IOLockUnlock(gSurfaceLock);
return ret;
}
IOReturn
IOSurfaceRootUserClient::trapIncrementUseCount(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
{
return useCountCommon((uint32_t) (uintptr_t) p1, (uint32_t) (uintptr_t) p2, 1);
}
IOReturn
IOSurfaceRootUserClient::trapDecrementUseCount(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
{
return useCountCommon((uint32_t) (uintptr_t) p1, (uint32_t) (uintptr_t) p2, -1);
}
/*
* Lock and unlock do not exclude anybody: the memory is shared and always mapped.
* What matters is the seed, which tells readers that the pixels may have changed.
*/
IOReturn
IOSurfaceRootUserClient::lockCommon(uint32_t sid, uint32_t options, user_addr_t seedOut)
{
struct minisurf_client_ref * ref;
uint32_t seed = 0;
IOLockLock(gSurfaceLock);
ref = findRef(sid, false);
if (ref != NULL) {
struct minisurf_shared * sh = ref->surface->shared();
if (!(options & kSurfaceLockReadOnly)) {
sh->seed++;
}
seed = sh->seed;
}
IOLockUnlock(gSurfaceLock);
if (ref == NULL) {
return kIOReturnNotFound;
}
if (seedOut != 0) {
copyout(&seed, seedOut, sizeof(seed));
}
return kIOReturnSuccess;
}
IOReturn
IOSurfaceRootUserClient::trapLock(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
{
return lockCommon((uint32_t) (uintptr_t) p1, (uint32_t) (uintptr_t) p2, (user_addr_t) (uintptr_t) p3);
}
IOReturn
IOSurfaceRootUserClient::trapUnlock(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
{
return lockCommon((uint32_t) (uintptr_t) p1, (uint32_t) (uintptr_t) p2, (user_addr_t) (uintptr_t) p3);
}
/*
* A client drops a surface in two steps: "detach" while it still holds its own lock, and
* "release" after it. A lookup of the same surface in between must make a fresh view.
*/
IOReturn
IOSurfaceRootUserClient::trapDetach(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
{
struct minisurf_client_ref * ref;
IOLockLock(gSurfaceLock);
ref = findRef((uint32_t) (uintptr_t) p1, false);
if (ref != NULL) {
ref->detached = true;
}
IOLockUnlock(gSurfaceLock);
return kIOReturnSuccess;
}
IOReturn
IOSurfaceRootUserClient::trapRelease(void * p1, void * p2, void * p3, void * p4, void * p5, void * p6)
{
struct minisurf_client_ref * ref;
uint32_t sid = (uint32_t) (uintptr_t) p1;
IOLockLock(gSurfaceLock);
ref = findRef(sid, true);
if (ref == NULL) {
ref = findRef(sid, false);
}
if (ref != NULL) {
dropRef(ref);
}
IOLockUnlock(gSurfaceLock);
return ref ? kIOReturnSuccess : kIOReturnNotFound;
}