Skip to content

Free fork of NotProton: Windows Steam games in Mac Steam, on CodeWeavers' Wine and Apple's Game Porting Toolkit

feat(overlay-shim): experimental frame generation behind NOTPROTON_FRAMEGEN with a panel toggle

raymondchinse7d79aee4edd
diff --git a/Makefile b/Makefile
index 1a3610c..f9f75db 100644
--- a/Makefile
+++ b/Makefile
@@ -331,6 +331,7 @@ panel-behavior:
node $(PANEL_TESTS)/launch-options.js $(OUT_DIR)/panel-emit && \
node $(PANEL_TESTS)/shell.js $(OUT_DIR)/panel-emit && \
node $(PANEL_TESTS)/migration.js $(OUT_DIR)/panel-emit && \
+ node $(PANEL_TESTS)/effective.js $(OUT_DIR)/panel-emit && \
echo "==> panel behavior: renders as expected, no stale arguments"
CX_ROOT ?= /Applications/CrossOver Preview.app
@@ -384,7 +385,7 @@ OVERLAY_SHIM_TEST_FLAGS := -mmacosx-version-min=$(MIN_VER) -O2 -Wall -Wextra \
overlay-shim: $(OVERLAY_SHIM)
-$(OVERLAY_SHIM): $(OVERLAY_SHIM_SRC)
+$(OVERLAY_SHIM): $(OVERLAY_SHIM_SRC) overlay-shim/framegen.m
@mkdir -p $(OUT_DIR)
$(CC) -arch arm64 $(OVERLAY_SHIM_FLAGS) -o $(OVERLAY_SHIM_ARM) $<
$(CC) -arch x86_64 $(OVERLAY_SHIM_FLAGS) -o $(OVERLAY_SHIM_X86) $<
@@ -407,7 +408,7 @@ overlay-shim-tests: $(OVERLAY_SHIM)
$(call SKIP,overlay-shim-tests,$(OVERLAY_SHIM_TESTS)); exit 0; fi; \
mkdir -p $(OUT_DIR); \
fail=0; ran=0; skipped=0; \
- for t in device bridge delivery; do \
+ for t in device bridge delivery framegen; do \
for a in arm64 x86_64; do \
$(CC) -arch $$a $(OVERLAY_SHIM_TEST_FLAGS) -o $(OUT_DIR)/$$t.$$a \
$(OVERLAY_SHIM_TESTS)/$$t.m || exit 1; \
diff --git a/docs/GAMEFIXES.md b/docs/GAMEFIXES.md
index 86e9833..8b3bef1 100644
--- a/docs/GAMEFIXES.md
+++ b/docs/GAMEFIXES.md
@@ -26,6 +26,7 @@ logged, when:
| `DLLOVERRIDES` | Wine syntax, e.g. `dinput8=n,b;xinput1_3=n` | Added to `WINEDLLOVERRIDES` after the backend's entries and before the launch options' |
| `MSYNC` | `0`, `1` | `1` turns msync on when the engine supports it; `0` turns it off |
| `RETINA` | `0`, `1` | Same as `NOTPROTON_RETINA` (High Resolution) |
+| `FRAMEGEN` | `0`, `1` | Same as `NOTPROTON_FRAMEGEN=1`: experimental MetalFX frame interpolation (2x) in overlay-shim, adds up to half a frame of latency |
| `NVAPI` | `0`, `1` | `1` turns on Apple's `nvapi64` on an engine with a linked Game Porting Toolkit, for `d3dmetal` only. Off by default, since an nvapi that answers sends some games down NVIDIA-only paths |
| `DXMT_*`, `DXVK_*`, `MVK_*`, `VKD3D_*`, `D3DM_*`, `STAGING_*` | any | Exported as is |
| `MTL_HUD_ENABLED`, `WINEDEBUG`, `WINE_LARGE_ADDRESS_AWARE`, `WINE_CPU_TOPOLOGY` | any | Exported as is |
diff --git a/dylib/feats/compat_run.sh b/dylib/feats/compat_run.sh
index 0555e67..f264c9f 100755
--- a/dylib/feats/compat_run.sh
+++ b/dylib/feats/compat_run.sh
@@ -256,7 +256,7 @@ gamefix_reason() {
case "$1" in
BACKEND)
case "$2" in dxmt|dxvk|wined3d|d3dmetal) ;; *) echo "not dxmt, dxvk, wined3d or d3dmetal" ;; esac ;;
- MSYNC|RETINA|NVAPI)
+ MSYNC|RETINA|NVAPI|FRAMEGEN)
case "$2" in 0|1) ;; *) echo "not 0 or 1" ;; esac ;;
DLLOVERRIDES)
valid_dll_overrides "$2" || echo "not Wine override syntax" ;;
@@ -340,6 +340,7 @@ load_gamefixes() {
gamefix_log "ARGS not applied, $verb is not the game launch"
fi ;;
RETINA) gamefix_apply_env NOTPROTON_RETINA "$value" "$from" ;;
+ FRAMEGEN) gamefix_apply_env NOTPROTON_FRAMEGEN "$value" "$from" ;;
*) gamefix_apply_env "$key" "$value" "$from" ;;
esac
done
@@ -1973,6 +1974,10 @@ if [ -n "\$STEAM_DYLD_INSERT_LIBRARIES" ]; then
else
export DYLD_INSERT_LIBRARIES="\$STEAM_DYLD_INSERT_LIBRARIES"
fi
+elif [ -f "\$shim" ] && [ "\${NOTPROTON_FRAMEGEN:-0}" != 0 ]; then
+ # Frame generation lives in the shim, so it loads without the Steam overlay too.
+ export DYLD_INSERT_LIBRARIES="\$shim"
+ export NOTPROTON_OVERLAY_SHIM="\$shim"
fi
[ -n "\$NOTPROTON_GAME_CWD" ] && cd "\$NOTPROTON_GAME_CWD"
"$WINELOADER" "\$@"
@@ -2109,7 +2114,7 @@ else
echo "=== client staged no overlay renderer, overlay disabled ===" >> "$log" 2>&1 || true
fi
set -- --args "$shim_exe" "$@"
-for name in $(env | sed -nE 's/^(CX_APPLEGPTK_LIBD3DSHARED_PATH|SDL_GAMECONTROLLERCONFIG(_FILE)?|SDL_JOYSTICK_MFI|WINE_LARGE_ADDRESS_AWARE|WINE_CPU_TOPOLOGY|Steam[A-Za-z0-9]*|(CX_GRAPHICS|D3DM_|DXMT_|DXVK_|MVK_|VKD3D_|STAGING_|MTL_|ROSETTA_)[A-Z0-9_]*)=.*/\1/p'); do
+for name in $(env | sed -nE 's/^(CX_APPLEGPTK_LIBD3DSHARED_PATH|SDL_GAMECONTROLLERCONFIG(_FILE)?|SDL_JOYSTICK_MFI|WINE_LARGE_ADDRESS_AWARE|WINE_CPU_TOPOLOGY|Steam[A-Za-z0-9]*|(CX_GRAPHICS|D3DM_|DXMT_|DXVK_|MVK_|VKD3D_|STAGING_|MTL_|ROSETTA_)[A-Z0-9_]*|NOTPROTON_FRAMEGEN)=.*/\1/p'); do
eval "value=\$$name"
# shellcheck disable=SC2154 # eval assigns value on the line above
set -- --env "$name=$value" "$@"
diff --git a/dylib/tests/gamefixescheck.sh b/dylib/tests/gamefixescheck.sh
index 9139bbc..09e30ad 100644
--- a/dylib/tests/gamefixescheck.sh
+++ b/dylib/tests/gamefixescheck.sh
@@ -106,7 +106,7 @@ launch() {
graphics_log "backend=$backend ($backend_from)"
compose_dll_overrides
eval "$bridge_line"
- for v in backend WINEDLLOVERRIDES WINEMSYNC msync_from NOTPROTON_RETINA CX_GRAPHICS_BACKEND \
+ for v in backend WINEDLLOVERRIDES WINEMSYNC msync_from NOTPROTON_RETINA NOTPROTON_FRAMEGEN CX_GRAPHICS_BACKEND \
DXVK_HUD VKD3D_CONFIG WINEDEBUG MTL_HUD_ENABLED gamefix_args; do
eval "printf \"%s=%s\\n\" \$v \"\${$v-<unset>}\""
done
@@ -166,7 +166,7 @@ default_conf '# shipped fix' '' ' DXVK_HUD=fps ' "VKD3D_CONFIG=dxr$(printf '\r
'WINEDEBUG=-all' 'PATH=/evil' 'DYLD_INSERT_LIBRARIES=/evil.dylib' 'lower=1' 'no equals sign' \
"DXMT_A=\$(touch $case_dir/pwned)" 'DXMT_B=`touch pwned`' 'DXMT_C="quoted"' "DXMT_D='single'" \
'BACKEND=vulkan' 'MSYNC=2' 'RETINA=yes' 'DLLOVERRIDES=dxgi=native' 'DLLOVERRIDES=;=n' \
- 'DXMT_=1' 'MTL_HUD_ENABLED=1'
+ 'DXMT_=1' 'MTL_HUD_ENABLED=1' 'FRAMEGEN=2'
launch waitforexitandrun "$case_dir/game/game.exe"
is "a padded value is trimmed" "fps" "$(got DXVK_HUD)"
is "a CRLF line ends cleanly" "dxr" "$(got VKD3D_CONFIG)"
@@ -494,13 +494,22 @@ is "RETINA maps to NOTPROTON_RETINA" "1" "$(got NOTPROTON_RETINA)"
launch waitforexitandrun "$case_dir/game/game.exe" NOTPROTON_RETINA=0
is "the High Resolution launch option wins" "0" "$(got NOTPROTON_RETINA)"
+echo "== FRAMEGEN"
+new_case
+full_engine
+user_conf 'FRAMEGEN=1'
+launch waitforexitandrun "$case_dir/game/game.exe"
+is "FRAMEGEN maps to NOTPROTON_FRAMEGEN" "1" "$(got NOTPROTON_FRAMEGEN)"
+launch waitforexitandrun "$case_dir/game/game.exe" NOTPROTON_FRAMEGEN=0
+is "the frame generation launch option wins" "0" "$(got NOTPROTON_FRAMEGEN)"
+
echo "== the launcher receives the allow-listed variables"
set --
out=$(env -i PATH="$PATH" forward="$forward" VKD3D_CONFIG=dxr MVK_CONFIG_LOG_LEVEL=1 \
STAGING_SHARED_MEMORY=1 WINE_CPU_TOPOLOGY=4:0,1,2,3 WINE_LARGE_ADDRESS_AWARE=1 \
- DXVK_HUD=fps NOTPROTON_UNRELATED=1 sh -c 'set --; eval "$forward"; printf "%s\n" "$@"' | grep -v '^--env$' | sort | tr '\n' ' ')
+ DXVK_HUD=fps NOTPROTON_FRAMEGEN=1 NOTPROTON_UNRELATED=1 sh -c 'set --; eval "$forward"; printf "%s\n" "$@"' | grep -v '^--env$' | sort | tr '\n' ' ')
is "VKD3D_, MVK_, STAGING_ and WINE_ topology variables are forwarded" \
- "DXVK_HUD=fps MVK_CONFIG_LOG_LEVEL=1 STAGING_SHARED_MEMORY=1 VKD3D_CONFIG=dxr WINE_CPU_TOPOLOGY=4:0,1,2,3 WINE_LARGE_ADDRESS_AWARE=1 " "$out"
+ "DXVK_HUD=fps MVK_CONFIG_LOG_LEVEL=1 NOTPROTON_FRAMEGEN=1 STAGING_SHARED_MEMORY=1 VKD3D_CONFIG=dxr WINE_CPU_TOPOLOGY=4:0,1,2,3 WINE_LARGE_ADDRESS_AWARE=1 " "$out"
echo "== the wineserver directory self-heal"
heal=$(sed -n '/^heal_wine_tmp() {$/,/^}$/p' "$SRC")
diff --git a/dylib/tests/panel-behavior/behavior.js b/dylib/tests/panel-behavior/behavior.js
index 1d21435..e6ff3f8 100644
--- a/dylib/tests/panel-behavior/behavior.js
+++ b/dylib/tests/panel-behavior/behavior.js
@@ -21,7 +21,7 @@ for (const form of Object.keys(FORMS)) {
'a shortcut renders the panel before a tool name reaches the page');
let r = nodes('');
- t.ok(toggles(r).length === 6 && sections(r).length === 3, 'automatic shows six toggles in three sections');
+ t.ok(toggles(r).length === 7 && sections(r).length === 4, 'automatic shows seven toggles in four sections');
t.ok(r.filter(x => x.type === 'Dropdown').length === 2, 'two dropdowns');
t.ok(!('label' in r.find(x => x.type === 'Dropdown').props), 'the backend dropdown carries no label column');
@@ -79,7 +79,7 @@ for (const form of Object.keys(FORMS)) {
const root = P({ details: details('') });
t.ok(root.props.className === 'MSCXPanel', 'the panel root carries its own class');
const classes = nodes('').filter(x => x.type === 'Toggle').map(x => x.props.className);
- t.ok(classes.length === 6 && classes.every(c => c === 'MSCXRow'), 'every checkbox carries the row class');
+ t.ok(classes.length === 7 && classes.every(c => c === 'MSCXRow'), 'every checkbox carries the row class');
const raw = o => nodes(o).find(x => x.props.label === 'Let games read controllers directly');
t.ok(sections(nodes('')).includes('Controllers') && raw('').props.checked === false,
'controller hiding is on unless the game says otherwise');
@@ -119,6 +119,25 @@ for (const form of Object.keys(FORMS)) {
t.ok(last().trim() === 'NOTPROTON_RAW_CONTROLLERS=0 %command%', `opting out of raw controllers writes 0 (${last()})`);
delete globalThis.MSCXFx;
+ // Frame generation: shown for every backend, one row.
+ const fg = (o) => nodes(o).find(x => x.props.label === 'MetalFX frame interpolation (2x)');
+ for (const b of ['', 'd3dmetal', 'dxmt', 'dxvk', 'wined3d']) {
+ t.ok(fg(b ? 'CX_GRAPHICS_BACKEND=' + b : '') !== undefined, `${b || 'automatic'} shows the frame generation row`);
+ }
+ t.ok(nodes('').some(x => x.type === 'Section' && x.props.label === 'Frame Generation (Experimental. Adds input lag)'),
+ 'the frame generation section warns about latency');
+ t.ok(fg('').props.checked === false, 'frame generation is off by default');
+ written.length = 0;
+ fg('').props.onChange(true);
+ t.ok(last().trim() === 'NOTPROTON_FRAMEGEN=1 %command%', `the frame generation row writes its flag (${last()})`);
+ t.ok(fg('NOTPROTON_FRAMEGEN=1').props.checked === true, 'the frame generation row reads its flag');
+ written.length = 0;
+ fg('NOTPROTON_FRAMEGEN=1 WINEMSYNC=1').props.onChange(false);
+ t.ok(last().trim() === 'WINEMSYNC=1 %command%', `turning frame generation off removes its flag (${last()})`);
+ globalThis.MSCXFx = { g: { NOTPROTON_FRAMEGEN: '1' }, a: {}, e: {}, d: {} };
+ t.ok(fg('').props.checked === true, 'a global conf value reads frame generation as on');
+ delete globalThis.MSCXFx;
+
const css = nodes('').filter(x => x.type === 'style');
t.ok(css.length === 1 && css[0].props.children.includes('.MSCXPanel .MSCXRow{'),
'the panel carries one stylesheet defining the row rule');
diff --git a/overlay-shim/framegen.m b/overlay-shim/framegen.m
new file mode 100644
index 0000000..6843ffa
--- /dev/null
+++ b/overlay-shim/framegen.m
@@ -0,0 +1,1632 @@
+// Frame generation for the overlay shim. Included textually by overlay_shim.m
+// so the Makefile keeps building one translation unit.
+//
+// Enabled only by NOTPROTON_FRAMEGEN=1. Off, nothing in here runs: no hook is
+// installed and no object is created, so the per-frame cost is the shim's own.
+//
+// How it works
+// Real frame N is the drawable the game presents. Its present is diverted:
+// the command buffer that carries the frame (the one calling presentDrawable:,
+// or the Metal 4 bridge buffer, or the last buffer that rendered into the
+// drawable) gets a blit of the drawable into a history texture and a signal
+// on an MTLEvent, and the drawable itself is never presented. A worker
+// thread then, on its own queue and ordered by that event, synthesises the
+// frame between N-1 and N and presents it on a second CAMetalLayer laid over
+// the game's layer, followed by a copy of real frame N half an interval
+// later. The game's layer stops presenting while framegen is active; the
+// overlay layer covers it exactly and inherits format, colorspace and EDR.
+//
+// Two engines: MetalFX's MTLFXFrameInterpolator (macOS 26+, fed with the two
+// colour frames, a constant depth and zero motion; it estimates motion
+// itself) and an in-house compute path (luma pyramid, coarse to fine block
+// matching in both directions, bidirectional warp with a mismatch fallback).
+// MetalFX is preferred when the device supports it; NOTPROTON_FRAMEGEN_ENGINE
+// = own|metalfx overrides.
+//
+// Pacing: the generated frame is presented at t_N + lead and the real frame
+// at t_N + lead + interval/2 through presentDrawable:atTime:. The real frame
+// therefore reaches the glass lead + interval/2 later than today; lead is the
+// measured generation latency, clamped to [3 ms, interval/2].
+//
+// Pass-through (the game's own present goes ahead, overlay layer hidden) on:
+// a second layer, a size or format change (until the worker has rebuilt),
+// game fps at or above the display refresh, the worker falling behind, no
+// drawable within the timeout, or any Metal failure.
+
+#import <QuartzCore/CATransaction.h>
+#import <dispatch/dispatch.h>
+#import <mach/mach_time.h>
+#import <pthread.h>
+#import <stdatomic.h>
+#import <stdlib.h>
+
+static int framegen_enabled;
+static int framegen_multiplier;
+static int framegen_engine_pref; // 0 auto, 1 own, 2 metalfx
+static int framegen_stats_log;
+
+enum fg_mode {
+ fg_mode_idle = 0,
+ fg_mode_configuring,
+ fg_mode_active,
+ fg_mode_passthrough,
+};
+
+enum fg_skip {
+ fg_skip_other_layer = 0,
+ fg_skip_mismatch,
+ fg_skip_fast,
+ fg_skip_lag,
+ fg_skip_no_drawable,
+ fg_skip_error,
+ fg_skip_unordered,
+ fg_skip_warmup,
+ fg_skip_settling,
+ fg_skip_count,
+};
+
+static const char *const fg_skip_names[fg_skip_count] = {
+ "other-layer", "mismatch", "fps>=refresh", "worker-lag", "no-drawable", "error", "unordered",
+ "metalfx-warmup", "settling",
+};
+
+// Shared between the game thread and the worker; fg_lock guards everything
+// below except the atomics.
+static os_unfair_lock fg_lock = OS_UNFAIR_LOCK_INIT;
+static _Atomic int fg_mode;
+static _Atomic int fg_inflight;
+static _Atomic unsigned fg_generation;
+static _Atomic int fg_hooks_installed;
+
+static id fg_game_layer; // unretained, identity only
+static id<MTLDevice> fg_device;
+static id<MTLCommandQueue> fg_queue;
+static id<MTLEvent> fg_event;
+static uint64_t fg_event_value;
+static CAMetalLayer *fg_layer;
+static dispatch_queue_t fg_worker;
+
+static NSUInteger fg_width, fg_height, fg_format;
+static id<MTLTexture> fg_hist[2];
+static int fg_hist_next;
+static int fg_hist_valid; // frames in history, caps at 2
+static double fg_last_present_time;
+static double fg_interval; // EMA of real present spacing, seconds
+static double fg_lead; // EMA of generation latency, seconds
+static double fg_refresh_interval;
+static int fg_slow_frames;
+
+// Stats, once per second when OVERLAY_SHIM_LOG is set
+static double fg_stat_start;
+static int fg_stat_real, fg_stat_generated, fg_stat_skips[fg_skip_count];
+static double fg_stat_gpu_ms, fg_stat_gpu_max_ms;
+static int fg_stat_gpu_n;
+static _Atomic int fg_total_generated, fg_total_real_copies;
+
+// Which engine the worker ended up with
+static int fg_engine_kind; // 0 none, 1 own, 2 metalfx
+static const char *fg_engine_name(int kind)
+{
+ return kind == 2 ? "metalfx" : kind == 1 ? "own" : "none";
+}
+
+// Per command buffer: the drawable it presents or the drawable texture it
+// rendered into, until commit.
+#define fg_max_pending 8
+
+struct fg_pending {
+ id buffer; // unretained key
+ id drawable; // retained when present was diverted
+ id<MTLTexture> texture; // unretained, identity only
+ int divert;
+ double min_duration; // presentDrawable:afterMinimumDuration: request, else 0
+};
+
+static struct fg_pending fg_pending[fg_max_pending];
+
+// The last blit encoded for a drawable texture that has not been presented yet
+static id<MTLTexture> fg_blit_texture; // unretained
+static uint64_t fg_blit_value;
+static int fg_blit_slot;
+static double fg_blit_time;
+
+static id fg_last_vended_texture; // unretained, from nextDrawable
+static id fg_passthrough_drawable; // unretained; its buffer-level present went ahead
+static int fg_unhide_pending; // the next real copy presented unhides the overlay layer
+static __thread int fg_internal; // set while the worker drives Metal itself
+
+static double fg_now(void)
+{
+ return CACurrentMediaTime();
+}
+
+static void fg_stat_skip(enum fg_skip reason)
+{
+ os_unfair_lock_lock(&fg_lock);
+ fg_stat_skips[reason]++;
+ os_unfair_lock_unlock(&fg_lock);
+}
+
+// ---------------------------------------------------------------------------
+// Engines
+
+static const char *const fg_shader_source =
+"#include <metal_stdlib>\n"
+"using namespace metal;\n"
+"constant sampler fg_lin(coord::normalized, address::clamp_to_edge, filter::linear);\n"
+"constant sampler fg_px(coord::pixel, address::clamp_to_edge, filter::nearest);\n"
+"struct fg_const_params { float value; };\n"
+"kernel void np_fg_const(texture2d<float, access::write> dst [[texture(0)]],\n"
+" constant fg_const_params &p [[buffer(0)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" if (gid.x >= dst.get_width() || gid.y >= dst.get_height()) return;\n"
+" dst.write(float4(p.value), gid);\n"
+"}\n"
+// Luma at a quarter of the resolution, each output pixel a 4x4 box of the input
+"kernel void np_fg_luma(texture2d<float, access::sample> src [[texture(0)]],\n"
+" texture2d<float, access::write> dst [[texture(1)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" if (gid.x >= dst.get_width() || gid.y >= dst.get_height()) return;\n"
+" float2 inv = 1.0 / float2(src.get_width(), src.get_height());\n"
+" float2 base = (float2(gid) * 4.0 + 1.0) * inv;\n"
+" float l = 0.0;\n"
+" for (int y = 0; y < 2; y++) for (int x = 0; x < 2; x++) {\n"
+" float3 c = src.sample(fg_lin, base + float2(x, y) * 2.0 * inv).rgb;\n"
+" l += dot(c, float3(0.299, 0.587, 0.114));\n"
+" }\n"
+" dst.write(float4(l * 0.25), gid);\n"
+"}\n"
+// One pyramid level of block matching: a -> b. Flow is in this level's pixels.
+// The radius is a template parameter so the cost window stays in registers.
+"template <int R, int HALF>\n"
+"static inline float2 fg_flow_search(texture2d<float, access::sample> a,\n"
+" texture2d<float, access::sample> b,\n"
+" float2 pos, float2 base) {\n"
+" float cost[2 * R + 1][2 * R + 1];\n"
+" float best = 1e30; int bi = R, bj = R;\n"
+" for (int dy = -R; dy <= R; dy++) for (int dx = -R; dx <= R; dx++) {\n"
+" float2 v = base + float2(dx, dy);\n"
+" float s = 0.0;\n"
+" for (int j = -HALF; j <= HALF; j++) for (int i = -HALF; i <= HALF; i++) {\n"
+" float2 q = pos + float2(i, j);\n"
+" s += abs(a.sample(fg_px, q).r - b.sample(fg_px, q + v).r);\n"
+" }\n"
+" cost[dy + R][dx + R] = s;\n"
+" if (s < best) { best = s; bi = dx + R; bj = dy + R; }\n"
+" }\n"
+" float2 sub = 0.0;\n"
+" if (bi > 0 && bi < 2 * R) {\n"
+" float l = cost[bj][bi - 1], rr = cost[bj][bi + 1];\n"
+" float d = l + rr - 2.0 * best;\n"
+" if (d > 1e-5) sub.x = clamp(0.5 * (l - rr) / d, -0.5, 0.5);\n"
+" }\n"
+" if (bj > 0 && bj < 2 * R) {\n"
+" float u = cost[bj - 1][bi], dn = cost[bj + 1][bi];\n"
+" float d = u + dn - 2.0 * best;\n"
+" if (d > 1e-5) sub.y = clamp(0.5 * (u - dn) / d, -0.5, 0.5);\n"
+" }\n"
+" return base + float2(bi - R, bj - R) + sub;\n"
+"}\n"
+"kernel void np_fg_flow3(texture2d<float, access::sample> a [[texture(0)]],\n"
+" texture2d<float, access::sample> b [[texture(1)]],\n"
+" texture2d<float, access::write> out [[texture(3)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" uint w = out.get_width(), h = out.get_height();\n"
+" if (gid.x >= w || gid.y >= h) return;\n"
+" float2 pos = float2(gid) + 0.5;\n"
+" out.write(float4(fg_flow_search<3, 2>(a, b, pos, float2(0.0)), 0.0, 0.0), gid);\n"
+"}\n"
+"kernel void np_fg_flow1(texture2d<float, access::sample> a [[texture(0)]],\n"
+" texture2d<float, access::sample> b [[texture(1)]],\n"
+" texture2d<float, access::sample> prev [[texture(2)]],\n"
+" texture2d<float, access::write> out [[texture(3)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" uint w = out.get_width(), h = out.get_height();\n"
+" if (gid.x >= w || gid.y >= h) return;\n"
+" float2 pos = float2(gid) + 0.5;\n"
+" float2 base = round(2.0 * prev.sample(fg_lin, pos / float2(w, h)).xy);\n"
+" out.write(float4(fg_flow_search<1, 2>(a, b, pos, base), 0.0, 0.0), gid);\n"
+"}\n"
+// The finest level with a 3x3 window: it carries most of the cost
+"kernel void np_fg_flow1f(texture2d<float, access::sample> a [[texture(0)]],\n"
+" texture2d<float, access::sample> b [[texture(1)]],\n"
+" texture2d<float, access::sample> prev [[texture(2)]],\n"
+" texture2d<float, access::write> out [[texture(3)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" uint w = out.get_width(), h = out.get_height();\n"
+" if (gid.x >= w || gid.y >= h) return;\n"
+" float2 pos = float2(gid) + 0.5;\n"
+" float2 base = round(2.0 * prev.sample(fg_lin, pos / float2(w, h)).xy);\n"
+" out.write(float4(fg_flow_search<1, 1>(a, b, pos, base), 0.0, 0.0), gid);\n"
+"}\n"
+// Quarter-resolution flow to full-resolution motion vectors in pixels, the
+// layout MetalFX wants: each pixel of the new frame points to where it was.
+"kernel void np_fg_motion(texture2d<float, access::sample> flow [[texture(0)]],\n"
+" texture2d<float, access::write> out [[texture(1)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" uint w = out.get_width(), h = out.get_height();\n"
+" if (gid.x >= w || gid.y >= h) return;\n"
+" float2 uv = (float2(gid) + 0.5) / float2(w, h);\n"
+" out.write(float4(flow.sample(fg_lin, uv).xy * 4.0, 0.0, 0.0), gid);\n"
+"}\n"
+// The frame between a and b, from the forward and backward flows at quarter
+// resolution. Where neither flow explains a pixel, lean on the newer frame.
+"kernel void np_fg_synth(texture2d<float, access::sample> a [[texture(0)]],\n"
+" texture2d<float, access::sample> b [[texture(1)]],\n"
+" texture2d<float, access::sample> fab [[texture(2)]],\n"
+" texture2d<float, access::sample> fba [[texture(3)]],\n"
+" texture2d<float, access::write> out [[texture(4)]],\n"
+" uint2 gid [[thread_position_in_grid]]) {\n"
+" uint w = out.get_width(), h = out.get_height();\n"
+" if (gid.x >= w || gid.y >= h) return;\n"
+" float2 size = float2(w, h);\n"
+" float2 uv = (float2(gid) + 0.5) / size;\n"
+" float2 f = fab.sample(fg_lin, uv).xy * 4.0 / size;\n"
+" float2 g = fba.sample(fg_lin, uv).xy * 4.0 / size;\n"
+" float4 a1 = a.sample(fg_lin, uv - 0.5 * f), b1 = b.sample(fg_lin, uv + 0.5 * f);\n"
+" float4 b2 = b.sample(fg_lin, uv - 0.5 * g), a2 = a.sample(fg_lin, uv + 0.5 * g);\n"
+" float e1 = dot(abs(a1.rgb - b1.rgb), float3(1.0 / 3.0));\n"
+" float e2 = dot(abs(a2.rgb - b2.rgb), float3(1.0 / 3.0));\n"
+" float4 c, newer; float e;\n"
+" if (e1 <= e2) { c = 0.5 * (a1 + b1); newer = b1; e = e1; }\n"
+" else { c = 0.5 * (a2 + b2); newer = b2; e = e2; }\n"
+" float k = smoothstep(0.08, 0.25, e);\n"
+" out.write(mix(c, newer, k), gid);\n"
+"}\n";
+
+#define fg_levels 5
+
+struct fg_engine {
+ id<MTLDevice> device;
+ NSUInteger width, height, format;
+ int kind;
+ // own
+ id<MTLComputePipelineState> ps_const, ps_luma, ps_flow1, ps_flow1f, ps_flow3, ps_motion, ps_synth;
+ id<MTLTexture> luma[2];
+ id<MTLTexture> luma_view[2][fg_levels];
+ id<MTLTexture> flow_ab[fg_levels], flow_ba[fg_levels];
+ id<MTLTexture> out;
+ // metalfx
+ id interpolator;
+ id<MTLTexture> depth, motion;
+ int reset_pending;
+ int warm; // MetalFX encodes since the last reset; the first two echo the new frame
+ int luma_valid[2];
+};
+
+// MetalFX output is a real interpolation only from this many encodes after a reset
+#define fg_metalfx_warmup 2
+
+@protocol np_fx_descriptor <NSObject>
+- (void)setColorTextureFormat:(NSUInteger)format;
+- (void)setOutputTextureFormat:(NSUInteger)format;
+- (void)setDepthTextureFormat:(NSUInteger)format;
+- (void)setMotionTextureFormat:(NSUInteger)format;
+- (void)setUITextureFormat:(NSUInteger)format;
+- (void)setInputWidth:(NSUInteger)value;
+- (void)setInputHeight:(NSUInteger)value;
+- (void)setOutputWidth:(NSUInteger)value;
+- (void)setOutputHeight:(NSUInteger)value;
+- (id)newFrameInterpolatorWithDevice:(id)device;
+@end
+
+@protocol np_fx_interpolator <NSObject>
+- (NSUInteger)colorTextureUsage;
+- (NSUInteger)depthTextureUsage;
+- (NSUInteger)motionTextureUsage;
+- (NSUInteger)outputTextureUsage;
+- (void)setColorTexture:(id)texture;
+- (void)setPrevColorTexture:(id)texture;
+- (void)setDepthTexture:(id)texture;
+- (void)setMotionTexture:(id)texture;
+- (void)setOutputTexture:(id)texture;
+- (void)setMotionVectorScaleX:(float)value;
+- (void)setMotionVectorScaleY:(float)value;
+- (void)setDeltaTime:(float)value;
+- (void)setNearPlane:(float)value;
+- (void)setFarPlane:(float)value;
+- (void)setFieldOfView:(float)value;
+- (void)setAspectRatio:(float)value;
+- (void)setJitterOffsetX:(float)value;
+- (void)setJitterOffsetY:(float)value;
+- (void)setShouldResetHistory:(BOOL)value;
+- (void)setDepthReversed:(BOOL)value;
+- (void)encodeToCommandBuffer:(id)buffer;
+@end
+
+// Foundation is not linked; NSString comes through the runtime
+static id fg_nsstring(const char *text)
+{
+ typedef id (*from_utf8_fn)(id, SEL, const char *);
+
+ return ((from_utf8_fn)objc_msgSend)((id)objc_getClass("NSString"),
+ sel_registerName("stringWithUTF8String:"), text);
+}
+
+static id<MTLTexture> fg_make_texture(id<MTLDevice> device, NSUInteger format, NSUInteger width,
+ NSUInteger height, MTLTextureUsage usage, int mipmapped)
+{
+ MTLTextureDescriptor *descriptor =
+ [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:(MTLPixelFormat)format
+ width:width
+ height:height
+ mipmapped:mipmapped ? YES : NO];
+ descriptor.usage = usage;
+ descriptor.storageMode = MTLStorageModePrivate;
+ return [device newTextureWithDescriptor:descriptor];
+}
+
+static void fg_release(id *slot)
+{
+ if (*slot != nil) {
+ [*slot release];
+ *slot = nil;
+ }
+}
+
+static void fg_engine_destroy(struct fg_engine *engine)
+{
+ fg_release((id *)&engine->ps_const);
+ fg_release((id *)&engine->ps_luma);
+ fg_release((id *)&engine->ps_flow1);
+ fg_release((id *)&engine->ps_flow1f);
+ fg_release((id *)&engine->ps_flow3);
+ fg_release((id *)&engine->ps_motion);
+ fg_release((id *)&engine->ps_synth);
+ for (int i = 0; i < 2; i++) {
+ fg_release((id *)&engine->luma[i]);
+ for (int l = 0; l < fg_levels; l++)
+ fg_release((id *)&engine->luma_view[i][l]);
+ }
+ for (int l = 0; l < fg_levels; l++) {
+ fg_release((id *)&engine->flow_ab[l]);
+ fg_release((id *)&engine->flow_ba[l]);
+ }
+ fg_release((id *)&engine->out);
+ fg_release(&engine->interpolator);
+ fg_release((id *)&engine->depth);
+ fg_release((id *)&engine->motion);
+ fg_release((id *)&engine->device);
+ memset(engine, 0, sizeof(*engine));
+}
+
+static id<MTLComputePipelineState> fg_pipeline(id<MTLDevice> device, id<MTLLibrary> library,
+ const char *name)
+{
+ id<MTLFunction> function = [library newFunctionWithName:fg_nsstring(name)];
+ if (function == nil)
+ return nil;
+
+ NSError *error = nil;
+ id<MTLComputePipelineState> state = [device newComputePipelineStateWithFunction:function
+ error:&error];
+ [function release];
+ if (state == nil)
+ shim_log("overlay_shim framegen: pipeline %s failed: %s\n", name,
+ error != nil ? [[error localizedDescription] UTF8String] : "no reason given");
+ return state;
+}
+
+static int fg_engine_build_own(struct fg_engine *engine)
+{
+ id<MTLDevice> device = engine->device;
+ NSError *error = nil;
+ MTLCompileOptions *options = [[MTLCompileOptions alloc] init];
+ id<MTLLibrary> library =
+ [device newLibraryWithSource:fg_nsstring(fg_shader_source)
+ options:options
+ error:&error];
+ [options release];
+ if (library == nil) {
+ shim_log("overlay_shim framegen: shader compile failed: %s\n",
+ error != nil ? [[error localizedDescription] UTF8String] : "no reason given");
+ return 0;
+ }
+
+ engine->ps_const = fg_pipeline(device, library, "np_fg_const");
+ engine->ps_luma = fg_pipeline(device, library, "np_fg_luma");
+ engine->ps_flow1 = fg_pipeline(device, library, "np_fg_flow1");
+ engine->ps_flow1f = fg_pipeline(device, library, "np_fg_flow1f");
+ engine->ps_flow3 = fg_pipeline(device, library, "np_fg_flow3");
+ engine->ps_motion = fg_pipeline(device, library, "np_fg_motion");
+ engine->ps_synth = fg_pipeline(device, library, "np_fg_synth");
+ [library release];
+ if (engine->ps_const == nil || engine->ps_luma == nil || engine->ps_flow1 == nil ||
+ engine->ps_flow1f == nil || engine->ps_flow3 == nil || engine->ps_motion == nil || engine->ps_synth == nil)
+ return 0;
+
+ NSUInteger lw = (engine->width + 3) / 4, lh = (engine->height + 3) / 4;
+ for (int i = 0; i < 2; i++) {
+ MTLTextureDescriptor *descriptor =
+ [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR16Float
+ width:lw
+ height:lh
+ mipmapped:YES];
+ descriptor.mipmapLevelCount = fg_levels;
+ descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
+ descriptor.storageMode = MTLStorageModePrivate;
+ engine->luma[i] = [device newTextureWithDescriptor:descriptor];
+ if (engine->luma[i] == nil)
+ return 0;
+ for (int l = 0; l < fg_levels; l++) {
+ engine->luma_view[i][l] =
+ [engine->luma[i] newTextureViewWithPixelFormat:MTLPixelFormatR16Float
+ textureType:MTLTextureType2D
+ levels:NSMakeRange(l, 1)
+ slices:NSMakeRange(0, 1)];
+ if (engine->luma_view[i][l] == nil)
+ return 0;
+ }
+ }
+ for (int l = 0; l < fg_levels; l++) {
+ NSUInteger w = lw >> l, h = lh >> l;
+ if (w == 0)
+ w = 1;
+ if (h == 0)
+ h = 1;
+ engine->flow_ab[l] = fg_make_texture(device, MTLPixelFormatRG16Float, w, h,
+ MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite, 0);
+ engine->flow_ba[l] = fg_make_texture(device, MTLPixelFormatRG16Float, w, h,
+ MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite, 0);
+ if (engine->flow_ab[l] == nil || engine->flow_ba[l] == nil)
+ return 0;
+ }
+ engine->out = fg_make_texture(device, engine->format, engine->width, engine->height,
+ MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite, 0);
+ return engine->out != nil;
+}
+
+static void fg_dispatch(id<MTLComputeCommandEncoder> encoder, id<MTLComputePipelineState> state,
+ NSUInteger width, NSUInteger height)
+{
+ NSUInteger tw = state.threadExecutionWidth;
+ NSUInteger th = state.maxTotalThreadsPerThreadgroup / tw;
+ if (th > 16)
+ th = 16;
+ [encoder setComputePipelineState:state];
+ [encoder dispatchThreads:MTLSizeMake(width, height, 1)
+ threadsPerThreadgroup:MTLSizeMake(tw, th, 1)];
+}
+
+static void fg_fill_constant(struct fg_engine *engine, id<MTLCommandBuffer> buffer,
+ id<MTLTexture> texture, float value)
+{
+ id<MTLComputeCommandEncoder> encoder = [buffer computeCommandEncoder];
+ struct { float value; } params = { value };
+ [encoder setTexture:texture atIndex:0];
+ [encoder setBytes:&params length:sizeof(params) atIndex:0];
+ fg_dispatch(encoder, engine->ps_const, texture.width, texture.height);
+ [encoder endEncoding];
+}
+
+static Class fg_metalfx_descriptor_class(void)
+{
+ static Class descriptor_class;
+ static int looked;
+
+ if (!looked) {
+ looked = 1;
+ dlopen("/System/Library/Frameworks/MetalFX.framework/MetalFX", RTLD_NOW | RTLD_GLOBAL);
+ descriptor_class = objc_getClass("MTLFXFrameInterpolatorDescriptor");
+ }
+ return descriptor_class;
+}
+
+static int fg_metalfx_supported(id<MTLDevice> device)
+{
+ Class descriptor_class = fg_metalfx_descriptor_class();
+ SEL supports = sel_registerName("supportsDevice:");
+
+ if (descriptor_class == Nil || !class_respondsToSelector(object_getClass(descriptor_class), supports))
+ return 0;
+ return ((BOOL (*)(Class, SEL, id))objc_msgSend)(descriptor_class, supports, device) ? 1 : 0;
+}
+
+static int fg_engine_build_metalfx(struct fg_engine *engine)
+{
+ Class descriptor_class = fg_metalfx_descriptor_class();
+ if (descriptor_class == Nil || !fg_metalfx_supported(engine->device))
+ return 0;
+
+ id<np_fx_descriptor> descriptor = [[descriptor_class alloc] init];
+ if (descriptor == nil)
+ return 0;
+ [descriptor setColorTextureFormat:engine->format];
+ [descriptor setOutputTextureFormat:engine->format];
+ [descriptor setDepthTextureFormat:MTLPixelFormatR32Float];
+ [descriptor setMotionTextureFormat:MTLPixelFormatRG16Float];
+ [descriptor setUITextureFormat:MTLPixelFormatInvalid];
+ [descriptor setInputWidth:engine->width];
+ [descriptor setInputHeight:engine->height];
+ [descriptor setOutputWidth:engine->width];
+ [descriptor setOutputHeight:engine->height];
+ // macOS 27 defaults to managing the previous colour itself, which would
+ // pair the new frame with a stale copy; we hand it the real previous frame
+ SEL requires_prev = sel_registerName("setRequiresPrevColorTexture:");
+ if ([descriptor respondsToSelector:requires_prev])
+ ((void (*)(id, SEL, BOOL))objc_msgSend)(descriptor, requires_prev, YES);
+ id<np_fx_interpolator> interpolator = [descriptor newFrameInterpolatorWithDevice:engine->device];
+ [descriptor release];
+ if (interpolator == nil) {
+ shim_log("overlay_shim framegen: MetalFX interpolator refused %lux%lu %s\n",
+ (unsigned long)engine->width, (unsigned long)engine->height,
+ format_name(engine->format));
+ return 0;
+ }
+ engine->interpolator = interpolator;
+
+ engine->depth = fg_make_texture(engine->device, MTLPixelFormatR32Float, engine->width,
+ engine->height,
+ [interpolator depthTextureUsage] | MTLTextureUsageShaderWrite, 0);
+ engine->motion = fg_make_texture(engine->device, MTLPixelFormatRG16Float, engine->width,
+ engine->height,
+ [interpolator motionTextureUsage] | MTLTextureUsageShaderWrite, 0);
+ engine->out = fg_make_texture(engine->device, engine->format, engine->width, engine->height,
+ [interpolator outputTextureUsage] | MTLTextureUsageShaderRead, 0);
+ if (engine->depth == nil || engine->motion == nil || engine->out == nil)
+ return 0;
+
+ id<MTLCommandQueue> queue = [engine->device newCommandQueue];
+ id<MTLCommandBuffer> buffer = [queue commandBuffer];
+ fg_fill_constant(engine, buffer, engine->depth, 0.5f);
+ fg_fill_constant(engine, buffer, engine->motion, 0.0f);
+ [buffer commit];
+ [buffer waitUntilCompleted];
+ int ok = buffer.error == nil;
+ [queue release];
+ engine->reset_pending = 1;
+ return ok;
+}
+
+// History textures need the usage both engines ask for
+static MTLTextureUsage fg_history_usage(void)
+{
+ return MTLTextureUsageShaderRead;
+}
+
+static int fg_engine_create(struct fg_engine *engine, id<MTLDevice> device, NSUInteger width,
+ NSUInteger height, NSUInteger format, int pref)
+{
+ memset(engine, 0, sizeof(*engine));
+ engine->device = [device retain];
+ engine->width = width;
+ engine->height = height;
+ engine->format = format;
+
+ // The own path's kernels are needed for the constant fills too
+ if (!fg_engine_build_own(engine)) {
+ fg_engine_destroy(engine);
+ return 0;
+ }
+ engine->kind = 1;
+
+ // MetalFX only on request: it needs our motion vectors anyway and costs
+ // 2-3x the in-house path (measured 2.6-4.5 ms against 1.4 ms at 3024x1964)
+ if (pref == 2 && fg_engine_build_metalfx(engine)) {
+ engine->kind = 2;
+ } else {
+ fg_release(&engine->interpolator);
+ fg_release((id *)&engine->depth);
+ fg_release((id *)&engine->motion);
+ if (pref == 2)
+ shim_log("overlay_shim framegen: metalfx requested but unavailable, using own\n");
+ fg_release((id *)&engine->out);
+ engine->out = fg_make_texture(device, format, width, height,
+ MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite, 0);
+ if (engine->out == nil) {
+ fg_engine_destroy(engine);
+ return 0;
+ }
+ }
+ return 1;
+}
+
+// Call once per real frame before interpolating against it (own engine)
+static void fg_engine_ingest(struct fg_engine *engine, id<MTLCommandBuffer> buffer,
+ id<MTLTexture> frame, int slot)
+{
+ id<MTLComputeCommandEncoder> encoder = [buffer computeCommandEncoder];
+ [encoder setTexture:frame atIndex:0];
+ [encoder setTexture:engine->luma_view[slot][0] atIndex:1];
+ fg_dispatch(encoder, engine->ps_luma, engine->luma[slot].width, engine->luma[slot].height);
+ [encoder endEncoding];
+
+ id<MTLBlitCommandEncoder> blit = [buffer blitCommandEncoder];
+ [blit generateMipmapsForTexture:engine->luma[slot]];
+ [blit endEncoding];
+ engine->luma_valid[slot] = 1;
+}
+
+static void fg_flow_pass(struct fg_engine *engine, id<MTLComputeCommandEncoder> encoder,
+ int from, int to, id<MTLTexture> *flows)
+{
+ for (int l = fg_levels - 1; l >= 0; l--) {
+ [encoder setTexture:engine->luma_view[from][l] atIndex:0];
+ [encoder setTexture:engine->luma_view[to][l] atIndex:1];
+ [encoder setTexture:l == fg_levels - 1 ? nil : flows[l + 1] atIndex:2];
+ [encoder setTexture:flows[l] atIndex:3];
+ fg_dispatch(encoder,
+ l == fg_levels - 1 ? engine->ps_flow3 : l == 0 ? engine->ps_flow1f : engine->ps_flow1,
+ flows[l].width, flows[l].height);
+ }
+}
+
+// Encodes the frame between prev (slot prev_slot) and cur (slot cur_slot) into engine->out
+static void fg_engine_interpolate(struct fg_engine *engine, id<MTLCommandBuffer> buffer,
+ id<MTLTexture> prev, int prev_slot, id<MTLTexture> cur,
+ int cur_slot, float delta)
+{
+ if (!engine->luma_valid[prev_slot])
+ fg_engine_ingest(engine, buffer, prev, prev_slot);
+ if (!engine->luma_valid[cur_slot])
+ fg_engine_ingest(engine, buffer, cur, cur_slot);
+
+ if (engine->kind == 2) {
+ // MetalFX does not estimate motion; it gets ours, new frame to previous
+ id<MTLComputeCommandEncoder> encoder = [buffer computeCommandEncoder];
+ fg_flow_pass(engine, encoder, cur_slot, prev_slot, engine->flow_ba);
+ [encoder setTexture:engine->flow_ba[0] atIndex:0];
+ [encoder setTexture:engine->motion atIndex:1];
+ fg_dispatch(encoder, engine->ps_motion, engine->width, engine->height);
+ [encoder endEncoding];
+
+ id<np_fx_interpolator> interpolator = engine->interpolator;
+ [interpolator setColorTexture:cur];
+ [interpolator setPrevColorTexture:prev];
+ [interpolator setDepthTexture:engine->depth];
+ [interpolator setMotionTexture:engine->motion];
+ [interpolator setOutputTexture:engine->out];
+ [interpolator setMotionVectorScaleX:1.0f];
+ [interpolator setMotionVectorScaleY:1.0f];
+ [interpolator setJitterOffsetX:0.0f];
+ [interpolator setJitterOffsetY:0.0f];
+ [interpolator setDeltaTime:delta > 0 ? delta : 1.0f / 60.0f];
+ [interpolator setNearPlane:0.1f];
+ [interpolator setFarPlane:1000.0f];
+ [interpolator setFieldOfView:60.0f];
+ [interpolator setAspectRatio:(float)engine->width / (float)engine->height];
+ [interpolator setDepthReversed:YES];
+ [interpolator setShouldResetHistory:engine->reset_pending ? YES : NO];
+ if (engine->reset_pending)
+ engine->warm = 0;
+ engine->reset_pending = 0;
+ engine->warm++;
+ [interpolator encodeToCommandBuffer:buffer];
+ return;
+ }
+
+ id<MTLComputeCommandEncoder> encoder = [buffer computeCommandEncoder];
+ fg_flow_pass(engine, encoder, prev_slot, cur_slot, engine->flow_ab);
+ fg_flow_pass(engine, encoder, cur_slot, prev_slot, engine->flow_ba);
+ [encoder setTexture:prev atIndex:0];
+ [encoder setTexture:cur atIndex:1];
+ [encoder setTexture:engine->flow_ab[0] atIndex:2];
+ [encoder setTexture:engine->flow_ba[0] atIndex:3];
+ [encoder setTexture:engine->out atIndex:4];
+ fg_dispatch(encoder, engine->ps_synth, engine->width, engine->height);
+ [encoder endEncoding];
+}
+
+static struct fg_engine fg_engine;
+
+// Test entry: interpolate a and b into out synchronously. engine 1 own,
+// 2 metalfx, 0 auto. Returns the engine used, or 0 on failure.
+__attribute__((visibility("default"))) int np_framegen_interpolate_sync(id device, id a, id b,
+ id out, int engine_pref,
+ double *gpu_ms)
+{
+ struct fg_engine engine;
+ id<MTLTexture> ta = a, tb = b, tout = out;
+
+ if (!fg_engine_create(&engine, device, ta.width, ta.height, ta.pixelFormat, engine_pref))
+ return 0;
+ if (engine_pref == 2 && engine.kind != 2) {
+ fg_engine_destroy(&engine);
+ return 0;
+ }
+
+ id<MTLCommandQueue> queue = [(id<MTLDevice>)device newCommandQueue];
+ int kind = engine.kind;
+ double total = 0;
+ int runs = gpu_ms != NULL || kind == 2 ? 6 : 1;
+ for (int i = 0; i < runs; i++) {
+ id<MTLCommandBuffer> buffer = [queue commandBuffer];
+ engine.luma_valid[0] = engine.luma_valid[1] = 0;
+ fg_engine_interpolate(&engine, buffer, ta, 0, tb, 1, 1.0f / 60.0f);
+ id<MTLBlitCommandEncoder> blit = [buffer blitCommandEncoder];
+ [blit copyFromTexture:engine.out toTexture:tout];
+ [blit endEncoding];
+ [buffer commit];
+ [buffer waitUntilCompleted];
+ if (buffer.error != nil)
+ kind = 0;
+ if (i > 0 || runs == 1)
+ total += buffer.GPUEndTime - buffer.GPUStartTime;
+ }
+ if (gpu_ms != NULL)
+ *gpu_ms = total / (runs > 1 ? runs - 1 : 1) * 1000.0;
+ [queue release];
+ fg_engine_destroy(&engine);
+ return kind;
+}
+
+// ---------------------------------------------------------------------------
+// Overlay layer and worker
+
+static double fg_display_interval(void)
+{
+ typedef id (*screen_fn)(id, SEL);
+ typedef NSInteger (*fps_fn)(id, SEL);
+
+ Class screen_class = objc_getClass("NSScreen");
+ if (screen_class != Nil) {
+ id screen = ((screen_fn)objc_msgSend)((id)screen_class, sel_registerName("mainScreen"));
+ SEL fps_sel = sel_registerName("maximumFramesPerSecond");
+ if (screen != nil && [screen respondsToSelector:fps_sel]) {
+ NSInteger fps = ((fps_fn)objc_msgSend)(screen, fps_sel);
+ if (fps > 0)
+ return 1.0 / (double)fps;
+ }
+ }
+
+ CGDisplayModeRef mode = CGDisplayCopyDisplayMode(CGMainDisplayID());
+ double rate = mode != NULL ? CGDisplayModeGetRefreshRate(mode) : 0;
+ if (mode != NULL)
+ CGDisplayModeRelease(mode);
+ return rate > 0 ? 1.0 / rate : 1.0 / 60.0;
+}
+
+static void fg_layer_set_hidden(int hidden)
+{
+ if (fg_layer == nil)
+ return;
+ [CATransaction begin];
+ [CATransaction setDisableActions:YES];
+ fg_layer.hidden = hidden ? YES : NO;
+ [CATransaction commit];
+}
+
+static void fg_layer_sync_geometry(CAMetalLayer *game)
+{
+ CGRect bounds = game.bounds;
+ if (!CGRectEqualToRect(fg_layer.frame, bounds) || fg_layer.contentsScale != game.contentsScale) {
+ [CATransaction begin];
+ [CATransaction setDisableActions:YES];
+ fg_layer.frame = bounds;
+ fg_layer.contentsScale = game.contentsScale;
+ [CATransaction commit];
+ }
+}
+
+static int fg_layer_create(CAMetalLayer *game, NSUInteger width, NSUInteger height, NSUInteger format)
+{
+ CAMetalLayer *layer = [[CAMetalLayer alloc] init];
+ if (layer == nil)
+ return 0;
+
+ [CATransaction begin];
+ [CATransaction setDisableActions:YES];
+ layer.device = fg_device;
+ layer.pixelFormat = (MTLPixelFormat)format;
+ layer.colorspace = game.colorspace;
+ layer.framebufferOnly = NO;
+ layer.maximumDrawableCount = 3;
+ layer.presentsWithTransaction = NO;
+ layer.displaySyncEnabled = game.displaySyncEnabled;
+ layer.opaque = YES;
+ layer.contentsGravity = game.contentsGravity;
+ layer.hidden = YES;
+ layer.zPosition = 1000;
+ if ([layer respondsToSelector:@selector(setWantsExtendedDynamicRangeContent:)])
+ layer.wantsExtendedDynamicRangeContent = game.wantsExtendedDynamicRangeContent;
+ if ([layer respondsToSelector:@selector(setEDRMetadata:)] && game.EDRMetadata != nil)
+ layer.EDRMetadata = game.EDRMetadata;
+ layer.contentsScale = game.contentsScale;
+ layer.frame = game.bounds;
+ layer.drawableSize = CGSizeMake(width, height);
+ [game addSublayer:layer];
+ [CATransaction commit];
+
+ fg_layer = layer;
+ return 1;
+}
+
+static void fg_layer_destroy(void)
+{
+ if (fg_layer == nil)
+ return;
+ [CATransaction begin];
+ [CATransaction setDisableActions:YES];
+ [fg_layer removeFromSuperlayer];
+ [CATransaction commit];
+ [fg_layer release];
+ fg_layer = nil;
+}
+
+// Worker side: tear down what the previous configuration built
+static void fg_resources_destroy(void)
+{
+ fg_engine_destroy(&fg_engine);
+ fg_engine_kind = 0;
+ for (int i = 0; i < 2; i++)
+ fg_release((id *)&fg_hist[i]);
+ fg_layer_destroy();
+}
+
+static void fg_configure(CAMetalLayer *game, NSUInteger width, NSUInteger height, NSUInteger format,
+ unsigned generation)
+{
+ double start = fg_now();
+
+ fg_resources_destroy();
+
+ int ok = 1;
+ for (int i = 0; i < 2 && ok; i++) {
+ fg_hist[i] = fg_make_texture(fg_device, format, width, height, fg_history_usage(), 0);
+ ok = fg_hist[i] != nil;
+ }
+ if (ok)
+ ok = fg_engine_create(&fg_engine, fg_device, width, height, format, framegen_engine_pref);
+ if (ok)
+ ok = fg_layer_create(game, width, height, format);
+
+ os_unfair_lock_lock(&fg_lock);
+ if (atomic_load(&fg_generation) != generation) {
+ // Reconfigured again meanwhile; the next configure replaces this
+ os_unfair_lock_unlock(&fg_lock);
+ return;
+ }
+ fg_width = width;
+ fg_height = height;
+ fg_format = format;
+ fg_hist_valid = 0;
+ fg_hist_next = 0;
+ fg_engine_kind = fg_engine.kind;
+ fg_refresh_interval = fg_display_interval();
+ fg_slow_frames = 0;
+ atomic_store(&fg_mode, ok ? fg_mode_passthrough : fg_mode_idle);
+ os_unfair_lock_unlock(&fg_lock);
+
+ if (!ok) {
+ fg_resources_destroy();
+ shim_log("overlay_shim framegen: setup failed for %lux%lu %s, framegen off for this layer\n",
+ (unsigned long)width, (unsigned long)height, format_name(format));
+ return;
+ }
+
+ shim_log("overlay_shim framegen: ready %lux%lu %s engine=%s refresh=%.0fHz in %.0f ms\n",
+ (unsigned long)width, (unsigned long)height, format_name(format),
+ fg_engine_name(fg_engine.kind), 1.0 / fg_refresh_interval, (fg_now() - start) * 1000.0);
+}
+
+struct fg_job {
+ int slot; // history slot holding this frame
+ uint64_t value; // event value signalled after the blit
+ double arrived; // game present time
+ double interval;
+ int interpolate;
+ int reset; // first pair after a history reset
+ int first; // unhide the overlay layer once the real copy is presented
+ unsigned generation;
+};
+
+static void fg_note_gpu(id<MTLCommandBuffer> buffer)
+{
+ double ms = (buffer.GPUEndTime - buffer.GPUStartTime) * 1000.0;
+ if (ms <= 0 || ms > 1000)
+ return;
+ os_unfair_lock_lock(&fg_lock);
+ fg_stat_gpu_ms += ms;
+ fg_stat_gpu_n++;
+ if (ms > fg_stat_gpu_max_ms)
+ fg_stat_gpu_max_ms = ms;
+ os_unfair_lock_unlock(&fg_lock);
+}
+
+static void fg_log_stats(double now)
+{
+ if (!framegen_stats_log)
+ return;
+
+ os_unfair_lock_lock(&fg_lock);
+ if (fg_stat_start == 0)
+ fg_stat_start = now;
+ double span = now - fg_stat_start;
+ if (span < 1.0) {
+ os_unfair_lock_unlock(&fg_lock);
+ return;
+ }
+ char skips[160];
+ int used = 0;
+ skips[0] = '\0';
+ for (int i = 0; i < fg_skip_count; i++) {
+ if (fg_stat_skips[i] == 0)
+ continue;
+ used += snprintf(skips + used, sizeof(skips) - (size_t)used, " %s=%d", fg_skip_names[i],
+ fg_stat_skips[i]);
+ if (used >= (int)sizeof(skips))
+ break;
+ }
+ shim_log("NP framegen stats: real=%.1f fps generated=%.1f fps gpu=%.2f ms avg %.2f ms max "
+ "engine=%s mode=%d lead=%.1f ms skips:%s\n",
+ fg_stat_real / span, fg_stat_generated / span,
+ fg_stat_gpu_n > 0 ? fg_stat_gpu_ms / fg_stat_gpu_n : 0.0, fg_stat_gpu_max_ms,
+ fg_engine_name(fg_engine_kind), atomic_load(&fg_mode), fg_lead * 1000.0,
+ skips[0] != '\0' ? skips : " none");
+ fg_stat_start = now;
+ fg_stat_real = fg_stat_generated = 0;
+ fg_stat_gpu_ms = fg_stat_gpu_max_ms = 0;
+ fg_stat_gpu_n = 0;
+ memset(fg_stat_skips, 0, sizeof(fg_stat_skips));
+ os_unfair_lock_unlock(&fg_lock);
+}
+
+// Per-second stats, driven from the game's present path
+static void framegen_tick(void)
+{
+ if (framegen_enabled && framegen_stats_log)
+ fg_log_stats(fg_now());
+}
+
+// Releases the game's drawable, no earlier than not_before: a game that asked
+// for presentDrawable:afterMinimumDuration: relies on the drawable staying in
+// flight that long to pace itself.
+static void fg_release_drawable_at(id drawable, double not_before)
+{
+ if (drawable == nil)
+ return;
+
+ double wait = not_before - fg_now();
+ if (wait > 0.0005) {
+ dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(wait * 1e9)), fg_worker, ^{
+ [drawable release];
+ });
+ } else {
+ [drawable release];
+ }
+}
+
+static void fg_run_job(struct fg_job *job)
+{
+ fg_internal = 1;
+
+ if (job->generation != atomic_load(&fg_generation) || fg_layer == nil ||
+ atomic_load(&fg_mode) != fg_mode_active) {
+ fg_internal = 0;
+ return;
+ }
+
+ fg_layer_sync_geometry((CAMetalLayer *)fg_game_layer);
+
+ int prev_slot = job->slot ^ 1;
+ double lead = fg_lead;
+ if (lead < 0.003)
+ lead = 0.003;
+ if (lead > job->interval * 0.5)
+ lead = job->interval * 0.5;
+ double target_generated = job->arrived + lead;
+ double target_real = target_generated + job->interval * 0.5;
+ int failed = 0;
+
+ if (job->interpolate && fg_engine.kind == 2 &&
+ (job->reset || fg_engine.warm < fg_metalfx_warmup)) {
+ // MetalFX echoes the new frame for two encodes after a reset: run them
+ // to warm it up, show nothing extra
+ id<MTLCommandBuffer> buffer = [fg_queue commandBuffer];
+ [buffer encodeWaitForEvent:fg_event value:job->value];
+ fg_engine.luma_valid[job->slot] = 0;
+ if (job->reset) {
+ fg_engine.luma_valid[prev_slot] = 0;
+ fg_engine.reset_pending = 1;
+ }
+ fg_engine_interpolate(&fg_engine, buffer, fg_hist[prev_slot], prev_slot,
+ fg_hist[job->slot], job->slot, (float)job->interval);
+ [buffer commit];
+ job->interpolate = 0;
+ job->reset = 0;
+ fg_stat_skip(fg_skip_warmup);
+ target_real = job->arrived + lead;
+ } else if (job->interpolate) {
+ id<CAMetalDrawable> drawable = [fg_layer nextDrawable];
+ if (drawable == nil) {
+ fg_stat_skip(fg_skip_no_drawable);
+ } else {
+ id<MTLCommandBuffer> buffer = [fg_queue commandBuffer];
+ [buffer encodeWaitForEvent:fg_event value:job->value];
+ fg_engine.luma_valid[job->slot] = 0;
+ if (job->reset) {
+ fg_engine.luma_valid[prev_slot] = 0;
+ fg_engine.reset_pending = 1;
+ }
+ fg_engine_interpolate(&fg_engine, buffer, fg_hist[prev_slot], prev_slot,
+ fg_hist[job->slot], job->slot, (float)job->interval);
+ id<MTLBlitCommandEncoder> blit = [buffer blitCommandEncoder];
+ [blit copyFromTexture:fg_engine.out toTexture:drawable.texture];
+ [blit endEncoding];
+ [buffer presentDrawable:drawable atTime:target_generated];
+ double started = fg_now();
+ [buffer addCompletedHandler:^(id<MTLCommandBuffer> done) {
+ fg_note_gpu(done);
+ // Submission to GPU completion, in the CACurrentMediaTime domain
+ double latency = done.GPUEndTime - started + 0.001;
+ if (latency < 0.001 || latency > 0.25)
+ latency = 0.003;
+ os_unfair_lock_lock(&fg_lock);
+ fg_lead = fg_lead == 0 ? latency : fg_lead * 0.8 + latency * 0.2;
+ if (done.error == nil) {
+ fg_stat_generated++;
+ atomic_fetch_add(&fg_total_generated, 1);
+ } else
+ fg_stat_skips[fg_skip_error]++;
+ os_unfair_lock_unlock(&fg_lock);
+ }];
+ [buffer commit];
+ }
+ } else {
+ target_real = job->arrived + 0.0005;
+ }
+
+ id<CAMetalDrawable> real = [fg_layer nextDrawable];
+ if (real == nil) {
+ fg_stat_skip(fg_skip_no_drawable);
+ failed = 1;
+ } else {
+ id<MTLCommandBuffer> buffer = [fg_queue commandBuffer];
+ if (!job->interpolate)
+ [buffer encodeWaitForEvent:fg_event value:job->value];
+ id<MTLBlitCommandEncoder> blit = [buffer blitCommandEncoder];
+ [blit copyFromTexture:fg_hist[job->slot] toTexture:real.texture];
+ [blit endEncoding];
+ [buffer presentDrawable:real atTime:target_real];
+ atomic_fetch_add(&fg_total_real_copies, 1);
+ int first = job->first;
+ if (first) {
+ [buffer addScheduledHandler:^(id<MTLCommandBuffer> done) {
+ (void)done;
+ fg_layer_set_hidden(0);
+ }];
+ }
+ [buffer commit];
+ }
+
+ fg_internal = 0;
+
+ if (failed && atomic_load(&fg_mode) == fg_mode_active) {
+ // The overlay layer cannot keep up; hand presentation back to the game
+ atomic_store(&fg_mode, fg_mode_passthrough);
+ fg_layer_set_hidden(1);
+ os_unfair_lock_lock(&fg_lock);
+ fg_hist_valid = 0;
+ fg_slow_frames = 0;
+ os_unfair_lock_unlock(&fg_lock);
+ }
+}
+
+static void fg_enqueue_job(struct fg_job job)
+{
+ atomic_fetch_add(&fg_inflight, 1);
+ dispatch_async(fg_worker, ^{
+ struct fg_job local = job;
+ fg_run_job(&local);
+ atomic_fetch_sub(&fg_inflight, 1);
+ });
+}
+
+// ---------------------------------------------------------------------------
+// Game-thread side
+
+static void framegen_init(void)
+{
+ const char *value = getenv("NOTPROTON_FRAMEGEN");
+ if (value == NULL || value[0] == '\0' || value[0] == '0')
+ return;
+
+ framegen_multiplier = atoi(value);
+ if (framegen_multiplier < 1)
+ return;
+ framegen_enabled = 1;
+
+ const char *engine = getenv("NOTPROTON_FRAMEGEN_ENGINE");
+ if (engine != NULL && strcmp(engine, "own") == 0)
+ framegen_engine_pref = 1;
+ else if (engine != NULL && strcmp(engine, "metalfx") == 0)
+ framegen_engine_pref = 2;
+
+ const char *log = getenv("OVERLAY_SHIM_LOG");
+ framegen_stats_log = log != NULL && log[0] != '\0' && log[0] != '0';
+
+ dispatch_queue_attr_t attr = dispatch_queue_attr_make_with_qos_class(
+ DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, 0);
+ fg_worker = dispatch_queue_create("com.notproton.framegen", attr);
+
+ shim_log("overlay_shim framegen: enabled x%d engine=%s stats=%s\n",
+ framegen_multiplier > 2 ? 2 : framegen_multiplier,
+ framegen_engine_pref == 1 ? "own" : framegen_engine_pref == 2 ? "metalfx" : "auto",
+ framegen_stats_log ? "on" : "off");
+ if (framegen_multiplier > 2)
+ shim_log("overlay_shim framegen: multiplier %d not available yet, generating one frame\n",
+ framegen_multiplier);
+}
+
+static struct fg_pending *fg_pending_find(id buffer, int create)
+{
+ struct fg_pending *free_slot = NULL;
+
+ for (int i = 0; i < fg_max_pending; i++) {
+ if (fg_pending[i].buffer == buffer)
+ return &fg_pending[i];
+ if (fg_pending[i].buffer == nil && free_slot == NULL)
+ free_slot = &fg_pending[i];
+ }
+ if (!create || free_slot == NULL)
+ return NULL;
+ memset(free_slot, 0, sizeof(*free_slot));
+ free_slot->buffer = buffer;
+ return free_slot;
+}
+
+static void fg_pending_clear(struct fg_pending *entry)
+{
+ fg_release(&entry->drawable);
+ memset(entry, 0, sizeof(*entry));
+}
+
+static void framegen_note_drawable(id layer, id drawable)
+{
+ (void)layer;
+ if (!framegen_enabled || fg_internal || drawable == nil)
+ return;
+ os_unfair_lock_lock(&fg_lock);
+ fg_last_vended_texture = [(id<CAMetalDrawable>)drawable texture];
+ os_unfair_lock_unlock(&fg_lock);
+}
+
+static void framegen_note_render_pass(id buffer, id attachment)
+{
+ if (!framegen_enabled || fg_internal || attachment == nil ||
+ atomic_load(&fg_mode) != fg_mode_active)
+ return;
+ os_unfair_lock_lock(&fg_lock);
+ if (attachment == fg_last_vended_texture) {
+ struct fg_pending *entry = fg_pending_find(buffer, 1);
+ if (entry != NULL && entry->texture == nil)
+ entry->texture = attachment;
+ }
+ os_unfair_lock_unlock(&fg_lock);
+}
+
+// Takes a layer as the one framegen follows. Lock held by caller. Returns 0
+// when the device offers no queue or event.
+static int fg_adopt(CAMetalLayer *layer, id<MTLTexture> texture)
+{
+ id<MTLDevice> device = texture.device;
+
+ if (fg_device != device) {
+ fg_release((id *)&fg_queue);
+ fg_release((id *)&fg_event);
+ fg_release((id *)&fg_device);
+ fg_device = [device retain];
+ fg_queue = [fg_device newCommandQueue];
+ fg_event = [fg_device newEvent];
+ fg_event_value = 0;
+ }
+ if (fg_queue == nil || fg_event == nil) {
+ shim_log("overlay_shim framegen: no queue or event on %s, framegen off\n",
+ [[fg_device name] UTF8String]);
+ return 0;
+ }
+ fg_release(&fg_game_layer);
+ fg_game_layer = [layer retain];
+ if (layer.framebufferOnly)
+ layer.framebufferOnly = NO;
+ fg_last_present_time = 0;
+ fg_interval = 0;
+ fg_blit_texture = nil;
+ shim_log("overlay_shim framegen: following layer %p on %s\n", (void *)layer,
+ [[fg_device name] UTF8String]);
+ return 1;
+}
+
+// Decides what to do with a real frame. Returns 1 to divert it. Holds no lock
+// on return; *interpolate is set for the job.
+static int fg_decide(id drawable, int *interpolate)
+{
+ id<CAMetalDrawable> d = drawable;
+ id<MTLTexture> texture = d.texture;
+ CAMetalLayer *layer = d.layer;
+ double now = fg_now();
+
+ if (texture == nil || layer == nil)
+ return 0;
+
+ framegen_tick();
+ os_unfair_lock_lock(&fg_lock);
+
+ if (layer != fg_game_layer) {
+ // A second layer takes over once the followed one has gone quiet
+ if (fg_game_layer != nil && now - fg_last_present_time < 0.5) {
+ fg_stat_skips[fg_skip_other_layer]++;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ int was_active = atomic_load(&fg_mode) == fg_mode_active;
+ atomic_fetch_add(&fg_generation, 1);
+ atomic_store(&fg_mode, fg_mode_idle);
+ fg_hist_valid = 0;
+ if (!fg_adopt(layer, texture)) {
+ os_unfair_lock_unlock(&fg_lock);
+ framegen_enabled = 0;
+ return 0;
+ }
+ fg_width = fg_height = 0;
+ os_unfair_lock_unlock(&fg_lock);
+ if (was_active)
+ fg_layer_set_hidden(1);
+ os_unfair_lock_lock(&fg_lock);
+ }
+
+ // Real-frame timing
+ double interval = fg_interval;
+ if (fg_last_present_time > 0) {
+ double gap = now - fg_last_present_time;
+ if (gap > 0.25 || gap < 0.0005) {
+ interval = 0; // a stall or a burst; start over
+ } else {
+ interval = interval == 0 ? gap : interval * 0.8 + gap * 0.2;
+ }
+ }
+ fg_last_present_time = now;
+ fg_interval = interval;
+ fg_stat_real++;
+
+ int mode = atomic_load(&fg_mode);
+ int mismatch = texture.width != fg_width || texture.height != fg_height ||
+ texture.pixelFormat != fg_format;
+
+ if (mode == fg_mode_idle || (mismatch && mode != fg_mode_configuring)) {
+ if (texture.framebufferOnly) {
+ // Cannot copy out of this drawable; the layer is set to stop making them
+ fg_stat_skips[fg_skip_mismatch]++;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ unsigned generation = atomic_fetch_add(&fg_generation, 1) + 1;
+ NSUInteger width = texture.width, height = texture.height, format = texture.pixelFormat;
+ atomic_store(&fg_mode, fg_mode_configuring);
+ fg_hist_valid = 0;
+ fg_blit_texture = nil;
+ os_unfair_lock_unlock(&fg_lock);
+ if (mode == fg_mode_active)
+ fg_layer_set_hidden(1);
+ fg_stat_skip(fg_skip_mismatch);
+ dispatch_async(fg_worker, ^{
+ fg_internal = 1;
+ fg_configure(layer, width, height, format, generation);
+ fg_internal = 0;
+ });
+ return 0;
+ }
+
+ if (mode == fg_mode_configuring) {
+ fg_stat_skips[fg_skip_mismatch]++;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+
+ // Near the refresh rate already there is nothing to add; generation starts
+ // below ~77% of it and stops above ~87%, so a few quick frames do not flap.
+ int fast = interval == 0 || interval < fg_refresh_interval * 1.15;
+ int slow = interval > fg_refresh_interval * 1.3;
+ if (fast) {
+ fg_slow_frames = 0;
+ if (mode == fg_mode_active) {
+ atomic_store(&fg_mode, fg_mode_passthrough);
+ fg_hist_valid = 0;
+ os_unfair_lock_unlock(&fg_lock);
+ fg_layer_set_hidden(1);
+ fg_stat_skip(fg_skip_fast);
+ return 0;
+ }
+ fg_stat_skips[fg_skip_fast]++;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ if (mode == fg_mode_passthrough) {
+ if (atomic_load(&fg_inflight) > 0) {
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ fg_slow_frames = slow ? fg_slow_frames + 1 : 0;
+ if (fg_slow_frames < 30) {
+ fg_stat_skips[fg_skip_settling]++;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ atomic_store(&fg_mode, fg_mode_active);
+ fg_hist_valid = 0;
+ fg_unhide_pending = 1;
+ }
+
+ int inflight = atomic_load(&fg_inflight);
+ if (inflight >= 4) {
+ atomic_store(&fg_mode, fg_mode_passthrough);
+ fg_hist_valid = 0;
+ fg_slow_frames = 0;
+ os_unfair_lock_unlock(&fg_lock);
+ fg_layer_set_hidden(1);
+ fg_stat_skip(fg_skip_lag);
+ return 0;
+ }
+ *interpolate = fg_hist_valid >= 1 && inflight < 2 && interval > 0;
+ if (!*interpolate && fg_hist_valid >= 1 && interval > 0)
+ fg_stat_skips[fg_skip_lag]++;
+ os_unfair_lock_unlock(&fg_lock);
+ return 1;
+}
+
+// Encodes the history copy for texture on buffer. Lock held by caller.
+static void fg_encode_copy(id<MTLCommandBuffer> buffer, id<MTLTexture> texture, double now)
+{
+ int slot;
+ if (fg_blit_texture == texture && fg_blit_texture != nil) {
+ slot = fg_blit_slot; // another buffer into the same frame: overwrite
+ } else {
+ slot = fg_hist_next;
+ fg_hist_next ^= 1;
+ }
+ if (fg_hist[slot] == nil || texture.width != fg_hist[slot].width ||
+ texture.height != fg_hist[slot].height || texture.pixelFormat != fg_hist[slot].pixelFormat)
+ return;
+
+ id<MTLBlitCommandEncoder> blit = [buffer blitCommandEncoder];
+ [blit copyFromTexture:texture toTexture:fg_hist[slot]];
+ [blit endEncoding];
+ [buffer encodeSignalEvent:fg_event value:++fg_event_value];
+ fg_blit_texture = texture;
+ fg_blit_value = fg_event_value;
+ fg_blit_slot = slot;
+ fg_blit_time = now;
+}
+
+// After the copy is encoded: hand the frame to the worker. Lock held by caller.
+static void fg_launch(int interpolate)
+{
+ struct fg_job job;
+ memset(&job, 0, sizeof(job));
+ job.slot = fg_blit_slot;
+ job.value = fg_blit_value;
+ job.arrived = fg_blit_time;
+ job.interval = fg_interval;
+ job.interpolate = interpolate && fg_hist_valid >= 1;
+ job.reset = fg_hist_valid < 2;
+ job.first = fg_unhide_pending;
+ fg_unhide_pending = 0;
+ job.generation = atomic_load(&fg_generation);
+ if (fg_hist_valid < 2)
+ fg_hist_valid++;
+ fg_blit_texture = nil;
+ fg_enqueue_job(job);
+}
+
+// Command buffer present hooks. These sit under the overlay's own swizzles, so
+// the overlay has drawn by the time they run.
+typedef void (*present_drawable_fn)(id, SEL, id);
+typedef void (*present_drawable_timed_fn)(id, SEL, id, CFTimeInterval);
+
+static present_drawable_fn next_present_drawable;
+static present_drawable_timed_fn next_present_drawable_after;
+static present_drawable_timed_fn next_present_drawable_at;
+static void (*next_commit)(id, SEL);
+
+static int fg_divert_present(id buffer, id drawable, double min_duration)
+{
+ if (!framegen_enabled || fg_internal || drawable == nil)
+ return 0;
+ if ([(id<CAMetalDrawable>)drawable layer] == fg_layer)
+ return 0;
+
+ int interpolate = 0;
+ int divert = fg_decide(drawable, &interpolate);
+
+ os_unfair_lock_lock(&fg_lock);
+ if (!divert) {
+ // The original present follows; the drawable-level hook must not count it again
+ fg_passthrough_drawable = drawable;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ struct fg_pending *entry = fg_pending_find(buffer, 1);
+ if (entry == NULL) {
+ fg_passthrough_drawable = drawable;
+ os_unfair_lock_unlock(&fg_lock);
+ fg_stat_skip(fg_skip_error);
+ return 0;
+ }
+ fg_release(&entry->drawable);
+ entry->drawable = [drawable retain];
+ entry->texture = [(id<CAMetalDrawable>)drawable texture];
+ entry->divert = 1 | (interpolate ? 2 : 0);
+ entry->min_duration = min_duration;
+ os_unfair_lock_unlock(&fg_lock);
+ return 1;
+}
+
+static void hook_present_drawable(id self, SEL selector, id drawable)
+{
+ if (fg_divert_present(self, drawable, 0))
+ return;
+ next_present_drawable(self, selector, drawable);
+}
+
+static void hook_present_drawable_after(id self, SEL selector, id drawable, CFTimeInterval duration)
+{
+ if (fg_divert_present(self, drawable, duration))
+ return;
+ next_present_drawable_after(self, selector, drawable, duration);
+}
+
+static void hook_present_drawable_at(id self, SEL selector, id drawable, CFTimeInterval time)
+{
+ if (fg_divert_present(self, drawable, 0))
+ return;
+ next_present_drawable_at(self, selector, drawable, time);
+}
+
+static void hook_commit(id self, SEL selector)
+{
+ if (!fg_internal && framegen_enabled && atomic_load(&fg_mode) == fg_mode_active) {
+ os_unfair_lock_lock(&fg_lock);
+ struct fg_pending *entry = fg_pending_find(self, 0);
+ if (entry != NULL) {
+ struct fg_pending local = *entry;
+ memset(entry, 0, sizeof(*entry));
+ if (local.texture != nil && fg_hist[0] != nil)
+ fg_encode_copy(self, local.texture, fg_now());
+ double arrived = fg_blit_time;
+ if (local.divert && fg_blit_texture == local.texture)
+ fg_launch((local.divert & 2) != 0);
+ else if (local.divert)
+ fg_stat_skips[fg_skip_error]++; // copy refused, frame lost
+ os_unfair_lock_unlock(&fg_lock);
+ if (local.drawable != nil) {
+ // The copy reads the drawable on this buffer (the game's queue,
+ // or the Metal 4 bridge queue): keep it out of the pool until
+ // the buffer is done, and as long as the game's frame cap asked
+ id held = local.drawable;
+ double hold_until = local.min_duration > 0 ? arrived + local.min_duration : 0;
+ [self addCompletedHandler:^(id<MTLCommandBuffer> done) {
+ (void)done;
+ fg_release_drawable_at(held, hold_until);
+ }];
+ }
+ } else {
+ os_unfair_lock_unlock(&fg_lock);
+ }
+ } else if (!fg_internal && framegen_enabled) {
+ os_unfair_lock_lock(&fg_lock);
+ struct fg_pending *entry = fg_pending_find(self, 0);
+ if (entry != NULL)
+ fg_pending_clear(entry);
+ os_unfair_lock_unlock(&fg_lock);
+ }
+ next_commit(self, selector);
+}
+
+// A drawable presented on its own ([drawable present], no buffer named). Only
+// divertable when a copy was already encoded for it at a commit.
+static int framegen_divert_plain_present(id drawable, double min_duration)
+{
+ if (!framegen_enabled || fg_internal || drawable == nil)
+ return 0;
+ if ([(id<CAMetalDrawable>)drawable layer] == fg_layer)
+ return 0;
+
+ id<MTLTexture> texture = [(id<CAMetalDrawable>)drawable texture];
+ os_unfair_lock_lock(&fg_lock);
+ if (fg_passthrough_drawable == drawable) {
+ fg_passthrough_drawable = nil;
+ os_unfair_lock_unlock(&fg_lock);
+ return 0;
+ }
+ int ready = fg_blit_texture == texture && texture != nil;
+ os_unfair_lock_unlock(&fg_lock);
+
+ int interpolate = 0;
+ if (!fg_decide(drawable, &interpolate))
+ return 0;
+ if (!ready) {
+ static int logged;
+ if (!logged) {
+ logged = 1;
+ shim_log("overlay_shim framegen: a drawable presented without a copy, this present "
+ "path passes through\n");
+ }
+ fg_stat_skip(fg_skip_unordered);
+ return 0;
+ }
+
+ os_unfair_lock_lock(&fg_lock);
+ double arrived = fg_blit_time;
+ fg_launch(interpolate);
+ os_unfair_lock_unlock(&fg_lock);
+ if (min_duration > 0)
+ fg_release_drawable_at([drawable retain], arrived + min_duration);
+ return 1;
+}
+
+// Hooks under the overlay's: the renamed original when the overlay already
+// swizzled, the selector itself otherwise. Either way the overlay has drawn
+// before these run.
+static const char *fg_inner_selector(Class target, const char *selector_name, char *scratch,
+ size_t size)
+{
+ snprintf(scratch, size, "steamoverlay_%s", selector_name);
+ return class_has_method(target, scratch) ? scratch : selector_name;
+}
+
+static void framegen_install_command_buffer_hooks(Class target)
+{
+ if (!framegen_enabled || target == Nil)
+ return;
+
+ char scratch[96];
+ int hooked = 0;
+ hooked += interpose(target, fg_inner_selector(target, "presentDrawable:", scratch, sizeof(scratch)),
+ (IMP)hook_present_drawable, (IMP *)&next_present_drawable);
+ hooked += interpose(target,
+ fg_inner_selector(target, "presentDrawable:afterMinimumDuration:", scratch,
+ sizeof(scratch)),
+ (IMP)hook_present_drawable_after, (IMP *)&next_present_drawable_after);
+ hooked += interpose(target, fg_inner_selector(target, "presentDrawable:atTime:", scratch, sizeof(scratch)),
+ (IMP)hook_present_drawable_at, (IMP *)&next_present_drawable_at);
+ hooked += interpose(target, fg_inner_selector(target, "commit", scratch, sizeof(scratch)),
+ (IMP)hook_commit, (IMP *)&next_commit);
+ atomic_store(&fg_hooks_installed, hooked);
+ shim_log("overlay_shim framegen: %s hooks=%d under_overlay=%s\n", class_getName(target), hooked,
+ class_has_method(target, "steamoverlay_commit") ? "yes" : "no");
+}
+
+typedef void (*set_bool_fn)(id, SEL, BOOL);
+static set_bool_fn next_set_framebuffer_only;
+
+static void hook_set_framebuffer_only(id layer, SEL selector, BOOL value)
+{
+ if (value && framegen_enabled) {
+ static int logged;
+ if (!logged) {
+ logged = 1;
+ shim_log("overlay_shim framegen: layer asked for framebufferOnly, kept off so frames "
+ "can be copied\n");
+ }
+ value = 0;
+ }
+ next_set_framebuffer_only(layer, selector, value);
+}
+
+static void framegen_install_layer_hooks(Class layer)
+{
+ if (!framegen_enabled)
+ return;
+ interpose(layer, "setFramebufferOnly:", (IMP)hook_set_framebuffer_only,
+ (IMP *)&next_set_framebuffer_only);
+}
+
+// Test entry: what framegen has done so far
+__attribute__((visibility("default"))) void np_framegen_probe(int *enabled, int *hooks, int *mode,
+ int *objects, int *generated,
+ int *real_copies)
+{
+ if (generated != NULL)
+ *generated = atomic_load(&fg_total_generated);
+ if (real_copies != NULL)
+ *real_copies = atomic_load(&fg_total_real_copies);
+ if (enabled != NULL)
+ *enabled = framegen_enabled;
+ if (hooks != NULL)
+ *hooks = atomic_load(&fg_hooks_installed) + (next_set_framebuffer_only != NULL ? 1 : 0);
+ if (mode != NULL)
+ *mode = atomic_load(&fg_mode);
+ if (objects != NULL)
+ *objects = (fg_worker != NULL) + (fg_queue != nil) + (fg_event != nil) + (fg_layer != nil) +
+ (fg_hist[0] != nil) + (fg_engine.kind != 0);
+}
diff --git a/overlay-shim/overlay_shim.m b/overlay-shim/overlay_shim.m
index 2d2678b..1624bd7 100644
--- a/overlay-shim/overlay_shim.m
+++ b/overlay-shim/overlay_shim.m
@@ -242,6 +242,9 @@ static int stage_claimed(Class seen, Class *held, const char *stage)
return 1;
}
+// Frame generation (NOTPROTON_FRAMEGEN=1), one translation unit with the shim
+#include "framegen.m"
+
static void encoder_mark(id encoder, NSUInteger format)
{
os_unfair_lock_lock(&encoder_lock);
@@ -461,6 +464,7 @@ static id hook_render_encoder(id self, SEL selector, MTLRenderPassDescriptor *de
NSUInteger format = attachment.pixelFormat;
encoder_mark(encoder, format);
+ framegen_note_render_pass(self, attachment);
if (!overlay_pass)
return encoder;
@@ -519,7 +523,8 @@ static void hook_signal_drawable(id self, SEL selector, id drawable)
signalled_drawable = nil;
- if (overlay_text_end == NULL || drawable == nil)
+ // Without the overlay the bridge has no reader, unless framegen wants the frame
+ if ((overlay_text_end == NULL && !framegen_enabled) || drawable == nil)
return;
id device = object_device(self);
@@ -580,6 +585,8 @@ static void hook_drawable_present(id self, SEL selector)
{
if (present_composited(self, 0.0, 0))
return;
+ if (framegen_divert_plain_present(self, 0))
+ return;
next_drawable_present(self, selector);
}
@@ -588,6 +595,8 @@ static void hook_drawable_present_after(id self, SEL selector, CFTimeInterval du
{
if (present_composited(self, duration, 1))
return;
+ if (framegen_divert_plain_present(self, duration))
+ return;
next_drawable_present_after(self, selector, duration);
}
@@ -626,6 +635,8 @@ static void ensure_command_buffer_hooks(id buffer)
shim_log("overlay_shim ensure_command_buffer_hooks: %s hooks=%d overlay_commit=%s\n",
class_getName(command_buffer_class), hooked,
class_has_method(command_buffer_class, "steamoverlay_commit") ? "present" : "missing");
+
+ framegen_install_command_buffer_hooks(command_buffer_class);
}
static void ensure_queue_hooks(id queue)
@@ -920,6 +931,12 @@ static id hook_next_drawable(id layer, SEL selector)
class_getName(object_getClass(layer)), drawable != nil ? "yes" : "no");
}
+ if (framegen_enabled) {
+ // Framegen needs the drawable present hooks on every path, not only Metal 4
+ ensure_drawable_hooks(drawable);
+ framegen_note_drawable(layer, drawable);
+ }
+
return drawable;
}
@@ -951,12 +968,14 @@ __attribute__((constructor)) static void overlay_shim_load(void)
}
_dyld_register_func_for_add_image(image_added);
+ framegen_init();
int hooked = interpose(layer, "setPixelFormat:", (IMP)hook_set_pixel_format,
(IMP *)&next_set_pixel_format);
hooked += interpose(layer, "setColorspace:", (IMP)hook_set_colorspace,
(IMP *)&next_set_colorspace);
hooked += interpose(layer, "setDevice:", (IMP)hook_set_device, (IMP *)&next_set_device);
+ framegen_install_layer_hooks(layer);
shim_log("overlay_shim_load: pid=%d exe=%s layer_hooks=%d overlay=%s\n", getpid(),
getprogname(), hooked, overlay_text_end != NULL ? "present" : "absent");
diff --git a/overlay-shim/tests/framegen.m b/overlay-shim/tests/framegen.m
new file mode 100644
index 0000000..3a6705e
--- /dev/null
+++ b/overlay-shim/tests/framegen.m
@@ -0,0 +1,283 @@
+#import <Metal/Metal.h>
+#import <QuartzCore/CAMetalLayer.h>
+#import <objc/runtime.h>
+
+#import <crt_externs.h>
+#import <dlfcn.h>
+#import <spawn.h>
+#import <stdlib.h>
+#import <string.h>
+#import <sys/wait.h>
+#import <unistd.h>
+
+#import "check.h"
+
+// Frame generation: inert without NOTPROTON_FRAMEGEN, and with it the
+// interpolated frame puts a moving square halfway between its two positions.
+
+typedef void (*probe_fn)(int *, int *, int *, int *, int *, int *);
+typedef int (*interpolate_fn)(id, id, id, id, int, double *);
+
+static void *load_shim(const char *path, probe_fn *probe, interpolate_fn *interpolate)
+{
+ void *shim = dlopen(path, RTLD_NOW | RTLD_LOCAL);
+ if (shim == NULL)
+ return NULL;
+
+ void (*install)(void) = (void (*)(void))dlsym(shim, "np_overlay_shim_install");
+ *probe = (probe_fn)dlsym(shim, "np_framegen_probe");
+ *interpolate = (interpolate_fn)dlsym(shim, "np_framegen_interpolate_sync");
+ if (install != NULL)
+ install();
+ return shim;
+}
+
+// Drives the present path the way a game would, offscreen
+static int present_frames(id<MTLDevice> device, CAMetalLayer *layer, int frames, int spacing_us)
+{
+ static id<MTLCommandQueue> queue;
+
+ if (queue == nil)
+ queue = [device newCommandQueue];
+
+ for (int i = 0; i < frames; i++) {
+ @autoreleasepool {
+ id<CAMetalDrawable> drawable = [layer nextDrawable];
+ if (drawable == nil)
+ return 0;
+ id<MTLCommandBuffer> buffer = [queue commandBuffer];
+ MTLRenderPassDescriptor *pass = [MTLRenderPassDescriptor renderPassDescriptor];
+ pass.colorAttachments[0].texture = drawable.texture;
+ pass.colorAttachments[0].loadAction = MTLLoadActionClear;
+ pass.colorAttachments[0].storeAction = MTLStoreActionStore;
+ id<MTLRenderCommandEncoder> encoder = [buffer renderCommandEncoderWithDescriptor:pass];
+ [encoder endEncoding];
+ [buffer presentDrawable:drawable];
+ [buffer commit];
+ [buffer waitUntilCompleted];
+ usleep(spacing_us);
+ }
+ }
+ return 1;
+}
+
+static CAMetalLayer *game_layer(id<MTLDevice> device)
+{
+ CAMetalLayer *layer = [CAMetalLayer layer];
+ layer.device = device;
+ layer.pixelFormat = MTLPixelFormatBGRA8Unorm;
+ layer.drawableSize = CGSizeMake(256, 256);
+ return layer;
+}
+
+static int inert_without_env(const char *path)
+{
+ probe_fn probe = NULL;
+ interpolate_fn interpolate = NULL;
+
+ CHECK(getenv("NOTPROTON_FRAMEGEN") == NULL, "NOTPROTON_FRAMEGEN is set in the parent");
+ CHECK(load_shim(path, &probe, &interpolate) != NULL, "%s does not load: %s", path, dlerror());
+ CHECK(probe != NULL, "%s exports no np_framegen_probe", path);
+ CHECK(interpolate != NULL, "%s exports no np_framegen_interpolate_sync", path);
+
+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+ SKIP_IF(device == nil, "no Metal device on this machine");
+ CHECK(present_frames(device, game_layer(device), 3, 20000), "offscreen layer vends no drawable");
+
+ int enabled = -1, hooks = -1, mode = -1, objects = -1, generated = -1, copies = -1;
+ probe(&enabled, &hooks, &mode, &objects, &generated, &copies);
+ CHECK(enabled == 0, "framegen enabled without NOTPROTON_FRAMEGEN");
+ CHECK(hooks == 0, "framegen installed %d hooks while off", hooks);
+ CHECK(mode == 0, "framegen mode %d while off", mode);
+ CHECK(objects == 0, "framegen created %d objects while off", objects);
+ printf("%-14s %s\n", "framegen off", "no hooks, no objects");
+ return 0;
+}
+
+static id<MTLTexture> square_frame(id<MTLDevice> device, NSUInteger width, NSUInteger height, int x0,
+ int y0, int size)
+{
+ MTLTextureDescriptor *descriptor =
+ [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
+ width:width
+ height:height
+ mipmapped:NO];
+ descriptor.usage = MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
+ descriptor.storageMode = MTLStorageModeShared;
+ id<MTLTexture> texture = [device newTextureWithDescriptor:descriptor];
+
+ uint32_t *pixels = calloc(width * height, 4);
+ for (NSUInteger y = 0; y < height; y++) {
+ for (NSUInteger x = 0; x < width; x++) {
+ int inside = (int)x >= x0 && (int)x < x0 + size && (int)y >= y0 && (int)y < y0 + size;
+ uint8_t g = (uint8_t)(((x / 8) * 5 + (y / 8) * 3) & 0x3f);
+ pixels[y * width + x] = inside ? 0xffffffffu : (0xff000000u | (g << 16) | (g << 8) | g);
+ }
+ }
+ [texture replaceRegion:MTLRegionMake2D(0, 0, width, height)
+ mipmapLevel:0
+ withBytes:pixels
+ bytesPerRow:width * 4];
+ free(pixels);
+ return texture;
+}
+
+// Bright pixels in the square's rows: how many and where their centre is
+static void bright_pixels(id<MTLTexture> texture, int y0, int size, int *count, double *centre)
+{
+ NSUInteger width = texture.width;
+ uint32_t *pixels = malloc(width * 4 * (size_t)size);
+ [texture getBytes:pixels
+ bytesPerRow:width * 4
+ fromRegion:MTLRegionMake2D(0, y0, width, size)
+ mipmapLevel:0];
+ double sum = 0;
+ int n = 0;
+ for (int y = 0; y < size; y++) {
+ for (NSUInteger x = 0; x < width; x++) {
+ uint32_t p = pixels[y * width + x];
+ if (((p >> 16) & 0xff) > 180 && ((p >> 8) & 0xff) > 180 && (p & 0xff) > 180) {
+ sum += x;
+ n++;
+ }
+ }
+ }
+ free(pixels);
+ *count = n;
+ *centre = n > 0 ? sum / n : -1;
+}
+
+static int midpoint_square(interpolate_fn interpolate, id<MTLDevice> device, int engine,
+ const char *name)
+{
+ const NSUInteger width = 1024, height = 768;
+ const int size = 120, y0 = 300, xa = 300, xb = 380;
+
+ id<MTLTexture> a = square_frame(device, width, height, xa, y0, size);
+ id<MTLTexture> b = square_frame(device, width, height, xb, y0, size);
+ id<MTLTexture> out = square_frame(device, width, height, 0, 0, 0);
+ double gpu_ms = 0;
+ int used = interpolate(device, a, b, out, engine, &gpu_ms);
+ if (used == 0 && engine == 2) {
+ printf("%-14s %s\n", name, "skipped: MetalFX frame interpolation unavailable here");
+ return 0;
+ }
+ CHECK(used == engine, "engine %s did not run (got %d)", name, used);
+
+ int count = 0;
+ double centre = -1;
+ bright_pixels(out, y0, size, &count, &centre);
+ double expected = (xa + xb) / 2.0 + size / 2.0;
+ printf("%-14s square centre %.1f (expected %.1f), %d bright pixels (square %d), %.2f ms GPU\n",
+ name, centre, expected, count, size * size, gpu_ms);
+ CHECK(count > size * size * 6 / 10 && count < size * size * 14 / 10,
+ "%s: %d bright pixels, a crossfade or a smear rather than a moved square", name, count);
+ CHECK(centre > expected - 12 && centre < expected + 12,
+ "%s: square centre %.1f is not near the midpoint %.1f", name, centre, expected);
+ return 0;
+}
+
+static int full_frame_cost(interpolate_fn interpolate, id<MTLDevice> device, int engine,
+ const char *name)
+{
+ const NSUInteger width = 3024, height = 1964;
+ id<MTLTexture> a = square_frame(device, width, height, 1000, 800, 200);
+ id<MTLTexture> b = square_frame(device, width, height, 1100, 800, 200);
+ id<MTLTexture> out = square_frame(device, width, height, 0, 0, 0);
+ double gpu_ms = 0;
+ int used = interpolate(device, a, b, out, engine, &gpu_ms);
+ if (used != engine)
+ return 0;
+ printf("%-14s %lux%lu: %.2f ms GPU per generated frame\n", name, (unsigned long)width,
+ (unsigned long)height, gpu_ms);
+ return 0;
+}
+
+static int child_with_env(const char *path)
+{
+ probe_fn probe = NULL;
+ interpolate_fn interpolate = NULL;
+
+ CHECK(load_shim(path, &probe, &interpolate) != NULL, "%s does not load: %s", path, dlerror());
+ CHECK(probe != NULL && interpolate != NULL, "%s exports no framegen entry points", path);
+
+ id<MTLDevice> device = MTLCreateSystemDefaultDevice();
+ SKIP_IF(device == nil, "no Metal device on this machine");
+
+ int enabled = 0, hooks = 0, mode = 0, objects = 0, generated = 0, copies = 0;
+ probe(&enabled, &hooks, &mode, &objects, &generated, &copies);
+ CHECK(enabled == 1, "NOTPROTON_FRAMEGEN=1 did not enable framegen");
+
+ // The present path must survive with framegen on, offscreen
+ CAMetalLayer *layer = game_layer(device);
+ CHECK(present_frames(device, layer, 8, 20000), "offscreen layer vends no drawable");
+ probe(&enabled, &hooks, &mode, &objects, &generated, &copies);
+ CHECK(hooks >= 4, "framegen installed %d hooks, expected the present family and commit", hooks);
+ printf("%-14s hooks=%d mode=%d objects=%d\n", "framegen on", hooks, mode, objects);
+
+ // Let the worker finish building, then run slow enough that it engages
+ for (int i = 0; i < 100 && mode == 1; i++) {
+ usleep(50000);
+ probe(&enabled, &hooks, &mode, &objects, &generated, &copies);
+ }
+ CHECK(mode == 3, "framegen mode %d after setup, expected pass-through", mode);
+ CHECK(present_frames(device, layer, 60, 25000), "offscreen layer vends no drawable");
+ usleep(200000);
+ probe(&enabled, &hooks, &mode, &objects, &generated, &copies);
+ CHECK(mode == 2, "framegen mode %d after 60 frames at 40 fps, expected active", mode);
+ CHECK(copies > 0, "no real frame copy presented while active");
+ CHECK(generated > 0, "no frame generated while active");
+ printf("%-14s %d real copies, %d generated at 40 fps\n", "framegen path", copies, generated);
+ double burst_start = CACurrentMediaTime();
+ CHECK(present_frames(device, layer, 40, 0), "offscreen layer vends no drawable");
+ double burst_ms = (CACurrentMediaTime() - burst_start) * 1000.0;
+ probe(&enabled, &hooks, &mode, &objects, &generated, &copies);
+ CHECK(mode == 3, "framegen mode %d after a fast burst (40 frames in %.1f ms), expected pass-through",
+ mode, burst_ms);
+ printf("%-14s back to pass-through above the refresh rate (40 frames in %.1f ms)\n",
+ "framegen fast", burst_ms);
+
+ int rc = midpoint_square(interpolate, device, 1, "own engine");
+ if (rc != 0)
+ return rc;
+ rc = midpoint_square(interpolate, device, 2, "metalfx");
+ if (rc != 0)
+ return rc;
+ full_frame_cost(interpolate, device, 1, "own engine");
+ full_frame_cost(interpolate, device, 2, "metalfx");
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ const char *path = getenv("NOTPROTON_OVERLAY_SHIM");
+
+ SKIP_IF(path == NULL, "NOTPROTON_OVERLAY_SHIM names the shim to load");
+
+ if (getenv("NP_FRAMEGEN_TEST_CHILD") != NULL)
+ return child_with_env(path);
+
+ int rc = inert_without_env(path);
+ if (rc != 0)
+ return rc;
+
+ // The shim reads the environment when it loads, so the on case is a child
+ char **environ_now = *_NSGetEnviron();
+ int count = 0;
+ while (environ_now[count] != NULL)
+ count++;
+ char **env = calloc((size_t)count + 3, sizeof(char *));
+ memcpy(env, environ_now, (size_t)count * sizeof(char *));
+ env[count] = "NOTPROTON_FRAMEGEN=1";
+ env[count + 1] = "NP_FRAMEGEN_TEST_CHILD=1";
+
+ fflush(stdout);
+ pid_t pid;
+ char *args[] = { argv[0], NULL };
+ (void)argc;
+ CHECK(posix_spawn(&pid, argv[0], NULL, NULL, args, env) == 0, "cannot spawn %s", argv[0]);
+ int status = 0;
+ CHECK(waitpid(pid, &status, 0) == pid, "waitpid failed");
+ CHECK(WIFEXITED(status), "framegen child did not exit cleanly (status %d)", status);
+ return WEXITSTATUS(status);
+}