Skip to content
TypeScript

fix(mobile): use H.264 for iOS previews and keep Close reachable (#28622)

shivamb28bb8fc9d58
diff --git a/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx
index 99aed81cf2b..1f6e01d7858 100644
--- a/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx
+++ b/apps/mobile/src/features/devices/DevicePreviewRouteScreen.tsx
@@ -88,7 +88,7 @@ export function DevicePreviewScreen({
}) {
const embedded = onMaximize !== undefined;
const insets = useSafeAreaInsets();
- // NativeColumnContent already insets the sides, so only a book fold moves the reveal handle.
+ // NativeColumnContent already insets the sides, so only a book fold moves the hidden-controls pill.
const handleBounds = deriveBottomControlInsets(useNativeColumnLayoutMetrics());
const { themeVariables } = useAppearancePreferences();
const focused = useIsFocused();
@@ -106,7 +106,7 @@ export function DevicePreviewScreen({
const preview = selectedThreadDevicePreview(previews, selectedKey);
const onInputConnected = async (connected: boolean) => {
setInputConnected(connected);
- if (connected && !embedded) setControlsVisible(false);
+ if (!connected) setControlsVisible(true);
};
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- closes the focused route through the navigator once the thread has no device previews
useEffect(() => {
@@ -348,20 +348,23 @@ export function DevicePreviewScreen({
</View>
</Animated.View>
) : (
- <Pressable
- accessibilityRole="button"
- accessibilityLabel="Show device controls"
- accessibilityHint="Shaking the phone also shows them"
- hitSlop={12}
- className="absolute h-11 w-16 items-center justify-center self-center"
+ <View
+ pointerEvents="box-none"
+ className="absolute self-center"
style={{
top: Math.max(insets.top, 12),
transform: [{ translateX: (handleBounds.left - handleBounds.right) / 2 }],
}}
- onPress={() => setControlsVisible(true)}
>
- <View className="h-1.5 w-10 rounded-full bg-white/40" />
- </Pressable>
+ <GlassSurface className="flex-row items-center gap-1 p-1">
+ <SheetCloseButton accessibilityLabel="Close device preview" onClose={onClose} />
+ <ControlPill
+ accessibilityLabel="Show device controls"
+ icon="chevron.down"
+ onPress={() => setControlsVisible(true)}
+ />
+ </GlassSurface>
+ </View>
)}
</View>
);
diff --git a/apps/mobile/src/features/devices/device-stream.browser.test.ts b/apps/mobile/src/features/devices/device-stream.browser.test.ts
new file mode 100644
index 00000000000..91a8e3bb82d
--- /dev/null
+++ b/apps/mobile/src/features/devices/device-stream.browser.test.ts
@@ -0,0 +1,108 @@
+import { describe, expect, it, vi } from "vite-plus/test";
+
+import { start, stop } from "./device-stream.browser";
+
+const envelope = (tag: number, payload: number[]) => {
+ const length = 1 + payload.length;
+ return [
+ length >>> 24,
+ (length >>> 16) & 0xff,
+ (length >>> 8) & 0xff,
+ length & 0xff,
+ tag,
+ ...payload,
+ ];
+};
+
+function viewerFixture(supported: boolean) {
+ vi.useFakeTimers();
+ const configured: string[] = [];
+ vi.stubGlobal(
+ "VideoDecoder",
+ class {
+ static isConfigSupported = async () => ({ supported });
+ state = "unconfigured";
+ decodeQueueSize = 0;
+ configure(config: VideoDecoderConfig) {
+ configured.push(config.codec);
+ this.state = "configured";
+ }
+ decode() {}
+ close() {
+ this.state = "closed";
+ }
+ },
+ );
+ vi.stubGlobal("EncodedVideoChunk", vi.fn());
+ vi.stubGlobal(
+ "WebSocket",
+ class {
+ static OPEN = 1;
+ readyState = 0;
+ send() {}
+ close() {}
+ },
+ );
+ const videoBodies: ReadableStreamDefaultController<Uint8Array>[] = [];
+ const requested: string[] = [];
+ vi.stubGlobal("fetch", (url: string, init: RequestInit) => {
+ requested.push(new URL(url).pathname);
+ if (!url.includes("stream.avcc")) return Promise.resolve(new Response(null));
+ return Promise.resolve(
+ new Response(
+ new ReadableStream<Uint8Array>({
+ start(body) {
+ videoBodies.push(body);
+ init.signal?.addEventListener("abort", () => body.error(new Error("aborted")));
+ },
+ }),
+ ),
+ );
+ });
+ vi.stubGlobal("ReactNativeWebView", { postMessage() {} });
+ start({
+ platform: "ios",
+ deviceId: "sim",
+ access: {
+ httpBase: "https://environment.test/api/device-hub",
+ wsBase: "wss://environment.test/api/device-hub",
+ credentials: false,
+ query: {},
+ },
+ colors: {
+ background: "black",
+ foreground: "white",
+ muted: "gray",
+ buttonBackground: "black",
+ buttonForeground: "white",
+ buttonBorder: "gray",
+ },
+ });
+ return { configured, requested, videoBodies };
+}
+
+describe("native iOS simulator viewer", () => {
+ it.each([
+ { supported: true, image: "" },
+ { supported: false, image: "/api/device-hub/vendor/serve-sim/helper/sim/stream.mjpeg" },
+ ])(
+ "streams H.264 when WebCodecs decodes it, else MJPEG (supported: $supported)",
+ async ({ supported, image }) => {
+ try {
+ const { configured, requested, videoBodies } = viewerFixture(supported);
+ await vi.advanceTimersByTimeAsync(0);
+ expect(requested).toContain("/api/device-hub/vendor/serve-sim/helper/sim/stream.avcc");
+ videoBodies[0]!.enqueue(new Uint8Array(envelope(1, [1, 0x64, 0x00, 0x33])));
+ await vi.advanceTimersByTimeAsync(0);
+ expect(configured).toEqual(supported ? ["avc1.640033"] : []);
+ const img = document.querySelector("img")!;
+ expect(img.src && new URL(img.src).pathname).toBe(image);
+ expect(img.style.display).toBe(supported ? "none" : "block");
+ } finally {
+ stop();
+ vi.useRealTimers();
+ vi.unstubAllGlobals();
+ }
+ },
+ );
+});
diff --git a/apps/mobile/src/features/devices/device-stream.browser.ts b/apps/mobile/src/features/devices/device-stream.browser.ts
index ed2092148d0..2434c35168b 100644
--- a/apps/mobile/src/features/devices/device-stream.browser.ts
+++ b/apps/mobile/src/features/devices/device-stream.browser.ts
@@ -130,28 +130,24 @@ export function start(configuration: DeviceStreamConfiguration) {
const unauthorized = () => {
if (activeClient === client) post({ type: "unauthorized" });
};
- const client = createDeviceStreamClient(
- { ...configuration, preferMjpeg: platform === "ios" },
- canvas,
- {
- onStatus: reportStatus,
- onScreen: layout,
- onMjpegFallback: () => {
- canvas.style.display = "none";
- image.style.display = "block";
- },
- onCanvasVideo: () => {
- image.style.display = "none";
- canvas.style.display = "";
- },
- onUnauthorized: unauthorized,
- onInputConnected: (connected) => {
- inputConnected = connected;
- inputStatus.style.display = streaming && !connected ? "block" : "none";
- post({ type: "input", connected });
- },
+ const client = createDeviceStreamClient(configuration, canvas, {
+ onStatus: reportStatus,
+ onScreen: layout,
+ onMjpegFallback: () => {
+ canvas.style.display = "none";
+ image.style.display = "block";
},
- );
+ onCanvasVideo: () => {
+ image.style.display = "none";
+ canvas.style.display = "";
+ },
+ onUnauthorized: unauthorized,
+ onInputConnected: (connected) => {
+ inputConnected = connected;
+ inputStatus.style.display = streaming && !connected ? "block" : "none";
+ post({ type: "input", connected });
+ },
+ });
activeClient = client;
client.setMjpegImage(image);
const touch = (event: PointerEvent, phase: "begin" | "move" | "end") => {
diff --git a/apps/mobile/vite.config.ts b/apps/mobile/vite.config.ts
index 75400348bfb..809b4ff4704 100644
--- a/apps/mobile/vite.config.ts
+++ b/apps/mobile/vite.config.ts
@@ -43,6 +43,7 @@ const reactTests = [
"src/features/home/home-list-options.test.tsx",
"src/features/workspace/useSearchScrollRestoration.test.tsx",
"src/features/connection/ConnectionsNewRouteScreen.test.tsx",
+ "src/features/devices/device-stream.browser.test.ts",
];
const sourceRoot = `${normalizePath(NodeURL.fileURLToPath(new URL("./src", import.meta.url)))}/`;