Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
fix(mobile): 修复 64 KB 及以下音视频无法播放 (#4642)
Lizi71d44b87ab84
diff --git a/apps/mobile/src/__tests__/mobileMakerTransport.test.ts b/apps/mobile/src/__tests__/mobileMakerTransport.test.tsindex 0ab72a6275..8d5080a120 100644--- a/apps/mobile/src/__tests__/mobileMakerTransport.test.ts+++ b/apps/mobile/src/__tests__/mobileMakerTransport.test.ts@@ -3,6 +3,7 @@ import { resolve } from "node:path";import { describe, expect, it, vi } from "vitest";import {CONTROLLER_CAPABILITY_PROVIDER_LOGO_KINDS_V2,+ FILE_INLINE_MAX_BYTES,REMOTE_INVOKE_ALLOWLIST,} from "@cindy/device-link";import {@@ -11,7 +12,7 @@ import {} from "@/device-link/mobileMakerTransport";import type { RemoteInvoke } from "@/device-link/mobileMakerTransport";import { installPeerFileDownload } from "@/device-link/peerFileRegistry";-import { resolveMobileRemoteMedia } from "@/session/remoteMedia";+import { isDirectPreviewableMediaUrl, resolveMobileRemoteMedia } from "@/session/remoteMedia";import {setMobileAuthOwner,__testing as authOwnerTesting,@@ -36,6 +37,63 @@ function harness() {}describe("mobile maker transport", () => {+ describe.each([+ ["video", "video/mp4", "mp4"],+ ["audio", "audio/mpeg", "mp3"],+ ] as const)("%s playback", (kind, mimeType, extension) => {+ it.each([FILE_INLINE_MAX_BYTES - 1, FILE_INLINE_MAX_BYTES, FILE_INLINE_MAX_BYTES + 1])(+ "resolves %i bytes to a URL accepted by the media player",+ async (size) => {+ const metadata = { ossKey: "", size, mimeType };+ const prepared = size <= FILE_INLINE_MAX_BYTES+ ? { ...metadata, inlineBase64: Buffer.alloc(size).toString("base64") }+ : { ...metadata, transferRequired: true };+ const uploaded = { ...metadata, ossKey: `media/playback.${extension}` };+ const invoke = vi.fn(async (_device, _channel, args) => {+ expect(args[0]).not.toHaveProperty("stream");+ return args[0].prepareOnly ? prepared : uploaded;+ });+ const getUrl = `https://media.example.invalid/playback.${extension}`;+ const presignGet = vi.fn(async () => ({ getUrl, expiresAt: "2099-01-01T00:00:00Z" }));+ const maker = createMobileMakerTransport({ deviceId: "d", invoke: invoke as RemoteInvoke });+ const resolved = await resolveMobileRemoteMedia(+ { kind, url: `cindy-media://blobs/${"a".repeat(64)}.${extension}` },+ { fetchRemoteMedia: maker.fetchRemoteMedia, presignGet },+ );+ expect(isDirectPreviewableMediaUrl(resolved.url)).toBe(true);+ expect(resolved).toMatchObject({ url: getUrl, mimeType, size, previewable: true });+ expect(resolved.inlineBase64).toBeUndefined();+ expect(presignGet).toHaveBeenCalledExactlyOnceWith(uploaded.ossKey);+ expect(invoke).toHaveBeenCalledTimes(2);+ },+ );+ it.each([FILE_INLINE_MAX_BYTES - 1, FILE_INLINE_MAX_BYTES])(+ "plays %i-byte inline media when an old host ignores prepareOnly",+ async (size) => {+ const metadata = { ossKey: "", size, mimeType };+ const inline = { ...metadata, inlineBase64: Buffer.alloc(size).toString("base64") };+ const invoke = vi.fn(async (_device, _channel, args) => {+ expect(args[0]).not.toHaveProperty("stream");+ return inline;+ });+ const presignGet = vi.fn();+ const maker = createMobileMakerTransport({ deviceId: "d", invoke: invoke as RemoteInvoke });+ const resolved = await resolveMobileRemoteMedia(+ { kind, url: `cindy-media://blobs/${"a".repeat(64)}.${extension}` },+ { fetchRemoteMedia: maker.fetchRemoteMedia, presignGet },+ );+ expect(isDirectPreviewableMediaUrl(resolved.url)).toBe(true);+ expect(resolved).toMatchObject({+ url: `data:${mimeType};base64,${inline.inlineBase64}`,+ mimeType,+ size,+ previewable: true,+ });+ expect(presignGet).not.toHaveBeenCalled();+ expect(invoke).toHaveBeenCalledTimes(2);+ },+ );+ });it.each(["audio/mpeg", "video/mp4", "image/png", "application/pdf", "application/octet-stream"])("retains %s previews while preserving peer for complete byte consumers", async (mimeType) => {const metadata = { ossKey: "", size: 70_000, mimeType, transferRequired: true };const direct = { ...metadata, transferRequired: false };diff --git a/apps/mobile/src/__tests__/remoteMedia.test.ts b/apps/mobile/src/__tests__/remoteMedia.test.tsindex 54e917c50a..3d51ccb61b 100644--- a/apps/mobile/src/__tests__/remoteMedia.test.ts+++ b/apps/mobile/src/__tests__/remoteMedia.test.ts@@ -3,6 +3,7 @@ import {canPreviewResolvedRemoteMedia,formatRemoteMediaSize,isDesktopLocalMediaUrl,+ isDirectPreviewableMediaUrl,isResolvedRemoteMediaFresh,localCopyResolvedMedia,REMOTE_MEDIA_NEVER_EXPIRES,@@ -10,6 +11,14 @@ import {} from "@/session/remoteMedia";describe("mobile remote media", () => {+ it("accepts inline audio and video data URLs for the media player", () => {+ expect(isDirectPreviewableMediaUrl("https://example.com/a.mp4")).toBe(true);+ expect(isDirectPreviewableMediaUrl("data:image/png;base64,aaa")).toBe(true);+ expect(isDirectPreviewableMediaUrl("data:audio/mpeg;base64,aaa")).toBe(true);+ expect(isDirectPreviewableMediaUrl("data:video/mp4;base64,aaa")).toBe(true);+ expect(isDirectPreviewableMediaUrl("data:text/plain;base64,aaa")).toBe(false);+ });+it("recognizes desktop-local media schemes only", () => {expect(isDesktopLocalMediaUrl("xdt-image://cache/a.png")).toBe(true);expect(isDesktopLocalMediaUrl("xdt-video://cache/a.mp4")).toBe(true);@@ -183,6 +192,28 @@ describe("mobile remote media", () => {expect(presignGet).not.toHaveBeenCalled();});+ it.each([+ ["audio", "audio/mpeg"],+ ["video", "video/mp4"],+ ] as const)("turns inline %s bytes into a player-accepted data URL", async (kind, mimeType) => {+ const inlineBase64 = "aGVsbG8=";+ const fetchRemoteMedia = vi.fn(async () => ({+ ossKey: "",+ mimeType,+ size: 5,+ inlineBase64,+ }));+ const presignGet = vi.fn();+ const resolved = await resolveMobileRemoteMedia(+ { kind, url: `cindy-media://blobs/${"a".repeat(64)}.bin` },+ { fetchRemoteMedia, presignGet },+ );+ expect(resolved.url).toBe(`data:${mimeType};base64,${inlineBase64}`);+ expect(isDirectPreviewableMediaUrl(resolved.url)).toBe(true);+ expect(resolved.previewable).toBe(true);+ expect(presignGet).not.toHaveBeenCalled();+ });+it("falls back to the presign path when an old desktop ignores the thumbnail flag", async () => {const fetchRemoteMedia = vi.fn(async () => ({ossKey: "cindy/device-link/user-1/a.png",diff --git a/apps/mobile/src/session/remoteMedia.ts b/apps/mobile/src/session/remoteMedia.tsindex e834cab502..a73afde479 100644--- a/apps/mobile/src/session/remoteMedia.ts+++ b/apps/mobile/src/session/remoteMedia.ts@@ -99,7 +99,10 @@ export function isDesktopLocalMediaUrl(url: unknown): url is string {}export function isDirectPreviewableMediaUrl(url: unknown): url is string {- return isPayloadDirectPreviewableUrl(url);+ return (+ isPayloadDirectPreviewableUrl(url)+ || (typeof url === "string" && (url.startsWith("data:audio/") || url.startsWith("data:video/")))+ );}export function canPreviewResolvedRemoteMedia(@@ -183,8 +186,9 @@ export async function resolveMobileRemoteMedia(expiresAt: peerMediaExpiry(fetched)!,previewable: canPreviewResolvedRemoteMedia(media.kind, fetched.mimeType),};- // inline 缩略图回包:字节已随 invoke 帧到手,无 OSS 对象,跳过 presign。- // url 先给 data URI 保证任何情况下可渲染;宿主(会话屏)会把字节落盘并换成 file://。+ // inline 回包:字节已随 invoke 帧到手,无 OSS 对象,跳过 presign。+ // 缩略图给 data:image;小音视频若仍以内联回包到达,也走 data URI,由播放器直接打开。+ // 图片宿主(会话屏)会把字节落盘并换成 file://。if (isValidInlineResult(fetched)) {return {url: `data:${fetched.mimeType};base64,${fetched.inlineBase64}`,diff --git a/packages/device-link/src/__tests__/fileAccess.test.ts b/packages/device-link/src/__tests__/fileAccess.test.tsindex e53cc5ab01..886bd532ca 100644--- a/packages/device-link/src/__tests__/fileAccess.test.ts+++ b/packages/device-link/src/__tests__/fileAccess.test.ts@@ -27,6 +27,50 @@ describe("shared file read policy", () => {expect(peer).toHaveBeenCalledTimes(peerCalls);expect(fallback).toHaveBeenCalledTimes(1 - peerCalls);});+ it.each([+ ["video/mp4", true],+ ["audio/mpeg", true],+ ["Audio/MP4", true],+ ["image/png", false],+ ["text/plain", false],+ ] as const)("uses a retained URL only for inline %s playback", async (mimeType, needsUrl) => {+ const prepared = { ossKey: "", size: 3, mimeType, inlineBase64: "YWJj" };+ const uploaded = { ossKey: "stream/key", size: 3, mimeType };+ const peer = vi.fn();+ const fallback = vi.fn(async () => uploaded);+ expect(await readDeviceFile({+ prepare: async () => prepared, peer, fallback, stream: true,+ })).toBe(needsUrl ? uploaded : prepared);+ expect(fallback).toHaveBeenCalledTimes(needsUrl ? 1 : 0);+ expect(peer).not.toHaveBeenCalled();++ // Download/share consumers still receive complete inline bytes without uploading.+ fallback.mockClear();+ expect(await readDeviceFile({+ prepare: async () => prepared, peer, fallback, stream: false,+ })).toBe(prepared);+ expect(fallback).not.toHaveBeenCalled();+ expect(peer).not.toHaveBeenCalled();+ });+ it("discards an inline video's uploaded result when playback is cancelled", async () => {+ const abort = new AbortController();+ const uploaded = { ossKey: "stream/key", size: 3, mimeType: "video/mp4" };+ const discard = vi.fn();+ const peer = vi.fn();+ await expect(readDeviceFile({+ prepare: async () => ({ ...uploaded, ossKey: "", inlineBase64: "YWJj" }),+ peer,+ fallback: async () => {+ abort.abort();+ return uploaded;+ },+ stream: true,+ signal: abort.signal,+ discard,+ })).rejects.toThrow("FILE_PEER_CANCELLED");+ expect(discard).toHaveBeenCalledExactlyOnceWith(uploaded);+ expect(peer).not.toHaveBeenCalled();+ });it("returns an empty inline file without peer negotiation or OSS", async () => {const result = {ossKey: "",@@ -42,12 +86,12 @@ describe("shared file read policy", () => {expect(peer).not.toHaveBeenCalled();expect(fallback).not.toHaveBeenCalled();});- it("accepts old hosts ordinary OSS result without uploading twice", async () => {- const result = { ossKey: "old/key", size: 1, mimeType: "text/plain" };+ it.each(["text/plain", "video/mp4", "audio/mpeg"])("accepts old hosts ordinary %s OSS result without uploading twice", async (mimeType) => {+ const result = { ossKey: "old/key", size: 1, mimeType };const peer = vi.fn(),fallback = vi.fn();expect(- await readDeviceFile({ prepare: async () => result, peer, fallback }),+ await readDeviceFile({ prepare: async () => result, peer, fallback, stream: true }),).toBe(result);expect(peer).not.toHaveBeenCalled();expect(fallback).not.toHaveBeenCalled();diff --git a/packages/device-link/src/fileAccess.ts b/packages/device-link/src/fileAccess.tsindex 3799953538..f92418f2ee 100644--- a/packages/device-link/src/fileAccess.ts+++ b/packages/device-link/src/fileAccess.ts@@ -42,9 +42,13 @@ export async function readDeviceFile<T extends DeviceFileResult>(options: {assertActive();const prepared = await options.prepare();if (!active()) return discardCancelled(prepared);- if (prepared.transferRequired !== true) return prepared;- const needsFallback = options.stream &&- (options.peerResultIsTransient || /^(audio|video)\//i.test(prepared.mimeType));+ const mediaStream = options.stream && /^(audio|video)\//i.test(prepared.mimeType);+ // Small audio/video files arrive inline too, but playback still needs a retained+ // URL. Reuse the streaming path without re-uploading an old host's OSS result.+ const inlineMediaStream = mediaStream && !prepared.ossKey &&+ typeof prepared.inlineBase64 === "string";+ if (prepared.transferRequired !== true && !inlineMediaStream) return prepared;+ const needsFallback = mediaStream || (options.stream && options.peerResultIsTransient);const direct = needsFallback ? null : await options.peer(prepared);if (direct && !active()) return discardCancelled(direct);else assertActive();