Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(mobile): 修复 64 KB 及以下音视频无法播放 (#4642)

Lizi71d44b87ab84
diff --git a/apps/mobile/src/__tests__/mobileMakerTransport.test.ts b/apps/mobile/src/__tests__/mobileMakerTransport.test.ts
index 0ab72a6275..8d5080a120 100644
--- a/apps/mobile/src/__tests__/mobileMakerTransport.test.ts
+++ b/apps/mobile/src/__tests__/mobileMakerTransport.test.ts
@@ -3,6 +3,7 @@ import { resolve } from "node:path";
import { describe, expect, it, vi } from "vitest";
import {
CONTROLLER_CAPABILITY_PROVIDER_LOGO_KINDS_V2,
+ FILE_INLINE_MAX_BYTES,
REMOTE_INVOKE_ALLOWLIST,
} from "@cindy/device-link";
import {
@@ -11,7 +12,7 @@ import {
} from "@/device-link/mobileMakerTransport";
import type { RemoteInvoke } from "@/device-link/mobileMakerTransport";
import { installPeerFileDownload } from "@/device-link/peerFileRegistry";
-import { resolveMobileRemoteMedia } from "@/session/remoteMedia";
+import { isDirectPreviewableMediaUrl, resolveMobileRemoteMedia } from "@/session/remoteMedia";
import {
setMobileAuthOwner,
__testing as authOwnerTesting,
@@ -36,6 +37,63 @@ function harness() {
}
describe("mobile maker transport", () => {
+ describe.each([
+ ["video", "video/mp4", "mp4"],
+ ["audio", "audio/mpeg", "mp3"],
+ ] as const)("%s playback", (kind, mimeType, extension) => {
+ it.each([FILE_INLINE_MAX_BYTES - 1, FILE_INLINE_MAX_BYTES, FILE_INLINE_MAX_BYTES + 1])(
+ "resolves %i bytes to a URL accepted by the media player",
+ async (size) => {
+ const metadata = { ossKey: "", size, mimeType };
+ const prepared = size <= FILE_INLINE_MAX_BYTES
+ ? { ...metadata, inlineBase64: Buffer.alloc(size).toString("base64") }
+ : { ...metadata, transferRequired: true };
+ const uploaded = { ...metadata, ossKey: `media/playback.${extension}` };
+ const invoke = vi.fn(async (_device, _channel, args) => {
+ expect(args[0]).not.toHaveProperty("stream");
+ return args[0].prepareOnly ? prepared : uploaded;
+ });
+ const getUrl = `https://media.example.invalid/playback.${extension}`;
+ const presignGet = vi.fn(async () => ({ getUrl, expiresAt: "2099-01-01T00:00:00Z" }));
+ const maker = createMobileMakerTransport({ deviceId: "d", invoke: invoke as RemoteInvoke });
+ const resolved = await resolveMobileRemoteMedia(
+ { kind, url: `cindy-media://blobs/${"a".repeat(64)}.${extension}` },
+ { fetchRemoteMedia: maker.fetchRemoteMedia, presignGet },
+ );
+ expect(isDirectPreviewableMediaUrl(resolved.url)).toBe(true);
+ expect(resolved).toMatchObject({ url: getUrl, mimeType, size, previewable: true });
+ expect(resolved.inlineBase64).toBeUndefined();
+ expect(presignGet).toHaveBeenCalledExactlyOnceWith(uploaded.ossKey);
+ expect(invoke).toHaveBeenCalledTimes(2);
+ },
+ );
+ it.each([FILE_INLINE_MAX_BYTES - 1, FILE_INLINE_MAX_BYTES])(
+ "plays %i-byte inline media when an old host ignores prepareOnly",
+ async (size) => {
+ const metadata = { ossKey: "", size, mimeType };
+ const inline = { ...metadata, inlineBase64: Buffer.alloc(size).toString("base64") };
+ const invoke = vi.fn(async (_device, _channel, args) => {
+ expect(args[0]).not.toHaveProperty("stream");
+ return inline;
+ });
+ const presignGet = vi.fn();
+ const maker = createMobileMakerTransport({ deviceId: "d", invoke: invoke as RemoteInvoke });
+ const resolved = await resolveMobileRemoteMedia(
+ { kind, url: `cindy-media://blobs/${"a".repeat(64)}.${extension}` },
+ { fetchRemoteMedia: maker.fetchRemoteMedia, presignGet },
+ );
+ expect(isDirectPreviewableMediaUrl(resolved.url)).toBe(true);
+ expect(resolved).toMatchObject({
+ url: `data:${mimeType};base64,${inline.inlineBase64}`,
+ mimeType,
+ size,
+ previewable: true,
+ });
+ expect(presignGet).not.toHaveBeenCalled();
+ expect(invoke).toHaveBeenCalledTimes(2);
+ },
+ );
+ });
it.each(["audio/mpeg", "video/mp4", "image/png", "application/pdf", "application/octet-stream"])("retains %s previews while preserving peer for complete byte consumers", async (mimeType) => {
const metadata = { ossKey: "", size: 70_000, mimeType, transferRequired: true };
const direct = { ...metadata, transferRequired: false };
diff --git a/apps/mobile/src/__tests__/remoteMedia.test.ts b/apps/mobile/src/__tests__/remoteMedia.test.ts
index 54e917c50a..3d51ccb61b 100644
--- a/apps/mobile/src/__tests__/remoteMedia.test.ts
+++ b/apps/mobile/src/__tests__/remoteMedia.test.ts
@@ -3,6 +3,7 @@ import {
canPreviewResolvedRemoteMedia,
formatRemoteMediaSize,
isDesktopLocalMediaUrl,
+ isDirectPreviewableMediaUrl,
isResolvedRemoteMediaFresh,
localCopyResolvedMedia,
REMOTE_MEDIA_NEVER_EXPIRES,
@@ -10,6 +11,14 @@ import {
} from "@/session/remoteMedia";
describe("mobile remote media", () => {
+ it("accepts inline audio and video data URLs for the media player", () => {
+ expect(isDirectPreviewableMediaUrl("https://example.com/a.mp4")).toBe(true);
+ expect(isDirectPreviewableMediaUrl("data:image/png;base64,aaa")).toBe(true);
+ expect(isDirectPreviewableMediaUrl("data:audio/mpeg;base64,aaa")).toBe(true);
+ expect(isDirectPreviewableMediaUrl("data:video/mp4;base64,aaa")).toBe(true);
+ expect(isDirectPreviewableMediaUrl("data:text/plain;base64,aaa")).toBe(false);
+ });
+
it("recognizes desktop-local media schemes only", () => {
expect(isDesktopLocalMediaUrl("xdt-image://cache/a.png")).toBe(true);
expect(isDesktopLocalMediaUrl("xdt-video://cache/a.mp4")).toBe(true);
@@ -183,6 +192,28 @@ describe("mobile remote media", () => {
expect(presignGet).not.toHaveBeenCalled();
});
+ it.each([
+ ["audio", "audio/mpeg"],
+ ["video", "video/mp4"],
+ ] as const)("turns inline %s bytes into a player-accepted data URL", async (kind, mimeType) => {
+ const inlineBase64 = "aGVsbG8=";
+ const fetchRemoteMedia = vi.fn(async () => ({
+ ossKey: "",
+ mimeType,
+ size: 5,
+ inlineBase64,
+ }));
+ const presignGet = vi.fn();
+ const resolved = await resolveMobileRemoteMedia(
+ { kind, url: `cindy-media://blobs/${"a".repeat(64)}.bin` },
+ { fetchRemoteMedia, presignGet },
+ );
+ expect(resolved.url).toBe(`data:${mimeType};base64,${inlineBase64}`);
+ expect(isDirectPreviewableMediaUrl(resolved.url)).toBe(true);
+ expect(resolved.previewable).toBe(true);
+ expect(presignGet).not.toHaveBeenCalled();
+ });
+
it("falls back to the presign path when an old desktop ignores the thumbnail flag", async () => {
const fetchRemoteMedia = vi.fn(async () => ({
ossKey: "cindy/device-link/user-1/a.png",
diff --git a/apps/mobile/src/session/remoteMedia.ts b/apps/mobile/src/session/remoteMedia.ts
index e834cab502..a73afde479 100644
--- a/apps/mobile/src/session/remoteMedia.ts
+++ b/apps/mobile/src/session/remoteMedia.ts
@@ -99,7 +99,10 @@ export function isDesktopLocalMediaUrl(url: unknown): url is string {
}
export function isDirectPreviewableMediaUrl(url: unknown): url is string {
- return isPayloadDirectPreviewableUrl(url);
+ return (
+ isPayloadDirectPreviewableUrl(url)
+ || (typeof url === "string" && (url.startsWith("data:audio/") || url.startsWith("data:video/")))
+ );
}
export function canPreviewResolvedRemoteMedia(
@@ -183,8 +186,9 @@ export async function resolveMobileRemoteMedia(
expiresAt: peerMediaExpiry(fetched)!,
previewable: canPreviewResolvedRemoteMedia(media.kind, fetched.mimeType),
};
- // inline 缩略图回包:字节已随 invoke 帧到手,无 OSS 对象,跳过 presign。
- // url 先给 data URI 保证任何情况下可渲染;宿主(会话屏)会把字节落盘并换成 file://。
+ // inline 回包:字节已随 invoke 帧到手,无 OSS 对象,跳过 presign。
+ // 缩略图给 data:image;小音视频若仍以内联回包到达,也走 data URI,由播放器直接打开。
+ // 图片宿主(会话屏)会把字节落盘并换成 file://。
if (isValidInlineResult(fetched)) {
return {
url: `data:${fetched.mimeType};base64,${fetched.inlineBase64}`,
diff --git a/packages/device-link/src/__tests__/fileAccess.test.ts b/packages/device-link/src/__tests__/fileAccess.test.ts
index e53cc5ab01..886bd532ca 100644
--- a/packages/device-link/src/__tests__/fileAccess.test.ts
+++ b/packages/device-link/src/__tests__/fileAccess.test.ts
@@ -27,6 +27,50 @@ describe("shared file read policy", () => {
expect(peer).toHaveBeenCalledTimes(peerCalls);
expect(fallback).toHaveBeenCalledTimes(1 - peerCalls);
});
+ it.each([
+ ["video/mp4", true],
+ ["audio/mpeg", true],
+ ["Audio/MP4", true],
+ ["image/png", false],
+ ["text/plain", false],
+ ] as const)("uses a retained URL only for inline %s playback", async (mimeType, needsUrl) => {
+ const prepared = { ossKey: "", size: 3, mimeType, inlineBase64: "YWJj" };
+ const uploaded = { ossKey: "stream/key", size: 3, mimeType };
+ const peer = vi.fn();
+ const fallback = vi.fn(async () => uploaded);
+ expect(await readDeviceFile({
+ prepare: async () => prepared, peer, fallback, stream: true,
+ })).toBe(needsUrl ? uploaded : prepared);
+ expect(fallback).toHaveBeenCalledTimes(needsUrl ? 1 : 0);
+ expect(peer).not.toHaveBeenCalled();
+
+ // Download/share consumers still receive complete inline bytes without uploading.
+ fallback.mockClear();
+ expect(await readDeviceFile({
+ prepare: async () => prepared, peer, fallback, stream: false,
+ })).toBe(prepared);
+ expect(fallback).not.toHaveBeenCalled();
+ expect(peer).not.toHaveBeenCalled();
+ });
+ it("discards an inline video's uploaded result when playback is cancelled", async () => {
+ const abort = new AbortController();
+ const uploaded = { ossKey: "stream/key", size: 3, mimeType: "video/mp4" };
+ const discard = vi.fn();
+ const peer = vi.fn();
+ await expect(readDeviceFile({
+ prepare: async () => ({ ...uploaded, ossKey: "", inlineBase64: "YWJj" }),
+ peer,
+ fallback: async () => {
+ abort.abort();
+ return uploaded;
+ },
+ stream: true,
+ signal: abort.signal,
+ discard,
+ })).rejects.toThrow("FILE_PEER_CANCELLED");
+ expect(discard).toHaveBeenCalledExactlyOnceWith(uploaded);
+ expect(peer).not.toHaveBeenCalled();
+ });
it("returns an empty inline file without peer negotiation or OSS", async () => {
const result = {
ossKey: "",
@@ -42,12 +86,12 @@ describe("shared file read policy", () => {
expect(peer).not.toHaveBeenCalled();
expect(fallback).not.toHaveBeenCalled();
});
- it("accepts old hosts ordinary OSS result without uploading twice", async () => {
- const result = { ossKey: "old/key", size: 1, mimeType: "text/plain" };
+ it.each(["text/plain", "video/mp4", "audio/mpeg"])("accepts old hosts ordinary %s OSS result without uploading twice", async (mimeType) => {
+ const result = { ossKey: "old/key", size: 1, mimeType };
const peer = vi.fn(),
fallback = vi.fn();
expect(
- await readDeviceFile({ prepare: async () => result, peer, fallback }),
+ await readDeviceFile({ prepare: async () => result, peer, fallback, stream: true }),
).toBe(result);
expect(peer).not.toHaveBeenCalled();
expect(fallback).not.toHaveBeenCalled();
diff --git a/packages/device-link/src/fileAccess.ts b/packages/device-link/src/fileAccess.ts
index 3799953538..f92418f2ee 100644
--- a/packages/device-link/src/fileAccess.ts
+++ b/packages/device-link/src/fileAccess.ts
@@ -42,9 +42,13 @@ export async function readDeviceFile<T extends DeviceFileResult>(options: {
assertActive();
const prepared = await options.prepare();
if (!active()) return discardCancelled(prepared);
- if (prepared.transferRequired !== true) return prepared;
- const needsFallback = options.stream &&
- (options.peerResultIsTransient || /^(audio|video)\//i.test(prepared.mimeType));
+ const mediaStream = options.stream && /^(audio|video)\//i.test(prepared.mimeType);
+ // Small audio/video files arrive inline too, but playback still needs a retained
+ // URL. Reuse the streaming path without re-uploading an old host's OSS result.
+ const inlineMediaStream = mediaStream && !prepared.ossKey &&
+ typeof prepared.inlineBase64 === "string";
+ if (prepared.transferRequired !== true && !inlineMediaStream) return prepared;
+ const needsFallback = mediaStream || (options.stream && options.peerResultIsTransient);
const direct = needsFallback ? null : await options.peer(prepared);
if (direct && !active()) return discardCancelled(direct);
else assertActive();