Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

16.1 KB
Raw Blame
import { peerMediaUri, peerMediaExpiry, releasePeerMedia } from "@/device-link/peerFileRegistry";
import type { NormalizedToolMedia } from "@/session/messageNormalize";
import {
  isPayloadDesktopLocalMediaUrl,
  isPayloadDirectPreviewableUrl,
} from "@cindy/maker-shared/payload-summary";
import { i18n } from "@/i18n";
import { errorText } from "@/debug/fileDiagnostics";
import { mobileDebugLog } from "@/debug/mobileDebugLog";

const EXPIRY_SAFETY_WINDOW_MS = 60 * 1000;

/** Match stable Host/transport codes, never expose raw errors containing local paths. */
export function mediaLoadFailureKey(error: unknown): 'message.lightbox.sourceMissing' | 'message.lightbox.deviceOffline' | 'message.lightbox.loadFailed' {
  const code = error && typeof error === 'object' && 'code' in error ? error.code : undefined;
  const message = error instanceof Error ? error.message : String(error ?? '');
  if (code === 'MEDIA_SOURCE_MISSING' || message.includes('[MEDIA_SOURCE_MISSING]')) return 'message.lightbox.sourceMissing';
  const offlineCodes = ['NOT_CONNECTED', 'DEVICE_LINK_NOT_CONNECTED', 'DEVICE_OFFLINE', 'PEER_OFFLINE'];
  if ((typeof code === 'string' && offlineCodes.includes(code))
    || /\b(?:NOT_CONNECTED|DEVICE_LINK_NOT_CONNECTED|DEVICE_OFFLINE|PEER_OFFLINE)\b/.test(message)) return 'message.lightbox.deviceOffline';
  return 'message.lightbox.loadFailed';
}

export interface MobileRemoteMediaFetchResult {
  version?: string;
  /** Local adapter only; never sent over device-link. Copies an OSS result into the shared file cache. */
  cacheDownload?: (downloadUrl: string) => Promise<MobileRemoteMediaFetchResult | null>;
  /** inline 缩略图回包(新被控端)时为空串。 */
  ossKey: string;
  mimeType: string;
  size: number;
  /** 被控端缩好的缩略图字节(base64,thumbnail 请求且被控端支持时才有)。 */
  inlineBase64?: string;
}

export interface MobileRemoteMediaPresignResult {
  getUrl: string;
  expiresAt: string;
}

export interface MobileResolvedRemoteMedia {
  /** Local-only consumer lease; never serialized over device-link. */
  protectLocalFile?(): () => void;
  url: string;
  /** inline 缩略图 / 本地磁盘缓存命中没有在世 OSS 对象,为空串(退屏清理跳过 DELETE)。 */
  ossKey: string;
  mimeType: string;
  size: number;
  expiresAt: string;
  previewable: boolean;
  /** inline 缩略图的原始字节(base64),供宿主写入磁盘缓存;消费端渲染用 url 即可。 */
  inlineBase64?: string;
}

export interface MobileRemoteMediaFetchOptions {
  /** Await Host version validation; still reuse bytes for that confirmed version. */
  revalidate?: boolean;
  /** Local callbacks: only connect on a cache miss or during background revalidation. */
  beforeFetch?: () => Promise<unknown>;
  retainBackground?: (result: MobileRemoteMediaFetchResult, isCurrent?: () => boolean) => Promise<void>;
  /** Local receiver bytes; never sent to the remote computer. */
  onProgress?: (loaded: number, total: number) => void;
  /** Local read intent only; false for consumers that copy complete bytes. */
  stream?: boolean;
  skipCache?: boolean;
  thumbnail?: boolean;
  signal?: AbortSignal;
  /** Return an undelivered OSS result to its existing cleanup owner before cancellation rejects. */
  onDiscardOssKey?: (ossKey: string) => void;
}

export interface MobileRemoteMediaResolverDeps {
  findLocalImage?(url: string): Promise<{ uri: string; mimeType: string; size: number } | null>;
  /** Retain completed video bytes before releasing the peer's short-lived staging file. */
  retainLocalVideo?(media: MobileResolvedRemoteMedia): Promise<MobileResolvedRemoteMedia>;
  fetchRemoteMedia(
    url: string,
    opts?: MobileRemoteMediaFetchOptions,
  ): Promise<MobileRemoteMediaFetchResult>;
  presignGet(ossKey: string): Promise<MobileRemoteMediaPresignResult>;
}

export interface MobileRemoteMediaResolveOptions {
  revalidate?: boolean;
  beforeFetch?: () => Promise<unknown>;
  discardBackgroundOssKey?: (key: string) => void;
  onProgress?: (loaded: number, total: number) => void;
  stream?: boolean;
  signal?: AbortSignal;
  /** 强制被控端绕过上传去重缓存(上次的 ossKey 已悬空时的自愈路径)。 */
  skipCache?: boolean;
  /** 只要聊天列表缩略图:被控端缩 1024px webp inline 回包;老被控端回落原图。 */
  thumbnail?: boolean;
  /**
   * 上传已成功、`ossKey` 已知,但 presign 还没做时**同步回调一次**。
   *
   * 给「用完就要把 OSS 对象删掉」的调用方用(HTML 渲染态的同目录资源取件):presign
   * 失败会让本函数抛错,调用方拿不到 resolved 结果,于是围绕 `media.ossKey` 写的
   * finally 根本不会执行 —— 对象已经上传却没人回收(review P1)。有了这个回调,key 一
   * 出现就落到调用方手里,失败路径也能 best-effort DELETE。
   *
   * inline 回包(缩略图字节随帧回来、无 OSS 对象)**不触发**:那条路没有 key 可回收。
   * 外层重试会让它按次触发,每次可能是不同的 key —— 调用方要**累加收集**,不能只记最后一个。
   */
  onOssKey?: (ossKey: string) => void;
}

/** inline 缩略图没有 presign 过期语义,给个远未来的过期时间(本地字节不过期)。 */
export const REMOTE_MEDIA_NEVER_EXPIRES = "9999-12-31T00:00:00.000Z";

/**
 * UI 层远程媒体取件回调:缩略图懒取件传 signal(滚出队列可取消)+ thumbnail
 * (被控端只回缩略图),查看器传 front(插队,取原图),Image 加载失败重试传
 * forceRefresh(跳过各级缓存)。thumbnail 与非 thumbnail 的取件在队列/缓存层
 * 按不同键隔离,查看器取原图不会命中缩略图缓存。
 */
export type ResolveRemoteMediaFn = (
  media: Pick<NormalizedToolMedia, "url" | "previewable"> & {
    kind: NormalizedToolMedia["kind"] | "file";
    thumbnail?: boolean;
  },
  opts?: {
    front?: boolean;
    signal?: AbortSignal;
    forceRefresh?: boolean;
    /** 不新建取件;与 front 同用可提升已有排队项。 */
    cachedOnly?: boolean;
  },
) => Promise<MobileResolvedRemoteMedia>;

export function isDesktopLocalMediaUrl(url: unknown): url is string {
  return isPayloadDesktopLocalMediaUrl(url);
}

export function isDirectPreviewableMediaUrl(url: unknown): url is string {
  return (
    isPayloadDirectPreviewableUrl(url)
    || (typeof url === "string" && (url.startsWith("data:audio/") || url.startsWith("data:video/")))
  );
}

export function canPreviewResolvedRemoteMedia(
  kind: NormalizedToolMedia["kind"] | "file",
  mimeType: string,
): boolean {
  if (kind === "image") return mimeType.startsWith("image/");
  if (kind === "video") return mimeType.startsWith("video/");
  if (kind === "audio") return mimeType.startsWith("audio/");
  return false;
}

/**
 * 原图落盘成功后的条目升级:把渲染地址从 presign OSS 换成本地 file://。
 *
 * 为什么必须换:presign 地址在有效期内会被取件队列当 fresh 缓存命中反复复用,
 * 同键请求再也不会重进磁盘 lookup,于是每次点开查看器都从 OSS 重新拉整张原图
 * (用户实测「已经打开过原图,关闭再打开又从黑屏开始」),盘上那份副本永远轮不
 * 到用。换成本地文件后再次点开零网络秒出。
 *
 * 升级是**事件驱动**的(落盘完成后经 RemoteMediaResolveHooks 回写队列缓存),不让
 * 调用方同步等待落盘——取件队列并发上限只有 2 且看不到 front,同步等待会让相邻页
 * 预取占住槽位,把用户正在看的那张饿住(PR #1125 review)。
 *
 * ossKey 必须保留:OSS 对象仍在世,退屏清理要靠它 DELETE(空 key 才跳过),队列也
 * 靠它识别「升级迟到、对象已被 forceRefresh 换掉」并丢弃本次升级。
 * hit 为空(落盘被跳过 / 失败)或 mime 不是图片时返回 null,由调用方放弃升级。
 */
export function localCopyResolvedMedia(
  resolved: MobileResolvedRemoteMedia,
  hit: { uri: string; mimeType: string; size: number } | null | undefined,
): MobileResolvedRemoteMedia | null {
  if (!hit || !hit.uri || !hit.mimeType.startsWith("image/")) return null;
  return {
    url: hit.uri,
    ossKey: resolved.ossKey,
    mimeType: hit.mimeType,
    size: hit.size,
    // 本地文件不过期;被 LRU/OS 清掉时 Image onError → forceRefresh 重取自愈。
    expiresAt: REMOTE_MEDIA_NEVER_EXPIRES,
    previewable: true,
  };
}

export function isResolvedRemoteMediaFresh(
  media: Pick<MobileResolvedRemoteMedia, "expiresAt">,
  now = Date.now(),
): boolean {
  const expiresAt = Date.parse(media.expiresAt);
  return (
    Number.isFinite(expiresAt) && expiresAt - now > EXPIRY_SAFETY_WINDOW_MS
  );
}

export async function resolveMobileRemoteMedia(
  media: { kind: NormalizedToolMedia["kind"] | "file"; url: string },
  deps: MobileRemoteMediaResolverDeps,
  opts?: MobileRemoteMediaResolveOptions,
): Promise<MobileResolvedRemoteMedia> {
  const retainVideo = media.kind === 'video' ? deps.retainLocalVideo : undefined;
  // Complete-file reads can use authenticated peer transport. Playback must own
  // a durable copy first: peer staging files expire after five minutes.
  const stream = retainVideo ? false : opts?.stream;
  const retain = async (resolved: MobileResolvedRemoteMedia) => {
    if (!retainVideo) return resolved;
    try { return await retainVideo(resolved); }
    finally { releasePeerMedia(resolved.url); }
  };
  if (media.kind === 'image' && !opts?.skipCache) {
    // Shared by task history, group chat and file viewers. Keeping this here
    // avoids fixing thumbnails in one screen while original-image reads still go online.
    const local = await (deps.findLocalImage
      ? deps.findLocalImage(media.url)
      : import('./sentAttachmentThumbStore').then((store) => store.resolveSentAttachmentThumb(media.url)))
      .catch(() => null);
    if (local) return {
      url: local.uri, mimeType: local.mimeType, size: local.size,
      ossKey: '', expiresAt: REMOTE_MEDIA_NEVER_EXPIRES, previewable: true,
    };
  }
  if (!isDesktopLocalMediaUrl(media.url)) {
    throw new Error(i18n.t("composer.attachments.notFetchableMedia"));
  }
  const fetchOpts = {
    // Old peer caches may carry octet-stream for a video. Revalidate metadata,
    // not bytes: a matching version still reuses the downloaded local file.
    ...(opts?.revalidate || media.kind === 'video' ? { revalidate: true } : {}),
    ...(opts?.beforeFetch ? { beforeFetch: opts.beforeFetch } : {}),
    ...(stream === false ? { retainBackground: async (result: MobileRemoteMediaFetchResult, isCurrent?: () => boolean) => {
      if (!result.ossKey || isCurrent?.() === false) return;
      try {
        const signed = await deps.presignGet(result.ossKey);
        const copy = await result.cacheDownload?.(signed.getUrl);
        const uri = copy && peerMediaUri(copy);
        if (uri) releasePeerMedia(uri);
      } finally { if (isCurrent?.() !== false) opts?.discardBackgroundOssKey?.(result.ossKey); }
    } } : {}),
    ...(opts?.onProgress ? { onProgress: opts.onProgress } : {}),
    ...(stream !== undefined ? { stream } : {}),
    ...(opts?.onOssKey ? { onDiscardOssKey: opts.onOssKey } : {}),
    ...(opts?.signal ? { signal: opts.signal } : {}),
    ...(opts?.skipCache ? { skipCache: true } : {}),
    ...(opts?.thumbnail ? { thumbnail: true } : {}),
  };
  const fetched = await deps.fetchRemoteMedia(
    media.url,
    Object.keys(fetchOpts).length > 0 ? fetchOpts : undefined,
  );
  const local = peerMediaUri(fetched);
  if (local)
    return retain({
      url: local,
      ossKey: "",
      size: fetched.size,
      mimeType: fetched.mimeType,
      expiresAt: peerMediaExpiry(fetched)!,
      previewable: canPreviewResolvedRemoteMedia(media.kind, fetched.mimeType),
    });
  // inline 回包:字节已随 invoke 帧到手,无 OSS 对象,跳过 presign。
  // 缩略图给 data:image;小音视频若仍以内联回包到达,也走 data URI,由播放器直接打开。
  // 图片宿主(会话屏)会把字节落盘并换成 file://。
  if (isValidInlineResult(fetched)) {
    return {
      url: `data:${fetched.mimeType};base64,${fetched.inlineBase64}`,
      ossKey: "",
      mimeType: fetched.mimeType,
      size: fetched.size,
      expiresAt: REMOTE_MEDIA_NEVER_EXPIRES,
      previewable: canPreviewResolvedRemoteMedia(media.kind, fetched.mimeType),
      inlineBase64: fetched.inlineBase64,
    };
  }
  // **在校验之前就把 key 交出去**(review P1 第三轮)。判据是「有没有在世的 OSS 对象」,
  // 不是「回包合不合法」—— 只要 ossKey 非空,被控端的 PUT 就已经完成了。放在 isValidFetchResult
  // 之后会漏掉一整类:合法的**零字节**文件(空的 .css / .js 完全正常)会因为 `size > 0` 这条
  // 校验先抛错,而对象已经上传 —— 调用方拿不到 key,那个对象永久遗留。
  if (typeof fetched?.ossKey === "string" && fetched.ossKey.length > 0) {
    opts?.onOssKey?.(fetched.ossKey);
  }
  if (!isValidFetchResult(fetched)) {
    throw new Error(i18n.t("composer.attachments.mediaResultInvalid"));
  }
  // Image thumbnails are high-volume; trace presign timing for full files, failures always.
  const presignStartedAt = Date.now();
  const signed = await deps.presignGet(fetched.ossKey).catch((error: unknown) => {
    mobileDebugLog("warn", "files", "remote media presign failed", {
      kind: media.kind,
      ms: Date.now() - presignStartedAt,
      error: errorText(error),
    });
    throw error;
  });
  if (!isValidPresignResult(signed)) {
    mobileDebugLog("warn", "files", "remote media presign invalid", { kind: media.kind });
    throw new Error(i18n.t("composer.attachments.mediaUrlInvalid"));
  }
  const downloaded = await fetched.cacheDownload?.(signed.getUrl);
  const downloadedUri = downloaded && peerMediaUri(downloaded);
  if (downloadedUri) return retain({
    url: downloadedUri, ossKey: fetched.ossKey, mimeType: fetched.mimeType, size: fetched.size,
    expiresAt: peerMediaExpiry(downloaded!)!,
    previewable: canPreviewResolvedRemoteMedia(media.kind, fetched.mimeType),
  });
  if (media.kind !== "image")
    mobileDebugLog("debug", "files", "remote media presigned", {
      kind: media.kind,
      ms: Date.now() - presignStartedAt,
      size: fetched.size,
    });
  return {
    url: signed.getUrl,
    ossKey: fetched.ossKey,
    mimeType: fetched.mimeType,
    size: fetched.size,
    expiresAt: signed.expiresAt,
    previewable: canPreviewResolvedRemoteMedia(media.kind, fetched.mimeType),
  };
}

export function formatRemoteMediaSize(bytes: number): string {
  if (!Number.isFinite(bytes) || bytes <= 0) return "";
  if (bytes < 1024) return `${bytes} B`;
  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
  if (bytes < 1024 * 1024 * 1024)
    return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
  return `${(bytes / 1024 / 1024 / 1024).toFixed(1)} GB`;
}

/** inline 缩略图回包有效性:字节 + 图片 mime + 正数 size 齐备才算。 */
function isValidInlineResult(
  value: MobileRemoteMediaFetchResult,
): value is MobileRemoteMediaFetchResult & { inlineBase64: string } {
  return (
    !!value &&
    typeof value.inlineBase64 === "string" &&
    typeof value.mimeType === "string" &&
    /^[\w.+-]+\/[\w.+-]+$/.test(value.mimeType) &&
    typeof value.size === "number" &&
    Number.isFinite(value.size) &&
    Number.isSafeInteger(value.size) &&
    value.size >= 0 &&
    value.inlineBase64.length <= 1024 * 1024 &&
    /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(
      value.inlineBase64,
    ) &&
    (value.inlineBase64.length / 4) * 3 -
      (value.inlineBase64.endsWith("==")
        ? 2
        : value.inlineBase64.endsWith("=")
          ? 1
          : 0) ===
      value.size
  );
}

function isValidFetchResult(value: MobileRemoteMediaFetchResult): boolean {
  return (
    !!value &&
    typeof value.ossKey === "string" &&
    value.ossKey.length > 0 &&
    typeof value.mimeType === "string" &&
    value.mimeType.length > 0 &&
    typeof value.size === "number" &&
    Number.isFinite(value.size) &&
    value.size >= 0
  );
}

function isValidPresignResult(value: MobileRemoteMediaPresignResult): boolean {
  return (
    !!value &&
    typeof value.getUrl === "string" &&
    value.getUrl.startsWith("http") &&
    typeof value.expiresAt === "string" &&
    Number.isFinite(Date.parse(value.expiresAt))
  );
}