Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

feat(editor): transcribe existing audio attachments (#6375)

beemines68db65243a16
diff --git a/web/src/components/MemoEditor/components/EditorMetadata.tsx b/web/src/components/MemoEditor/components/EditorMetadata.tsx
index 83e1e439a..9ed2729d7 100644
--- a/web/src/components/MemoEditor/components/EditorMetadata.tsx
+++ b/web/src/components/MemoEditor/components/EditorMetadata.tsx
@@ -5,7 +5,15 @@ import { extractManagedAttachmentUIDs } from "@/utils/managed-attachment";
import { useEditorContext, useEditorSelector } from "../state";
import type { EditorMetadataProps } from "../types";
-export const EditorMetadata: FC<EditorMetadataProps> = ({ memoName, uploadingLocalFileURLs, onInsertAttachments, onInsertLocalFiles }) => {
+export const EditorMetadata: FC<EditorMetadataProps> = ({
+ memoName,
+ uploadingLocalFileURLs,
+ onInsertAttachments,
+ onInsertLocalFiles,
+ onTranscribeAttachment,
+ transcribingAttachment,
+ transcriptionDisabled,
+}) => {
const { actions, dispatch } = useEditorContext();
const attachments = useEditorSelector((s) => s.metadata.attachments);
const localFiles = useEditorSelector((s) => s.localFiles);
@@ -30,6 +38,9 @@ export const EditorMetadata: FC<EditorMetadataProps> = ({ memoName, uploadingLoc
onInsertLocalFiles={onInsertLocalFiles}
placementActionsDisabled={placementActionsDisabled}
uploadingLocalFileURLs={uploadingLocalFileURLs}
+ onTranscribeAttachment={onTranscribeAttachment}
+ transcribingAttachment={transcribingAttachment}
+ transcriptionDisabled={transcriptionDisabled}
/>
<RelationListEditor
diff --git a/web/src/components/MemoEditor/hooks/index.ts b/web/src/components/MemoEditor/hooks/index.ts
index 86623fabc..0bb377e1a 100644
--- a/web/src/components/MemoEditor/hooks/index.ts
+++ b/web/src/components/MemoEditor/hooks/index.ts
@@ -1,5 +1,6 @@
// Custom hooks for MemoEditor (internal use only)
+export { useAttachmentTranscription } from "./useAttachmentTranscription";
export { useAudioRecorder } from "./useAudioRecorder";
export { useAudioWaveform } from "./useAudioWaveform";
export { useAutoSave } from "./useAutoSave";
diff --git a/web/src/components/MemoEditor/hooks/useAttachmentTranscription.ts b/web/src/components/MemoEditor/hooks/useAttachmentTranscription.ts
new file mode 100644
index 000000000..ee7750931
--- /dev/null
+++ b/web/src/components/MemoEditor/hooks/useAttachmentTranscription.ts
@@ -0,0 +1,45 @@
+import { ConnectError } from "@connectrpc/connect";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { toast } from "react-hot-toast";
+import type { Attachment } from "@/types/proto/api/v1/attachment_service_pb";
+import { useTranslate } from "@/utils/i18n";
+import { transcriptionService } from "../services";
+
+/** Transcribes an existing attachment into the draft without changing the original audio. */
+export const useAttachmentTranscription = (onText: (text: string) => void) => {
+ const t = useTranslate();
+ const [transcribingAttachment, setTranscribingAttachment] = useState<string>();
+ const request = useRef<AbortController | undefined>(undefined);
+
+ useEffect(() => () => request.current?.abort(), []);
+
+ const transcribeAttachment = useCallback(
+ async (attachment: Attachment) => {
+ if (request.current) return;
+ const controller = new AbortController();
+ request.current = controller;
+ setTranscribingAttachment(attachment.name);
+ try {
+ const text = (await transcriptionService.transcribeAttachment(attachment, controller.signal)).trim();
+ if (controller.signal.aborted) return;
+ if (!text) {
+ toast.error(t("editor.audio-recorder.transcribe-empty"));
+ return;
+ }
+ onText(text);
+ toast.success(t("editor.audio-recorder.transcribe-success"));
+ } catch (error) {
+ if (!controller.signal.aborted) {
+ // Server errors explain themselves; download and network failures get the translated fallback.
+ toast.error((error instanceof ConnectError && error.rawMessage) || t("editor.audio-recorder.transcribe-error"));
+ }
+ } finally {
+ request.current = undefined;
+ if (!controller.signal.aborted) setTranscribingAttachment(undefined);
+ }
+ },
+ [onText, t],
+ );
+
+ return { transcribeAttachment, transcribingAttachment };
+};
diff --git a/web/src/components/MemoEditor/index.tsx b/web/src/components/MemoEditor/index.tsx
index 8a8ede251..c1a07489e 100644
--- a/web/src/components/MemoEditor/index.tsx
+++ b/web/src/components/MemoEditor/index.tsx
@@ -15,6 +15,7 @@ import type { EditorFileOrigin } from "./Editor/extensions";
import {
splitInlineLocalFiles,
toLocalFiles,
+ useAttachmentTranscription,
useAudioRecorder,
useAutoSave,
useBlobUrls,
@@ -149,6 +150,8 @@ const MemoEditorImpl: React.FC<MemoEditorProps> = ({
editor.scrollToCursor();
}, []);
+ const { transcribeAttachment, transcribingAttachment } = useAttachmentTranscription(insertTranscribedText);
+
const handleTranscribeRecordedAudio = useCallback(
async (localFile: LocalFile) => {
if (!canTranscribe) {
@@ -206,6 +209,12 @@ const MemoEditorImpl: React.FC<MemoEditorProps> = ({
dispatch(actions.setRecorderBusy(audioRecorder.isBusy));
}, [audioRecorder.isBusy, actions, dispatch]);
+ // Both transcription paths insert into the draft when they land, so saving
+ // waits for either one; the recorder itself is idle by then.
+ useEffect(() => {
+ dispatch(actions.setLoading("transcribing", isTranscribingAudio || Boolean(transcribingAttachment)));
+ }, [isTranscribingAudio, transcribingAttachment, actions, dispatch]);
+
useEffect(() => {
if (!isAudioRecorderOpen) {
return;
@@ -389,6 +398,9 @@ const MemoEditorImpl: React.FC<MemoEditorProps> = ({
uploadingLocalFileURLs={inlineImageUpload.uploadingLocalFileURLs}
onInsertAttachments={inlineImageUpload.insertRemoteImages}
onInsertLocalFiles={inlineImageUpload.insertLocalImages}
+ onTranscribeAttachment={canTranscribe ? transcribeAttachment : undefined}
+ transcribingAttachment={transcribingAttachment}
+ transcriptionDisabled={isSaving || audioRecorder.isBusy || isTranscribingAudio}
/>
{!memo && !parentMemoName && isInitialized && suggestions.length > 0 && (
<EditorSuggestions suggestions={suggestions} controllerRef={editorRef} space={editorSpace} />
diff --git a/web/src/components/MemoEditor/services/transcriptionService.ts b/web/src/components/MemoEditor/services/transcriptionService.ts
index c4a191e2a..853b524cd 100644
--- a/web/src/components/MemoEditor/services/transcriptionService.ts
+++ b/web/src/components/MemoEditor/services/transcriptionService.ts
@@ -1,23 +1,49 @@
import { create } from "@bufbuild/protobuf";
+import { Code, ConnectError } from "@connectrpc/connect";
import { aiServiceClient } from "@/connect";
import { TranscribeRequestSchema, TranscriptionAudioSchema } from "@/types/proto/api/v1/ai_service_pb";
+import type { Attachment } from "@/types/proto/api/v1/attachment_service_pb";
+import { getAttachmentUrl } from "@/utils/attachment";
-export const transcriptionService = {
- async transcribeFile(file: File): Promise<string> {
- const content = new Uint8Array(await file.arrayBuffer());
- const response = await aiServiceClient.transcribe(
- create(TranscribeRequestSchema, {
- audio: create(TranscriptionAudioSchema, {
- source: {
- case: "content",
- value: content,
- },
- filename: file.name,
- contentType: file.type,
- }),
+// Mirrors maxTranscriptionAudioSizeBytes in server/api/v1/ai_service.go so an
+// oversized attachment is rejected before it is downloaded, with the server's error.
+const MAX_AUDIO_SIZE = 25 * 1024 * 1024;
+
+const checkAudioSize = (size: number) => {
+ if (size > MAX_AUDIO_SIZE) throw new ConnectError("audio file is too large; maximum size is 25 MiB", Code.InvalidArgument);
+};
+
+const transcribe = async (content: Uint8Array, filename: string, contentType: string, signal?: AbortSignal): Promise<string> => {
+ signal?.throwIfAborted();
+ const response = await aiServiceClient.transcribe(
+ create(TranscribeRequestSchema, {
+ audio: create(TranscriptionAudioSchema, {
+ source: {
+ case: "content",
+ value: content,
+ },
+ filename,
+ contentType,
}),
- );
+ }),
+ { signal },
+ );
+
+ return response.text;
+};
+
+export const transcriptionService = {
+ async transcribeAttachment(attachment: Attachment, signal?: AbortSignal): Promise<string> {
+ checkAudioSize(Number(attachment.size));
+ const response = await fetch(getAttachmentUrl(attachment), { signal, credentials: "same-origin" });
+ if (!response.ok) throw new Error(`Failed to download audio (${response.status})`);
+ checkAudioSize(Number(response.headers.get("Content-Length")));
+ const content = new Uint8Array(await response.arrayBuffer());
+ checkAudioSize(content.byteLength);
+ return transcribe(content, attachment.filename, attachment.type, signal);
+ },
- return response.text;
+ async transcribeFile(file: File, signal?: AbortSignal): Promise<string> {
+ return transcribe(new Uint8Array(await file.arrayBuffer()), file.name, file.type, signal);
},
};
diff --git a/web/src/components/MemoEditor/services/validationService.ts b/web/src/components/MemoEditor/services/validationService.ts
index 776c57e15..875649ebd 100644
--- a/web/src/components/MemoEditor/services/validationService.ts
+++ b/web/src/components/MemoEditor/services/validationService.ts
@@ -36,6 +36,10 @@ export const validationService = {
return { valid: false, reason: "editor.validation.unsupported-attachment-image-url", detail: invalidReference };
}
+ if (state.ui.isLoading.transcribing) {
+ return { valid: false, reason: "editor.audio-recorder.transcribing" };
+ }
+
// Cannot save while audio recorder is active
if (state.recorderBusy) {
return { valid: false, reason: "editor.validation.finish-audio-recording" };
diff --git a/web/src/components/MemoEditor/state/types.ts b/web/src/components/MemoEditor/state/types.ts
index e5cb7811e..3146592ed 100644
--- a/web/src/components/MemoEditor/state/types.ts
+++ b/web/src/components/MemoEditor/state/types.ts
@@ -3,7 +3,7 @@ import type { Location, MemoRelation } from "@/types/proto/api/v1/memo_service_p
import { Visibility } from "@/types/proto/api/v1/memo_service_pb";
import type { LocalFile } from "../types/attachment";
-export type LoadingKey = "saving" | "uploading" | "loading";
+export type LoadingKey = "saving" | "uploading" | "loading" | "transcribing";
export type ContentSource = "editor" | "external";
export interface EditorState {
@@ -22,6 +22,7 @@ export interface EditorState {
saving: boolean;
uploading: boolean;
loading: boolean;
+ transcribing: boolean;
};
/** Save landed and the editor is about to close; the toolbar shows a brief
* confirmation instead of the commit verb. Only hosts that unmount after
@@ -73,6 +74,7 @@ const defaultState: EditorState = {
saving: false,
uploading: false,
loading: false,
+ transcribing: false,
},
justSaved: false,
},
diff --git a/web/src/components/MemoEditor/types/components.ts b/web/src/components/MemoEditor/types/components.ts
index 36da5dc23..a4619fb40 100644
--- a/web/src/components/MemoEditor/types/components.ts
+++ b/web/src/components/MemoEditor/types/components.ts
@@ -80,6 +80,9 @@ export interface EditorToolbarProps {
}
export interface EditorMetadataProps {
+ onTranscribeAttachment?: (attachment: Attachment) => void;
+ transcribingAttachment?: string;
+ transcriptionDisabled?: boolean;
memoName?: string;
uploadingLocalFileURLs: ReadonlySet<string>;
onInsertAttachments: (attachments: Attachment[]) => void;
diff --git a/web/src/components/MemoMetadata/Attachment/AttachmentListEditor.tsx b/web/src/components/MemoMetadata/Attachment/AttachmentListEditor.tsx
index e0c4f296c..f93e44a3d 100644
--- a/web/src/components/MemoMetadata/Attachment/AttachmentListEditor.tsx
+++ b/web/src/components/MemoMetadata/Attachment/AttachmentListEditor.tsx
@@ -1,4 +1,15 @@
-import { ChevronDownIcon, ChevronUpIcon, FileAudioIcon, FileIcon, ImagePlusIcon, PauseIcon, PlayIcon, XIcon } from "lucide-react";
+import {
+ ChevronDownIcon,
+ ChevronUpIcon,
+ FileAudioIcon,
+ FileIcon,
+ FileTextIcon,
+ ImagePlusIcon,
+ LoaderIcon,
+ PauseIcon,
+ PlayIcon,
+ XIcon,
+} from "lucide-react";
import { type FC, type MouseEvent, useMemo, useRef, useState } from "react";
import type { AttachmentItem, LocalFile } from "@/components/MemoEditor/types/attachment";
import { getAudioRecordingTimeLabel, toAttachmentItems } from "@/components/MemoEditor/types/attachment";
@@ -24,6 +35,9 @@ const collectMembers = <T,>(byId: ReadonlyMap<string, T>, memberIds: string[]):
memberIds.map((memberId) => byId.get(memberId)).filter((member): member is T => member !== undefined);
interface AttachmentListEditorProps {
+ onTranscribeAttachment?: (attachment: Attachment) => void;
+ transcribingAttachment?: string;
+ transcriptionDisabled?: boolean;
attachments: Attachment[];
localFiles?: LocalFile[];
onAttachmentsChange?: (attachments: Attachment[]) => void;
@@ -38,19 +52,34 @@ interface AttachmentListEditorProps {
/** Trailing controls of an attachment row: quiet 24px squares that show once the row is engaged. */
const AttachmentItemActions: FC<{
+ transcriptionAction?: { label: string; onClick: () => void; disabled: boolean; busy: boolean };
placementAction?: { label: string; onClick: () => void; disabled?: boolean };
onRemove?: () => void;
onMoveUp?: () => void;
onMoveDown?: () => void;
canMoveUp?: boolean;
canMoveDown?: boolean;
-}> = ({ placementAction, onRemove, onMoveUp, onMoveDown, canMoveUp = true, canMoveDown = true }) => {
+}> = ({ transcriptionAction, placementAction, onRemove, onMoveUp, onMoveDown, canMoveUp = true, canMoveDown = true }) => {
const stopPropagation = (event: MouseEvent) => {
event.stopPropagation();
};
return (
<div className={METADATA_ROW_CONTROLS_CLASSES}>
+ {transcriptionAction && (
+ <Button
+ variant="quiet"
+ size="sm"
+ disabled={transcriptionAction.disabled}
+ onClick={(event) => {
+ stopPropagation(event);
+ transcriptionAction.onClick();
+ }}
+ >
+ {transcriptionAction.busy ? <LoaderIcon className="size-3.5 animate-spin" /> : <FileTextIcon className="size-3.5" />}
+ {transcriptionAction.label}
+ </Button>
+ )}
{placementAction && (
<Button
variant="quiet"
@@ -116,6 +145,7 @@ const AttachmentItemActions: FC<{
};
const AttachmentItemCard: FC<{
+ transcriptionAction?: { label: string; onClick: () => void; disabled: boolean; busy: boolean };
item: AttachmentItem;
onPreview?: () => void;
onRemove?: () => void;
@@ -125,7 +155,18 @@ const AttachmentItemCard: FC<{
canMoveDown?: boolean;
isUploadingInline?: boolean;
placementAction?: { label: string; onClick: () => void; disabled?: boolean };
-}> = ({ item, onPreview, onRemove, onMoveUp, onMoveDown, canMoveUp = true, canMoveDown = true, isUploadingInline, placementAction }) => {
+}> = ({
+ item,
+ onPreview,
+ onRemove,
+ onMoveUp,
+ onMoveDown,
+ canMoveUp = true,
+ canMoveDown = true,
+ isUploadingInline,
+ placementAction,
+ transcriptionAction,
+}) => {
const t = useTranslate();
const { category, filename, thumbnailUrl, mimeType, size, sourceUrl, isVoiceNote, audioMeta } = item;
const audioRef = useRef<HTMLAudioElement>(null);
@@ -226,6 +267,7 @@ const AttachmentItemCard: FC<{
<MetadataRowDetail parts={detailParts} />
<AttachmentItemActions
+ transcriptionAction={transcriptionAction}
placementAction={placementAction}
onRemove={onRemove}
onMoveUp={onMoveUp}
@@ -248,6 +290,9 @@ const AttachmentListEditor: FC<AttachmentListEditorProps> = ({
onInsertLocalFiles,
placementActionsDisabled = false,
uploadingLocalFileURLs = new Set(),
+ onTranscribeAttachment,
+ transcribingAttachment,
+ transcriptionDisabled = false,
}) => {
const t = useTranslate();
const [previewState, setPreviewState] = useState<{ open: boolean; initialIndex: number }>({ open: false, initialIndex: 0 });
@@ -385,6 +430,19 @@ const AttachmentListEditor: FC<AttachmentListEditorProps> = ({
const itemAttachments = collectMembers(attachmentsByName, item.memberIds);
const itemLocalFiles = collectMembers(localFilesByPreviewUrl, item.memberIds);
const isUploadingInline = item.isLocal && item.memberIds.some((memberID) => uploadingLocalFileURLs.has(memberID));
+ const isTranscribing = item.id === transcribingAttachment;
+ // External links are fetched cross-origin by the browser, which most hosts refuse.
+ const transcribableAttachment =
+ !item.isLocal && item.category === "audio" && !itemAttachments[0]?.externalLink ? itemAttachments[0] : undefined;
+ const transcriptionAction =
+ transcribableAttachment && onTranscribeAttachment
+ ? {
+ label: t(isTranscribing ? "editor.audio-recorder.transcribing" : "editor.audio-recorder.transcribe"),
+ onClick: () => onTranscribeAttachment(transcribableAttachment),
+ disabled: transcriptionDisabled || Boolean(transcribingAttachment),
+ busy: isTranscribing,
+ }
+ : undefined;
const canInsert =
(item.category === "image" || item.category === "motion") && (item.isLocal || itemAttachments.some(canInlineAttachment));
const placementAction = canInsert
@@ -402,12 +460,13 @@ const AttachmentListEditor: FC<AttachmentListEditorProps> = ({
item={item}
isUploadingInline={isUploadingInline}
placementAction={placementAction}
+ transcriptionAction={transcriptionAction}
onPreview={
item.category === "image" || item.category === "video" || item.category === "motion"
? () => handlePreviewItem(item)
: undefined
}
- onRemove={isUploadingInline ? undefined : () => handleRemoveItem(item)}
+ onRemove={isUploadingInline || isTranscribing ? undefined : () => handleRemoveItem(item)}
onMoveUp={item.isLocal ? () => handleMoveLocalFiles(item.id, -1) : () => handleMoveAttachments(item.id, -1)}
onMoveDown={item.isLocal ? () => handleMoveLocalFiles(item.id, 1) : () => handleMoveAttachments(item.id, 1)}
canMoveUp={itemIndex > 0}
diff --git a/web/tests/attachment-transcription-action.test.tsx b/web/tests/attachment-transcription-action.test.tsx
new file mode 100644
index 000000000..7a223fb0d
--- /dev/null
+++ b/web/tests/attachment-transcription-action.test.tsx
@@ -0,0 +1,54 @@
+import { create } from "@bufbuild/protobuf";
+import { fireEvent, render, screen } from "@testing-library/react";
+import { beforeAll, describe, expect, it, vi } from "vitest";
+import AttachmentListEditor from "@/components/MemoMetadata/Attachment/AttachmentListEditor";
+import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
+
+const audio = create(AttachmentSchema, { name: "attachments/voice", filename: "voice.wav", type: "audio/wav" });
+
+beforeAll(() => {
+ Object.defineProperty(window, "matchMedia", {
+ writable: true,
+ value: () => ({ matches: false, addEventListener: () => undefined, removeEventListener: () => undefined }),
+ });
+});
+
+describe("attachment transcription action", () => {
+ it("only offers transcription for persisted audio when the provider is configured", () => {
+ const props = {
+ attachments: [audio, create(AttachmentSchema, { name: "attachments/document", filename: "document.pdf", type: "application/pdf" })],
+ };
+ const { rerender } = render(<AttachmentListEditor {...props} />);
+ expect(screen.queryByRole("button", { name: "Transcribe" })).not.toBeInTheDocument();
+ const onTranscribeAttachment = vi.fn();
+ rerender(<AttachmentListEditor {...props} onTranscribeAttachment={onTranscribeAttachment} />);
+ expect(screen.getAllByRole("button", { name: "Transcribe" })).toHaveLength(1);
+ fireEvent.click(screen.getByRole("button", { name: "Transcribe" }));
+ expect(onTranscribeAttachment).toHaveBeenCalledExactlyOnceWith(audio);
+ });
+
+ it("shows progress, prevents duplicate transcription and preserves the active recording", () => {
+ render(
+ <AttachmentListEditor
+ attachments={[audio]}
+ onTranscribeAttachment={vi.fn()}
+ transcribingAttachment={audio.name}
+ onAttachmentsChange={vi.fn()}
+ />,
+ );
+ expect(screen.getByRole("button", { name: "Transcribing..." })).toBeDisabled();
+ expect(screen.queryByRole("button", { name: "Remove attachment" })).not.toBeInTheDocument();
+ expect(screen.getByText("voice.wav")).toBeInTheDocument();
+ });
+
+ it("does not offer transcription for externally linked audio", () => {
+ const linked = create(AttachmentSchema, { ...audio, name: "attachments/linked", externalLink: "https://example.com/voice.wav" });
+ render(<AttachmentListEditor attachments={[linked]} onTranscribeAttachment={vi.fn()} />);
+ expect(screen.queryByRole("button", { name: "Transcribe" })).not.toBeInTheDocument();
+ });
+
+ it("disables transcription while the editor is saving or recording", () => {
+ render(<AttachmentListEditor attachments={[audio]} onTranscribeAttachment={vi.fn()} transcriptionDisabled />);
+ expect(screen.getByRole("button", { name: "Transcribe" })).toBeDisabled();
+ });
+});
diff --git a/web/tests/memo-editor-transcription.test.tsx b/web/tests/memo-editor-transcription.test.tsx
new file mode 100644
index 000000000..5bca2592b
--- /dev/null
+++ b/web/tests/memo-editor-transcription.test.tsx
@@ -0,0 +1,69 @@
+import { create } from "@bufbuild/protobuf";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { act, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { describe, expect, it, vi } from "vitest";
+import MemoEditor from "@/components/MemoEditor";
+import { transcriptionService } from "@/components/MemoEditor/services";
+import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
+import { MemoSchema, Visibility } from "@/types/proto/api/v1/memo_service_pb";
+
+vi.mock("@/hooks/useUserQueries", () => ({ useTagCounts: () => ({ data: {} }) }));
+vi.mock("@/hooks/useCurrentUser", () => ({ default: () => ({ name: "users/1" }) }));
+vi.mock("@/contexts/AuthContext", () => ({ useAuth: () => ({ userGeneralSetting: {} }) }));
+vi.mock("@/contexts/InstanceContext", () => ({
+ useInstance: () => ({
+ aiSetting: { providers: [{ id: "configured", apiKeySet: true }], transcription: { providerId: "configured" } },
+ fetchSetting: vi.fn().mockResolvedValue(undefined),
+ }),
+}));
+vi.mock("@/contexts/NewMemoContext", () => ({ useNewMemo: () => ({ markNewMemo: vi.fn() }) }));
+const clients = vi.hoisted(() => ({ getMemo: vi.fn(), updateMemo: vi.fn(), listMemoAttachments: vi.fn() }));
+vi.mock("@/connect", () => ({ attachmentServiceClient: {}, memoServiceClient: clients, userServiceClient: {} }));
+vi.stubGlobal("matchMedia", () => ({ matches: false, addEventListener() {}, removeEventListener() {} }));
+vi.stubGlobal(
+ "ResizeObserver",
+ class {
+ observe() {}
+ unobserve() {}
+ disconnect() {}
+ },
+);
+
+describe("existing memo audio transcription", () => {
+ it("blocks save while transcribing, inserts into the editable draft and retains the original attachment on save", async () => {
+ const attachment = create(AttachmentSchema, { name: "attachments/voice", filename: "voice.wav", type: "audio/wav" });
+ const memo = create(MemoSchema, {
+ name: "memos/meeting",
+ creator: "users/1",
+ content: "Original meeting note",
+ visibility: Visibility.PRIVATE,
+ attachments: [attachment],
+ });
+ clients.getMemo.mockResolvedValue(memo);
+ clients.updateMemo.mockImplementation(async ({ memo: update }) => create(MemoSchema, { ...memo, ...update }));
+ const pending = Promise.withResolvers<string>();
+ vi.spyOn(transcriptionService, "transcribeAttachment").mockReturnValue(pending.promise);
+ const { container } = render(
+ <QueryClientProvider client={new QueryClient()}>
+ <MemoEditor memo={memo} />
+ </QueryClientProvider>,
+ );
+ await waitFor(() => expect(container.querySelector(".cm-content")?.textContent).toBe(memo.content));
+ fireEvent.click(screen.getByRole("button", { name: "Transcribe" }));
+ expect(screen.getByRole("button", { name: "Update" })).toBeDisabled();
+ await act(async () => {
+ pending.resolve("Recognized meeting discussion");
+ await pending.promise;
+ });
+ await waitFor(() => expect(container.querySelector(".cm-content")?.textContent).toContain("Recognized meeting discussion"));
+ expect(container.querySelector(".cm-content")?.textContent).toContain(memo.content);
+ expect(screen.getByText("voice.wav")).toBeInTheDocument();
+ expect(clients.updateMemo).not.toHaveBeenCalled();
+ fireEvent.click(screen.getByRole("button", { name: "Update" }));
+ await waitFor(() => expect(clients.updateMemo).toHaveBeenCalledOnce());
+ const request = clients.updateMemo.mock.calls[0][0];
+ expect(request.memo.content).toContain("Recognized meeting discussion");
+ expect(request.memo.content).toContain(memo.content);
+ expect(request.updateMask.paths).not.toContain("attachments");
+ });
+});
diff --git a/web/tests/transcription-service.test.ts b/web/tests/transcription-service.test.ts
new file mode 100644
index 000000000..13fa86026
--- /dev/null
+++ b/web/tests/transcription-service.test.ts
@@ -0,0 +1,80 @@
+import { create } from "@bufbuild/protobuf";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
+
+const { transcribe } = vi.hoisted(() => ({ transcribe: vi.fn() }));
+vi.mock("@/connect", () => ({ aiServiceClient: { transcribe } }));
+
+import { transcriptionService } from "@/components/MemoEditor/services/transcriptionService";
+
+const audio = create(AttachmentSchema, { name: "attachments/voice", filename: "voice.wav", type: "audio/wav", size: 3n });
+
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.unstubAllGlobals();
+ transcribe.mockReset();
+});
+
+describe("existing audio transcription", () => {
+ it("downloads authenticated audio and sends its bytes and metadata to the configured AI service", async () => {
+ const fetchAudio = vi.fn().mockResolvedValue(new Response("abc"));
+ vi.stubGlobal("fetch", fetchAudio);
+ transcribe.mockResolvedValue({ text: "Meeting notes" });
+ const controller = new AbortController();
+
+ await expect(transcriptionService.transcribeAttachment(audio, controller.signal)).resolves.toBe("Meeting notes");
+
+ expect(fetchAudio).toHaveBeenCalledWith(`${window.location.origin}/file/attachments/voice/voice.wav`, {
+ credentials: "same-origin",
+ signal: controller.signal,
+ });
+ expect(transcribe).toHaveBeenCalledWith(
+ expect.objectContaining({
+ audio: expect.objectContaining({
+ filename: "voice.wav",
+ contentType: "audio/wav",
+ source: { case: "content", value: new Uint8Array([97, 98, 99]) },
+ }),
+ }),
+ { signal: controller.signal },
+ );
+ });
+
+ it("does not send download failures to the AI provider", async () => {
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response(null, { status: 403 })));
+ await expect(transcriptionService.transcribeAttachment(audio)).rejects.toThrow("403");
+ expect(transcribe).not.toHaveBeenCalled();
+ });
+
+ it("rejects oversized metadata before downloading", async () => {
+ const fetchAudio = vi.fn();
+ vi.stubGlobal("fetch", fetchAudio);
+ await expect(transcriptionService.transcribeAttachment(create(AttachmentSchema, { ...audio, size: 26214401n }))).rejects.toThrow(
+ "25 MiB",
+ );
+ expect(fetchAudio).not.toHaveBeenCalled();
+ });
+
+ it("rejects oversized download headers before reading the body", async () => {
+ const response = new Response(null, { headers: { "Content-Length": "26214401" } });
+ const read = vi.spyOn(response, "arrayBuffer");
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response));
+ await expect(transcriptionService.transcribeAttachment(audio)).rejects.toThrow("25 MiB");
+ expect(read).not.toHaveBeenCalled();
+ expect(transcribe).not.toHaveBeenCalled();
+ });
+
+ it("checks actual body size when the server omits Content-Length", async () => {
+ const body = new ArrayBuffer(26214401);
+ vi.stubGlobal("fetch", vi.fn().mockResolvedValue({ ok: true, headers: new Headers(), arrayBuffer: async () => body }));
+ await expect(transcriptionService.transcribeAttachment(audio)).rejects.toThrow("25 MiB");
+ expect(transcribe).not.toHaveBeenCalled();
+ });
+
+ it("does not start a transcription after cancellation", async () => {
+ const controller = new AbortController();
+ controller.abort();
+ await expect(transcriptionService.transcribeFile(new File(["abc"], "voice.wav"), controller.signal)).rejects.toThrow();
+ expect(transcribe).not.toHaveBeenCalled();
+ });
+});
diff --git a/web/tests/use-attachment-transcription.test.tsx b/web/tests/use-attachment-transcription.test.tsx
new file mode 100644
index 000000000..ad7b237ae
--- /dev/null
+++ b/web/tests/use-attachment-transcription.test.tsx
@@ -0,0 +1,93 @@
+import { create } from "@bufbuild/protobuf";
+import { Code, ConnectError } from "@connectrpc/connect";
+import { act, renderHook } from "@testing-library/react";
+import { toast } from "react-hot-toast";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { useAttachmentTranscription } from "@/components/MemoEditor/hooks/useAttachmentTranscription";
+import { transcriptionService } from "@/components/MemoEditor/services";
+import { AttachmentSchema } from "@/types/proto/api/v1/attachment_service_pb";
+
+vi.mock("react-hot-toast", () => ({ toast: { error: vi.fn(), success: vi.fn() } }));
+const audio = create(AttachmentSchema, { name: "attachments/voice", filename: "voice.wav", type: "audio/wav" });
+afterEach(() => {
+ vi.restoreAllMocks();
+ vi.clearAllMocks();
+});
+
+describe("attachment transcription lifecycle", () => {
+ it("inserts trimmed text once, keeping the attachment unchanged and rejecting concurrent requests", async () => {
+ const pending = Promise.withResolvers<string>();
+ const transcribe = vi.spyOn(transcriptionService, "transcribeAttachment").mockReturnValue(pending.promise);
+ const onText = vi.fn();
+ const original = create(AttachmentSchema, audio);
+ const { result } = renderHook(() => useAttachmentTranscription(onText));
+ let completion: Promise<void>;
+ act(() => {
+ completion = result.current.transcribeAttachment(audio);
+ });
+ expect(result.current.transcribingAttachment).toBe(audio.name);
+ await act(() => result.current.transcribeAttachment(audio));
+ expect(transcribe).toHaveBeenCalledTimes(1);
+ await act(async () => {
+ pending.resolve(" Meeting notes ");
+ await completion;
+ });
+ expect(onText).toHaveBeenCalledExactlyOnceWith("Meeting notes");
+ expect(audio).toEqual(original);
+ expect(result.current.transcribingAttachment).toBeUndefined();
+ });
+
+ it.each(["", " "])("leaves the draft unchanged on an empty transcript", async (text) => {
+ vi.spyOn(transcriptionService, "transcribeAttachment").mockResolvedValue(text);
+ const onText = vi.fn();
+ const { result } = renderHook(() => useAttachmentTranscription(onText));
+ await act(() => result.current.transcribeAttachment(audio));
+ expect(onText).not.toHaveBeenCalled();
+ expect(toast.error).toHaveBeenCalledWith("No speech detected");
+ expect(result.current.transcribingAttachment).toBeUndefined();
+ });
+
+ it("allows retry after a provider failure without modifying the draft", async () => {
+ vi.spyOn(transcriptionService, "transcribeAttachment")
+ .mockRejectedValueOnce(new ConnectError("Provider unavailable", Code.Unavailable))
+ .mockResolvedValueOnce("Retry works");
+ const onText = vi.fn();
+ const { result } = renderHook(() => useAttachmentTranscription(onText));
+ await act(() => result.current.transcribeAttachment(audio));
+ expect(onText).not.toHaveBeenCalled();
+ expect(toast.error).toHaveBeenCalledWith("Provider unavailable");
+ expect(result.current.transcribingAttachment).toBeUndefined();
+ await act(() => result.current.transcribeAttachment(audio));
+ expect(onText).toHaveBeenCalledExactlyOnceWith("Retry works");
+ });
+
+ it("reports download and network failures with the translated fallback", async () => {
+ vi.spyOn(transcriptionService, "transcribeAttachment").mockRejectedValue(new TypeError("Failed to fetch"));
+ const onText = vi.fn();
+ const { result } = renderHook(() => useAttachmentTranscription(onText));
+ await act(() => result.current.transcribeAttachment(audio));
+ expect(onText).not.toHaveBeenCalled();
+ expect(toast.error).toHaveBeenCalledWith("Failed to transcribe audio");
+ expect(result.current.transcribingAttachment).toBeUndefined();
+ });
+
+ it("cancels on editor unmount and ignores a late transcript", async () => {
+ const pending = Promise.withResolvers<string>();
+ const transcribe = vi.spyOn(transcriptionService, "transcribeAttachment").mockReturnValue(pending.promise);
+ const onText = vi.fn();
+ const { result, unmount } = renderHook(() => useAttachmentTranscription(onText));
+ let completion: Promise<void>;
+ act(() => {
+ completion = result.current.transcribeAttachment(audio);
+ });
+ const signal = transcribe.mock.calls[0][1];
+ unmount();
+ expect(signal?.aborted).toBe(true);
+ await act(async () => {
+ pending.resolve("Late result");
+ await completion;
+ });
+ expect(onText).not.toHaveBeenCalled();
+ expect(toast.success).not.toHaveBeenCalled();
+ });
+});