There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(core): call real endpoint for audio transcription (#11139)
pengx17dccd7c20aac3
diff --git a/blocksuite/affine/model/src/blocks/transcription/transcription-model.ts b/blocksuite/affine/model/src/blocks/transcription/transcription-model.tsindex 5821e8d125..1e19d8971f 100644--- a/blocksuite/affine/model/src/blocks/transcription/transcription-model.ts+++ b/blocksuite/affine/model/src/blocks/transcription/transcription-model.ts@@ -6,12 +6,15 @@ import {export const TranscriptionBlockFlavour = 'affine:transcription';+const defaultProps: TranscriptionBlockProps = {+ transcription: {},+ jobId: undefined,+ createdBy: undefined, // the user id of the creator+};+export const TranscriptionBlockSchema = defineBlockSchema({flavour: TranscriptionBlockFlavour,- props: () => ({- transcription: {},- jobId: '',- }),+ props: () => defaultProps,metadata: {version: 1,role: 'attachment-viewer',@@ -23,7 +26,8 @@ export const TranscriptionBlockSchema = defineBlockSchema({export type TranscriptionBlockProps = {transcription: Record<string, any>;- jobId: string;+ jobId?: string;+ createdBy?: string;};export class TranscriptionBlockModel extends BlockModel<TranscriptionBlockProps> {}diff --git a/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts b/packages/frontend/apps/electron-renderer/src/app/effects/recording.tsindex 827485f64c..91ac241cd2 100644--- a/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts+++ b/packages/frontend/apps/electron-renderer/src/app/effects/recording.ts@@ -97,7 +97,9 @@ export function setupRecordingEvents(frameworkProvider: FrameworkProvider) {using audioAttachment = workspace.scope.get(AudioAttachmentService).get(model);- audioAttachment?.obj.transcribe();+ audioAttachment?.obj.transcribe().catch(err => {+ logger.error('Failed to transcribe recording', err);+ });}})().catch(console.error);},diff --git a/packages/frontend/component/src/ui/modal/confirm-modal.tsx b/packages/frontend/component/src/ui/modal/confirm-modal.tsxindex be315c3ce4..e50d8649cf 100644--- a/packages/frontend/component/src/ui/modal/confirm-modal.tsx+++ b/packages/frontend/component/src/ui/modal/confirm-modal.tsx@@ -16,7 +16,7 @@ export interface ConfirmModalProps extends ModalProps {confirmButtonOptions?: Omit<ButtonProps, 'children'>;childrenContentClassName?: string;onConfirm?: (() => void) | (() => Promise<void>);- onCancel?: () => void;+ onCancel?: (() => void) | false;confirmText?: React.ReactNode;cancelText?: React.ReactNode;cancelButtonOptions?: Omit<ButtonProps, 'children'>;@@ -52,6 +52,12 @@ export const ConfirmModal = ({console.error(err);});}, [onConfirm]);+ const handleCancel = useCallback(() => {+ if (onCancel === false) {+ return;+ }+ onCancel?.();+ }, [onCancel]);return (<ModalcontentOptions={{@@ -59,12 +65,12 @@ export const ConfirmModal = ({className: clsx(styles.container, contentOptions?.className),onPointerDownOutside: e => {e.stopPropagation();- onCancel?.();+ handleCancel();},}}width={width}closeButtonOptions={{- onClick: onCancel,+ onClick: handleCancel,}}headerClassName={clsx(styles.header, headerClassName)}descriptionClassName={clsx(styles.description, descriptionClassName)}@@ -81,21 +87,23 @@ export const ConfirmModal = ({reverse: reverseFooter,})}>- <DialogTrigger asChild>- <Button- className={styles.action}- onClick={onCancel}- data-testid="confirm-modal-cancel"- {...cancelButtonOptions}- variant={- cancelButtonOptions?.variant- ? cancelButtonOptions.variant- : 'secondary'- }- >- {cancelText}- </Button>- </DialogTrigger>+ {onCancel !== false ? (+ <DialogTrigger asChild>+ <Button+ className={styles.action}+ onClick={handleCancel}+ data-testid="confirm-modal-cancel"+ {...cancelButtonOptions}+ variant={+ cancelButtonOptions?.variant+ ? cancelButtonOptions.variant+ : 'secondary'+ }+ >+ {cancelText}+ </Button>+ </DialogTrigger>+ ) : null}{CustomConfirmButton ? (<CustomConfirmButton data-testid="confirm-modal-confirm" />) : (diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.tsindex 3e2e9d4c23..8182ffac72 100644--- a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts+++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.css.ts@@ -22,4 +22,17 @@ export const notesButtonIcon = style({fontSize: 24,width: '1em',height: '1em',+ display: 'inline-flex',+ alignItems: 'center',+ justifyContent: 'center',+});++export const error = style({+ color: cssVarV2('aI/errorText'),+});++export const publicUserLabel = style({+ fontSize: cssVar('fontXs'),+ fontWeight: 500,+ userSelect: 'none',});diff --git a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsxindex 895bd6f87f..ab54c6ab9d 100644--- a/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx+++ b/packages/frontend/core/src/blocksuite/attachment-viewer/audio/audio-block.tsx@@ -1,13 +1,17 @@-import { Button, Tooltip } from '@affine/component';+import { Button, Tooltip, useConfirmModal } from '@affine/component';import { AudioPlayer } from '@affine/core/components/audio-player';import { AnimatedTranscribeIcon } from '@affine/core/components/audio-player/lottie/animated-transcribe-icon';import { useSeekTime } from '@affine/core/components/audio-player/use-seek-time';import { useEnableAI } from '@affine/core/components/hooks/affine/use-enable-ai';+import { useAsyncCallback } from '@affine/core/components/hooks/affine-async-hooks';+import { CurrentServerScopeProvider } from '@affine/core/components/providers/current-server-scope';+import { PublicUserLabel } from '@affine/core/modules/cloud/views/public-user';+import { GlobalDialogService } from '@affine/core/modules/dialogs';import type { AudioAttachmentBlock } from '@affine/core/modules/media/entities/audio-attachment-block';import { useAttachmentMediaBlock } from '@affine/core/modules/media/views/use-attachment-media';-import { useI18n } from '@affine/i18n';-import { useLiveData } from '@toeverything/infra';-import { useCallback, useMemo } from 'react';+import { Trans, useI18n } from '@affine/i18n';+import { useLiveData, useService } from '@toeverything/infra';+import { useCallback, useMemo, useState } from 'react';import type { AttachmentViewerProps } from '../types';import * as styles from './audio-block.css';@@ -19,12 +23,15 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {const stats = useLiveData(audioMedia.stats$);const loading = useLiveData(audioMedia.loading$);const expanded = useLiveData(block.expanded$);- const transcribing = useLiveData(block.transcribing$);- const transcribed = useLiveData(block.transcribed$);+ const [preflightChecking, setPreflightChecking] = useState(false);+ const transcribing =+ useLiveData(block.transcriptionJob.transcribing$) || preflightChecking;+ const error = useLiveData(block.transcriptionJob.error$);+ const transcribed = useLiveData(block.hasTranscription$);const handleClick = useCallback((e: React.MouseEvent<HTMLDivElement>) => {e.stopPropagation();}, []);-+ const confirmModal = useConfirmModal();const seekTime = useSeekTime(playbackState, stats.duration);const handlePlay = useCallback(() => {@@ -50,6 +57,66 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {const enableAi = useEnableAI();+ const globalDialogService = useService(GlobalDialogService);++ const handleNotesClick = useAsyncCallback(async () => {+ if (!enableAi || transcribing) {+ return;+ }++ if (transcribed) {+ block.expanded$.setValue(!expanded);+ return;+ }++ if (!block.transcriptionJob.currentUserId) {+ confirmModal.openConfirmModal({+ title: t['com.affine.ai.login-required.dialog-title'](),+ description: t['com.affine.ai.login-required.dialog-content'](),+ confirmText: t['com.affine.ai.login-required.dialog-confirm'](),+ confirmButtonOptions: {+ variant: 'primary',+ },+ cancelText: t['com.affine.ai.login-required.dialog-cancel'](),+ onConfirm: () => {+ globalDialogService.open('sign-in', {});+ },+ });+ return;+ }++ setPreflightChecking(true);+ const result = await block.transcriptionJob.preflightCheck();+ setPreflightChecking(false);+ if (result?.error === 'created-by-others') {+ confirmModal.openConfirmModal({+ title: t['com.affine.audio.transcribe.non-owner.confirm.title'](),+ description: (+ <Trans i18nKey="com.affine.audio.transcribe.non-owner.confirm.message">+ Please contact <PublicUserLabel id={result.userId} /> to upgrade AI+ rights or resend the attachment.+ </Trans>+ ),+ onCancel: false,+ confirmText: t['Confirm'](),+ confirmButtonOptions: {+ variant: 'primary',+ },+ });+ } else {+ await block.transcribe();+ }+ }, [+ enableAi,+ transcribing,+ transcribed,+ block,+ expanded,+ confirmModal,+ t,+ globalDialogService,+ ]);+const notesEntry = useMemo(() => {if (!enableAi) {return null;@@ -62,37 +129,37 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {state={transcribing ? 'transcribing' : 'idle'}/>}- disabled={transcribing}size="large"prefixClassName={styles.notesButtonIcon}className={styles.notesButton}- onClick={() => {- if (transcribed) {- block.expanded$.setValue(!expanded);- } else {- block.transcribe();- }- }}+ onClick={handleNotesClick}>- {t['com.affine.attachmentViewer.audio.notes']()}+ {transcribing+ ? t['com.affine.audio.transcribing']()+ : t['com.affine.audio.notes']()}</Button>);if (transcribing) {return (- <Tooltip- content={t['com.affine.attachmentViewer.audio.transcribing']()}- >+ <Tooltip content={t['com.affine.audio.transcribing']()}>{inner}</Tooltip>);}return inner;- }, [enableAi, transcribing, t, transcribed, block, expanded]);+ }, [enableAi, transcribing, handleNotesClick, t]);++ const sizeEntry = useMemo(() => {+ if (error) {+ return <div className={styles.error}>{error.message}</div>;+ }+ return block.props.props.size;+ }, [error, block.props.props.size]);return (<AudioPlayername={block.props.props.name}- size={block.props.props.size}+ size={sizeEntry}loading={loading}playbackState={playbackState?.state || 'idle'}waveform={stats.waveform}@@ -103,7 +170,9 @@ const AttachmentAudioPlayer = ({ block }: { block: AudioAttachmentBlock }) => {onPause={handlePause}onStop={handleStop}onSeek={handleSeek}- notesEntry={notesEntry}+ notesEntry={+ <CurrentServerScopeProvider>{notesEntry}</CurrentServerScopeProvider>+ }/>);};diff --git a/packages/frontend/core/src/components/audio-player/audio-player.tsx b/packages/frontend/core/src/components/audio-player/audio-player.tsxindex 193053df5b..058167ef66 100644--- a/packages/frontend/core/src/components/audio-player/audio-player.tsx+++ b/packages/frontend/core/src/components/audio-player/audio-player.tsx@@ -23,7 +23,7 @@ const formatTime = (seconds: number): string => {export interface AudioPlayerProps {// Audio metadataname: string;- size: number;+ size: number | ReactNode; // the size entry may be used for drawing error messagewaveform: number[] | null;// Playback stateplaybackState: 'idle' | 'playing' | 'paused' | 'stopped';@@ -97,7 +97,9 @@ export const AudioPlayer = ({<div className={styles.nameLabel}>{name}</div></div><div className={styles.upperRow}>- <div className={styles.sizeInfo}>{bytes(size)}</div>+ <div className={styles.sizeInfo}>+ {typeof size === 'number' ? bytes(size) : size}+ </div></div></div><div className={styles.upperRight}>diff --git a/packages/frontend/core/src/components/audio-player/lottie/styles.css.ts b/packages/frontend/core/src/components/audio-player/lottie/styles.css.tsindex ccaf638058..cf8e30183d 100644--- a/packages/frontend/core/src/components/audio-player/lottie/styles.css.ts+++ b/packages/frontend/core/src/components/audio-player/lottie/styles.css.ts@@ -1,6 +1,8 @@import { cssVarV2 } from '@toeverything/theme/v2';import { globalStyle, style } from '@vanilla-extract/css';-export const root = style({});+export const root = style({+ display: 'inline-flex',+});// replace primary colors to cssVarV2('icon/primary')const iconPrimaryColors = [diff --git a/packages/frontend/core/src/components/audio-player/use-seek-time.ts b/packages/frontend/core/src/components/audio-player/use-seek-time.tsindex e80b4d77a2..7d01ddf74d 100644--- a/packages/frontend/core/src/components/audio-player/use-seek-time.ts+++ b/packages/frontend/core/src/components/audio-player/use-seek-time.ts@@ -26,7 +26,7 @@ export const useSeekTime = (playbackState.state === 'playing'? (Date.now() - playbackState.updateTime) / 1000: 0;- // if timeElapsed + playbackState.seekOffset is closed to duration,+ // if timeElapsed + playbackState.seekOffset is close to duration,// set seekTime to duration// this is to avoid the seek time being set to a value that is not exactly the same as the duration// at the end of the audiodiff --git a/packages/frontend/core/src/components/providers/current-server-scope.tsx b/packages/frontend/core/src/components/providers/current-server-scope.tsxnew file mode 100644index 0000000000..bf28a6715d--- /dev/null+++ b/packages/frontend/core/src/components/providers/current-server-scope.tsx@@ -0,0 +1,47 @@+import { ServersService } from '@affine/core/modules/cloud';+import { GlobalContextService } from '@affine/core/modules/global-context';+import { FrameworkScope, useLiveData, useService } from '@toeverything/infra';+import { useMemo } from 'react';++export const CurrentServerScopeProvider = ({+ children,+}: {+ children: React.ReactNode;+}) => {+ const globalContext = useService(GlobalContextService).globalContext;+ const serversService = useService(ServersService);+ const currentServerId = useLiveData(globalContext.serverId.$);+ const serverService = useLiveData(+ useMemo(() => {+ if (!currentServerId) {+ return null;+ }+ return serversService.server$(currentServerId);+ }, [currentServerId, serversService])+ );++ if (!serverService) {+ // todo(@pengx17): render a loading/error component here if not found?+ return null;+ }++ return (+ <FrameworkScope scope={serverService.scope}>{children}</FrameworkScope>+ );+};++export const useCurrentServerService = () => {+ const globalContext = useService(GlobalContextService).globalContext;+ const serversService = useService(ServersService);+ const currentServerId = useLiveData(globalContext.serverId.$);+ const serverService = useLiveData(+ useMemo(() => {+ if (!currentServerId) {+ return null;+ }+ return serversService.server$(currentServerId);+ }, [currentServerId, serversService])+ );++ return serverService ?? undefined;+};diff --git a/packages/frontend/core/src/modules/cloud/views/public-user.css.ts b/packages/frontend/core/src/modules/cloud/views/public-user.css.tsnew file mode 100644index 0000000000..eba825fd8e--- /dev/null+++ b/packages/frontend/core/src/modules/cloud/views/public-user.css.ts@@ -0,0 +1,20 @@+import { style } from '@vanilla-extract/css';++export const publicUserLabel = style({+ fontSize: 'inherit',+});++export const publicUserLabelLoading = style([+ publicUserLabel,+ {+ opacity: 0.5,+ },+]);++export const publicUserLabelRemoved = style([+ publicUserLabel,+ {+ opacity: 0.5,+ textDecoration: 'line-through',+ },+]);diff --git a/packages/frontend/core/src/modules/cloud/views/public-user.tsx b/packages/frontend/core/src/modules/cloud/views/public-user.tsxnew file mode 100644index 0000000000..be2d48ac5e--- /dev/null+++ b/packages/frontend/core/src/modules/cloud/views/public-user.tsx@@ -0,0 +1,38 @@+import { useCurrentServerService } from '@affine/core/components/providers/current-server-scope';+import { useI18n } from '@affine/i18n';+import { useLiveData } from '@toeverything/infra';+import { useLayoutEffect, useMemo } from 'react';++import { PublicUserService } from '../services/public-user';+import * as styles from './public-user.css';++export const PublicUserLabel = ({ id }: { id: string }) => {+ const serverService = useCurrentServerService();+ const publicUser = useMemo(() => {+ return serverService?.scope.get(PublicUserService);+ }, [serverService]);++ useLayoutEffect(() => {+ if (publicUser) {+ publicUser.revalidate(id);+ }+ }, [id, publicUser]);++ const user = useLiveData(publicUser?.publicUser$(id));+ const isLoading = useLiveData(publicUser?.isLoading$(id));+ const t = useI18n();++ if (isLoading && !user) {+ return <span className={styles.publicUserLabelLoading}>...</span>;+ }++ if (user?.removed) {+ return (+ <span className={styles.publicUserLabelRemoved}>+ {t['Unknown User']()}+ </span>+ );+ }++ return <span className={styles.publicUserLabel}>{user?.name}</span>;+};diff --git a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.tsindex 3696114767..879fe44687 100644--- a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts+++ b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts@@ -1,3 +1,5 @@+import { DebugLogger } from '@affine/debug';+import { AiJobStatus } from '@affine/graphql';import {type AttachmentBlockModel,TranscriptionBlockFlavour,@@ -6,31 +8,16 @@ import {import type { AffineTextAttributes } from '@blocksuite/affine/shared/types';import { type DeltaInsert, Text } from '@blocksuite/affine/store';import { computed } from '@preact/signals-core';-import {- catchErrorInto,- effect,- Entity,- fromPromise,- LiveData,- onComplete,- onStart,-} from '@toeverything/infra';+import { Entity, LiveData } from '@toeverything/infra';import { cssVarV2 } from '@toeverything/theme/v2';-import { EMPTY, mergeMap, switchMap } from 'rxjs';+import type { WorkspaceService } from '../../workspace';import type { AudioMediaManagerService } from '../services/audio-media-manager';import type { AudioMedia } from './audio-media';+import { AudioTranscriptionJob } from './audio-transcription-job';+import type { TranscriptionResult } from './types';-export interface TranscriptionResult {- title: string;- summary: string;- segments: {- speaker: string;- start_time: string;- end_time: string;- transcription: string;- }[];-}+const logger = new DebugLogger('audio-attachment-block');// BlockSuiteError: yText must not contain "\r" because it will break the range synchronizationfunction sanitizeText(text: string) {@@ -41,42 +28,22 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {private readonly refCount$ = new LiveData<number>(0);readonly audioMedia: AudioMedia;constructor(- public readonly audioMediaManagerService: AudioMediaManagerService+ readonly audioMediaManagerService: AudioMediaManagerService,+ readonly workspaceService: WorkspaceService) {super();const mediaRef = audioMediaManagerService.ensureMediaEntity(this.props);this.audioMedia = mediaRef.media;this.disposables.push(() => mediaRef.release());+ this.disposables.push(() => {+ this.transcriptionJob.dispose();+ });}// rendering means the attachment is visible in the editor+ // it is used to determine if we should show show the audio player on the sidebarrendering$ = this.refCount$.map(refCount => refCount > 0);expanded$ = new LiveData<boolean>(true);- transcribing$ = new LiveData<boolean>(false);- transcriptionError$ = new LiveData<Error | null>(null);- transcribed$ = LiveData.computed(get => {- const transcriptionBlock = get(this.transcriptionBlock$);- if (!transcriptionBlock) {- return null;- }- const childMap = get(LiveData.fromSignal(transcriptionBlock.childMap));- return childMap.size > 0;- });-- transcribe = effect(- switchMap(() =>- fromPromise(this.doTranscribe()).pipe(- mergeMap(result => {- // attach transcription result to the block- this.fillTranscriptionResult(result);- return EMPTY;- }),- catchErrorInto(this.transcriptionError$),- onStart(() => this.transcribing$.setValue(true)),- onComplete(() => this.transcribing$.setValue(false))- )- )- );readonly transcriptionBlock$ = LiveData.fromSignal(computed(() => {@@ -91,59 +58,108 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {}));- // TODO: use real implementation- private readonly doTranscribe = async (): Promise<TranscriptionResult> => {- try {- const buffer = await this.audioMedia.getBuffer();- if (!buffer) {- throw new Error('No audio buffer available');- }+ hasTranscription$ = LiveData.computed(get => {+ const transcriptionBlock = get(this.transcriptionBlock$);+ if (!transcriptionBlock) {+ return null;+ }+ const childMap = get(LiveData.fromSignal(transcriptionBlock.childMap));+ return childMap.size > 0;+ });- // Send binary audio data directly- const blob = new Blob([buffer], { type: 'audio/wav' }); // adjust mime type if needed- const formData = new FormData();- formData.append('audio', blob);+ transcriptionJob: AudioTranscriptionJob = this.createTranscriptionJob();- const response = await fetch('http://localhost:6544/transcribe', {- method: 'POST',- body: formData,+ mount() {+ if (+ this.transcriptionJob.isCreator() &&+ this.transcriptionJob.status$.value.status === 'waiting-for-job' &&+ !this.hasTranscription$.value+ ) {+ this.transcribe().catch(error => {+ logger.error('Error transcribing audio:', error);});+ }- if (!response.ok) {- throw new Error(`Transcription failed: ${response.statusText}`);- }+ this.refCount$.setValue(this.refCount$.value + 1);+ }- const result = await response.json();- return result.transcription;+ unmount() {+ this.refCount$.setValue(this.refCount$.value - 1);+ }++ private createTranscriptionJob() {+ if (!this.props.props.sourceId) {+ throw new Error('No source id');+ }++ let transcriptionBlockProps = this.transcriptionBlock$.value?.props;++ if (!transcriptionBlockProps) {+ // transcription block is not created yet, we need to create it+ this.props.doc.addBlock(+ 'affine:transcription',+ {+ transcription: {},+ },+ this.props.id+ );+ transcriptionBlockProps = this.transcriptionBlock$.value?.props;+ }++ if (!transcriptionBlockProps) {+ throw new Error('No transcription block props');+ }++ const job = this.framework.createEntity(AudioTranscriptionJob, {+ blobId: this.props.props.sourceId,+ blockProps: transcriptionBlockProps,+ getAudioFile: async () => {+ const buffer = await this.audioMedia.getBuffer();+ if (!buffer) {+ throw new Error('No audio buffer available');+ }+ const blob = new Blob([buffer], { type: this.props.props.type });+ const file = new File([blob], this.props.props.name, {+ type: this.props.props.type,+ });+ return file;+ },+ });++ return job;+ }++ readonly transcribe = async () => {+ try {+ // if job is already running, we should not start it again+ if (this.transcriptionJob.status$.value.status !== 'waiting-for-job') {+ return;+ }+ const status = await this.transcriptionJob.start();+ if (status.status === AiJobStatus.claimed) {+ this.fillTranscriptionResult(status.result);+ }} catch (error) {- console.error('Error transcribing audio:', error);+ logger.error('Error transcribing audio:', error);throw error;}};private readonly fillTranscriptionResult = (result: TranscriptionResult) => {- this.props.props.caption = result.title;- // todo: add transcription block schema etc.- const transcriptionBlockId = this.props.doc.addBlock(- 'affine:transcription',- {- transcription: result,- },- this.props.id- );+ this.props.props.caption = result.title ?? '';const calloutId = this.props.doc.addBlock('affine:callout',{emoji: '💬',},- transcriptionBlockId+ this.transcriptionBlock$.value?.id);// todo: refactor- const spearkerToColors = new Map<string, string>();+ const speakerToColors = new Map<string, string>();for (const segment of result.segments) {- let color = spearkerToColors.get(segment.speaker);+ let color = speakerToColors.get(segment.speaker);const colorOptions = [cssVarV2.text.highlight.fg.red,cssVarV2.text.highlight.fg.green,@@ -156,12 +172,12 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {cssVarV2.text.highlight.fg.magenta,];if (!color) {- color = colorOptions[spearkerToColors.size % colorOptions.length];- spearkerToColors.set(segment.speaker, color);+ color = colorOptions[speakerToColors.size % colorOptions.length];+ speakerToColors.set(segment.speaker, color);}const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [{- insert: sanitizeText(segment.start_time + ' ' + segment.speaker),+ insert: sanitizeText(segment.start + ' ' + segment.speaker),attributes: {color,bold: true,@@ -180,12 +196,4 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {);}};-- mount() {- this.refCount$.setValue(this.refCount$.value + 1);- }-- unmount() {- this.refCount$.setValue(this.refCount$.value - 1);- }}diff --git a/packages/frontend/core/src/modules/media/entities/audio-media.ts b/packages/frontend/core/src/modules/media/entities/audio-media.tsindex 964f3eecc5..cff3570290 100644--- a/packages/frontend/core/src/modules/media/entities/audio-media.ts+++ b/packages/frontend/core/src/modules/media/entities/audio-media.ts@@ -162,7 +162,7 @@ export class AudioMedia extends Entity<AudioSource> {const startTime = performance.now();// calculating audio stats is expensive. Maybe persist the result in cache?- const stats = await this.calcuateStatsFromBuffer(blob);+ const stats = await this.calculateStatsFromBuffer(blob);logger.debug(`Calculate audio stats time: ${performance.now() - startTime}ms`);@@ -177,9 +177,8 @@ export class AudioMedia extends Entity<AudioSource> {return fromPromise(async () => {return this.loadAudioBuffer();}).pipe(- mergeMap(({ blob, duration, waveform }) => {+ mergeMap(({ blob, waveform }) => {const url = URL.createObjectURL(blob);- this.duration$.setValue(duration);// Set the audio element sourcethis.audioElement.src = url;this.waveform$.setValue(waveform);@@ -187,6 +186,9 @@ export class AudioMedia extends Entity<AudioSource> {if (this.playbackState$.getValue().state === 'playing') {this.play(true);}+ this.audioElement.onloadedmetadata = () => {+ this.duration$.setValue(this.audioElement.duration);+ };return EMPTY;}),onStart(() => this.loading$.setValue(true)),@@ -397,13 +399,13 @@ export class AudioMedia extends Entity<AudioSource> {return this.playbackState$.getValue();}- private async calcuateStatsFromBuffer(buffer: Blob) {+ private async calculateStatsFromBuffer(buffer: Blob) {const audioContext = new AudioContext();const audioBuffer = await audioContext.decodeAudioData(await buffer.arrayBuffer());const waveform = await this.calculateWaveform(audioBuffer);- return { waveform, duration: audioBuffer.duration };+ return { waveform };}/**diff --git a/packages/frontend/core/src/modules/media/entities/audio-transcription-job-store.ts b/packages/frontend/core/src/modules/media/entities/audio-transcription-job-store.tsnew file mode 100644index 0000000000..82ad498f3b--- /dev/null+++ b/packages/frontend/core/src/modules/media/entities/audio-transcription-job-store.ts@@ -0,0 +1,96 @@+import {+ claimAudioTranscriptionMutation,+ getAudioTranscriptionQuery,+ submitAudioTranscriptionMutation,+} from '@affine/graphql';+import { Entity } from '@toeverything/infra';++import type { DefaultServerService, WorkspaceServerService } from '../../cloud';+import { GraphQLService } from '../../cloud/services/graphql';+import type { WorkspaceService } from '../../workspace';++export class AudioTranscriptionJobStore extends Entity<{+ readonly blobId: string;+ readonly getAudioFile: () => Promise<File>;+}> {+ constructor(+ private readonly workspaceService: WorkspaceService,+ private readonly workspaceServerService: WorkspaceServerService,+ private readonly defaultServerService: DefaultServerService+ ) {+ super();+ }++ private get serverService() {+ return (+ this.workspaceServerService.server || this.defaultServerService.server+ );+ }++ private get graphqlService() {+ return this.serverService?.scope.get(GraphQLService);+ }++ private get currentWorkspaceId() {+ return this.workspaceService.workspace.id;+ }++ submitAudioTranscription = async () => {+ const graphqlService = this.graphqlService;+ if (!graphqlService) {+ throw new Error('No graphql service available');+ }+ const file = await this.props.getAudioFile();+ const response = await graphqlService.gql({+ query: submitAudioTranscriptionMutation,+ variables: {+ workspaceId: this.currentWorkspaceId,+ blobId: this.props.blobId,+ blob: file,+ },+ });+ if (!response.submitAudioTranscription?.id) {+ throw new Error('Failed to submit audio transcription');+ }+ return response.submitAudioTranscription;+ };++ getAudioTranscription = async (blobId: string, jobId?: string) => {+ const graphqlService = this.graphqlService;+ if (!graphqlService) {+ throw new Error('No graphql service available');+ }+ const currentWorkspaceId = this.currentWorkspaceId;+ if (!currentWorkspaceId) {+ throw new Error('No current workspace id');+ }+ const response = await graphqlService.gql({+ query: getAudioTranscriptionQuery,+ variables: {+ workspaceId: currentWorkspaceId,+ jobId,+ blobId,+ },+ });+ if (!response.currentUser?.copilot?.audioTranscription) {+ return null;+ }+ return response.currentUser.copilot.audioTranscription;+ };+ claimAudioTranscription = async (jobId: string) => {+ const graphqlService = this.graphqlService;+ if (!graphqlService) {+ throw new Error('No graphql service available');+ }+ const response = await graphqlService.gql({+ query: claimAudioTranscriptionMutation,+ variables: {+ jobId,+ },+ });+ if (!response.claimAudioTranscription) {+ throw new Error('Failed to claim transcription result');+ }+ return response.claimAudioTranscription;+ };+}diff --git a/packages/frontend/core/src/modules/media/entities/audio-transcription-job.ts b/packages/frontend/core/src/modules/media/entities/audio-transcription-job.tsnew file mode 100644index 0000000000..2a5d1a0eea--- /dev/null+++ b/packages/frontend/core/src/modules/media/entities/audio-transcription-job.ts@@ -0,0 +1,281 @@+import { shallowEqual } from '@affine/component';+import { DebugLogger } from '@affine/debug';+import { UserFriendlyError } from '@affine/error';+import { AiJobStatus } from '@affine/graphql';+import type { TranscriptionBlockProps } from '@blocksuite/affine/model';+import { Entity, LiveData } from '@toeverything/infra';++import type { DefaultServerService, WorkspaceServerService } from '../../cloud';+import { AuthService } from '../../cloud/services/auth';+import { AudioTranscriptionJobStore } from './audio-transcription-job-store';+import type { TranscriptionResult } from './types';++// The UI status of the transcription job+export type TranscriptionStatus =+ | {+ status: 'waiting-for-job';+ }+ | {+ status: 'started';+ }+ | {+ status: AiJobStatus.pending;+ }+ | {+ status: AiJobStatus.running;+ }+ | {+ status: AiJobStatus.failed;+ error: UserFriendlyError; // <<- this is not visible on UI yet+ }+ | {+ status: AiJobStatus.finished; // ready to be claimed, but may be rejected because of insufficient credits+ }+ | {+ status: AiJobStatus.claimed;+ result: TranscriptionResult;+ };++const logger = new DebugLogger('audio-transcription-job');++// facts on transcription job ownership+// 1. jobid + blobid is unique for a given user+// 2. only the creator can claim the job+// 3. all users can query the claimed job result+// 4. claim a job requires AI credits+export class AudioTranscriptionJob extends Entity<{+ readonly blockProps: TranscriptionBlockProps;+ readonly blobId: string;+ readonly getAudioFile: () => Promise<File>;+}> {+ constructor(+ private readonly workspaceServerService: WorkspaceServerService,+ private readonly defaultServerService: DefaultServerService+ ) {+ super();+ this.disposables.push(() => {+ this.disposed = true;+ });+ }++ disposed = false;++ private readonly _status$ = new LiveData<TranscriptionStatus>({+ status: 'waiting-for-job',+ });++ private readonly store = this.framework.createEntity(+ AudioTranscriptionJobStore,+ {+ blobId: this.props.blobId,+ getAudioFile: this.props.getAudioFile,+ }+ );++ status$ = this._status$.distinctUntilChanged(shallowEqual);+ transcribing$ = this.status$.map(status => {+ return (+ status.status === 'started' ||+ status.status === AiJobStatus.pending ||+ status.status === AiJobStatus.running ||+ status.status === AiJobStatus.finished+ );+ });++ error$ = this.status$.map(status => {+ if (status.status === AiJobStatus.failed) {+ return status.error;+ }+ return null;+ });++ // check if we can kick start the transcription job+ readonly preflightCheck = async () => {+ // if the job id is given, check if the job exists+ if (this.props.blockProps.jobId) {+ const existingJob = await this.store.getAudioTranscription(+ this.props.blobId,+ this.props.blockProps.jobId+ );++ if (existingJob?.status === AiJobStatus.claimed) {+ // if job exists, anyone can query it+ return;+ }++ if (+ !existingJob &&+ this.props.blockProps.createdBy &&+ this.props.blockProps.createdBy !== this.currentUserId+ ) {+ return {+ error: 'created-by-others',+ userId: this.props.blockProps.createdBy,+ };+ }+ }++ // if no job id, anyone can start a new job+ return;+ };++ async start() {+ if (this.disposed) {+ logger.debug('Job already disposed, cannot start');+ throw new Error('Job already disposed');+ }++ this._status$.value = {+ status: 'started',+ };++ try {+ // firstly check if there is a job already+ logger.debug('Checking for existing transcription job', {+ blobId: this.props.blobId,+ jobId: this.props.blockProps.jobId,+ });+ let job: {+ id: string;+ status: AiJobStatus;+ } | null = await this.store.getAudioTranscription(+ this.props.blobId,+ this.props.blockProps.jobId+ );++ if (!job) {+ logger.debug('No existing job found, submitting new transcription job');+ job = await this.store.submitAudioTranscription();+ } else {+ logger.debug('Found existing job', {+ jobId: job.id,+ status: job.status,+ });+ }++ this.props.blockProps.jobId = job.id;+ this.props.blockProps.createdBy = this.currentUserId;++ if (job.status !== AiJobStatus.failed) {+ this._status$.value = {+ status: AiJobStatus.pending,+ };+ } else {+ logger.debug('Job submission failed');+ throw UserFriendlyError.fromAny('failed to submit transcription');+ }++ await this.untilJobFinishedOrClaimed();+ await this.claim();+ } catch (err) {+ logger.debug('Error during job submission', { error: err });+ this._status$.value = {+ status: AiJobStatus.failed,+ error: UserFriendlyError.fromAny(err),+ };+ }+ return this.status$.value;+ }++ private async untilJobFinishedOrClaimed() {+ while (+ !this.disposed &&+ this.props.blockProps.jobId &&+ this.props.blockProps.createdBy === this.currentUserId+ ) {+ logger.debug('Polling job status', {+ jobId: this.props.blockProps.jobId,+ });+ const job = await this.store.getAudioTranscription(+ this.props.blobId,+ this.props.blockProps.jobId+ );++ if (!job || job?.status === 'failed') {+ logger.debug('Job failed during polling', {+ jobId: this.props.blockProps.jobId,+ });+ throw UserFriendlyError.fromAny('Transcription job failed');+ }++ if (job?.status === 'finished' || job?.status === 'claimed') {+ logger.debug('Job finished, ready to claim', {+ jobId: this.props.blockProps.jobId,+ });+ this._status$.value = {+ status: AiJobStatus.finished,+ };+ return;+ }++ // Add delay between polling attempts+ await new Promise(resolve => setTimeout(resolve, 3000));+ }+ }++ async claim() {+ if (this.disposed) {+ logger.debug('Job already disposed, cannot claim');+ throw new Error('Job already disposed');+ }++ logger.debug('Attempting to claim job', {+ jobId: this.props.blockProps.jobId,+ });++ if (!this.props.blockProps.jobId) {+ logger.debug('No job id found, cannot claim');+ throw new Error('No job id found');+ }++ const claimedJob = await this.store.claimAudioTranscription(+ this.props.blockProps.jobId+ );++ if (claimedJob) {+ logger.debug('Successfully claimed job', {+ jobId: this.props.blockProps.jobId,+ });+ const result: TranscriptionResult = {+ summary: claimedJob.summary ?? '',+ title: claimedJob.title ?? '',+ segments:+ claimedJob.transcription?.map(segment => ({+ speaker: segment.speaker,+ start: segment.start,+ end: segment.end,+ transcription: segment.transcription,+ })) ?? [],+ };++ this._status$.value = {+ status: AiJobStatus.claimed,+ result,+ };+ } else {+ throw new Error('Failed to claim transcription result');+ }+ }++ isCreator() {+ return (+ this.props.blockProps.jobId &&+ this.props.blockProps.createdBy &&+ this.props.blockProps.createdBy === this.currentUserId+ );+ }++ private get serverService() {+ return (+ this.workspaceServerService.server || this.defaultServerService.server+ );+ }++ get currentUserId() {+ const authService = this.serverService?.scope.getOptional(AuthService);+ if (!authService) {+ return;+ }+ return authService.session.account$.value?.id;+ }+}diff --git a/packages/frontend/core/src/modules/media/entities/types.ts b/packages/frontend/core/src/modules/media/entities/types.tsnew file mode 100644index 0000000000..f1794c3466--- /dev/null+++ b/packages/frontend/core/src/modules/media/entities/types.ts@@ -0,0 +1,10 @@+export interface TranscriptionResult {+ title: string;+ summary: string;+ segments: {+ speaker: string;+ start: string;+ end: string;+ transcription: string;+ }[];+}diff --git a/packages/frontend/core/src/modules/media/index.ts b/packages/frontend/core/src/modules/media/index.tsindex bbb3f47592..54b76cc5e7 100644--- a/packages/frontend/core/src/modules/media/index.ts+++ b/packages/frontend/core/src/modules/media/index.ts@@ -1,11 +1,14 @@import type { Framework } from '@toeverything/infra';+import { DefaultServerService, WorkspaceServerService } from '../cloud';import { DesktopApiService } from '../desktop-api';import { GlobalState } from '../storage';import { WorkbenchService } from '../workbench';import { WorkspaceScope, WorkspaceService } from '../workspace';import { AudioAttachmentBlock } from './entities/audio-attachment-block';import { AudioMedia } from './entities/audio-media';+import { AudioTranscriptionJob } from './entities/audio-transcription-job';+import { AudioTranscriptionJobStore } from './entities/audio-transcription-job-store';import {ElectronGlobalMediaStateProvider,GlobalMediaStateProvider,@@ -15,31 +18,40 @@ import { AudioAttachmentService } from './services/audio-attachment';import { AudioMediaManagerService } from './services/audio-media-manager';export function configureMediaModule(framework: Framework) {+ framework+ .scope(WorkspaceScope)+ .entity(AudioMedia, [WorkspaceService])+ .entity(AudioAttachmentBlock, [AudioMediaManagerService, WorkspaceService])+ .entity(AudioTranscriptionJob, [+ WorkspaceServerService,+ DefaultServerService,+ ])+ .entity(AudioTranscriptionJobStore, [+ WorkspaceService,+ WorkspaceServerService,+ DefaultServerService,+ ])+ .service(AudioAttachmentService);+if (BUILD_CONFIG.isElectron) {framework.impl(GlobalMediaStateProvider, ElectronGlobalMediaStateProvider, [GlobalState,]).scope(WorkspaceScope)- .entity(AudioMedia, [WorkspaceService])- .entity(AudioAttachmentBlock, [AudioMediaManagerService]).service(AudioMediaManagerService, [GlobalMediaStateProvider,WorkbenchService,DesktopApiService,- ])- .service(AudioAttachmentService);+ ]);} else {framework.impl(GlobalMediaStateProvider, WebGlobalMediaStateProvider).scope(WorkspaceScope)- .entity(AudioMedia, [WorkspaceService])- .entity(AudioAttachmentBlock, [AudioMediaManagerService]).service(AudioMediaManagerService, [GlobalMediaStateProvider,WorkbenchService,- ])- .service(AudioAttachmentService);+ ]);}}