There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(core): add summary to transcription block (#11753)
pengx17ebf1d5476dd0
diff --git a/blocksuite/affine/blocks/callout/src/callout-block.ts b/blocksuite/affine/blocks/callout/src/callout-block.tsindex 5e09c52368..dd42e39ddd 100644--- a/blocksuite/affine/blocks/callout/src/callout-block.ts+++ b/blocksuite/affine/blocks/callout/src/callout-block.ts@@ -21,16 +21,21 @@ export class CalloutBlockComponent extends CaptionedBlockComponent<CalloutBlockM.affine-callout-block-container {display: flex;- padding: 12px 16px;+ padding: 5px 10px;border-radius: 8px;background-color: ${unsafeCSSVarV2('block/callout/background/grey')};}.affine-callout-emoji-container {- margin-right: 12px;- margin-top: 10px;+ margin-right: 10px;+ margin-top: 14px;user-select: none;font-size: 1.2em;+ width: 24px;+ height: 24px;+ display: flex;+ align-items: center;+ justify-content: center;}.affine-callout-emoji:hover {cursor: pointer;@@ -40,6 +45,7 @@ export class CalloutBlockComponent extends CaptionedBlockComponent<CalloutBlockM.affine-callout-children {flex: 1;min-width: 0;+ padding-left: 10px;}`;diff --git a/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts b/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.tsindex b6d3f58f59..ff4897479f 100644--- a/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts+++ b/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts@@ -10,24 +10,15 @@ export class LitTranscriptionBlock extends BlockComponent<TranscriptionBlockModecss`transcription-block {outline: none;+ display: flex;+ flex-direction: column;+ gap: 4px;}`,];- get lastCalloutBlock() {- for (const child of this.model.children.toReversed()) {- if (child.flavour === 'affine:callout') {- return child;- }- }- return null;- }-override render() {- return this.std.host.renderChildren(this.model, model => {- // if model is the last transcription block, we should render it- return model === this.lastCalloutBlock;- });+ return this.std.host.renderChildren(this.model);}@property({ type: String, attribute: 'data-block-id' })diff --git a/packages/frontend/core/src/blocksuite/ai/slides/index.ts b/packages/frontend/core/src/blocksuite/ai/slides/index.tsindex bf2cf6e9b9..2788c6d373 100644--- a/packages/frontend/core/src/blocksuite/ai/slides/index.ts+++ b/packages/frontend/core/src/blocksuite/ai/slides/index.ts@@ -56,7 +56,7 @@ export const PPTBuilder = (host: EditorHost) => {return {process: async (text: string) => {try {- const snapshot = await markdownToSnapshot(text, host);+ const snapshot = await markdownToSnapshot(text, host.doc, host);const block = snapshot.snapshot?.content[0];if (!block) {@@ -73,7 +73,7 @@ export const PPTBuilder = (host: EditorHost) => {return { contents, images: allImages };},done: async (text: string) => {- const snapshot = await markdownToSnapshot(text, host);+ const snapshot = await markdownToSnapshot(text, host.doc, host);const block = snapshot.snapshot?.content[0];if (!block) {return;diff --git a/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts b/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.tsindex 9f6ab7c2e1..08d68578f7 100644--- a/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts+++ b/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts@@ -110,7 +110,11 @@ export const replace = async (if (textSelection) {host.std.command.exec(deleteTextCommand, { textSelection });- const { snapshot, transformer } = await markdownToSnapshot(content, host);+ const { snapshot, transformer } = await markdownToSnapshot(+ content,+ host.doc,+ host+ );if (snapshot) {await transformer.snapshotToSlice(snapshot,diff --git a/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts b/packages/frontend/core/src/blocksuite/utils/markdown-utils.tsindex 5e2c64e737..c73236bbf4 100644--- a/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts+++ b/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts@@ -102,18 +102,19 @@ export async function getContentFromSlice(export const markdownToSnapshot = async (markdown: string,- host: EditorHost+ store: Store,+ host?: EditorHost) => {- const transformer = host.std.store.getTransformer([- defaultImageProxyMiddleware,- pasteMiddleware(host.std),- ]);- const markdownAdapter = new MixTextAdapter(transformer, host.std.provider);+ const middlewares = host+ ? [defaultImageProxyMiddleware, pasteMiddleware(host.std)]+ : [defaultImageProxyMiddleware];+ const transformer = store.getTransformer(middlewares);+ const markdownAdapter = new MixTextAdapter(transformer, store.provider);const payload = {file: markdown,assets: transformer.assetsManager,- workspaceId: host.std.store.workspace.id,- pageId: host.std.store.id,+ workspaceId: store.workspace.id,+ pageId: store.id,};const snapshot = await markdownAdapter.toSliceSnapshot(payload);@@ -125,13 +126,17 @@ export const markdownToSnapshot = async (};export async function insertFromMarkdown(- host: EditorHost,+ host: EditorHost | undefined,markdown: string,doc: Store,parent?: string,index?: number) {- const { snapshot, transformer } = await markdownToSnapshot(markdown, host);+ const { snapshot, transformer } = await markdownToSnapshot(+ markdown,+ doc,+ host+ );const snapshots = snapshot?.content.flatMap(x => x.children) ?? [];diff --git a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.tsindex 531432947c..45bbf37bfa 100644--- a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts+++ b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts@@ -1,3 +1,4 @@+import { insertFromMarkdown } from '@affine/core/blocksuite/utils';import { encodeAudioBlobToOpusSlices } from '@affine/core/utils/webm-encoding';import { DebugLogger } from '@affine/debug';import { AiJobStatus } from '@affine/graphql';@@ -26,6 +27,18 @@ function sanitizeText(text: string) {return text.replace(/\r/g, '');}+const colorOptions = [+ cssVarV2.text.highlight.fg.red,+ cssVarV2.text.highlight.fg.green,+ cssVarV2.text.highlight.fg.blue,+ cssVarV2.text.highlight.fg.yellow,+ cssVarV2.text.highlight.fg.purple,+ cssVarV2.text.highlight.fg.orange,+ cssVarV2.text.highlight.fg.teal,+ cssVarV2.text.highlight.fg.grey,+ cssVarV2.text.highlight.fg.magenta,+];+export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {private readonly refCount$ = new LiveData<number>(0);readonly audioMedia: AudioMedia;@@ -142,7 +155,7 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {}const status = await this.transcriptionJob.start();if (status.status === AiJobStatus.claimed) {- this.fillTranscriptionResult(status.result);+ await this.fillTranscriptionResult(status.result);}} catch (error) {track.doc.editor.audioBlock.transcribeRecording({@@ -154,55 +167,77 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {}};- private readonly fillTranscriptionResult = (result: TranscriptionResult) => {+ private readonly fillTranscriptionResult = async (+ result: TranscriptionResult+ ) => {this.props.props.caption = result.title ?? '';- const calloutId = this.props.doc.addBlock(- 'affine:callout',- {- emoji: '💬',- },- this.transcriptionBlock$.value?.id- );-- // todo: refactor- const speakerToColors = new Map<string, string>();- for (const segment of result.segments) {- let color = speakerToColors.get(segment.speaker);- const colorOptions = [- cssVarV2.text.highlight.fg.red,- cssVarV2.text.highlight.fg.green,- cssVarV2.text.highlight.fg.blue,- cssVarV2.text.highlight.fg.yellow,- cssVarV2.text.highlight.fg.purple,- cssVarV2.text.highlight.fg.orange,- cssVarV2.text.highlight.fg.teal,- cssVarV2.text.highlight.fg.grey,- cssVarV2.text.highlight.fg.magenta,- ];- if (!color) {- color = colorOptions[speakerToColors.size % colorOptions.length];- speakerToColors.set(segment.speaker, color);- }- const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [- {- insert: sanitizeText(segment.start + ' ' + segment.speaker),- attributes: {- color,- bold: true,- },- },+ const addCalloutBlock = (emoji: string, title: string) => {+ const calloutId = this.props.doc.addBlock(+ 'affine:callout',{- insert: ': ' + sanitizeText(segment.transcription),+ emoji,},- ];+ this.transcriptionBlock$.value?.id+ );this.props.doc.addBlock('affine:paragraph',{- text: new Text(deltaInserts),+ type: 'h6',+ text: new Text([+ {+ insert: title,+ },+ ]),},calloutId);- }+ return calloutId;+ };+ const fillTranscription = (segments: TranscriptionResult['segments']) => {+ const calloutId = addCalloutBlock('💬', 'Transcript');++ const speakerToColors = new Map<string, string>();+ for (const segment of segments) {+ let color = speakerToColors.get(segment.speaker);+ if (!color) {+ color = colorOptions[speakerToColors.size % colorOptions.length];+ speakerToColors.set(segment.speaker, color);+ }+ const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [+ {+ insert: sanitizeText(segment.start + ' ' + segment.speaker),+ attributes: {+ color,+ bold: true,+ },+ },+ {+ insert: ': ' + sanitizeText(segment.transcription),+ },+ ];+ this.props.doc.addBlock(+ 'affine:paragraph',+ {+ text: new Text(deltaInserts),+ },+ calloutId+ );+ }+ };++ const fillSummary = async (summary: TranscriptionResult['summary']) => {+ const calloutId = addCalloutBlock('📑', 'Summary');+ await insertFromMarkdown(+ undefined,+ summary,+ this.props.doc,+ calloutId,+ 1+ );+ };++ fillTranscription(result.segments);+ await fillSummary(result.summary);};}