Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(core): add summary to transcription block (#11753)

pengx17ebf1d5476dd0
diff --git a/blocksuite/affine/blocks/callout/src/callout-block.ts b/blocksuite/affine/blocks/callout/src/callout-block.ts
index 5e09c52368..dd42e39ddd 100644
--- a/blocksuite/affine/blocks/callout/src/callout-block.ts
+++ b/blocksuite/affine/blocks/callout/src/callout-block.ts
@@ -21,16 +21,21 @@ export class CalloutBlockComponent extends CaptionedBlockComponent<CalloutBlockM
.affine-callout-block-container {
display: flex;
- padding: 12px 16px;
+ padding: 5px 10px;
border-radius: 8px;
background-color: ${unsafeCSSVarV2('block/callout/background/grey')};
}
.affine-callout-emoji-container {
- margin-right: 12px;
- margin-top: 10px;
+ margin-right: 10px;
+ margin-top: 14px;
user-select: none;
font-size: 1.2em;
+ width: 24px;
+ height: 24px;
+ display: flex;
+ align-items: center;
+ justify-content: center;
}
.affine-callout-emoji:hover {
cursor: pointer;
@@ -40,6 +45,7 @@ export class CalloutBlockComponent extends CaptionedBlockComponent<CalloutBlockM
.affine-callout-children {
flex: 1;
min-width: 0;
+ padding-left: 10px;
}
`;
diff --git a/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts b/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts
index b6d3f58f59..ff4897479f 100644
--- a/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts
+++ b/packages/frontend/core/src/blocksuite/ai/blocks/ai-chat-block/ai-transcription-block.ts
@@ -10,24 +10,15 @@ export class LitTranscriptionBlock extends BlockComponent<TranscriptionBlockMode
css`
transcription-block {
outline: none;
+ display: flex;
+ flex-direction: column;
+ gap: 4px;
}
`,
];
- get lastCalloutBlock() {
- for (const child of this.model.children.toReversed()) {
- if (child.flavour === 'affine:callout') {
- return child;
- }
- }
- return null;
- }
-
override render() {
- return this.std.host.renderChildren(this.model, model => {
- // if model is the last transcription block, we should render it
- return model === this.lastCalloutBlock;
- });
+ return this.std.host.renderChildren(this.model);
}
@property({ type: String, attribute: 'data-block-id' })
diff --git a/packages/frontend/core/src/blocksuite/ai/slides/index.ts b/packages/frontend/core/src/blocksuite/ai/slides/index.ts
index bf2cf6e9b9..2788c6d373 100644
--- a/packages/frontend/core/src/blocksuite/ai/slides/index.ts
+++ b/packages/frontend/core/src/blocksuite/ai/slides/index.ts
@@ -56,7 +56,7 @@ export const PPTBuilder = (host: EditorHost) => {
return {
process: async (text: string) => {
try {
- const snapshot = await markdownToSnapshot(text, host);
+ const snapshot = await markdownToSnapshot(text, host.doc, host);
const block = snapshot.snapshot?.content[0];
if (!block) {
@@ -73,7 +73,7 @@ export const PPTBuilder = (host: EditorHost) => {
return { contents, images: allImages };
},
done: async (text: string) => {
- const snapshot = await markdownToSnapshot(text, host);
+ const snapshot = await markdownToSnapshot(text, host.doc, host);
const block = snapshot.snapshot?.content[0];
if (!block) {
return;
diff --git a/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts b/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts
index 9f6ab7c2e1..08d68578f7 100644
--- a/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts
+++ b/packages/frontend/core/src/blocksuite/ai/utils/editor-actions.ts
@@ -110,7 +110,11 @@ export const replace = async (
if (textSelection) {
host.std.command.exec(deleteTextCommand, { textSelection });
- const { snapshot, transformer } = await markdownToSnapshot(content, host);
+ const { snapshot, transformer } = await markdownToSnapshot(
+ content,
+ host.doc,
+ host
+ );
if (snapshot) {
await transformer.snapshotToSlice(
snapshot,
diff --git a/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts b/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts
index 5e2c64e737..c73236bbf4 100644
--- a/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts
+++ b/packages/frontend/core/src/blocksuite/utils/markdown-utils.ts
@@ -102,18 +102,19 @@ export async function getContentFromSlice(
export const markdownToSnapshot = async (
markdown: string,
- host: EditorHost
+ store: Store,
+ host?: EditorHost
) => {
- const transformer = host.std.store.getTransformer([
- defaultImageProxyMiddleware,
- pasteMiddleware(host.std),
- ]);
- const markdownAdapter = new MixTextAdapter(transformer, host.std.provider);
+ const middlewares = host
+ ? [defaultImageProxyMiddleware, pasteMiddleware(host.std)]
+ : [defaultImageProxyMiddleware];
+ const transformer = store.getTransformer(middlewares);
+ const markdownAdapter = new MixTextAdapter(transformer, store.provider);
const payload = {
file: markdown,
assets: transformer.assetsManager,
- workspaceId: host.std.store.workspace.id,
- pageId: host.std.store.id,
+ workspaceId: store.workspace.id,
+ pageId: store.id,
};
const snapshot = await markdownAdapter.toSliceSnapshot(payload);
@@ -125,13 +126,17 @@ export const markdownToSnapshot = async (
};
export async function insertFromMarkdown(
- host: EditorHost,
+ host: EditorHost | undefined,
markdown: string,
doc: Store,
parent?: string,
index?: number
) {
- const { snapshot, transformer } = await markdownToSnapshot(markdown, host);
+ const { snapshot, transformer } = await markdownToSnapshot(
+ markdown,
+ doc,
+ host
+ );
const snapshots = snapshot?.content.flatMap(x => x.children) ?? [];
diff --git a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts
index 531432947c..45bbf37bfa 100644
--- a/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts
+++ b/packages/frontend/core/src/modules/media/entities/audio-attachment-block.ts
@@ -1,3 +1,4 @@
+import { insertFromMarkdown } from '@affine/core/blocksuite/utils';
import { encodeAudioBlobToOpusSlices } from '@affine/core/utils/webm-encoding';
import { DebugLogger } from '@affine/debug';
import { AiJobStatus } from '@affine/graphql';
@@ -26,6 +27,18 @@ function sanitizeText(text: string) {
return text.replace(/\r/g, '');
}
+const colorOptions = [
+ cssVarV2.text.highlight.fg.red,
+ cssVarV2.text.highlight.fg.green,
+ cssVarV2.text.highlight.fg.blue,
+ cssVarV2.text.highlight.fg.yellow,
+ cssVarV2.text.highlight.fg.purple,
+ cssVarV2.text.highlight.fg.orange,
+ cssVarV2.text.highlight.fg.teal,
+ cssVarV2.text.highlight.fg.grey,
+ cssVarV2.text.highlight.fg.magenta,
+];
+
export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
private readonly refCount$ = new LiveData<number>(0);
readonly audioMedia: AudioMedia;
@@ -142,7 +155,7 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
}
const status = await this.transcriptionJob.start();
if (status.status === AiJobStatus.claimed) {
- this.fillTranscriptionResult(status.result);
+ await this.fillTranscriptionResult(status.result);
}
} catch (error) {
track.doc.editor.audioBlock.transcribeRecording({
@@ -154,55 +167,77 @@ export class AudioAttachmentBlock extends Entity<AttachmentBlockModel> {
}
};
- private readonly fillTranscriptionResult = (result: TranscriptionResult) => {
+ private readonly fillTranscriptionResult = async (
+ result: TranscriptionResult
+ ) => {
this.props.props.caption = result.title ?? '';
- const calloutId = this.props.doc.addBlock(
- 'affine:callout',
- {
- emoji: '💬',
- },
- this.transcriptionBlock$.value?.id
- );
-
- // todo: refactor
- const speakerToColors = new Map<string, string>();
- for (const segment of result.segments) {
- let color = speakerToColors.get(segment.speaker);
- const colorOptions = [
- cssVarV2.text.highlight.fg.red,
- cssVarV2.text.highlight.fg.green,
- cssVarV2.text.highlight.fg.blue,
- cssVarV2.text.highlight.fg.yellow,
- cssVarV2.text.highlight.fg.purple,
- cssVarV2.text.highlight.fg.orange,
- cssVarV2.text.highlight.fg.teal,
- cssVarV2.text.highlight.fg.grey,
- cssVarV2.text.highlight.fg.magenta,
- ];
- if (!color) {
- color = colorOptions[speakerToColors.size % colorOptions.length];
- speakerToColors.set(segment.speaker, color);
- }
- const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [
- {
- insert: sanitizeText(segment.start + ' ' + segment.speaker),
- attributes: {
- color,
- bold: true,
- },
- },
+ const addCalloutBlock = (emoji: string, title: string) => {
+ const calloutId = this.props.doc.addBlock(
+ 'affine:callout',
{
- insert: ': ' + sanitizeText(segment.transcription),
+ emoji,
},
- ];
+ this.transcriptionBlock$.value?.id
+ );
this.props.doc.addBlock(
'affine:paragraph',
{
- text: new Text(deltaInserts),
+ type: 'h6',
+ text: new Text([
+ {
+ insert: title,
+ },
+ ]),
},
calloutId
);
- }
+ return calloutId;
+ };
+ const fillTranscription = (segments: TranscriptionResult['segments']) => {
+ const calloutId = addCalloutBlock('💬', 'Transcript');
+
+ const speakerToColors = new Map<string, string>();
+ for (const segment of segments) {
+ let color = speakerToColors.get(segment.speaker);
+ if (!color) {
+ color = colorOptions[speakerToColors.size % colorOptions.length];
+ speakerToColors.set(segment.speaker, color);
+ }
+ const deltaInserts: DeltaInsert<AffineTextAttributes>[] = [
+ {
+ insert: sanitizeText(segment.start + ' ' + segment.speaker),
+ attributes: {
+ color,
+ bold: true,
+ },
+ },
+ {
+ insert: ': ' + sanitizeText(segment.transcription),
+ },
+ ];
+ this.props.doc.addBlock(
+ 'affine:paragraph',
+ {
+ text: new Text(deltaInserts),
+ },
+ calloutId
+ );
+ }
+ };
+
+ const fillSummary = async (summary: TranscriptionResult['summary']) => {
+ const calloutId = addCalloutBlock('📑', 'Summary');
+ await insertFromMarkdown(
+ undefined,
+ summary,
+ this.props.doc,
+ calloutId,
+ 1
+ );
+ };
+
+ fillTranscription(result.segments);
+ await fillSummary(result.summary);
};
}