Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(core): support chat panel chips and suggest current doc for embedding (#9747)

akumatusd048ac6c91c4
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts
new file mode 100644
index 0000000000..4882e5ce0d
--- /dev/null
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-config.ts
@@ -0,0 +1,17 @@
+import type { Store } from '@blocksuite/store';
+import type { Signal } from '@preact/signals-core';
+
+export interface AINetworkSearchConfig {
+ visible: Signal<boolean | undefined>;
+ enabled: Signal<boolean | undefined>;
+ setEnabled: (state: boolean) => void;
+}
+
+export interface DocDisplayConfig {
+ getIcon: (docId: string) => any;
+ getTitle: (docId: string) => {
+ signal: Signal<string>;
+ cleanup: () => void;
+ };
+ getDoc: (docId: string) => Store | null;
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts
index 41b49c5060..c2c2eb126b 100644
--- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-context.ts
@@ -24,13 +24,28 @@ export type ChatStatus =
| 'idle'
| 'transmitting';
+export interface DocContext {
+ docId: string;
+ plaintext?: string;
+ markdown?: string;
+ images?: File[];
+}
+
export type ChatContextValue = {
+ // history messages of the chat
items: ChatItem[];
status: ChatStatus;
error: AIError | null;
+ // plain-text of the selected content
quote: string;
+ // markdown of the selected content
markdown: string;
+ // images of the selected content or user uploaded
images: File[];
+ // chips of workspace doc or user uploaded file
+ chips: ChatChip[];
+ // content of selected workspace doc
+ docs: DocContext[];
abortController: AbortController | null;
chatSessionId: string | null;
};
@@ -40,3 +55,34 @@ export type ChatBlockMessage = ChatMessage & {
userName?: string;
avatarUrl?: string;
};
+
+export type ChipState =
+ | 'candidate'
+ | 'uploading'
+ | 'embedding'
+ | 'success'
+ | 'failed';
+
+export interface BaseChip {
+ /**
+ * candidate: the chip is a candidate for the chat
+ * uploading: the chip is uploading
+ * embedding: the chip is embedding
+ * success: the chip is successfully embedded
+ * failed: the chip is failed to embed
+ */
+ state: ChipState;
+ tooltip?: string;
+}
+
+export interface DocChip extends BaseChip {
+ docId: string;
+}
+
+export interface FileChip extends BaseChip {
+ fileName: string;
+ fileId: string;
+ fileType: string;
+}
+
+export type ChatChip = DocChip | FileChip;
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts
new file mode 100644
index 0000000000..966ce99f3d
--- /dev/null
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-chips.ts
@@ -0,0 +1,59 @@
+import {
+ type EditorHost,
+ ShadowlessElement,
+} from '@blocksuite/affine/block-std';
+import { WithDisposable } from '@blocksuite/affine/global/utils';
+import { css, html } from 'lit';
+import { property } from 'lit/decorators.js';
+import { repeat } from 'lit/directives/repeat.js';
+
+import type { DocDisplayConfig } from './chat-config';
+import type { ChatContextValue } from './chat-context';
+import { getChipKey, isDocChip, isFileChip } from './components/utils';
+
+export class ChatPanelChips extends WithDisposable(ShadowlessElement) {
+ static override styles = css`
+ .chip-list {
+ display: flex;
+ flex-wrap: wrap;
+ }
+ `;
+
+ @property({ attribute: false })
+ accessor host!: EditorHost;
+
+ @property({ attribute: false })
+ accessor chatContextValue!: ChatContextValue;
+
+ @property({ attribute: false })
+ accessor updateContext!: (context: Partial<ChatContextValue>) => void;
+
+ @property({ attribute: false })
+ accessor docDisplayConfig!: DocDisplayConfig;
+
+ override render() {
+ return html`<div class="chip-list">
+ ${repeat(
+ this.chatContextValue.chips,
+ chip => getChipKey(chip),
+ chip => {
+ if (isDocChip(chip)) {
+ return html`<chat-panel-doc-chip
+ .chip=${chip}
+ .docDisplayConfig=${this.docDisplayConfig}
+ .host=${this.host}
+ .chatContextValue=${this.chatContextValue}
+ .updateContext=${this.updateContext}
+ ></chat-panel-doc-chip>`;
+ }
+ if (isFileChip(chip)) {
+ return html`<chat-panel-file-chip
+ .chip=${chip}
+ ></chat-panel-file-chip>`;
+ }
+ return null;
+ }
+ )}
+ </div>`;
+ }
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts
index 38c46bd847..53b3399e1d 100644
--- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/chat-panel-input.ts
@@ -8,7 +8,6 @@ import {
} from '@blocksuite/affine/global/utils';
import { unsafeCSSVarV2 } from '@blocksuite/affine-shared/theme';
import { ImageIcon, PublishIcon } from '@blocksuite/icons/lit';
-import type { Signal } from '@preact/signals-core';
import { css, html, LitElement, nothing } from 'lit';
import { property, query, state } from 'lit/decorators.js';
import { repeat } from 'lit/directives/repeat.js';
@@ -22,6 +21,7 @@ import {
import { AIProvider } from '../provider';
import { reportResponse } from '../utils/action-reporter';
import { readBlobAsURL } from '../utils/image';
+import type { AINetworkSearchConfig } from './chat-config';
import type { ChatContextValue, ChatMessage } from './chat-context';
const MaximumImageCount = 32;
@@ -31,12 +31,6 @@ function getFirstTwoLines(text: string) {
return lines.slice(0, 2);
}
-export interface AINetworkSearchConfig {
- visible: Signal<boolean | undefined>;
- enabled: Signal<boolean | undefined>;
- setEnabled: (state: boolean) => void;
-}
-
export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) {
static override styles = css`
.chat-panel-input {
@@ -510,7 +504,7 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) {
};
send = async (text: string) => {
- const { status, markdown } = this.chatContextValue;
+ const { status, markdown, docs } = this.chatContextValue;
if (status === 'loading' || status === 'transmitting') return;
const { images } = this.chatContextValue;
@@ -531,7 +525,8 @@ export class ChatPanelInput extends SignalWatcher(WithDisposable(LitElement)) {
images?.map(image => readBlobAsURL(image))
);
- const content = (markdown ? `${markdown}\n` : '') + text;
+ const refDocs = docs.map(doc => doc.markdown).join('\n');
+ const content = (markdown ? `${markdown}\n` : '') + `${refDocs}\n` + text;
this.updateContext({
items: [
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts
new file mode 100644
index 0000000000..d4f21d81d8
--- /dev/null
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/chip.ts
@@ -0,0 +1,106 @@
+import { ShadowlessElement } from '@blocksuite/affine/block-std';
+import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils';
+import { CloseIcon } from '@blocksuite/icons/lit';
+import { css, html, type TemplateResult } from 'lit';
+import { property } from 'lit/decorators.js';
+
+import type { ChipState } from '../chat-context';
+
+export class ChatPanelChip extends SignalWatcher(
+ WithDisposable(ShadowlessElement)
+) {
+ static override styles = css`
+ .chip-card {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ padding: 4px;
+ margin: 4px;
+ border-radius: 4px;
+ border: 0.5px solid var(--affine-border-color);
+ background: var(--affine-background-primary-color);
+ }
+ .chip-card[data-state='candidate'] {
+ border-width: 0.5px;
+ border-style: dashed;
+ background: var(--affine-background-secondary-color);
+ }
+ .chip-card[data-state='failed'] {
+ color: var(--affine-error-color);
+ background: var(--affine-background-error-color);
+ }
+ .chip-card[data-state='failed'] svg {
+ color: var(--affine-error-color);
+ }
+ .chip-card svg {
+ width: 16px;
+ height: 16px;
+ color: var(--affine-v2-icon-primary);
+ }
+ .chip-card-title {
+ display: inline-block;
+ margin: 0 4px;
+ font-size: 12px;
+ min-width: 16px;
+ max-width: 124px;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ cursor: pointer;
+ }
+ .chip-card-close {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ cursor: pointer;
+ border-radius: 4px;
+ }
+ .chip-card-close:hover {
+ background: var(--affine-hover-color);
+ }
+ `;
+
+ @property({ attribute: false })
+ accessor state!: ChipState;
+
+ @property({ attribute: false })
+ accessor name!: string;
+
+ @property({ attribute: false })
+ accessor tooltip!: string;
+
+ @property({ attribute: false })
+ accessor icon!: TemplateResult<1>;
+
+ @property({ attribute: false })
+ accessor closeable: boolean = false;
+
+ @property({ attribute: false })
+ accessor onChipDelete: () => void = () => {};
+
+ @property({ attribute: false })
+ accessor onChipClick: () => void = () => {};
+
+ override render() {
+ return html`
+ <div
+ class="chip-card"
+ data-testid="chat-panel-chip"
+ data-state=${this.state}
+ >
+ ${this.icon}
+ <span class="chip-card-title" @click=${this.onChipClick}>
+ <affine-tooltip>${this.tooltip}</affine-tooltip>
+ <span data-testid="chat-panel-chip-title">${this.name}</span>
+ </span>
+ ${this.closeable
+ ? html`
+ <div class="chip-card-close" @click=${this.onChipDelete}>
+ ${CloseIcon()}
+ </div>
+ `
+ : ''}
+ </div>
+ `;
+ }
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts
new file mode 100644
index 0000000000..e56934af2b
--- /dev/null
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/doc-chip.ts
@@ -0,0 +1,127 @@
+import {
+ type EditorHost,
+ ShadowlessElement,
+} from '@blocksuite/affine/block-std';
+import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils';
+import { Signal } from '@preact/signals-core';
+import { html } from 'lit';
+import { property } from 'lit/decorators.js';
+
+import { extractMarkdownFromDoc } from '../../utils/extract';
+import type { DocDisplayConfig } from '../chat-config';
+import type { ChatContextValue, DocChip } from '../chat-context';
+import { getChipIcon, getChipTooltip, isDocChip } from './utils';
+
+export class ChatPanelDocChip extends SignalWatcher(
+ WithDisposable(ShadowlessElement)
+) {
+ @property({ attribute: false })
+ accessor chip!: DocChip;
+
+ @property({ attribute: false })
+ accessor docDisplayConfig!: DocDisplayConfig;
+
+ @property({ attribute: false })
+ accessor host!: EditorHost;
+
+ @property({ attribute: false })
+ accessor chatContextValue!: ChatContextValue;
+
+ @property({ attribute: false })
+ accessor updateContext!: (context: Partial<ChatContextValue>) => void;
+
+ private name = new Signal<string>('');
+
+ private cleanup?: () => any;
+
+ override connectedCallback() {
+ super.connectedCallback();
+ const { signal, cleanup } = this.docDisplayConfig.getTitle(this.chip.docId);
+ this.name = signal;
+ this.cleanup = cleanup;
+ }
+
+ override disconnectedCallback() {
+ super.disconnectedCallback();
+ this.cleanup?.();
+ }
+
+ private readonly onChipClick = () => {
+ if (this.chip.state === 'candidate') {
+ const doc = this.docDisplayConfig.getDoc(this.chip.docId);
+ if (!doc) {
+ return;
+ }
+ this.updateChipContext({
+ state: 'embedding',
+ });
+ extractMarkdownFromDoc(doc, this.host.std.provider)
+ .then(result => {
+ this.updateChipContext({
+ state: 'success',
+ });
+ this.updateContext({
+ docs: [...this.chatContextValue.docs, result],
+ });
+ })
+ .catch(e => {
+ this.updateChipContext({
+ state: 'failed',
+ tooltip: e.message,
+ });
+ });
+ }
+ };
+
+ private updateChipContext(options: Partial<DocChip>) {
+ const index = this.chatContextValue.chips.findIndex(item => {
+ return isDocChip(item) && item.docId === this.chip.docId;
+ });
+ const nextChip: DocChip = {
+ ...this.chip,
+ ...options,
+ };
+ this.updateContext({
+ chips: [
+ ...this.chatContextValue.chips.slice(0, index),
+ nextChip,
+ ...this.chatContextValue.chips.slice(index + 1),
+ ],
+ });
+ }
+
+ private readonly onChipDelete = () => {
+ if (this.chip.state === 'success') {
+ this.updateContext({
+ docs: this.chatContextValue.docs.filter(
+ doc => doc.docId !== this.chip.docId
+ ),
+ });
+ }
+
+ this.updateContext({
+ chips: this.chatContextValue.chips.filter(
+ chip => isDocChip(chip) && chip.docId !== this.chip.docId
+ ),
+ });
+ };
+
+ override render() {
+ const { state, docId } = this.chip;
+ const isLoading = state === 'embedding' || state === 'uploading';
+ const getIcon = this.docDisplayConfig.getIcon(docId);
+ const docIcon = typeof getIcon === 'function' ? getIcon() : getIcon;
+ const icon = getChipIcon(state, docIcon);
+ const tooltip = getChipTooltip(state, this.name.value, this.chip.tooltip);
+
+ return html`<chat-panel-chip
+ .state=${state}
+ .name=${this.name.value}
+ .tooltip=${tooltip}
+ .icon=${icon}
+ .closeable=${!isLoading}
+ .onChipClick=${this.onChipClick}
+ .onChipDelete=${this.onChipDelete}
+ ></chat-panel-chip>`;
+ }
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts
new file mode 100644
index 0000000000..7897507906
--- /dev/null
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/file-chip.ts
@@ -0,0 +1,31 @@
+import { ShadowlessElement } from '@blocksuite/affine/block-std';
+import { SignalWatcher, WithDisposable } from '@blocksuite/affine/global/utils';
+import { getAttachmentFileIcons } from '@blocksuite/affine-components/icons';
+import { html } from 'lit';
+import { property } from 'lit/decorators.js';
+
+import type { FileChip } from '../chat-context';
+import { getChipIcon, getChipTooltip } from './utils';
+
+export class ChatPanelFileChip extends SignalWatcher(
+ WithDisposable(ShadowlessElement)
+) {
+ @property({ attribute: false })
+ accessor chip!: FileChip;
+
+ override render() {
+ const { state, fileName, fileType } = this.chip;
+ const isLoading = state === 'embedding' || state === 'uploading';
+ const tooltip = getChipTooltip(state, fileName, this.chip.tooltip);
+ const fileIcon = getAttachmentFileIcons(fileType);
+ const icon = getChipIcon(state, fileIcon);
+
+ return html`<chat-panel-chip
+ .state=${state}
+ .name=${fileName}
+ .tooltip=${tooltip}
+ .icon=${icon}
+ .closeable=${!isLoading}
+ ></chat-panel-chip>`;
+ }
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts
new file mode 100644
index 0000000000..9bf9e99dda
--- /dev/null
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/components/utils.ts
@@ -0,0 +1,61 @@
+import { WarningIcon } from '@blocksuite/icons/lit';
+import { type TemplateResult } from 'lit';
+
+import { LoadingIcon } from '../../../blocks/_common/icon';
+import type { ChatChip, ChipState, DocChip, FileChip } from '../chat-context';
+
+export function getChipTooltip(
+ state: ChipState,
+ title: string,
+ tooltip?: string
+) {
+ if (tooltip) {
+ return tooltip;
+ }
+ if (state === 'candidate') {
+ return 'Click to add doc';
+ }
+ if (state === 'embedding') {
+ return 'Embedding...';
+ }
+ if (state === 'uploading') {
+ return 'Uploading...';
+ }
+ if (state === 'failed') {
+ return 'Failed to embed';
+ }
+ return title;
+}
+
+export function getChipIcon(
+ state: ChipState,
+ icon: TemplateResult<1>
+): TemplateResult<1> {
+ const isLoading = state === 'embedding' || state === 'uploading';
+ const isFailed = state === 'failed';
+ if (isFailed) {
+ return WarningIcon();
+ }
+ if (isLoading) {
+ return LoadingIcon;
+ }
+ return icon;
+}
+
+export function isDocChip(chip: ChatChip): chip is DocChip {
+ return 'docId' in chip;
+}
+
+export function isFileChip(chip: ChatChip): chip is FileChip {
+ return 'fileId' in chip;
+}
+
+export function getChipKey(chip: ChatChip) {
+ if (isDocChip(chip)) {
+ return chip.docId;
+ }
+ if (isFileChip(chip)) {
+ return chip.fileId;
+ }
+ return null;
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts
index ba273ce5bd..94a82bca30 100644
--- a/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/chat-panel/index.ts
@@ -17,8 +17,13 @@ import {
getSelectedImagesAsBlobs,
getSelectedTextContent,
} from '../utils/selection-utils';
-import type { ChatAction, ChatContextValue, ChatItem } from './chat-context';
-import type { AINetworkSearchConfig } from './chat-panel-input';
+import type { AINetworkSearchConfig, DocDisplayConfig } from './chat-config';
+import type {
+ ChatAction,
+ ChatContextValue,
+ ChatItem,
+ DocChip,
+} from './chat-context';
import type { ChatPanelMessages } from './chat-panel-messages';
export class ChatPanel extends WithDisposable(ShadowlessElement) {
@@ -125,6 +130,13 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
AIProvider.LAST_ROOT_SESSION_ID = history.sessionId;
}
+ const { chips } = this.chatContextValue;
+ const defaultChip: DocChip = {
+ docId: this.doc.id,
+ state: 'candidate',
+ };
+ const nextChips =
+ items.length === 0 && chips.length === 0 ? [defaultChip] : chips;
this.chatContextValue = {
...this.chatContextValue,
items: items.sort((a, b) => {
@@ -132,6 +144,7 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime()
);
}),
+ chips: nextChips,
};
this.isLoading = false;
@@ -148,6 +161,9 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
@property({ attribute: false })
accessor networkSearchConfig!: AINetworkSearchConfig;
+ @property({ attribute: false })
+ accessor docDisplayConfig!: DocDisplayConfig;
+
@state()
accessor isLoading = false;
@@ -157,6 +173,8 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
images: [],
abortController: null,
items: [],
+ chips: [],
+ docs: [],
status: 'idle',
error: null,
markdown: '',
@@ -198,6 +216,9 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
if (_changedProperties.has('doc')) {
requestAnimationFrame(() => {
this.chatContextValue.chatSessionId = null;
+ // TODO get from CopilotContext
+ this.chatContextValue.chips = [];
+ this.chatContextValue.docs = [];
this._resetItems();
});
}
@@ -281,6 +302,12 @@ export class ChatPanel extends WithDisposable(ShadowlessElement) {
.host=${this.host}
.isLoading=${this.isLoading}
></chat-panel-messages>
+ <chat-panel-chips
+ .host=${this.host}
+ .chatContextValue=${this.chatContextValue}
+ .updateContext=${this.updateContext}
+ .docDisplayConfig=${this.docDisplayConfig}
+ ></chat-panel-chips>
<chat-panel-input
.chatContextValue=${this.chatContextValue}
.networkSearchConfig=${this.networkSearchConfig}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/utils/extract.ts b/packages/frontend/core/src/blocksuite/presets/ai/utils/extract.ts
index 1108cc334a..0f6b6d34e7 100644
--- a/packages/frontend/core/src/blocksuite/presets/ai/utils/extract.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/utils/extract.ts
@@ -1,20 +1,27 @@
import type { EditorHost } from '@blocksuite/affine/block-std';
import {
+ BlocksUtils,
DocModeProvider,
+ embedSyncedDocMiddleware,
type ImageBlockModel,
isInsideEdgelessEditor,
+ MarkdownAdapter,
type NoteBlockModel,
NoteDisplayMode,
+ titleMiddleware,
} from '@blocksuite/affine/blocks';
-import type { BlockModel } from '@blocksuite/affine/store';
+import type { BlockModel, Store } from '@blocksuite/affine/store';
+import { Slice, toDraftModel, Transformer } from '@blocksuite/affine/store';
+import type { ServiceProvider } from '@blocksuite/global/di';
-import type { ChatContextValue } from '../chat-panel/chat-context';
+import type { ChatContextValue, DocContext } from '../chat-panel/chat-context';
import {
allToCanvas,
getSelectedImagesAsBlobs,
getSelectedTextContent,
getTextContentFromBlockModels,
selectedToCanvas,
+ traverse,
} from './selection-utils';
export async function extractSelectedContent(
@@ -95,7 +102,7 @@ export async function extractAllContent(
}
}
-async function extractEdgelessAll(
+export async function extractEdgelessAll(
host: EditorHost
): Promise<Partial<ChatContextValue> | null> {
if (!isInsideEdgelessEditor(host)) return null;
@@ -113,21 +120,10 @@ async function extractEdgelessAll(
};
}
-async function extractPageAll(
+export async function extractPageAll(
host: EditorHost
): Promise<Partial<ChatContextValue> | null> {
- const notes = host.doc
- .getBlocksByFlavour('affine:note')
- .filter(
- note =>
- (note.model as NoteBlockModel).displayMode !==
- NoteDisplayMode.EdgelessOnly
- )
- .map(note => note.model as NoteBlockModel);
- const blockModels = notes.reduce((acc, note) => {
- acc.push(...note.children);
- return acc;
- }, [] as BlockModel[]);
+ const blockModels = getNoteBlockModels(host.doc);
const text = await getTextContentFromBlockModels(
host,
blockModels,
@@ -156,3 +152,64 @@ async function extractPageAll(
images,
};
}
+
+export async function extractMarkdownFromDoc(
+ doc: Store,
+ provider: ServiceProvider
+): Promise<DocContext> {
+ const transformer = await getTransformer(doc);
+ const adapter = new MarkdownAdapter(transformer, provider);
+ const blockModels = getNoteBlockModels(doc);
+ const textModels = blockModels.filter(
+ model =>
+ !BlocksUtils.matchFlavours(model, ['affine:image', 'affine:database'])
+ );
+ const drafts = textModels.map(toDraftModel);
+ drafts.forEach(draft => traverse(draft, drafts));
+ const slice = Slice.fromModels(doc, drafts);
+
+ const snapshot = transformer.sliceToSnapshot(slice);
+ if (!snapshot) {
+ throw new Error('Failed to extract markdown, snapshot is undefined');
+ }
+ const content = await adapter.fromSliceSnapshot({
+ snapshot,
+ assets: transformer.assetsManager,
+ });
+ return {
+ docId: doc.id,
+ markdown: content.file,
+ };
+}
+
+function getNoteBlockModels(doc: Store) {
+ const notes = doc
+ .getBlocksByFlavour('affine:note')
+ .filter(
+ note =>
+ (note.model as NoteBlockModel).displayMode !==
+ NoteDisplayMode.EdgelessOnly
+ )
+ .map(note => note.model as NoteBlockModel);
+ const blockModels = notes.reduce((acc, note) => {
+ acc.push(...note.children);
+ return acc;
+ }, [] as BlockModel[]);
+ return blockModels;
+}
+
+async function getTransformer(doc: Store) {
+ return new Transformer({
+ schema: doc.workspace.schema,
+ blobCRUD: doc.workspace.blobSync,
+ docCRUD: {
+ create: (id: string) => doc.workspace.createDoc({ id }),
+ get: (id: string) => doc.workspace.getDoc(id),
+ delete: (id: string) => doc.workspace.removeDoc(id),
+ },
+ middlewares: [
+ titleMiddleware(doc.workspace.meta.docMetas),
+ embedSyncedDocMiddleware('content'),
+ ],
+ });
+}
diff --git a/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts b/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts
index 660991d95e..8a6c156b95 100644
--- a/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts
+++ b/packages/frontend/core/src/blocksuite/presets/ai/utils/selection-utils.ts
@@ -105,7 +105,7 @@ export function getSelectedModels(editorHost: EditorHost) {
return selectedModels;
}
-function traverse(model: DraftModel, drafts: DraftModel[]) {
+export function traverse(model: DraftModel, drafts: DraftModel[]) {
const isDatabase = model.flavour === 'affine:database';
const children = isDatabase
? model.children
diff --git a/packages/frontend/core/src/blocksuite/presets/effects.ts b/packages/frontend/core/src/blocksuite/presets/effects.ts
index 4b35112efb..fa8f6d1dee 100644
--- a/packages/frontend/core/src/blocksuite/presets/effects.ts
+++ b/packages/frontend/core/src/blocksuite/presets/effects.ts
@@ -15,8 +15,12 @@ import { ActionMindmap } from './ai/chat-panel/actions/mindmap';
import { ActionSlides } from './ai/chat-panel/actions/slides';
import { ActionText } from './ai/chat-panel/actions/text';
import { AILoading } from './ai/chat-panel/ai-loading';
+import { ChatPanelChips } from './ai/chat-panel/chat-panel-chips';
import { ChatPanelInput } from './ai/chat-panel/chat-panel-input';
import { ChatPanelMessages } from './ai/chat-panel/chat-panel-messages';
+import { ChatPanelChip } from './ai/chat-panel/components/chip';
+import { ChatPanelDocChip } from './ai/chat-panel/components/doc-chip';
+import { ChatPanelFileChip } from './ai/chat-panel/components/file-chip';
import { AIErrorWrapper } from './ai/messages/error';
import { AISlidesRenderer } from './ai/messages/slides-renderer';
import { AIAnswerWrapper } from './ai/messages/wrapper';
@@ -57,6 +61,10 @@ export function registerBlocksuitePresetsCustomComponents() {
customElements.define('chat-panel-input', ChatPanelInput);
customElements.define('chat-panel-messages', ChatPanelMessages);
customElements.define('chat-panel', ChatPanel);
+ customElements.define('chat-panel-chips', ChatPanelChips);
+ customElements.define('chat-panel-doc-chip', ChatPanelDocChip);
+ customElements.define('chat-panel-file-chip', ChatPanelFileChip);
+ customElements.define('chat-panel-chip', ChatPanelChip);
customElements.define('ai-error-wrapper', AIErrorWrapper);
customElements.define('ai-slides-renderer', AISlidesRenderer);
customElements.define('ai-answer-wrapper', AIAnswerWrapper);
diff --git a/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx b/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx
index 63344a9788..d0a70520e2 100644
--- a/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx
+++ b/packages/frontend/core/src/desktop/pages/workspace/detail-page/tabs/chat.tsx
@@ -1,10 +1,13 @@
import { ChatPanel } from '@affine/core/blocksuite/presets/ai';
import { AINetworkSearchService } from '@affine/core/modules/ai-button/services/network-search';
+import { DocDisplayMetaService } from '@affine/core/modules/doc-display-meta';
+import { WorkspaceService } from '@affine/core/modules/workspace';
import {
DocModeProvider,
RefNodeSlotsProvider,
} from '@blocksuite/affine/blocks';
import type { AffineEditorContainer } from '@blocksuite/affine/presets';
+import { createSignalFromObservable } from '@blocksuite/affine-shared/utils';
import { useFramework } from '@toeverything/infra';
import { forwardRef, useEffect, useRef } from 'react';
@@ -49,12 +52,26 @@ export const EditorChatPanel = forwardRef(function EditorChatPanel(
chatPanelRef.current.doc = editor.doc;
containerRef.current?.append(chatPanelRef.current);
const searchService = framework.get(AINetworkSearchService);
- const networkSearchConfig = {
+ const docDisplayMetaService = framework.get(DocDisplayMetaService);
+ const workspaceService = framework.get(WorkspaceService);
+ chatPanelRef.current.networkSearchConfig = {
visible: searchService.visible,
enabled: searchService.enabled,
setEnabled: searchService.setEnabled,
};
- chatPanelRef.current.networkSearchConfig = networkSearchConfig;
+ chatPanelRef.current.docDisplayConfig = {
+ getIcon: (docId: string) => {
+ return docDisplayMetaService.icon$(docId, { type: 'lit' }).value;
+ },
+ getTitle: (docId: string) => {
+ const title$ = docDisplayMetaService.title$(docId);
+ return createSignalFromObservable(title$, '');
+ },
+ getDoc: (docId: string) => {
+ const doc = workspaceService.workspace.docCollection.getDoc(docId);
+ return doc;
+ },
+ };
} else {
chatPanelRef.current.host = editor.host;
chatPanelRef.current.doc = editor.doc;
diff --git a/tests/affine-cloud-copilot/e2e/copilot.spec.ts b/tests/affine-cloud-copilot/e2e/copilot.spec.ts
index 599aac3925..55ee19c2bc 100644
--- a/tests/affine-cloud-copilot/e2e/copilot.spec.ts
+++ b/tests/affine-cloud-copilot/e2e/copilot.spec.ts
@@ -86,6 +86,13 @@ const typeChat = async (page: Page, content: string) => {
await page.keyboard.type(content);
};
+const typeChatSequentially = async (page: Page, content: string) => {
+ const input = await page.locator('chat-panel-input textarea').nth(0);
+ await input.pressSequentially(content, {
+ delay: 50,
+ });
+};
+
const makeChat = async (page: Page, content: string) => {
await openChat(page);
await typeChat(page, content);
@@ -210,8 +217,8 @@ test.describe('chat panel', () => {
await createLocalWorkspace({ name: 'test' }, page);
await clickNewPageButton(page);
const sendButton = await page.getByTestId('chat-panel-send');
- // oxlint-disable-next-line unicorn/prefer-dom-node-dataset
await openChat(page);
+ // oxlint-disable-next-line unicorn/prefer-dom-node-dataset
expect(await sendButton.getAttribute('aria-disabled')).toBe('true');
await typeChat(page, 'hello');
// oxlint-disable-next-line unicorn/prefer-dom-node-dataset
@@ -399,11 +406,12 @@ test.describe('chat panel', () => {
await page.getByTestId('modal-close-button').click();
await openChat(page);
await page.getByTestId('chat-network-search').click();
- await makeChat(page, 'hello');
+ await typeChatSequentially(page, 'What is the weather in Shanghai today?');
+ await page.keyboard.press('Enter');
let history = await collectChat(page);
expect(history[0]).toEqual({
name: 'You',
- content: 'hello',
+ content: 'What is the weather in Shanghai today?',
});
expect(history[1].name).toBe('AFFiNE AI');
expect(
@@ -413,11 +421,12 @@ test.describe('chat panel', () => {
await clearChat(page);
expect((await collectChat(page)).length).toBe(0);
await page.getByTestId('chat-network-search').click();
- await makeChat(page, 'hello');
+ await typeChatSequentially(page, 'What is the weather in Shanghai today?');
+ await page.keyboard.press('Enter');
history = await collectChat(page);
expect(history[0]).toEqual({
name: 'You',
- content: 'hello',
+ content: 'What is the weather in Shanghai today?',
});
expect(history[1].name).toBe('AFFiNE AI');
expect(await page.locator('chat-panel affine-link').count()).toBe(0);
@@ -704,3 +713,64 @@ test.describe('chat with block', () => {
});
});
});
+
+test.describe('chat with doc', () => {
+ let user: {
+ email: string;
+ password: string;
+ };
+
+ test.beforeEach(async ({ page }) => {
+ user = await getUser();
+ await loginUser(page, user);
+ });
+
+ test('can chat with current doc', async ({ page }) => {
+ await page.reload();
+ await clickSideBarAllPageButton(page);
+ await page.waitForTimeout(200);
+ await createLocalWorkspace({ name: 'test' }, page);
+ await clickNewPageButton(page);
+
+ await openChat(page);
+ const chipTitle = await page.getByTestId('chat-panel-chip-title');
+ expect(await chipTitle.textContent()).toBe('Untitled');
+
+ const editorTitle = await page.locator('doc-title .inline-editor').nth(0);
+ await editorTitle.pressSequentially('AFFiNE AI', {
+ delay: 50,
+ });
+ await page.keyboard.press('Enter', { delay: 50 });
+ // Wait for the editor to be ready and focused
+ await page.waitForSelector('page-editor affine-paragraph .inline-editor');
+ const richText = await page
+ .locator('page-editor affine-paragraph .inline-editor')
+ .nth(0);
+ await richText.click(); // Ensure proper focus
+ await page.keyboard.type(
+ 'AFFiNE AI is an assistant with the ability to create well-structured outlines for any given content.',
+ {
+ delay: 50,
+ }
+ );
+
+ expect(await chipTitle.textContent()).toBe('AFFiNE AI');
+ const chip = await page.getByTestId('chat-panel-chip');
+ // oxlint-disable-next-line unicorn/prefer-dom-node-dataset
+ expect(await chip.getAttribute('data-state')).toBe('candidate');
+ await chip.click();
+ // oxlint-disable-next-line unicorn/prefer-dom-node-dataset
+ expect(await chip.getAttribute('data-state')).toBe('success');
+
+ await makeChat(page, 'summarize');
+ const history = await collectChat(page);
+ expect(history[0]).toEqual({
+ name: 'You',
+ content:
+ 'AFFiNE AI is an assistant with the ability to create well-structured outlines for any given content.\nsummarize',
+ });
+ expect(history[1].name).toBe('AFFiNE AI');
+ await clearChat(page);
+ expect((await collectChat(page)).length).toBe(0);
+ });
+});