Skip to content

There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.

feat(core): add markdown preview for backlinks (#8883)

EYHNcee5d02f7173
diff --git a/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch b/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch
new file mode 100644
index 0000000000..716784a96c
--- /dev/null
+++ b/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch
@@ -0,0 +1,13 @@
+diff --git a/package.json b/package.json
+index 5fef2811aa86f3f1f8228daef7d867863e71db72..b795fbd2a0e1cba0b6389ff051220f4e3c52fc13 100644
+--- a/package.json
++++ b/package.json
+@@ -34,7 +34,7 @@
+ "deno": "./index.js",
+ "react-native": "./index.js",
+ "worker": "./index.js",
+- "browser": "./index.dom.js",
++ "browser": "./index.js",
+ "default": "./index.js"
+ }
+ },
diff --git a/package.json b/package.json
index 02ac846bfa..776decf65e 100644
--- a/package.json
+++ b/package.json
@@ -156,6 +156,7 @@
"which-typed-array": "npm:@nolyfill/which-typed-array@latest",
"macos-alias": "npm:@napi-rs/[email protected]",
"fs-xattr": "npm:@napi-rs/xattr@latest",
- "vite": "6.0.1"
+ "vite": "6.0.1",
+ "decode-named-character-reference@npm:^1.0.0": "patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch"
}
}
diff --git a/packages/common/infra/src/sync/indexer/document.ts b/packages/common/infra/src/sync/indexer/document.ts
index d21397e8d3..e2e22d6994 100644
--- a/packages/common/infra/src/sync/indexer/document.ts
+++ b/packages/common/infra/src/sync/indexer/document.ts
@@ -40,7 +40,7 @@ export class Document<S extends Schema = any> {
}
} else {
for (const key in map) {
- if (map[key] === undefined) {
+ if (map[key] === undefined || map[key] === null) {
continue;
}
doc.insert(key, map[key]);
diff --git a/packages/frontend/component/src/ui/property/property.tsx b/packages/frontend/component/src/ui/property/property.tsx
index 622dbe9407..cdce501221 100644
--- a/packages/frontend/component/src/ui/property/property.tsx
+++ b/packages/frontend/component/src/ui/property/property.tsx
@@ -39,7 +39,7 @@ export const PropertyCollapsibleSection = forwardRef<
collapsed?: boolean;
onCollapseChange?: (collapsed: boolean) => void;
}> &
- HTMLProps<HTMLDivElement>
+ Omit<HTMLProps<HTMLDivElement>, 'title'>
>(
(
{
diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx b/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx
index 891156687f..3bc66ae915 100644
--- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx
+++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx
@@ -5,7 +5,7 @@ import {
} from '@affine/core/modules/doc-link';
import { useI18n } from '@affine/i18n';
import { LiveData, useLiveData, useServices } from '@toeverything/infra';
-import { useCallback, useState } from 'react';
+import { Fragment, useCallback, useState } from 'react';
import { AffinePageReference } from '../../affine/reference-link';
import * as styles from './bi-directional-link-panel.css';
@@ -52,9 +52,14 @@ export const BiDirectionalLinkPanel = () => {
{t['com.affine.page-properties.backlinks']()} · {backlinks.length}
</div>
{backlinks.map(link => (
- <div key={link.docId} className={styles.link}>
- <AffinePageReference key={link.docId} pageId={link.docId} />
- </div>
+ <Fragment key={link.docId}>
+ <div className={styles.link}>
+ <AffinePageReference key={link.docId} pageId={link.docId} />
+ </div>
+ <br />
+ <pre style={{ opacity: 0.5 }}>{link.markdownPreview}</pre>
+ <br />
+ </Fragment>
))}
</div>
<div className={styles.linksContainer}>
diff --git a/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx b/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx
index c2202780e5..d7bba354b9 100644
--- a/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx
+++ b/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx
@@ -122,7 +122,6 @@ const DatabaseBacklinkRow = ({
return (
<PropertyCollapsibleSection
- // @ts-expect-error fix type
title={
<span className={styles.databaseNameWrapper}>
<span className={styles.databaseName}>
diff --git a/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts b/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts
index 8ca7c6d985..cee54f9810 100644
--- a/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts
+++ b/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts
@@ -7,6 +7,7 @@ export interface Backlink {
docId: string;
blockId: string;
title: string;
+ markdownPreview?: string;
}
export class DocBacklinks extends Entity {
diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
index e1f5bbba2c..f10d77f521 100644
--- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
+++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts
@@ -36,7 +36,7 @@ export class DocsIndexer extends Entity {
/**
* increase this number to re-index all docs
*/
- static INDEXER_VERSION = 6;
+ static INDEXER_VERSION = 10;
private readonly jobQueue: JobQueue<IndexerJobPayload> =
new IndexedDBJobQueue<IndexerJobPayload>(
diff --git a/packages/frontend/core/src/modules/docs-search/schema.ts b/packages/frontend/core/src/modules/docs-search/schema.ts
index ec395320a4..9c8fe09ed1 100644
--- a/packages/frontend/core/src/modules/docs-search/schema.ts
+++ b/packages/frontend/core/src/modules/docs-search/schema.ts
@@ -28,6 +28,7 @@ export const blockIndexSchema = defineSchema({
// additional info
// { "databaseName": "xxx" }
additional: { type: 'String', index: false },
+ markdownPreview: { type: 'String', index: false },
});
export type BlockIndexSchema = typeof blockIndexSchema;
diff --git a/packages/frontend/core/src/modules/docs-search/services/docs-search.ts b/packages/frontend/core/src/modules/docs-search/services/docs-search.ts
index 6d03bc1661..fe3d05ee4b 100644
--- a/packages/frontend/core/src/modules/docs-search/services/docs-search.ts
+++ b/packages/frontend/core/src/modules/docs-search/services/docs-search.ts
@@ -478,7 +478,7 @@ export class DocsSearchService extends Service {
'docId',
{
hits: {
- fields: ['docId', 'blockId'],
+ fields: ['docId', 'blockId', 'markdownPreview'],
pagination: {
limit: 1,
},
@@ -499,10 +499,16 @@ export class DocsSearchService extends Service {
const title =
docData.find(doc => doc.id === bucket.key)?.get('title') ?? '';
const blockId = bucket.hits.nodes[0]?.fields.blockId ?? '';
+ const markdownPreview =
+ bucket.hits.nodes[0]?.fields.markdownPreview ?? '';
return {
docId: bucket.key,
blockId: typeof blockId === 'string' ? blockId : blockId[0],
title: typeof title === 'string' ? title : title[0],
+ markdownPreview:
+ typeof markdownPreview === 'string'
+ ? markdownPreview
+ : markdownPreview[0],
};
});
});
diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
index 49e6982e09..16c35761d2 100644
--- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
+++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts
@@ -1,6 +1,16 @@
-import type { AffineTextAttributes } from '@blocksuite/affine/blocks';
+import {
+ type AffineTextAttributes,
+ MarkdownAdapter,
+} from '@blocksuite/affine/blocks';
+import {
+ createYProxy,
+ DocCollection,
+ type DraftModel,
+ Job,
+ type YBlock,
+} from '@blocksuite/affine/store';
import type { DeltaInsert } from '@blocksuite/inline';
-import { Document } from '@toeverything/infra';
+import { Document, getAFFiNEWorkspaceSchema } from '@toeverything/infra';
import { toHexString } from 'lib0/buffer.js';
import { digest as lib0Digest } from 'lib0/hash/sha256';
import { difference, uniq } from 'lodash-es';
@@ -20,6 +30,8 @@ import type {
WorkerOutput,
} from './types';
+const blocksuiteSchema = getAFFiNEWorkspaceSchema();
+
const LRU_CACHE_SIZE = 5;
// lru cache for ydoc instances, last used at the end of the array
@@ -61,6 +73,92 @@ async function getOrCreateCachedYDoc(data: Uint8Array) {
}
}
+function yblockToDraftModal(yblock: YBlock): DraftModel | null {
+ const flavour = yblock.get('sys:flavour');
+ const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);
+ if (!blockSchema) {
+ return null;
+ }
+ const keys = Array.from(yblock.keys())
+ .filter(key => key.startsWith('prop:'))
+ .map(key => key.substring(5));
+
+ const props = Object.fromEntries(
+ keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])
+ );
+
+ return {
+ ...props,
+ id: yblock.get('sys:id'),
+ flavour,
+ children: [],
+ role: blockSchema.model.role,
+ version: (yblock.get('sys:version') as number) ?? blockSchema.version,
+ keys: Array.from(yblock.keys())
+ .filter(key => key.startsWith('prop:'))
+ .map(key => key.substring(5)),
+ };
+}
+
+const markdownAdapter = new MarkdownAdapter(
+ new Job({
+ collection: new DocCollection({
+ id: 'indexer',
+ schema: blocksuiteSchema,
+ }),
+ })
+);
+
+interface BlockDocumentInfo {
+ docId: string;
+ blockId: string;
+ content?: string | string[];
+ flavour: string;
+ blob?: string[];
+ refDocId?: string[];
+ ref?: string[];
+ parentFlavour?: string;
+ parentBlockId?: string;
+ additional?: { databaseName?: string };
+ yblock: YMap<any>;
+ markdownPreview?: string;
+}
+
+const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();
+const generateMarkdownPreview = async (block: BlockDocumentInfo) => {
+ if (markdownPreviewCache.has(block)) {
+ return markdownPreviewCache.get(block);
+ }
+ const flavour = block.flavour;
+ let markdown: string | null = null;
+ if (
+ flavour === 'affine:paragraph' ||
+ flavour === 'affine:list' ||
+ flavour === 'affine:code'
+ ) {
+ const draftModel = yblockToDraftModal(block.yblock);
+ markdown =
+ block.parentFlavour === 'affine:database'
+ ? `database · ${block.additional?.databaseName}\n`
+ : ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)
+ ?.file ?? null);
+ }
+ if (
+ flavour === 'affine:embed-linked-doc' ||
+ flavour === 'affine:embed-synced-doc'
+ ) {
+ markdown = '🔗\n';
+ }
+ if (flavour === 'affine:attachment') {
+ markdown = '📃\n';
+ }
+ if (flavour === 'affine:image') {
+ markdown = '🖼️\n';
+ }
+ markdownPreviewCache.set(block, markdown);
+ return markdown;
+};
+
async function crawlingDocData({
docBuffer,
storageDocId,
@@ -110,7 +208,7 @@ async function crawlingDocData({
let docTitle = '';
let summaryLenNeeded = 1000;
let summary = '';
- const blockDocuments: Document<BlockIndexSchema>[] = [];
+ const blockDocuments: BlockDocumentInfo[] = [];
const blocks = ydoc.getMap<any>('blocks');
@@ -147,6 +245,7 @@ async function crawlingDocData({
}
};
+ // #region first loop - generate block base info
while (queue.length) {
const next = queue.pop();
if (!next) {
@@ -167,14 +266,13 @@ async function crawlingDocData({
if (flavour === 'affine:page') {
docTitle = block.get('prop:title').toString();
- blockDocuments.push(
- Document.from(`${docId}:${blockId}`, {
- docId,
- flavour,
- blockId,
- content: docTitle,
- })
- );
+ blockDocuments.push({
+ docId,
+ flavour,
+ blockId,
+ content: docTitle,
+ yblock: block,
+ });
}
if (
@@ -183,6 +281,7 @@ async function crawlingDocData({
flavour === 'affine:code'
) {
const text = block.get('prop:text') as YText;
+
if (!text) {
continue;
}
@@ -213,27 +312,24 @@ async function crawlingDocData({
? parentBlock?.get('prop:title')?.toString()
: undefined;
- blockDocuments.push(
- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
- docId,
- flavour,
- blockId,
- content: text.toString(),
- ...refs.reduce<{ refDocId: string[]; ref: string[] }>(
- (prev, curr) => {
- prev.refDocId.push(curr.refDocId);
- prev.ref.push(curr.ref);
- return prev;
- },
- { refDocId: [], ref: [] }
- ),
- parentFlavour,
- parentBlockId,
- additional: databaseName
- ? JSON.stringify({ databaseName })
- : undefined,
- })
- );
+ blockDocuments.push({
+ docId,
+ flavour,
+ blockId,
+ content: text.toString(),
+ ...refs.reduce<{ refDocId: string[]; ref: string[] }>(
+ (prev, curr) => {
+ prev.refDocId.push(curr.refDocId);
+ prev.ref.push(curr.ref);
+ return prev;
+ },
+ { refDocId: [], ref: [] }
+ ),
+ parentFlavour,
+ parentBlockId,
+ additional: { databaseName },
+ yblock: block,
+ });
if (summaryLenNeeded > 0) {
summary += text.toString();
@@ -249,33 +345,31 @@ async function crawlingDocData({
if (typeof pageId === 'string') {
// reference info
const params = block.get('prop:params') ?? {};
- blockDocuments.push(
- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
- docId,
- flavour,
- blockId,
- refDocId: [pageId],
- ref: [JSON.stringify({ docId: pageId, ...params })],
- parentFlavour,
- parentBlockId,
- })
- );
+ blockDocuments.push({
+ docId,
+ flavour,
+ blockId,
+ refDocId: [pageId],
+ ref: [JSON.stringify({ docId: pageId, ...params })],
+ parentFlavour,
+ parentBlockId,
+ yblock: block,
+ });
}
}
if (flavour === 'affine:attachment' || flavour === 'affine:image') {
const blobId = block.get('prop:sourceId');
if (typeof blobId === 'string') {
- blockDocuments.push(
- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
- docId,
- flavour,
- blockId,
- blob: [blobId],
- parentFlavour,
- parentBlockId,
- })
- );
+ blockDocuments.push({
+ docId,
+ flavour,
+ blockId,
+ blob: [blobId],
+ parentFlavour,
+ parentBlockId,
+ yblock: block,
+ });
}
}
@@ -308,16 +402,15 @@ async function crawlingDocData({
texts.push(text.toString());
}
- blockDocuments.push(
- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
- docId,
- flavour,
- blockId,
- content: texts,
- parentFlavour,
- parentBlockId,
- })
- );
+ blockDocuments.push({
+ docId,
+ flavour,
+ blockId,
+ content: texts,
+ parentFlavour,
+ parentBlockId,
+ yblock: block,
+ });
}
if (flavour === 'affine:database') {
@@ -356,16 +449,81 @@ async function crawlingDocData({
}
}
- blockDocuments.push(
- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {
- docId,
- flavour,
- blockId,
- content: texts,
- })
- );
+ blockDocuments.push({
+ docId,
+ flavour,
+ blockId,
+ content: texts,
+ yblock: block,
+ });
}
}
+ // #endregion
+
+ // #region second loop - generate markdown preview
+ const TARGET_PREVIEW_CHARACTER = 500;
+ const TARGET_PREVIOUS_BLOCK = 1;
+ const TARGET_FOLLOW_BLOCK = 4;
+ for (let i = 0; i < blockDocuments.length; i++) {
+ const block = blockDocuments[i];
+ if (block.ref) {
+ // only generate markdown preview for reference blocks
+ let previewText = (await generateMarkdownPreview(block)) ?? '';
+ let previousBlock = 0;
+ let followBlock = 0;
+ let previousIndex = i;
+ let followIndex = i;
+
+ while (
+ !(
+ (
+ previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit
+ ((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) &&
+ (followBlock >= TARGET_FOLLOW_BLOCK ||
+ followIndex >= blockDocuments.length))
+ ) // stop if no more blocks, or preview block reaches the limit
+ )
+ ) {
+ if (previousBlock < TARGET_PREVIOUS_BLOCK) {
+ previousIndex--;
+ const block =
+ previousIndex >= 0 ? blockDocuments.at(previousIndex) : null;
+ const markdown = block
+ ? await generateMarkdownPreview(block)
+ : null;
+ if (
+ markdown &&
+ !previewText.startsWith(
+ markdown
+ ) /* A small hack to skip blocks with the same content */
+ ) {
+ previewText = markdown + previewText;
+ previousBlock++;
+ }
+ }
+
+ if (followBlock < TARGET_FOLLOW_BLOCK) {
+ followIndex++;
+ const block = blockDocuments.at(followIndex);
+ const markdown = block
+ ? await generateMarkdownPreview(block)
+ : null;
+ if (
+ markdown &&
+ !previewText.endsWith(
+ markdown
+ ) /* A small hack to skip blocks with the same content */
+ ) {
+ previewText = previewText + markdown;
+ followBlock++;
+ }
+ }
+ }
+
+ block.markdownPreview = previewText;
+ }
+ }
+ // #endregion
return {
addedDoc: [
@@ -375,7 +533,23 @@ async function crawlingDocData({
title: docTitle,
summary,
}),
- blocks: blockDocuments,
+ blocks: blockDocuments.map(block =>
+ Document.from<BlockIndexSchema>(`${docId}:${block.blockId}`, {
+ docId: block.docId,
+ blockId: block.blockId,
+ content: block.content,
+ flavour: block.flavour,
+ blob: block.blob,
+ refDocId: block.refDocId,
+ ref: block.ref,
+ parentFlavour: block.parentFlavour,
+ parentBlockId: block.parentBlockId,
+ additional: block.additional
+ ? JSON.stringify(block.additional)
+ : undefined,
+ markdownPreview: block.markdownPreview,
+ })
+ ),
},
],
};
diff --git a/yarn.lock b/yarn.lock
index 6a425c271b..bac79efb3a 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -18060,7 +18060,7 @@ __metadata:
languageName: node
linkType: hard
-"decode-named-character-reference@npm:^1.0.0":
+"decode-named-character-reference@npm:1.0.2":
version: 1.0.2
resolution: "decode-named-character-reference@npm:1.0.2"
dependencies:
@@ -18069,6 +18069,15 @@ __metadata:
languageName: node
linkType: hard
+"decode-named-character-reference@patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch":
+ version: 1.0.2
+ resolution: "decode-named-character-reference@patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch::version=1.0.2&hash=2c2160"
+ dependencies:
+ character-entities: "npm:^2.0.0"
+ checksum: 10/bd6e42b2cc162f55351a34fa5123cee0bfdebd983aa3690e5347c9ec23ce8b7f701fce0f77099b3a51eb1451b6a17e66f41d69e7cfc482ea3a0a1e38fe2442bf
+ languageName: node
+ linkType: hard
+
"decode-uri-component@npm:^0.2.2":
version: 0.2.2
resolution: "decode-uri-component@npm:0.2.2"