There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(core): add markdown preview for backlinks (#8883)
EYHNcee5d02f7173
diff --git a/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch b/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patchnew file mode 100644index 0000000000..716784a96c--- /dev/null+++ b/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch@@ -0,0 +1,13 @@+diff --git a/package.json b/package.json+index 5fef2811aa86f3f1f8228daef7d867863e71db72..b795fbd2a0e1cba0b6389ff051220f4e3c52fc13 100644+--- a/package.json++++ b/package.json+@@ -34,7 +34,7 @@+ "deno": "./index.js",+ "react-native": "./index.js",+ "worker": "./index.js",+- "browser": "./index.dom.js",++ "browser": "./index.js",+ "default": "./index.js"+ }+ },diff --git a/package.json b/package.jsonindex 02ac846bfa..776decf65e 100644--- a/package.json+++ b/package.json@@ -156,6 +156,7 @@"which-typed-array": "npm:@nolyfill/which-typed-array@latest","macos-alias": "npm:@napi-rs/[email protected]","fs-xattr": "npm:@napi-rs/xattr@latest",- "vite": "6.0.1"+ "vite": "6.0.1",+ "decode-named-character-reference@npm:^1.0.0": "patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch"}}diff --git a/packages/common/infra/src/sync/indexer/document.ts b/packages/common/infra/src/sync/indexer/document.tsindex d21397e8d3..e2e22d6994 100644--- a/packages/common/infra/src/sync/indexer/document.ts+++ b/packages/common/infra/src/sync/indexer/document.ts@@ -40,7 +40,7 @@ export class Document<S extends Schema = any> {}} else {for (const key in map) {- if (map[key] === undefined) {+ if (map[key] === undefined || map[key] === null) {continue;}doc.insert(key, map[key]);diff --git a/packages/frontend/component/src/ui/property/property.tsx b/packages/frontend/component/src/ui/property/property.tsxindex 622dbe9407..cdce501221 100644--- a/packages/frontend/component/src/ui/property/property.tsx+++ b/packages/frontend/component/src/ui/property/property.tsx@@ -39,7 +39,7 @@ export const PropertyCollapsibleSection = forwardRef<collapsed?: boolean;onCollapseChange?: (collapsed: boolean) => void;}> &- HTMLProps<HTMLDivElement>+ Omit<HTMLProps<HTMLDivElement>, 'title'>>(({diff --git a/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx b/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsxindex 891156687f..3bc66ae915 100644--- a/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx+++ b/packages/frontend/core/src/components/blocksuite/block-suite-editor/bi-directional-link-panel.tsx@@ -5,7 +5,7 @@ import {} from '@affine/core/modules/doc-link';import { useI18n } from '@affine/i18n';import { LiveData, useLiveData, useServices } from '@toeverything/infra';-import { useCallback, useState } from 'react';+import { Fragment, useCallback, useState } from 'react';import { AffinePageReference } from '../../affine/reference-link';import * as styles from './bi-directional-link-panel.css';@@ -52,9 +52,14 @@ export const BiDirectionalLinkPanel = () => {{t['com.affine.page-properties.backlinks']()} · {backlinks.length}</div>{backlinks.map(link => (- <div key={link.docId} className={styles.link}>- <AffinePageReference key={link.docId} pageId={link.docId} />- </div>+ <Fragment key={link.docId}>+ <div className={styles.link}>+ <AffinePageReference key={link.docId} pageId={link.docId} />+ </div>+ <br />+ <pre style={{ opacity: 0.5 }}>{link.markdownPreview}</pre>+ <br />+ </Fragment>))}</div><div className={styles.linksContainer}>diff --git a/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx b/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsxindex c2202780e5..d7bba354b9 100644--- a/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx+++ b/packages/frontend/core/src/modules/doc-info/views/database-properties/doc-database-backlink-info.tsx@@ -122,7 +122,6 @@ const DatabaseBacklinkRow = ({return (<PropertyCollapsibleSection- // @ts-expect-error fix typetitle={<span className={styles.databaseNameWrapper}><span className={styles.databaseName}>diff --git a/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts b/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.tsindex 8ca7c6d985..cee54f9810 100644--- a/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts+++ b/packages/frontend/core/src/modules/doc-link/entities/doc-backlinks.ts@@ -7,6 +7,7 @@ export interface Backlink {docId: string;blockId: string;title: string;+ markdownPreview?: string;}export class DocBacklinks extends Entity {diff --git a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.tsindex e1f5bbba2c..f10d77f521 100644--- a/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts+++ b/packages/frontend/core/src/modules/docs-search/entities/docs-indexer.ts@@ -36,7 +36,7 @@ export class DocsIndexer extends Entity {/*** increase this number to re-index all docs*/- static INDEXER_VERSION = 6;+ static INDEXER_VERSION = 10;private readonly jobQueue: JobQueue<IndexerJobPayload> =new IndexedDBJobQueue<IndexerJobPayload>(diff --git a/packages/frontend/core/src/modules/docs-search/schema.ts b/packages/frontend/core/src/modules/docs-search/schema.tsindex ec395320a4..9c8fe09ed1 100644--- a/packages/frontend/core/src/modules/docs-search/schema.ts+++ b/packages/frontend/core/src/modules/docs-search/schema.ts@@ -28,6 +28,7 @@ export const blockIndexSchema = defineSchema({// additional info// { "databaseName": "xxx" }additional: { type: 'String', index: false },+ markdownPreview: { type: 'String', index: false },});export type BlockIndexSchema = typeof blockIndexSchema;diff --git a/packages/frontend/core/src/modules/docs-search/services/docs-search.ts b/packages/frontend/core/src/modules/docs-search/services/docs-search.tsindex 6d03bc1661..fe3d05ee4b 100644--- a/packages/frontend/core/src/modules/docs-search/services/docs-search.ts+++ b/packages/frontend/core/src/modules/docs-search/services/docs-search.ts@@ -478,7 +478,7 @@ export class DocsSearchService extends Service {'docId',{hits: {- fields: ['docId', 'blockId'],+ fields: ['docId', 'blockId', 'markdownPreview'],pagination: {limit: 1,},@@ -499,10 +499,16 @@ export class DocsSearchService extends Service {const title =docData.find(doc => doc.id === bucket.key)?.get('title') ?? '';const blockId = bucket.hits.nodes[0]?.fields.blockId ?? '';+ const markdownPreview =+ bucket.hits.nodes[0]?.fields.markdownPreview ?? '';return {docId: bucket.key,blockId: typeof blockId === 'string' ? blockId : blockId[0],title: typeof title === 'string' ? title : title[0],+ markdownPreview:+ typeof markdownPreview === 'string'+ ? markdownPreview+ : markdownPreview[0],};});});diff --git a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts b/packages/frontend/core/src/modules/docs-search/worker/in-worker.tsindex 49e6982e09..16c35761d2 100644--- a/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts+++ b/packages/frontend/core/src/modules/docs-search/worker/in-worker.ts@@ -1,6 +1,16 @@-import type { AffineTextAttributes } from '@blocksuite/affine/blocks';+import {+ type AffineTextAttributes,+ MarkdownAdapter,+} from '@blocksuite/affine/blocks';+import {+ createYProxy,+ DocCollection,+ type DraftModel,+ Job,+ type YBlock,+} from '@blocksuite/affine/store';import type { DeltaInsert } from '@blocksuite/inline';-import { Document } from '@toeverything/infra';+import { Document, getAFFiNEWorkspaceSchema } from '@toeverything/infra';import { toHexString } from 'lib0/buffer.js';import { digest as lib0Digest } from 'lib0/hash/sha256';import { difference, uniq } from 'lodash-es';@@ -20,6 +30,8 @@ import type {WorkerOutput,} from './types';+const blocksuiteSchema = getAFFiNEWorkspaceSchema();+const LRU_CACHE_SIZE = 5;// lru cache for ydoc instances, last used at the end of the array@@ -61,6 +73,92 @@ async function getOrCreateCachedYDoc(data: Uint8Array) {}}+function yblockToDraftModal(yblock: YBlock): DraftModel | null {+ const flavour = yblock.get('sys:flavour');+ const blockSchema = blocksuiteSchema.flavourSchemaMap.get(flavour);+ if (!blockSchema) {+ return null;+ }+ const keys = Array.from(yblock.keys())+ .filter(key => key.startsWith('prop:'))+ .map(key => key.substring(5));++ const props = Object.fromEntries(+ keys.map(key => [key, createYProxy(yblock.get(`prop:${key}`))])+ );++ return {+ ...props,+ id: yblock.get('sys:id'),+ flavour,+ children: [],+ role: blockSchema.model.role,+ version: (yblock.get('sys:version') as number) ?? blockSchema.version,+ keys: Array.from(yblock.keys())+ .filter(key => key.startsWith('prop:'))+ .map(key => key.substring(5)),+ };+}++const markdownAdapter = new MarkdownAdapter(+ new Job({+ collection: new DocCollection({+ id: 'indexer',+ schema: blocksuiteSchema,+ }),+ })+);++interface BlockDocumentInfo {+ docId: string;+ blockId: string;+ content?: string | string[];+ flavour: string;+ blob?: string[];+ refDocId?: string[];+ ref?: string[];+ parentFlavour?: string;+ parentBlockId?: string;+ additional?: { databaseName?: string };+ yblock: YMap<any>;+ markdownPreview?: string;+}++const markdownPreviewCache = new WeakMap<BlockDocumentInfo, string | null>();+const generateMarkdownPreview = async (block: BlockDocumentInfo) => {+ if (markdownPreviewCache.has(block)) {+ return markdownPreviewCache.get(block);+ }+ const flavour = block.flavour;+ let markdown: string | null = null;+ if (+ flavour === 'affine:paragraph' ||+ flavour === 'affine:list' ||+ flavour === 'affine:code'+ ) {+ const draftModel = yblockToDraftModal(block.yblock);+ markdown =+ block.parentFlavour === 'affine:database'+ ? `database · ${block.additional?.databaseName}\n`+ : ((draftModel ? await markdownAdapter.fromBlock(draftModel) : null)+ ?.file ?? null);+ }+ if (+ flavour === 'affine:embed-linked-doc' ||+ flavour === 'affine:embed-synced-doc'+ ) {+ markdown = '🔗\n';+ }+ if (flavour === 'affine:attachment') {+ markdown = '📃\n';+ }+ if (flavour === 'affine:image') {+ markdown = '🖼️\n';+ }+ markdownPreviewCache.set(block, markdown);+ return markdown;+};+async function crawlingDocData({docBuffer,storageDocId,@@ -110,7 +208,7 @@ async function crawlingDocData({let docTitle = '';let summaryLenNeeded = 1000;let summary = '';- const blockDocuments: Document<BlockIndexSchema>[] = [];+ const blockDocuments: BlockDocumentInfo[] = [];const blocks = ydoc.getMap<any>('blocks');@@ -147,6 +245,7 @@ async function crawlingDocData({}};+ // #region first loop - generate block base infowhile (queue.length) {const next = queue.pop();if (!next) {@@ -167,14 +266,13 @@ async function crawlingDocData({if (flavour === 'affine:page') {docTitle = block.get('prop:title').toString();- blockDocuments.push(- Document.from(`${docId}:${blockId}`, {- docId,- flavour,- blockId,- content: docTitle,- })- );+ blockDocuments.push({+ docId,+ flavour,+ blockId,+ content: docTitle,+ yblock: block,+ });}if (@@ -183,6 +281,7 @@ async function crawlingDocData({flavour === 'affine:code') {const text = block.get('prop:text') as YText;+if (!text) {continue;}@@ -213,27 +312,24 @@ async function crawlingDocData({? parentBlock?.get('prop:title')?.toString(): undefined;- blockDocuments.push(- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {- docId,- flavour,- blockId,- content: text.toString(),- ...refs.reduce<{ refDocId: string[]; ref: string[] }>(- (prev, curr) => {- prev.refDocId.push(curr.refDocId);- prev.ref.push(curr.ref);- return prev;- },- { refDocId: [], ref: [] }- ),- parentFlavour,- parentBlockId,- additional: databaseName- ? JSON.stringify({ databaseName })- : undefined,- })- );+ blockDocuments.push({+ docId,+ flavour,+ blockId,+ content: text.toString(),+ ...refs.reduce<{ refDocId: string[]; ref: string[] }>(+ (prev, curr) => {+ prev.refDocId.push(curr.refDocId);+ prev.ref.push(curr.ref);+ return prev;+ },+ { refDocId: [], ref: [] }+ ),+ parentFlavour,+ parentBlockId,+ additional: { databaseName },+ yblock: block,+ });if (summaryLenNeeded > 0) {summary += text.toString();@@ -249,33 +345,31 @@ async function crawlingDocData({if (typeof pageId === 'string') {// reference infoconst params = block.get('prop:params') ?? {};- blockDocuments.push(- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {- docId,- flavour,- blockId,- refDocId: [pageId],- ref: [JSON.stringify({ docId: pageId, ...params })],- parentFlavour,- parentBlockId,- })- );+ blockDocuments.push({+ docId,+ flavour,+ blockId,+ refDocId: [pageId],+ ref: [JSON.stringify({ docId: pageId, ...params })],+ parentFlavour,+ parentBlockId,+ yblock: block,+ });}}if (flavour === 'affine:attachment' || flavour === 'affine:image') {const blobId = block.get('prop:sourceId');if (typeof blobId === 'string') {- blockDocuments.push(- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {- docId,- flavour,- blockId,- blob: [blobId],- parentFlavour,- parentBlockId,- })- );+ blockDocuments.push({+ docId,+ flavour,+ blockId,+ blob: [blobId],+ parentFlavour,+ parentBlockId,+ yblock: block,+ });}}@@ -308,16 +402,15 @@ async function crawlingDocData({texts.push(text.toString());}- blockDocuments.push(- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {- docId,- flavour,- blockId,- content: texts,- parentFlavour,- parentBlockId,- })- );+ blockDocuments.push({+ docId,+ flavour,+ blockId,+ content: texts,+ parentFlavour,+ parentBlockId,+ yblock: block,+ });}if (flavour === 'affine:database') {@@ -356,16 +449,81 @@ async function crawlingDocData({}}- blockDocuments.push(- Document.from<BlockIndexSchema>(`${docId}:${blockId}`, {- docId,- flavour,- blockId,- content: texts,- })- );+ blockDocuments.push({+ docId,+ flavour,+ blockId,+ content: texts,+ yblock: block,+ });}}+ // #endregion++ // #region second loop - generate markdown preview+ const TARGET_PREVIEW_CHARACTER = 500;+ const TARGET_PREVIOUS_BLOCK = 1;+ const TARGET_FOLLOW_BLOCK = 4;+ for (let i = 0; i < blockDocuments.length; i++) {+ const block = blockDocuments[i];+ if (block.ref) {+ // only generate markdown preview for reference blocks+ let previewText = (await generateMarkdownPreview(block)) ?? '';+ let previousBlock = 0;+ let followBlock = 0;+ let previousIndex = i;+ let followIndex = i;++ while (+ !(+ (+ previewText.length > TARGET_PREVIEW_CHARACTER || // stop if preview text reaches the limit+ ((previousBlock >= TARGET_PREVIOUS_BLOCK || previousIndex < 0) &&+ (followBlock >= TARGET_FOLLOW_BLOCK ||+ followIndex >= blockDocuments.length))+ ) // stop if no more blocks, or preview block reaches the limit+ )+ ) {+ if (previousBlock < TARGET_PREVIOUS_BLOCK) {+ previousIndex--;+ const block =+ previousIndex >= 0 ? blockDocuments.at(previousIndex) : null;+ const markdown = block+ ? await generateMarkdownPreview(block)+ : null;+ if (+ markdown &&+ !previewText.startsWith(+ markdown+ ) /* A small hack to skip blocks with the same content */+ ) {+ previewText = markdown + previewText;+ previousBlock++;+ }+ }++ if (followBlock < TARGET_FOLLOW_BLOCK) {+ followIndex++;+ const block = blockDocuments.at(followIndex);+ const markdown = block+ ? await generateMarkdownPreview(block)+ : null;+ if (+ markdown &&+ !previewText.endsWith(+ markdown+ ) /* A small hack to skip blocks with the same content */+ ) {+ previewText = previewText + markdown;+ followBlock++;+ }+ }+ }++ block.markdownPreview = previewText;+ }+ }+ // #endregionreturn {addedDoc: [@@ -375,7 +533,23 @@ async function crawlingDocData({title: docTitle,summary,}),- blocks: blockDocuments,+ blocks: blockDocuments.map(block =>+ Document.from<BlockIndexSchema>(`${docId}:${block.blockId}`, {+ docId: block.docId,+ blockId: block.blockId,+ content: block.content,+ flavour: block.flavour,+ blob: block.blob,+ refDocId: block.refDocId,+ ref: block.ref,+ parentFlavour: block.parentFlavour,+ parentBlockId: block.parentBlockId,+ additional: block.additional+ ? JSON.stringify(block.additional)+ : undefined,+ markdownPreview: block.markdownPreview,+ })+ ),},],};diff --git a/yarn.lock b/yarn.lockindex 6a425c271b..bac79efb3a 100644--- a/yarn.lock+++ b/yarn.lock@@ -18060,7 +18060,7 @@ __metadata:languageName: nodelinkType: hard-"decode-named-character-reference@npm:^1.0.0":+"decode-named-character-reference@npm:1.0.2":version: 1.0.2resolution: "decode-named-character-reference@npm:1.0.2"dependencies:@@ -18069,6 +18069,15 @@ __metadata:languageName: nodelinkType: hard+"decode-named-character-reference@patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch":+ version: 1.0.2+ resolution: "decode-named-character-reference@patch:decode-named-character-reference@npm%3A1.0.2#~/.yarn/patches/decode-named-character-reference-npm-1.0.2-db17a755fd.patch::version=1.0.2&hash=2c2160"+ dependencies:+ character-entities: "npm:^2.0.0"+ checksum: 10/bd6e42b2cc162f55351a34fa5123cee0bfdebd983aa3690e5347c9ec23ce8b7f701fce0f77099b3a51eb1451b6a17e66f41d69e7cfc482ea3a0a1e38fe2442bf+ languageName: node+ linkType: hard+"decode-uri-component@npm:^0.2.2":version: 0.2.2resolution: "decode-uri-component@npm:0.2.2"