There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
refactor(server): move bin content parser to doc reader (#10302)
fengmk2b4097aef8e31
diff --git a/packages/backend/server/src/__tests__/event/eventbus.spec.ts b/packages/backend/server/src/__tests__/event/eventbus.spec.tsindex be247227a7..afb4b23054 100644--- a/packages/backend/server/src/__tests__/event/eventbus.spec.ts+++ b/packages/backend/server/src/__tests__/event/eventbus.spec.ts@@ -187,7 +187,7 @@ test('should generate request id for event', async t => {await eventbus.emitAsync('__test__.requestId', {});- t.true(listeners.onRequestId.lastCall.returnValue.includes(':event/'));+ t.true(listeners.onRequestId.lastCall.returnValue.includes(':event:'));});test('should continuously use the same request id', async t => {diff --git a/packages/backend/server/src/__tests__/team.e2e.ts b/packages/backend/server/src/__tests__/team.e2e.tsindex b968dc7aff..5aa05e4fd9 100644--- a/packages/backend/server/src/__tests__/team.e2e.ts+++ b/packages/backend/server/src/__tests__/team.e2e.ts@@ -10,7 +10,7 @@ import Sinon from 'sinon';import { AppModule } from '../app.module';import { EventBus } from '../base';import { AuthService } from '../core/auth';-import { DocContentService } from '../core/doc-renderer';+import { DocReader } from '../core/doc';import { DocRole, PermissionService, WorkspaceRole } from '../core/permission';import { WorkspaceType } from '../core/workspaces';import { Models } from '../models';@@ -53,7 +53,7 @@ test.before(async t => {module.overrideProvider(EventBus).useValue(Sinon.createStubInstance(EventBus));- module.overrideProvider(DocContentService).useValue({+ module.overrideProvider(DocReader).useValue({getWorkspaceContent() {return {name: 'test',diff --git a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.tsindex a0fe401c2c..1eeb40a3da 100644--- a/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts+++ b/packages/backend/server/src/base/job/queue/__tests__/queue.spec.ts@@ -226,7 +226,7 @@ test('should generate request id', async t => {await executor.run('nightly.__test__requestId', {});- t.true(spy.returnValues.some(v => v.includes(':job/')));+ t.true(spy.returnValues.some(v => v.includes(':job:')));spy.restore();});diff --git a/packages/backend/server/src/base/utils/request.ts b/packages/backend/server/src/base/utils/request.tsindex f39646d42d..ee9e0642be 100644--- a/packages/backend/server/src/base/utils/request.ts+++ b/packages/backend/server/src/base/utils/request.ts@@ -94,7 +94,7 @@ export function parseCookies(export type RequestType = GqlContextType | 'event' | 'job';export function genRequestId(type: RequestType) {- return `${AFFiNE.flavor.type}:${type}/${randomUUID()}`;+ return `${AFFiNE.flavor.type}:${type}:${randomUUID()}`;}export function getOrGenRequestId(type: RequestType) {diff --git a/packages/backend/server/src/core/doc-renderer/__tests__/service.spec.ts b/packages/backend/server/src/core/doc-renderer/__tests__/service.spec.tsdeleted file mode 100644index f7f104773f..0000000000--- a/packages/backend/server/src/core/doc-renderer/__tests__/service.spec.ts+++ /dev/null@@ -1,85 +0,0 @@-import { randomUUID } from 'node:crypto';--import { User, Workspace } from '@prisma/client';-import ava, { TestFn } from 'ava';-import { Doc as YDoc } from 'yjs';--import { createTestingApp, type TestingApp } from '../../../__tests__/utils';-import { AppModule } from '../../../app.module';-import { Config } from '../../../base';-import { ConfigModule } from '../../../base/config';-import { Models } from '../../../models';-import { PgWorkspaceDocStorageAdapter } from '../../doc';-import { DocContentService } from '..';--const test = ava as TestFn<{- models: Models;- app: TestingApp;- docContentService: DocContentService;- config: Config;- adapter: PgWorkspaceDocStorageAdapter;-}>;--test.before(async t => {- const app = await createTestingApp({- imports: [- ConfigModule.forRoot({- flavor: {- doc: false,- },- docService: {- endpoint: '',- },- }),- AppModule,- ],- });-- t.context.models = app.get(Models);- t.context.docContentService = app.get(DocContentService);- t.context.config = app.get(Config);- t.context.adapter = app.get(PgWorkspaceDocStorageAdapter);- t.context.app = app;-});--let user: User;-let workspace: Workspace;--test.beforeEach(async t => {- t.context.config.docService.endpoint = t.context.app.url();- await t.context.app.initTestingDB();- user = await t.context.models.user.create({- email: '[email protected]',- });- workspace = await t.context.models.workspace.create(user.id);-});--test.after.always(async t => {- await t.context.app.close();-});--test('should get doc content from doc service rpc', async t => {- const docId = randomUUID();- const { docContentService } = t.context;-- const doc = new YDoc();- const text = doc.getText('content');- const updates: Buffer[] = [];-- doc.on('update', update => {- updates.push(Buffer.from(update));- });-- text.insert(0, 'hello');- text.insert(5, 'world');- text.insert(5, ' ');-- await t.context.adapter.pushDocUpdates(workspace.id, docId, updates, user.id);-- const docContent = await docContentService.getPageContent(- workspace.id,- docId- );- // TODO(@fengmk2): should create a test ydoc with blocks- t.is(docContent, null);-});diff --git a/packages/backend/server/src/core/doc-renderer/controller.ts b/packages/backend/server/src/core/doc-renderer/controller.tsindex 76998f9515..4bce9c34a6 100644--- a/packages/backend/server/src/core/doc-renderer/controller.ts+++ b/packages/backend/server/src/core/doc-renderer/controller.ts@@ -8,8 +8,8 @@ import isMobile from 'is-mobile';import { Config, metrics, URLHelper } from '../../base';import { htmlSanitize } from '../../native';import { Public } from '../auth';+import { DocReader } from '../doc';import { PermissionService } from '../permission';-import { DocContentService } from './service';interface RenderOptions {title: string;@@ -50,7 +50,7 @@ export class DocRendererController {private readonly mobileAssets: HtmlAssets = defaultAssets;constructor(- private readonly doc: DocContentService,+ private readonly doc: DocReader,private readonly permission: PermissionService,private readonly config: Config,private readonly url: URLHelper@@ -114,7 +114,7 @@ export class DocRendererController {}if (allowUrlPreview) {- return this.doc.getPageContent(workspaceId, docId);+ return this.doc.getDocContent(workspaceId, docId);}return null;diff --git a/packages/backend/server/src/core/doc-renderer/event.ts b/packages/backend/server/src/core/doc-renderer/event.tsdeleted file mode 100644index 778b104f95..0000000000--- a/packages/backend/server/src/core/doc-renderer/event.ts+++ /dev/null@@ -1,27 +0,0 @@-import { FactoryProvider } from '@nestjs/common';--import { Config, OnEvent } from '../../base';-import { DocContentService } from './service';--class DocEventsListener {- constructor(private readonly doc: DocContentService) {}-- @OnEvent('doc.snapshot.updated')- async handleDocSnapshotUpdated({- workspaceId,- docId,- }: Events['doc.snapshot.updated']) {- await this.doc.markDocContentCacheStale(workspaceId, docId);- }-}--export const DocEventsListenerProvider: FactoryProvider = {- provide: DocEventsListener,- useFactory: (config: Config, doc: DocContentService) => {- if (config.flavor.renderer) {- return new DocEventsListener(doc);- }- return;- },- inject: [Config, DocContentService],-};diff --git a/packages/backend/server/src/core/doc-renderer/index.ts b/packages/backend/server/src/core/doc-renderer/index.tsindex de7c3f570c..203eea66f0 100644--- a/packages/backend/server/src/core/doc-renderer/index.ts+++ b/packages/backend/server/src/core/doc-renderer/index.ts@@ -3,15 +3,9 @@ import { Module } from '@nestjs/common';import { DocStorageModule } from '../doc';import { PermissionModule } from '../permission';import { DocRendererController } from './controller';-import { DocEventsListenerProvider } from './event';-import { DocContentService } from './service';@Module({imports: [DocStorageModule, PermissionModule],- providers: [DocContentService, DocEventsListenerProvider],controllers: [DocRendererController],- exports: [DocContentService],})export class DocRendererModule {}--export { DocContentService };diff --git a/packages/backend/server/src/core/doc-renderer/service.ts b/packages/backend/server/src/core/doc-renderer/service.tsdeleted file mode 100644index 842b4d359f..0000000000--- a/packages/backend/server/src/core/doc-renderer/service.ts+++ /dev/null@@ -1,88 +0,0 @@-import { Injectable } from '@nestjs/common';-import { applyUpdate, Doc } from 'yjs';--import { Cache } from '../../base';-import { DocReader } from '../doc';-import {- type PageDocContent,- parsePageDoc,- parseWorkspaceDoc,- type WorkspaceDocContent,-} from '../utils/blocksuite';--@Injectable()-export class DocContentService {- constructor(- private readonly cache: Cache,- private readonly docReader: DocReader- ) {}-- async getPageContent(- workspaceId: string,- guid: string- ): Promise<PageDocContent | null> {- const cacheKey = `workspace:${workspaceId}:doc:${guid}:content`;- const cachedResult = await this.cache.get<PageDocContent>(cacheKey);-- if (cachedResult) {- return cachedResult;- }-- const docRecord = await this.docReader.getDoc(workspaceId, guid);- if (!docRecord) {- return null;- }-- const doc = new Doc();- applyUpdate(doc, docRecord.bin);-- const content = parsePageDoc(doc);-- if (content) {- await this.cache.set(cacheKey, content, {- ttl:- 7 *- 24 *- 60 *- 60 *- 1000 /* TODO(@forehalo): we need time constants helper */,- });- }- return content;- }-- async getWorkspaceContent(- workspaceId: string- ): Promise<WorkspaceDocContent | null> {- const cacheKey = `workspace:${workspaceId}:content`;- const cachedResult = await this.cache.get<WorkspaceDocContent>(cacheKey);-- if (cachedResult) {- return cachedResult;- }-- const docRecord = await this.docReader.getDoc(workspaceId, workspaceId);- if (!docRecord) {- return null;- }-- const doc = new Doc();- applyUpdate(doc, docRecord.bin);-- const content = parseWorkspaceDoc(doc);-- if (content) {- await this.cache.set(cacheKey, content);- }-- return content;- }-- async markDocContentCacheStale(workspaceId: string, docId: string) {- const key =- workspaceId === docId- ? `workspace:${workspaceId}:content`- : `workspace:${workspaceId}:doc:${docId}:content`;- await this.cache.delete(key);- }-}diff --git a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.tsindex a4cb646a78..b56a006c8a 100644--- a/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts+++ b/packages/backend/server/src/core/doc-service/__tests__/controller.spec.ts@@ -1,4 +1,5 @@import { randomUUID } from 'node:crypto';+import { mock } from 'node:test';import { User, Workspace } from '@prisma/client';import ava, { TestFn } from 'ava';@@ -8,11 +9,13 @@ import { AppModule } from '../../../app.module';import { CryptoHelper } from '../../../base';import { ConfigModule } from '../../../base/config';import { Models } from '../../../models';+import { DatabaseDocReader } from '../../doc';const test = ava as TestFn<{models: Models;app: TestingApp;crypto: CryptoHelper;+ databaseDocReader: DatabaseDocReader;}>;test.before(async t => {@@ -23,6 +26,7 @@ test.before(async t => {t.context.models = app.get(Models);t.context.crypto = app.get(CryptoHelper);t.context.app = app;+ t.context.databaseDocReader = app.get(DatabaseDocReader);});let user: User;@@ -119,3 +123,101 @@ test('should return doc when found', async t => {t.is(res.headers['x-doc-timestamp'], timestamp.toString());t.is(res.headers['x-doc-editor-id'], user.id);});++test('should 404 when doc diff not found', async t => {+ const { app } = t.context;++ const workspaceId = '123';+ const docId = '123';+ await app+ .POST(`/rpc/workspaces/${workspaceId}/docs/${docId}/diff`)+ .set('x-access-token', t.context.crypto.sign(docId))+ .expect({+ status: 404,+ code: 'Not Found',+ type: 'RESOURCE_NOT_FOUND',+ name: 'NOT_FOUND',+ message: 'Doc not found',+ })+ .expect(404);+ t.pass();+});++test('should 404 when doc content not found', async t => {+ const { app } = t.context;++ const workspaceId = '123';+ const docId = '123';+ await app+ .GET(`/rpc/workspaces/${workspaceId}/docs/${docId}/content`)+ .set('x-access-token', t.context.crypto.sign(docId))+ .expect({+ status: 404,+ code: 'Not Found',+ type: 'RESOURCE_NOT_FOUND',+ name: 'NOT_FOUND',+ message: 'Doc not found',+ })+ .expect(404);+ t.pass();+});++test('should get doc content in json format', async t => {+ const { app } = t.context;+ mock.method(t.context.databaseDocReader, 'getDocContent', async () => {+ return {+ title: 'test title',+ summary: 'test summary',+ };+ });++ const docId = randomUUID();+ await app+ .GET(`/rpc/workspaces/${workspace.id}/docs/${docId}/content`)+ .set('x-access-token', t.context.crypto.sign(docId))+ .expect({+ title: 'test title',+ summary: 'test summary',+ })+ .expect(200);+ t.pass();+});++test('should 404 when workspace content not found', async t => {+ const { app } = t.context;++ const workspaceId = '123';+ await app+ .GET(`/rpc/workspaces/${workspaceId}/content`)+ .set('x-access-token', t.context.crypto.sign(workspaceId))+ .expect({+ status: 404,+ code: 'Not Found',+ type: 'RESOURCE_NOT_FOUND',+ name: 'NOT_FOUND',+ message: 'Workspace not found',+ })+ .expect(404);+ t.pass();+});++test('should get workspace content in json format', async t => {+ const { app } = t.context;+ mock.method(t.context.databaseDocReader, 'getWorkspaceContent', async () => {+ return {+ name: 'test name',+ avatarKey: 'avatar key',+ };+ });++ const workspaceId = randomUUID();+ await app+ .GET(`/rpc/workspaces/${workspaceId}/content`)+ .set('x-access-token', t.context.crypto.sign(workspaceId))+ .expect(200)+ .expect({+ name: 'test name',+ avatarKey: 'avatar key',+ });+ t.pass();+});diff --git a/packages/backend/server/src/core/doc-service/controller.ts b/packages/backend/server/src/core/doc-service/controller.tsindex 1f949be967..9bf3d9aa75 100644--- a/packages/backend/server/src/core/doc-service/controller.ts+++ b/packages/backend/server/src/core/doc-service/controller.ts@@ -70,4 +70,31 @@ export class DocRpcController {);res.send(Buffer.concat([diff.missing, diff.state]));}++ @SkipThrottle()+ @Internal()+ @Get('/workspaces/:workspaceId/docs/:docId/content')+ async getDocContent(+ @Param('workspaceId') workspaceId: string,+ @Param('docId') docId: string+ ) {+ const content = await this.docReader.getDocContent(workspaceId, docId);+ if (!content) {+ throw new NotFound('Doc not found');+ }+ this.logger.log(`get doc content ${docId} from workspace ${workspaceId}`);+ return content;+ }++ @SkipThrottle()+ @Internal()+ @Get('/workspaces/:workspaceId/content')+ async getWorkspaceContent(@Param('workspaceId') workspaceId: string) {+ const content = await this.docReader.getWorkspaceContent(workspaceId);+ if (!content) {+ throw new NotFound('Workspace not found');+ }+ this.logger.log(`get workspace content ${workspaceId}`);+ return content;+ }}diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.tsindex 6abc8b2df9..6156cad4a8 100644--- a/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts+++ b/packages/backend/server/src/core/doc/__tests__/reader-from-database.spec.ts@@ -8,13 +8,14 @@ import { createTestingApp, type TestingApp } from '../../../__tests__/utils';import { AppModule } from '../../../app.module';import { ConfigModule } from '../../../base/config';import { Models } from '../../../models';-import { DocReader } from '..';+import { DocReader, PgWorkspaceDocStorageAdapter } from '..';import { DatabaseDocReader } from '../reader';const test = ava as TestFn<{models: Models;app: TestingApp;docReader: DocReader;+ adapter: PgWorkspaceDocStorageAdapter;}>;test.before(async t => {@@ -24,6 +25,7 @@ test.before(async t => {t.context.models = app.get(Models);t.context.docReader = app.get(DocReader);+ t.context.adapter = app.get(PgWorkspaceDocStorageAdapter);t.context.app = app;});@@ -136,3 +138,53 @@ test('should return doc diff', async t => {applyUpdate(doc2, diff3!.missing);t.is(doc2.getText('content').toString(), 'hello world!@');});++test('should get doc content', async t => {+ const docId = randomUUID();+ const { docReader } = t.context;++ const doc = new YDoc();+ const text = doc.getText('content');+ const updates: Buffer[] = [];++ doc.on('update', update => {+ updates.push(Buffer.from(update));+ });++ text.insert(0, 'hello');+ text.insert(5, 'world');+ text.insert(5, ' ');++ await t.context.adapter.pushDocUpdates(workspace.id, docId, updates, user.id);++ const docContent = await docReader.getDocContent(workspace.id, docId);+ // TODO(@fengmk2): should create a test ydoc with blocks+ t.is(docContent, null);+});++test('should get workspace content', async t => {+ const { docReader } = t.context;++ const doc = new YDoc();+ const text = doc.getText('content');+ const updates: Buffer[] = [];++ doc.on('update', update => {+ updates.push(Buffer.from(update));+ });++ text.insert(0, 'hello');+ text.insert(5, 'world');+ text.insert(5, ' ');++ await t.context.adapter.pushDocUpdates(+ workspace.id,+ workspace.id,+ updates,+ user.id+ );++ const workspaceContent = await docReader.getWorkspaceContent(workspace.id);+ // TODO(@fengmk2): should create a test ydoc with blocks+ t.is(workspaceContent, null);+});diff --git a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.tsindex 271f1af43c..0444e38387 100644--- a/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts+++ b/packages/backend/server/src/core/doc/__tests__/reader-from-rpc.spec.ts@@ -10,7 +10,7 @@ import { AppModule } from '../../../app.module';import { Config, UserFriendlyError } from '../../../base';import { ConfigModule } from '../../../base/config';import { Models } from '../../../models';-import { DatabaseDocReader, DocReader } from '..';+import { DatabaseDocReader, DocReader, PgWorkspaceDocStorageAdapter } from '..';import { RpcDocReader } from '../reader';const test = ava as TestFn<{@@ -18,6 +18,7 @@ const test = ava as TestFn<{app: TestingApp;docReader: DocReader;databaseDocReader: DatabaseDocReader;+ adapter: PgWorkspaceDocStorageAdapter;config: Config;}>;@@ -39,6 +40,7 @@ test.before(async t => {t.context.models = app.get(Models);t.context.docReader = app.get(DocReader);t.context.databaseDocReader = app.get(DatabaseDocReader);+ t.context.adapter = app.get(PgWorkspaceDocStorageAdapter);t.context.config = app.get(Config);t.context.app = app;});@@ -71,12 +73,36 @@ test('should return null when doc not found', async t => {});test('should throw error when doc service internal error', async t => {- const { docReader, databaseDocReader } = t.context;+ const { docReader, adapter } = t.context;const docId = randomUUID();- mock.method(databaseDocReader, 'getDoc', async () => {+ mock.method(adapter, 'getDoc', async () => {throw new Error('mock doc service internal error');});- const err = await t.throwsAsync(docReader.getDoc(workspace.id, docId), {+ let err = await t.throwsAsync(docReader.getDoc(workspace.id, docId), {+ instanceOf: UserFriendlyError,+ message: 'An internal error occurred.',+ name: 'internal_server_error',+ });+ t.is(err.type, 'internal_server_error');+ t.is(err.status, 500);++ err = await t.throwsAsync(docReader.getDocDiff(workspace.id, docId), {+ instanceOf: UserFriendlyError,+ message: 'An internal error occurred.',+ name: 'internal_server_error',+ });+ t.is(err.type, 'internal_server_error');+ t.is(err.status, 500);++ err = await t.throwsAsync(docReader.getDocContent(workspace.id, docId), {+ instanceOf: UserFriendlyError,+ message: 'An internal error occurred.',+ name: 'internal_server_error',+ });+ t.is(err.type, 'internal_server_error');+ t.is(err.status, 500);++ err = await t.throwsAsync(docReader.getWorkspaceContent(workspace.id), {instanceOf: UserFriendlyError,message: 'An internal error occurred.',name: 'internal_server_error',@@ -232,3 +258,84 @@ test('should get doc diff fallback to database doc reader when endpoint networkapplyUpdate(doc2, diff!.missing);t.is(doc2.getText('content').toString(), 'hello world!');});++test('should get doc content', async t => {+ const docId = randomUUID();+ const { docReader, databaseDocReader } = t.context;+ mock.method(databaseDocReader, 'getDocContent', async () => {+ return {+ title: 'test title',+ summary: 'test summary',+ };+ });+ const docContent = await docReader.getDocContent(workspace.id, docId);+ t.deepEqual(docContent, {+ title: 'test title',+ summary: 'test summary',+ });+});++test('should return null when doc content not exists', async t => {+ const docId = randomUUID();+ const { docReader, adapter } = t.context;++ const doc = new YDoc();+ const text = doc.getText('content');+ const updates: Buffer[] = [];++ doc.on('update', update => {+ updates.push(Buffer.from(update));+ });++ text.insert(0, 'hello');+ text.insert(5, 'world');+ text.insert(5, ' ');++ await adapter.pushDocUpdates(workspace.id, docId, updates, user.id);++ const docContent = await docReader.getDocContent(workspace.id, docId);+ t.is(docContent, null);++ const notExists = await docReader.getDocContent(workspace.id, randomUUID());+ t.is(notExists, null);+});++test('should get workspace content from doc service rpc', async t => {+ const { docReader, databaseDocReader } = t.context;+ mock.method(databaseDocReader, 'getWorkspaceContent', async () => {+ return {+ name: 'test name',+ avatarKey: 'avatar key',+ };+ });++ const workspaceContent = await docReader.getWorkspaceContent(workspace.id);+ t.deepEqual(workspaceContent, {+ name: 'test name',+ avatarKey: 'avatar key',+ });+});++test('should return null when workspace bin meta not exists', async t => {+ const { docReader, adapter } = t.context;+ const doc = new YDoc();+ const text = doc.getText('content');+ const updates: Buffer[] = [];++ doc.on('update', update => {+ updates.push(Buffer.from(update));+ });++ text.insert(0, 'hello');+ text.insert(5, 'world');+ text.insert(5, ' ');++ await adapter.pushDocUpdates(workspace.id, workspace.id, updates, user.id);++ const workspaceContent = await docReader.getWorkspaceContent(workspace.id);+ t.is(workspaceContent, null);++ // workspace not exists+ const notExists = await docReader.getWorkspaceContent(randomUUID());+ t.is(notExists, null);+});diff --git a/packages/backend/server/src/core/doc/adapters/workspace.ts b/packages/backend/server/src/core/doc/adapters/workspace.tsindex a1c35f8e67..8cd74dcf49 100644--- a/packages/backend/server/src/core/doc/adapters/workspace.ts+++ b/packages/backend/server/src/core/doc/adapters/workspace.ts@@ -344,7 +344,7 @@ export class PgWorkspaceDocStorageAdapter extends DocStorageAdapter {});if (updatedSnapshot) {- this.event.broadcast('doc.snapshot.updated', {+ this.event.emit('doc.snapshot.updated', {workspaceId: snapshot.spaceId,docId: snapshot.docId,});diff --git a/packages/backend/server/src/core/doc/event.ts b/packages/backend/server/src/core/doc/event.tsnew file mode 100644index 0000000000..1f90641737--- /dev/null+++ b/packages/backend/server/src/core/doc/event.ts@@ -0,0 +1,17 @@+import { Injectable } from '@nestjs/common';++import { OnEvent } from '../../base';+import { DocReader } from './reader';++@Injectable()+export class DocEventsListener {+ constructor(private readonly doc: DocReader) {}++ @OnEvent('doc.snapshot.updated')+ async markDocContentCacheStale({+ workspaceId,+ docId,+ }: Events['doc.snapshot.updated']) {+ await this.doc.markDocContentCacheStale(workspaceId, docId);+ }+}diff --git a/packages/backend/server/src/core/doc/index.ts b/packages/backend/server/src/core/doc/index.tsindex 3b55b1e6d0..d563380674 100644--- a/packages/backend/server/src/core/doc/index.ts+++ b/packages/backend/server/src/core/doc/index.ts@@ -6,6 +6,7 @@ import { PermissionModule } from '../permission';import { QuotaModule } from '../quota';import { PgUserspaceDocStorageAdapter } from './adapters/userspace';import { PgWorkspaceDocStorageAdapter } from './adapters/workspace';+import { DocEventsListener } from './event';import { DocStorageCronJob } from './job';import { DocStorageOptions } from './options';import { DatabaseDocReader, DocReader, DocReaderProvider } from './reader';@@ -19,6 +20,7 @@ import { DatabaseDocReader, DocReader, DocReaderProvider } from './reader';DocStorageCronJob,DocReaderProvider,DatabaseDocReader,+ DocEventsListener,],exports: [DatabaseDocReader,diff --git a/packages/backend/server/src/core/doc/reader.ts b/packages/backend/server/src/core/doc/reader.tsindex 3508ea4012..e527800456 100644--- a/packages/backend/server/src/core/doc/reader.ts+++ b/packages/backend/server/src/core/doc/reader.ts@@ -1,17 +1,33 @@import { FactoryProvider, Injectable, Logger } from '@nestjs/common';import { ModuleRef } from '@nestjs/core';-import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';+import {+ applyUpdate,+ diffUpdate,+ Doc as YDoc,+ encodeStateVectorFromUpdate,+} from 'yjs';import {+ Cache,Config,CryptoHelper,getOrGenRequestId,UserFriendlyError,} from '../../base';+import {+ type PageDocContent,+ parsePageDoc,+ parseWorkspaceDoc,+ type WorkspaceDocContent,+} from '../utils/blocksuite';import { PgWorkspaceDocStorageAdapter } from './adapters/workspace';import { type DocDiff, type DocRecord } from './storage';+const DOC_CONTENT_CACHE_7_DAYS = 7 * 24 * 60 * 60 * 1000;+export abstract class DocReader {+ constructor(protected readonly cache: Cache) {}+abstract getDoc(workspaceId: string,docId: string@@ -23,6 +39,60 @@ export abstract class DocReader {stateVector?: Uint8Array): Promise<DocDiff | null>;+ async getDocContent(+ workspaceId: string,+ docId: string+ ): Promise<PageDocContent | null> {+ const cacheKey = this.cacheKey(workspaceId, docId);+ const cachedResult = await this.cache.get<PageDocContent>(cacheKey);+ if (cachedResult) {+ return cachedResult;+ }++ const content = await this.getDocContentWithoutCache(workspaceId, docId);+ if (content) {+ await this.cache.set(cacheKey, content, {+ ttl: DOC_CONTENT_CACHE_7_DAYS,+ });+ }+ return content;+ }++ async getWorkspaceContent(+ workspaceId: string+ ): Promise<WorkspaceDocContent | null> {+ const cacheKey = this.cacheKey(workspaceId, workspaceId);+ const cachedResult = await this.cache.get<WorkspaceDocContent>(cacheKey);+ if (cachedResult) {+ return cachedResult;+ }++ const content = await this.getWorkspaceContentWithoutCache(workspaceId);+ if (content) {+ await this.cache.set(cacheKey, content);+ }+ return content;+ }++ async markDocContentCacheStale(workspaceId: string, docId: string) {+ await this.cache.delete(this.cacheKey(workspaceId, docId));+ }++ private cacheKey(workspaceId: string, docId: string) {+ return workspaceId === docId+ ? `workspace:${workspaceId}:content`+ : `workspace:${workspaceId}:doc:${docId}:content`;+ }++ protected abstract getDocContentWithoutCache(+ workspaceId: string,+ guid: string+ ): Promise<PageDocContent | null>;++ protected abstract getWorkspaceContentWithoutCache(+ workspaceId: string+ ): Promise<WorkspaceDocContent | null>;+protected docDiff(update: Uint8Array, stateVector?: Uint8Array) {const missing = stateVector ? diffUpdate(update, stateVector) : update;const state = encodeStateVectorFromUpdate(update);@@ -35,8 +105,11 @@ export abstract class DocReader {@Injectable()export class DatabaseDocReader extends DocReader {- constructor(protected readonly workspace: PgWorkspaceDocStorageAdapter) {- super();+ constructor(+ protected override readonly cache: Cache,+ protected readonly workspace: PgWorkspaceDocStorageAdapter+ ) {+ super(cache);}async getDoc(workspaceId: string, docId: string): Promise<DocRecord | null> {@@ -57,6 +130,31 @@ export class DatabaseDocReader extends DocReader {timestamp: doc.timestamp,};}++ protected override async getDocContentWithoutCache(+ workspaceId: string,+ guid: string+ ): Promise<PageDocContent | null> {+ const docRecord = await this.workspace.getDoc(workspaceId, guid);+ if (!docRecord) {+ return null;+ }+ const doc = new YDoc();+ applyUpdate(doc, docRecord.bin);+ return parsePageDoc(doc);+ }++ protected override async getWorkspaceContentWithoutCache(+ workspaceId: string+ ): Promise<WorkspaceDocContent | null> {+ const docRecord = await this.workspace.getDoc(workspaceId, workspaceId);+ if (!docRecord) {+ return null;+ }+ const doc = new YDoc();+ applyUpdate(doc, docRecord.bin);+ return parseWorkspaceDoc(doc);+ }}@Injectable()@@ -66,9 +164,10 @@ export class RpcDocReader extends DatabaseDocReader {constructor(private readonly config: Config,private readonly crypto: CryptoHelper,+ protected override readonly cache: Cache,protected override readonly workspace: PgWorkspaceDocStorageAdapter) {- super(workspace);+ super(cache, workspace);}private async fetch(@@ -128,11 +227,8 @@ export class RpcDocReader extends DatabaseDocReader {// other errorthis.logger.error(`Failed to fetch doc ${url}, fallback to database doc reader`,- err.stack+ err);- if (err.cause instanceof Error) {- this.logger.error(err.cause.stack);- }// fallback to database doc reader if the error is not user friendly, like network errorreturn await super.getDoc(workspaceId, docId);}@@ -172,15 +268,61 @@ export class RpcDocReader extends DatabaseDocReader {const err = e as Error;this.logger.error(`Failed to fetch doc diff ${url}, fallback to database doc reader`,- err.stack+ err);- if (err.cause instanceof Error) {- this.logger.error(err.cause.stack);- }// fallback to database doc reader if the error is not user friendly, like network errorreturn await super.getDocDiff(workspaceId, docId, stateVector);}}++ protected override async getDocContentWithoutCache(+ workspaceId: string,+ docId: string+ ): Promise<PageDocContent | null> {+ const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/docs/${docId}/content`;+ const accessToken = this.crypto.sign(docId);+ try {+ const res = await this.fetch(accessToken, url, 'GET');+ if (!res) {+ return null;+ }+ return (await res.json()) as PageDocContent;+ } catch (e) {+ if (e instanceof UserFriendlyError) {+ throw e;+ }+ const err = e as Error;+ this.logger.error(+ `Failed to fetch doc content ${url}, fallback to database doc reader`,+ err+ );+ return await super.getDocContentWithoutCache(workspaceId, docId);+ }+ }++ protected override async getWorkspaceContentWithoutCache(+ workspaceId: string+ ): Promise<WorkspaceDocContent | null> {+ const url = `${this.config.docService.endpoint}/rpc/workspaces/${workspaceId}/content`;+ const accessToken = this.crypto.sign(workspaceId);+ try {+ const res = await this.fetch(accessToken, url, 'GET');+ if (!res) {+ return null;+ }+ return (await res.json()) as WorkspaceDocContent;+ } catch (e) {+ if (e instanceof UserFriendlyError) {+ throw e;+ }+ const err = e as Error;+ this.logger.error(+ `Failed to fetch workspace content ${url}, fallback to database doc reader`,+ err+ );+ return await super.getWorkspaceContentWithoutCache(workspaceId);+ }+ }}export const DocReaderProvider: FactoryProvider = {diff --git a/packages/backend/server/src/core/workspaces/resolvers/service.ts b/packages/backend/server/src/core/workspaces/resolvers/service.tsindex 4955d39c82..87f5f48f0a 100644--- a/packages/backend/server/src/core/workspaces/resolvers/service.ts+++ b/packages/backend/server/src/core/workspaces/resolvers/service.ts@@ -10,7 +10,7 @@ import {UserNotFound,} from '../../../base';import { Models } from '../../../models';-import { DocContentService } from '../../doc-renderer';+import { DocReader } from '../../doc';import { PermissionService, WorkspaceRole } from '../../permission';import { WorkspaceBlobStorage } from '../../storage';@@ -30,7 +30,7 @@ export class WorkspaceService {constructor(private readonly blobStorage: WorkspaceBlobStorage,private readonly cache: Cache,- private readonly doc: DocContentService,+ private readonly doc: DocReader,private readonly mailer: MailService,private readonly permission: PermissionService,private readonly prisma: PrismaClient,