Bring projects, wikis, and teams together with AI. AppFlowy is the AI collaborative workspace where you achieve more without losing control of your data. The leading open source Notion alternative.
Dart blogconfluence-alternativecontent-managementcontent-servicesdocumentationflutternote-takingnotion-alternativeproject-managementtask-managementteam-collaborationwiki
chore: improvements to caching strategies (#5488)
Kilu.He86696b271e35
diff --git a/frontend/appflowy_web_app/src/application/database-yjs/selector.ts b/frontend/appflowy_web_app/src/application/database-yjs/selector.tsindex 49fceb885c..884c58516d 100644--- a/frontend/appflowy_web_app/src/application/database-yjs/selector.ts+++ b/frontend/appflowy_web_app/src/application/database-yjs/selector.ts@@ -71,7 +71,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {const [key] = viewItem;const view = folderViews?.get(key);- console.log('view', view?.get(YjsFolderKey.bid), iidIndex);if (visibleViewsId.includes(key) &&view &&@@ -81,7 +80,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {}}- console.log('viewsId', viewsId);setViewIds(viewsId);};diff --git a/frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.ts b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.tsnew file mode 100644index 0000000000..9a7e3fcb9d--- /dev/null+++ b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.ts@@ -0,0 +1,310 @@+import { CollabType } from '@/application/collab.type';+import * as Y from 'yjs';+import { withTestingYDoc } from '@/application/slate-yjs/__tests__/withTestingYjsEditor';+import { expect } from '@jest/globals';+import { getCollab, batchCollab, collabTypeToDBType } from '../cache';+import { applyYDoc } from '@/application/ydoc/apply';+import { getCollabDBName, openCollabDB } from '../cache/db';+import { StrategyType } from '../cache/types';++jest.mock('@/application/ydoc/apply', () => ({+ applyYDoc: jest.fn(),+}));+jest.mock('../cache/db', () => ({+ openCollabDB: jest.fn(),+ getCollabDBName: jest.fn(),+}));++const emptyDoc = new Y.Doc();+const normalDoc = withTestingYDoc('1');+const mockFetcher = jest.fn();+const mockBatchFetcher = jest.fn();++describe('Cache functions', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ describe('getCollab', () => {+ describe('with CACHE_ONLY strategy', () => {+ it('should throw error when no cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ await expect(+ getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.CACHE_ONLY+ )+ ).rejects.toThrow('No cache found');+ });+ it('should fetch collab with CACHE_ONLY strategy and existing cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ const result = await getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.CACHE_ONLY+ );++ expect(result).toBe(normalDoc);+ expect(mockFetcher).not.toHaveBeenCalled();+ expect(applyYDoc).not.toHaveBeenCalled();+ });+ });++ describe('with CACHE_FIRST strategy', () => {+ it('should fetch collab with CACHE_FIRST strategy and existing cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ mockFetcher.mockResolvedValue({ state: new Uint8Array() });++ const result = await getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.CACHE_FIRST+ );++ expect(result).toBe(normalDoc);+ expect(mockFetcher).not.toHaveBeenCalled();+ expect(applyYDoc).not.toHaveBeenCalled();+ });++ it('should fetch collab with CACHE_FIRST strategy and no cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ mockFetcher.mockResolvedValue({ state: new Uint8Array() });++ const result = await getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.CACHE_FIRST+ );++ expect(result).toBe(emptyDoc);+ expect(mockFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });+ });++ describe('with CACHE_AND_NETWORK strategy', () => {+ it('should fetch collab with CACHE_AND_NETWORK strategy and existing cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ mockFetcher.mockResolvedValue({ state: new Uint8Array() });++ const result = await getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.CACHE_AND_NETWORK+ );++ expect(result).toBe(normalDoc);+ expect(mockFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });++ it('should fetch collab with CACHE_AND_NETWORK strategy and no cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ mockFetcher.mockResolvedValue({ state: new Uint8Array() });++ const result = await getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.CACHE_AND_NETWORK+ );++ expect(result).toBe(emptyDoc);+ expect(mockFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });+ });++ describe('with default strategy', () => {+ it('should fetch collab with default strategy', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ mockFetcher.mockResolvedValue({ state: new Uint8Array() });++ const result = await getCollab(+ mockFetcher,+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ StrategyType.NETWORK_ONLY+ );++ expect(result).toBe(normalDoc);+ expect(mockFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });+ });+ });++ describe('batchCollab', () => {+ describe('with CACHE_ONLY strategy', () => {+ it('should batch fetch collabs with CACHE_ONLY strategy and no cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);++ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ await expect(+ batchCollab(+ mockBatchFetcher,+ [+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ ],+ StrategyType.CACHE_ONLY+ )+ ).rejects.toThrow('No cache found');+ });++ it('should batch fetch collabs with CACHE_ONLY strategy and existing cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);++ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ await batchCollab(+ mockBatchFetcher,+ [+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ ],+ StrategyType.CACHE_ONLY+ );++ expect(mockBatchFetcher).not.toHaveBeenCalled();+ });+ });++ describe('with CACHE_FIRST strategy', () => {+ it('should batch fetch collabs with CACHE_FIRST strategy and existing cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);++ (getCollabDBName as jest.Mock).mockReturnValue('testDB');++ await batchCollab(+ mockBatchFetcher,+ [+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ ],+ StrategyType.CACHE_FIRST+ );++ expect(mockBatchFetcher).not.toHaveBeenCalled();+ });++ it('should batch fetch collabs with CACHE_FIRST strategy and no cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);++ (getCollabDBName as jest.Mock).mockReturnValue('testDB');+ mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });++ await batchCollab(+ mockBatchFetcher,+ [+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ ],+ StrategyType.CACHE_FIRST+ );++ expect(mockBatchFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });+ });++ describe('with CACHE_AND_NETWORK strategy', () => {+ it('should batch fetch collabs with CACHE_AND_NETWORK strategy', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);++ (getCollabDBName as jest.Mock).mockReturnValue('testDB');+ mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });++ await batchCollab(+ mockBatchFetcher,+ [+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ ],+ StrategyType.CACHE_AND_NETWORK+ );++ expect(mockBatchFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });++ it('should batch fetch collabs with CACHE_AND_NETWORK strategy and no cache', async () => {+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);++ (getCollabDBName as jest.Mock).mockReturnValue('testDB');+ mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });++ await batchCollab(+ mockBatchFetcher,+ [+ {+ collabId: 'id1',+ collabType: CollabType.Document,+ },+ ],+ StrategyType.CACHE_AND_NETWORK+ );++ expect(mockBatchFetcher).toHaveBeenCalled();+ expect(applyYDoc).toHaveBeenCalled();+ });+ });+ });+});++describe('collabTypeToDBType', () => {+ it('should return correct DB type', () => {+ expect(collabTypeToDBType(CollabType.Document)).toBe('document');+ expect(collabTypeToDBType(CollabType.Folder)).toBe('folder');+ expect(collabTypeToDBType(CollabType.Database)).toBe('database');+ expect(collabTypeToDBType(CollabType.WorkspaceDatabase)).toBe('databases');+ expect(collabTypeToDBType(CollabType.DatabaseRow)).toBe('database_row');+ expect(collabTypeToDBType(CollabType.UserAwareness)).toBe('user_awareness');+ expect(collabTypeToDBType(CollabType.Empty)).toBe('');+ });+});diff --git a/frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.ts b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.tsnew file mode 100644index 0000000000..afdc418c2a--- /dev/null+++ b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.ts@@ -0,0 +1,57 @@+import { expect } from '@jest/globals';+import { fetchCollab, batchFetchCollab } from '../fetch';+import { CollabType } from '@/application/collab.type';+import { APIService } from '@/application/services/js-services/wasm';++jest.mock('@/application/services/js-services/wasm', () => {+ return {+ APIService: {+ getCollab: jest.fn(),+ batchGetCollab: jest.fn(),+ },+ };+});++describe('Collab fetch functions with deduplication', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ });++ describe('fetchCollab', () => {+ it('should fetch collab without duplicating requests', async () => {+ const workspaceId = 'workspace1';+ const id = 'id1';+ const type = CollabType.Document;+ const mockResponse = { data: 'mockData' };++ (APIService.getCollab as jest.Mock).mockResolvedValue(mockResponse);++ const result1 = fetchCollab(workspaceId, id, type);+ const result2 = fetchCollab(workspaceId, id, type);++ expect(result1).toBe(result2);+ await expect(result1).resolves.toEqual(mockResponse);+ expect(APIService.getCollab).toHaveBeenCalledTimes(1);+ });+ });++ describe('batchFetchCollab', () => {+ it('should batch fetch collabs without duplicating requests', async () => {+ const workspaceId = 'workspace1';+ const params = [+ { collabId: 'id1', collabType: CollabType.Document },+ { collabId: 'id2', collabType: CollabType.Folder },+ ];+ const mockResponse = { data: 'mockData' };++ (APIService.batchGetCollab as jest.Mock).mockResolvedValue(mockResponse);++ const result1 = batchFetchCollab(workspaceId, params);+ const result2 = batchFetchCollab(workspaceId, params);++ expect(result1).toBe(result2);+ await expect(result1).resolves.toEqual(mockResponse);+ expect(APIService.batchGetCollab).toHaveBeenCalledTimes(1);+ });+ });+});diff --git a/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/auth.service.tsindex 7119497775..7f80c9f871 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts@@ -1,8 +1,8 @@import { AuthService } from '@/application/services/services.type';import { ProviderType, SignUpWithEmailPasswordParams } from '@/application/user.type';import { APIService } from 'src/application/services/js-services/wasm';-import { signInSuccess } from '@/application/services/js-services/storage/auth';-import { invalidToken } from '@/application/services/js-services/storage';+import { signInSuccess } from '@/application/services/js-services/session/auth';+import { invalidToken } from 'src/application/services/js-services/session';import { afterSignInDecorator } from '@/application/services/js-services/decorator';export class JSAuthService implements AuthService {diff --git a/frontend/appflowy_web_app/src/application/services/js-services/db/index.ts b/frontend/appflowy_web_app/src/application/services/js-services/cache/db.tssimilarity index 76%rename from frontend/appflowy_web_app/src/application/services/js-services/db/index.tsrename to frontend/appflowy_web_app/src/application/services/js-services/cache/db.tsindex f17ac50531..a4d888498b 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/db/index.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/cache/db.ts@@ -1,9 +1,10 @@import { YDoc } from '@/application/collab.type';import { databasePrefix } from '@/application/constants';-import { getAuthInfo } from '@/application/services/js-services/storage';import { IndexeddbPersistence } from 'y-indexeddb';import * as Y from 'yjs';+const openedSet = new Set<string>();+/*** Open the collaboration database, and return a function to close it*/@@ -19,6 +20,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {});provider.on('synced', () => {+ if (!openedSet.has(name)) {+ openedSet.add(name);+ }+resolve(true);});@@ -27,9 +32,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {return doc as YDoc;}-export function getDBName(id: string, type: string) {- const { uuid } = getAuthInfo() || {};+export function getCollabDBName(id: string, type: string, uuid?: string) {+ if (!uuid) {+ return `${type}_${id}`;+ }- if (!uuid) throw new Error('No user found');return `${uuid}_${type}_${id}`;}diff --git a/frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts b/frontend/appflowy_web_app/src/application/services/js-services/cache/index.tsnew file mode 100644index 0000000000..1f7fe670f1--- /dev/null+++ b/frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts@@ -0,0 +1,165 @@+import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';+import { applyYDoc } from '@/application/ydoc/apply';+import { getCollabDBName, openCollabDB } from './db';+import { Fetcher, StrategyType } from './types';++export function collabTypeToDBType(type: CollabType) {+ switch (type) {+ case CollabType.Folder:+ return 'folder';+ case CollabType.Document:+ return 'document';+ case CollabType.Database:+ return 'database';+ case CollabType.WorkspaceDatabase:+ return 'databases';+ case CollabType.DatabaseRow:+ return 'database_row';+ case CollabType.UserAwareness:+ return 'user_awareness';+ default:+ return '';+ }+}++const collabSharedRootKeyMap = {+ [CollabType.Folder]: YjsEditorKey.folder,+ [CollabType.Document]: YjsEditorKey.document,+ [CollabType.Database]: YjsEditorKey.database,+ [CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,+ [CollabType.DatabaseRow]: YjsEditorKey.database_row,+ [CollabType.UserAwareness]: YjsEditorKey.user_awareness,+ [CollabType.Empty]: YjsEditorKey.empty,+};++export function hasCache(doc: YDoc, type: CollabType) {+ const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;++ return data.has(collabSharedRootKeyMap[type] as string);+}++export async function getCollab(+ fetcher: Fetcher<{+ state: Uint8Array;+ }>,+ {+ collabId,+ collabType,+ uuid,+ }: {+ uuid?: string;+ collabId: string;+ collabType: CollabType;+ },+ strategy: StrategyType = StrategyType.CACHE_AND_NETWORK+) {+ const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);+ const collab = await openCollabDB(name);+ const exist = hasCache(collab, collabType);++ switch (strategy) {+ case StrategyType.CACHE_ONLY: {+ if (!exist) {+ throw new Error('No cache found');+ }++ return collab;+ }++ case StrategyType.CACHE_FIRST: {+ if (!exist) {+ await revalidateCollab(fetcher, collab);+ }++ return collab;+ }++ case StrategyType.CACHE_AND_NETWORK: {+ if (!exist) {+ await revalidateCollab(fetcher, collab);+ } else {+ void revalidateCollab(fetcher, collab);+ }++ return collab;+ }++ default: {+ await revalidateCollab(fetcher, collab);++ return collab;+ }+ }+}++async function revalidateCollab(+ fetcher: Fetcher<{+ state: Uint8Array;+ }>,+ collab: YDoc+) {+ const { state } = await fetcher();++ applyYDoc(collab, state);+}++export async function batchCollab(+ batchFetcher: Fetcher<Record<string, number[]>>,+ collabs: {+ collabId: string;+ collabType: CollabType;+ uuid?: string;+ }[],+ strategy: StrategyType = StrategyType.CACHE_AND_NETWORK,+ itemCallback?: (id: string, doc: YDoc) => void+) {+ const collabMap = new Map<string, YDoc>();++ for (const { collabId, collabType, uuid } of collabs) {+ const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);+ const collab = await openCollabDB(name);+ const exist = hasCache(collab, collabType);++ collabMap.set(collabId, collab);+ if (exist) {+ itemCallback?.(collabId, collab);+ }+ }++ const notCacheIds = collabs.filter(({ collabId, collabType }) => {+ const id = collabMap.get(collabId);++ if (!id) return false;++ return !hasCache(id, collabType);+ });++ if (strategy === StrategyType.CACHE_ONLY) {+ if (notCacheIds.length > 0) {+ throw new Error('No cache found');+ }++ return;+ }++ if (strategy === StrategyType.CACHE_FIRST && notCacheIds.length === 0) {+ return;+ }++ const states = await batchFetcher();++ for (const [collabId, data] of Object.entries(states)) {+ const info = collabs.find((item) => item.collabId === collabId);+ const collab = collabMap.get(collabId);++ if (!info || !collab) {+ continue;+ }++ const state = new Uint8Array(data);++ applyYDoc(collab, state);++ itemCallback?.(collabId, collab);+ }+}diff --git a/frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts b/frontend/appflowy_web_app/src/application/services/js-services/cache/types.tsnew file mode 100644index 0000000000..1c1a949723--- /dev/null+++ b/frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts@@ -0,0 +1,12 @@+export enum StrategyType {+ // Cache only: return the cache if it exists, otherwise throw an error+ CACHE_ONLY = 'CACHE_ONLY',+ // Cache first: return the cache if it exists, otherwise fetch from the network+ CACHE_FIRST = 'CACHE_FIRST',+ // Cache and network: return the cache if it exists, otherwise fetch from the network and update the cache+ CACHE_AND_NETWORK = 'CACHE_AND_NETWORK',+ // Network only: fetch from the network and update the cache+ NETWORK_ONLY = 'NETWORK_ONLY',+}++export type Fetcher<T> = () => Promise<T>;diff --git a/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/database.service.tsindex b1dcfe8d0f..cf29b221dd 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts@@ -1,16 +1,16 @@import { CollabType, YDatabase, YDoc, YjsDatabaseKey, YjsEditorKey } from '@/application/collab.type';-import {- batchCollabs,- getCollabStorage,- getCollabStorageWithAPICall,- getCurrentWorkspace,-} from '@/application/services/js-services/storage';+import { batchCollab, getCollab } from '@/application/services/js-services/cache';+import { StrategyType } from '@/application/services/js-services/cache/types';+import { batchFetchCollab, fetchCollab } from '@/application/services/js-services/fetch';+import { getCurrentWorkspace } from 'src/application/services/js-services/session';import { DatabaseService } from '@/application/services/services.type';import * as Y from 'yjs';export class JSDatabaseService implements DatabaseService {private loadedDatabaseId: Set<string> = new Set();+ private loadedWorkspaceId: Set<string> = new Set();+private cacheDatabaseRowDocMap: Map<string, Y.Doc> = new Map();constructor() {@@ -28,22 +28,30 @@ export class JSDatabaseService implements DatabaseService {throw new Error('Workspace database not found');}- const workspaceDatabase = await getCollabStorageWithAPICall(- workspace.id,- workspace.workspaceDatabaseId,- CollabType.WorkspaceDatabase+ const isLoaded = this.loadedWorkspaceId.has(workspace.id);++ const workspaceDatabase = await getCollab(+ () => {+ return fetchCollab(workspace.id, workspace.workspaceDatabaseId, CollabType.WorkspaceDatabase);+ },+ {+ collabId: workspace.workspaceDatabaseId,+ collabType: CollabType.WorkspaceDatabase,+ },+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK);+ if (!isLoaded) {+ this.loadedWorkspaceId.add(workspace.id);+ }+return workspaceDatabase.getMap(YjsEditorKey.data_section).get(YjsEditorKey.workspace_database).toJSON() as {views: string[];database_id: string;}[];}- async openDatabase(- databaseId: string,- rowIds?: string[]- ): Promise<{+ async openDatabase(databaseId: string): Promise<{databaseDoc: YDoc;rows: Y.Map<YDoc>;}> {@@ -68,13 +76,18 @@ export class JSDatabaseService implements DatabaseService {const rowsFolder: Y.Map<YDoc> = rootRowsDoc.getMap();- let databaseDoc: YDoc | undefined = undefined;+ const databaseDoc = await getCollab(+ () => {+ return fetchCollab(workspaceId, databaseId, CollabType.Database);+ },+ {+ collabId: databaseId,+ collabType: CollabType.Database,+ },+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK+ );- if (isLoaded) {- databaseDoc = (await getCollabStorage(databaseId, CollabType.Database)).doc;- } else {- databaseDoc = await getCollabStorageWithAPICall(workspaceId, databaseId, CollabType.Database);- }+ if (!isLoaded) this.loadedDatabaseId.add(databaseId);const database = databaseDoc.getMap(YjsEditorKey.data_section)?.get(YjsEditorKey.database) as YDatabase;const viewId = database.get(YjsDatabaseKey.metas)?.get(YjsDatabaseKey.iid)?.toString();@@ -87,47 +100,50 @@ export class JSDatabaseService implements DatabaseService {throw new Error('Database rows not found');}- const ids = rowIds ? rowIds : rowOrdersIds.map((item) => item.id);-- if (isLoaded) {- for (const id of ids) {- const { doc } = await getCollabStorage(id, CollabType.DatabaseRow);-+ const rowsParams = rowOrdersIds.map((item) => ({+ collabId: item.id,+ collabType: CollabType.DatabaseRow,+ }));++ void batchCollab(+ () => {+ return batchFetchCollab(workspaceId, rowsParams);+ },+ rowsParams,+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK,+ (id: string, doc: YDoc) => {if (!rowsFolder.has(id)) {rowsFolder.set(id, doc);}}- } else {- void this.loadDatabaseRows(workspaceId, ids, (id, row) => {- if (!rowsFolder.has(id)) {- rowsFolder.set(id, row);- }- });- }+ );- this.loadedDatabaseId.add(databaseId);-- if (!rowIds) {- // Update rows if new rows are added- rowOrders?.observe((event) => {- if (event.changes.added.size > 0) {- const rowIds = rowOrders.toJSON() as {- id: string;- }[];-- console.log('Update rows', rowIds);- void this.loadDatabaseRows(- workspaceId,- rowIds.map((item) => item.id),- (rowId: string, rowDoc) => {- if (!rowsFolder.has(rowId)) {- rowsFolder.set(rowId, rowDoc);- }+ // Update rows if there are new rows added after the database has been loaded+ rowOrders?.observe((event) => {+ if (event.changes.added.size > 0) {+ const rowIds = rowOrders.toJSON() as {+ id: string;+ }[];++ const params = rowIds.map((item) => ({+ collabId: item.id,+ collabType: CollabType.DatabaseRow,+ }));++ void batchCollab(+ () => {+ return batchFetchCollab(workspaceId, params);+ },+ params,+ StrategyType.CACHE_AND_NETWORK,+ (id: string, doc: YDoc) => {+ if (!rowsFolder.has(id)) {+ rowsFolder.set(id, doc);}- );- }- });- }+ }+ );+ }+ });return {databaseDoc,@@ -135,21 +151,6 @@ export class JSDatabaseService implements DatabaseService {};}- async loadDatabaseRows(workspaceId: string, rowIds: string[], rowCallback: (rowId: string, rowDoc: YDoc) => void) {- try {- await batchCollabs(- workspaceId,- rowIds.map((id) => ({- object_id: id,- collab_type: CollabType.DatabaseRow,- })),- rowCallback- );- } catch (e) {- console.error(e);- }- }-async closeDatabase(databaseId: string) {this.cacheDatabaseRowDocMap.delete(databaseId);}diff --git a/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/document.service.tsindex cc3d3dd411..bcf73ae550 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts@@ -1,8 +1,13 @@import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';-import { getCollabStorageWithAPICall, getCurrentWorkspace } from '@/application/services/js-services/storage';+import { getCollab } from '@/application/services/js-services/cache';+import { StrategyType } from '@/application/services/js-services/cache/types';+import { fetchCollab } from '@/application/services/js-services/fetch';+import { getCurrentWorkspace } from 'src/application/services/js-services/session';import { DocumentService } from '@/application/services/services.type';export class JSDocumentService implements DocumentService {+ private loaded: Set<string> = new Set();+constructor() {//}@@ -14,8 +19,20 @@ export class JSDocumentService implements DocumentService {throw new Error('Workspace database not found');}- const doc = await getCollabStorageWithAPICall(workspace.id, docId, CollabType.Document);+ const isLoaded = this.loaded.has(docId);++ const doc = await getCollab(+ () => {+ return fetchCollab(workspace.id, docId, CollabType.Document);+ },+ {+ collabId: docId,+ collabType: CollabType.Document,+ },+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK+ );+ if (!isLoaded) this.loaded.add(docId);const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {if (origin === CollabOrigin.LocalSync) {// Send the update to the serverdiff --git a/frontend/appflowy_web_app/src/application/services/js-services/fetch.ts b/frontend/appflowy_web_app/src/application/services/js-services/fetch.tsnew file mode 100644index 0000000000..7ae4dc8902--- /dev/null+++ b/frontend/appflowy_web_app/src/application/services/js-services/fetch.ts@@ -0,0 +1,66 @@+import { CollabType } from '@/application/collab.type';+import { APIService } from '@/application/services/js-services/wasm';++const pendingRequests = new Map();++function generateRequestKey<T>(url: string, params: T) {+ if (!params) return url;++ try {+ return `${url}_${JSON.stringify(params)}`;+ } catch (_e) {+ return `${url}_${params}`;+ }+}++// Deduplication fetch requests+// When multiple requests are made to the same URL with the same params, only one request is made+// and the result is shared with all the requests+function fetchWithDeduplication<Req, Res>(url: string, params: Req, fetchFunction: () => Promise<Res>): Promise<Res> {+ const requestKey = generateRequestKey<Req>(url, params);++ if (pendingRequests.has(requestKey)) {+ return pendingRequests.get(requestKey);+ }++ const fetchPromise = fetchFunction().finally(() => {+ pendingRequests.delete(requestKey);+ });++ pendingRequests.set(requestKey, fetchPromise);+ return fetchPromise;+}++/**+ * Fetch collab+ * @param workspaceId+ * @param id+ * @param type [CollabType]+ */+export function fetchCollab(workspaceId: string, id: string, type: CollabType) {+ const fetchFunction = () => APIService.getCollab(workspaceId, id, type);++ return fetchWithDeduplication(`fetchCollab_${workspaceId}`, { id, type }, fetchFunction);+}++/**+ * Batch fetch collab+ * Usage:+ * // load database rows+ * const rows = await batchFetchCollab(workspaceId, databaseRows.map((row) => ({ collabId: row.id, collabType: CollabType.DatabaseRow })));+ *+ * @param workspaceId+ * @param params [{ collabId: string; collabType: CollabType }]+ */+export function batchFetchCollab(workspaceId: string, params: { collabId: string; collabType: CollabType }[]) {+ const fetchFunction = () =>+ APIService.batchGetCollab(+ workspaceId,+ params.map(({ collabId, collabType }) => ({+ object_id: collabId,+ collab_type: collabType,+ }))+ );++ return fetchWithDeduplication(`batchFetchCollab_${workspaceId}`, params, fetchFunction);+}diff --git a/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/folder.service.tsindex c475cfa935..f145480c18 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts@@ -1,14 +1,30 @@import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';-import { getCollabStorageWithAPICall } from '@/application/services/js-services/storage';+import { getCollab } from '@/application/services/js-services/cache';+import { StrategyType } from '@/application/services/js-services/cache/types';+import { fetchCollab } from '@/application/services/js-services/fetch';import { FolderService } from '@/application/services/services.type';export class JSFolderService implements FolderService {+ private loaded: Set<string> = new Set();+constructor() {//}async openWorkspace(workspaceId: string): Promise<YDoc> {- const doc = await getCollabStorageWithAPICall(workspaceId, workspaceId, CollabType.Folder);+ const isLoaded = this.loaded.has(workspaceId);+ const doc = await getCollab(+ () => {+ return fetchCollab(workspaceId, workspaceId, CollabType.Folder);+ },+ {+ collabId: workspaceId,+ collabType: CollabType.Folder,+ },+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK+ );++ if (!isLoaded) this.loaded.add(workspaceId);const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {if (origin === CollabOrigin.LocalSync) {// Send the update to the serverdiff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/auth.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/auth.tssimilarity index 100%rename from frontend/appflowy_web_app/src/application/services/js-services/storage/auth.tsrename to frontend/appflowy_web_app/src/application/services/js-services/session/auth.tsdiff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/index.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/index.tssimilarity index 73%rename from frontend/appflowy_web_app/src/application/services/js-services/storage/index.tsrename to frontend/appflowy_web_app/src/application/services/js-services/session/index.tsindex f0b9cab2d6..c618a85cfd 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/storage/index.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/session/index.ts@@ -1,4 +1,3 @@export * from './token';export * from './user';-export * from './collab';export * from './auth';diff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/token.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/token.tssimilarity index 100%rename from frontend/appflowy_web_app/src/application/services/js-services/storage/token.tsrename to frontend/appflowy_web_app/src/application/services/js-services/session/token.tsdiff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/user.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/user.tssimilarity index 100%rename from frontend/appflowy_web_app/src/application/services/js-services/storage/user.tsrename to frontend/appflowy_web_app/src/application/services/js-services/session/user.tsdiff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/collab.ts b/frontend/appflowy_web_app/src/application/services/js-services/storage/collab.tsdeleted file mode 100644index 6b315658f5..0000000000--- a/frontend/appflowy_web_app/src/application/services/js-services/storage/collab.ts+++ /dev/null@@ -1,118 +0,0 @@-import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';-import { getDBName, openCollabDB } from '@/application/services/js-services/db';-import { APIService } from '@/application/services/js-services/wasm';-import { applyYDoc } from '@/application/ydoc/apply';--export function fetchCollab(workspaceId: string, id: string, type: CollabType) {- return APIService.getCollab(workspaceId, id, type);-}--export function batchFetchCollab(workspaceId: string, params: { object_id: string; collab_type: CollabType }[]) {- return APIService.batchGetCollab(workspaceId, params);-}--function collabTypeToDBType(type: CollabType) {- switch (type) {- case CollabType.Folder:- return 'folder';- case CollabType.Document:- return 'document';- case CollabType.Database:- return 'database';- case CollabType.WorkspaceDatabase:- return 'databases';- case CollabType.DatabaseRow:- return 'database_row';- case CollabType.UserAwareness:- return 'user_awareness';- default:- return '';- }-}--const collabSharedRootKeyMap = {- [CollabType.Folder]: YjsEditorKey.folder,- [CollabType.Document]: YjsEditorKey.document,- [CollabType.Database]: YjsEditorKey.database,- [CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,- [CollabType.DatabaseRow]: YjsEditorKey.database_row,- [CollabType.UserAwareness]: YjsEditorKey.user_awareness,- [CollabType.Empty]: YjsEditorKey.empty,-};--export async function getCollabStorage(id: string, type: CollabType) {- const name = getDBName(id, collabTypeToDBType(type));-- const doc = await openCollabDB(name);- let localExist = false;- const existData = doc.share.has(YjsEditorKey.data_section);-- if (existData) {- const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;-- localExist = data.has(collabSharedRootKeyMap[type] as string);- }-- return {- doc,- localExist,- };-}--export async function getCollabStorageWithAPICall(workspaceId: string, id: string, type: CollabType) {- const { doc, localExist } = await getCollabStorage(id, type);- const asyncApply = async () => {- const res = await fetchCollab(workspaceId, id, type);-- applyYDoc(doc, res.state);- };-- // If the document exists locally, apply the state asynchronously,- // otherwise, apply the state synchronously- if (localExist) {- void asyncApply();- } else {- await asyncApply();- }-- return doc;-}--export async function batchCollabs(- workspaceId: string,- params: {- object_id: string;- collab_type: CollabType;- }[],- rowCallback?: (id: string, doc: YDoc) => void-) {- console.log('Fetching collab data:', params);- // Create or get Y.Doc from local storage- for (const item of params) {- const { object_id, collab_type } = item;-- const { doc, localExist } = await getCollabStorage(object_id, collab_type);-- if (rowCallback && localExist) {- rowCallback(object_id, doc);- }- }-- const res = await batchFetchCollab(workspaceId, params);-- console.log('Fetched collab data:', res);- for (const id of Object.keys(res)) {- const type = params.find((param) => param.object_id === id)?.collab_type;- const data = res[id];-- if (type === undefined || !data) {- continue;- }-- const { doc } = await getCollabStorage(id, type);-- applyYDoc(doc, new Uint8Array(data));-- rowCallback?.(id, doc);- }-}diff --git a/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/user.service.tsindex c4853f850d..ce912bd50f 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts@@ -8,7 +8,7 @@ import {invalidToken,setSignInUser,setUserWorkspace,-} from '@/application/services/js-services/storage';+} from 'src/application/services/js-services/session';import { asyncDataDecorator } from '@/application/services/js-services/decorator';async function getUser() {diff --git a/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts b/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.tsindex 6b50ca27e5..cd09cb74d1 100644--- a/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts+++ b/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts@@ -2,7 +2,7 @@ import { CollabType } from '@/application/collab.type';import { ClientAPI } from '@appflowyinc/client-api-wasm';import { UserProfile, UserWorkspace } from '@/application/user.type';import { AFCloudConfig } from '@/application/services/services.type';-import { invalidToken, readTokenStr, writeToken } from '@/application/services/js-services/storage';+import { invalidToken, readTokenStr, writeToken } from 'src/application/services/js-services/session';let client: ClientAPI;diff --git a/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx b/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsxindex 76d0f7575c..a7ef1d2684 100644--- a/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx+++ b/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx@@ -1,4 +1,4 @@-import { getCurrentWorkspace } from '@/application/services/js-services/storage';+import { getCurrentWorkspace } from 'src/application/services/js-services/session';import { Button, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle } from '@mui/material';import React from 'react';import { useNavigate } from 'react-router-dom';diff --git a/frontend/appflowy_web_app/src/components/database/Database.tsx b/frontend/appflowy_web_app/src/components/database/Database.tsxindex 6c7c3a4868..0c590462f9 100644--- a/frontend/appflowy_web_app/src/components/database/Database.tsx+++ b/frontend/appflowy_web_app/src/components/database/Database.tsx@@ -12,7 +12,6 @@ export const Database = memo(viewId: string;onNavigateToView: (viewId: string) => void;}) => {- console.log('Database', viewId, iidIndex);return (<div className='appflowy-database relative flex w-full flex-1 select-text flex-col overflow-y-hidden'><DatabaseViews iidIndex={iidIndex} onChangeView={onNavigateToView} viewId={viewId} />diff --git a/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx b/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsxindex a563557ab7..1e6bbb151a 100644--- a/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx+++ b/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx@@ -1,6 +1,6 @@import { ReactComponent as ExpandMoreIcon } from '$icons/16x/full_view.svg';import { useNavigateToView } from '@/application/folder-yjs';-import { getCurrentWorkspace } from '@/application/services/js-services/storage';+import { getCurrentWorkspace } from 'src/application/services/js-services/session';import { IdProvider } from '@/components/_shared/context-provider/IdProvider';import { Database } from '@/components/database';import { useGetDatabaseId, useLoadDatabase } from '@/components/database/Database.hooks';diff --git a/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx b/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsxindex 1f58950e58..7d857ac893 100644--- a/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx+++ b/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx@@ -9,19 +9,19 @@ import { ReactComponent as CalendarSvg } from '$icons/16x/date.svg';const renderCrumbIcon = (icon: string) => {if (Number(icon) === ViewLayout.Grid) {- return <GridSvg />;+ return <GridSvg className={'h-4 w-4'} />;}if (Number(icon) === ViewLayout.Board) {- return <BoardSvg />;+ return <BoardSvg className={'h-4 w-4'} />;}if (Number(icon) === ViewLayout.Calendar) {- return <CalendarSvg />;+ return <CalendarSvg className={'h-4 w-4'} />;}if (Number(icon) === ViewLayout.Document) {- return <DocumentSvg />;+ return <DocumentSvg className={'h-4 w-4'} />;}return icon;@@ -41,7 +41,7 @@ function Item({ crumb, disableClick = false }: { crumb: Crumb; disableClick?: boonNavigateToView?.(viewId);}}>- <span className={'h-4 w-4'}>{renderCrumbIcon(icon)}</span>+ {renderCrumbIcon(icon)}<spanclassName={!disableClick ? 'max-w-[250px] truncate hover:text-fill-default hover:underline' : 'flex-1 truncate'}>